<commit_before>package dupfinder

import (
	"testing"
	"strings"
	"io/ioutil"
	"os"
	"strconv"
	"github.com/janosgyerik/dupfinder/finder"
	"path/filepath"
	"errors"
)

const testdataBasedir = "testdata"

func Test_cmpResult(t*testing.T) {
	err := errors.New("dummy")

	cmpResultTests := []struct {
		label            string
		actual, expected cmpResult
	}{
		{
			"done 1",
			done(1),
			cmpResult{cmp: 1, success: true, done: true},
		},
		{
			"done -1",
			done(-1),
			cmpResult{cmp: -1, success: true, done: true},
		},
		{
			"done 0",
			done(0),
			cmpResult{cmp: 0, success: true, done: true},
		},
		{
			"undecided",
			undecided(),
			cmpResult{},
		},
		{
			"errFirst",
			errFirst(err),
			cmpResult{errFirst: err, done: true},
		},
		{
			"errSecond",
			errSecond(err),
			cmpResult{errSecond: err, done: true},
		},
	}
	for _, tt := range cmpResultTests {
		if tt.expected != tt.actual {
			t.Errorf("got %#v for %s, expected %#v", tt.actual, tt.label, tt.expected)
		}
	}
}

func Test_compareReaders_same_file(t*testing.T) {
	content := "dummy content"
	reader1 := strings.NewReader(content)
	reader2 := strings.NewReader(content)

	expected := 0

	r := compareReaders(reader1, reader2)
	if !r.success {
		t.Errorf("Compare(f, f) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(f, f) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareReaders_size_ascending(t*testing.T) {
	smaller := strings.NewReader("dummy content")
	bigger := strings.NewReader("longer dummy content")

	expected := -1

	r := compareReaders(smaller, bigger)
	if !r.success {
		t.Errorf("Compare(smaller, bigger) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(smaller, bigger) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareReaders_size_descending(t*testing.T) {
	smaller := strings.NewReader("dummy content")
	bigger := strings.NewReader("longer dummy content")

	expected := 1

	r := compareReaders(bigger, smaller)
	if !r.success {
		t.Errorf("Compare(bigger, smaller) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(bigger, smaller) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareReaders_same_size_content_ascending(t*testing.T) {
	lower := strings.NewReader("dummy content a")
	higher := strings.NewReader("dummy content b")

	expected := -1

	r := compareReaders(lower, higher)
	if !r.success {
		t.Errorf("Compare(lower, higher) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(lower, higher) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareReaders_same_size_content_descending(t*testing.T) {
	lower := strings.NewReader("dummy content a")
	higher := strings.NewReader("dummy content b")

	expected := 1

	r := compareReaders(higher, lower)
	if !r.success {
		t.Errorf("Compare(higher, lower) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(higher, lower) == %v, want %v", r.cmp, expected)
	}
}

func newTempFile(t*testing.T, name string) *os.File {
	file, err := ioutil.TempFile(os.TempDir(), name)

	if err != nil {
		t.Error("Failed to create temporary file")
	}

	return file
}

func Test_compareFiles_equal_if_both_same_empty_dummy(t*testing.T) {
	file := newTempFile(t, "dummy")
	defer os.Remove(file.Name())

	expected := 0

	r := compareFiles(file.Name(), file.Name())
	if !r.success {
		t.Errorf("Compare(dummy, dummy) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(dummy, dummy) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareFiles_equal_if_both_empty_dummy(t*testing.T) {
	dummy1 := newTempFile(t, "dummy1")
	defer os.Remove(dummy1.Name())

	dummy2 := newTempFile(t, "dummy2")
	defer os.Remove(dummy2.Name())

	expected := 0

	r := compareFiles(dummy1.Name(), dummy2.Name())
	if !r.success {
		t.Errorf("Compare(dummy1, dummy2) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(dummy1, dummy2) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareFiles_empty_comes_before_nonempty(t*testing.T) {
	empty := newTempFile(t, "empty")
	defer os.Remove(empty.Name())

	nonempty := newTempFile(t, "nonempty")
	defer os.Remove(nonempty.Name())

	nonempty.WriteString("something")

	expected := -1

	r := compareFiles(empty.Name(), nonempty.Name())
	if !r.success {
		t.Errorf("Compare(empty, nonempty) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(empty, nonempty) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareFiles_fails_if_both_nonexistent(t*testing.T) {
	r := compareFiles("/nonexistent1", "/nonexistent2")
	if r.errFirst == nil {
		t.Error("Compare(nonexistent1, nonexistent2) should have raised error")
	}
}

func Test_compareFiles_fails_if_first_nonexistent(t*testing.T) {
	file := newTempFile(t, "dummy")
	defer os.Remove(file.Name())

	r := compareFiles("/nonexistent", file.Name())
	if r.success || r.errFirst == nil || r.errSecond != nil || !r.done {
		t.Error("Compare(nonexistent, dummy) should have raised error")
	}
}

func Test_compareFiles_fails_if_second_nonexistent(t*testing.T) {
	file := newTempFile(t, "dummy")
	defer os.Remove(file.Name())

	r := compareFiles(file.Name(), "/nonexistent")
	if r.success || r.errFirst != nil || r.errSecond == nil || !r.done {
		t.Error("Compare(dummy, nonexistent) should have raised error")
	}
}

func createTempFileWithContent(content string) *os.File {
	file, _ := ioutil.TempFile(os.TempDir(), "prefix")
	file.WriteString(content)
	return file
}

func findDuplicates(distinctCount int, groupSizes... int) []Duplicates {
	paths := make([]string, 0)

	for i := 0; i < distinctCount; i++ {
		file := createTempFileWithContent("dummy distinct " + strconv.Itoa(i))
		defer os.Remove(file.Name())
		paths = append(paths, file.Name())
	}

	for group, groupSize := range groupSizes {
		content := "dummy group " + strconv.Itoa(group)
		for i := 0; i < groupSize; i++ {
			file := createTempFileWithContent(content)
			defer os.Remove(file.Name())
			paths = append(paths, file.Name())
		}
	}

	return FindDuplicates(paths)
}

func Test_FindDuplicates_two_duplicates(t*testing.T) {
	duplicates := findDuplicates(0, 2)
	if len(duplicates) != 1 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 1)
	}
	if duplicates[0].count() != 2 {
		t.Errorf("Found %d duplicate files, expected %d", duplicates[0].count(), 2)
	}
}

func Test_FindDuplicates_three_duplicates(t*testing.T) {
	duplicates := findDuplicates(0, 3)
	if len(duplicates) != 1 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 1)
	}
	if duplicates[0].count() != 3 {
		t.Errorf("Found %d duplicate files, expected %d", duplicates[0].count(), 3)
	}
}

func Test_FindDuplicates_two_different(t*testing.T) {
	duplicates := findDuplicates(2)
	if len(duplicates) != 0 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 0)
	}
}

func Test_FindDuplicates_three_different(t*testing.T) {
	duplicates := findDuplicates(3)
	if len(duplicates) != 0 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 0)
	}
}

func Test_FindDuplicates_two_duplicate_groups(t*testing.T) {
	duplicates := findDuplicates(0, 2, 3)
	if len(duplicates) != 2 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 2)
	}
	if duplicates[0].count() != 2 {
		t.Errorf("Found %d duplicate files in group 1, expected %d", duplicates[0].count(), 2)
	}
	if duplicates[1].count() != 3 {
		t.Errorf("Found %d duplicate files in group 2, expected %d", duplicates[1].count(), 3)
	}
}

func Test_dupTracker_add_and_merge(t*testing.T) {
	tracker := newDupTracker()
	tracker.add("path1-1", "path1-2")
	tracker.add("path1-2", "path1-3")
	tracker.add("path2-1", "path2-2")
	tracker.add("path2-3", "path2-2")
	tracker.add("path1-1", "path2-2")

	duplicates := tracker.getDuplicates()
	if len(duplicates) != 1 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 1)
	}
	if duplicates[0].count() != 6 {
		t.Errorf("Found %d duplicate files, expected %d", duplicates[0].count(), 6)
	}
}

func findPaths(basename string) []string {
	return finder.NewFinder().FindAll(filepath.Join(testdataBasedir, basename))
}

func Test_nodups(t*testing.T) {
	duplicates := FindDuplicates(findPaths("nodups"))

	if len(duplicates) > 0 {
		t.Fatal("found duplicates in different files:", duplicates)
	}
}

func Test_samesize(t*testing.T) {
	duplicates := FindDuplicates(findPaths("samesize"))

	if len(duplicates) > 0 {
		t.Fatal("found duplicates in different files with same size:", duplicates)
	}
}

func Test_alldups(t*testing.T) {
	duplicates := FindDuplicates(findPaths("alldups"))

	if len(duplicates) != 1 {
		t.Fatalf("got %d duplicate groups, expected 1", len(duplicates))
	}
}
<commit_msg>Do not ignore errors when creating temp files with content<commit_after>package dupfinder

import (
	"testing"
	"strings"
	"io/ioutil"
	"os"
	"strconv"
	"github.com/janosgyerik/dupfinder/finder"
	"path/filepath"
	"errors"
)

const testdataBasedir = "testdata"

func Test_cmpResult(t*testing.T) {
	err := errors.New("dummy")

	cmpResultTests := []struct {
		label            string
		actual, expected cmpResult
	}{
		{
			"done 1",
			done(1),
			cmpResult{cmp: 1, success: true, done: true},
		},
		{
			"done -1",
			done(-1),
			cmpResult{cmp: -1, success: true, done: true},
		},
		{
			"done 0",
			done(0),
			cmpResult{cmp: 0, success: true, done: true},
		},
		{
			"undecided",
			undecided(),
			cmpResult{},
		},
		{
			"errFirst",
			errFirst(err),
			cmpResult{errFirst: err, done: true},
		},
		{
			"errSecond",
			errSecond(err),
			cmpResult{errSecond: err, done: true},
		},
	}
	for _, tt := range cmpResultTests {
		if tt.expected != tt.actual {
			t.Errorf("got %#v for %s, expected %#v", tt.actual, tt.label, tt.expected)
		}
	}
}

func Test_compareReaders_same_file(t*testing.T) {
	content := "dummy content"
	reader1 := strings.NewReader(content)
	reader2 := strings.NewReader(content)

	expected := 0

	r := compareReaders(reader1, reader2)
	if !r.success {
		t.Errorf("Compare(f, f) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(f, f) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareReaders_size_ascending(t*testing.T) {
	smaller := strings.NewReader("dummy content")
	bigger := strings.NewReader("longer dummy content")

	expected := -1

	r := compareReaders(smaller, bigger)
	if !r.success {
		t.Errorf("Compare(smaller, bigger) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(smaller, bigger) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareReaders_size_descending(t*testing.T) {
	smaller := strings.NewReader("dummy content")
	bigger := strings.NewReader("longer dummy content")

	expected := 1

	r := compareReaders(bigger, smaller)
	if !r.success {
		t.Errorf("Compare(bigger, smaller) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(bigger, smaller) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareReaders_same_size_content_ascending(t*testing.T) {
	lower := strings.NewReader("dummy content a")
	higher := strings.NewReader("dummy content b")

	expected := -1

	r := compareReaders(lower, higher)
	if !r.success {
		t.Errorf("Compare(lower, higher) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(lower, higher) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareReaders_same_size_content_descending(t*testing.T) {
	lower := strings.NewReader("dummy content a")
	higher := strings.NewReader("dummy content b")

	expected := 1

	r := compareReaders(higher, lower)
	if !r.success {
		t.Errorf("Compare(higher, lower) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(higher, lower) == %v, want %v", r.cmp, expected)
	}
}

func newTempFile(t*testing.T, name string) *os.File {
	file, err := ioutil.TempFile(os.TempDir(), name)

	if err != nil {
		t.Error("Failed to create temporary file")
	}

	return file
}

func newTempFileWithContent(t*testing.T, content string) *os.File {
	file := newTempFile(t, "dummy")
	file.WriteString(content)
	return file
}

func Test_compareFiles_equal_if_both_same_empty_dummy(t*testing.T) {
	file := newTempFile(t, "dummy")
	defer os.Remove(file.Name())

	expected := 0

	r := compareFiles(file.Name(), file.Name())
	if !r.success {
		t.Errorf("Compare(dummy, dummy) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(dummy, dummy) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareFiles_equal_if_both_empty_dummy(t*testing.T) {
	dummy1 := newTempFile(t, "dummy1")
	defer os.Remove(dummy1.Name())

	dummy2 := newTempFile(t, "dummy2")
	defer os.Remove(dummy2.Name())

	expected := 0

	r := compareFiles(dummy1.Name(), dummy2.Name())
	if !r.success {
		t.Errorf("Compare(dummy1, dummy2) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(dummy1, dummy2) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareFiles_empty_comes_before_nonempty(t*testing.T) {
	empty := newTempFile(t, "empty")
	defer os.Remove(empty.Name())

	nonempty := newTempFile(t, "nonempty")
	defer os.Remove(nonempty.Name())

	nonempty.WriteString("something")

	expected := -1

	r := compareFiles(empty.Name(), nonempty.Name())
	if !r.success {
		t.Errorf("Compare(empty, nonempty) failed: %v", r)
	}
	if r.cmp != expected {
		t.Errorf("Compare(empty, nonempty) == %v, want %v", r.cmp, expected)
	}
}

func Test_compareFiles_fails_if_both_nonexistent(t*testing.T) {
	r := compareFiles("/nonexistent1", "/nonexistent2")
	if r.errFirst == nil {
		t.Error("Compare(nonexistent1, nonexistent2) should have raised error")
	}
}

func Test_compareFiles_fails_if_first_nonexistent(t*testing.T) {
	file := newTempFile(t, "dummy")
	defer os.Remove(file.Name())

	r := compareFiles("/nonexistent", file.Name())
	if r.success || r.errFirst == nil || r.errSecond != nil || !r.done {
		t.Error("Compare(nonexistent, dummy) should have raised error")
	}
}

func Test_compareFiles_fails_if_second_nonexistent(t*testing.T) {
	file := newTempFile(t, "dummy")
	defer os.Remove(file.Name())

	r := compareFiles(file.Name(), "/nonexistent")
	if r.success || r.errFirst != nil || r.errSecond == nil || !r.done {
		t.Error("Compare(dummy, nonexistent) should have raised error")
	}
}

func findDuplicates(t*testing.T, distinctCount int, groupSizes... int) []Duplicates {
	paths := make([]string, 0)

	for i := 0; i < distinctCount; i++ {
		file := newTempFileWithContent(t, "dummy distinct " + strconv.Itoa(i))
		defer os.Remove(file.Name())
		paths = append(paths, file.Name())
	}

	for group, groupSize := range groupSizes {
		content := "dummy group " + strconv.Itoa(group)
		for i := 0; i < groupSize; i++ {
			file := newTempFileWithContent(t, content)
			defer os.Remove(file.Name())
			paths = append(paths, file.Name())
		}
	}

	return FindDuplicates(paths)
}

func Test_FindDuplicates_two_duplicates(t*testing.T) {
	duplicates := findDuplicates(t, 0, 2)
	if len(duplicates) != 1 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 1)
	}
	if duplicates[0].count() != 2 {
		t.Errorf("Found %d duplicate files, expected %d", duplicates[0].count(), 2)
	}
}

func Test_FindDuplicates_three_duplicates(t*testing.T) {
	duplicates := findDuplicates(t, 0, 3)
	if len(duplicates) != 1 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 1)
	}
	if duplicates[0].count() != 3 {
		t.Errorf("Found %d duplicate files, expected %d", duplicates[0].count(), 3)
	}
}

func Test_FindDuplicates_two_different(t*testing.T) {
	duplicates := findDuplicates(t, 2)
	if len(duplicates) != 0 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 0)
	}
}

func Test_FindDuplicates_three_different(t*testing.T) {
	duplicates := findDuplicates(t, 3)
	if len(duplicates) != 0 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 0)
	}
}

func Test_FindDuplicates_two_duplicate_groups(t*testing.T) {
	duplicates := findDuplicates(t, 0, 2, 3)
	if len(duplicates) != 2 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 2)
	}
	if duplicates[0].count() != 2 {
		t.Errorf("Found %d duplicate files in group 1, expected %d", duplicates[0].count(), 2)
	}
	if duplicates[1].count() != 3 {
		t.Errorf("Found %d duplicate files in group 2, expected %d", duplicates[1].count(), 3)
	}
}

func Test_dupTracker_add_and_merge(t*testing.T) {
	tracker := newDupTracker()
	tracker.add("path1-1", "path1-2")
	tracker.add("path1-2", "path1-3")
	tracker.add("path2-1", "path2-2")
	tracker.add("path2-3", "path2-2")
	tracker.add("path1-1", "path2-2")

	duplicates := tracker.getDuplicates()
	if len(duplicates) != 1 {
		t.Errorf("Found %d duplicate groups, expected %d", len(duplicates), 1)
	}
	if duplicates[0].count() != 6 {
		t.Errorf("Found %d duplicate files, expected %d", duplicates[0].count(), 6)
	}
}

func findPaths(basename string) []string {
	return finder.NewFinder().FindAll(filepath.Join(testdataBasedir, basename))
}

func Test_nodups(t*testing.T) {
	duplicates := FindDuplicates(findPaths("nodups"))

	if len(duplicates) > 0 {
		t.Fatal("found duplicates in different files:", duplicates)
	}
}

func Test_samesize(t*testing.T) {
	duplicates := FindDuplicates(findPaths("samesize"))

	if len(duplicates) > 0 {
		t.Fatal("found duplicates in different files with same size:", duplicates)
	}
}

func Test_alldups(t*testing.T) {
	duplicates := FindDuplicates(findPaths("alldups"))

	if len(duplicates) != 1 {
		t.Fatalf("got %d duplicate groups, expected 1", len(duplicates))
	}
}
<|endoftext|>


<commit_before>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| encoding/string_decoder.go                               |
|                                                          |
| LastModified: Apr 25, 2020                               |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package encoding

// stringDecoder is the implementation of ValueDecoder for string.
type stringDecoder struct{}

var strdec stringDecoder

func (stringDecoder) Decode(dec *Decoder, p interface{}, tag byte) {
	if pv, ok := p.(*string); ok {
		switch tag {
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			*pv = string(tag)
		case TagNull, TagEmpty:
			*pv = ""
		case TagTrue:
			*pv = "true"
		case TagFalse:
			*pv = "false"
		case TagNaN:
			*pv = "NaN"
		case TagInfinity:
			if dec.NextByte() == TagNeg {
				*pv = "-Inf"
			} else {
				*pv = "+Inf"
			}
		case TagInteger, TagLong, TagDouble:
			*pv = string(dec.Until(TagSemicolon))
		case TagUTF8Char:
			*pv = dec.readSafeString(1)
		case TagString:
			*pv = dec.ReadString()
		default:
			dec.decodeError(p, tag)
		}
	}
}

func (dec *Decoder) fastReadStringAsBytes(utf16Length int) (data []byte) {
	buf := dec.buf[dec.head:dec.tail]
	off := 0
	for ; utf16Length > 0; utf16Length-- {
		b := buf[off]
		switch b >> 4 {
		case 0, 1, 2, 3, 4, 5, 6, 7:
			off++
		case 12, 13:
			off += 2
		case 14:
			off += 3
		case 15:
			if b&8 == 8 {
				if dec.Error == nil {
					dec.Error = ErrInvalidUTF8
				}
				return
			}
			off += 4
			utf16Length--
		default:
			if dec.Error == nil {
				dec.Error = ErrInvalidUTF8
			}
			return
		}
	}
	dec.head += off
	return buf[:off]
}

func (dec *Decoder) readStringAsBytes(utf16Length int) (data []byte) {
	if (utf16Length == 0) || (dec.head == dec.tail) && !dec.loadMore() {
		return
	}
	length := dec.tail - dec.head
	if length >= utf16Length*3 {
		return dec.fastReadStringAsBytes(utf16Length)
	}
	for {
		buf := dec.buf[dec.head:dec.tail]
		off := 0
		for ; utf16Length > 0 && off < length; utf16Length-- {
			b := buf[off]
			switch b >> 4 {
			case 0, 1, 2, 3, 4, 5, 6, 7:
				off++
			case 12, 13:
				off += 2
			case 14:
				off += 3
			case 15:
				if b&8 == 8 {
					if dec.Error == nil {
						dec.Error = ErrInvalidUTF8
					}
					return
				}
				off += 4
				utf16Length--
			default:
				if dec.Error == nil {
					dec.Error = ErrInvalidUTF8
				}
				return
			}
		}
		remains := length - off
		if remains > 0 {
			dec.head += off
			if data == nil {
				return buf[:off]
			}
			data = append(data, buf[:off]...)
			return
		}
		data = append(data, buf...)
		if !dec.loadMore() {
			if remains < 0 {
				if dec.Error == nil {
					dec.Error = ErrInvalidUTF8
				}
			}
			return
		}
		data = append(data, dec.buf[dec.head:dec.head-remains]...)
		dec.head -= remains
		length = dec.tail - dec.head
	}
}

func (dec *Decoder) readUnsafeString(utf16Length int) (s string) {
	data := dec.readStringAsBytes(utf16Length)
	if data == nil {
		return
	}
	return unsafeString(data)
}

func (dec *Decoder) readSafeString(utf16Length int) (s string) {
	data := dec.readStringAsBytes(utf16Length)
	if data == nil {
		return
	}
	return string(data)
}

// ReadUnsafeString read unsafe string
func (dec *Decoder) ReadUnsafeString() (s string) {
	s = dec.readUnsafeString(dec.ReadInt())
	dec.Skip()
	return
}

// ReadSafeString read safe string
func (dec *Decoder) ReadSafeString() (s string) {
	s = dec.readSafeString(dec.ReadInt())
	dec.Skip()
	return
}

// ReadString read safe string and add reference
func (dec *Decoder) ReadString() (s string) {
	s = dec.ReadSafeString()
	dec.AddReference(s)
	return
}
<commit_msg>Update string_decoder.go<commit_after>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| encoding/string_decoder.go                               |
|                                                          |
| LastModified: Apr 25, 2020                               |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package encoding

// stringDecoder is the implementation of ValueDecoder for string.
type stringDecoder struct{}

var strdec stringDecoder

func (valdec stringDecoder) decode(dec *Decoder, p interface{}, tag byte) string {
	switch tag {
	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
		return string(tag)
	case TagNull, TagEmpty:
		return ""
	case TagTrue:
		return "true"
	case TagFalse:
		return "false"
	case TagNaN:
		return "NaN"
	case TagInfinity:
		if dec.NextByte() == TagNeg {
			return "-Inf"
		}
		return "+Inf"
	case TagInteger, TagLong, TagDouble:
		return string(dec.Until(TagSemicolon))
	case TagUTF8Char:
		return dec.readSafeString(1)
	case TagString:
		return dec.ReadString()
	default:
		dec.decodeError(p, tag)
	}
	return ""
}

func (valdec stringDecoder) Decode(dec *Decoder, p interface{}, tag byte) {
	if tag == TagNull {
		switch pv := p.(type) {
		case **string:
			*pv = nil
		case *string:
			*pv = ""
		}
		return
	}
	s := valdec.decode(dec, p, tag)
	if dec.Error != nil {
		return
	}
	switch pv := p.(type) {
	case **string:
		*pv = &s
	case *string:
		*pv = s
	}
}

func (dec *Decoder) fastReadStringAsBytes(utf16Length int) (data []byte) {
	buf := dec.buf[dec.head:dec.tail]
	off := 0
	for ; utf16Length > 0; utf16Length-- {
		b := buf[off]
		switch b >> 4 {
		case 0, 1, 2, 3, 4, 5, 6, 7:
			off++
		case 12, 13:
			off += 2
		case 14:
			off += 3
		case 15:
			if b&8 == 8 {
				if dec.Error == nil {
					dec.Error = ErrInvalidUTF8
				}
				return
			}
			off += 4
			utf16Length--
		default:
			if dec.Error == nil {
				dec.Error = ErrInvalidUTF8
			}
			return
		}
	}
	dec.head += off
	return buf[:off]
}

func (dec *Decoder) readStringAsBytes(utf16Length int) (data []byte) {
	if (utf16Length == 0) || (dec.head == dec.tail) && !dec.loadMore() {
		return
	}
	length := dec.tail - dec.head
	if length >= utf16Length*3 {
		return dec.fastReadStringAsBytes(utf16Length)
	}
	for {
		buf := dec.buf[dec.head:dec.tail]
		off := 0
		for ; utf16Length > 0 && off < length; utf16Length-- {
			b := buf[off]
			switch b >> 4 {
			case 0, 1, 2, 3, 4, 5, 6, 7:
				off++
			case 12, 13:
				off += 2
			case 14:
				off += 3
			case 15:
				if b&8 == 8 {
					if dec.Error == nil {
						dec.Error = ErrInvalidUTF8
					}
					return
				}
				off += 4
				utf16Length--
			default:
				if dec.Error == nil {
					dec.Error = ErrInvalidUTF8
				}
				return
			}
		}
		remains := length - off
		if remains > 0 {
			dec.head += off
			if data == nil {
				return buf[:off]
			}
			data = append(data, buf[:off]...)
			return
		}
		data = append(data, buf...)
		if !dec.loadMore() {
			if remains < 0 {
				if dec.Error == nil {
					dec.Error = ErrInvalidUTF8
				}
			}
			return
		}
		data = append(data, dec.buf[dec.head:dec.head-remains]...)
		dec.head -= remains
		length = dec.tail - dec.head
	}
}

func (dec *Decoder) readUnsafeString(utf16Length int) (s string) {
	data := dec.readStringAsBytes(utf16Length)
	if data == nil {
		return
	}
	return unsafeString(data)
}

func (dec *Decoder) readSafeString(utf16Length int) (s string) {
	data := dec.readStringAsBytes(utf16Length)
	if data == nil {
		return
	}
	return string(data)
}

// ReadUnsafeString read unsafe string
func (dec *Decoder) ReadUnsafeString() (s string) {
	s = dec.readUnsafeString(dec.ReadInt())
	dec.Skip()
	return
}

// ReadSafeString read safe string
func (dec *Decoder) ReadSafeString() (s string) {
	s = dec.readSafeString(dec.ReadInt())
	dec.Skip()
	return
}

// ReadString read safe string and add reference
func (dec *Decoder) ReadString() (s string) {
	s = dec.ReadSafeString()
	dec.AddReference(s)
	return
}
<|endoftext|>


<commit_before>package elastic

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"

	"github.com/juju/errors"
)

// Although there are many Elasticsearch clients with Go, I still want to implement one by myself.
// Because we only need some very simple usages.
type Client struct {
	Addr string

	c *http.Client
}

func NewClient(addr string) *Client {
	c := new(Client)

	c.Addr = addr

	c.c = &http.Client{}

	return c
}

type ResponseItem struct {
	ID      string                 `json:"_id"`
	Index   string                 `json:"_index"`
	Type    string                 `json:"_type"`
	Version int                    `json:"_version"`
	Found   bool                   `json:"found"`
	Source  map[string]interface{} `json:"_source"`
}

type Response struct {
	Code int
	ResponseItem
}

// See http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/bulk.html
const (
	ActionCreate = "create"
	ActionUpdate = "update"
	ActionDelete = "delete"
	ActionIndex  = "index"
)

type BulkRequest struct {
	Action string
	Index  string
	Type   string
	ID     string
	Parent string

	Data map[string]interface{}
}

func (r *BulkRequest) bulk(buf *bytes.Buffer) error {
	meta := make(map[string]map[string]string)
	metaData := make(map[string]string)
	if len(r.Index) > 0 {
		metaData["_index"] = r.Index
	}
	if len(r.Type) > 0 {
		metaData["_type"] = r.Type
	}

	if len(r.ID) > 0 {
		metaData["_id"] = r.ID
	}
	if len(r.Parent) > 0 {
		metaData["_parent"] = r.Parent
	}

	meta[r.Action] = metaData

	data, err := json.Marshal(meta)
	if err != nil {
		return errors.Trace(err)
	}

	buf.Write(data)
	buf.WriteByte('\n')

	switch r.Action {
	case ActionDelete:
		//nothing to do
	case ActionUpdate:
		doc := map[string]interface{}{
			"doc": r.Data,
		}
		data, err = json.Marshal(doc)
		if err != nil {
			return errors.Trace(err)
		}

		buf.Write(data)
		buf.WriteByte('\n')
	default:
		//for create and index
		data, err = json.Marshal(r.Data)
		if err != nil {
			return errors.Trace(err)
		}

		buf.Write(data)
		buf.WriteByte('\n')
	}

	return nil
}

type BulkResponse struct {
	Code   int
	Took   int  `json:"took"`
	Errors bool `json:"errors"`

	Items []map[string]*BulkResponseItem `json:"items"`
}

type BulkResponseItem struct {
	Index   string `json:"_index"`
	Type    string `json:"_type"`
	ID      string `json:"_id"`
	Version int    `json:"_version"`
	Status  int    `json:"status"`
	Error   string `json:"error"`
	Found   bool   `json:"found"`
}

func (c *Client) Do(method string, url string, body map[string]interface{}) (*Response, error) {
	bodyData, err := json.Marshal(body)
	if err != nil {
		return nil, errors.Trace(err)
	}

	buf := bytes.NewBuffer(bodyData)

	req, err := http.NewRequest(method, url, buf)
	if err != nil {
		return nil, errors.Trace(err)
	}

	resp, err := c.c.Do(req)
	if err != nil {
		return nil, errors.Trace(err)
	}

	ret := new(Response)
	ret.Code = resp.StatusCode

	if resp.ContentLength > 0 {
		d := json.NewDecoder(resp.Body)
		err = d.Decode(&ret.ResponseItem)
	}

	resp.Body.Close()

	return ret, err
}

func (c *Client) DoBulk(url string, items []*BulkRequest) (*BulkResponse, error) {
	var buf bytes.Buffer

	for _, item := range items {
		if err := item.bulk(&buf); err != nil {
			return nil, errors.Trace(err)
		}
	}

	req, err := http.NewRequest("POST", url, &buf)
	if err != nil {
		return nil, errors.Trace(err)
	}

	resp, err := c.c.Do(req)
	if err != nil {
		return nil, errors.Trace(err)
	}

	ret := new(BulkResponse)
	ret.Code = resp.StatusCode

	if resp.ContentLength > 0 {
		d := json.NewDecoder(resp.Body)
		err = d.Decode(&ret)
	}

	resp.Body.Close()

	return ret, err
}

func (c *Client) CreateMapping(index string, docType string, mapping map[string]interface{}) error {
	reqUrl := fmt.Sprintf("http://%s/%s", c.Addr,
		url.QueryEscape(index))

	r, err := c.Do("HEAD", reqUrl, nil)
	if err != nil {
		return errors.Trace(err)
	}

	// index doesn't exist, create index first
	if r.Code != http.StatusOK {
		_, err = c.Do("POST", reqUrl, nil)

		if err != nil {
			return errors.Trace(err)
		}
	}

	reqUrl = fmt.Sprintf("http://%s/%s/%s/_mapping", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType))

	_, err = c.Do("POST", reqUrl, mapping)
	return errors.Trace(err)
}

func (c *Client) DeleteIndex(index string) error {
	reqUrl := fmt.Sprintf("http://%s/%s", c.Addr,
		url.QueryEscape(index))

	r, err := c.Do("DELETE", reqUrl, nil)
	if err != nil {
		return errors.Trace(err)
	}

	if r.Code == http.StatusOK || r.Code == http.StatusNotFound {
		return nil
	} else {
		return errors.Errorf("Error: %s, code: %d", http.StatusText(r.Code), r.Code)
	}
}

func (c *Client) Get(index string, docType string, id string) (*Response, error) {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/%s", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType),
		url.QueryEscape(id))

	return c.Do("GET", reqUrl, nil)
}

// Can use Update to create or update the data
func (c *Client) Update(index string, docType string, id string, data map[string]interface{}) error {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/%s", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType),
		url.QueryEscape(id))

	r, err := c.Do("PUT", reqUrl, data)
	if err != nil {
		return errors.Trace(err)
	}

	if r.Code == http.StatusOK || r.Code == http.StatusCreated {
		return nil
	} else {
		return errors.Errorf("Error: %s, code: %d", http.StatusText(r.Code), r.Code)
	}
}

func (c *Client) Exists(index string, docType string, id string) (bool, error) {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/%s", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType),
		url.QueryEscape(id))

	r, err := c.Do("HEAD", reqUrl, nil)
	if err != nil {
		return false, err
	}

	return r.Code == http.StatusOK, nil
}

func (c *Client) Delete(index string, docType string, id string) error {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/%s", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType),
		url.QueryEscape(id))

	r, err := c.Do("DELETE", reqUrl, nil)
	if err != nil {
		return errors.Trace(err)
	}

	if r.Code == http.StatusOK || r.Code == http.StatusNotFound {
		return nil
	} else {
		return errors.Errorf("Error: %s, code: %d", http.StatusText(r.Code), r.Code)
	}
}

// only support parent in 'Bulk' related apis
func (c *Client) Bulk(items []*BulkRequest) (*BulkResponse, error) {
	reqUrl := fmt.Sprintf("http://%s/_bulk", c.Addr)

	return c.DoBulk(reqUrl, items)
}

func (c *Client) IndexBulk(index string, items []*BulkRequest) (*BulkResponse, error) {
	reqUrl := fmt.Sprintf("http://%s/%s/_bulk", c.Addr,
		url.QueryEscape(index))

	return c.DoBulk(reqUrl, items)
}

func (c *Client) IndexTypeBulk(index string, docType string, items []*BulkRequest) (*BulkResponse, error) {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/_bulk", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType))

	return c.DoBulk(reqUrl, items)
}
<commit_msg>changing BulkResponseItem.Error to json.rawMessage<commit_after>package elastic

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"

	"github.com/juju/errors"
)

// Although there are many Elasticsearch clients with Go, I still want to implement one by myself.
// Because we only need some very simple usages.
type Client struct {
	Addr string

	c *http.Client
}

func NewClient(addr string) *Client {
	c := new(Client)

	c.Addr = addr

	c.c = &http.Client{}

	return c
}

type ResponseItem struct {
	ID      string                 `json:"_id"`
	Index   string                 `json:"_index"`
	Type    string                 `json:"_type"`
	Version int                    `json:"_version"`
	Found   bool                   `json:"found"`
	Source  map[string]interface{} `json:"_source"`
}

type Response struct {
	Code int
	ResponseItem
}

// See http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/bulk.html
const (
	ActionCreate = "create"
	ActionUpdate = "update"
	ActionDelete = "delete"
	ActionIndex  = "index"
)

type BulkRequest struct {
	Action string
	Index  string
	Type   string
	ID     string
	Parent string

	Data map[string]interface{}
}

func (r *BulkRequest) bulk(buf *bytes.Buffer) error {
	meta := make(map[string]map[string]string)
	metaData := make(map[string]string)
	if len(r.Index) > 0 {
		metaData["_index"] = r.Index
	}
	if len(r.Type) > 0 {
		metaData["_type"] = r.Type
	}

	if len(r.ID) > 0 {
		metaData["_id"] = r.ID
	}
	if len(r.Parent) > 0 {
		metaData["_parent"] = r.Parent
	}

	meta[r.Action] = metaData

	data, err := json.Marshal(meta)
	if err != nil {
		return errors.Trace(err)
	}

	buf.Write(data)
	buf.WriteByte('\n')

	switch r.Action {
	case ActionDelete:
		//nothing to do
	case ActionUpdate:
		doc := map[string]interface{}{
			"doc": r.Data,
		}
		data, err = json.Marshal(doc)
		if err != nil {
			return errors.Trace(err)
		}

		buf.Write(data)
		buf.WriteByte('\n')
	default:
		//for create and index
		data, err = json.Marshal(r.Data)
		if err != nil {
			return errors.Trace(err)
		}

		buf.Write(data)
		buf.WriteByte('\n')
	}

	return nil
}

type BulkResponse struct {
	Code   int
	Took   int  `json:"took"`
	Errors bool `json:"errors"`

	Items []map[string]*BulkResponseItem `json:"items"`
}

type BulkResponseItem struct {
	Index   string `json:"_index"`
	Type    string `json:"_type"`
	ID      string `json:"_id"`
	Version int    `json:"_version"`
	Status  int    `json:"status"`
	Error   json.RawMessage `json:"error"`
	Found   bool   `json:"found"`
}

func (c *Client) Do(method string, url string, body map[string]interface{}) (*Response, error) {
	bodyData, err := json.Marshal(body)
	if err != nil {
		return nil, errors.Trace(err)
	}

	buf := bytes.NewBuffer(bodyData)

	req, err := http.NewRequest(method, url, buf)
	if err != nil {
		return nil, errors.Trace(err)
	}

	resp, err := c.c.Do(req)
	if err != nil {
		return nil, errors.Trace(err)
	}

	ret := new(Response)
	ret.Code = resp.StatusCode

	if resp.ContentLength > 0 {
		d := json.NewDecoder(resp.Body)
		err = d.Decode(&ret.ResponseItem)
	}

	resp.Body.Close()

	return ret, err
}

func (c *Client) DoBulk(url string, items []*BulkRequest) (*BulkResponse, error) {
	var buf bytes.Buffer

	for _, item := range items {
		if err := item.bulk(&buf); err != nil {
			return nil, errors.Trace(err)
		}
	}

	req, err := http.NewRequest("POST", url, &buf)
	if err != nil {
		return nil, errors.Trace(err)
	}

	resp, err := c.c.Do(req)
	if err != nil {
		return nil, errors.Trace(err)
	}

	ret := new(BulkResponse)
	ret.Code = resp.StatusCode

	if resp.ContentLength > 0 {
		d := json.NewDecoder(resp.Body)
		err = d.Decode(&ret)
	}

	resp.Body.Close()

	return ret, err
}

func (c *Client) CreateMapping(index string, docType string, mapping map[string]interface{}) error {
	reqUrl := fmt.Sprintf("http://%s/%s", c.Addr,
		url.QueryEscape(index))

	r, err := c.Do("HEAD", reqUrl, nil)
	if err != nil {
		return errors.Trace(err)
	}

	// index doesn't exist, create index first
	if r.Code != http.StatusOK {
		_, err = c.Do("POST", reqUrl, nil)

		if err != nil {
			return errors.Trace(err)
		}
	}

	reqUrl = fmt.Sprintf("http://%s/%s/%s/_mapping", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType))

	_, err = c.Do("POST", reqUrl, mapping)
	return errors.Trace(err)
}

func (c *Client) DeleteIndex(index string) error {
	reqUrl := fmt.Sprintf("http://%s/%s", c.Addr,
		url.QueryEscape(index))

	r, err := c.Do("DELETE", reqUrl, nil)
	if err != nil {
		return errors.Trace(err)
	}

	if r.Code == http.StatusOK || r.Code == http.StatusNotFound {
		return nil
	} else {
		return errors.Errorf("Error: %s, code: %d", http.StatusText(r.Code), r.Code)
	}
}

func (c *Client) Get(index string, docType string, id string) (*Response, error) {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/%s", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType),
		url.QueryEscape(id))

	return c.Do("GET", reqUrl, nil)
}

// Can use Update to create or update the data
func (c *Client) Update(index string, docType string, id string, data map[string]interface{}) error {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/%s", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType),
		url.QueryEscape(id))

	r, err := c.Do("PUT", reqUrl, data)
	if err != nil {
		return errors.Trace(err)
	}

	if r.Code == http.StatusOK || r.Code == http.StatusCreated {
		return nil
	} else {
		return errors.Errorf("Error: %s, code: %d", http.StatusText(r.Code), r.Code)
	}
}

func (c *Client) Exists(index string, docType string, id string) (bool, error) {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/%s", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType),
		url.QueryEscape(id))

	r, err := c.Do("HEAD", reqUrl, nil)
	if err != nil {
		return false, err
	}

	return r.Code == http.StatusOK, nil
}

func (c *Client) Delete(index string, docType string, id string) error {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/%s", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType),
		url.QueryEscape(id))

	r, err := c.Do("DELETE", reqUrl, nil)
	if err != nil {
		return errors.Trace(err)
	}

	if r.Code == http.StatusOK || r.Code == http.StatusNotFound {
		return nil
	} else {
		return errors.Errorf("Error: %s, code: %d", http.StatusText(r.Code), r.Code)
	}
}

// only support parent in 'Bulk' related apis
func (c *Client) Bulk(items []*BulkRequest) (*BulkResponse, error) {
	reqUrl := fmt.Sprintf("http://%s/_bulk", c.Addr)

	return c.DoBulk(reqUrl, items)
}

func (c *Client) IndexBulk(index string, items []*BulkRequest) (*BulkResponse, error) {
	reqUrl := fmt.Sprintf("http://%s/%s/_bulk", c.Addr,
		url.QueryEscape(index))

	return c.DoBulk(reqUrl, items)
}

func (c *Client) IndexTypeBulk(index string, docType string, items []*BulkRequest) (*BulkResponse, error) {
	reqUrl := fmt.Sprintf("http://%s/%s/%s/_bulk", c.Addr,
		url.QueryEscape(index),
		url.QueryEscape(docType))

	return c.DoBulk(reqUrl, items)
}
<|endoftext|>


<commit_before>// Copyright 2013 Hui Chen
// Copyright 2016 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package engine

import (
	// "fmt"
	"strings"

	"github.com/go-ego/gpy"
	"github.com/go-ego/riot/types"
)

type segmenterRequest struct {
	docId uint64
	hash  uint32
	data  types.DocIndexData
	// data        types.DocumentIndexData
	forceUpdate bool
}

// Map defines the type map[string][]int
type Map map[string][]int

func (engine *Engine) splitData(request segmenterRequest) (Map, int) {
	tokensMap := make(map[string][]int)
	// split data
	var (
		sqlitStr  string
		num       int
		numTokens int
	)

	if request.data.Content != "" {
		request.data.Content = strings.ToLower(request.data.Content)
		if engine.initOptions.Using == 4 {
			// use segmenter
			segments := engine.segmenter.Segment([]byte(request.data.Content))
			for _, segment := range segments {
				token := segment.Token().Text()
				if !engine.stopTokens.IsStopToken(token) {
					tokensMap[token] = append(tokensMap[token], segment.Start())
				}
			}
			numTokens += len(segments)
		}

		splitData := strings.Split(request.data.Content, "")
		num = len(splitData)
		for i := 0; i < num; i++ {
			if splitData[i] != "" {
				if !engine.stopTokens.IsStopToken(splitData[i]) {
					numTokens++
					tokensMap[splitData[i]] = append(tokensMap[splitData[i]], numTokens)
				}
				sqlitStr += splitData[i]

				if !engine.stopTokens.IsStopToken(sqlitStr) {
					numTokens++
					tokensMap[sqlitStr] = append(tokensMap[sqlitStr], numTokens)
				}

				if engine.initOptions.Using == 0 {
					// more combination
					var sqlitsStr string
					for s := i + 1; s < len(splitData); s++ {
						sqlitsStr += splitData[s]
						if !engine.stopTokens.IsStopToken(sqlitsStr) {
							numTokens++
							tokensMap[sqlitsStr] = append(tokensMap[sqlitsStr], numTokens)
						}
					}
				}

			}
		}
	}

	for _, t := range request.data.Tokens {
		if !engine.stopTokens.IsStopToken(t.Text) {
			tokensMap[t.Text] = t.Locations
		}
	}

	numTokens += len(request.data.Tokens)

	// fmt.Println("fmt.Println(tokensMap)------------", tokensMap)
	return tokensMap, numTokens
}

func (engine *Engine) segmenterData(request segmenterRequest) (Map, int) {
	tokensMap := make(map[string][]int)
	numTokens := 0

	if engine.initOptions.Using == 1 && request.data.Content != "" {
		// Content分词, 当文档正文不为空时，优先从内容分词中得到关键词
		segments := engine.segmenter.Segment([]byte(request.data.Content))
		for _, segment := range segments {
			token := segment.Token().Text()
			if !engine.stopTokens.IsStopToken(token) {
				tokensMap[token] = append(tokensMap[token], segment.Start())
			}
		}
		numTokens = len(segments)

		return tokensMap, numTokens
	}

	if engine.initOptions.Using == 2 || ((engine.initOptions.Using == 1 || engine.initOptions.Using == 3) && request.data.Content == "") {
		for _, t := range request.data.Tokens {
			if !engine.stopTokens.IsStopToken(t.Text) {
				tokensMap[t.Text] = t.Locations
			}
		}

		numTokens = len(request.data.Tokens)

		return tokensMap, numTokens
	}

	if engine.initOptions.Using == 3 && request.data.Content != "" {
		// Content分词, 当文档正文不为空时，优先从内容分词中得到关键词
		segments := engine.segmenter.Segment([]byte(request.data.Content))
		for _, segment := range segments {
			token := segment.Token().Text()
			if !engine.stopTokens.IsStopToken(token) {
				tokensMap[token] = append(tokensMap[token], segment.Start())
			}
		}

		for _, t := range request.data.Tokens {
			if !engine.stopTokens.IsStopToken(t.Text) {
				tokensMap[t.Text] = t.Locations
			}
		}

		numTokens = len(segments) + len(request.data.Tokens)

		return tokensMap, numTokens
	}

	tokenMap, lenSplitData := engine.splitData(request)

	return tokenMap, lenSplitData
}

func (engine *Engine) segmenterWorker() {
	for {
		request := <-engine.segmenterChannel
		if request.docId == 0 {
			if request.forceUpdate {
				for i := 0; i < engine.initOptions.NumShards; i++ {
					engine.indexerAddDocChannels[i] <- indexerAddDocumentRequest{forceUpdate: true}
				}
			}
			continue
		}

		shard := engine.getShard(request.hash)
		tokensMap, numTokens := engine.segmenterData(request)

		// 加入非分词的文档标签
		for _, label := range request.data.Labels {
			if !engine.initOptions.NotUsingSegmenter {
				if !engine.stopTokens.IsStopToken(label) {
					//当正文中已存在关键字时，若不判断，位置信息将会丢失
					if _, ok := tokensMap[label]; !ok {
						tokensMap[label] = []int{}
					}
				}
			} else {
				//当正文中已存在关键字时，若不判断，位置信息将会丢失
				if _, ok := tokensMap[label]; !ok {
					tokensMap[label] = []int{}
				}
			}
		}

		indexerRequest := indexerAddDocumentRequest{
			document: &types.DocumentIndex{
				DocId:       request.docId,
				TokenLength: float32(numTokens),
				Keywords:    make([]types.KeywordIndex, len(tokensMap)),
			},
			forceUpdate: request.forceUpdate,
		}
		iTokens := 0
		for k, v := range tokensMap {
			indexerRequest.document.Keywords[iTokens] = types.KeywordIndex{
				Text: k,
				// 非分词标注的词频设置为0，不参与tf-idf计算
				Frequency: float32(len(v)),
				Starts:    v}
			iTokens++
		}

		engine.indexerAddDocChannels[shard] <- indexerRequest
		if request.forceUpdate {
			for i := 0; i < engine.initOptions.NumShards; i++ {
				if i == shard {
					continue
				}
				engine.indexerAddDocChannels[i] <- indexerAddDocumentRequest{forceUpdate: true}
			}
		}
		rankerRequest := rankerAddDocRequest{
			// docId: request.docId, fields: request.data.Fields}
			docId: request.docId, fields: request.data.Fields, content: request.data.Content, attri: request.data.Attri}
		engine.rankerAddDocChannels[shard] <- rankerRequest
	}
}

// PinYin get the Chinese alphabet and abbreviation
func (engine *Engine) PinYin(hans string) []string {
	var (
		str      string
		pystr    string
		strArr   []string
		sqlitStr string
		// sqlitArr []string
	)

	//
	splitHans := strings.Split(hans, "")
	for i := 0; i < len(splitHans); i++ {
		if splitHans[i] != "" {
			strArr = append(strArr, splitHans[i])
			sqlitStr += splitHans[i]
		}
		strArr = append(strArr, sqlitStr)
	}

	// Segment 分词
	if engine.initOptions.NotUsingSegmenter {
		sehans := engine.Segment(hans)
		for h := 0; h < len(sehans); h++ {
			strArr = append(strArr, sehans[h])
		}
	}
	//
	// py := pinyin.LazyConvert(sehans[h], nil)
	py := gpy.LazyConvert(hans, nil)

	for i := 0; i < len(py); i++ {
		str += py[i][0:1]
		pystr += py[i]
		strArr = append(strArr, pystr)
		strArr = append(strArr, str)
	}
	// }

	return strArr
}
<commit_msg>Update segment<commit_after>// Copyright 2013 Hui Chen
// Copyright 2016 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package engine

import (
	// "fmt"
	"strings"

	"github.com/go-ego/gpy"
	"github.com/go-ego/riot/types"
)

type segmenterRequest struct {
	docId uint64
	hash  uint32
	data  types.DocIndexData
	// data        types.DocumentIndexData
	forceUpdate bool
}

// Map defines the type map[string][]int
type Map map[string][]int

// Segspl split seg
func (engine *Engine) Segspl(splitData []string, num int) (Map, int) {
	var (
		numTokens int
		sqlitStr  string
	)
	tokensMap := make(map[string][]int)
	for i := 0; i < num; i++ {
		if splitData[i] != "" {
			if !engine.stopTokens.IsStopToken(splitData[i]) {
				numTokens++
				tokensMap[splitData[i]] = append(tokensMap[splitData[i]], numTokens)
			}

			sqlitStr += splitData[i]
			if !engine.stopTokens.IsStopToken(sqlitStr) {
				numTokens++
				tokensMap[sqlitStr] = append(tokensMap[sqlitStr], numTokens)
			}

			if engine.initOptions.Using == 6 {
				// more combination
				var sqlitsStr string
				for s := i + 1; s < len(splitData); s++ {
					sqlitsStr += splitData[s]

					if !engine.stopTokens.IsStopToken(sqlitsStr) {
						numTokens++
						tokensMap[sqlitsStr] = append(tokensMap[sqlitsStr], numTokens)
					}
				}
			}

		}
	}

	return tokensMap, numTokens
}

func (engine *Engine) splitData(request segmenterRequest) (Map, int) {
	tokensMap := make(map[string][]int)
	var (
		num       int
		numTokens int
	)

	if request.data.Content != "" {
		request.data.Content = strings.ToLower(request.data.Content)
		if engine.initOptions.Using == 4 {
			// use segmenter
			segments := engine.segmenter.Segment([]byte(request.data.Content))
			for _, segment := range segments {
				token := segment.Token().Text()
				if !engine.stopTokens.IsStopToken(token) {
					tokensMap[token] = append(tokensMap[token], segment.Start())
				}
			}
			numTokens += len(segments)
		}

		if engine.initOptions.Using == 5 {
			// use segmenter
			splitSpaData := strings.Split(request.data.Content, " ")
			num := len(splitSpaData)
			tokenMap, numToken := engine.Segspl(splitSpaData, num)
			numTokens += numToken
			for key, val := range tokenMap {
				tokensMap[key] = val
			}
		}

		if engine.initOptions.Using != 5 {
			splitData := strings.Split(request.data.Content, "")
			num = len(splitData)
			tokenMap, numToken := engine.Segspl(splitData, num)
			numTokens += numToken
			for key, val := range tokenMap {
				tokensMap[key] = val
			}
		}
	}

	for _, t := range request.data.Tokens {
		if !engine.stopTokens.IsStopToken(t.Text) {
			tokensMap[t.Text] = t.Locations
		}
	}

	numTokens += len(request.data.Tokens)

	return tokensMap, numTokens
}

func (engine *Engine) segmenterData(request segmenterRequest) (Map, int) {
	tokensMap := make(map[string][]int)
	numTokens := 0

	if engine.initOptions.Using == 1 && request.data.Content != "" {
		// Content分词, 当文档正文不为空时，优先从内容分词中得到关键词
		segments := engine.segmenter.Segment([]byte(request.data.Content))
		for _, segment := range segments {
			token := segment.Token().Text()
			if !engine.stopTokens.IsStopToken(token) {
				tokensMap[token] = append(tokensMap[token], segment.Start())
			}
		}
		numTokens = len(segments)

		return tokensMap, numTokens
	}

	if engine.initOptions.Using == 2 || ((engine.initOptions.Using == 1 || engine.initOptions.Using == 3) && request.data.Content == "") {
		for _, t := range request.data.Tokens {
			if !engine.stopTokens.IsStopToken(t.Text) {
				tokensMap[t.Text] = t.Locations
			}
		}

		numTokens = len(request.data.Tokens)

		return tokensMap, numTokens
	}

	if engine.initOptions.Using == 3 && request.data.Content != "" {
		// Content分词, 当文档正文不为空时，优先从内容分词中得到关键词
		segments := engine.segmenter.Segment([]byte(request.data.Content))
		for _, segment := range segments {
			token := segment.Token().Text()
			if !engine.stopTokens.IsStopToken(token) {
				tokensMap[token] = append(tokensMap[token], segment.Start())
			}
		}

		for _, t := range request.data.Tokens {
			if !engine.stopTokens.IsStopToken(t.Text) {
				tokensMap[t.Text] = t.Locations
			}
		}

		numTokens = len(segments) + len(request.data.Tokens)

		return tokensMap, numTokens
	}

	tokenMap, lenSplitData := engine.splitData(request)

	return tokenMap, lenSplitData
}

func (engine *Engine) segmenterWorker() {
	for {
		request := <-engine.segmenterChannel
		if request.docId == 0 {
			if request.forceUpdate {
				for i := 0; i < engine.initOptions.NumShards; i++ {
					engine.indexerAddDocChannels[i] <- indexerAddDocumentRequest{forceUpdate: true}
				}
			}
			continue
		}

		shard := engine.getShard(request.hash)
		tokensMap, numTokens := engine.segmenterData(request)

		// 加入非分词的文档标签
		for _, label := range request.data.Labels {
			if !engine.initOptions.NotUsingSegmenter {
				if !engine.stopTokens.IsStopToken(label) {
					//当正文中已存在关键字时，若不判断，位置信息将会丢失
					if _, ok := tokensMap[label]; !ok {
						tokensMap[label] = []int{}
					}
				}
			} else {
				//当正文中已存在关键字时，若不判断，位置信息将会丢失
				if _, ok := tokensMap[label]; !ok {
					tokensMap[label] = []int{}
				}
			}
		}

		indexerRequest := indexerAddDocumentRequest{
			document: &types.DocumentIndex{
				DocId:       request.docId,
				TokenLength: float32(numTokens),
				Keywords:    make([]types.KeywordIndex, len(tokensMap)),
			},
			forceUpdate: request.forceUpdate,
		}
		iTokens := 0
		for k, v := range tokensMap {
			indexerRequest.document.Keywords[iTokens] = types.KeywordIndex{
				Text: k,
				// 非分词标注的词频设置为0，不参与tf-idf计算
				Frequency: float32(len(v)),
				Starts:    v}
			iTokens++
		}

		engine.indexerAddDocChannels[shard] <- indexerRequest
		if request.forceUpdate {
			for i := 0; i < engine.initOptions.NumShards; i++ {
				if i == shard {
					continue
				}
				engine.indexerAddDocChannels[i] <- indexerAddDocumentRequest{forceUpdate: true}
			}
		}
		rankerRequest := rankerAddDocRequest{
			// docId: request.docId, fields: request.data.Fields}
			docId: request.docId, fields: request.data.Fields, content: request.data.Content, attri: request.data.Attri}
		engine.rankerAddDocChannels[shard] <- rankerRequest
	}
}

// PinYin get the Chinese alphabet and abbreviation
func (engine *Engine) PinYin(hans string) []string {
	var (
		str      string
		pystr    string
		strArr   []string
		sqlitStr string
		// sqlitArr []string
	)

	//
	splitHans := strings.Split(hans, "")
	for i := 0; i < len(splitHans); i++ {
		if splitHans[i] != "" {
			strArr = append(strArr, splitHans[i])
			sqlitStr += splitHans[i]
		}
		strArr = append(strArr, sqlitStr)
	}

	// Segment 分词
	if engine.initOptions.NotUsingSegmenter {
		sehans := engine.Segment(hans)
		for h := 0; h < len(sehans); h++ {
			strArr = append(strArr, sehans[h])
		}
	}
	//
	// py := pinyin.LazyConvert(sehans[h], nil)
	py := gpy.LazyConvert(hans, nil)

	for i := 0; i < len(py); i++ {
		str += py[i][0:1]
		pystr += py[i]
		strArr = append(strArr, pystr)
		strArr = append(strArr, str)
	}
	// }

	return strArr
}
<|endoftext|>


<commit_before>package ec2_test

import (
	"fmt"
	"io"
	. "launchpad.net/gocheck"
	"launchpad.net/juju/go/environs/ec2"
	"net/http"
	"os"
	"path/filepath"
	"testing"
)

type imageSuite struct{}

var _ = Suite(imageSuite{})

// N.B. the image IDs in this test will need updating
// if the image directory is regenerated.
var imageTests = []struct {
	constraint ec2.ImageConstraint
	imageId    string
	err        string
}{
	{*ec2.DefaultImageConstraint, "ami-a7f539ce", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "natty",
		Architecture:      "amd64",
		PersistentStorage: false,
		Region:            "eu-west-1",
		Daily:             true,
		Desktop:           true,
	}, "ami-19fdc16d", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "natty",
		Architecture:      "i386",
		PersistentStorage: true,
		Region:            "ap-northeast-1",
		Daily:             true,
		Desktop:           true,
	}, "ami-cc9621cd", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "natty",
		Architecture:      "i386",
		PersistentStorage: false,
		Region:            "ap-northeast-1",
		Daily:             true,
		Desktop:           true,
	}, "ami-62962163", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "natty",
		Architecture:      "amd64",
		PersistentStorage: false,
		Region:            "ap-northeast-1",
		Daily:             true,
		Desktop:           true,
	}, "ami-a69621a7", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "zingy",
		Architecture:      "amd64",
		PersistentStorage: false,
		Region:            "eu-west-1",
		Daily:             true,
		Desktop:           true,
	}, "", "error getting instance types:.*"},
}

func (imageSuite) TestFindImageSpec(c *C) {
	// set up http so that all requests will be satisfied from the images directory.
	defer setTransport(setTransport(http.NewFileTransport(http.Dir("images"))))

	for i, t := range imageTests {
		id, err := ec2.FindImageSpec(&t.constraint)
		if t.err != "" {
			c.Check(err, ErrorMatches, t.err, Commentf("test %d", i))
			c.Check(id, IsNil, Commentf("test %d", i))
			continue
		}
		if !c.Check(err, IsNil, Commentf("test %d", i)) {
			continue
		}
		if !c.Check(id, NotNil, Commentf("test %d", i)) {
			continue
		}
		c.Check(id.ImageId, Equals, t.imageId)
	}
}

func setTransport(t http.RoundTripper) (old http.RoundTripper) {
	old = http.DefaultTransport
	http.DefaultTransport = t
	return
}

// regenerate all data inside the images directory.
// N.B. this second-guesses the logic inside images.go
func regenerateImages(t *testing.T) {
	if err := os.RemoveAll(imagesRoot); err != nil {
		t.Errorf("cannot remove old images: %v", err)
		return
	}
	for _, variant := range []string{"desktop", "server"} {
		for _, version := range []string{"daily", "released"} {
			for _, release := range []string{"natty", "oneiric"} {
				s := fmt.Sprintf("query/%s/%s/%s.current.txt", release, variant, version)
				t.Logf("regenerating images from %q", s)
				err := copylocal(s)
				if err != nil {
					t.Logf("regenerate: %v", err)
				}
			}
		}
	}
}

var imagesRoot = "images"

func copylocal(s string) error {
	r, err := http.Get("http://uec-images.ubuntu.com/" + s)
	if err != nil {
		return fmt.Errorf("get %q: %v", s, err)
	}
	defer r.Body.Close()
	if r.StatusCode != 200 {
		return fmt.Errorf("status on %q: %s", s, r.Status)
	}
	path := filepath.Join(filepath.FromSlash(imagesRoot), filepath.FromSlash(s))
	d, _ := filepath.Split(path)
	if err := os.MkdirAll(d, 0777); err != nil {
		return err
	}
	file, err := os.Create(path)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = io.Copy(file, r.Body)
	if err != nil {
		return fmt.Errorf("error copying image file: %v", err)
	}
	return nil
}
<commit_msg>make image_test.go use UseLocalImages<commit_after>package ec2_test

import (
	"fmt"
	"io"
	. "launchpad.net/gocheck"
	"launchpad.net/juju/go/environs/ec2"
	"net/http"
	"os"
	"path/filepath"
	"testing"
)

type imageSuite struct{}

var _ = Suite(imageSuite{})

func (imageSuite) SetUpSuite(c *C) {
	ec2.UseLocalImages(true)
}

func (imageSuite) TearDownSuite(c *C) {
	ec2.UseLocalImages(false)
}

// N.B. the image IDs in this test will need updating
// if the image directory is regenerated.
var imageTests = []struct {
	constraint ec2.ImageConstraint
	imageId    string
	err        string
}{
	{*ec2.DefaultImageConstraint, "ami-a7f539ce", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "natty",
		Architecture:      "amd64",
		PersistentStorage: false,
		Region:            "eu-west-1",
		Daily:             true,
		Desktop:           true,
	}, "ami-19fdc16d", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "natty",
		Architecture:      "i386",
		PersistentStorage: true,
		Region:            "ap-northeast-1",
		Daily:             true,
		Desktop:           true,
	}, "ami-cc9621cd", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "natty",
		Architecture:      "i386",
		PersistentStorage: false,
		Region:            "ap-northeast-1",
		Daily:             true,
		Desktop:           true,
	}, "ami-62962163", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "natty",
		Architecture:      "amd64",
		PersistentStorage: false,
		Region:            "ap-northeast-1",
		Daily:             true,
		Desktop:           true,
	}, "ami-a69621a7", ""},
	{ec2.ImageConstraint{
		UbuntuRelease:     "zingy",
		Architecture:      "amd64",
		PersistentStorage: false,
		Region:            "eu-west-1",
		Daily:             true,
		Desktop:           true,
	}, "", "error getting instance types:.*"},
}

func (imageSuite) TestFindImageSpec(c *C) {

	for i, t := range imageTests {
		id, err := ec2.FindImageSpec(&t.constraint)
		if t.err != "" {
			c.Check(err, ErrorMatches, t.err, Commentf("test %d", i))
			c.Check(id, IsNil, Commentf("test %d", i))
			continue
		}
		if !c.Check(err, IsNil, Commentf("test %d", i)) {
			continue
		}
		if !c.Check(id, NotNil, Commentf("test %d", i)) {
			continue
		}
		c.Check(id.ImageId, Equals, t.imageId)
	}
}

// regenerate all data inside the images directory.
// N.B. this second-guesses the logic inside images.go
func regenerateImages(t *testing.T) {
	if err := os.RemoveAll(imagesRoot); err != nil {
		t.Errorf("cannot remove old images: %v", err)
		return
	}
	for _, variant := range []string{"desktop", "server"} {
		for _, version := range []string{"daily", "released"} {
			for _, release := range []string{"natty", "oneiric"} {
				s := fmt.Sprintf("query/%s/%s/%s.current.txt", release, variant, version)
				t.Logf("regenerating images from %q", s)
				err := copylocal(s)
				if err != nil {
					t.Logf("regenerate: %v", err)
				}
			}
		}
	}
}

var imagesRoot = "images"

func copylocal(s string) error {
	r, err := http.Get("http://uec-images.ubuntu.com/" + s)
	if err != nil {
		return fmt.Errorf("get %q: %v", s, err)
	}
	defer r.Body.Close()
	if r.StatusCode != 200 {
		return fmt.Errorf("status on %q: %s", s, r.Status)
	}
	path := filepath.Join(filepath.FromSlash(imagesRoot), filepath.FromSlash(s))
	d, _ := filepath.Split(path)
	if err := os.MkdirAll(d, 0777); err != nil {
		return err
	}
	file, err := os.Create(path)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = io.Copy(file, r.Body)
	if err != nil {
		return fmt.Errorf("error copying image file: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"bufio"
	"context"
	"errors"
	"fmt"
	"log"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/strslice"
	"github.com/docker/docker/client"
	"github.com/docker/go-connections/nat"
)

func CreateContainer(ctx context.Context, s ServiceConfig, networkName, formattedImageName, dockerComposeFile string) (string, error) {
	cli, err := client.NewEnvClient()
	if err != nil {
		return "", &popagateError{
			originalErr: err,
			newErr:      errors.New(" :unable to intialize docker client")}
	}
	defer cli.Close()

	// 2.1 make labels
	labelsMap := make(map[string]string)
	if len(s.Labels) > 0 {
		for _, v := range s.Labels {
			onelabel := fomatLabels(v)
			labelsMap[onelabel[0]] = onelabel[1]
		}
	}
	//2.2 make ports
	portsMap := make(map[nat.Port]struct{})
	portBindingMap := make(map[nat.Port][]nat.PortBinding)
	if len(s.Ports) > 0 {
		for _, v := range s.Ports {
			oneport := fomatPorts(v)
			hostport := oneport[0]
			containerport := oneport[1]
			port, err := nat.NewPort("tcp", containerport)
			myPortBinding := nat.PortBinding{HostPort: hostport}
			if err != nil {
				log.Println(err, " :unable to create a nat.Port")
			}
			portsMap[port] = EmptyStruct{}
			portBindingMap[port] = []nat.PortBinding{myPortBinding}
		}
	}
	//2.3 create command
	cmd := strslice.StrSlice{}
	if s.Command != "" {
		sliceCommand := strings.Fields(s.Command)
		cmd = strslice.StrSlice(sliceCommand)
	}
	//2.4 create restart policy
	restartPolicy := container.RestartPolicy{}
	if s.Restart != "" {
		// you cannot set MaximumRetryCount for the following restart policies;
		// always, no, unless-stopped
		if s.Restart == "on-failure" {
			restartPolicy = container.RestartPolicy{Name: s.Restart, MaximumRetryCount: 3}
		} else {
			restartPolicy = container.RestartPolicy{Name: s.Restart}
		}

	}
	//2.5 build image
	imageName := s.Image
	if s.Build != (Buildstruct{}) {
		dockerFile := s.Build.Dockerfile
		if dockerFile == "" {
			dockerFile = "Dockerfile"
		}
		pathToDockerFile := formatComposePath(dockerComposeFile)[0]
		dockerFile = pathToDockerFile + "/" + dockerFile
		imageName, err = BuildDockerImage(ctx, dockerFile)
		if err != nil {
			return "", &popagateError{originalErr: err}
		}
	}

	//2.6 add volumes
	volume := make(map[string]struct{})
	binds := []string{}
	if len(s.Volumes) > 0 {
		vol := fomatServiceVolumes(s.Volumes[0])
		volume[vol[1]] = EmptyStruct{}
		// TODO: handle other read/write modes
		whatToBind := "meli_" + vol[0] + ":" + vol[1] + ":rw"
		binds = append(binds, whatToBind)
	}

	// TODO: we should skip creating the container again if already exists
	// instead of creating a uniquely named container name
	containerCreateResp, err := cli.ContainerCreate(
		ctx,
		&container.Config{
			Image:        imageName,
			Labels:       labelsMap,
			Env:          s.Environment,
			ExposedPorts: portsMap,
			Cmd:          cmd,
			Volumes:      volume},
		&container.HostConfig{
			PublishAllPorts: false,
			PortBindings:    portBindingMap,
			NetworkMode:     container.NetworkMode(networkName),
			RestartPolicy:   restartPolicy,
			Binds:           binds},
		nil,
		formattedImageName)
	if err != nil {
		if err != nil {
			return "", &popagateError{
				originalErr: err,
				newErr:      errors.New(" :unable to create container")}
		}
	}

	return containerCreateResp.ID, nil
}

func ContainerStart(ctx context.Context, containerId string) error {
	cli, err := client.NewEnvClient()
	if err != nil {
		return &popagateError{
			originalErr: err,
			newErr:      errors.New(" :unable to intialize docker client")}
	}
	defer cli.Close()

	err = cli.ContainerStart(
		ctx,
		containerId,
		types.ContainerStartOptions{})
	if err != nil {
		return &popagateError{
			originalErr: err,
			newErr:      fmt.Errorf(" :unable to start container %s", containerId)}

	}
	return nil
}

func ContainerLogs(ctx context.Context, containerId string, followLogs bool) error {
	cli, err := client.NewEnvClient()
	if err != nil {
		return &popagateError{
			originalErr: err,
			newErr:      errors.New(" :unable to intialize docker client")}
	}
	defer cli.Close()

	containerLogResp, err := cli.ContainerLogs(
		ctx,
		containerId,
		types.ContainerLogsOptions{
			ShowStdout: true,
			ShowStderr: true,
			Timestamps: true,
			Follow:     followLogs,
			Details:    true,
			Tail:       "all"})

	if err != nil {
		if err != nil {
			return &popagateError{
				originalErr: err,
				newErr:      fmt.Errorf(" :unable to get container logs %s", containerId)}
		}
	}
	defer containerLogResp.Close()

	scanner := bufio.NewScanner(containerLogResp)
	for scanner.Scan() {
		output := strings.Replace(scanner.Text(), "u003e", ">", -1)
		log.Println(output)
	}
	err = scanner.Err()
	if err != nil {
		log.Println(err, "error in scanning")
	}
	return nil
}
<commit_msg>fix issues/38<commit_after>package api

import (
	"bufio"
	"context"
	"errors"
	"fmt"
	"log"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/strslice"
	"github.com/docker/docker/client"
	"github.com/docker/go-connections/nat"
)

func CreateContainer(ctx context.Context, s ServiceConfig, networkName, formattedImageName, dockerComposeFile string) (string, error) {
	cli, err := client.NewEnvClient()
	if err != nil {
		return "", &popagateError{
			originalErr: err,
			newErr:      errors.New(" :unable to intialize docker client")}
	}
	defer cli.Close()

	// 2.1 make labels
	labelsMap := make(map[string]string)
	if len(s.Labels) > 0 {
		for _, v := range s.Labels {
			onelabel := fomatLabels(v)
			labelsMap[onelabel[0]] = onelabel[1]
		}
	}
	//2.2 make ports
	portsMap := make(map[nat.Port]struct{})
	portBindingMap := make(map[nat.Port][]nat.PortBinding)
	if len(s.Ports) > 0 {
		for _, v := range s.Ports {
			oneport := fomatPorts(v)
			hostport := oneport[0]
			containerport := oneport[1]
			port, err := nat.NewPort("tcp", containerport)
			myPortBinding := nat.PortBinding{HostPort: hostport}
			if err != nil {
				log.Println(err, " :unable to create a nat.Port")
			}
			portsMap[port] = EmptyStruct{}
			portBindingMap[port] = []nat.PortBinding{myPortBinding}
		}
	}
	//2.3 create command
	cmd := strslice.StrSlice{}
	if s.Command != "" {
		sliceCommand := strings.Fields(s.Command)
		cmd = strslice.StrSlice(sliceCommand)
	}
	//2.4 create restart policy
	restartPolicy := container.RestartPolicy{}
	if s.Restart != "" {
		// you cannot set MaximumRetryCount for the following restart policies;
		// always, no, unless-stopped
		if s.Restart == "on-failure" {
			restartPolicy = container.RestartPolicy{Name: s.Restart, MaximumRetryCount: 3}
		} else {
			restartPolicy = container.RestartPolicy{Name: s.Restart}
		}

	}
	//2.5 build image
	imageName := s.Image
	if s.Build != (Buildstruct{}) {
		dockerFile := s.Build.Dockerfile
		if dockerFile == "" {
			dockerFile = "Dockerfile"
		}
		pathToDockerFile := formatComposePath(dockerComposeFile)[0]
		if pathToDockerFile != "docker-compose.yml" {
			dockerFile = pathToDockerFile + "/" + dockerFile
		}
		imageName, err = BuildDockerImage(ctx, dockerFile)
		if err != nil {
			return "", &popagateError{originalErr: err}
		}
	}

	//2.6 add volumes
	volume := make(map[string]struct{})
	binds := []string{}
	if len(s.Volumes) > 0 {
		vol := fomatServiceVolumes(s.Volumes[0])
		volume[vol[1]] = EmptyStruct{}
		// TODO: handle other read/write modes
		whatToBind := "meli_" + vol[0] + ":" + vol[1] + ":rw"
		binds = append(binds, whatToBind)
	}

	// TODO: we should skip creating the container again if already exists
	// instead of creating a uniquely named container name
	containerCreateResp, err := cli.ContainerCreate(
		ctx,
		&container.Config{
			Image:        imageName,
			Labels:       labelsMap,
			Env:          s.Environment,
			ExposedPorts: portsMap,
			Cmd:          cmd,
			Volumes:      volume},
		&container.HostConfig{
			PublishAllPorts: false,
			PortBindings:    portBindingMap,
			NetworkMode:     container.NetworkMode(networkName),
			RestartPolicy:   restartPolicy,
			Binds:           binds},
		nil,
		formattedImageName)
	if err != nil {
		if err != nil {
			return "", &popagateError{
				originalErr: err,
				newErr:      errors.New(" :unable to create container")}
		}
	}

	return containerCreateResp.ID, nil
}

func ContainerStart(ctx context.Context, containerId string) error {
	cli, err := client.NewEnvClient()
	if err != nil {
		return &popagateError{
			originalErr: err,
			newErr:      errors.New(" :unable to intialize docker client")}
	}
	defer cli.Close()

	err = cli.ContainerStart(
		ctx,
		containerId,
		types.ContainerStartOptions{})
	if err != nil {
		return &popagateError{
			originalErr: err,
			newErr:      fmt.Errorf(" :unable to start container %s", containerId)}

	}
	return nil
}

func ContainerLogs(ctx context.Context, containerId string, followLogs bool) error {
	cli, err := client.NewEnvClient()
	if err != nil {
		return &popagateError{
			originalErr: err,
			newErr:      errors.New(" :unable to intialize docker client")}
	}
	defer cli.Close()

	containerLogResp, err := cli.ContainerLogs(
		ctx,
		containerId,
		types.ContainerLogsOptions{
			ShowStdout: true,
			ShowStderr: true,
			Timestamps: true,
			Follow:     followLogs,
			Details:    true,
			Tail:       "all"})

	if err != nil {
		if err != nil {
			return &popagateError{
				originalErr: err,
				newErr:      fmt.Errorf(" :unable to get container logs %s", containerId)}
		}
	}
	defer containerLogResp.Close()

	scanner := bufio.NewScanner(containerLogResp)
	for scanner.Scan() {
		output := strings.Replace(scanner.Text(), "u003e", ">", -1)
		log.Println(output)
	}
	err = scanner.Err()
	if err != nil {
		log.Println(err, "error in scanning")
	}
	return nil
}
<|endoftext|>


<commit_before>package currency

import (
	"encoding/xml"
	"github.com/mbanzon/simplehttp"
	"time"
)

const (
	ecbResourceUrl = "http://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"
	dateFormat     = "2006-01-02"
)

type currencyEnvelope struct {
	Sender string `xml:"Sender>name"`
	Cube   []cube `xml:"Cube>Cube>Cube"`
}

type timeEnvelope struct {
	Time timeCube `xml:"Cube>Cube"`
}

type timeCube struct {
	Time string `xml:"time,attr"`
}

type cube struct {
	Name string  `xml:"currency,attr"`
	Rate float64 `xml:"rate,attr"`
}

func parseEcbData() (time.Time, map[string]float64, error) {
	r := simplehttp.NewGetRequest(ecbResourceUrl)
	data, err := r.MakeRequest()
	if err != nil {
		return time.Time{}, nil, err
	}

	var c currencyEnvelope
	err = xml.Unmarshal(data, &c) // first unmarshal to get currencies
	if err != nil {
		return time.Time{}, nil, err
	}

	var t timeEnvelope
	err = xml.Unmarshal(data, &t) // second unmarshal to get time
	if err != nil {
		return time.Time{}, nil, err
	}

	currencyTime, err := time.Parse(dateFormat, t.Time.Time)
	if err != nil {
		return time.Time{}, nil, err
	}

	currencies := make(map[string]float64)

	currencies[eur] = 1

	for _, currency := range c.Cube {
		currencies[currency.Name] = currency.Rate
	}

	return currencyTime, currencies, nil
}
<commit_msg>Comment.<commit_after>package currency

import (
	"encoding/xml"
	"github.com/mbanzon/simplehttp"
	"time"
)

const (
	ecbResourceUrl = "http://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"
	dateFormat     = "2006-01-02"
)

type currencyEnvelope struct {
	Sender string `xml:"Sender>name"`
	Cube   []cube `xml:"Cube>Cube>Cube"`
}

type timeEnvelope struct {
	Time timeCube `xml:"Cube>Cube"`
}

type timeCube struct {
	Time string `xml:"time,attr"`
}

// structure used for parsing the currencies.
type cube struct {
	Name string  `xml:"currency,attr"`
	Rate float64 `xml:"rate,attr"`
}

func parseEcbData() (time.Time, map[string]float64, error) {
	r := simplehttp.NewGetRequest(ecbResourceUrl)
	data, err := r.MakeRequest()
	if err != nil {
		return time.Time{}, nil, err
	}

	var c currencyEnvelope
	err = xml.Unmarshal(data, &c) // first unmarshal to get currencies
	if err != nil {
		return time.Time{}, nil, err
	}

	var t timeEnvelope
	err = xml.Unmarshal(data, &t) // second unmarshal to get time
	if err != nil {
		return time.Time{}, nil, err
	}

	currencyTime, err := time.Parse(dateFormat, t.Time.Time)
	if err != nil {
		return time.Time{}, nil, err
	}

	currencies := make(map[string]float64)

	currencies[eur] = 1

	for _, currency := range c.Cube {
		currencies[currency.Name] = currency.Rate
	}

	return currencyTime, currencies, nil
}
<|endoftext|>


<commit_before>package gofakeit

import (
	"math"
	"math/rand"
	"strconv"
	"strings"
	"time"

	"github.com/brianvoe/gofakeit/v5/data"
)

// CurrencyInfo is a struct of currency information
type CurrencyInfo struct {
	Short string `json:"short" xml:"short"`
	Long  string `json:"long" xml:"long"`
}

// Currency will generate a struct with random currency information
func Currency() *CurrencyInfo {
	index := rand.Intn(len(data.Data["currency"]["short"]))
	return &CurrencyInfo{
		Short: data.Data["currency"]["short"][index],
		Long:  data.Data["currency"]["long"][index],
	}
}

// CurrencyShort will generate a random short currency value
func CurrencyShort() string {
	return getRandValue([]string{"currency", "short"})
}

// CurrencyLong will generate a random long currency name
func CurrencyLong() string {
	return getRandValue([]string{"currency", "long"})
}

// Price will take in a min and max value and return a formatted price
func Price(min, max float64) float64 {
	return math.Floor(randFloat64Range(min, max)*100) / 100
}

// CreditCardInfo is a struct containing credit variables
type CreditCardInfo struct {
	Type   string `json:"type" xml:"type"`
	Number string `json:"number" xml:"number"`
	Exp    string `json:"exp" xml:"exp"`
	Cvv    string `json:"cvv" xml:"cvv"`
}

// CreditCard will generate a struct full of credit card information
func CreditCard() *CreditCardInfo {
	ccType := RandomString(data.CreditCardTypes)
	return &CreditCardInfo{
		Type:   data.CreditCards[RandomString(data.CreditCardTypes)].Display,
		Number: CreditCardNumber(&CreditCardOptions{Types: []string{ccType}}),
		Exp:    CreditCardExp(),
		Cvv:    Generate(strings.Repeat("#", int(data.CreditCards[RandomString(data.CreditCardTypes)].Code.Size))),
	}
}

// CreditCardType will generate a random credit card type string
func CreditCardType() string {
	return data.CreditCards[RandomString(data.CreditCardTypes)].Display
}

// CreditCardOptions is the options for credit card number
type CreditCardOptions struct {
	Types []string
	Gaps  bool
}

// CreditCardNumber will generate a random luhn credit card number
func CreditCardNumber(cco *CreditCardOptions) string {
	if cco == nil {
		cco = &CreditCardOptions{}
	}
	if cco.Types == nil || len(cco.Types) == 0 {
		cco.Types = data.CreditCardTypes
	}
	ccType := RandomString(cco.Types)

	// Get Card info
	var cardInfo data.CreditCardInfo
	if info, ok := data.CreditCards[ccType]; ok {
		cardInfo = info
	} else {
		ccType = RandomString(data.CreditCardTypes)
		cardInfo = data.CreditCards[ccType]
	}

	// Get length and pattern
	length := RandomUint(cardInfo.Lengths)
	numStr := strconv.FormatUint(uint64(RandomUint(cardInfo.Patterns)), 10)
	numStr += strings.Repeat("#", int(length)-len(numStr))
	numStr = Numerify(numStr)
	ui, _ := strconv.ParseUint(numStr, 10, 64)

	// Loop through until its a valid luhn
	for {
		valid := isLuhn(strconv.FormatUint(ui, 10))
		if valid {
			break
		}
		ui++
	}
	numStr = strconv.FormatUint(ui, 10)

	// Add gaps to number
	if cco.Gaps {
		for i, spot := range cardInfo.Gaps {
			numStr = numStr[:(int(spot)+i)] + " " + numStr[(int(spot)+i):]
		}
	}

	return numStr
}

// CreditCardExp will generate a random credit card expiration date string
// Exp date will always be a future date
func CreditCardExp() string {
	month := strconv.Itoa(randIntRange(1, 12))
	if len(month) == 1 {
		month = "0" + month
	}

	var currentYear = time.Now().Year() - 2000
	return month + "/" + strconv.Itoa(randIntRange(currentYear+1, currentYear+10))
}

// CreditCardCvv will generate a random CVV number
// Its a string because you could have 017 as an exp date
func CreditCardCvv() string {
	return Numerify("###")
}

// isLuhn check is used for checking if credit card is a valid luhn card
func isLuhn(s string) bool {
	var t = [...]int{0, 2, 4, 6, 8, 1, 3, 5, 7, 9}
	odd := len(s) & 1
	var sum int
	for i, c := range s {
		if c < '0' || c > '9' {
			return false
		}
		if i&1 == odd {
			sum += t[c-'0']
		} else {
			sum += int(c - '0')
		}
	}
	return sum%10 == 0
}

func addPaymentLookup() {
	AddFuncLookup("currency", Info{
		Display:     "Currency",
		Category:    "payment",
		Description: "Random currency data set",
		Example:     `{short: "USD", long: "United States Dollar"}`,
		Output:      "map[string]string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CurrencyShort(), nil
		},
	})

	AddFuncLookup("currencyshort", Info{
		Display:     "Currency Short",
		Category:    "payment",
		Description: "Random currency abbreviated",
		Example:     "USD",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CurrencyShort(), nil
		},
	})

	AddFuncLookup("currencylong", Info{
		Display:     "Currency Long",
		Category:    "payment",
		Description: "Random currency",
		Example:     "United States Dollar",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CurrencyLong(), nil
		},
	})

	AddFuncLookup("price", Info{
		Display:     "Price",
		Category:    "payment",
		Description: "Random monitary price",
		Example:     "92.26",
		Output:      "float64",
		Params: []Param{
			{Field: "min", Display: "Min", Type: "float", Default: "0", Description: "Minimum price value"},
			{Field: "max", Display: "Max", Type: "float", Default: "1000", Description: "Maximum price value"},
		},
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			min, err := info.GetFloat64(m, "min")
			if err != nil {
				return nil, err
			}

			max, err := info.GetFloat64(m, "max")
			if err != nil {
				return nil, err
			}

			return Price(min, max), nil
		},
	})

	AddFuncLookup("creditcard", Info{
		Display:     "Credit Card",
		Category:    "payment",
		Description: "Random credit card data set",
		Example:     `{type: "Visa", number: "4136459948995369", exp: "01/21", cvv: "513"}`,
		Output:      "map[string]interface",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CreditCard(), nil
		},
	})

	AddFuncLookup("creditcardtype", Info{
		Display:     "Credit Card Type",
		Category:    "payment",
		Description: "Random credit card type",
		Example:     "Visa",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CreditCardType(), nil
		},
	})

	AddFuncLookup("creditcardnumber", Info{
		Display:     "Credit Card Number",
		Category:    "payment",
		Description: "Random credit card number",
		Example:     "4136459948995369",
		Output:      "int",
		Params: []Param{
			{
				Field: "types", Display: "Types", Type: "[]string", Default: "all",
				Options:     []string{"visa", "mastercard", "american-express", "diners-club", "discover", "jcb", "unionpay", "maestro", "elo", "hiper", "hipercard"},
				Description: "Minimum price value",
			},
			{Field: "gaps", Display: "Gaps", Type: "bool", Default: "false", Description: "Maximum price value"},
		},
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			types, err := info.GetStringArray(m, "types")
			if err != nil {
				return nil, err
			}
			if len(types) == 1 && types[0] == "all" {
				types = []string{}
			}

			gaps, err := info.GetBool(m, "gaps")
			if err != nil {
				return nil, err
			}

			options := CreditCardOptions{
				Types: types,
				Gaps:  gaps,
			}
			return CreditCardNumber(&options), nil
		},
	})

	AddFuncLookup("creditcardexp", Info{
		Display:     "Credit Card Exp",
		Category:    "payment",
		Description: "Random credit card expiraction date",
		Example:     "01/21",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CreditCardExp(), nil
		},
	})

	AddFuncLookup("creditcardcvv", Info{
		Display:     "Credit Card CVV",
		Category:    "payment",
		Description: "Random credit card number",
		Example:     "513",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CreditCardCvv(), nil
		},
	})
}
<commit_msg>payment - credit card params description fix<commit_after>package gofakeit

import (
	"math"
	"math/rand"
	"strconv"
	"strings"
	"time"

	"github.com/brianvoe/gofakeit/v5/data"
)

// CurrencyInfo is a struct of currency information
type CurrencyInfo struct {
	Short string `json:"short" xml:"short"`
	Long  string `json:"long" xml:"long"`
}

// Currency will generate a struct with random currency information
func Currency() *CurrencyInfo {
	index := rand.Intn(len(data.Data["currency"]["short"]))
	return &CurrencyInfo{
		Short: data.Data["currency"]["short"][index],
		Long:  data.Data["currency"]["long"][index],
	}
}

// CurrencyShort will generate a random short currency value
func CurrencyShort() string {
	return getRandValue([]string{"currency", "short"})
}

// CurrencyLong will generate a random long currency name
func CurrencyLong() string {
	return getRandValue([]string{"currency", "long"})
}

// Price will take in a min and max value and return a formatted price
func Price(min, max float64) float64 {
	return math.Floor(randFloat64Range(min, max)*100) / 100
}

// CreditCardInfo is a struct containing credit variables
type CreditCardInfo struct {
	Type   string `json:"type" xml:"type"`
	Number string `json:"number" xml:"number"`
	Exp    string `json:"exp" xml:"exp"`
	Cvv    string `json:"cvv" xml:"cvv"`
}

// CreditCard will generate a struct full of credit card information
func CreditCard() *CreditCardInfo {
	ccType := RandomString(data.CreditCardTypes)
	return &CreditCardInfo{
		Type:   data.CreditCards[RandomString(data.CreditCardTypes)].Display,
		Number: CreditCardNumber(&CreditCardOptions{Types: []string{ccType}}),
		Exp:    CreditCardExp(),
		Cvv:    Generate(strings.Repeat("#", int(data.CreditCards[RandomString(data.CreditCardTypes)].Code.Size))),
	}
}

// CreditCardType will generate a random credit card type string
func CreditCardType() string {
	return data.CreditCards[RandomString(data.CreditCardTypes)].Display
}

// CreditCardOptions is the options for credit card number
type CreditCardOptions struct {
	Types []string
	Gaps  bool
}

// CreditCardNumber will generate a random luhn credit card number
func CreditCardNumber(cco *CreditCardOptions) string {
	if cco == nil {
		cco = &CreditCardOptions{}
	}
	if cco.Types == nil || len(cco.Types) == 0 {
		cco.Types = data.CreditCardTypes
	}
	ccType := RandomString(cco.Types)

	// Get Card info
	var cardInfo data.CreditCardInfo
	if info, ok := data.CreditCards[ccType]; ok {
		cardInfo = info
	} else {
		ccType = RandomString(data.CreditCardTypes)
		cardInfo = data.CreditCards[ccType]
	}

	// Get length and pattern
	length := RandomUint(cardInfo.Lengths)
	numStr := strconv.FormatUint(uint64(RandomUint(cardInfo.Patterns)), 10)
	numStr += strings.Repeat("#", int(length)-len(numStr))
	numStr = Numerify(numStr)
	ui, _ := strconv.ParseUint(numStr, 10, 64)

	// Loop through until its a valid luhn
	for {
		valid := isLuhn(strconv.FormatUint(ui, 10))
		if valid {
			break
		}
		ui++
	}
	numStr = strconv.FormatUint(ui, 10)

	// Add gaps to number
	if cco.Gaps {
		for i, spot := range cardInfo.Gaps {
			numStr = numStr[:(int(spot)+i)] + " " + numStr[(int(spot)+i):]
		}
	}

	return numStr
}

// CreditCardExp will generate a random credit card expiration date string
// Exp date will always be a future date
func CreditCardExp() string {
	month := strconv.Itoa(randIntRange(1, 12))
	if len(month) == 1 {
		month = "0" + month
	}

	var currentYear = time.Now().Year() - 2000
	return month + "/" + strconv.Itoa(randIntRange(currentYear+1, currentYear+10))
}

// CreditCardCvv will generate a random CVV number
// Its a string because you could have 017 as an exp date
func CreditCardCvv() string {
	return Numerify("###")
}

// isLuhn check is used for checking if credit card is a valid luhn card
func isLuhn(s string) bool {
	var t = [...]int{0, 2, 4, 6, 8, 1, 3, 5, 7, 9}
	odd := len(s) & 1
	var sum int
	for i, c := range s {
		if c < '0' || c > '9' {
			return false
		}
		if i&1 == odd {
			sum += t[c-'0']
		} else {
			sum += int(c - '0')
		}
	}
	return sum%10 == 0
}

func addPaymentLookup() {
	AddFuncLookup("currency", Info{
		Display:     "Currency",
		Category:    "payment",
		Description: "Random currency data set",
		Example:     `{short: "USD", long: "United States Dollar"}`,
		Output:      "map[string]string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CurrencyShort(), nil
		},
	})

	AddFuncLookup("currencyshort", Info{
		Display:     "Currency Short",
		Category:    "payment",
		Description: "Random currency abbreviated",
		Example:     "USD",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CurrencyShort(), nil
		},
	})

	AddFuncLookup("currencylong", Info{
		Display:     "Currency Long",
		Category:    "payment",
		Description: "Random currency",
		Example:     "United States Dollar",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CurrencyLong(), nil
		},
	})

	AddFuncLookup("price", Info{
		Display:     "Price",
		Category:    "payment",
		Description: "Random monitary price",
		Example:     "92.26",
		Output:      "float64",
		Params: []Param{
			{Field: "min", Display: "Min", Type: "float", Default: "0", Description: "Minimum price value"},
			{Field: "max", Display: "Max", Type: "float", Default: "1000", Description: "Maximum price value"},
		},
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			min, err := info.GetFloat64(m, "min")
			if err != nil {
				return nil, err
			}

			max, err := info.GetFloat64(m, "max")
			if err != nil {
				return nil, err
			}

			return Price(min, max), nil
		},
	})

	AddFuncLookup("creditcard", Info{
		Display:     "Credit Card",
		Category:    "payment",
		Description: "Random credit card data set",
		Example:     `{type: "Visa", number: "4136459948995369", exp: "01/21", cvv: "513"}`,
		Output:      "map[string]interface",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CreditCard(), nil
		},
	})

	AddFuncLookup("creditcardtype", Info{
		Display:     "Credit Card Type",
		Category:    "payment",
		Description: "Random credit card type",
		Example:     "Visa",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CreditCardType(), nil
		},
	})

	AddFuncLookup("creditcardnumber", Info{
		Display:     "Credit Card Number",
		Category:    "payment",
		Description: "Random credit card number",
		Example:     "4136459948995369",
		Output:      "int",
		Params: []Param{
			{
				Field: "types", Display: "Types", Type: "[]string", Default: "all",
				Options:     []string{"visa", "mastercard", "american-express", "diners-club", "discover", "jcb", "unionpay", "maestro", "elo", "hiper", "hipercard"},
				Description: "A select number of types you want to use when generating a credit card number",
			},
			{Field: "gaps", Display: "Gaps", Type: "bool", Default: "false", Description: "Whether or not to have gaps in number"},
		},
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			types, err := info.GetStringArray(m, "types")
			if err != nil {
				return nil, err
			}
			if len(types) == 1 && types[0] == "all" {
				types = []string{}
			}

			gaps, err := info.GetBool(m, "gaps")
			if err != nil {
				return nil, err
			}

			options := CreditCardOptions{
				Types: types,
				Gaps:  gaps,
			}
			return CreditCardNumber(&options), nil
		},
	})

	AddFuncLookup("creditcardexp", Info{
		Display:     "Credit Card Exp",
		Category:    "payment",
		Description: "Random credit card expiraction date",
		Example:     "01/21",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CreditCardExp(), nil
		},
	})

	AddFuncLookup("creditcardcvv", Info{
		Display:     "Credit Card CVV",
		Category:    "payment",
		Description: "Random credit card number",
		Example:     "513",
		Output:      "string",
		Call: func(m *map[string][]string, info *Info) (interface{}, error) {
			return CreditCardCvv(), nil
		},
	})
}
<|endoftext|>