<commit_before>package oembed

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

// Oembed contains list of available oembed items (official endpoints)
type Oembed struct {
	items []*Item
}

// Endpoint contains single endpoint to check against
type Endpoint struct {
	URL       string   `json:"url"`
	Discovery bool     `json:"discovery,omitempty"`
	Schemes   []string `json:"schemes,omitempty"`
}

// Provider contains a single provider which can have multiple endpoints
type Provider struct {
	Name      string     `json:"provider_name"`
	URL       string     `json:"provider_url"`
	Endpoints []Endpoint `json:"endpoints"`
}

// Item contains data for a schema
type Item struct {
	EndpointURL  string
	ProviderName string
	ProviderURL  string
	regex        *regexp.Regexp
}

// ComposeURL returns url of oembed resource ready to be queried
func (item *Item) ComposeURL(u string) string {
	return item.EndpointURL + url.QueryEscape(u)
}

// FetchOembed return oembed info from an url containing it
func (item *Item) FetchOembed(u string, client *http.Client) (*Info, error) {
	resURL := item.ComposeURL(u)

	var resp *http.Response
	var err error

	if client != nil {
		resp, err = client.Get(resURL)
	} else {
		resp, err = http.Get(resURL)
	}

	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	if resp.StatusCode > 200 {
		return &Info{Status: resp.StatusCode}, nil
	}

	reader := io.LimitReader(resp.Body, 40000) // 40 KB max

	info := NewInfo()
	err = info.FillFromJSON(reader)

	if err != nil {
		return nil, err
	}

	if len(info.URL) == 0 {
		info.URL = u
	}

	if len(info.ProviderURL) == 0 {
		info.ProviderURL = item.ProviderURL
	}

	if len(info.ProviderName) == 0 {
		info.ProviderName = item.ProviderName
	}

	return info, nil
}

// MatchURL tests if given url applies to the endpoint
func (item *Item) MatchURL(url string) bool {
	return item.regex.MatchString(strings.Trim(url, "\r\n"))
}

// NewOembed creates Oembed instance
func NewOembed() *Oembed {
	return &Oembed{}
}

// ParseProviders build oembed endpoint list based on provided json stream
func (o *Oembed) ParseProviders(buf io.Reader) error {
	var providers []Provider

	data, err := ioutil.ReadAll(buf)

	if err != nil {
		return err
	}

	err = json.Unmarshal(data, &providers)

	if err != nil {
		return err
	}

	var items []*Item

	for _, provider := range providers {
		for _, endpoint := range provider.Endpoints {
			if len(endpoint.Schemes) == 0 {
				endpoint.Schemes = append(endpoint.Schemes, strings.TrimRight(provider.URL, "/")+"/*")
			}
			for _, schema := range endpoint.Schemes {
				or := &Item{ProviderName: provider.Name, ProviderURL: provider.URL}
				or.EndpointURL = o.prepareEndpointURL(endpoint.URL)
				or.regex = o.convertSchemaURL2Regexp(schema)
				items = append(items, or)
			}
		}
	}

	o.items = items

	return nil
}

// FindItem returns Oembed item based on provided url
func (o *Oembed) FindItem(url string) *Item {
	for _, or := range o.items {
		if or.MatchURL(url) {
			return or
		}
	}

	return nil
}

// TODO: add more intelligent parameters parsing
func (o *Oembed) prepareEndpointURL(url string) string {
	url = strings.Replace(url, "{format}", "json", -1)
	url = strings.Replace(url, "/*", "", -1) // hack for Ora TV.. wtf they put in?
	if strings.IndexRune(url, '?') == -1 {
		url += "?format=json&url="
	} else {
		url += "&format=json&url="
	}

	return url
}

// TODO: precompile and move out regexes from the function
func (o *Oembed) convertSchemaURL2Regexp(url string) *regexp.Regexp {
	// domain replacements
	url = strings.Replace(url, "?", "\\?", -1)
	re1 := regexp.MustCompile("^(https?://[^/]*?)\\*(.+)$")
	url = re1.ReplaceAllString(url, "${1}[^/]%?${2}")
	re2 := regexp.MustCompile("^(https?://[^/]*?/.*?)\\*(.+)$")
	url = re2.ReplaceAllString(url, "${1}.%?${2}")
	re3 := regexp.MustCompile("^(https?://.*?)\\*$")
	url = re3.ReplaceAllString(url, "${1}.%")
	re4 := regexp.MustCompile("^http://")
	url = re4.ReplaceAllString(url, "https?://")
	url = strings.Replace(url, "%", "*", -1)
	////
	res, err := regexp.Compile("^" + url + "$")

	if err != nil {
		panic(err)
	}

	return res
}
<commit_msg>added new flag to oembed.Item which decides if we need to compose url or not<commit_after>package oembed

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

// Oembed contains list of available oembed items (official endpoints)
type Oembed struct {
	items []*Item
}

// Endpoint contains single endpoint to check against
type Endpoint struct {
	URL       string   `json:"url"`
	Discovery bool     `json:"discovery,omitempty"`
	Schemes   []string `json:"schemes,omitempty"`
}

// Provider contains a single provider which can have multiple endpoints
type Provider struct {
	Name      string     `json:"provider_name"`
	URL       string     `json:"provider_url"`
	Endpoints []Endpoint `json:"endpoints"`
}

// Item contains data for a schema
type Item struct {
	IsEndpointURLComplete bool
	EndpointURL           string
	ProviderName          string
	ProviderURL           string
	regex                 *regexp.Regexp
}

// ComposeURL returns url of oembed resource ready to be queried
func (item *Item) ComposeURL(u string) string {
	if item.IsEndpointURLComplete {
		return item.EndpointURL
	}

	return item.EndpointURL + url.QueryEscape(u)
}

// FetchOembed return oembed info from an url containing it
func (item *Item) FetchOembed(u string, client *http.Client) (*Info, error) {
	resURL := item.ComposeURL(u)

	var resp *http.Response
	var err error

	if client != nil {
		resp, err = client.Get(resURL)
	} else {
		resp, err = http.Get(resURL)
	}

	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	if resp.StatusCode > 200 {
		return &Info{Status: resp.StatusCode}, nil
	}

	reader := io.LimitReader(resp.Body, 40000) // 40 KB max

	info := NewInfo()
	err = info.FillFromJSON(reader)

	if err != nil {
		return nil, err
	}

	if len(info.URL) == 0 {
		info.URL = u
	}

	if len(info.ProviderURL) == 0 {
		info.ProviderURL = item.ProviderURL
	}

	if len(info.ProviderName) == 0 {
		info.ProviderName = item.ProviderName
	}

	return info, nil
}

// MatchURL tests if given url applies to the endpoint
func (item *Item) MatchURL(url string) bool {
	return item.regex.MatchString(strings.Trim(url, "\r\n"))
}

// NewOembed creates Oembed instance
func NewOembed() *Oembed {
	return &Oembed{}
}

// ParseProviders build oembed endpoint list based on provided json stream
func (o *Oembed) ParseProviders(buf io.Reader) error {
	var providers []Provider

	data, err := ioutil.ReadAll(buf)

	if err != nil {
		return err
	}

	err = json.Unmarshal(data, &providers)

	if err != nil {
		return err
	}

	var items []*Item

	for _, provider := range providers {
		for _, endpoint := range provider.Endpoints {
			if len(endpoint.Schemes) == 0 {
				endpoint.Schemes = append(endpoint.Schemes, strings.TrimRight(provider.URL, "/")+"/*")
			}
			for _, schema := range endpoint.Schemes {
				or := &Item{ProviderName: provider.Name, ProviderURL: provider.URL}
				or.EndpointURL = o.prepareEndpointURL(endpoint.URL)
				or.regex = o.convertSchemaURL2Regexp(schema)
				items = append(items, or)
			}
		}
	}

	o.items = items

	return nil
}

// FindItem returns Oembed item based on provided url
func (o *Oembed) FindItem(url string) *Item {
	for _, or := range o.items {
		if or.MatchURL(url) {
			return or
		}
	}

	return nil
}

// TODO: add more intelligent parameters parsing
func (o *Oembed) prepareEndpointURL(url string) string {
	url = strings.Replace(url, "{format}", "json", -1)
	url = strings.Replace(url, "/*", "", -1) // hack for Ora TV.. wtf they put in?
	if strings.IndexRune(url, '?') == -1 {
		url += "?format=json&url="
	} else {
		url += "&format=json&url="
	}

	return url
}

// TODO: precompile and move out regexes from the function
func (o *Oembed) convertSchemaURL2Regexp(url string) *regexp.Regexp {
	// domain replacements
	url = strings.Replace(url, "?", "\\?", -1)
	re1 := regexp.MustCompile("^(https?://[^/]*?)\\*(.+)$")
	url = re1.ReplaceAllString(url, "${1}[^/]%?${2}")
	re2 := regexp.MustCompile("^(https?://[^/]*?/.*?)\\*(.+)$")
	url = re2.ReplaceAllString(url, "${1}.%?${2}")
	re3 := regexp.MustCompile("^(https?://.*?)\\*$")
	url = re3.ReplaceAllString(url, "${1}.%")
	re4 := regexp.MustCompile("^http://")
	url = re4.ReplaceAllString(url, "https?://")
	url = strings.Replace(url, "%", "*", -1)
	////
	res, err := regexp.Compile("^" + url + "$")

	if err != nil {
		panic(err)
	}

	return res
}
<|endoftext|>


<commit_before>package orm

import (
	//	"fmt"
	"testing"
)

func TestSave(t *testing.T) {
	//connect to Postgres
	orm, scream := ConnectToPostgres()
	if scream != nil {
		panic(scream)
	}

	ormTest, err := orm.NewHandler(DSLTest{})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	//check if the object is stored in the table and if the ID is populated after insert
	ormTest.DropTable()
	ormTest.CreateTable()
	dslTest := DSLTest{FieldString: "teststring", FieldInt: 123}
	err = ormTest.Save(&dslTest)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	id := dslTest.ID
	if id != 1 {
		t.Fatalf("want: 1; got: %v", id)
	}

	dslTestFind, err := ormTest.Select().ByID(id)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj := dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	obj.FieldInt = 222
	err = ormTest.Save(&obj)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	dslTestFindUptated, err := ormTest.Select().ByID(id)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj = dslTestFindUptated.(DSLTest)

	if dslTestFindUptated == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 222 {
		t.Fatalf("want: 222, got: %v", obj.FieldInt)
	}

	dslTestNegativeID := DSLTest{FieldString: "teststring", FieldInt: 123, ID: -1}
	err = ormTest.Save(&dslTestNegativeID)
	if err.Error() != "Negative ID not allowed: {-1 teststring 123}" {
		t.Fatalf("want: `Negative ID not allowed: {-1 teststring 123}`, got `%v`", err)
	}
}

func TestAssembleValuesArray(t *testing.T) {

}

func TestAssembleSQLInsertStatement(t *testing.T) {
	var handler Handler
	handler.table = DSLTest{}

	//check if the sqlInsert string is assembled correctly
	handler.assembleSQLInsert()
	expected := `insert into DSLTest(FieldString, FieldInt) values ($1, $2) RETURNING id;`
	got := handler.insertSQL
	if got != expected {
		t.Fatalf("\nExpected:\t %v\nGot:\t\t %v\n", expected, got)
	}

	//expectedMap := []saveField{}
	mapInsert := handler.insertMap
	if len(mapInsert) != 2 {
		t.Fatalf("\nmapInsert lengh:\t %v\nWant:\t\t\t 2\n", len(mapInsert))
	}

	if mapInsert[0].name != "FieldString" || mapInsert[0].fieldType != "string" {
		t.Fatalf("\nmapInsert[0] got:\t %v\nWant:\t\t\t {FieldString string}\n", mapInsert[0])
	}

	if mapInsert[1].name != "FieldInt" || mapInsert[1].fieldType != "int" {
		t.Fatalf("\nmapInsert[0] got:\t %v\nWant:\t\t\t {FieldInt int}\n", mapInsert[1])
	}
}

func TestAssembleSQLUpdate(t *testing.T) {
	var handler Handler
	handler.table = DSLTest{}

	handler.assembleSQLUpdate()
	expected := `update DSLTest set FieldString = $1, FieldInt = $2 where id = $3`
	got := handler.updateSQL
	if got != expected {
		t.Fatalf("\nExpected:\t %v\nGot:\t\t %v\n", expected, got)
	}

	updateMap := handler.updateMap
	if len(updateMap) != 3 {
		t.Fatalf("updateMap lenght: %v, want: 3", len(updateMap))
	}

	if updateMap[0].name != "FieldString" || updateMap[0].fieldType != "string" {
		t.Fatalf("updateMap[0] got:\t %v\nWant:\t\t\t {FieldString string}\n", updateMap[0])
	}

	if updateMap[1].name != "FieldInt" || updateMap[1].fieldType != "int" {
		t.Fatalf("updateMap[1] got:\t %v\nWant:\t\t\t {FieldInt int}\n", updateMap[1])
	}

	if updateMap[2].name != "ID" || updateMap[2].fieldType != "int" {
		t.Fatalf("updateMap[2] got: %v, want: {ID int}\n", updateMap[2])
	}
}

func TestInsert(t *testing.T) {
	//connect to Postgres
	orm, scream := ConnectToPostgres()
	if scream != nil {
		panic(scream)
	}

	ormTest, err := orm.NewHandler(DSLTest{})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	//check if the object is stored in the table and if the ID is populated after insert
	ormTest.DropTable()
	ormTest.CreateTable()
	dslTest := DSLTest{FieldString: "teststring", FieldInt: 123}
	err = ormTest.Save(&dslTest)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	if dslTest.ID != 1 {
		t.Fatalf("\ndslTest.ID got:\t %v\nWant:\t\t\t 1\n", dslTest.ID)
	}

	//force an error by passing an object with a wrong type
	dslTestWithoutID := DSLTestWithoutID{FieldString: "teststring", FieldInt: 123}
	err = ormTest.Save(&dslTestWithoutID)
	if err.Error() != "Object table name (DSLTestWithoutID) is diferent from handler table name (DSLTest)" {
		t.Fatalf("Error expected: 'Object table name (DSLTestWithoutID) is diferent from handler table name (DSLTest)'\nError got: %v\n", err)
	}

	//force an error by droping the table before the insert
	ormTest.DropTable()
	err = ormTest.Save(&dslTest)
	if err.Error() != `pq: relation "dsltest" does not exist` {
		t.Fatalf("Error expected: 'pq: relation \"dsltest\" does not exist'\nError got: %v\n", err)
	}
}
<commit_msg>Add unit test for Insert and Update methods<commit_after>package orm

import (
	"reflect"
	"testing"
)

func TestSave(t *testing.T) {
	//connect to Postgres
	orm, scream := ConnectToPostgres()
	if scream != nil {
		panic(scream)
	}

	ormTest, err := orm.NewHandler(DSLTest{})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	//check if the object is stored in the table and if the ID is populated after insert
	ormTest.DropTable()
	ormTest.CreateTable()
	dslTest := DSLTest{FieldString: "teststring", FieldInt: 123}
	err = ormTest.Save(&dslTest)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	id := dslTest.ID
	if id != 1 {
		t.Fatalf("want: 1; got: %v", id)
	}

	dslTestFind, err := ormTest.Select().ByID(id)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj := dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	obj.FieldInt = 222
	err = ormTest.Save(&obj)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	dslTestFindUptated, err := ormTest.Select().ByID(id)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj = dslTestFindUptated.(DSLTest)

	if dslTestFindUptated == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 222 {
		t.Fatalf("want: 222, got: %v", obj.FieldInt)
	}

	dslTestNegativeID := DSLTest{FieldString: "teststring", FieldInt: 123, ID: -1}
	err = ormTest.Save(&dslTestNegativeID)
	if err.Error() != "Negative ID not allowed: {-1 teststring 123}" {
		t.Fatalf("want: `Negative ID not allowed: {-1 teststring 123}`, got `%v`", err)
	}
}

func TestInsertAndUpdate(t *testing.T) {
	//connect to Postgres
	orm, scream := ConnectToPostgres()
	if scream != nil {
		panic(scream)
	}

	//create a new handler for DSLTest structure
	ormTest, err := orm.NewHandler(DSLTest{})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	//DropTable & CreateTable
	ormTest.DropTable()
	ormTest.CreateTable()

	//create a test object
	dslTest := DSLTest{FieldString: "teststring", FieldInt: 123}
	object := reflect.ValueOf(&dslTest).Elem()

	//insert on db
	err = ormTest.insert(object)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	if dslTest.ID != 1 {
		t.Fatalf("\ndslTest.ID got:\t %v\nWant:\t\t\t 1\n", dslTest.ID)
	}

	//check if the object was persisted
	dslTestFind, err := ormTest.Select().ByID(1)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj := dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 123 {
		t.Fatalf("want: 123, got: %v", obj.FieldInt)
	}

	//change FieldInt atribute
	obj.FieldInt = 222

	//update on db
	object = reflect.ValueOf(&obj).Elem()
	err = ormTest.update(object)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	//check if the object was correctly updated
	dslTestFind, err = ormTest.Select().ByID(1)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj = dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 222 {
		t.Fatalf("want: 222, got: %v", obj.FieldInt)
	}

	//force a error on insert
	oldInsertSQL := ormTest.insertSQL
	ormTest.insertSQL = "wrong-sql"
	err = ormTest.insert(object)
	if err.Error() != "pq: syntax error at or near \"wrong\"" {
		t.Fatalf("want: `pq: syntax error at or near \"wrong\"`, got: `%v`", err)
	}
	ormTest.insertSQL = oldInsertSQL

	//force a error on insert
	oldUpdateSQL := ormTest.updateSQL
	ormTest.updateSQL = "wrong-sql"
	err = ormTest.update(object)
	if err.Error() != "pq: syntax error at or near \"wrong\"" {
		t.Fatalf("want: `pq: syntax error at or near \"wrong\"`, got: `%v`", err)
	}
	ormTest.updateSQL = oldUpdateSQL
}

func TestAssembleValuesArray(t *testing.T) {

}

func TestAssembleSQLInsertStatement(t *testing.T) {
	var handler Handler
	handler.table = DSLTest{}

	//check if the sqlInsert string is assembled correctly
	handler.assembleSQLInsert()
	expected := `insert into DSLTest(FieldString, FieldInt) values ($1, $2) RETURNING id;`
	got := handler.insertSQL
	if got != expected {
		t.Fatalf("\nExpected:\t %v\nGot:\t\t %v\n", expected, got)
	}

	//expectedMap := []saveField{}
	mapInsert := handler.insertMap
	if len(mapInsert) != 2 {
		t.Fatalf("\nmapInsert lengh:\t %v\nWant:\t\t\t 2\n", len(mapInsert))
	}

	if mapInsert[0].name != "FieldString" || mapInsert[0].fieldType != "string" {
		t.Fatalf("\nmapInsert[0] got:\t %v\nWant:\t\t\t {FieldString string}\n", mapInsert[0])
	}

	if mapInsert[1].name != "FieldInt" || mapInsert[1].fieldType != "int" {
		t.Fatalf("\nmapInsert[0] got:\t %v\nWant:\t\t\t {FieldInt int}\n", mapInsert[1])
	}
}

func TestAssembleSQLUpdate(t *testing.T) {
	var handler Handler
	handler.table = DSLTest{}

	handler.assembleSQLUpdate()
	expected := `update DSLTest set FieldString = $1, FieldInt = $2 where id = $3`
	got := handler.updateSQL
	if got != expected {
		t.Fatalf("\nExpected:\t %v\nGot:\t\t %v\n", expected, got)
	}

	updateMap := handler.updateMap
	if len(updateMap) != 3 {
		t.Fatalf("updateMap lenght: %v, want: 3", len(updateMap))
	}

	if updateMap[0].name != "FieldString" || updateMap[0].fieldType != "string" {
		t.Fatalf("updateMap[0] got:\t %v\nWant:\t\t\t {FieldString string}\n", updateMap[0])
	}

	if updateMap[1].name != "FieldInt" || updateMap[1].fieldType != "int" {
		t.Fatalf("updateMap[1] got:\t %v\nWant:\t\t\t {FieldInt int}\n", updateMap[1])
	}

	if updateMap[2].name != "ID" || updateMap[2].fieldType != "int" {
		t.Fatalf("updateMap[2] got: %v, want: {ID int}\n", updateMap[2])
	}
}
<|endoftext|>


<commit_before>package todolite

import (
	"fmt"
)

type Attachments map[string]interface{}

type TodoItem struct {
	Revision    string      `json:"_rev"`
	Id          string      `json:"_id"`
	Checked     bool        `json:"checked"`
	CreatedAt   string      `json:"created_at"`
	ListId      string      `json:"list_id"`
	Title       string      `json:"title"`
	Type        string      `json:"task"`
	OcrDecoded  string      `json:"ocr_decoded"`
	Attachments Attachments `json:"_attachments"`
}

func (t TodoItem) AttachmentUrl(dbUrl string) string {

	attachmentUrl := ""
	for k, _ := range t.Attachments {
		attachmentUrl = fmt.Sprintf("%s/%s/%s", dbUrl, t.Id, k)
		return attachmentUrl
	}
	return attachmentUrl

}
<commit_msg>type field was being mangled<commit_after>package todolite

import (
	"fmt"
)

type Attachments map[string]interface{}

type TodoItem struct {
	Revision    string      `json:"_rev"`
	Id          string      `json:"_id"`
	Checked     bool        `json:"checked"`
	CreatedAt   string      `json:"created_at"`
	ListId      string      `json:"list_id"`
	Title       string      `json:"title"`
	Type        string      `json:"type"`
	OcrDecoded  string      `json:"ocr_decoded"`
	Attachments Attachments `json:"_attachments"`
}

func (t TodoItem) AttachmentUrl(dbUrl string) string {

	attachmentUrl := ""
	for k, _ := range t.Attachments {
		attachmentUrl = fmt.Sprintf("%s/%s/%s", dbUrl, t.Id, k)
		return attachmentUrl
	}
	return attachmentUrl

}
<|endoftext|>


<commit_before>package tokenex

func NewConfig(baseUrl string, id string, apiKey string) Config {
	var config Config
	config.init(baseUrl, id, apiKey)
	return config
}

func Tokenize(data string, tokenScheme int, config Config) (TokenResponse, error) {
	response := TokenResponse{}
	request := map[string]interface{}{
		"Action":      "Tokenize",
		"Data":        data,
		"TokenScheme": tokenScheme,
	}
	err := response.get(&response, request, config)
	return response, err
}

func TokenizeFromEncryptedValue(data string, tokenScheme int, config Config) (TokenResponse, error) {
	response := TokenResponse{}
	request := map[string]interface{}{
		"Action":       "TokenizeFromEncryptedValue",
		"EcryptedData": data,
		"TokenScheme":  tokenScheme,
	}

	err := response.get(&response, request, config)
	return response, err
}

func Detokenize(token string, config Config) (ValueResponse, error) {
	response := ValueResponse{}
	request := map[string]interface{}{
		"Action": "Detokenize",
		"Token":  token,
	}
	err := response.get(&response, request, config)
	return response, err
}

func Validate(token string, config Config) (ValidateResponse, error) {
	response := ValidateResponse{}
	request := map[string]interface{}{
		"Action": "ValidateToken",
		"Token":  token,
	}
	err := response.get(&response, request, config)
	return response, err
}

func Delete(token string, config Config) (DeleteResponse, error) {
	response := DeleteResponse{}
	request := map[string]interface{}{
		"Action": "DeleteToken",
		"Token":  token,
	}
	err := response.get(&response, request, config)
	return response, err
}
<commit_msg>:penguin: Formatting [ci skip]<commit_after>package tokenex

func NewConfig(baseUrl string, id string, apiKey string) Config {
	var config Config
	config.init(baseUrl, id, apiKey)
	return config
}

func Tokenize(data string, tokenScheme int, config Config) (TokenResponse, error) {
	response := TokenResponse{}
	request := map[string]interface{}{
		"Action":      "Tokenize",
		"Data":        data,
		"TokenScheme": tokenScheme,
	}
	err := response.get(&response, request, config)
	return response, err
}

func TokenizeFromEncryptedValue(data string, tokenScheme int, config Config) (TokenResponse, error) {
	response := TokenResponse{}
	request := map[string]interface{}{
		"Action":       "TokenizeFromEncryptedValue",
		"EcryptedData": data,
		"TokenScheme":  tokenScheme,
	}
	err := response.get(&response, request, config)
	return response, err
}

func Detokenize(token string, config Config) (ValueResponse, error) {
	response := ValueResponse{}
	request := map[string]interface{}{
		"Action": "Detokenize",
		"Token":  token,
	}
	err := response.get(&response, request, config)
	return response, err
}

func Validate(token string, config Config) (ValidateResponse, error) {
	response := ValidateResponse{}
	request := map[string]interface{}{
		"Action": "ValidateToken",
		"Token":  token,
	}
	err := response.get(&response, request, config)
	return response, err
}

func Delete(token string, config Config) (DeleteResponse, error) {
	response := DeleteResponse{}
	request := map[string]interface{}{
		"Action": "DeleteToken",
		"Token":  token,
	}
	err := response.get(&response, request, config)
	return response, err
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Sean Hickey <sean@headzoo.io>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ocean

import (
	"bufio"
	"bytes"
	"errors"
	"io"
)

// TokenClass is a top-level token; a word, space, comment, unknown.
type TokenClass string

// TokenValue is the value of the token, usually a string.
type TokenValue string

// TokenState is used within the lexer state machine to keep track of the current state.
type TokenState int

const (
	TOKEN_UNKNOWN  TokenClass = "UNKNOWN"
	TOKEN_WORD     TokenClass = "WORD"
	TOKEN_SPACE    TokenClass = "SPACE"
	TOKEN_PIPE     TokenClass = "PIPE"
	TOKEN_REDIRECT TokenClass = "REDIRECT"

	STATE_START           TokenState = 0
	STATE_WORD            TokenState = 1
	STATE_ESCAPING        TokenState = 2
	STATE_ESCAPING_QUOTED TokenState = 3
	STATE_QUOTED_ESCAPING TokenState = 4
	STATE_QUOTED          TokenState = 5
	STATE_COMMENT         TokenState = 6
	STATE_EMIT            TokenState = 7
)

// Token represents a single "token" found within a stream.
type Token struct {
	Class TokenClass
	Value TokenValue
}

// Creates and returns a new Token instance.
func NewToken(class TokenClass, value TokenValue) *Token {
	return &Token{
		Class: class,
		Value: value,
	}
}

// Tokenizer turns an input stream in to a sequence of typed tokens.
type Tokenizer struct {
	input      *bufio.Reader
	classifier *Classifier
}

// Creates and returns a new tokenizer.
func NewTokenizer(reader io.Reader) *Tokenizer {
	return &Tokenizer{
		input:      bufio.NewReader(reader),
		classifier: NewClassifier(),
	}
}

// NextToken returns the next token in the stream, and an error value. If there are no more
// tokens available, the error value will be io.EOF.
func (tokenizer *Tokenizer) NextToken() (*Token, error) {
	var (
		err       error
		next      rune
		class     RuneClass
		tokenType TokenClass
	)

	state := STATE_START
	buffer := bytes.NewBuffer(make([]byte, 0, 1000))

SCAN:
	for state != STATE_EMIT {
		next, class, err = tokenizer.readRune()
		if err != nil {
			return nil, err
		}

		switch state {
		case STATE_START: // no runes read yet
			{
				switch class {
				case RUNE_EOF:
					{
						return nil, io.EOF
					}
				case RUNE_CHAR:
					{
						tokenType = TOKEN_WORD
						buffer.WriteRune(next)
						state = STATE_WORD
					}
				case RUNE_SPACE:
					{
					}
				case RUNE_QUOTE_DOUBLE:
					{
						tokenType = TOKEN_WORD
						state = STATE_QUOTED_ESCAPING
					}
				case RUNE_QUOTE_SINGLE:
					{
						tokenType = TOKEN_WORD
						state = STATE_QUOTED
					}
				case RUNE_ESCAPE:
					{
						tokenType = TOKEN_WORD
						state = STATE_ESCAPING
					}
				case RUNE_PIPE:
					{
						tokenType = TOKEN_PIPE
						buffer.WriteRune(next)
						state = STATE_EMIT
					}
				case RUNE_REDIRECT:
					{
						tokenType = TOKEN_REDIRECT
						buffer.WriteRune(next)
						state = STATE_EMIT

						n, _, e := tokenizer.readRune()
						if e != nil {
							return nil, err
						}
						if n == next {
							buffer.WriteRune(next)
						} else {
							tokenizer.unreadRune()
						}
					}
				default:
					{
						return nil, errorf("Unknown rune: %v", next)
					}
				}
			}
		case STATE_WORD: // in a regular word
			{
				switch class {
				case RUNE_EOF:
					{
						break SCAN
					}
				case RUNE_CHAR:
					{
						buffer.WriteRune(next)
					}
				case RUNE_SPACE:
					{
						tokenizer.unreadRune()
						break SCAN
					}
				case RUNE_QUOTE_DOUBLE:
					{
						state = STATE_QUOTED_ESCAPING
					}
				case RUNE_QUOTE_SINGLE:
					{
						state = STATE_QUOTED
					}
				case RUNE_ESCAPE:
					{
						state = STATE_ESCAPING
					}
				case RUNE_PIPE:
					{
						tokenizer.unreadRune()
						state = STATE_EMIT
					}
				case RUNE_REDIRECT:
					{
						tokenizer.unreadRune()
						state = STATE_EMIT
					}
				default:
					{
						return nil, errorf("Unknown rune: %v", next)
					}
				}
			}
		case STATE_ESCAPING: // the next rune after an escape character
			{
				switch class {
				case RUNE_EOF:
					{
						err = errors.New("EOF found after escape character")
						break SCAN
					}
				case RUNE_CHAR, RUNE_SPACE, RUNE_QUOTE_DOUBLE, RUNE_QUOTE_SINGLE, RUNE_ESCAPE,
					RUNE_PIPE, RUNE_REDIRECT:
					{
						state = STATE_WORD
						buffer.WriteRune(next)
					}
				default:
					{
						return nil, errorf("Unknown rune: %v", next)
					}
				}
			}
		case STATE_ESCAPING_QUOTED: // the next rune after an escape character, in double quotes
			{
				switch class {
				case RUNE_EOF:
					{
						err = errors.New("EOF found after escape character")
						break SCAN
					}
				case RUNE_CHAR, RUNE_SPACE, RUNE_QUOTE_DOUBLE, RUNE_QUOTE_SINGLE, RUNE_ESCAPE,
					RUNE_PIPE, RUNE_REDIRECT:
					{
						state = STATE_QUOTED_ESCAPING
						buffer.WriteRune(next)
					}
				default:
					{
						return nil, errorf("Unknown rune: %v", next)
					}
				}
			}
		case STATE_QUOTED_ESCAPING: // in escaping double quotes
			{
				switch class {
				case RUNE_EOF:
					{
						err = errors.New("EOF found when expecting closing quote.")
						break SCAN
					}
				case RUNE_CHAR, RUNE_SPACE, RUNE_QUOTE_SINGLE,
					RUNE_PIPE, RUNE_REDIRECT:
					{
						buffer.WriteRune(next)
					}
				case RUNE_QUOTE_DOUBLE:
					{
						state = STATE_WORD
					}
				case RUNE_ESCAPE:
					{
						state = STATE_ESCAPING_QUOTED
					}
				default:
					{
						return nil, errorf("Unknown rune: %v", next)
					}
				}
			}
		case STATE_QUOTED: // in non-escaping single quotes
			{
				switch class {
				case RUNE_EOF:
					{
						err = errors.New("EOF found when expecting closing quote.")
						break SCAN
					}
				case RUNE_CHAR, RUNE_SPACE, RUNE_QUOTE_DOUBLE, RUNE_ESCAPE,
					RUNE_PIPE, RUNE_REDIRECT:
					{
						buffer.WriteRune(next)
					}
				case RUNE_QUOTE_SINGLE:
					{
						state = STATE_WORD
					}
				default:
					{
						return nil, errorf("Unknown rune: %v", next)
					}
				}
			}
		case STATE_COMMENT:
			{
				switch class {
				case RUNE_EOF:
					{
						break SCAN
					}
				case RUNE_CHAR, RUNE_QUOTE_DOUBLE, RUNE_ESCAPE,
					RUNE_QUOTE_SINGLE, RUNE_PIPE, RUNE_REDIRECT:
					{
						buffer.WriteRune(next)
					}
				case RUNE_SPACE:
					{
						if next == '\n' {
							state = STATE_START
							break SCAN
						} else {
							buffer.WriteRune(next)
						}
					}
				default:
					{
						return nil, errorf("Unknown rune: %v", next)
					}
				}
			}
		default:
			{
				panic(errorf("Unexpected state: %v", state))
			}
		}
	}

	return NewToken(tokenType, TokenValue(buffer.String())), err
}

// readRune returns the next rune in the stream.
func (tokenizer *Tokenizer) readRune() (rune, RuneClass, error) {
	next, _, err := tokenizer.input.ReadRune()
	class := tokenizer.classifier.Classify(next)

	if err != nil {
		if err == io.EOF {
			class = RUNE_EOF
			err = nil
		}
	}

	return next, class, err
}

// unread puts the previously read rune back into the stream.
func (tokenizer *Tokenizer) unreadRune() {
	tokenizer.input.UnreadRune()
}
<commit_msg>Add debug information for tokenizer<commit_after>/*
Copyright 2014 Sean Hickey <sean@headzoo.io>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ocean

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
)

// TokenClass is a top-level token; a word, space, comment, unknown.
type TokenClass string

// TokenValue is the value of the token, usually a string.
type TokenValue string

// TokenState is used within the lexer state machine to keep track of the current state.
type TokenState int

const (
	TOKEN_UNKNOWN  TokenClass = "UNKNOWN"
	TOKEN_WORD     TokenClass = "WORD"
	TOKEN_SPACE    TokenClass = "SPACE"
	TOKEN_PIPE     TokenClass = "PIPE"
	TOKEN_REDIRECT TokenClass = "REDIRECT"

	STATE_START           TokenState = 0
	STATE_WORD            TokenState = 1
	STATE_ESCAPING        TokenState = 2
	STATE_ESCAPING_QUOTED TokenState = 3
	STATE_QUOTED_ESCAPING TokenState = 4
	STATE_QUOTED          TokenState = 5
	STATE_COMMENT         TokenState = 6
	STATE_EMIT            TokenState = 7
)

func (ts TokenState) String() string {
	switch ts {
		case STATE_START:
			return "START"
		case STATE_WORD:
			return "WORD"
		case STATE_ESCAPING:
			return "ESCAPING"
		case STATE_ESCAPING_QUOTED:
			return "ESCAPING_QUOTED"
		case STATE_QUOTED_ESCAPING:
			return "QUOTED_ESCAPING"
		case STATE_QUOTED:
			return "QUOTED"
		case STATE_COMMENT:
			return "COMMENT"
		case STATE_EMIT:
			return "EMIT"
	}

	return fmt.Sprintf("!? %d", ts)
}

// Token represents a single "token" found within a stream.
type Token struct {
	Class TokenClass
	Value TokenValue
}

// Creates and returns a new Token instance.
func NewToken(class TokenClass, value TokenValue) *Token {
	return &Token{
		Class: class,
		Value: value,
	}
}

// Tokenizer turns an input stream in to a sequence of typed tokens.
type Tokenizer struct {
	input      *bufio.Reader
	classifier *Classifier
}

// Creates and returns a new tokenizer.
func NewTokenizer(reader io.Reader) *Tokenizer {
	return &Tokenizer{
		input:      bufio.NewReader(reader),
		classifier: NewClassifier(),
	}
}

// NextToken returns the next token in the stream, and an error value. If there are no more
// tokens available, the error value will be io.EOF.
func (tokenizer *Tokenizer) NextToken() (*Token, error) {
	var (
		err       error
		next      rune
		class     RuneClass
		tokenType TokenClass
	)

	state := STATE_START
	buffer := bytes.NewBuffer(make([]byte, 0, 1000))

SCAN:
	for state != STATE_EMIT {
		next, class, err = tokenizer.readRune()
		if err != nil {
			return nil, err
		}

		switch state {
		case STATE_START: // no runes read yet
			{
				switch class {
				case RUNE_EOF:
					{
						return nil, io.EOF
					}
				case RUNE_CHAR:
					{
						tokenType = TOKEN_WORD
						buffer.WriteRune(next)
						state = STATE_WORD
					}
				case RUNE_SPACE:
					{
					}
				case RUNE_QUOTE_DOUBLE:
					{
						tokenType = TOKEN_WORD
						state = STATE_QUOTED_ESCAPING
					}
				case RUNE_QUOTE_SINGLE:
					{
						tokenType = TOKEN_WORD
						state = STATE_QUOTED
					}
				case RUNE_ESCAPE:
					{
						tokenType = TOKEN_WORD
						state = STATE_ESCAPING
					}
				case RUNE_PIPE:
					{
						tokenType = TOKEN_PIPE
						buffer.WriteRune(next)
						state = STATE_EMIT
					}
				case RUNE_REDIRECT:
					{
						tokenType = TOKEN_REDIRECT
						buffer.WriteRune(next)
						state = STATE_EMIT

						n, _, e := tokenizer.readRune()
						if e != nil {
							return nil, err
						}
						if n == next {
							buffer.WriteRune(next)
						} else {
							tokenizer.unreadRune()
						}
					}
				default:
					{
						return nil, errorf("Unknown rune in state %s: %v", state, next)
					}
				}
			}
		case STATE_WORD: // in a regular word
			{
				switch class {
				case RUNE_EOF:
					{
						break SCAN
					}
				case RUNE_CHAR:
					{
						buffer.WriteRune(next)
					}
				case RUNE_SPACE:
					{
						tokenizer.unreadRune()
						break SCAN
					}
				case RUNE_QUOTE_DOUBLE:
					{
						state = STATE_QUOTED_ESCAPING
					}
				case RUNE_QUOTE_SINGLE:
					{
						state = STATE_QUOTED
					}
				case RUNE_ESCAPE:
					{
						state = STATE_ESCAPING
					}
				case RUNE_PIPE:
					{
						tokenizer.unreadRune()
						state = STATE_EMIT
					}
				case RUNE_REDIRECT:
					{
						tokenizer.unreadRune()
						state = STATE_EMIT
					}
				default:
					{
						return nil, errorf("Unknown rune in state %s: %v", state, next)
					}
				}
			}
		case STATE_ESCAPING: // the next rune after an escape character
			{
				switch class {
				case RUNE_EOF:
					{
						err = errors.New("EOF found after escape character")
						break SCAN
					}
				case RUNE_CHAR, RUNE_SPACE, RUNE_QUOTE_DOUBLE, RUNE_QUOTE_SINGLE, RUNE_ESCAPE,
					RUNE_PIPE, RUNE_REDIRECT:
					{
						state = STATE_WORD
						buffer.WriteRune(next)
					}
				default:
					{
						return nil, errorf("Unknown rune in state %s: %v", state, next)
					}
				}
			}
		case STATE_ESCAPING_QUOTED: // the next rune after an escape character, in double quotes
			{
				switch class {
				case RUNE_EOF:
					{
						err = errors.New("EOF found after escape character")
						break SCAN
					}
				case RUNE_CHAR, RUNE_SPACE, RUNE_QUOTE_DOUBLE, RUNE_QUOTE_SINGLE, RUNE_ESCAPE,
					RUNE_PIPE, RUNE_REDIRECT:
					{
						state = STATE_QUOTED_ESCAPING
						buffer.WriteRune(next)
					}
				default:
					{
						return nil, errorf("Unknown rune in state %s: %v", state, next)
					}
				}
			}
		case STATE_QUOTED_ESCAPING: // in escaping double quotes
			{
				switch class {
				case RUNE_EOF:
					{
						err = errors.New("EOF found when expecting closing quote.")
						break SCAN
					}
				case RUNE_CHAR, RUNE_SPACE, RUNE_QUOTE_SINGLE,
					RUNE_PIPE, RUNE_REDIRECT:
					{
						buffer.WriteRune(next)
					}
				case RUNE_QUOTE_DOUBLE:
					{
						state = STATE_WORD
					}
				case RUNE_ESCAPE:
					{
						state = STATE_ESCAPING_QUOTED
					}
				default:
					{
						return nil, errorf("Unknown rune in state %s: %v", state, next)
					}
				}
			}
		case STATE_QUOTED: // in non-escaping single quotes
			{
				switch class {
				case RUNE_EOF:
					{
						err = errors.New("EOF found when expecting closing quote.")
						break SCAN
					}
				case RUNE_CHAR, RUNE_SPACE, RUNE_QUOTE_DOUBLE, RUNE_ESCAPE,
					RUNE_PIPE, RUNE_REDIRECT:
					{
						buffer.WriteRune(next)
					}
				case RUNE_QUOTE_SINGLE:
					{
						state = STATE_WORD
					}
				default:
					{
						return nil, errorf("Unknown rune in state %s: %v", state, next)
					}
				}
			}
		case STATE_COMMENT:
			{
				switch class {
				case RUNE_EOF:
					{
						break SCAN
					}
				case RUNE_CHAR, RUNE_QUOTE_DOUBLE, RUNE_ESCAPE,
					RUNE_QUOTE_SINGLE, RUNE_PIPE, RUNE_REDIRECT:
					{
						buffer.WriteRune(next)
					}
				case RUNE_SPACE:
					{
						if next == '\n' {
							state = STATE_START
							break SCAN
						} else {
							buffer.WriteRune(next)
						}
					}
				default:
					{
						return nil, errorf("Unknown rune in state %s: %v", state, next)
					}
				}
			}
		default:
			{
				panic(errorf("Unexpected state: %v", state))
			}
		}
	}

	return NewToken(tokenType, TokenValue(buffer.String())), err
}

// readRune returns the next rune in the stream.
func (tokenizer *Tokenizer) readRune() (rune, RuneClass, error) {
	next, _, err := tokenizer.input.ReadRune()
	class := tokenizer.classifier.Classify(next)

	if err != nil {
		if err == io.EOF {
			class = RUNE_EOF
			err = nil
		}
	}

	return next, class, err
}

// unread puts the previously read rune back into the stream.
func (tokenizer *Tokenizer) unreadRune() {
	tokenizer.input.UnreadRune()
}
<|endoftext|>


<commit_before>// Testing support for go-toml

package toml

import (
	"testing"
)

func TestTomlHas(t *testing.T) {
	tree, _ := Load(`
		[test]
		key = "value"
	`)

	if !tree.Has("test.key") {
		t.Errorf("Has - expected test.key to exists")
	}
}

func TestTomlHasPath(t *testing.T) {
	tree, _ := Load(`
		[test]
		key = "value"
	`)

	if !tree.HasPath([]string{"test", "key"}) {
		t.Errorf("HasPath - expected test.key to exists")
	}
}

func TestTomlGetPath(t *testing.T) {
	node := newTomlTree()
	//TODO: set other node data

	for idx, item := range []struct {
		Path     []string
		Expected *TomlTree
	}{
		{ // empty path test
			[]string{},
			node,
		},
	} {
		result := node.GetPath(item.Path)
		if result != item.Expected {
			t.Errorf("GetPath[%d] %v - expected %v, got %v instead.", idx, item.Path, item.Expected, result)
		}
	}
}

func TestTomlQuery(t *testing.T) {
	tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
	if err != nil {
		t.Error(err)
		return
	}
	result, err := tree.Query("$.foo.bar")
	if err != nil {
		t.Error(err)
		return
	}
	values := result.Values()
	if len(values) != 1 {
		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
	}

	if tt, ok := values[0].(*TomlTree); !ok {
		t.Errorf("Expected type of TomlTree: %T", values[0])
	} else if tt.Get("a") != int64(1) {
		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
	} else if tt.Get("b") != int64(2) {
		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
	}
}
<commit_msg>Add couple tests<commit_after>// Testing support for go-toml

package toml

import (
	"testing"
)

func TestTomlHas(t *testing.T) {
	tree, _ := Load(`
		[test]
		key = "value"
	`)

	if !tree.Has("test.key") {
		t.Errorf("Has - expected test.key to exists")
	}

	if tree.Has("") {
		t.Errorf("Should return false if the key is not provided")
	}
}

func TestTomlHasPath(t *testing.T) {
	tree, _ := Load(`
		[test]
		key = "value"
	`)

	if !tree.HasPath([]string{"test", "key"}) {
		t.Errorf("HasPath - expected test.key to exists")
	}
}

func TestTomlGetPath(t *testing.T) {
	node := newTomlTree()
	//TODO: set other node data

	for idx, item := range []struct {
		Path     []string
		Expected *TomlTree
	}{
		{ // empty path test
			[]string{},
			node,
		},
	} {
		result := node.GetPath(item.Path)
		if result != item.Expected {
			t.Errorf("GetPath[%d] %v - expected %v, got %v instead.", idx, item.Path, item.Expected, result)
		}
	}
}

func TestTomlQuery(t *testing.T) {
	tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6")
	if err != nil {
		t.Error(err)
		return
	}
	result, err := tree.Query("$.foo.bar")
	if err != nil {
		t.Error(err)
		return
	}
	values := result.Values()
	if len(values) != 1 {
		t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values)
	}

	if tt, ok := values[0].(*TomlTree); !ok {
		t.Errorf("Expected type of TomlTree: %T", values[0])
	} else if tt.Get("a") != int64(1) {
		t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a"))
	} else if tt.Get("b") != int64(2) {
		t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b"))
	}
}

func TestTomlFromMap(t *testing.T) {
	simpleMap := map[string]interface{}{"hello": 42}
	tree := TreeFromMap(simpleMap)
	if tree.Get("hello") != 42 {
		t.Fatal("hello should be 42, not", tree.Get("hello"))
	}
}
<|endoftext|>


<commit_before>package tools

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"os"

	"github.com/vektra/cypress"
	"github.com/vektra/cypress/cli/commands"
)

type Format int

const (
	KV Format = iota
	HUMAN
	JSON
	NATIVE
)

type Cat struct {
	out io.Writer
	gen cypress.Generator

	format Format

	buf []byte
}

func NewCat(out io.Writer, gen cypress.Generator, format Format) (*Cat, error) {
	return &Cat{out, gen, format, make([]byte, 128)}, nil
}

var nl = []byte{'\n'}

func (c *Cat) Run() error {
	defer c.gen.Close()

	var buf bytes.Buffer

	for {
		m, err := c.gen.Generate()
		if err != nil {
			return err
		}

		if m == nil {
			break
		}

		switch c.format {
		case KV:
			buf.Reset()

			m.KVStringInto(&buf)
			c.out.Write(buf.Bytes())
			c.out.Write(nl)
		case HUMAN:
			c.out.Write([]byte(m.HumanString()))
			c.out.Write(nl)
		case JSON:
			json.NewEncoder(c.out).Encode(m)
		case NATIVE:
			bytes, err := m.Marshal()
			if err != nil {
				return err
			}

			binary.BigEndian.PutUint64(c.buf[:8], uint64(len(bytes)))
			c.out.Write(c.buf[:8])
			c.out.Write(bytes)
		}
	}

	return nil
}

type CatCLI struct {
	Kv     bool `short:"k" description:"display in key=value format"`
	Human  bool `short:"H" description:"display in easy to read format"`
	Json   bool `short:"j" description:"display in json"`
	Native bool `short:"n" description:"output as native binary"`
}

func (c *CatCLI) Execute(args []string) error {
	var cnt int

	if c.Kv {
		cnt++
	}

	if c.Human {
		cnt++
	}

	if c.Json {
		cnt++
	}

	if c.Native {
		cnt++
	}

	var format Format

	switch {
	case cnt == 0:
		format = KV
	case cnt > 1:
		return fmt.Errorf("multiple display types requested, only use one")
	case c.Kv:
		format = KV
	case c.Human:
		format = HUMAN
	case c.Json:
		format = JSON
	case c.Native:
		format = NATIVE
	}

	dec, err := cypress.NewStreamDecoder(os.Stdin)
	if err != nil {
		return err
	}

	cat, err := NewCat(os.Stdout, dec, format)
	if err != nil {
		return err
	}

	err = cat.Run()
	if err != nil {
		if err == io.EOF {
			return nil
		}

		return err
	}

	return nil
}

func init() {
	long := `Given a stream on stdin, the cat command will read those messages in and print them out.`

	commands.Add("cat", "display messages", long, &CatCLI{})
}
<commit_msg>Encode using a StreamEncoder with cat -n<commit_after>package tools

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"

	"github.com/vektra/cypress"
	"github.com/vektra/cypress/cli/commands"
)

type Format int

const (
	KV Format = iota
	HUMAN
	JSON
	NATIVE
)

type Cat struct {
	out io.Writer
	gen cypress.Generator

	format Format

	buf []byte
}

func NewCat(out io.Writer, gen cypress.Generator, format Format) (*Cat, error) {
	return &Cat{out, gen, format, make([]byte, 128)}, nil
}

var nl = []byte{'\n'}

func (c *Cat) Run() error {
	defer c.gen.Close()

	var (
		buf bytes.Buffer
		enc *cypress.StreamEncoder
	)

	if c.format == NATIVE {
		enc = cypress.NewStreamEncoder(c.out)
	}

	for {
		m, err := c.gen.Generate()
		if err != nil {
			return err
		}

		if m == nil {
			break
		}

		switch c.format {
		case KV:
			buf.Reset()

			m.KVStringInto(&buf)
			c.out.Write(buf.Bytes())
			c.out.Write(nl)
		case HUMAN:
			c.out.Write([]byte(m.HumanString()))
			c.out.Write(nl)
		case JSON:
			json.NewEncoder(c.out).Encode(m)
		case NATIVE:
			err := enc.Receive(m)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

type CatCLI struct {
	Kv     bool `short:"k" description:"display in key=value format"`
	Human  bool `short:"H" description:"display in easy to read format"`
	Json   bool `short:"j" description:"display in json"`
	Native bool `short:"n" description:"output as native binary"`
}

func (c *CatCLI) Execute(args []string) error {
	var cnt int

	if c.Kv {
		cnt++
	}

	if c.Human {
		cnt++
	}

	if c.Json {
		cnt++
	}

	if c.Native {
		cnt++
	}

	var format Format

	switch {
	case cnt == 0:
		format = KV
	case cnt > 1:
		return fmt.Errorf("multiple display types requested, only use one")
	case c.Kv:
		format = KV
	case c.Human:
		format = HUMAN
	case c.Json:
		format = JSON
	case c.Native:
		format = NATIVE
	}

	dec, err := cypress.NewStreamDecoder(os.Stdin)
	if err != nil {
		return err
	}

	cat, err := NewCat(os.Stdout, dec, format)
	if err != nil {
		return err
	}

	err = cat.Run()
	if err != nil {
		if err == io.EOF {
			return nil
		}

		return err
	}

	return nil
}

func init() {
	long := `Given a stream on stdin, the cat command will read those messages in and print them out.`

	commands.Add("cat", "display messages", long, &CatCLI{})
}
<|endoftext|>


<commit_before>package parser

// Type tags so we know what kind of token we have
type lexeme int
const (
  LPAREN = iota
  RPAREN
  LBRACE
  RBRACE
  COMMA
  EQUAL
  STRING
  REGEXP
  POS
  GVAR
  LVAR
  KWD
  ID
  FUNC
  TYPE
  IMPORT
  READ
  EOF
)
// Map the numeric tags back to names for debugging purposes.
var lexemeName [18]string

// A token has a type (aka lexeme), a value, and a line number
type token struct {
  Lexeme lexeme
  Value string
  LineNum int
}

/*
  Represent a tokenizer with an stuct containing the remaining source text
  and the line number. Easier than using a stateless tokenizing function that
  returns them as extra values and requires the parser to keep track of them.
*/
type tokenizer struct {
  Source []byte
  LineNum int
}

// Discard leading spaces (excluding newlines) in the source text.
func (t *tokenizer) discardSpaces() {
  var i int
  var c byte
  for i, c = 0, t.Source[0]; c == ' ' || c == '\t'; i, c = i+1, t.Source[i+1] { }
  t.Source = t.Source[i:]
}

// Discard leading text until a newline is found.
func(t *tokenizer) discardLine() {
  var i int
  var c byte
  for i, c = 0, t.Source[0]; c != '\n'; i, c = i+1, t.Source[i+1] { }
  t.Source = t.Source[i:]
}

// Discard leading comments in the source text.
func (t *tokenizer) discardComments() {
  switch t.Source[0] {
  case '#':
    t.discardLine()
  case '/':
    switch t.Source[1] {
    case '/':
      t.discardLine()
    case '*':
      t.discardBlockComment()
    }
  }
}

// Helper for discarding block comments.
// TO DO: ERROR HANDLING FOR UNTERMINATED COMMENTS
func (t *tokenizer) discardBlockComment() {
  depth, i, length := 1, 2, len(t.Source)
  for depth > 0 {
    if i >= length {
      // ERROR
    }
    switch t.Source[i] {
    case '\n':
      t.LineNum++
    case '/':
      i++
      if i >= length {
        // ERROR
      }
      if t.Source[i] == '*' {
        depth++
      }
    case '*':
      i++
      if i >= length {
        // ERROR
      }
      if t.Source[i] == '/' {
        depth--
      }
    }
    i++
  }
  t.Source = t.Source[i:]
}

func init() {
  // Is there a more elegant way to do this?
  lexemeName[LPAREN] = "LPAREN"
  lexemeName[RPAREN] = "RPAREN"
  lexemeName[LBRACE] = "LBRACE"
  lexemeName[RBRACE] = "RBRACE"
  lexemeName[COMMA]  = "COMMA"
  lexemeName[EQUAL]  = "EQUAL"
  lexemeName[STRING] = "STRING"
  lexemeName[REGEXP] = "REGEXP"
  lexemeName[POS]    = "POS"
  lexemeName[GVAR]   = "GVAR"
  lexemeName[LVAR]   = "LVAR"
  lexemeName[KWD]    = "KWD"
  lexemeName[ID]     = "ID"
  lexemeName[FUNC]   = "FUNC"
  lexemeName[TYPE]   = "TYPE"
  lexemeName[IMPORT] = "IMPORT"
  lexemeName[READ]   = "READ"
  lexemeName[EOF]    = "EOF"
}
<commit_msg>Gah, finished the whitespace and comment handling functions.<commit_after>package main

import (
  "bytes"
  //"strconv"
  "fmt"
)

// Type tags so we know what kind of token we have
type lexeme int
const (
  LPAREN = iota
  RPAREN
  LBRACE
  RBRACE
  COMMA
  EQUAL
  STRING
  REGEXP
  POS
  GVAR
  LVAR
  KWD
  ID
  FUNC
  TYPE
  IMPORT
  READ
  EOF
)
// Map the numeric tags back to names for debugging purposes.
var lexemeName [18]string

// A token has a type (aka lexeme), a value, and a line number
type token struct {
  Lexeme lexeme
  Value string
  LineNum int
}

/*
  Represent a tokenizer with a struct containing the remaining source text and
  the line number. Easier than using a stateless tokenizing function that
  returns them as extra values and requires the parser to keep track of them.
*/
type Tokenizer struct {
  Source []byte
  LineNum int
}

func (t *Tokenizer) hasPrefix(s string) bool {
  return bytes.HasPrefix(t.Source, []byte(s))
}

// Discard leading spaces (excluding newlines) in the source text.
func (t *Tokenizer) discardSpaces() {
  t.Source = bytes.TrimLeft(t.Source, " \t")
}

// Discard leading text until a newline (or EOF) is found.
func(t *Tokenizer) discardLine() {
  if i := bytes.IndexByte(t.Source, '\n'); i >= 0 {
    t.Source = t.Source[i:]
  } else {
    t.Source = t.Source[len(t.Source):]
  }
}

// Discard the leading comment in the source text.
func (t *Tokenizer) discardComment() {
  if t.hasPrefix("#") || t.hasPrefix("//") {
    t.discardLine()
  } else if t.hasPrefix("/*") {
    t.discardBlockComment()
  }
}

// Helper for discarding block comments.
// TO DO: ERROR HANDLING FOR UNTERMINATED COMMENTS
func (t *Tokenizer) discardBlockComment() {
  depth, i, length := 1, 2, len(t.Source)
  for depth > 0 {
    if i >= length {
      // ERROR
    }
    switch t.Source[i] {
    case '\n':
      t.LineNum++
    case '/':
      i++
      if i >= length {
        // ERROR
      }
      if t.Source[i] == '*' {
        depth++
      }
    case '*':
      i++
      if i >= length {
        // ERROR
      }
      if t.Source[i] == '/' {
        depth--
      }
    }
    i++
  }
  t.Source = t.Source[i:]
}

// Discard all leading whitespace and comments from the source text. Need to
// tally up the newlines to keep LineNum up to date.
func (t *Tokenizer) discardWhitespaceAndComments() {
  for len(t.Source) > 0 {
    switch {
    case t.hasPrefix("\n"):
      fmt.Println("newline")
      t.LineNum++
      t.Source = t.Source[1:]
    case t.hasPrefix(" ") || t.hasPrefix("\t"):
      fmt.Println("spaces")
      t.discardSpaces()
    case t.hasPrefix("#") || t.hasPrefix("//") || t.hasPrefix("/*"):
      fmt.Println("comment")
      t.discardComment()
    default:
      return
    }
  }
}

func main() {
  t := Tokenizer{ Source: []byte("  \t // blah \n/* a /* b */ c */  hello"), LineNum: 1 }
  // t.discardSpaces()
  // t.discardComment()
  // t.Source = t.Source[1:]
  // t.discardComment()
  // t.discardSpaces()
  t.discardWhitespaceAndComments()

  fmt.Println(string(t.Source))
}

func init() {
  // Is there a more elegant way to do this?
  lexemeName[LPAREN] = "LPAREN"
  lexemeName[RPAREN] = "RPAREN"
  lexemeName[LBRACE] = "LBRACE"
  lexemeName[RBRACE] = "RBRACE"
  lexemeName[COMMA]  = "COMMA"
  lexemeName[EQUAL]  = "EQUAL"
  lexemeName[STRING] = "STRING"
  lexemeName[REGEXP] = "REGEXP"
  lexemeName[POS]    = "POS"
  lexemeName[GVAR]   = "GVAR"
  lexemeName[LVAR]   = "LVAR"
  lexemeName[KWD]    = "KWD"
  lexemeName[ID]     = "ID"
  lexemeName[FUNC]   = "FUNC"
  lexemeName[TYPE]   = "TYPE"
  lexemeName[IMPORT] = "IMPORT"
  lexemeName[READ]   = "READ"
  lexemeName[EOF]    = "EOF"
}
<|endoftext|>


<commit_before>package backend

import "sort"

// IDSet is a set of IDs.
type IDSet map[ID]struct{}

// NewIDSet returns a new IDSet, populated with ids.
func NewIDSet(ids ...ID) IDSet {
	m := make(IDSet)
	for _, id := range ids {
		m[id] = struct{}{}
	}

	return m
}

// Has returns true iff id is contained in the set.
func (s IDSet) Has(id ID) bool {
	_, ok := s[id]
	return ok
}

// Insert adds id to the set.
func (s IDSet) Insert(id ID) {
	s[id] = struct{}{}
}

// Delete removes id from the set.
func (s IDSet) Delete(id ID) {
	delete(s, id)
}

// List returns a slice of all IDs in the set.
func (s IDSet) List() IDs {
	list := make(IDs, 0, len(s))
	for id := range s {
		list = append(list, id)
	}

	sort.Sort(list)

	return list
}

// Equals returns true iff s equals other.
func (s IDSet) Equals(other IDSet) bool {
	if len(s) != len(other) {
		return false
	}

	for id := range s {
		if _, ok := other[id]; !ok {
			return false
		}
	}

	for id := range other {
		if _, ok := s[id]; !ok {
			return false
		}
	}

	return true
}

func (s IDSet) String() string {
	str := s.List().String()
	if len(str) < 2 {
		return "{}"
	}

	return "{" + str[1:len(str)-1] + "}"
}
<commit_msg>idset.go: micro-optimise away redundant scan<commit_after>package backend

import "sort"

// IDSet is a set of IDs.
type IDSet map[ID]struct{}

// NewIDSet returns a new IDSet, populated with ids.
func NewIDSet(ids ...ID) IDSet {
	m := make(IDSet)
	for _, id := range ids {
		m[id] = struct{}{}
	}

	return m
}

// Has returns true iff id is contained in the set.
func (s IDSet) Has(id ID) bool {
	_, ok := s[id]
	return ok
}

// Insert adds id to the set.
func (s IDSet) Insert(id ID) {
	s[id] = struct{}{}
}

// Delete removes id from the set.
func (s IDSet) Delete(id ID) {
	delete(s, id)
}

// List returns a slice of all IDs in the set.
func (s IDSet) List() IDs {
	list := make(IDs, 0, len(s))
	for id := range s {
		list = append(list, id)
	}

	sort.Sort(list)

	return list
}

// Equals returns true iff s equals other.
func (s IDSet) Equals(other IDSet) bool {
	if len(s) != len(other) {
		return false
	}

	for id := range s {
		if _, ok := other[id]; !ok {
			return false
		}
	}

	// length + one-way comparison is sufficient implication of equality

	return true
}

func (s IDSet) String() string {
	str := s.List().String()
	if len(str) < 2 {
		return "{}"
	}

	return "{" + str[1:len(str)-1] + "}"
}
<|endoftext|>


<commit_before>/*
Package markov is a minor adaptation of https://golang.org/doc/codewalk/markov.go
with the difference being that it attempts to load text from a url linking to
a .txt file and only returns chains that are < textLength characters long.

This is because I wrote it for implementation in an IRCBOT and the IRC protocol
is limited in characters per message
*/
package markov

import (
	"bufio"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"regexp"
	"strings"
	"time"
)

var supportedFile = regexp.MustCompile(`.*\.txt$`)
var textLength = 400

// Prefix is a Markov chain prefix of one or more words.
type Prefix []string

// String returns the Prefix as a string (for use as a map key).
func (p Prefix) String() string {
	return strings.Join(p, " ")
}

// Shift removes the first word from the Prefix and appends the given word.
func (p Prefix) Shift(word string) {
	copy(p, p[1:])
	p[len(p)-1] = word
}

// Chain contains a map ("chain") of prefixes to a list of suffixes.
// A prefix is a string of prefixLen words joined with spaces.
// A suffix is a single word. A prefix can have multiple suffixes.
type Chain struct {
	chain     map[string][]string
	prefixLen int
}

// NewChain returns a new Chain with prefixes of prefixLen words.
func NewChain(prefixLen int) *Chain {
	return &Chain{make(map[string][]string), prefixLen}
}

// Build reads text from the provided Reader and
// parses it into prefixes and suffixes that are stored in Chain.
func (c *Chain) Build(r io.Reader) {
	br := bufio.NewReader(r)
	p := make(Prefix, c.prefixLen)
	for {
		var s string
		if _, err := fmt.Fscan(br, &s); err != nil {
			break
		}
		key := p.String()
		c.chain[key] = append(c.chain[key], s)
		p.Shift(s)
	}
}

// Generate returns a string of at most n words generated from Chain.
func (c *Chain) Generate(n int) string {
	rand.Seed(time.Now().UnixNano())
	p := make(Prefix, c.prefixLen)
	var words []string
	for i := 0; i < n; i++ {
		choices := c.chain[p.String()]
		if len(choices) == 0 {
			break
		}
		next := choices[rand.Intn(len(choices))]
		words = append(words, next)
		p.Shift(next)
	}
	return strings.Join(words, " ")
}

// GoNuts retrieves url (if it is supported) and returns a markov chain
// based on its contents
func GoNuts(url string) (string, error) {
	if !supportedFile.MatchString(url) {
		return "", fmt.Errorf("Unsupported filetype: %v", url)
	}

	resp, err := http.Get(url)
	if err != nil {
		return "", fmt.Errorf("Could not retrieve file: %v", err)
	}
	defer resp.Body.Close()

	c := NewChain(2)   // Initialize a new Chain.
	c.Build(resp.Body) // Build chains from standard input.

	tLen := textLength
	text := c.Generate(tLen)
	for len(text) > tLen {
		text = c.Generate(tLen) // Generate text.
		tLen--
	}
	return text, nil
}
<commit_msg>tidied global var declaration<commit_after>/*
Package markov is a minor adaptation of https://golang.org/doc/codewalk/markov.go
with the difference being that it attempts to load text from a url linking to
a .txt file and only returns chains that are < textLength characters long.

This is because I wrote it for implementation in an IRCBOT and the IRC protocol
is limited in characters per message
*/
package markov

import (
	"bufio"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"regexp"
	"strings"
	"time"
)

var (
	supportedFile = regexp.MustCompile(`.*\.txt$`)
	textLength    = 400
)

// Prefix is a Markov chain prefix of one or more words.
type Prefix []string

// String returns the Prefix as a string (for use as a map key).
func (p Prefix) String() string {
	return strings.Join(p, " ")
}

// Shift removes the first word from the Prefix and appends the given word.
func (p Prefix) Shift(word string) {
	copy(p, p[1:])
	p[len(p)-1] = word
}

// Chain contains a map ("chain") of prefixes to a list of suffixes.
// A prefix is a string of prefixLen words joined with spaces.
// A suffix is a single word. A prefix can have multiple suffixes.
type Chain struct {
	chain     map[string][]string
	prefixLen int
}

// NewChain returns a new Chain with prefixes of prefixLen words.
func NewChain(prefixLen int) *Chain {
	return &Chain{make(map[string][]string), prefixLen}
}

// Build reads text from the provided Reader and
// parses it into prefixes and suffixes that are stored in Chain.
func (c *Chain) Build(r io.Reader) {
	br := bufio.NewReader(r)
	p := make(Prefix, c.prefixLen)
	for {
		var s string
		if _, err := fmt.Fscan(br, &s); err != nil {
			break
		}
		key := p.String()
		c.chain[key] = append(c.chain[key], s)
		p.Shift(s)
	}
}

// Generate returns a string of at most n words generated from Chain.
func (c *Chain) Generate(n int) string {
	rand.Seed(time.Now().UnixNano())
	p := make(Prefix, c.prefixLen)
	var words []string
	for i := 0; i < n; i++ {
		choices := c.chain[p.String()]
		if len(choices) == 0 {
			break
		}
		next := choices[rand.Intn(len(choices))]
		words = append(words, next)
		p.Shift(next)
	}
	return strings.Join(words, " ")
}

// GoNuts retrieves url (if it is supported) and returns a markov chain
// based on its contents
func GoNuts(url string) (string, error) {
	if !supportedFile.MatchString(url) {
		return "", fmt.Errorf("Unsupported filetype: %v", url)
	}

	resp, err := http.Get(url)
	if err != nil {
		return "", fmt.Errorf("Could not retrieve file: %v", err)
	}
	defer resp.Body.Close()

	c := NewChain(2)   // Initialize a new Chain.
	c.Build(resp.Body) // Build chains from standard input.

	tLen := textLength
	text := c.Generate(tLen)
	for len(text) > tLen {
		text = c.Generate(tLen) // Generate text.
		tLen--
	}
	return text, nil
}
<|endoftext|>


<commit_before>/*
 * portal - marshal
 *
 * a library that implements an algorithm for doing consumer coordination within Kafka, rather
 * than using Zookeeper or another external system.
 *
 */

package marshal

import (
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/optiopay/kafka"
	"github.com/optiopay/kafka/proto"
)

// claim is instantiated for each partition "claim" we have. This type is responsible for
// pulling data from Kafka and managing its cursors, heartbeating as necessary, and health
// checking itself.
type claim struct {
	// These items are read-only. They are never changed after the object is created,
	// so access to these may be done without the lock.
	topic  string
	partID int

	// lock protects all access to the member variables of this struct except for the
	// messages channel, which can be read from or written to without holding the lock.
	lock          sync.RWMutex
	marshal       *Marshaler
	rand          *rand.Rand
	claimed       *int32
	lastHeartbeat int64
	consumer      kafka.Consumer
	messages      chan *proto.Message

	// Number of heartbeat cycles this claim has been lagging, i.e., consumption is going
	// too slowly (defined as being behind by more than 2 heartbeat cycles)
	cyclesBehind int

	// A Kafka partition consists of N messages with offsets. In the basic case, you
	// can think of an offset like an array index. With log compaction and other trickery
	// it acts more like a sparse array, but it's a close enough metaphor.
	//
	// We keep track of four values for offsets:
	//
	//    offsets       1     2     3     7     9    10    11
	//   partition  [ msg1, msg2, msg3, msg4, msg5, msg6, msg7, ... ]
	//                 ^                  ^                      ^
	//                 \- offsetEarliest  |                      |
	//                                    \- offsetCurrent   offsetLatest
	//                                     and startOffset
	//
	// In this example, offsetEarliest is 1 which is the "oldest" offset within the
	// partition. At any given time this offset might become invalid (if a log rolls)
	// so we might update it. startOffset is simply the value of offsetCurrent at
	// the very beginning of consumption (for tracking velocity).
	//
	// offsetCurrent is 7, which is the offset of the NEXT message i.e. this message
	// has not been consumed yet.
	//
	// offsetLatest is 12, which is the offset that Kafka will assign to the message
	// that next gets committed to the partition. This offset does not yet exist,
	// and mit might never.
	offsetCurrent  int64
	offsetEarliest int64
	offsetLatest   int64

	// History arrays used for calculating average veolocity for health checking.
	offsetCurrentHistory [10]int64
	offsetLatestHistory  [10]int64
}

// newClaim returns an internal claim object, used by the consumer to manage the
// claim of a single partition.
func newClaim(topic string, partID int, marshal *Marshaler) *claim {
	// Get all available offset information
	oEarly, oLate, oCur, err := marshal.GetPartitionOffsets(topic, partID)
	if err != nil {
		log.Errorf("%s:%d failed to get offsets: %s", topic, partID, err)
		return nil
	}
	log.Debugf("%s:%d consumer offsets: early = %d, cur = %d, late = %d",
		topic, partID, oEarly, oCur, oLate)

	// Construct object and set it up
	obj := &claim{
		marshal:        marshal,
		topic:          topic,
		partID:         partID,
		claimed:        new(int32),
		offsetEarliest: oEarly,
		offsetLatest:   oLate,
		offsetCurrent:  oCur,
		messages:       make(chan *proto.Message, 100),
		rand:           rand.New(rand.NewSource(time.Now().UnixNano())),
	}
	atomic.StoreInt32(obj.claimed, 1)

	// Now try to actually claim it, this can block a while
	log.Infof("%s:%d consumer attempting to claim", topic, partID)
	if !marshal.ClaimPartition(topic, partID) {
		log.Infof("%s:%d consumer failed to claim", topic, partID)
		return nil
	}

	// If that worked, kick off the main setup loop and return
	obj.setup()
	return obj
}

// updateOffsets polls Kafka periodically to get information about the partition's
// state.
func (c *claim) updateOffsetsLoop() {
	ctr := 0
	for c.Claimed() {
		time.Sleep(<-c.marshal.jitters)
		c.updateOffsets(ctr)
		ctr++
	}
	log.Infof("%s:%d no longer claimed, offset loop exiting", c.topic, c.partID)
}

// setup is the initial worker that initializes the claim structure. Until this is done,
// our internal state is inconsistent.
func (c *claim) setup() {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Of course, if the current offset is greater than the earliest, we must reset
	// to the earliest known
	if c.offsetCurrent < c.offsetEarliest {
		log.Warningf("%s:%d consumer fast-forwarding from %d to %d",
			c.topic, c.partID, c.offsetCurrent, c.offsetEarliest)
		c.offsetCurrent = c.offsetEarliest
	}

	// Since it's claimed, we now want to heartbeat with the last seen offset
	err := c.marshal.Heartbeat(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d consumer failed to heartbeat: %s", c.topic, c.partID, err)
		atomic.StoreInt32(c.claimed, 0)
		return
	}
	c.lastHeartbeat = time.Now().Unix()

	// Set up Kafka consumer
	consumerConf := kafka.NewConsumerConf(c.topic, int32(c.partID))
	consumerConf.StartOffset = c.offsetCurrent
	kafkaConsumer, err := c.marshal.kafka.Consumer(consumerConf)
	if err != nil {
		log.Errorf("%s:%d consumer failed to create Kafka Consumer: %s",
			c.topic, c.partID, err)
		// TODO: There is an optimization here where we could release the partition.
		// As it stands, we're not doing anything,
		atomic.StoreInt32(c.claimed, 0)
		return
	}
	c.consumer = kafkaConsumer

	// Start our maintenance goroutines that keep this system healthy
	go c.updateOffsetsLoop()
	go c.healthCheckLoop()
	go c.messagePump()

	// Totally done, let the world know and move on
	log.Infof("%s:%d consumer claimed at offset %d (is %d behind)",
		c.topic, c.partID, c.offsetCurrent, c.offsetLatest-c.offsetCurrent)
}

// Given a message offset (which should be our current offset), return whether or
// not that offset is allowed to be consumed (i.e., we're still claimed)
func (c *claim) Consumed(offset int64) bool {
	if atomic.LoadInt32(c.claimed) != 1 {
		return false
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	c.offsetCurrent = offset + 1
	return true
}

// Claimed returns whether or not this claim structure is alive and well and believes
// that it is still an active claim.
func (c *claim) Claimed() bool {
	return atomic.LoadInt32(c.claimed) == 1
}

// GetCurrentLag returns this partition's cursor lag.
func (c *claim) GetCurrentLag() int64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.offsetCurrent < c.offsetLatest {
		return c.offsetLatest - c.offsetCurrent
	}
	return 0
}

// Release will invoke our release mechanism if and only if we are still claimed.
func (c *claim) Release() bool {
	if !atomic.CompareAndSwapInt32(c.claimed, 1, 0) {
		return false
	}

	// Holds the lock through a Kafka transaction, but since we're releasing I think this
	// is reasonable. Held because of using offsetCurrent below.
	c.lock.RLock()
	defer c.lock.RUnlock()

	log.Infof("%s:%d releasing partition claim", c.topic, c.partID)
	err := c.marshal.ReleasePartition(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d failed to release: %s", c.topic, c.partID, err)
		return false
	}
	return true
}

// messagePump continuously pulls message from Kafka for this partition and makes them
// available for consumption.
func (c *claim) messagePump() {
	// This method MUST NOT make changes to the claim structure. Since we might
	// be running while someone else has the lock, and we can't get it ourselves, we are
	// forbidden to touch anything other than the consumer and the message channel.
	for c.Claimed() {
		msg, err := c.consumer.Consume()
		if err == proto.ErrOffsetOutOfRange {
			// Fell out of range, presumably because we're handling this too slow, so
			// let's abandon this consumer
			log.Errorf("%s:%d error consuming: out of range, abandoning partition",
				c.topic, c.partID)
			go c.Release()
			return
		}
		if err != nil {
			log.Errorf("%s:%d error consuming: %s", c.topic, c.partID, err)

			// Often a consumption error is caused by data going away, such as if we're consuming
			// from the head and Kafka has deleted the data. In that case we need to wait for
			// the next offset update, so let's not go crazy
			time.Sleep(1 * time.Second)
			continue
		}

		c.messages <- msg
	}
	log.Debugf("%s:%d no longer claimed, pump exiting", c.topic, c.partID)
}

// heartbeat is the internal "send a heartbeat" function. Calling this will immediately
// send a heartbeat to Kafka. If we fail to send a heartbeat, we will release the
// partition.
func (c *claim) heartbeat() {
	c.lock.Lock()
	defer c.lock.Unlock()

	err := c.marshal.Heartbeat(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d failed to heartbeat, releasing", c.topic, c.partID)
		go c.Release()
	}
	c.lastHeartbeat = time.Now().Unix()
}

// healthCheck performs a single health check against the claim. If we have failed
// too many times, this will also start a partition release. Returns true if the
// partition is healthy, else false.
func (c *claim) healthCheck() bool {
	// Get velocities; these functions both use the locks so we have to do this before
	// we personally take the lock (to avoid deadlock)
	consumerVelocity := c.ConsumerVelocity()
	partitionVelocity := c.PartitionVelocity()

	c.lock.Lock()
	defer c.lock.Unlock()

	// If our heartbeat is expired, we are definitely unhealthy... don't even bother
	// with checking velocity
	if c.lastHeartbeat < time.Now().Unix()-HeartbeatInterval {
		log.Warningf("%s:%d consumer unhealthy by heartbeat test, releasing",
			c.topic, c.partID)
		go c.Release()
		return false
	}

	// If current has gone forward of the latest (which is possible, but unlikely)
	// then we are by definition caught up
	if c.offsetCurrent >= c.offsetLatest {
		c.cyclesBehind = 0
		return true
	}

	// If velocity is good, reset cycles behind and exit
	if partitionVelocity <= consumerVelocity {
		c.cyclesBehind = 0
		return true
	}

	// Unhealthy, so increase the cycle count so we know when it's been unhealthy for
	// too long and we want to give it up
	c.cyclesBehind++

	// If were behind by too many cycles, then we should try to release the
	// partition. If so, do this in a goroutine since it will involve calling out
	// to Kafka and releasing the partition.
	if c.cyclesBehind >= 3 {
		log.Errorf("%s:%d consumer unhealthy, releasing",
			c.topic, c.partID)
		go c.Release()
		return false
	} else {
		log.Warningf("%s:%d consumer unhealthy: CV %0.2f < PV %0.2f",
			c.topic, c.partID, consumerVelocity, partitionVelocity)
	}

	// Clearly we haven't been behind for long enough, so we're still "healthy"
	return true
}

// healthCheckLoop runs regularly and will perform a health check. Exits when we are no longer
// a claimed partition
func (c *claim) healthCheckLoop() {
	for c.Claimed() {
		time.Sleep(<-c.marshal.jitters)
		if c.healthCheck() {
			go c.heartbeat()
		}
	}
	log.Debugf("%s:%d health check loop exiting", c.topic, c.partID)
}

// average returns the average of a given slice of int64s. It ignores 0s as
// those are "uninitialized" elements.
func average(vals []int64) float64 {
	min, max, ct := int64(0), int64(0), int64(0)
	for _, val := range vals {
		if val <= 0 {
			continue
		}
		if min == 0 || val < min {
			min = val
		}
		if max == 0 || val > max {
			max = val
		}
		ct++
	}
	if ct == 0 {
		return 0
	}
	return float64(max-min) / float64(ct)
}

// ConsumerVelocity returns the average of our consumers' velocity
func (c *claim) ConsumerVelocity() float64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	return average(c.offsetCurrentHistory[0:])
}

// PartitionVelocity returns the average of the partition's velocity
func (c *claim) PartitionVelocity() float64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	return average(c.offsetLatestHistory[0:])
}

// updateOffsets will update the offsets of our current partition.
func (c *claim) updateOffsets(ctr int) error {
	// Slow, hits Kafka. Run in a goroutine.
	oEarly, oLate, _, err := c.marshal.GetPartitionOffsets(c.topic, c.partID)
	if err != nil {
		log.Errorf("%s:%d failed to get offsets: %s", c.topic, c.partID, err)
		return err
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	// Update the earliest/latest offsets that are presently available within the
	// partition
	c.offsetEarliest = oEarly
	c.offsetLatest = oLate

	// Do update our "history" values, this is used for calculating moving averages
	// in the health checking function
	c.offsetLatestHistory[ctr%10] = oLate
	c.offsetCurrentHistory[ctr%10] = c.offsetCurrent

	return nil
}
<commit_msg>Move sleep to end<commit_after>/*
 * portal - marshal
 *
 * a library that implements an algorithm for doing consumer coordination within Kafka, rather
 * than using Zookeeper or another external system.
 *
 */

package marshal

import (
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/optiopay/kafka"
	"github.com/optiopay/kafka/proto"
)

// claim is instantiated for each partition "claim" we have. This type is responsible for
// pulling data from Kafka and managing its cursors, heartbeating as necessary, and health
// checking itself.
type claim struct {
	// These items are read-only. They are never changed after the object is created,
	// so access to these may be done without the lock.
	topic  string
	partID int

	// lock protects all access to the member variables of this struct except for the
	// messages channel, which can be read from or written to without holding the lock.
	lock          sync.RWMutex
	marshal       *Marshaler
	rand          *rand.Rand
	claimed       *int32
	lastHeartbeat int64
	consumer      kafka.Consumer
	messages      chan *proto.Message

	// Number of heartbeat cycles this claim has been lagging, i.e., consumption is going
	// too slowly (defined as being behind by more than 2 heartbeat cycles)
	cyclesBehind int

	// A Kafka partition consists of N messages with offsets. In the basic case, you
	// can think of an offset like an array index. With log compaction and other trickery
	// it acts more like a sparse array, but it's a close enough metaphor.
	//
	// We keep track of four values for offsets:
	//
	//    offsets       1     2     3     7     9    10    11
	//   partition  [ msg1, msg2, msg3, msg4, msg5, msg6, msg7, ... ]
	//                 ^                  ^                      ^
	//                 \- offsetEarliest  |                      |
	//                                    \- offsetCurrent   offsetLatest
	//                                     and startOffset
	//
	// In this example, offsetEarliest is 1 which is the "oldest" offset within the
	// partition. At any given time this offset might become invalid (if a log rolls)
	// so we might update it. startOffset is simply the value of offsetCurrent at
	// the very beginning of consumption (for tracking velocity).
	//
	// offsetCurrent is 7, which is the offset of the NEXT message i.e. this message
	// has not been consumed yet.
	//
	// offsetLatest is 12, which is the offset that Kafka will assign to the message
	// that next gets committed to the partition. This offset does not yet exist,
	// and mit might never.
	offsetCurrent  int64
	offsetEarliest int64
	offsetLatest   int64

	// History arrays used for calculating average veolocity for health checking.
	offsetCurrentHistory [10]int64
	offsetLatestHistory  [10]int64
}

// newClaim returns an internal claim object, used by the consumer to manage the
// claim of a single partition.
func newClaim(topic string, partID int, marshal *Marshaler) *claim {
	// Get all available offset information
	oEarly, oLate, oCur, err := marshal.GetPartitionOffsets(topic, partID)
	if err != nil {
		log.Errorf("%s:%d failed to get offsets: %s", topic, partID, err)
		return nil
	}
	log.Debugf("%s:%d consumer offsets: early = %d, cur = %d, late = %d",
		topic, partID, oEarly, oCur, oLate)

	// Construct object and set it up
	obj := &claim{
		marshal:        marshal,
		topic:          topic,
		partID:         partID,
		claimed:        new(int32),
		offsetEarliest: oEarly,
		offsetLatest:   oLate,
		offsetCurrent:  oCur,
		messages:       make(chan *proto.Message, 100),
		rand:           rand.New(rand.NewSource(time.Now().UnixNano())),
	}
	atomic.StoreInt32(obj.claimed, 1)

	// Now try to actually claim it, this can block a while
	log.Infof("%s:%d consumer attempting to claim", topic, partID)
	if !marshal.ClaimPartition(topic, partID) {
		log.Infof("%s:%d consumer failed to claim", topic, partID)
		return nil
	}

	// If that worked, kick off the main setup loop and return
	obj.setup()
	return obj
}

// updateOffsets polls Kafka periodically to get information about the partition's
// state.
func (c *claim) updateOffsetsLoop() {
	ctr := 0
	time.Sleep(<-c.marshal.jitters)
	for c.Claimed() {
		c.updateOffsets(ctr)
		ctr++
		time.Sleep(<-c.marshal.jitters)
	}
	log.Debugf("%s:%d no longer claimed, offset loop exiting", c.topic, c.partID)
}

// setup is the initial worker that initializes the claim structure. Until this is done,
// our internal state is inconsistent.
func (c *claim) setup() {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Of course, if the current offset is greater than the earliest, we must reset
	// to the earliest known
	if c.offsetCurrent < c.offsetEarliest {
		log.Warningf("%s:%d consumer fast-forwarding from %d to %d",
			c.topic, c.partID, c.offsetCurrent, c.offsetEarliest)
		c.offsetCurrent = c.offsetEarliest
	}

	// Since it's claimed, we now want to heartbeat with the last seen offset
	err := c.marshal.Heartbeat(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d consumer failed to heartbeat: %s", c.topic, c.partID, err)
		atomic.StoreInt32(c.claimed, 0)
		return
	}
	c.lastHeartbeat = time.Now().Unix()

	// Set up Kafka consumer
	consumerConf := kafka.NewConsumerConf(c.topic, int32(c.partID))
	consumerConf.StartOffset = c.offsetCurrent
	kafkaConsumer, err := c.marshal.kafka.Consumer(consumerConf)
	if err != nil {
		log.Errorf("%s:%d consumer failed to create Kafka Consumer: %s",
			c.topic, c.partID, err)
		// TODO: There is an optimization here where we could release the partition.
		// As it stands, we're not doing anything,
		atomic.StoreInt32(c.claimed, 0)
		return
	}
	c.consumer = kafkaConsumer

	// Start our maintenance goroutines that keep this system healthy
	go c.updateOffsetsLoop()
	go c.healthCheckLoop()
	go c.messagePump()

	// Totally done, let the world know and move on
	log.Infof("%s:%d consumer claimed at offset %d (is %d behind)",
		c.topic, c.partID, c.offsetCurrent, c.offsetLatest-c.offsetCurrent)
}

// Given a message offset (which should be our current offset), return whether or
// not that offset is allowed to be consumed (i.e., we're still claimed)
func (c *claim) Consumed(offset int64) bool {
	if atomic.LoadInt32(c.claimed) != 1 {
		return false
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	c.offsetCurrent = offset + 1
	return true
}

// Claimed returns whether or not this claim structure is alive and well and believes
// that it is still an active claim.
func (c *claim) Claimed() bool {
	return atomic.LoadInt32(c.claimed) == 1
}

// GetCurrentLag returns this partition's cursor lag.
func (c *claim) GetCurrentLag() int64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.offsetCurrent < c.offsetLatest {
		return c.offsetLatest - c.offsetCurrent
	}
	return 0
}

// Release will invoke our release mechanism if and only if we are still claimed.
func (c *claim) Release() bool {
	if !atomic.CompareAndSwapInt32(c.claimed, 1, 0) {
		return false
	}

	// Holds the lock through a Kafka transaction, but since we're releasing I think this
	// is reasonable. Held because of using offsetCurrent below.
	c.lock.RLock()
	defer c.lock.RUnlock()

	log.Infof("%s:%d releasing partition claim", c.topic, c.partID)
	err := c.marshal.ReleasePartition(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d failed to release: %s", c.topic, c.partID, err)
		return false
	}
	return true
}

// messagePump continuously pulls message from Kafka for this partition and makes them
// available for consumption.
func (c *claim) messagePump() {
	// This method MUST NOT make changes to the claim structure. Since we might
	// be running while someone else has the lock, and we can't get it ourselves, we are
	// forbidden to touch anything other than the consumer and the message channel.
	for c.Claimed() {
		msg, err := c.consumer.Consume()
		if err == proto.ErrOffsetOutOfRange {
			// Fell out of range, presumably because we're handling this too slow, so
			// let's abandon this consumer
			log.Errorf("%s:%d error consuming: out of range, abandoning partition",
				c.topic, c.partID)
			go c.Release()
			return
		}
		if err != nil {
			log.Errorf("%s:%d error consuming: %s", c.topic, c.partID, err)

			// Often a consumption error is caused by data going away, such as if we're consuming
			// from the head and Kafka has deleted the data. In that case we need to wait for
			// the next offset update, so let's not go crazy
			time.Sleep(1 * time.Second)
			continue
		}

		c.messages <- msg
	}
	log.Debugf("%s:%d no longer claimed, pump exiting", c.topic, c.partID)
}

// heartbeat is the internal "send a heartbeat" function. Calling this will immediately
// send a heartbeat to Kafka. If we fail to send a heartbeat, we will release the
// partition.
func (c *claim) heartbeat() {
	c.lock.Lock()
	defer c.lock.Unlock()

	err := c.marshal.Heartbeat(c.topic, c.partID, c.offsetCurrent)
	if err != nil {
		log.Errorf("%s:%d failed to heartbeat, releasing", c.topic, c.partID)
		go c.Release()
	}
	c.lastHeartbeat = time.Now().Unix()
}

// healthCheck performs a single health check against the claim. If we have failed
// too many times, this will also start a partition release. Returns true if the
// partition is healthy, else false.
func (c *claim) healthCheck() bool {
	// Get velocities; these functions both use the locks so we have to do this before
	// we personally take the lock (to avoid deadlock)
	consumerVelocity := c.ConsumerVelocity()
	partitionVelocity := c.PartitionVelocity()

	c.lock.Lock()
	defer c.lock.Unlock()

	// If our heartbeat is expired, we are definitely unhealthy... don't even bother
	// with checking velocity
	if c.lastHeartbeat < time.Now().Unix()-HeartbeatInterval {
		log.Warningf("%s:%d consumer unhealthy by heartbeat test, releasing",
			c.topic, c.partID)
		go c.Release()
		return false
	}

	// If current has gone forward of the latest (which is possible, but unlikely)
	// then we are by definition caught up
	if c.offsetCurrent >= c.offsetLatest {
		c.cyclesBehind = 0
		return true
	}

	// If velocity is good, reset cycles behind and exit
	if partitionVelocity <= consumerVelocity {
		c.cyclesBehind = 0
		return true
	}

	// Unhealthy, so increase the cycle count so we know when it's been unhealthy for
	// too long and we want to give it up
	c.cyclesBehind++

	// If were behind by too many cycles, then we should try to release the
	// partition. If so, do this in a goroutine since it will involve calling out
	// to Kafka and releasing the partition.
	if c.cyclesBehind >= 3 {
		log.Errorf("%s:%d consumer unhealthy, releasing",
			c.topic, c.partID)
		go c.Release()
		return false
	} else {
		log.Warningf("%s:%d consumer unhealthy: CV %0.2f < PV %0.2f",
			c.topic, c.partID, consumerVelocity, partitionVelocity)
	}

	// Clearly we haven't been behind for long enough, so we're still "healthy"
	return true
}

// healthCheckLoop runs regularly and will perform a health check. Exits when we are no longer
// a claimed partition
func (c *claim) healthCheckLoop() {
	time.Sleep(<-c.marshal.jitters)
	for c.Claimed() {
		if c.healthCheck() {
			go c.heartbeat()
		}
		time.Sleep(<-c.marshal.jitters)
	}
	log.Debugf("%s:%d health check loop exiting", c.topic, c.partID)
}

// average returns the average of a given slice of int64s. It ignores 0s as
// those are "uninitialized" elements.
func average(vals []int64) float64 {
	min, max, ct := int64(0), int64(0), int64(0)
	for _, val := range vals {
		if val <= 0 {
			continue
		}
		if min == 0 || val < min {
			min = val
		}
		if max == 0 || val > max {
			max = val
		}
		ct++
	}
	if ct == 0 {
		return 0
	}
	return float64(max-min) / float64(ct)
}

// ConsumerVelocity returns the average of our consumers' velocity
func (c *claim) ConsumerVelocity() float64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	return average(c.offsetCurrentHistory[0:])
}

// PartitionVelocity returns the average of the partition's velocity
func (c *claim) PartitionVelocity() float64 {
	c.lock.RLock()
	defer c.lock.RUnlock()

	return average(c.offsetLatestHistory[0:])
}

// updateOffsets will update the offsets of our current partition.
func (c *claim) updateOffsets(ctr int) error {
	// Slow, hits Kafka. Run in a goroutine.
	oEarly, oLate, _, err := c.marshal.GetPartitionOffsets(c.topic, c.partID)
	if err != nil {
		log.Errorf("%s:%d failed to get offsets: %s", c.topic, c.partID, err)
		return err
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	// Update the earliest/latest offsets that are presently available within the
	// partition
	c.offsetEarliest = oEarly
	c.offsetLatest = oLate

	// Do update our "history" values, this is used for calculating moving averages
	// in the health checking function
	c.offsetLatestHistory[ctr%10] = oLate
	c.offsetCurrentHistory[ctr%10] = c.offsetCurrent

	return nil
}
<|endoftext|>


<commit_before>package structfield

import (
	"reflect"
	"strings"
)

// Transformer is the interface which provides the Transform method.
//
// Transform takes a field, whose name is field and value is value,
// and returns a new field name and new value.
type Transformer interface {
	Transform(field string, value interface{}) (string, interface{})
}

// Transform takes a struct or ptr to a struct and converts it to a map,
// applying a set of transformers on it.
//
// transformers is a map of field names to Transformer.
// Before the transformers are run, Transform honors the json struct tag of the field.
//
// Note that Transform stays at depth 1: only the fields of the struct are processed, not nested data structures.
// e.g: T.Foo is processed, but T.Foo.Bar isn't.
func Transform(v interface{}, transformers map[string]Transformer) map[string]interface{} {
	rv := reflect.ValueOf(v)
KindTest:
	for {
		switch rv.Kind() {
		case reflect.Ptr:
			rv = rv.Elem()
		case reflect.Struct:
			break KindTest
		default:
			panic("structfield.Transform: not a ptr or a struct")
		}
	}
	rt := rv.Type()
	nFields := rv.NumField()
	m := make(map[string]interface{}, nFields)
	for i := 0; i < nFields; i++ {
		var fieldName string
		tagParts := strings.Split(rt.Field(i).Tag.Get("json"), ",")
		if len(tagParts) > 0 {
			fieldName = strings.TrimSpace(tagParts[0])
		}
		if fieldName == "-" {
			continue
		}
		if fieldName == "" {
			fieldName = rt.Field(i).Name
		}

		var omitempty bool
		for _, tagPart := range tagParts[1:] {
			if tagPart == "omitempty" {
				omitempty = true
				break
			}
		}
		if omitempty && isEmptyValue(rv.Field(i)) {
			continue
		}
		transformer, ok := transformers[fieldName]
		if !ok {
			m[fieldName] = rv.Field(i).Interface()
			continue
		}
		newField, newValue := transformer.Transform(fieldName, rv.Field(i).Interface())
		if newField != "" {
			m[newField] = newValue
		}
	}
	return m
}

// Borrowed from encoding/json.
func isEmptyValue(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
		return v.Len() == 0
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return false
}

// TransformerFunc is to Transformer what http.HandlerFunc is to http.Handler.
type TransformerFunc func(field string, value interface{}) (string, interface{})

// Transform calls tf(field, value).
func (tf TransformerFunc) Transform(field string, value interface{}) (string, interface{}) {
	return tf(field, value)
}
<commit_msg>document field discarding when ""<commit_after>package structfield

import (
	"reflect"
	"strings"
)

// Transformer is the interface which provides the Transform method.
//
// Transform takes a field, whose name is field and value is value,
// and returns a new field name and new value.
type Transformer interface {
	Transform(field string, value interface{}) (string, interface{})
}

// Transform takes a struct or ptr to a struct and converts it to a map,
// applying a set of transformers on it.
//
// transformers is a map of field names to Transformer.
// Before the transformers are run, Transform honors the json struct tag of the field.
//
// When the Transform method of a Transformer returns an empty string for the new field name, the field is discarded.
//
// Note that Transform stays at depth 1: only the fields of the struct are processed, not nested data structures.
// e.g: T.Foo is processed, but T.Foo.Bar isn't.
func Transform(v interface{}, transformers map[string]Transformer) map[string]interface{} {
	rv := reflect.ValueOf(v)
KindTest:
	for {
		switch rv.Kind() {
		case reflect.Ptr:
			rv = rv.Elem()
		case reflect.Struct:
			break KindTest
		default:
			panic("structfield.Transform: not a ptr or a struct")
		}
	}
	rt := rv.Type()
	nFields := rv.NumField()
	m := make(map[string]interface{}, nFields)
	for i := 0; i < nFields; i++ {
		var fieldName string
		tagParts := strings.Split(rt.Field(i).Tag.Get("json"), ",")
		if len(tagParts) > 0 {
			fieldName = strings.TrimSpace(tagParts[0])
		}
		if fieldName == "-" {
			continue
		}
		if fieldName == "" {
			fieldName = rt.Field(i).Name
		}

		var omitempty bool
		for _, tagPart := range tagParts[1:] {
			if tagPart == "omitempty" {
				omitempty = true
				break
			}
		}
		if omitempty && isEmptyValue(rv.Field(i)) {
			continue
		}
		transformer, ok := transformers[fieldName]
		if !ok {
			m[fieldName] = rv.Field(i).Interface()
			continue
		}
		newField, newValue := transformer.Transform(fieldName, rv.Field(i).Interface())
		if newField != "" {
			m[newField] = newValue
		}
	}
	return m
}

// Borrowed from encoding/json.
func isEmptyValue(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
		return v.Len() == 0
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return false
}

// TransformerFunc is to Transformer what http.HandlerFunc is to http.Handler.
type TransformerFunc func(field string, value interface{}) (string, interface{})

// Transform calls tf(field, value).
func (tf TransformerFunc) Transform(field string, value interface{}) (string, interface{}) {
	return tf(field, value)
}
<|endoftext|>


<commit_before>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	//"fmt"
	check "launchpad.net/gocheck"
	"math"

	"github.com/gonum/floats"
)

func isUpperTriangular(a *Dense) bool {
	rows, cols := a.Dims()
	for c := 0; c < cols-1; c++ {
		for r := c + 1; r < rows; r++ {
			if math.Abs(a.At(r, c)) > 1e-14 {
				return false
			}
		}
	}
	return true
}

func isOrthogonal(a *Dense) bool {
	rows, cols := a.Dims()
	col1 := make([]float64, rows)
	col2 := make([]float64, rows)
	for i := 0; i < cols-1; i++ {
		for j := i + 1; j < cols; j++ {
			a.Col(col1, i)
			a.Col(col2, j)
			dot := floats.Dot(col1, col2)
			if math.Abs(dot) > 1e-14 {
				return false
			}
		}
	}
	return true
}

func (s *S) TestQRD(c *check.C) {
	for _, test := range []struct {
		a    [][]float64
		name string
	}{
		{
			name: "Square",
			a: [][]float64{
				{1.3, 2.4, 8.9},
				{-2.6, 8.7, 9.1},
				{5.6, 5.8, 2.1},
			},
		},
		{
			name: "Skinny",
			a: [][]float64{
				{1.3, 2.4, 8.9},
				{-2.6, 8.7, 9.1},
				{5.6, 5.8, 2.1},
				{19.4, 5.2, -26.1},
			},
		},
		/*
			{
				name: "Fat",
				a: [][]float64{
					{1.3, 2.4, 8.9, 19.4},
					{-2.6, 8.7, 9.1, 5.2},
					{5.6, 5.8, 2.1, -26.1},
				},
			},
		*/
	} {

		a := NewDense(flatten(test.a))
		qf := QR(DenseCopyOf(a))
		r := qf.R()
		q := qf.Q()

		rows, cols := a.Dims()
		newA := NewDense(rows, cols, nil)
		newA.Mul(q, r)

		c.Check(isOrthogonal(q), check.Equals, true, check.Commentf("Test %v: Q not orthogonal", test.name))
		c.Check(isUpperTriangular(r), check.Equals, true, check.Commentf("Test %v: R not upper triangular", test.name))
		c.Check(a.EqualsApprox(newA, 1e-14), check.Equals, true, check.Commentf("Test %v: Q*R != A", test.name))
	}
}
<commit_msg>Clean imports<commit_after>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	check "launchpad.net/gocheck"
	"math"

	"github.com/gonum/floats"
)

func isUpperTriangular(a *Dense) bool {
	rows, cols := a.Dims()
	for c := 0; c < cols-1; c++ {
		for r := c + 1; r < rows; r++ {
			if math.Abs(a.At(r, c)) > 1e-14 {
				return false
			}
		}
	}
	return true
}

func isOrthogonal(a *Dense) bool {
	rows, cols := a.Dims()
	col1 := make([]float64, rows)
	col2 := make([]float64, rows)
	for i := 0; i < cols-1; i++ {
		for j := i + 1; j < cols; j++ {
			a.Col(col1, i)
			a.Col(col2, j)
			dot := floats.Dot(col1, col2)
			if math.Abs(dot) > 1e-14 {
				return false
			}
		}
	}
	return true
}

func (s *S) TestQRD(c *check.C) {
	for _, test := range []struct {
		a    [][]float64
		name string
	}{
		{
			name: "Square",
			a: [][]float64{
				{1.3, 2.4, 8.9},
				{-2.6, 8.7, 9.1},
				{5.6, 5.8, 2.1},
			},
		},
		{
			name: "Skinny",
			a: [][]float64{
				{1.3, 2.4, 8.9},
				{-2.6, 8.7, 9.1},
				{5.6, 5.8, 2.1},
				{19.4, 5.2, -26.1},
			},
		},
		/*
			{
				name: "Fat",
				a: [][]float64{
					{1.3, 2.4, 8.9, 19.4},
					{-2.6, 8.7, 9.1, 5.2},
					{5.6, 5.8, 2.1, -26.1},
				},
			},
		*/
	} {

		a := NewDense(flatten(test.a))
		qf := QR(DenseCopyOf(a))
		r := qf.R()
		q := qf.Q()

		rows, cols := a.Dims()
		newA := NewDense(rows, cols, nil)
		newA.Mul(q, r)

		c.Check(isOrthogonal(q), check.Equals, true, check.Commentf("Test %v: Q not orthogonal", test.name))
		c.Check(isUpperTriangular(r), check.Equals, true, check.Commentf("Test %v: R not upper triangular", test.name))
		c.Check(a.EqualsApprox(newA, 1e-14), check.Equals, true, check.Commentf("Test %v: Q*R != A", test.name))
	}
}
<|endoftext|>