<commit_before>package multiverse

import (
	"strings"
	"sync"
	"unicode"

	"github.com/CasualSuperman/Diorite/trie"
	"github.com/CasualSuperman/sift3"
	"github.com/CasualSuperman/phonetics"
)

func generatePhoneticsMaps(cards []*Card) trie.Trie {
	metaphoneMap := trie.Alt()

	for i, c := range cards {
		name := preventUnicode(c.Name)
		for _, word := range strings.Split(name, " ") {
			if len(word) < 4 {
				continue
			}
			mtp := phonetics.EncodeMetaphone(word)

			others, ok := metaphoneMap.Get(mtp)
			if ok {
				slice := others.([]int)
				slice = append(slice, i)
				metaphoneMap.Remove(mtp)
				metaphoneMap.Add(mtp, slice)
			} else {
				metaphoneMap.Add(mtp, []int{i})
			}
		}
	}

	return metaphoneMap
}

var phoneticsLock sync.RWMutex
var phoneticsCache = make(map[string]string)

func getMetaphone(s string) string {
	phoneticsLock.RLock()
	if cached, ok := phoneticsCache[s]; ok {
		phoneticsLock.RUnlock()
		return cached
	}
	phoneticsLock.RUnlock()

	m := phonetics.EncodeMetaphone(s)
	phoneticsLock.Lock()
	phoneticsCache[s] = m
	phoneticsLock.Unlock()
	return m
}

var unicodeLock sync.RWMutex
var unicodeCache = make(map[string]string)

func preventUnicode(name string) string {
	unicodeLock.RLock()
	if cached, ok := unicodeCache[name]; ok {
		unicodeLock.RUnlock()
		return cached
	}
	oldName := name
	name = strings.ToLower(name)
	if cached, ok := unicodeCache[name]; ok {
		unicodeLock.RUnlock()
		unicodeLock.Lock()
		unicodeCache[oldName] = cached
		unicodeLock.Unlock()
		return cached
	}

	unicodeLock.RUnlock()

	clean := ""
	for _, r := range name {
		if r > 128 {
			switch r {
			case 'á', 'à', 'â':
				clean += "a"
			case 'é':
				clean += "e"
			case 'í':
				clean += "i"
			case 'ö':
				clean += "o"
			case 'û', 'ú':
				clean += "u"

			case 'Æ', 'æ':
				clean += "ae"

			case '®':
				// We know this is an option but we're explicitly ignoring it.

			default:
			}
		} else {
			if r == ' ' || unicode.IsLetter(r) {
				clean += string(r)
			}
		}
	}

	unicodeLock.Lock()
	unicodeCache[oldName] = clean
	unicodeCache[name] = clean
	unicodeLock.Unlock()

	return clean
}

type fuzzySearchList []struct {
	index      int
	similarity float32
}

// FuzzyNameSearch searches for a card with a similar name to the searchPhrase, and returns count or less of the most likely results.
func (m Multiverse) FuzzyNameSearch(searchPhrase string, count int) []*Card {
	var aggregator = make(fuzzySearchList, 0, count)
	searchPhrase = preventUnicode(searchPhrase)
	searchGrams2 := newNGram(searchPhrase, 2)
	searchGrams3 := newNGram(searchPhrase, 3)

	for _, searchTerm := range strings.Split(searchPhrase, " ") {
		for _, candidate := range m.Pronunciations.Search(getMetaphone(searchTerm)) {
			cardIndices, _ := m.Pronunciations.Get(candidate)
		cardLoop:
			for _, cardIndex := range cardIndices.([]int) {
				name := preventUnicode(m.Cards.List[cardIndex].Name)

				bestMatch := 0
				for _, word := range strings.Split(name, " ") {
					match := phonetics.DifferenceSoundex(word, searchTerm)
					if match > bestMatch {
						bestMatch = match
					}
				}

				similarity := searchGrams2.Similarity(name)
				similarity *= searchGrams3.Similarity(name)
				similarity *= float32(len(name) * bestMatch)
				similarity /= float32(sift3.Sift(searchPhrase, name))

				if strings.Contains(name, searchPhrase) {
					similarity *= 10
				}

				var app = struct {
					index      int
					similarity float32
				}{
					cardIndex,
					similarity,
				}

				for i, candidate := range aggregator {
					if candidate.index == app.index && app.similarity > candidate.similarity {
						aggregator[i].similarity = app.similarity
						continue cardLoop
					}
				}

				if len(aggregator) < cap(aggregator) {
					i := len(aggregator)
					aggregator = aggregator[:i+1]
					aggregator[i] = app
				} else {
					for i := count - 1; i >= 0; i-- {
						if aggregator[i].similarity < app.similarity {
							if i < count-1 {
								aggregator[i+1] = aggregator[i]
							}
							aggregator[i] = app
						} else {
							i = 0
						}
					}
				}
			}
		}
	levenshteinLoop:
		for cardIndex, card := range m.Cards.List {
			for _, word := range strings.Split(preventUnicode(card.Name), " ") {
				if sift3.Sift(word, searchTerm) <= len(searchTerm)/3 {

					name := preventUnicode(card.Name)
					similarity := searchGrams2.Similarity(name)
					similarity *= searchGrams3.Similarity(name)
					similarity *= float32(len(name) * phonetics.DifferenceSoundex(word, searchTerm)) / 10.0
					similarity /= float32(sift3.Sift(searchPhrase, name))
					var app = struct {
						index      int
						similarity float32
					}{
						cardIndex,
						similarity,
					}

					for i, ci := range aggregator {
						if cardIndex == ci.index {
							if ci.similarity < similarity {
								aggregator[i] = app
							}
							continue levenshteinLoop
						}
					}

					if len(aggregator) < cap(aggregator) {
						i := len(aggregator)
						aggregator = aggregator[:i+1]
						aggregator[i] = app
					} else {
						for i := count - 1; i >= 0; i-- {
							if aggregator[i].similarity < app.similarity {
								if i < count-1 {
									aggregator[i+1] = aggregator[i]
								}
								aggregator[i] = app
							} else {
								i = 0
							}
						}
					}
				}
			}
		}
	}

	if len(aggregator) < count {
		count = len(aggregator)
	}

	results := make([]*Card, count)

	for i, card := range aggregator {
		results[i] = m.Cards.List[card.index]
	}

	return results
}
<commit_msg>Simplified fuzzysearch logic.<commit_after>package multiverse

import (
	"strings"
	"sync"
	"unicode"

	"github.com/CasualSuperman/Diorite/trie"
	"github.com/CasualSuperman/phonetics"
	"github.com/CasualSuperman/sift3"
)

func generatePhoneticsMaps(cards []*Card) trie.Trie {
	metaphoneMap := trie.Alt()

	for i, c := range cards {
		name := preventUnicode(c.Name)
		for _, word := range strings.Split(name, " ") {
			if len(word) < 4 {
				continue
			}
			mtp := phonetics.EncodeMetaphone(word)

			others, ok := metaphoneMap.Get(mtp)
			if ok {
				slice := others.([]int)
				slice = append(slice, i)
				metaphoneMap.Remove(mtp)
				metaphoneMap.Add(mtp, slice)
			} else {
				metaphoneMap.Add(mtp, []int{i})
			}
		}
	}

	return metaphoneMap
}

var phoneticsLock sync.RWMutex
var phoneticsCache = make(map[string]string)

func getMetaphone(s string) string {
	phoneticsLock.RLock()
	if cached, ok := phoneticsCache[s]; ok {
		phoneticsLock.RUnlock()
		return cached
	}
	phoneticsLock.RUnlock()

	m := phonetics.EncodeMetaphone(s)
	phoneticsLock.Lock()
	phoneticsCache[s] = m
	phoneticsLock.Unlock()
	return m
}

var unicodeLock sync.RWMutex
var unicodeCache = make(map[string]string)

func preventUnicode(name string) string {
	unicodeLock.RLock()
	if cached, ok := unicodeCache[name]; ok {
		unicodeLock.RUnlock()
		return cached
	}
	oldName := name
	name = strings.ToLower(name)
	if cached, ok := unicodeCache[name]; ok {
		unicodeLock.RUnlock()
		unicodeLock.Lock()
		unicodeCache[oldName] = cached
		unicodeLock.Unlock()
		return cached
	}

	unicodeLock.RUnlock()

	clean := ""
	for _, r := range name {
		if r > 128 {
			switch r {
			case 'á', 'à', 'â':
				clean += "a"
			case 'é':
				clean += "e"
			case 'í':
				clean += "i"
			case 'ö':
				clean += "o"
			case 'û', 'ú':
				clean += "u"

			case 'Æ', 'æ':
				clean += "ae"

			case '®':
				// We know this is an option but we're explicitly ignoring it.

			default:
			}
		} else {
			if r == ' ' || unicode.IsLetter(r) {
				clean += string(r)
			}
		}
	}

	unicodeLock.Lock()
	unicodeCache[oldName] = clean
	unicodeCache[name] = clean
	unicodeLock.Unlock()

	return clean
}

type fuzzySearchList []struct {
	index      int
	similarity float32
}

func (f *fuzzySearchList) Add(index int, similarity float32) {
	for i, item := range *f {
		if item.index == index {
			if (*f)[i].similarity < similarity {
				(*f)[i].similarity = similarity
			}
			return
		}
	}

	myLen := len(*f)

	if myLen < cap(*f) {
		(*f) = (*f)[:myLen+1]
		myLen++
	}

	for i := myLen - 1; i >= 0; i-- {
		if (*f)[i].similarity < similarity {
			if i < myLen-1 {
				(*f)[i+1] = (*f)[i]
			}
			(*f)[i].index = index
			(*f)[i].similarity = similarity
		} else {
			return
		}
	}
}

// FuzzyNameSearch searches for a card with a similar name to the searchPhrase, and returns count or less of the most likely results.
func (m Multiverse) FuzzyNameSearch(searchPhrase string, count int) []*Card {
	var aggregator = make(fuzzySearchList, 0, count)
	searchPhrase = preventUnicode(searchPhrase)
	searchGrams2 := newNGram(searchPhrase, 2)
	searchGrams3 := newNGram(searchPhrase, 3)

	for _, searchTerm := range strings.Split(searchPhrase, " ") {
		for _, candidate := range m.Pronunciations.Search(getMetaphone(searchTerm)) {
			cardIndices, _ := m.Pronunciations.Get(candidate)

			for _, cardIndex := range cardIndices.([]int) {
				name := preventUnicode(m.Cards.List[cardIndex].Name)

				bestMatch := 0
				for _, word := range strings.Split(name, " ") {
					match := phonetics.DifferenceSoundex(word, searchTerm)
					if match > bestMatch {
						bestMatch = match
					}
				}

				similarity := searchGrams2.Similarity(name)
				similarity += searchGrams3.Similarity(name)
				similarity *= float32(len(name) * bestMatch)
				similarity /= float32(sift3.Sift(searchPhrase, name))

				if strings.Contains(name, searchPhrase) {
					similarity *= 50
				}

				aggregator.Add(cardIndex, similarity)
			}
		}

		for cardIndex, card := range m.Cards.List {
			for _, word := range strings.Split(preventUnicode(card.Name), " ") {
				if sift3.Sift(word, searchTerm) <= len(searchTerm)/3 {

					name := preventUnicode(card.Name)
					similarity := searchGrams2.Similarity(name)
					similarity += searchGrams3.Similarity(name)
					similarity *= float32(len(name)*phonetics.DifferenceSoundex(word, searchTerm)) / 10.0
					similarity /= float32(sift3.Sift(searchPhrase, name))

					aggregator.Add(cardIndex, similarity)
				}
			}
		}
	}

	if len(aggregator) < count {
		count = len(aggregator)
	}

	results := make([]*Card, count)

	for i, card := range aggregator {
		results[i] = m.Cards.List[card.index]
	}

	return results
}
<|endoftext|>


<commit_before>// Copyright 2015-2016 Zack Scholl. All rights reserved.
// Use of this source code is governed by a AGPL
// license that can be found in the LICENSE file.

// fingerprint.go contains structures and functions for handling fingerprints.

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"strconv"

	"net/http"
	"path"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	"github.com/gin-gonic/gin"
)

// Fingerprint is the prototypical information from the fingerprinting device
// IF you change Fingerprint, follow these steps to re-generate fingerprint_ffjson.go
// find ./ -name "*.go" -type f | xargs sed -i  's/package main/package main/g'
// Uncomment json.Marshal/Unmarshal functions
// $GOPATH/bin/ffjson fingerprint.go
// find ./ -name "*.go" -type f | xargs sed -i  's/package main/package main/g'
// Comment json.Marshal/Unmarshal functions
type Fingerprint struct {
	Group           string   `json:"group"`
	Username        string   `json:"username"`
	Location        string   `json:"location"`
	Timestamp       int64    `json:"timestamp"`
	WifiFingerprint []Router `json:"wifi-fingerprint"`
}

// Router is the router information for each invdividual mac address
type Router struct {
	Mac  string `json:"mac"`
	Rssi int    `json:"rssi"`
}

var jsonExample = `{
	"group": "whatevergroup",
	"username": "iamauser",
	"location": null,
	"wififingerprint": [{
		"mac": "AA:AA:AA:AA:AA:AA",
		"rssi": -45
	}, {
		"mac": "BB:BB:BB:BB:BB:BB",
		"rssi": -55
	}]
}`

// compression 9 us -> 900 us
func dumpFingerprint(res Fingerprint) []byte {
	dumped, _ := res.MarshalJSON()
	//dumped, _ := json.Marshal(res)
	return compressByte(dumped)
}

// compression 30 us -> 600 us
func loadFingerprint(jsonByte []byte) Fingerprint {
	res := Fingerprint{}
	//json.Unmarshal(decompressByte(jsonByte), res)
	res.UnmarshalJSON(decompressByte(jsonByte))
	return res
}

func cleanFingerprint(res *Fingerprint) {
	res.Group = strings.ToLower(res.Group)
	res.Location = strings.ToLower(res.Location)
	res.Username = strings.ToLower(res.Username)
	deleteIndex := -1
	for r := range res.WifiFingerprint {
		if res.WifiFingerprint[r].Rssi >= 0 { // https://stackoverflow.com/questions/15797920/how-to-convert-wifi-signal-strength-from-quality-percent-to-rssi-dbm
			res.WifiFingerprint[r].Rssi = int(res.WifiFingerprint[r].Rssi/2) - 100
		}
		if res.WifiFingerprint[r].Mac == "00:00:00:00:00:00" {
			deleteIndex = r
		}
	}
	if deleteIndex > -1 {
		res.WifiFingerprint[deleteIndex] = res.WifiFingerprint[len(res.WifiFingerprint)-1]
		res.WifiFingerprint = res.WifiFingerprint[:len(res.WifiFingerprint)-1]
	}
}

func putFingerprintIntoDatabase(res Fingerprint, database string) error {
	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, res.Group+".db"), 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(database))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}

		if res.Timestamp == 0 {
			res.Timestamp = time.Now().UnixNano()
		}
		err = bucket.Put([]byte(strconv.FormatInt(res.Timestamp, 10)), dumpFingerprint(res))
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}
		return err
	})
	return err
}

func trackFingerprintPOST(c *gin.Context) {
	var jsonFingerprint Fingerprint
	if c.BindJSON(&jsonFingerprint) == nil {
		message, success, locationGuess, bayes, svm := trackFingerprint(jsonFingerprint)
		if success {
			c.JSON(http.StatusOK, gin.H{"message": message, "success": true, "location": locationGuess, "bayes": bayes, "svm": svm})
		} else {
			c.JSON(http.StatusOK, gin.H{"message": message, "success": false})
		}
	} else {
		Warning.Println("Could not bind JSON")
		c.JSON(http.StatusOK, gin.H{"message": "Could not bind JSON", "success": false})
	}
}

func learnFingerprintPOST(c *gin.Context) {
	var jsonFingerprint Fingerprint
	if c.BindJSON(&jsonFingerprint) == nil {
		message, success := learnFingerprint(jsonFingerprint)
		Debug.Println(message)
		if !success {
			Debug.Println(jsonFingerprint)
		}
		c.JSON(http.StatusOK, gin.H{"message": message, "success": success})
	} else {
		Warning.Println("Could not bind JSON")
		c.JSON(http.StatusOK, gin.H{"message": "Could not bind JSON", "success": false})
	}
}

func learnFingerprint(jsonFingerprint Fingerprint) (string, bool) {
	cleanFingerprint(&jsonFingerprint)
	if len(jsonFingerprint.Group) == 0 {
		return "Need to define your group name in request, see API", false
	}
	if len(jsonFingerprint.WifiFingerprint) == 0 {
		return "No fingerprints found to insert, see API", false
	}
	putFingerprintIntoDatabase(jsonFingerprint, "fingerprints")
	isLearning[strings.ToLower(jsonFingerprint.Group)] = true
	message := "Inserted fingerprint containing " + strconv.Itoa(len(jsonFingerprint.WifiFingerprint)) + " APs for " + jsonFingerprint.Username + " at " + jsonFingerprint.Location
	return message, true
}

func trackFingerprint(jsonFingerprint Fingerprint) (string, bool, string, map[string]float64, map[string]float64) {
	bayes := make(map[string]float64)
	svmData := make(map[string]float64)
	cleanFingerprint(&jsonFingerprint)
	if !groupExists(jsonFingerprint.Group) || len(jsonFingerprint.Group) == 0 {
		return "You should insert fingerprints before tracking", false, "", bayes, make(map[string]float64)
	}
	if len(jsonFingerprint.WifiFingerprint) == 0 {
		return "No fingerprints found to track, see API", false, "", bayes, make(map[string]float64)
	}
	if len(jsonFingerprint.Username) == 0 {
		return "No username defined, see API", false, "", bayes, make(map[string]float64)
	}
	if wasLearning, ok := isLearning[strings.ToLower(jsonFingerprint.Group)]; ok {
		if wasLearning {
			Debug.Println("Was learning, calculating priors")
			group := strings.ToLower(jsonFingerprint.Group)
			isLearning[group] = false
			optimizePriorsThreaded(group)
			if RuntimeArgs.Svm {
				dumpFingerprintsSVM(group)
				calculateSVM(group)
			}
			if _, ok := usersCache[group]; ok {
				if len(usersCache[group]) == 0 {
					usersCache[group] = append([]string{}, strings.ToLower(jsonFingerprint.Username))
				}
			}
		}
	}
	locationGuess1, bayes := calculatePosterior(jsonFingerprint, *NewFullParameters())
	percentGuess1 := float64(0)
	total := float64(0)
	for _, locBayes := range bayes {
		total += math.Exp(locBayes)
		if locBayes > percentGuess1 {
			percentGuess1 = locBayes
		}
	}
	percentGuess1 = math.Exp(bayes[locationGuess1]) / total * 100.0

	jsonFingerprint.Location = locationGuess1
	putFingerprintIntoDatabase(jsonFingerprint, "fingerprints-track")

	Debug.Println("Tracking fingerprint containing " + strconv.Itoa(len(jsonFingerprint.WifiFingerprint)) + " APs for " + jsonFingerprint.Username + " (" + jsonFingerprint.Group + ") at " + jsonFingerprint.Location + " (guess)")
	message := "Current location: " + locationGuess1 + " (" + strconv.Itoa(int(percentGuess1)) + "% confidence)"

	// Process SVM if needed
	if RuntimeArgs.Svm {
		locationGuess2, svmData2 := classify(jsonFingerprint)
		percentGuess2 := int(100 * math.Exp(svmData2[locationGuess2]))
		if percentGuess2 > 100 {
			percentGuess2 = percentGuess2 / 10
		}
		//message = "NB: " + locationGuess1 + " (" + strconv.Itoa(int(percentGuess1)) + "%)" + ", SVM: " + locationGuess2 + " (" + strconv.Itoa(int(percentGuess2)) + "%)"
		svmData = svmData2
	}

	// Send MQTT if needed
	if RuntimeArgs.Mqtt {
		type FingerprintResponse struct {
			LocationGuess string             `json:"location"`
			Bayes         map[string]float64 `json:"bayes"`
			Svm           map[string]float64 `json:"svm"`
		}
		mqttMessage, _ := json.Marshal(FingerprintResponse{
			LocationGuess: locationGuess1,
			Bayes:         bayes,
			Svm:           svmData,
		})
		go sendMQTTLocation(string(mqttMessage), jsonFingerprint.Group, jsonFingerprint.Username)
	}

	// Send out the final responses
	var userJSON UserPositionJSON
	userJSON.Location = locationGuess1
	userJSON.Bayes = bayes
	userJSON.Svm = svmData
	userJSON.Time = time.Now().String()
	userPositionCache[strings.ToLower(jsonFingerprint.Group)+strings.ToLower(jsonFingerprint.Username)] = userJSON

	return message, true, locationGuess1, bayes, svmData

}
<commit_msg>Removed SVM from message<commit_after>// Copyright 2015-2016 Zack Scholl. All rights reserved.
// Use of this source code is governed by a AGPL
// license that can be found in the LICENSE file.

// fingerprint.go contains structures and functions for handling fingerprints.

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"strconv"

	"net/http"
	"path"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	"github.com/gin-gonic/gin"
)

// Fingerprint is the prototypical information from the fingerprinting device
// IF you change Fingerprint, follow these steps to re-generate fingerprint_ffjson.go
// find ./ -name "*.go" -type f | xargs sed -i  's/package main/package main/g'
// Uncomment json.Marshal/Unmarshal functions
// $GOPATH/bin/ffjson fingerprint.go
// find ./ -name "*.go" -type f | xargs sed -i  's/package main/package main/g'
// Comment json.Marshal/Unmarshal functions
type Fingerprint struct {
	Group           string   `json:"group"`
	Username        string   `json:"username"`
	Location        string   `json:"location"`
	Timestamp       int64    `json:"timestamp"`
	WifiFingerprint []Router `json:"wifi-fingerprint"`
}

// Router is the router information for each invdividual mac address
type Router struct {
	Mac  string `json:"mac"`
	Rssi int    `json:"rssi"`
}

var jsonExample = `{
	"group": "whatevergroup",
	"username": "iamauser",
	"location": null,
	"wififingerprint": [{
		"mac": "AA:AA:AA:AA:AA:AA",
		"rssi": -45
	}, {
		"mac": "BB:BB:BB:BB:BB:BB",
		"rssi": -55
	}]
}`

// compression 9 us -> 900 us
func dumpFingerprint(res Fingerprint) []byte {
	dumped, _ := res.MarshalJSON()
	//dumped, _ := json.Marshal(res)
	return compressByte(dumped)
}

// compression 30 us -> 600 us
func loadFingerprint(jsonByte []byte) Fingerprint {
	res := Fingerprint{}
	//json.Unmarshal(decompressByte(jsonByte), res)
	res.UnmarshalJSON(decompressByte(jsonByte))
	return res
}

func cleanFingerprint(res *Fingerprint) {
	res.Group = strings.ToLower(res.Group)
	res.Location = strings.ToLower(res.Location)
	res.Username = strings.ToLower(res.Username)
	deleteIndex := -1
	for r := range res.WifiFingerprint {
		if res.WifiFingerprint[r].Rssi >= 0 { // https://stackoverflow.com/questions/15797920/how-to-convert-wifi-signal-strength-from-quality-percent-to-rssi-dbm
			res.WifiFingerprint[r].Rssi = int(res.WifiFingerprint[r].Rssi/2) - 100
		}
		if res.WifiFingerprint[r].Mac == "00:00:00:00:00:00" {
			deleteIndex = r
		}
	}
	if deleteIndex > -1 {
		res.WifiFingerprint[deleteIndex] = res.WifiFingerprint[len(res.WifiFingerprint)-1]
		res.WifiFingerprint = res.WifiFingerprint[:len(res.WifiFingerprint)-1]
	}
}

func putFingerprintIntoDatabase(res Fingerprint, database string) error {
	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, res.Group+".db"), 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(database))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}

		if res.Timestamp == 0 {
			res.Timestamp = time.Now().UnixNano()
		}
		err = bucket.Put([]byte(strconv.FormatInt(res.Timestamp, 10)), dumpFingerprint(res))
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}
		return err
	})
	return err
}

func trackFingerprintPOST(c *gin.Context) {
	var jsonFingerprint Fingerprint
	if c.BindJSON(&jsonFingerprint) == nil {
		message, success, locationGuess, bayes, svm := trackFingerprint(jsonFingerprint)
		if success {
			c.JSON(http.StatusOK, gin.H{"message": message, "success": true, "location": locationGuess, "bayes": bayes, "svm": svm})
		} else {
			c.JSON(http.StatusOK, gin.H{"message": message, "success": false})
		}
	} else {
		Warning.Println("Could not bind JSON")
		c.JSON(http.StatusOK, gin.H{"message": "Could not bind JSON", "success": false})
	}
}

func learnFingerprintPOST(c *gin.Context) {
	var jsonFingerprint Fingerprint
	if c.BindJSON(&jsonFingerprint) == nil {
		message, success := learnFingerprint(jsonFingerprint)
		Debug.Println(message)
		if !success {
			Debug.Println(jsonFingerprint)
		}
		c.JSON(http.StatusOK, gin.H{"message": message, "success": success})
	} else {
		Warning.Println("Could not bind JSON")
		c.JSON(http.StatusOK, gin.H{"message": "Could not bind JSON", "success": false})
	}
}

func learnFingerprint(jsonFingerprint Fingerprint) (string, bool) {
	cleanFingerprint(&jsonFingerprint)
	if len(jsonFingerprint.Group) == 0 {
		return "Need to define your group name in request, see API", false
	}
	if len(jsonFingerprint.WifiFingerprint) == 0 {
		return "No fingerprints found to insert, see API", false
	}
	putFingerprintIntoDatabase(jsonFingerprint, "fingerprints")
	isLearning[strings.ToLower(jsonFingerprint.Group)] = true
	message := "Inserted fingerprint containing " + strconv.Itoa(len(jsonFingerprint.WifiFingerprint)) + " APs for " + jsonFingerprint.Username + " at " + jsonFingerprint.Location
	return message, true
}

func trackFingerprint(jsonFingerprint Fingerprint) (string, bool, string, map[string]float64, map[string]float64) {
	bayes := make(map[string]float64)
	svmData := make(map[string]float64)
	cleanFingerprint(&jsonFingerprint)
	if !groupExists(jsonFingerprint.Group) || len(jsonFingerprint.Group) == 0 {
		return "You should insert fingerprints before tracking", false, "", bayes, make(map[string]float64)
	}
	if len(jsonFingerprint.WifiFingerprint) == 0 {
		return "No fingerprints found to track, see API", false, "", bayes, make(map[string]float64)
	}
	if len(jsonFingerprint.Username) == 0 {
		return "No username defined, see API", false, "", bayes, make(map[string]float64)
	}
	if wasLearning, ok := isLearning[strings.ToLower(jsonFingerprint.Group)]; ok {
		if wasLearning {
			Debug.Println("Was learning, calculating priors")
			group := strings.ToLower(jsonFingerprint.Group)
			isLearning[group] = false
			optimizePriorsThreaded(group)
			if RuntimeArgs.Svm {
				dumpFingerprintsSVM(group)
				calculateSVM(group)
			}
			if _, ok := usersCache[group]; ok {
				if len(usersCache[group]) == 0 {
					usersCache[group] = append([]string{}, strings.ToLower(jsonFingerprint.Username))
				}
			}
		}
	}
	locationGuess1, bayes := calculatePosterior(jsonFingerprint, *NewFullParameters())
	percentGuess1 := float64(0)
	total := float64(0)
	for _, locBayes := range bayes {
		total += math.Exp(locBayes)
		if locBayes > percentGuess1 {
			percentGuess1 = locBayes
		}
	}
	percentGuess1 = math.Exp(bayes[locationGuess1]) / total * 100.0

	jsonFingerprint.Location = locationGuess1
	putFingerprintIntoDatabase(jsonFingerprint, "fingerprints-track")

	Debug.Println("Tracking fingerprint containing " + strconv.Itoa(len(jsonFingerprint.WifiFingerprint)) + " APs for " + jsonFingerprint.Username + " (" + jsonFingerprint.Group + ") at " + jsonFingerprint.Location + " (guess)")
	message := "Current location: " + locationGuess1 //+ " (" + strconv.Itoa(int(percentGuess1)) + "% confidence)"

	// Process SVM if needed
	if RuntimeArgs.Svm {
		locationGuess2, svmData2 := classify(jsonFingerprint)
		percentGuess2 := int(100 * math.Exp(svmData2[locationGuess2]))
		if percentGuess2 > 100 {
			percentGuess2 = percentGuess2 / 10
		}
		//message = "NB: " + locationGuess1 + " (" + strconv.Itoa(int(percentGuess1)) + "%)" + ", SVM: " + locationGuess2 + " (" + strconv.Itoa(int(percentGuess2)) + "%)"
		svmData = svmData2
	}

	// Send MQTT if needed
	if RuntimeArgs.Mqtt {
		type FingerprintResponse struct {
			LocationGuess string             `json:"location"`
			Bayes         map[string]float64 `json:"bayes"`
			Svm           map[string]float64 `json:"svm"`
		}
		mqttMessage, _ := json.Marshal(FingerprintResponse{
			LocationGuess: locationGuess1,
			Bayes:         bayes,
			Svm:           svmData,
		})
		go sendMQTTLocation(string(mqttMessage), jsonFingerprint.Group, jsonFingerprint.Username)
	}

	// Send out the final responses
	var userJSON UserPositionJSON
	userJSON.Location = locationGuess1
	userJSON.Bayes = bayes
	userJSON.Svm = svmData
	userJSON.Time = time.Now().String()
	userPositionCache[strings.ToLower(jsonFingerprint.Group)+strings.ToLower(jsonFingerprint.Username)] = userJSON

	return message, true, locationGuess1, bayes, svmData

}
<|endoftext|>


<commit_before>package db

import (
	"io/ioutil"
	"os"
	"testing"
)

func TestIdxCRUD(t *testing.T) {
	os.RemoveAll(TEST_DATA_DIR)
	defer os.RemoveAll(TEST_DATA_DIR)
	if err := os.MkdirAll(TEST_DATA_DIR, 0700); err != nil {
		t.Fatal(err)
	}
	if err := ioutil.WriteFile(TEST_DATA_DIR+"/number_of_partitions", []byte("2"), 0600); err != nil {
		t.Fatal(err)
	}
	db, err := OpenDB(TEST_DATA_DIR)
	if err != nil {
		t.Fatal(err)
	}
	if err = db.Create("col"); err != nil {
		t.Fatal(err)
	}
	col := db.Use("col")
	if len(col.AllIndexes()) != 0 {
		t.Fatal(col.AllIndexes())
	}
	// Create index & verify
	if err = col.Index([]string{"a", "b"}); err != nil {
		t.Fatal(err)
	}
	if col.Index([]string{"a", "b"}) == nil {
		t.Fatal(col.indexPaths, "Did not error")
	}
	if len(col.AllIndexes()) != 1 || col.AllIndexes()[0][0] != "a" || col.AllIndexes()[0][1] != "b" {
		t.Fatal(col.AllIndexes())
	}
	if err = col.Index([]string{"c"}); err != nil {
		t.Fatal(err)
	}
	if len(col.AllIndexes()) != 2 || !((col.AllIndexes()[0][0] == "a" && col.AllIndexes()[0][1] == "b" && col.AllIndexes()[1][0] == "c") || (col.AllIndexes()[0][0] == "c" && col.AllIndexes()[1][0] == "a" && col.AllIndexes()[1][1] == "b")) {
		t.Fatal(col.AllIndexes())
	}
	// Unindex & verify
	if col.Unindex([]string{"%&^*"}) == nil {
		t.Fatal("Did not error")
	}
	if err = col.Unindex([]string{"c"}); err != nil {
		t.Fatal(err)
	}
	if len(col.AllIndexes()) != 1 || col.AllIndexes()[0][0] != "a" || col.AllIndexes()[0][1] != "b" {
		t.Fatal(col.AllIndexes())
	}
	if err = col.Unindex([]string{"a", "b"}); err != nil {
		t.Fatal(err)
	}
	if len(col.AllIndexes()) != 0 {
		t.Fatal(col.AllIndexes())
	}
	if err = col.Sync(); err != nil {
		t.Fatal(err)
	}
	if err := db.Close(); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>fix test case, again<commit_after>package db

import (
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

func TestIdxCRUD(t *testing.T) {
	os.RemoveAll(TEST_DATA_DIR)
	defer os.RemoveAll(TEST_DATA_DIR)
	if err := os.MkdirAll(TEST_DATA_DIR, 0700); err != nil {
		t.Fatal(err)
	}
	if err := ioutil.WriteFile(TEST_DATA_DIR+"/number_of_partitions", []byte("2"), 0600); err != nil {
		t.Fatal(err)
	}
	db, err := OpenDB(TEST_DATA_DIR)
	if err != nil {
		t.Fatal(err)
	}
	if err = db.Create("col"); err != nil {
		t.Fatal(err)
	}
	col := db.Use("col")
	if len(col.AllIndexes()) != 0 {
		t.Fatal(col.AllIndexes())
	}
	// Create index & verify
	if err = col.Index([]string{"a", "b"}); err != nil {
		t.Fatal(err)
	}
	if col.Index([]string{"a", "b"}) == nil {
		t.Fatal(col.indexPaths, "Did not error")
	}
	if len(col.AllIndexes()) != 1 || col.AllIndexes()[0][0] != "a" || col.AllIndexes()[0][1] != "b" {
		t.Fatal(col.AllIndexes())
	}
	if err = col.Index([]string{"c"}); err != nil {
		t.Fatal(err)
	}
	allIndexes := col.AllIndexes()
	idx0 := strings.Join(allIndexes[0], ",")
	idx1 := strings.Join(allIndexes[1], ",")
	if !(idx0 == "a,b" && idx1 == "c" || idx0 == "c" && idx1 == "a,b") {
		t.Fatal(allIndexes)
	}
	// Unindex & verify
	if col.Unindex([]string{"%&^*"}) == nil {
		t.Fatal("Did not error")
	}
	if err = col.Unindex([]string{"c"}); err != nil {
		t.Fatal(err)
	}
	if len(col.AllIndexes()) != 1 || col.AllIndexes()[0][0] != "a" || col.AllIndexes()[0][1] != "b" {
		t.Fatal(col.AllIndexes())
	}
	if err = col.Unindex([]string{"a", "b"}); err != nil {
		t.Fatal(err)
	}
	if len(col.AllIndexes()) != 0 {
		t.Fatal(col.AllIndexes())
	}
	if err = col.Sync(); err != nil {
		t.Fatal(err)
	}
	if err := db.Close(); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package sparkle

import (
	"errors"
	"net/http"
	"reflect"
	"strconv"
)

var ErrNilUnmarshalTarget = errors.New("Passed nil object as unmarshal target")
var ErrNotPointerTarget = errors.New("Target is not a pointer")
var ErrNotStructTarget = errors.New("Target pointer does not point to struct")
var ErrCouldNotGetContextRequest = errors.New("Could not obtain request from context")
var ErrUnsupportedType = errors.New("Can not unmarshall to field type as it is unsupported")

// A map type that has both string keys and values
type KeyValueStringMap map[string]string

// An interface that supplies UnmarshalParameters with the data in which
// to unmarshal
type ParameterValueProvider interface {
	Values(c *Context) (KeyValueStringMap, error)
}

// A ValueProvider that provides values from the current request query string and
// form inputs
type FormValueProvider struct{}

var maxMemoryForMultipartForm int64
var currentValueProvider ParameterValueProvider

func init() {
	currentValueProvider = &FormValueProvider{}
	// Default to 32 MB
	maxMemoryForMultipartForm = 32 * 1024 * 1024
}

// Sets the ParameterValueProvider that sparkle will use as data for unmarshalling paramters
func SetParameterValueProvider(p ParameterValueProvider) {
	currentValueProvider = p
}

// Provides a KeyValueStringMap of the Query String and Form parameters
func (p *ParameterValueProvider) Values(c *Context) (KeyValueStringMap, error) {
	request := c.Request()
	if request == nil {
		// Should never happen
		return nil, ErrCouldNotGetContextRequest
	}

	if err := parseFormData(request); err != nil {
		return nil, err
	}

	return request.Form, nil
}

// Sets the Max Memory to be used when UnmarshalParameters encounters a
// multipart form.
func SetMaxMemory(maxMemory int64) {
	maxMemoryForMultipartForm = maxMemory
}

func canUnmarshal(v interface{}) error {
	if v == nil {
		return ErrNilUnmarshalTarget
	}

	it := reflect.ValueOf(v)
	if it.Kind() != reflect.Ptr {
		return ErrNotPointerTarget
	}

	if rt := it.Elem(); rt.Kind() != reflect.Struct {
		return ErrNotStructTarget
	}

	return nil
}

func parseFormData(r *http.Request) error {
	if r.Header.Get("Content-Type") == "multipart/form-data" {
		return r.ParseMultipartForm(maxMemoryForMultipartForm)
	}

	return r.ParseForm()
}

func getFieldAndKind(rt reflect.Value, fieldName string) (reflect.Value, reflect.Kind) {
	// Look for property on structure with the same name
	fieldValue := rt.FieldByName(fieldName)
	fieldKind := fieldValue.Kind()
	if fieldKind == reflect.Ptr {
		// Step one in if it's a pointer
		fieldValue = fieldValue.Elem()
		fieldKind = fieldValue.Kind()
	}

	return fieldValue, fieldKind
}

// Unmarshals Query and Post parameters to an object supplied in v
//
// If a parameter doesn't have a corresponding field in the struct, it is
// ignored. If the struct has a field that doesn't have a corresponding
// parameter, then the struct field will not be altered.
//
// If successful returns nil with the values in v set accordingly
// Returns ErrNilUnmarshalTarget if v was nil
// Returns ErrNotPointerTarget if v was not a pointer
// Returns ErrNotStructTarget if v was not a pointer to a struct
func (c *Context) UnmarshalParameters(v interface{}) error {

	if err := canUnmarshal(v); err != nil {
		return err
	}

	rt := reflect.ValueOf(v).Elem()

	valueMap, err := currentValueProvider(c)
	if err != nil {
		return err
	}

	// Okay, so we should step through the values in the form now
	for key, value := range valueMap {
		fieldValue, fieldKind := getFieldAndKind(rt, key)

		// if we can't set the result, then ignore it
		if !fieldValue.CanSet() {
			continue
		}

		switch fieldKind {
		case reflect.String:
			fieldValue.SetString(value[0])

		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			if v, err := strconv.ParseUint(value[0], 10, 64); err != nil {
				return err
			} else {
				fieldValue.SetUint(v)
			}
		case reflect.Int:
			if v, err := strconv.ParseInt(value[0], 10, 64); err != nil {
				return err
			} else {
				fieldValue.SetInt(v)
			}
		case reflect.Bool:
			if v, err := strconv.ParseBool(value[0]); err != nil {
				return err
			} else {
				fieldValue.SetBool(v)
			}
		default:
			return ErrUnsupportedType
		}
	}

	return nil
}
<commit_msg>Updated documentation and reorder the order of types and functions in params.go<commit_after>package sparkle

import (
	"errors"
	"net/http"
	"reflect"
	"strconv"
)

var ErrNilUnmarshalTarget = errors.New("Passed nil object as unmarshal target")
var ErrNotPointerTarget = errors.New("Target is not a pointer")
var ErrNotStructTarget = errors.New("Target pointer does not point to struct")
var ErrCouldNotGetContextRequest = errors.New("Could not obtain request from context")
var ErrUnsupportedType = errors.New("Can not unmarshall to field type as it is unsupported")

// A ValueProvider that provides values from the current request query string and
// form inputs
var currentValueProvider ParameterValueProvider
var maxMemoryForMultipartForm int64

func init() {
	currentValueProvider = &FormValueProvider{}
	// Default to 32 MB
	maxMemoryForMultipartForm = 32 * 1024 * 1024
}

// A map type that has both string keys and values
type KeyValueStringMap map[string]string

// ParameterValueProvider is an interface which when implemented provides
// the mechinism neccisary for (*Context).UnmarshalParameters to get data
// for use in unmarshalling parameters.
type ParameterValueProvider interface {
	Values(c *Context) (KeyValueStringMap, error)
}

// SetParameterValueProvider sets the ParameterValueProvider that sparkle
// will use as data for unmarshalling paramters
func SetParameterValueProvider(p ParameterValueProvider) {
	currentValueProvider = p
}

// FormValueProvider is a ParameterValueProvider that obtains data from
// the request query string and form paramaeters
type FormValueProvider struct{}

// Values provides a KeyValueStringMap of the Query String and Form parameters
func (p *FormValueProvider) Values(c *Context) (KeyValueStringMap, error) {
	request := c.Request()
	if request == nil {
		// Should never happen
		return nil, ErrCouldNotGetContextRequest
	}

	if err := parseFormData(request); err != nil {
		return nil, err
	}

	return request.Form, nil
}

// Sets the Max Memory to be used when UnmarshalParameters encounters a
// multipart form.
func SetMaxMemory(maxMemory int64) {
	maxMemoryForMultipartForm = maxMemory
}

func canUnmarshal(v interface{}) error {
	if v == nil {
		return ErrNilUnmarshalTarget
	}

	it := reflect.ValueOf(v)
	if it.Kind() != reflect.Ptr {
		return ErrNotPointerTarget
	}

	if rt := it.Elem(); rt.Kind() != reflect.Struct {
		return ErrNotStructTarget
	}

	return nil
}

func parseFormData(r *http.Request) error {
	if r.Header.Get("Content-Type") == "multipart/form-data" {
		return r.ParseMultipartForm(maxMemoryForMultipartForm)
	}

	return r.ParseForm()
}

func getFieldAndKind(rt reflect.Value, fieldName string) (reflect.Value, reflect.Kind) {
	// Look for property on structure with the same name
	fieldValue := rt.FieldByName(fieldName)
	fieldKind := fieldValue.Kind()
	if fieldKind == reflect.Ptr {
		// Step one in if it's a pointer
		fieldValue = fieldValue.Elem()
		fieldKind = fieldValue.Kind()
	}

	return fieldValue, fieldKind
}

// Unmarshals Query and Post parameters to an object supplied in v
//
// If a parameter doesn't have a corresponding field in the struct, it is
// ignored. If the struct has a field that doesn't have a corresponding
// parameter, then the struct field will not be altered.
//
// If successful returns nil with the values in v set accordingly
// Returns ErrNilUnmarshalTarget if v was nil
// Returns ErrNotPointerTarget if v was not a pointer
// Returns ErrNotStructTarget if v was not a pointer to a struct
func (c *Context) UnmarshalParameters(v interface{}) error {

	if err := canUnmarshal(v); err != nil {
		return err
	}

	rt := reflect.ValueOf(v).Elem()

	valueMap, err := currentValueProvider(c)
	if err != nil {
		return err
	}

	// Okay, so we should step through the values in the form now
	for key, value := range valueMap {
		fieldValue, fieldKind := getFieldAndKind(rt, key)

		// if we can't set the result, then ignore it
		if !fieldValue.CanSet() {
			continue
		}

		switch fieldKind {
		case reflect.String:
			fieldValue.SetString(value[0])

		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			if v, err := strconv.ParseUint(value[0], 10, 64); err != nil {
				return err
			} else {
				fieldValue.SetUint(v)
			}
		case reflect.Int:
			if v, err := strconv.ParseInt(value[0], 10, 64); err != nil {
				return err
			} else {
				fieldValue.SetInt(v)
			}
		case reflect.Bool:
			if v, err := strconv.ParseBool(value[0]); err != nil {
				return err
			} else {
				fieldValue.SetBool(v)
			}
		default:
			return ErrUnsupportedType
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"html/template"
	"log"
	"math/rand"
	"net/http"
	"runtime"
	"sort"
	"strconv"
)

const (
	connectionString = "localhost"
	helloWorldString = "Hello, world!"
	worldRowCount    = 10000
)

var (
	tmpl = template.Must(template.ParseFiles("templates/layout.html", "templates/fortune.html"))

	database *mgo.Database
	fortunes *mgo.Collection
	worlds   *mgo.Collection
)

type Message struct {
	Message string `json:"message"`
}

type World struct {
	Id           uint16 `bson:"id" json:"id"`
	RandomNumber uint16 `bson:"randomNumber" json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `bson:"id" json:"id"`
	Message string `bson:"message" json:"message"`
}

type Fortunes []Fortune

func (s Fortunes) Len() int {
	return len(s)
}

func (s Fortunes) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

type ByMessage struct{ Fortunes }

func (s ByMessage) Less(i, j int) bool {
	return s.Fortunes[i].Message < s.Fortunes[j].Message
}

func main() {
	port := ":8080"
	runtime.GOMAXPROCS(runtime.NumCPU())
	if session, err := mgo.Dial(connectionString); err != nil {
		log.Fatalf("Error opening database: %v", err)
	} else {
		defer session.Close()
		session.SetPoolLimit(5)
		database = session.DB("hello_world")
		worlds = database.C("world")
		fortunes = database.C("fortune")
		http.HandleFunc("/json", jsonHandler)
		http.HandleFunc("/db", dbHandler)
		http.HandleFunc("/fortune", fortuneHandler)
		http.HandleFunc("/queries", queriesHandler)
		http.HandleFunc("/update", updateHandler)
		http.HandleFunc("/plaintext", plaintextHandler)
		http.ListenAndServe(port, nil)
	}
}

// Helper for random numbers
func getRandomNumber() uint16 {
	return uint16(rand.Intn(worldRowCount) + 1)
}

// Test 1: JSON serialization
func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/javascript")
	json.NewEncoder(w).Encode(&Message{helloWorldString})
}

func dbHandler(w http.ResponseWriter, r *http.Request) {
	var world World
	query := bson.M{"id": getRandomNumber()}
	if worlds != nil {
		if err := worlds.Find(query).One(&world); err != nil {
			log.Fatalf("Error finding world with id: %s", err.Error())
			return
		} else {
			w.Header().Set("Content-Type", "application/json")
			json.NewEncoder(w).Encode(&world)
			return
		}
	} else {
		log.Fatal("Collection not initialized properly")
	}
}

func queriesHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) > 0 {
		n, _ = strconv.Atoi(nStr)
	}

	if n > 500 {
		n = 500
	}
	var result []World
	if n <= 1 {
		result = make([]World, 1)
		query := bson.M{"id": getRandomNumber()}
		if err := worlds.Find(query).One(&result[0]); err != nil {
			log.Fatalf("Error finding world with id: %s", err.Error())
			return
		}
	} else {
		result = make([]World, n)
		for i := 0; i < n; i++ {
			query := bson.M{"id": getRandomNumber()}
			if err := worlds.Find(query).One(&result[i]); err != nil {
				log.Fatalf("Error finding world with id: %s", err.Error())
				return
			}
		}
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(&result)
}

func fortuneHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	f := make(Fortunes, 16)
	if err := fortunes.Find(nil).All(&f); err == nil {
		f = append(f, Fortune{
			Message: "Additional fortune added at request time.",
		})
		sort.Sort(ByMessage{f})
		if err := tmpl.Execute(w, f); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}

}

func updateHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) > 0 {
		n, _ = strconv.Atoi(nStr)
	}

	w.Header().Set("Content-Type", "application/json")
	encoder := json.NewEncoder(w)

	if n <= 1 {
		var world World
		colQuery := bson.M{"id": getRandomNumber()}
		update := bson.M{"$set": bson.M{"randomNumber": getRandomNumber()}}
		if err := worlds.Update(colQuery, update); err != nil {
			log.Fatalf("Error updating world with id: %s", err.Error())
		} else {
			world.Id = colQuery["id"].(uint16)
			world.RandomNumber = update["$set"].(bson.M)["randomNumber"].(uint16)
		}
		encoder.Encode(world)
	} else {
		if n > 500 {
			n = 500
		}
		result := make([]World, n)
		for _, world := range result {
			colQuery := bson.M{"id": getRandomNumber()}
			update := bson.M{"$set": bson.M{"randomNumber": getRandomNumber()}}
			if err := worlds.Update(colQuery, update); err != nil {
				log.Fatalf("Error updating world with id: %s", err.Error())
			} else {
				world.Id = colQuery["id"].(uint16)
				world.RandomNumber = update["$set"].(bson.M)["randomNumber"].(uint16)
			}
		}
		encoder.Encode(&result)
	}
}

func plaintextHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte(helloWorldString))
}
<commit_msg>Fixed empty fields in response struct for update<commit_after>package main

import (
	"encoding/json"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"html/template"
	"log"
	"math/rand"
	"net/http"
	"runtime"
	"sort"
	"strconv"
)

const (
	connectionString = "localhost"
	helloWorldString = "Hello, world!"
	worldRowCount    = 10000
)

var (
	tmpl = template.Must(template.ParseFiles("templates/layout.html", "templates/fortune.html"))

	database *mgo.Database
	fortunes *mgo.Collection
	worlds   *mgo.Collection
)

type Message struct {
	Message string `json:"message"`
}

type World struct {
	Id           uint16 `bson:"id" json:"id"`
	RandomNumber uint16 `bson:"randomNumber" json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `bson:"id" json:"id"`
	Message string `bson:"message" json:"message"`
}

type Fortunes []Fortune

func (s Fortunes) Len() int {
	return len(s)
}

func (s Fortunes) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

type ByMessage struct{ Fortunes }

func (s ByMessage) Less(i, j int) bool {
	return s.Fortunes[i].Message < s.Fortunes[j].Message
}

func main() {
	port := ":8080"
	runtime.GOMAXPROCS(runtime.NumCPU())
	if session, err := mgo.Dial(connectionString); err != nil {
		log.Fatalf("Error opening database: %v", err)
	} else {
		defer session.Close()
		session.SetPoolLimit(5)
		database = session.DB("hello_world")
		worlds = database.C("world")
		fortunes = database.C("fortune")
		http.HandleFunc("/json", jsonHandler)
		http.HandleFunc("/db", dbHandler)
		http.HandleFunc("/fortune", fortuneHandler)
		http.HandleFunc("/queries", queriesHandler)
		http.HandleFunc("/update", updateHandler)
		http.HandleFunc("/plaintext", plaintextHandler)
		http.ListenAndServe(port, nil)
	}
}

// Helper for random numbers
func getRandomNumber() uint16 {
	return uint16(rand.Intn(worldRowCount) + 1)
}

// Test 1: JSON serialization
func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/javascript")
	json.NewEncoder(w).Encode(&Message{helloWorldString})
}

func dbHandler(w http.ResponseWriter, r *http.Request) {
	var world World
	query := bson.M{"id": getRandomNumber()}
	if worlds != nil {
		if err := worlds.Find(query).One(&world); err != nil {
			log.Fatalf("Error finding world with id: %s", err.Error())
			return
		} else {
			w.Header().Set("Content-Type", "application/json")
			json.NewEncoder(w).Encode(&world)
			return
		}
	} else {
		log.Fatal("Collection not initialized properly")
	}
}

func queriesHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) > 0 {
		n, _ = strconv.Atoi(nStr)
	}

	if n > 500 {
		n = 500
	}

	w.Header().Set("Content-Type", "application/json")
	encoder := json.NewEncoder(w)

	if n <= 1 {
		result := make([]World, 1)
		query := bson.M{"id": getRandomNumber()}
		if err := worlds.Find(query).One(&result[0]); err != nil {
			log.Fatalf("Error finding world with id: %s", err.Error())
			return
		}
		encoder.Encode(&result)
	} else {
		result := make([]World, n)
		for i := 0; i < n; i++ {
			query := bson.M{"id": getRandomNumber()}
			if err := worlds.Find(query).One(&result[i]); err != nil {
				log.Fatalf("Error finding world with id: %s", err.Error())
				return
			}
		}
		encoder.Encode(&result)
	}
}

func fortuneHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	f := make(Fortunes, 16)
	if err := fortunes.Find(nil).All(&f); err == nil {
		f = append(f, Fortune{
			Message: "Additional fortune added at request time.",
		})
		sort.Sort(ByMessage{f})
		if err := tmpl.Execute(w, f); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}

}

func updateHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) > 0 {
		n, _ = strconv.Atoi(nStr)
	}

	w.Header().Set("Content-Type", "application/json")
	encoder := json.NewEncoder(w)

	if n <= 1 {
		result := make([]World, 1)
		colQuery := bson.M{"id": getRandomNumber()}
		update := bson.M{"$set": bson.M{"randomNumber": getRandomNumber()}}
		if err := worlds.Update(colQuery, update); err != nil {
			log.Fatalf("Error updating world with id: %s", err.Error())
		} else {
			result[0].Id = colQuery["id"].(uint16)
			result[0].RandomNumber = update["$set"].(bson.M)["randomNumber"].(uint16)
		}
		encoder.Encode(&result)
	} else {
		if n > 500 {
			n = 500
		}
		result := make([]World, n)
		for i := 0; i < n; i++ {
			colQuery := bson.M{"id": getRandomNumber()}
			update := bson.M{"$set": bson.M{"randomNumber": getRandomNumber()}}
			if err := worlds.Update(colQuery, update); err != nil {
				log.Fatalf("Error updating world with id: %s", err.Error())
			} else {
				result[i].Id = colQuery["id"].(uint16)
				result[i].RandomNumber = update["$set"].(bson.M)["randomNumber"].(uint16)
			}
		}
		encoder.Encode(&result)
	}
}

func plaintextHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte(helloWorldString))
}
<|endoftext|>


<commit_before>package config

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"unicode"
)

// Parser is an parser interface.
type Parser interface {
	// Name returns the name of the parser to identify it.
	Name() string

	// If the parser needs some configurations, it can return all those names
	// by the method, and mark whether they must be required.
	//
	// For example, the method returns {"ip": true, "port": false},
	// which indicates the configuration must pass the parser the value of the
	// option 'ip' when calling the method Parse, but the value of 'port' is
	// optional. These option values will be acquired from the default group.
	GetKeys() map[string]bool

	// Parse the value of the registered options.
	//
	// The first argument, defaultGroupName, is the name of the default group.
	//
	// The second argument, opts, is the parsed option information. The key is
	// the group name, and the value is the parsed option list.
	//
	// The third argument, conf, is the configuration information. The key is
	// the key of the map value returned by the method GetKeys(), and the value
	// is pulled from the default group which has just been parsed.
	//
	// For example, for the redis parser, its the method GetKeys() maybe
	// returns {"connection": false}, then there may be a key-value pair,
	// {"connection": "redis://1.2.3.4:6379/1"} in the argument conf of the
	// method Parse, but there may be not. Ff so, the redis parser maybe use
	/// the default value, "redis://127.0.0.1:6379/0". For the builtin ini
	// parser, NewSimpleIniParser, it's "config-file" by default, but you can
	// change it when a new ini parser is created.
	//
	// For the first result, a map, the key is the group name, and the value is
	// the key-value pairs about the options defined in that group, which the
	// option key is the name of the registered option.
	//
	// If a certain option has no value, the parser should not return a default
	// one instead.
	Parse(defaultGroupName string, opts map[string][]Opt,
		conf map[string]interface{}) (results map[string]map[string]interface{},
		err error)
}

// CliParser is an interface to parse the CLI arguments.
type CliParser interface {
	// Name returns the name of the CLI parser to identify a CLI parser.
	Name() string

	// The argument is the CLI arguments, but it may be nil.
	//
	// The key of the result map is the group name, and the value of that is
	// the key-value pairs, which the key is the name of the registered option.
	//
	// If a certain option has no value, the CLI parser should not return a
	// default one.

	// Parse the value of the registered CLI options.
	//
	// The first argument, defaultGroupName, is the name of the default group.
	//
	// The second argument, opts, is the parsed option information. The key is
	// the group name, and the value is the parsed option list.
	//
	// The third argument, arguments, is the CLI arguments, which must be
	// a string slice, not nil, but it maybe have no elements. The parser
	// implementor should use os.Args[1:] when it's nil or empty, because
	// it has been confirmed.
	//
	// For the first result, a map, the key is the group name, and the value
	// is the key-value pairs about the options defined in that group, which
	// the option key is the name of the registered option. NOTICE: the parser
	// can parse the value of the option to a special type by calling the
	// mehtod Parse() of the option, but it's not necessary. Whether the parser
	// calls opt.Parse(value) or not, the configuration engine will call it.
	//
	// The second result is the rest of the CLI arguments, which are not the
	// options starting with the prefix "-", "--" or others, etc.
	//
	// If a certain option has no value, the parser should not return a default
	// one instead.
	Parse(defaultGroupName string, opts map[string][]Opt, arguments []string) (
		results map[string]map[string]interface{}, args []string, err error)
}

type flagParser struct {
	flagSet    *flag.FlagSet
	name       string
	errhandler flag.ErrorHandling
}

// NewDefaultFlagCliParser returns a new CLI parser based on flag.
//
// The parser will use flag.CommandLine to parse the CLI arguments.
func NewDefaultFlagCliParser() CliParser {
	return flagParser{
		flagSet: flag.CommandLine,
	}
}

// NewFlagCliParser returns a new CLI parser based on flag.FlagSet.
//
// The arguments is the same as that of flag.NewFlagSet(), but if the name is
// "", it will be filepath.Base(os.Args[0]).
//
// When other libraries use the default global flag.FlagSet, that's
// flag.CommandLine, such as github.com/golang/glog, please use
// NewDefaultFlagCliParser(), not this function.
func NewFlagCliParser(appName string, errhandler flag.ErrorHandling) CliParser {
	if appName == "" {
		appName = filepath.Base(os.Args[0])
	}
	return flagParser{
		name:       appName,
		errhandler: errhandler,
	}
}

func (f flagParser) Name() string {
	return "flag"
}

func (f flagParser) Parse(_default string, opts map[string][]Opt, as []string) (
	results map[string]map[string]interface{}, args []string, err error) {
	// Register the options into flag.FlagSet.
	flagSet := f.flagSet
	if flagSet == nil {
		flagSet = flag.NewFlagSet(f.name, f.errhandler)
	}

	name2group := make(map[string]string, 8)
	name2opt := make(map[string]string, 8)
	for group, _opts := range opts {
		for _, opt := range _opts {
			name := opt.Name()
			if group != _default {
				name = fmt.Sprintf("%s_%s", group, name)
			}
			name2group[name] = group
			name2opt[name] = opt.Name()

			if opt.IsBool() {
				var _default bool
				if v := opt.Default(); v != nil {
					_default = v.(bool)
				}
				flagSet.Bool(name, _default, opt.Help())
			} else {
				flagSet.String(name, "", opt.Help())
			}
		}
	}

	// Parse the CLI arguments.
	if err = flagSet.Parse(as); err != nil {
		return
	}

	// Acquire the result.
	args = flagSet.Args()
	results = make(map[string]map[string]interface{}, len(name2group))
	flagSet.Visit(func(fg *flag.Flag) {
		if group, ok := results[name2group[fg.Name]]; ok {
			group[name2opt[fg.Name]] = fg.Value.String()
		} else {
			results[name2group[fg.Name]] = map[string]interface{}{
				name2opt[fg.Name]: fg.Value.String(),
			}
		}
	})
	return
}

type iniParser struct {
	sep     string
	optName string
}

// NewSimpleIniParser returns a new ini parser based on the file.
//
// The argument is the option name which the parser needs. It should be
// registered, and parsed before this parser runs.
//
// The ini parser supports the line comments starting with "#" or "//".
// The key and the value is separated by an equal sign, that's =.
//
// Notice: the options that have not been assigned to a certain group will be
// divided into the default group.
func NewSimpleIniParser(optName string) Parser {
	return iniParser{optName: optName, sep: "="}
}

func (p iniParser) Name() string {
	return "ini"
}

func (p iniParser) GetKeys() map[string]bool {
	return map[string]bool{
		p.optName: false,
	}
}

func (p iniParser) Parse(_default string, opts map[string][]Opt,
	conf map[string]interface{}) (results map[string]map[string]interface{},
	err error) {
	// Read the content of the config file.
	filename, ok := conf[p.optName].(string)
	if !ok || filename == "" {
		return
	}

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	// Convert the format of the optons.
	options := make(map[string]map[string]struct{}, len(opts))
	for group, _opts := range opts {
		g, ok := options[group]
		if !ok {
			g = make(map[string]struct{}, len(_opts))
			options[group] = g
		}
		for _, opt := range _opts {
			g[opt.Name()] = struct{}{}
		}
	}

	// Parse the config file.
	group := make(map[string]interface{}, 8)
	results = make(map[string]map[string]interface{}, len(options))
	results[_default] = group
	lines := strings.Split(string(data), "\n")
	for _, line := range lines {
		line = strings.TrimSpace(line)

		// Ignore the empty line.
		if len(line) == 0 {
			continue
		}

		// Ignore the line comments starting with "#" or "//".
		if (line[0] == '#') ||
			(len(line) > 1 && line[0] == '/' && line[1] == '/') {
			continue
		}

		// Start a new group
		if line[0] == '[' && line[len(line)-1] == ']' {
			gname := strings.TrimSpace(line[1 : len(line)-1])
			if gname == "" {
				return nil, fmt.Errorf("the group is empty")
			}
			if group = results[gname]; group == nil {
				group = make(map[string]interface{}, 4)
				results[gname] = group
			}
			continue
		}

		n := strings.Index(line, p.sep)
		if n == -1 {
			err = fmt.Errorf("the line misses the separator %s", p.sep)
			return
		}

		key := strings.TrimSpace(line[0:n])
		for _, r := range key {
			if !unicode.IsNumber(r) && !unicode.IsLetter(r) {
				err = fmt.Errorf("the key is not an valid identifier")
				return
			}
		}
		group[key] = strings.TrimSpace(line[n+len(p.sep) : len(line)])
	}
	return
}
<commit_msg>Add the default for CLI parser.<commit_after>package config

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"unicode"
)

// Parser is an parser interface.
type Parser interface {
	// Name returns the name of the parser to identify it.
	Name() string

	// If the parser needs some configurations, it can return all those names
	// by the method, and mark whether they must be required.
	//
	// For example, the method returns {"ip": true, "port": false},
	// which indicates the configuration must pass the parser the value of the
	// option 'ip' when calling the method Parse, but the value of 'port' is
	// optional. These option values will be acquired from the default group.
	GetKeys() map[string]bool

	// Parse the value of the registered options.
	//
	// The first argument, defaultGroupName, is the name of the default group.
	//
	// The second argument, opts, is the parsed option information. The key is
	// the group name, and the value is the parsed option list.
	//
	// The third argument, conf, is the configuration information. The key is
	// the key of the map value returned by the method GetKeys(), and the value
	// is pulled from the default group which has just been parsed.
	//
	// For example, for the redis parser, its the method GetKeys() maybe
	// returns {"connection": false}, then there may be a key-value pair,
	// {"connection": "redis://1.2.3.4:6379/1"} in the argument conf of the
	// method Parse, but there may be not. Ff so, the redis parser maybe use
	/// the default value, "redis://127.0.0.1:6379/0". For the builtin ini
	// parser, NewSimpleIniParser, it's "config-file" by default, but you can
	// change it when a new ini parser is created.
	//
	// For the first result, a map, the key is the group name, and the value is
	// the key-value pairs about the options defined in that group, which the
	// option key is the name of the registered option.
	//
	// If a certain option has no value, the parser should not return a default
	// one instead.
	Parse(defaultGroupName string, opts map[string][]Opt,
		conf map[string]interface{}) (results map[string]map[string]interface{},
		err error)
}

// CliParser is an interface to parse the CLI arguments.
type CliParser interface {
	// Name returns the name of the CLI parser to identify a CLI parser.
	Name() string

	// The argument is the CLI arguments, but it may be nil.
	//
	// The key of the result map is the group name, and the value of that is
	// the key-value pairs, which the key is the name of the registered option.
	//
	// If a certain option has no value, the CLI parser should not return a
	// default one.

	// Parse the value of the registered CLI options.
	//
	// The first argument, defaultGroupName, is the name of the default group.
	//
	// The second argument, opts, is the parsed option information. The key is
	// the group name, and the value is the parsed option list.
	//
	// The third argument, arguments, is the CLI arguments, which must be
	// a string slice, not nil, but it maybe have no elements. The parser
	// implementor should use os.Args[1:] when it's nil or empty, because
	// it has been confirmed.
	//
	// For the first result, a map, the key is the group name, and the value
	// is the key-value pairs about the options defined in that group, which
	// the option key is the name of the registered option. NOTICE: the parser
	// can parse the value of the option to a special type by calling the
	// mehtod Parse() of the option, but it's not necessary. Whether the parser
	// calls opt.Parse(value) or not, the configuration engine will call it.
	//
	// The second result is the rest of the CLI arguments, which are not the
	// options starting with the prefix "-", "--" or others, etc.
	//
	// If a certain option has no value, the parser should not return a default
	// one instead.
	Parse(defaultGroupName string, opts map[string][]Opt, arguments []string) (
		results map[string]map[string]interface{}, args []string, err error)
}

type flagParser struct {
	flagSet    *flag.FlagSet
	name       string
	errhandler flag.ErrorHandling
}

// NewDefaultFlagCliParser returns a new CLI parser based on flag.
//
// The parser will use flag.CommandLine to parse the CLI arguments.
func NewDefaultFlagCliParser() CliParser {
	return flagParser{
		flagSet: flag.CommandLine,
	}
}

// NewFlagCliParser returns a new CLI parser based on flag.FlagSet.
//
// The arguments is the same as that of flag.NewFlagSet(), but if the name is
// "", it will be filepath.Base(os.Args[0]).
//
// When other libraries use the default global flag.FlagSet, that's
// flag.CommandLine, such as github.com/golang/glog, please use
// NewDefaultFlagCliParser(), not this function.
func NewFlagCliParser(appName string, errhandler flag.ErrorHandling) CliParser {
	if appName == "" {
		appName = filepath.Base(os.Args[0])
	}
	return flagParser{
		name:       appName,
		errhandler: errhandler,
	}
}

func (f flagParser) Name() string {
	return "flag"
}

func (f flagParser) Parse(_default string, opts map[string][]Opt, as []string) (
	results map[string]map[string]interface{}, args []string, err error) {
	// Register the options into flag.FlagSet.
	flagSet := f.flagSet
	if flagSet == nil {
		flagSet = flag.NewFlagSet(f.name, f.errhandler)
	}

	name2group := make(map[string]string, 8)
	name2opt := make(map[string]string, 8)
	for group, _opts := range opts {
		for _, opt := range _opts {
			name := opt.Name()
			if group != _default {
				name = fmt.Sprintf("%s_%s", group, name)
			}
			name2group[name] = group
			name2opt[name] = opt.Name()

			if opt.IsBool() {
				var _default bool
				if v := opt.Default(); v != nil {
					_default = v.(bool)
				}
				flagSet.Bool(name, _default, opt.Help())
			} else {
				_default := ""
				if opt.Default() != nil {
					_default = fmt.Sprintf("%v", opt.Default())
				}
				flagSet.String(name, _default, opt.Help())
			}
		}
	}

	// Parse the CLI arguments.
	if err = flagSet.Parse(as); err != nil {
		return
	}

	// Acquire the result.
	args = flagSet.Args()
	results = make(map[string]map[string]interface{}, len(name2group))
	flagSet.Visit(func(fg *flag.Flag) {
		if group, ok := results[name2group[fg.Name]]; ok {
			group[name2opt[fg.Name]] = fg.Value.String()
		} else {
			results[name2group[fg.Name]] = map[string]interface{}{
				name2opt[fg.Name]: fg.Value.String(),
			}
		}
	})
	return
}

type iniParser struct {
	sep     string
	optName string
}

// NewSimpleIniParser returns a new ini parser based on the file.
//
// The argument is the option name which the parser needs. It should be
// registered, and parsed before this parser runs.
//
// The ini parser supports the line comments starting with "#" or "//".
// The key and the value is separated by an equal sign, that's =.
//
// Notice: the options that have not been assigned to a certain group will be
// divided into the default group.
func NewSimpleIniParser(optName string) Parser {
	return iniParser{optName: optName, sep: "="}
}

func (p iniParser) Name() string {
	return "ini"
}

func (p iniParser) GetKeys() map[string]bool {
	return map[string]bool{
		p.optName: false,
	}
}

func (p iniParser) Parse(_default string, opts map[string][]Opt,
	conf map[string]interface{}) (results map[string]map[string]interface{},
	err error) {
	// Read the content of the config file.
	filename, ok := conf[p.optName].(string)
	if !ok || filename == "" {
		return
	}

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	// Convert the format of the optons.
	options := make(map[string]map[string]struct{}, len(opts))
	for group, _opts := range opts {
		g, ok := options[group]
		if !ok {
			g = make(map[string]struct{}, len(_opts))
			options[group] = g
		}
		for _, opt := range _opts {
			g[opt.Name()] = struct{}{}
		}
	}

	// Parse the config file.
	group := make(map[string]interface{}, 8)
	results = make(map[string]map[string]interface{}, len(options))
	results[_default] = group
	lines := strings.Split(string(data), "\n")
	for _, line := range lines {
		line = strings.TrimSpace(line)

		// Ignore the empty line.
		if len(line) == 0 {
			continue
		}

		// Ignore the line comments starting with "#" or "//".
		if (line[0] == '#') ||
			(len(line) > 1 && line[0] == '/' && line[1] == '/') {
			continue
		}

		// Start a new group
		if line[0] == '[' && line[len(line)-1] == ']' {
			gname := strings.TrimSpace(line[1 : len(line)-1])
			if gname == "" {
				return nil, fmt.Errorf("the group is empty")
			}
			if group = results[gname]; group == nil {
				group = make(map[string]interface{}, 4)
				results[gname] = group
			}
			continue
		}

		n := strings.Index(line, p.sep)
		if n == -1 {
			err = fmt.Errorf("the line misses the separator %s", p.sep)
			return
		}

		key := strings.TrimSpace(line[0:n])
		for _, r := range key {
			if !unicode.IsNumber(r) && !unicode.IsLetter(r) {
				err = fmt.Errorf("the key is not an valid identifier")
				return
			}
		}
		group[key] = strings.TrimSpace(line[n+len(p.sep) : len(line)])
	}
	return
}
<|endoftext|>


<commit_before>package pot

//-----------------------------------------------------------------------------

// Parser interface implemented by Dict, DictKey, List and String parsers.
type Parser interface {
	// Get the next parser, nil or an error.
	Next() (Parser, error)

	// Get text the parser was initialized with.
	Bytes() []byte

	// Get parser start location in the original text input.
	// The location is reset when using NewParser, NewDictParser or NewListParser.
	Location() Location
}

//-----------------------------------------------------------------------------

// Root level parser capable of parsing multiple root level objects from the
// same text input.
type Root struct {
	org parserBuf  // Text the parser was initialized with.
	buf *parserBuf // Text buffer the parser operates on.
}

// Create a new root level parser parsing the supplied text.
func NewParser(pot []byte) Parser {
	buf := newParserBuf(pot)
	return &Root{*buf, buf}
}

// Get the next parser or nil on end of input or an error.
// The returned parser may be a Dict, List or String.
func (root *Root) Next() (Parser, error) {
	return scanValue(root.buf)
}

// Get text the parser was initialized with.
func (root *Root) Bytes() []byte {
	return root.org.bytes
}

// Get parser start location in the original text input.
func (root *Root) Location() Location {
	return root.org.location
}

//-----------------------------------------------------------------------------

// Dictionary parser.
type Dict struct {
	org   parserBuf  // Text the parser was initialized with.
	buf   *parserBuf // Text buffer the parser operates on.
	count int        // Number of returned parsers.
}

// Create a new dictionary parser parsing the supplied text.
func NewDictParser(pot []byte) *Dict {
	return newDictParser(newParserBuf(pot))
}

// Create a new dictionary parser parsing the supplied parser buffer.
func newDictParser(buf *parserBuf) *Dict {
	dict := &Dict{*buf, buf, 0}
	buf.stripBlock('{', '}')
	// Trim space to make IsEmpty() work out of the gate.
	buf.trimSpaceLeft()
	return dict
}

// Get the next parser or nil on end of input or an error.
// Every even call returns a key which is of type DictKey.
// Every odd call returns a value which may be a Dict, List or String.
func (dict *Dict) Next() (parser Parser, err error) {
	if dict.count%2 == 0 {
		parser, err = scanKey(dict.buf)
	} else {
		parser, err = scanValue(dict.buf)
		if parser == nil && err == nil {
			err = dict.buf.errorf("key without value in dictionary")
		}
	}
	if parser != nil {
		dict.count++
	}
	return
}

// Check if the parser has consumed all data.
func (dict *Dict) IsEmpty() bool {
	return len(dict.buf.bytes) == 0
}

// Get text the parser was initialized with.
func (dict *Dict) Bytes() []byte {
	return dict.org.bytes
}

// Get parser start location in the original text input.
func (dict *Dict) Location() Location {
	return dict.org.location
}

//-----------------------------------------------------------------------------

// Dictionary key type.
// A unique "string" type is used to be able to separate keys from string values
// in type switches.
type DictKey String

// Returns nil as keys does not contain sub parsers.
func (key *DictKey) Next() (Parser, error) {
	return nil, nil
}

// Get text the parser was initialized with.
func (key *DictKey) Bytes() []byte {
	return (*String)(key).Bytes()
}

// Format as a POT dictionary key.
func (key *DictKey) String() string {
	return string(key.Bytes()) + ":"
}

// Get parser start location in the original text input.
func (key *DictKey) Location() Location {
	return (*String)(key).Location()
}

//-----------------------------------------------------------------------------

// List parser.
type List struct {
	org parserBuf  // Text the parser was initialized with.
	buf *parserBuf // Text buffer the parser operates on.
}

// Create a new list parser parsing the supplied text.
func NewListParser(pot []byte) *List {
	return newListParser(newParserBuf(pot))
}

// Create a new list parser parsing the supplied parser buffer.
func newListParser(buf *parserBuf) *List {
	list := &List{*buf, buf}
	buf.stripBlock('[', ']')
	return list
}

// Get the next parser or nil on end of input or an error.
// The returned parser may be a Dict, List or String.
func (list *List) Next() (Parser, error) {
	return scanValue(list.buf)
}

// Get text the parser was initialized with.
func (list *List) Bytes() []byte {
	return list.org.bytes
}

// Get parser start location in the original text input.
func (list *List) Location() Location {
	return list.org.location
}

//-----------------------------------------------------------------------------

// String parser.
type String parserBuf

// Returns nil as strings does not contain sub parsers.
func (str *String) Next() (Parser, error) {
	return nil, nil
}

var charToEscapeCode = map[byte]byte{
	'\n': 'n',
	'\r': 'r',
	'\t': 't',
	'\\': '\\',
	'"':  '"',
}

// Format as a POT string value.
func (str *String) String() string {
	quote := false
	newBuf := false

	t := str.bytes
	for i, c := range str.bytes {
		switch c {
		case '{', '}', '[', ']', ':', ' ':
			quote = true
		case '\n', '\r', '\t', '\\', '"':
			if !newBuf {
				t = make([]byte, 0, len(str.bytes))
				t = append(t, str.bytes[:i]...)
				newBuf = true
			}
			t = append(t, '\\', charToEscapeCode[c])
		default:
			if newBuf {
				t = append(t, c)
			}
		}
	}
	if quote || len(t) == 0 {
		return "\"" + string(t) + "\""
	}
	return string(t)
}

// Get text the parser was initialized with.
func (str *String) Bytes() []byte {
	return str.bytes
}

// Get parser start location in the original text input.
func (str *String) Location() Location {
	return str.location
}

//-----------------------------------------------------------------------------

// Scans a DictKey from a text buffer.
// Returns a parser or nil when there is no more input or an error.
func scanKey(buf *parserBuf) (Parser, error) {
	buf.trimSpaceLeft()
	if len(buf.bytes) == 0 {
		return nil, nil
	}
	for i, c := range buf.bytes {
		switch {
		case validKeyChar(i, c):
		case c == ':' && i > 0:
			t := buf.split(i)
			buf.trimBytesLeft(1) // Eat ':'
			return (*DictKey)(t), nil
		default:
			buf.trimBytesLeft(i)
			return nil, buf.errorf("invalid character '%c' in key", c)
		}
	}
	buf.trimAll()
	return nil, buf.errorf("end of input while parsing key")
}

// Scans a Dict, List or String from a text buffer.
// Returns a parser or nil when there is no more input or an error.
func scanValue(buf *parserBuf) (parser Parser, err error) {
	buf.trimSpaceLeft()
	if len(buf.bytes) > 0 {
		switch buf.bytes[0] {
		case '{':
			if buf, err = scanBlock(buf, '{', '}'); err == nil {
				parser = newDictParser(buf)
			}
		case '[':
			if buf, err = scanBlock(buf, '[', ']'); err == nil {
				parser = newListParser(buf)
			}
		default:
			parser, err = scanString(buf)
		}
	}
	return
}

// Scans a Dict or List block from a text buffer.
// Returns a new text buffer containing the block or an error.
func scanBlock(buf *parserBuf, begChar, endChar byte) (*parserBuf, error) {
	quoted := false
	escaped := false
	scope := 0
	for i, c := range buf.bytes {
		switch c {
		case '\\':
			escaped = !escaped
		case '"':
			if !escaped {
				quoted = !quoted
			}
			escaped = false
		case begChar:
			if !quoted && !escaped {
				scope++
			}
			escaped = false
		case endChar:
			if !quoted && !escaped {
				scope--
				if scope == 0 {
					return buf.split(i + 1), nil
				}
			}
			escaped = false
		default:
			escaped = false
		}
	}
	buf.trimAll()
	return nil, buf.errorf("end of input while parsing '%c%c' block", begChar, endChar)
}

// Scans a String from a text buffer.
// Returns a parser or an error.
func scanString(buf *parserBuf) (Parser, error) {
	quoted := false
	escaped := false
	eval := false

	var i int
	var c byte
loop:
	for i, c = range buf.bytes {
		switch c {
		case '\\':
			escaped = !escaped
			eval = true
		case '"':
			if !escaped {
				quoted = !quoted
			}
			escaped = false
			eval = true
		case '{', '}', '[', ']', ' ', '\n', '\r', '\t':
			if !quoted && !escaped {
				if i == 0 {
					buf.trimBytesLeft(i)
					return nil, buf.errorf("invalid character '%c' in string", c)
				} else {
					i--
					break loop
				}
			}
			escaped = false
		case ':':
			if !quoted && !escaped {
				buf.trimBytesLeft(i)
				return nil, buf.errorf("invalid character '%c' in string", c)
			}
			escaped = false
		}
	}

	str := buf.split(i + 1)
	if eval {
		return evalStringBuffer(str)
	}
	return (*String)(str), nil
}

var escapeCodeToChar = map[byte]byte{
	'n': '\n',
	'r': '\r',
	't': '\t',
}

// Evaluate escape codes and quotes in string.
// Returns the modified parser buffer as a string parser or an error.
func evalStringBuffer(buf *parserBuf) (Parser, error) {
	quoted := false
	escaped := false

	tr := make([]byte, 0, len(buf.bytes))
	for i, c := range buf.bytes {
		switch c {
		case '\\':
			if escaped {
				tr = append(tr, c)
			}
			escaped = !escaped
		case '"':
			if escaped {
				tr = append(tr, c)
			} else {
				quoted = !quoted
			}
			escaped = false
		case 'n', 'r', 't':
			if escaped {
				tr = append(tr, escapeCodeToChar[c])
			} else {
				tr = append(tr, c)
			}
			escaped = false
		case '{', '}', '[', ']', ':', ' ':
			tr = append(tr, c)
			escaped = false
		default:
			if escaped {
				buf.trimBytesLeft(i)
				return nil, buf.errorf("invalid escape code \\%c", c)
			} else {
				tr = append(tr, c)
			}
		}
	}

	if quoted {
		buf.trimAll()
		return nil, buf.errorf("miss-matched quotes in string")
	}
	if escaped {
		buf.trimAll()
		return nil, buf.errorf("unterminated escape code in string")
	}

	buf.bytes = tr
	return (*String)(buf), nil
}

// Check if 'c' is a valid key character at index 'i'.
func validKeyChar(i int, c byte) bool {
	if c >= 'a' && c < 'z' || c >= 'A' && c < 'Z' || c >= '0' && c < '9' {
		return true
	}
	if i > 0 && c == '-' {
		return true
	}
	return false
}

//-----------------------------------------------------------------------------
<commit_msg>Add Name method to parser interface<commit_after>package pot

// Parser interface implemented by Dict, DictKey, List and String parsers.
type Parser interface {
	// Parser name.
	Name() string

	// Get the next parser, nil or an error.
	Next() (Parser, error)

	// Get text the parser was initialized with.
	Bytes() []byte

	// Get parser start location in the original text input.
	// The location is reset when using NewParser, NewDictParser or NewListParser.
	Location() Location
}

// Root level parser capable of parsing multiple root level objects from the
// same text input.
type Root struct {
	org parserBuf  // Text the parser was initialized with.
	buf *parserBuf // Text buffer the parser operates on.
}

// Create a new root level parser parsing the supplied text.
func NewParser(pot []byte) Parser {
	buf := newParserBuf(pot)
	return &Root{*buf, buf}
}

func (root *Root) Name() string {
	return "root"
}

// Get the next parser or nil on end of input or an error.
// The returned parser may be a Dict, List or String.
func (root *Root) Next() (Parser, error) {
	return scanValue(root.buf)
}

// Get text the parser was initialized with.
func (root *Root) Bytes() []byte {
	return root.org.bytes
}

// Get parser start location in the original text input.
func (root *Root) Location() Location {
	return root.org.location
}

// Dictionary parser.
type Dict struct {
	org   parserBuf  // Text the parser was initialized with.
	buf   *parserBuf // Text buffer the parser operates on.
	count int        // Number of returned parsers.
}

// Create a new dictionary parser parsing the supplied text.
func NewDictParser(pot []byte) *Dict {
	return newDictParser(newParserBuf(pot))
}

// Create a new dictionary parser parsing the supplied parser buffer.
func newDictParser(buf *parserBuf) *Dict {
	dict := &Dict{*buf, buf, 0}
	buf.stripBlock('{', '}')
	// Trim space to make IsEmpty() work out of the gate.
	buf.trimSpaceLeft()
	return dict
}

func (dict *Dict) Name() string {
	return "dictionary"
}

// Get the next parser or nil on end of input or an error.
// Every even call returns a key which is of type DictKey.
// Every odd call returns a value which may be a Dict, List or String.
func (dict *Dict) Next() (parser Parser, err error) {
	if dict.count%2 == 0 {
		parser, err = scanKey(dict.buf)
	} else {
		parser, err = scanValue(dict.buf)
		if parser == nil && err == nil {
			err = dict.buf.errorf("key without value in dictionary")
		}
	}
	if parser != nil {
		dict.count++
	}
	return
}

// Check if the parser has consumed all data.
func (dict *Dict) IsEmpty() bool {
	return len(dict.buf.bytes) == 0
}

// Get text the parser was initialized with.
func (dict *Dict) Bytes() []byte {
	return dict.org.bytes
}

// Get parser start location in the original text input.
func (dict *Dict) Location() Location {
	return dict.org.location
}

// Dictionary key type.
// A unique "string" type is used to be able to separate keys from string values
// in type switches.
type DictKey String

func (key *DictKey) Name() string {
	return "dictionary-key"
}

// Returns nil as keys does not contain sub parsers.
func (key *DictKey) Next() (Parser, error) {
	return nil, nil
}

// Get text the parser was initialized with.
func (key *DictKey) Bytes() []byte {
	return (*String)(key).Bytes()
}

// Format as a POT dictionary key.
func (key *DictKey) String() string {
	return string(key.Bytes()) + ":"
}

// Get parser start location in the original text input.
func (key *DictKey) Location() Location {
	return (*String)(key).Location()
}

// List parser.
type List struct {
	org parserBuf  // Text the parser was initialized with.
	buf *parserBuf // Text buffer the parser operates on.
}

// Create a new list parser parsing the supplied text.
func NewListParser(pot []byte) *List {
	return newListParser(newParserBuf(pot))
}

// Create a new list parser parsing the supplied parser buffer.
func newListParser(buf *parserBuf) *List {
	list := &List{*buf, buf}
	buf.stripBlock('[', ']')
	return list
}

func (list *List) Name() string {
	return "list"
}

// Get the next parser or nil on end of input or an error.
// The returned parser may be a Dict, List or String.
func (list *List) Next() (Parser, error) {
	return scanValue(list.buf)
}

// Get text the parser was initialized with.
func (list *List) Bytes() []byte {
	return list.org.bytes
}

// Get parser start location in the original text input.
func (list *List) Location() Location {
	return list.org.location
}

// String parser.
type String parserBuf

func (str *String) Name() string {
	return "string"
}

// Returns nil as strings does not contain sub parsers.
func (str *String) Next() (Parser, error) {
	return nil, nil
}

var charToEscapeCode = map[byte]byte{
	'\n': 'n',
	'\r': 'r',
	'\t': 't',
	'\\': '\\',
	'"':  '"',
}

// Format as a POT string value.
func (str *String) String() string {
	quote := false
	newBuf := false

	t := str.bytes
	for i, c := range str.bytes {
		switch c {
		case '{', '}', '[', ']', ':', ' ':
			quote = true
		case '\n', '\r', '\t', '\\', '"':
			if !newBuf {
				t = make([]byte, 0, len(str.bytes))
				t = append(t, str.bytes[:i]...)
				newBuf = true
			}
			t = append(t, '\\', charToEscapeCode[c])
		default:
			if newBuf {
				t = append(t, c)
			}
		}
	}
	if quote || len(t) == 0 {
		return "\"" + string(t) + "\""
	}
	return string(t)
}

// Get text the parser was initialized with.
func (str *String) Bytes() []byte {
	return str.bytes
}

// Get parser start location in the original text input.
func (str *String) Location() Location {
	return str.location
}

// Scans a DictKey from a text buffer.
// Returns a parser or nil when there is no more input or an error.
func scanKey(buf *parserBuf) (Parser, error) {
	buf.trimSpaceLeft()
	if len(buf.bytes) == 0 {
		return nil, nil
	}
	for i, c := range buf.bytes {
		switch {
		case validKeyChar(i, c):
		case c == ':' && i > 0:
			t := buf.split(i)
			buf.trimBytesLeft(1) // Eat ':'
			return (*DictKey)(t), nil
		default:
			buf.trimBytesLeft(i)
			return nil, buf.errorf("invalid character '%c' in key", c)
		}
	}
	buf.trimAll()
	return nil, buf.errorf("end of input while parsing key")
}

// Scans a Dict, List or String from a text buffer.
// Returns a parser or nil when there is no more input or an error.
func scanValue(buf *parserBuf) (parser Parser, err error) {
	buf.trimSpaceLeft()
	if len(buf.bytes) > 0 {
		switch buf.bytes[0] {
		case '{':
			if buf, err = scanBlock(buf, '{', '}'); err == nil {
				parser = newDictParser(buf)
			}
		case '[':
			if buf, err = scanBlock(buf, '[', ']'); err == nil {
				parser = newListParser(buf)
			}
		default:
			parser, err = scanString(buf)
		}
	}
	return
}

// Scans a Dict or List block from a text buffer.
// Returns a new text buffer containing the block or an error.
func scanBlock(buf *parserBuf, begChar, endChar byte) (*parserBuf, error) {
	quoted := false
	escaped := false
	scope := 0
	for i, c := range buf.bytes {
		switch c {
		case '\\':
			escaped = !escaped
		case '"':
			if !escaped {
				quoted = !quoted
			}
			escaped = false
		case begChar:
			if !quoted && !escaped {
				scope++
			}
			escaped = false
		case endChar:
			if !quoted && !escaped {
				scope--
				if scope == 0 {
					return buf.split(i + 1), nil
				}
			}
			escaped = false
		default:
			escaped = false
		}
	}
	buf.trimAll()
	return nil, buf.errorf("end of input while parsing '%c%c' block", begChar, endChar)
}

// Scans a String from a text buffer.
// Returns a parser or an error.
func scanString(buf *parserBuf) (Parser, error) {
	quoted := false
	escaped := false
	eval := false

	var i int
	var c byte
loop:
	for i, c = range buf.bytes {
		switch c {
		case '\\':
			escaped = !escaped
			eval = true
		case '"':
			if !escaped {
				quoted = !quoted
			}
			escaped = false
			eval = true
		case '{', '}', '[', ']', ' ', '\n', '\r', '\t':
			if !quoted && !escaped {
				if i == 0 {
					buf.trimBytesLeft(i)
					return nil, buf.errorf("invalid character '%c' in string", c)
				} else {
					i--
					break loop
				}
			}
			escaped = false
		case ':':
			if !quoted && !escaped {
				buf.trimBytesLeft(i)
				return nil, buf.errorf("invalid character '%c' in string", c)
			}
			escaped = false
		}
	}

	str := buf.split(i + 1)
	if eval {
		return evalStringBuffer(str)
	}
	return (*String)(str), nil
}

var escapeCodeToChar = map[byte]byte{
	'n': '\n',
	'r': '\r',
	't': '\t',
}

// Evaluate escape codes and quotes in string.
// Returns the modified parser buffer as a string parser or an error.
func evalStringBuffer(buf *parserBuf) (Parser, error) {
	quoted := false
	escaped := false

	tr := make([]byte, 0, len(buf.bytes))
	for i, c := range buf.bytes {
		switch c {
		case '\\':
			if escaped {
				tr = append(tr, c)
			}
			escaped = !escaped
		case '"':
			if escaped {
				tr = append(tr, c)
			} else {
				quoted = !quoted
			}
			escaped = false
		case 'n', 'r', 't':
			if escaped {
				tr = append(tr, escapeCodeToChar[c])
			} else {
				tr = append(tr, c)
			}
			escaped = false
		case '{', '}', '[', ']', ':', ' ':
			tr = append(tr, c)
			escaped = false
		default:
			if escaped {
				buf.trimBytesLeft(i)
				return nil, buf.errorf("invalid escape code \\%c", c)
			} else {
				tr = append(tr, c)
			}
		}
	}

	if quoted {
		buf.trimAll()
		return nil, buf.errorf("miss-matched quotes in string")
	}
	if escaped {
		buf.trimAll()
		return nil, buf.errorf("unterminated escape code in string")
	}

	buf.bytes = tr
	return (*String)(buf), nil
}

// Check if 'c' is a valid key character at index 'i'.
func validKeyChar(i int, c byte) bool {
	if c >= 'a' && c < 'z' || c >= 'A' && c < 'Z' || c >= '0' && c < '9' {
		return true
	}
	if i > 0 && c == '-' {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	text  string
	lex   *lexer
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek().typ; p {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemText, itemStrong, itemItalic, itemStrike, itemCode,
			itemLink, itemAutoLink, itemGfmLink, itemImage:
			n = t.parseParagraph()
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			// 0 for the depth
			n = t.parseList(0)
		default:
			fmt.Println("Nothing to do", p)
		}
		t.append(n)
	}
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseParagraph scan until itemBr occur.
func (t *Tree) parseParagraph() *ParagraphNode {
	token := t.next()
	p := t.newParagraph(token.pos)
Loop:
	for {
		var node Node
		switch token.typ {
		case eof, itemError, itemHeading, itemList:
			t.backup()
			break Loop
		case itemNewLine:
			// Two or more lines continuosly, or block below
			if typ := t.peek().typ; typ == itemNewLine || isBlock(typ) {
				t.backup2(token)
				break Loop
			}
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemText:
			node = t.newText(token.pos, token.val)
		case itemStrong, itemItalic, itemStrike, itemCode:
			match := span[token.typ].FindStringSubmatch(token.val)
			text := match[len(match)-1]
			if text == "" {
				text = match[1]
			}
			node = t.newEmphasis(token.pos, token.typ, text)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		}
		p.append(node)
		token = t.next()
	}
	return p
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// itemLHeading will always be in level 1.
		node = t.newHeading(token.pos, 1, match[1])
	}
	return
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		lang, text = match[1], match[2]
		if text == "" {
			text = match[4]
		}
	} else {
		text = regexp.MustCompile("(?m)( {4}|\t)").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

// parse list
func (t *Tree) parseList(depth int) *ListNode {
	token := t.next()
	list := t.newList(token.pos, depth, isDigit(token.val))
	item := new(ListItemNode)
Loop:
	for {
		switch token = t.next(); token.typ {
		case eof, itemError:
			break Loop
		// It's actually a listItem
		case itemList:
			// List, but not the same type
			if list.Ordered != isDigit(token.val) || depth > 0 {
				t.backup()
				break Loop
			}
			item = t.parseListItem(token.pos, list)
		case itemNewLine:
			if t.peek().typ == itemNewLine {
				break Loop
			}
			fallthrough
		case itemIndent:
			if depth == len(token.val) {
				item = t.parseListItem(token.pos, list)
			}
			t.backup()
			break Loop
		default:
			t.backup()
			item = t.parseListItem(token.pos, list)
		}
		list.append(item)
	}
	return list
}

func (t *Tree) parseListItem(pos Pos, list *ListNode) *ListItemNode {
	item := t.newListItem(pos, list)
	var n Node
Loop:
	for {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemList:
			t.backup()
			break Loop
		case itemNewLine:
			switch typ := t.peek().typ; typ {
			case itemNewLine, eof, itemError, itemList, itemIndent:
				continue
			default:
				n = t.newLine(token.pos)
			}
		case itemIndent:
			if t.peek().typ == itemList {
				depth := len(token.val)
				// If it's in the same depth - sibling
				// or if it's less-than - exit
				if depth <= item.List.Depth {
					t.backup2(token)
					break Loop
				}
				n = t.parseList(depth)
			} else {
				n = t.newText(token.pos, token.val)
			}
		default:
			n = t.newText(token.pos, token.val)
		}
		item.append(n)
	}
	return item
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) bool {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		return true
	default:
		return false
	}
}
<commit_msg>feat(parser): use parseParagraph to parse line-text<commit_after>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	text  string
	lex   *lexer
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek().typ; p {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemText, itemStrong, itemItalic, itemStrike, itemCode,
			itemLink, itemAutoLink, itemGfmLink, itemImage:
			n = t.parseParagraph()
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			// 0 for the depth
			n = t.parseList(0)
		default:
			fmt.Println("Nothing to do", p)
		}
		t.append(n)
	}
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseParagraph scan until itemBr occur.
func (t *Tree) parseParagraph() *ParagraphNode {
	token := t.next()
	p := t.newParagraph(token.pos)
Loop:
	for {
		var node Node
		switch token.typ {
		case eof, itemError, itemHeading, itemList, itemIndent:
			t.backup()
			break Loop
		case itemNewLine:
			// Two or more lines continuosly, or block below
			if typ := t.peek().typ; typ == itemNewLine || isBlock(typ) {
				t.backup2(token)
				break Loop
			}
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemText:
			node = t.newText(token.pos, token.val)
		case itemStrong, itemItalic, itemStrike, itemCode:
			match := span[token.typ].FindStringSubmatch(token.val)
			text := match[len(match)-1]
			if text == "" {
				text = match[1]
			}
			node = t.newEmphasis(token.pos, token.typ, text)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		}
		p.append(node)
		token = t.next()
	}
	return p
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// itemLHeading will always be in level 1.
		node = t.newHeading(token.pos, 1, match[1])
	}
	return
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		lang, text = match[1], match[2]
		if text == "" {
			text = match[4]
		}
	} else {
		text = regexp.MustCompile("(?m)( {4}|\t)").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

// parse list
func (t *Tree) parseList(depth int) *ListNode {
	token := t.next()
	list := t.newList(token.pos, depth, isDigit(token.val))
	item := new(ListItemNode)
Loop:
	for {
		switch token = t.next(); token.typ {
		case eof, itemError:
			break Loop
		// It's actually a listItem
		case itemList:
			// List, but not the same type
			if list.Ordered != isDigit(token.val) || depth > 0 {
				t.backup()
				break Loop
			}
			item = t.parseListItem(token.pos, list)
		case itemNewLine:
			if t.peek().typ == itemNewLine {
				break Loop
			}
			fallthrough
		case itemIndent:
			if depth == len(token.val) {
				item = t.parseListItem(token.pos, list)
			}
			t.backup()
			break Loop
		default:
			t.backup()
			item = t.parseListItem(token.pos, list)
		}
		list.append(item)
	}
	return list
}

func (t *Tree) parseListItem(pos Pos, list *ListNode) *ListItemNode {
	item := t.newListItem(pos, list)
	var n Node
Loop:
	for {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemList:
			t.backup()
			break Loop
		case itemNewLine:
			switch typ := t.peek().typ; typ {
			case itemNewLine, eof, itemError, itemList, itemIndent:
				continue
			default:
				n = t.newLine(token.pos)
			}
		case itemIndent:
			if t.peek().typ == itemList {
				depth := len(token.val)
				// If it's in the same depth - sibling
				// or if it's less-than - exit
				if depth <= item.List.Depth {
					t.backup2(token)
					break Loop
				}
				n = t.parseList(depth)
			} else {
				n = t.newText(token.pos, token.val)
			}
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		default:
			// DRY
			for _, n := range t.parseParagraph().Nodes {
				if n.Type() != NodeNewLine {
					item.append(n)
				}
			}
			continue
		}
		item.append(n)
	}
	return item
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) bool {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		return true
	default:
		return false
	}
}
<|endoftext|>