<commit_before>package message

import (
	"DNA/common"
	"DNA/common/log"
	"DNA/common/serialization"
	"DNA/core/ledger"
	. "DNA/net/protocol"
	"bytes"
	"crypto/sha256"
	"encoding/binary"
	"errors"
	"math/rand"
	"time"
)

type headersReq struct {
	hdr msgHdr
	p   struct {
		len       uint8
		hashStart [HASHLEN]byte
		hashEnd   [HASHLEN]byte
	}
}

type blkHeader struct {
	hdr    msgHdr
	cnt    uint32
	blkHdr []ledger.Header
}

func NewHeadersReq() ([]byte, error) {
	var h headersReq

	h.p.len = 1
	buf := ledger.DefaultLedger.Store.GetCurrentHeaderHash()
	copy(h.p.hashEnd[:], buf[:])

	p := new(bytes.Buffer)
	err := binary.Write(p, binary.LittleEndian, &(h.p))
	if err != nil {
		log.Error("Binary Write failed at new headersReq")
		return nil, err
	}

	s := checkSum(p.Bytes())
	h.hdr.init("getheaders", s, uint32(len(p.Bytes())))

	m, err := h.Serialization()
	return m, err
}

func (msg headersReq) Verify(buf []byte) error {
	// TODO Verify the message Content
	err := msg.hdr.Verify(buf)
	return err
}

func (msg blkHeader) Verify(buf []byte) error {
	// TODO Verify the message Content
	err := msg.hdr.Verify(buf)
	return err
}

func (msg headersReq) Serialization() ([]byte, error) {
	hdrBuf, err := msg.hdr.Serialization()
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(hdrBuf)
	err = binary.Write(buf, binary.LittleEndian, msg.p.len)
	if err != nil {
		return nil, err
	}
	err = binary.Write(buf, binary.LittleEndian, msg.p.hashStart)
	if err != nil {
		return nil, err
	}

	err = binary.Write(buf, binary.LittleEndian, msg.p.hashEnd)
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), err
}

func (msg *headersReq) Deserialization(p []byte) error {
	buf := bytes.NewBuffer(p)
	err := binary.Read(buf, binary.LittleEndian, &(msg.hdr))
	if err != nil {
		return err
	}

	err = binary.Read(buf, binary.LittleEndian, &(msg.p.len))
	if err != nil {
		return err
	}

	err = binary.Read(buf, binary.LittleEndian, &(msg.p.hashStart))
	if err != nil {
		return err
	}

	err = binary.Read(buf, binary.LittleEndian, &(msg.p.hashEnd))
	return err
}

func (msg blkHeader) Serialization() ([]byte, error) {
	hdrBuf, err := msg.hdr.Serialization()
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(hdrBuf)
	err = binary.Write(buf, binary.LittleEndian, msg.cnt)
	if err != nil {
		return nil, err
	}

	for _, header := range msg.blkHdr {
		header.Serialize(buf)
	}
	return buf.Bytes(), err
}

func (msg *blkHeader) Deserialization(p []byte) error {
	buf := bytes.NewBuffer(p)
	err := binary.Read(buf, binary.LittleEndian, &(msg.hdr))
	if err != nil {
		return err
	}

	err = binary.Read(buf, binary.LittleEndian, &(msg.cnt))
	if err != nil {
		return err
	}

	for i := 0; i < int(msg.cnt); i++ {
		var headers ledger.Header
		err := (&headers).Deserialize(buf)
		msg.blkHdr = append(msg.blkHdr, headers)
		if err != nil {
			log.Debug("blkHeader Deserialization failed")
			goto blkHdrErr
		}
	}

blkHdrErr:
	return err
}

func (msg headersReq) Handle(node Noder) error {
	log.Debug()
	// lock
	var startHash [HASHLEN]byte
	var stopHash [HASHLEN]byte
	startHash = msg.p.hashStart
	stopHash = msg.p.hashEnd
	//FIXME if HeaderHashCount > 1
	headers, cnt, err := GetHeadersFromHash(startHash, stopHash)
	if err != nil {
		return err
	}
	buf, err := NewHeaders(headers, cnt)
	if err != nil {
		return err
	}
	go node.Tx(buf)
	return nil
}

func SendMsgSyncHeaders(node Noder) {
	buf, err := NewHeadersReq()
	if err != nil {
		log.Error("failed build a new headersReq")
	} else {
		go node.Tx(buf)
	}
}

func ReqBlkHdrFromOthers(node Noder) {
	noders := node.LocalNode().GetNeighborNoder()
	if len(noders) == 0 {
		return
	}
	rand.Seed(time.Now().UnixNano())
	index := rand.Intn(len(noders))
	if noders[index].GetID() == node.GetID() {
		index = (index + 1) % len(noders)
	}
	n := noders[index]
	SendMsgSyncHeaders(n)

}

func (msg blkHeader) Handle(node Noder) error {
	log.Debug()
	node.StopRetryTimer()
	err := ledger.DefaultLedger.Store.AddHeaders(msg.blkHdr, ledger.DefaultLedger)
	if err != nil {
		log.Warn("Add block Header error")
		ReqBlkHdrFromOthers(node)
		return errors.New("Add block Header error, send new header request to another node\n")
	}
	if msg.cnt == MAXBLKHDRCNT {
		SendMsgSyncHeaders(node)
		node.StartRetryTimer()
	}
	return nil
}

func GetHeadersFromHash(startHash common.Uint256, stopHash common.Uint256) ([]ledger.Header, uint32, error) {
	var count uint32 = 0
	var empty [HASHLEN]byte
	headers := []ledger.Header{}
	var startHeight uint32
	var stopHeight uint32
	curHeight := ledger.DefaultLedger.GetLocalBlockChainHeight()
	if startHash == empty {
		if stopHash == empty {
			if curHeight > MAXBLKHDRCNT {
				count = MAXBLKHDRCNT
			} else {
				count = curHeight
			}
		} else {
			bkstop, err := ledger.DefaultLedger.Store.GetHeader(stopHash)
			if err != nil {
				return nil, 0, err
			}
			stopHeight = bkstop.Blockdata.Height
			count = curHeight - stopHeight
			if count > MAXBLKHDRCNT {
				count = MAXBLKHDRCNT
			}
		}
	} else {
		bkstart, err := ledger.DefaultLedger.Store.GetHeader(startHash)
		if err != nil {
			return nil, 0, err
		}
		startHeight = bkstart.Blockdata.Height
		if stopHash != empty {
			bkstop, err := ledger.DefaultLedger.Store.GetHeader(stopHash)
			if err != nil {
				return nil, 0, err
			}
			stopHeight = bkstop.Blockdata.Height
			count = startHeight - stopHeight
			if count >= MAXBLKHDRCNT {
				count = MAXBLKHDRCNT
				stopHeight = startHeight - MAXBLKHDRCNT
			}
		} else {

			if startHeight > MAXBLKHDRCNT {
				count = MAXBLKHDRCNT
			} else {
				count = startHeight
			}
		}
	}

	var i uint32
	for i = 1; i <= count; i++ {
		hash, err := ledger.DefaultLedger.Store.GetBlockHash(stopHeight + i)
		hd, err := ledger.DefaultLedger.Store.GetHeader(hash)
		if err != nil {
			log.Error("GetBlockWithHeight failed ", err.Error())
			return nil, 0, err
		}
		headers = append(headers, *hd)
	}

	return headers, count, nil
}

func NewHeaders(headers []ledger.Header, count uint32) ([]byte, error) {
	var msg blkHeader
	msg.cnt = count
	msg.blkHdr = headers
	msg.hdr.Magic = NETMAGIC
	cmd := "headers"
	copy(msg.hdr.CMD[0:len(cmd)], cmd)

	tmpBuffer := bytes.NewBuffer([]byte{})
	serialization.WriteUint32(tmpBuffer, msg.cnt)
	for _, header := range headers {
		header.Serialize(tmpBuffer)
	}
	b := new(bytes.Buffer)
	err := binary.Write(b, binary.LittleEndian, tmpBuffer.Bytes())
	if err != nil {
		log.Error("Binary Write failed at new Msg")
		return nil, err
	}
	s := sha256.Sum256(b.Bytes())
	s2 := s[:]
	s = sha256.Sum256(s2)
	buf := bytes.NewBuffer(s[:4])
	binary.Read(buf, binary.LittleEndian, &(msg.hdr.Checksum))
	msg.hdr.Length = uint32(len(b.Bytes()))

	m, err := msg.Serialization()
	if err != nil {
		log.Error("Error Convert net message ", err.Error())
		return nil, err
	}
	return m, nil
}
<commit_msg>fix: integer underflow<commit_after>package message

import (
	"DNA/common"
	"DNA/common/log"
	"DNA/common/serialization"
	"DNA/core/ledger"
	. "DNA/net/protocol"
	"bytes"
	"crypto/sha256"
	"encoding/binary"
	"errors"
	"math/rand"
	"time"
)

type headersReq struct {
	hdr msgHdr
	p   struct {
		len       uint8
		hashStart [HASHLEN]byte
		hashEnd   [HASHLEN]byte
	}
}

type blkHeader struct {
	hdr    msgHdr
	cnt    uint32
	blkHdr []ledger.Header
}

func NewHeadersReq() ([]byte, error) {
	var h headersReq

	h.p.len = 1
	buf := ledger.DefaultLedger.Store.GetCurrentHeaderHash()
	copy(h.p.hashEnd[:], buf[:])

	p := new(bytes.Buffer)
	err := binary.Write(p, binary.LittleEndian, &(h.p))
	if err != nil {
		log.Error("Binary Write failed at new headersReq")
		return nil, err
	}

	s := checkSum(p.Bytes())
	h.hdr.init("getheaders", s, uint32(len(p.Bytes())))

	m, err := h.Serialization()
	return m, err
}

func (msg headersReq) Verify(buf []byte) error {
	// TODO Verify the message Content
	err := msg.hdr.Verify(buf)
	return err
}

func (msg blkHeader) Verify(buf []byte) error {
	// TODO Verify the message Content
	err := msg.hdr.Verify(buf)
	return err
}

func (msg headersReq) Serialization() ([]byte, error) {
	hdrBuf, err := msg.hdr.Serialization()
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(hdrBuf)
	err = binary.Write(buf, binary.LittleEndian, msg.p.len)
	if err != nil {
		return nil, err
	}
	err = binary.Write(buf, binary.LittleEndian, msg.p.hashStart)
	if err != nil {
		return nil, err
	}

	err = binary.Write(buf, binary.LittleEndian, msg.p.hashEnd)
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), err
}

func (msg *headersReq) Deserialization(p []byte) error {
	buf := bytes.NewBuffer(p)
	err := binary.Read(buf, binary.LittleEndian, &(msg.hdr))
	if err != nil {
		return err
	}

	err = binary.Read(buf, binary.LittleEndian, &(msg.p.len))
	if err != nil {
		return err
	}

	err = binary.Read(buf, binary.LittleEndian, &(msg.p.hashStart))
	if err != nil {
		return err
	}

	err = binary.Read(buf, binary.LittleEndian, &(msg.p.hashEnd))
	return err
}

func (msg blkHeader) Serialization() ([]byte, error) {
	hdrBuf, err := msg.hdr.Serialization()
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(hdrBuf)
	err = binary.Write(buf, binary.LittleEndian, msg.cnt)
	if err != nil {
		return nil, err
	}

	for _, header := range msg.blkHdr {
		header.Serialize(buf)
	}
	return buf.Bytes(), err
}

func (msg *blkHeader) Deserialization(p []byte) error {
	buf := bytes.NewBuffer(p)
	err := binary.Read(buf, binary.LittleEndian, &(msg.hdr))
	if err != nil {
		return err
	}

	err = binary.Read(buf, binary.LittleEndian, &(msg.cnt))
	if err != nil {
		return err
	}

	for i := 0; i < int(msg.cnt); i++ {
		var headers ledger.Header
		err := (&headers).Deserialize(buf)
		msg.blkHdr = append(msg.blkHdr, headers)
		if err != nil {
			log.Debug("blkHeader Deserialization failed")
			goto blkHdrErr
		}
	}

blkHdrErr:
	return err
}

func (msg headersReq) Handle(node Noder) error {
	log.Debug()
	// lock
	var startHash [HASHLEN]byte
	var stopHash [HASHLEN]byte
	startHash = msg.p.hashStart
	stopHash = msg.p.hashEnd
	//FIXME if HeaderHashCount > 1
	headers, cnt, err := GetHeadersFromHash(startHash, stopHash)
	if err != nil {
		return err
	}
	buf, err := NewHeaders(headers, cnt)
	if err != nil {
		return err
	}
	go node.Tx(buf)
	return nil
}

func SendMsgSyncHeaders(node Noder) {
	buf, err := NewHeadersReq()
	if err != nil {
		log.Error("failed build a new headersReq")
	} else {
		go node.Tx(buf)
	}
}

func ReqBlkHdrFromOthers(node Noder) {
	noders := node.LocalNode().GetNeighborNoder()
	if len(noders) == 0 {
		return
	}
	rand.Seed(time.Now().UnixNano())
	index := rand.Intn(len(noders))
	if noders[index].GetID() == node.GetID() {
		index = (index + 1) % len(noders)
	}
	n := noders[index]
	SendMsgSyncHeaders(n)

}

func (msg blkHeader) Handle(node Noder) error {
	log.Debug()
	node.StopRetryTimer()
	err := ledger.DefaultLedger.Store.AddHeaders(msg.blkHdr, ledger.DefaultLedger)
	if err != nil {
		log.Warn("Add block Header error")
		ReqBlkHdrFromOthers(node)
		return errors.New("Add block Header error, send new header request to another node\n")
	}
	if msg.cnt == MAXBLKHDRCNT {
		SendMsgSyncHeaders(node)
		node.StartRetryTimer()
	}
	return nil
}

func GetHeadersFromHash(startHash common.Uint256, stopHash common.Uint256) ([]ledger.Header, uint32, error) {
	var count uint32 = 0
	var empty [HASHLEN]byte
	headers := []ledger.Header{}
	var startHeight uint32
	var stopHeight uint32
	curHeight := ledger.DefaultLedger.Store.GetHeaderHeight()
	if startHash == empty {
		if stopHash == empty {
			if curHeight > MAXBLKHDRCNT {
				count = MAXBLKHDRCNT
			} else {
				count = curHeight
			}
		} else {
			bkstop, err := ledger.DefaultLedger.Store.GetHeader(stopHash)
			if err != nil {
				return nil, 0, err
			}
			stopHeight = bkstop.Blockdata.Height
			count = curHeight - stopHeight
			if count > MAXBLKHDRCNT {
				count = MAXBLKHDRCNT
			}
		}
	} else {
		bkstart, err := ledger.DefaultLedger.Store.GetHeader(startHash)
		if err != nil {
			return nil, 0, err
		}
		startHeight = bkstart.Blockdata.Height
		if stopHash != empty {
			bkstop, err := ledger.DefaultLedger.Store.GetHeader(stopHash)
			if err != nil {
				return nil, 0, err
			}
			stopHeight = bkstop.Blockdata.Height

			// avoid unsigned integer underflow
			if startHeight < stopHeight {
				return nil, 0, errors.New("do not have header to send")
			}
			count = startHeight - stopHeight

			if count >= MAXBLKHDRCNT {
				count = MAXBLKHDRCNT
				stopHeight = startHeight - MAXBLKHDRCNT
			}
		} else {

			if startHeight > MAXBLKHDRCNT {
				count = MAXBLKHDRCNT
			} else {
				count = startHeight
			}
		}
	}

	var i uint32
	for i = 1; i <= count; i++ {
		hash, err := ledger.DefaultLedger.Store.GetBlockHash(stopHeight + i)
		hd, err := ledger.DefaultLedger.Store.GetHeader(hash)
		if err != nil {
			log.Error("GetBlockWithHeight failed ", err.Error())
			return nil, 0, err
		}
		headers = append(headers, *hd)
	}

	return headers, count, nil
}

func NewHeaders(headers []ledger.Header, count uint32) ([]byte, error) {
	var msg blkHeader
	msg.cnt = count
	msg.blkHdr = headers
	msg.hdr.Magic = NETMAGIC
	cmd := "headers"
	copy(msg.hdr.CMD[0:len(cmd)], cmd)

	tmpBuffer := bytes.NewBuffer([]byte{})
	serialization.WriteUint32(tmpBuffer, msg.cnt)
	for _, header := range headers {
		header.Serialize(tmpBuffer)
	}
	b := new(bytes.Buffer)
	err := binary.Write(b, binary.LittleEndian, tmpBuffer.Bytes())
	if err != nil {
		log.Error("Binary Write failed at new Msg")
		return nil, err
	}
	s := sha256.Sum256(b.Bytes())
	s2 := s[:]
	s = sha256.Sum256(s2)
	buf := bytes.NewBuffer(s[:4])
	binary.Read(buf, binary.LittleEndian, &(msg.hdr.Checksum))
	msg.hdr.Length = uint32(len(b.Bytes()))

	m, err := msg.Serialization()
	if err != nil {
		log.Error("Error Convert net message ", err.Error())
		return nil, err
	}
	return m, nil
}
<|endoftext|>


<commit_before>package rake

/*
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"unicode"
)

var sentenceDelimiters map[string]bool = map[string]bool{
	"[": true, "]": true, "\n": true, ".": true, "!": true, "?": true, ",": true, ";": true,
	":": true, "\t": true, "-": true, "\"": true, "(": true, ")": true, "'": true, "\u2019": true,
	"\u2013": true,
}
var wordSep *regexp.Regexp = regexp.MustCompile("\\s+")
var wordSplitter *regexp.Regexp = regexp.MustCompile("[^\\p{L}0-9\\+\\-/]")

func SplitSentences(text string) []string {
	indices := []int{0}
	for i := 0; i < len(text); i++ {
		s := text[i : i+1]
		_, match := sentenceDelimiters[s]
		if match {
			indices = append(indices, i)
		}
	}
	indices = append(indices, len(text))
	result := make([]string, 0, len(indices)+1)
	for i, index := range indices {
		j := i + 1
		if j < len(indices) {
			index2 := indices[j]
			if index2-index > 1 {
				result = append(result, text[index+1:index2])
			}
		}
	}
	return result
	//return sentenceDelimiters.Split(text, -1)
}

func IsAcceptable(phrase string, minCharLength int, maxWordsLength int) bool {
	// a phrase must have a min length in characters
	if len(phrase) < minCharLength {
		return false
	}

	// a phrase must have a max number of words
	words := wordSep.Split(phrase, -1)
	if len(words) > maxWordsLength {
		return false
	}
	digits := 0
	alpha := 0
	for _, c := range phrase {
		if unicode.IsDigit(c) {
			digits++
		} else if unicode.IsLetter(c) {
			alpha++
		}
	}
	// a phrase must have at least one alpha character
	if alpha == 0 {
		return false
	}
	// a phrase must have more alpha than digits characters
	if digits > alpha {
		return false
	}
	return true
}

func findStopwordIndices(words []string, stopwords map[string]bool) []int {
	result := []int{}
	if stopwords != nil {
		for i, word := range words {
			if stopwords[strings.ToLower(word)] {
				result = append(result, i)
			}
		}
	}
	return result
}

func GenerateCandidateKeywords(sentenceList []string, stopwords map[string]bool, minCharLength int, maxWordsLength int) []string {
	phraseList := []string{}
	for _, s := range sentenceList {
		words := strings.Fields(s)
		stopwordIndices := findStopwordIndices(words, stopwords)

		if len(stopwordIndices) > 0 {
			if stopwordIndices[0] != 0 {
				phraseWords := words[0:stopwordIndices[0]]
				phrase := strings.Join(phraseWords, " ")
				if phrase != "" && IsAcceptable(phrase, minCharLength, maxWordsLength) {
					phraseList = append(phraseList, phrase)
				}
			}
			for i, index := range stopwordIndices {
				j := i + 1
				if j < len(stopwordIndices) {
					index2 := stopwordIndices[j]
					if index2-index == 1 {
						continue
					}
					phraseWords := words[index+1 : index2]
					phrase := strings.Join(phraseWords, " ")
					if phrase != "" && IsAcceptable(phrase, minCharLength, maxWordsLength) {
						phraseList = append(phraseList, phrase)
					}
				}
			}
			if stopwordIndices[len(stopwordIndices)-1] != len(words)-1 {
				index := stopwordIndices[len(stopwordIndices)-1]
				phraseWords := words[index+1:]
				phrase := strings.Join(phraseWords, " ")
				if phrase != "" && IsAcceptable(phrase, minCharLength, maxWordsLength) {
					phraseList = append(phraseList, phrase)
				}
			}
		} else {
			phrase := strings.Join(words, " ")
			if phrase != "" && IsAcceptable(phrase, minCharLength, maxWordsLength) {
				phraseList = append(phraseList, phrase)
			}
		}
	}
	return phraseList
}

func IsNumber(s string) bool {
	if strings.Index(s, ".") != -1 {
		_, err := strconv.ParseFloat(s, 32)
		return err == nil
	} else {
		_, err := strconv.ParseInt(s, 10, 32)
		return err == nil
	}
}

func SeparateWords(text string, minWordReturnSize int) []string {
	//    Utility function to return a list of all words that are have a length
	//    greater than a specified number of characters.
	//    @param text The text that must be split in to words.
	//    @param min_word_return_size The minimum no of characters
	//           a word must have to be included.
	words := []string{}
	for _, singleWord := range wordSplitter.Split(text, -1) {
		currentWord := strings.ToLower(strings.TrimSpace(singleWord))
		// leave numbers in phrase, but don't count as words, since they tend to invalidate scores of their phrases
		if len(currentWord) > minWordReturnSize && currentWord != "" && !IsNumber(currentWord) {
			words = append(words, currentWord)
		}
	}
	return words
}

func CalculateWordScores(phraseList []string) map[string]float64 {
	wordFrequency := map[string]int{}
	wordDegree := map[string]int{}
	for _, phrase := range phraseList {
		wordList := SeparateWords(phrase, 0)
		wordListLength := len(wordList)
		wordListDegree := wordListLength - 1
		// if word_list_degree > 3: word_list_degree = 3 #exp.
		for _, word := range wordList {
			wordFrequency[word]++
			wordDegree[word] += wordListDegree
		}
	}
	for item := range wordFrequency {
		wordDegree[item] = wordDegree[item] + wordFrequency[item]
	}
	// Calculate Word scores = deg(w)/frew(w)
	wordScore := map[string]float64{}
	for item := range wordFrequency {
		wordScore[item] = float64(wordDegree[item]) / float64(wordFrequency[item])
	}
	return wordScore
}

func StringCount(stringList []string) map[string]int {
	stringCount := map[string]int{}
	for _, item := range stringList {
		stringCount[item]++
	}
	return stringCount
}

func GenerateCandidateKeywordScores(phraseList []string, wordScore map[string]float64, minKeywordFrequency int) (map[string]float64, map[string]int) {
	keywordCandidates := map[string]float64{}
	phraseCounts := StringCount(phraseList)
	for _, phrase := range phraseList {
		if minKeywordFrequency > 1 {
			if phraseCounts[phrase] < minKeywordFrequency {
				continue
			}
		}
		wordList := SeparateWords(phrase, 0)
		var candiateScore float64 = 0
		for _, word := range wordList {
			candiateScore += wordScore[word]
		}
		keywordCandidates[phrase] = candiateScore
	}

	return keywordCandidates, phraseCounts
}

func MapToKeywordScores(keywordScoreMap map[string]float64, keywordCountMap map[string]int) []KeywordScore {
	keywordScores := []KeywordScore{}
	for key, value := range keywordScoreMap {
		keywordScore := KeywordScore{
			Keyword: key,
			Score:   value,
		}
		count, _ := keywordCountMap[key]
		keywordScore.Count = count
		keywordScores = append(keywordScores, keywordScore)
	}
	return keywordScores
}

func loadStopWords(stopWordFile string) []string {
	stopWords := []string{}
	contents, err := ioutil.ReadFile(stopWordFile)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	lines := strings.Split(string(contents), "\n")
	for _, line := range lines {
		line = strings.ToLower(strings.TrimSpace(line))
		if strings.Index(line, "#") != 0 {
			// in case more than one per line
			for _, word := range strings.Fields(line) {
				stopWords = append(stopWords, word)
			}

		}
	}
	return stopWords
}

func BuildStopWordMap(stopWordList []string) map[string]bool {
	stopWordMap := make(map[string]bool, len(stopWordList))
	for _, word := range stopWordList {
		stopWordMap[strings.ToLower(word)] = true
	}
	return stopWordMap
}

// ByScore implements sort.Interface for []KeywordScore based on
// the Score field. Sorts in reverse order.
type ByScore []KeywordScore

func (a ByScore) Len() int           { return len(a) }
func (a ByScore) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByScore) Less(i, j int) bool { return a[i].Score > a[j].Score }

type Rake struct {
	stopWordsPath       string
	stopWords           map[string]bool
	minCharLength       int
	maxWordsLength      int
	minKeywordFrequency int
}

type KeywordScore struct {
	Keyword string
	Score   float64
	Count   int
}

func NewRake(stopWordsPath string, minCharLength int, maxWordsLength int, minKeywordFrequency int) *Rake {
	rake := &Rake{
		minCharLength:       minCharLength,
		maxWordsLength:      maxWordsLength,
		minKeywordFrequency: minKeywordFrequency,
	}

	if stopWordsPath != "" {
		rake.stopWords = BuildStopWordMap(loadStopWords(stopWordsPath))
	}

	return rake
}

func (rake *Rake) SetStopWords(stopWordsList []string) {
	rake.stopWords = BuildStopWordMap(stopWordsList)
}

func (rake *Rake) Run(text string) []KeywordScore {
	sentenceList := SplitSentences(text)
	phraseList := GenerateCandidateKeywords(sentenceList, rake.stopWords, rake.minCharLength, rake.maxWordsLength)
	wordScores := CalculateWordScores(phraseList)
	keywordCandidates, keywordCounts := GenerateCandidateKeywordScores(phraseList, wordScores, rake.minKeywordFrequency)
	keywordScores := MapToKeywordScores(keywordCandidates, keywordCounts)
	sort.Sort(ByScore(keywordScores))
	return keywordScores
}
<commit_msg>Improve sentence separation<commit_after>package rake

/*
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"unicode"
)

var wordSep *regexp.Regexp = regexp.MustCompile("\\s+")
var wordSplitter *regexp.Regexp = regexp.MustCompile("[^\\p{L}0-9\\+\\-/]")

func sentenceSplitter(r rune) bool {
	// Ignore hyphens, usually they are used to join words (e.g., "ice-cream-flavored candy")
	return unicode.IsPunct(r) && !unicode.Is(unicode.Properties["Hyphen"], r)
}

func IsAcceptable(phrase string, minCharLength int, maxWordsLength int) bool {
	// a phrase must have a min length in characters
	if len(phrase) < minCharLength {
		return false
	}

	// a phrase must have a max number of words
	words := wordSep.Split(phrase, -1)
	if len(words) > maxWordsLength {
		return false
	}
	digits := 0
	alpha := 0
	for _, c := range phrase {
		if unicode.IsDigit(c) {
			digits++
		} else if unicode.IsLetter(c) {
			alpha++
		}
	}
	// a phrase must have at least one alpha character
	if alpha == 0 {
		return false
	}
	// a phrase must have more alpha than digits characters
	if digits > alpha {
		return false
	}
	return true
}

func findStopwordIndices(words []string, stopwords map[string]bool) []int {
	result := []int{}
	if stopwords != nil {
		for i, word := range words {
			if stopwords[strings.ToLower(word)] {
				result = append(result, i)
			}
		}
	}
	return result
}

func GenerateCandidateKeywords(sentenceList []string, stopwords map[string]bool, minCharLength int, maxWordsLength int) []string {
	phraseList := []string{}
	for _, s := range sentenceList {
		words := strings.Fields(s)
		stopwordIndices := findStopwordIndices(words, stopwords)

		if len(stopwordIndices) > 0 {
			if stopwordIndices[0] != 0 {
				phraseWords := words[0:stopwordIndices[0]]
				phrase := strings.Join(phraseWords, " ")
				if phrase != "" && IsAcceptable(phrase, minCharLength, maxWordsLength) {
					phraseList = append(phraseList, phrase)
				}
			}
			for i, index := range stopwordIndices {
				j := i + 1
				if j < len(stopwordIndices) {
					index2 := stopwordIndices[j]
					if index2-index == 1 {
						continue
					}
					phraseWords := words[index+1 : index2]
					phrase := strings.Join(phraseWords, " ")
					if phrase != "" && IsAcceptable(phrase, minCharLength, maxWordsLength) {
						phraseList = append(phraseList, phrase)
					}
				}
			}
			if stopwordIndices[len(stopwordIndices)-1] != len(words)-1 {
				index := stopwordIndices[len(stopwordIndices)-1]
				phraseWords := words[index+1:]
				phrase := strings.Join(phraseWords, " ")
				if phrase != "" && IsAcceptable(phrase, minCharLength, maxWordsLength) {
					phraseList = append(phraseList, phrase)
				}
			}
		} else {
			phrase := strings.Join(words, " ")
			if phrase != "" && IsAcceptable(phrase, minCharLength, maxWordsLength) {
				phraseList = append(phraseList, phrase)
			}
		}
	}
	return phraseList
}

func IsNumber(s string) bool {
	if strings.Index(s, ".") != -1 {
		_, err := strconv.ParseFloat(s, 32)
		return err == nil
	} else {
		_, err := strconv.ParseInt(s, 10, 32)
		return err == nil
	}
}

func SeparateWords(text string, minWordReturnSize int) []string {
	//    Utility function to return a list of all words that are have a length
	//    greater than a specified number of characters.
	//    @param text The text that must be split in to words.
	//    @param min_word_return_size The minimum no of characters
	//           a word must have to be included.
	words := []string{}
	for _, singleWord := range wordSplitter.Split(text, -1) {
		currentWord := strings.ToLower(strings.TrimSpace(singleWord))
		// leave numbers in phrase, but don't count as words, since they tend to invalidate scores of their phrases
		if len(currentWord) > minWordReturnSize && currentWord != "" && !IsNumber(currentWord) {
			words = append(words, currentWord)
		}
	}
	return words
}

func CalculateWordScores(phraseList []string) map[string]float64 {
	wordFrequency := map[string]int{}
	wordDegree := map[string]int{}
	for _, phrase := range phraseList {
		wordList := SeparateWords(phrase, 0)
		wordListLength := len(wordList)
		wordListDegree := wordListLength - 1
		// if word_list_degree > 3: word_list_degree = 3 #exp.
		for _, word := range wordList {
			wordFrequency[word]++
			wordDegree[word] += wordListDegree
		}
	}
	for item := range wordFrequency {
		wordDegree[item] = wordDegree[item] + wordFrequency[item]
	}
	// Calculate Word scores = deg(w)/frew(w)
	wordScore := map[string]float64{}
	for item := range wordFrequency {
		wordScore[item] = float64(wordDegree[item]) / float64(wordFrequency[item])
	}
	return wordScore
}

func StringCount(stringList []string) map[string]int {
	stringCount := map[string]int{}
	for _, item := range stringList {
		stringCount[item]++
	}
	return stringCount
}

func GenerateCandidateKeywordScores(phraseList []string, wordScore map[string]float64, minKeywordFrequency int) (map[string]float64, map[string]int) {
	keywordCandidates := map[string]float64{}
	phraseCounts := StringCount(phraseList)
	for _, phrase := range phraseList {
		if minKeywordFrequency > 1 {
			if phraseCounts[phrase] < minKeywordFrequency {
				continue
			}
		}
		wordList := SeparateWords(phrase, 0)
		var candiateScore float64 = 0
		for _, word := range wordList {
			candiateScore += wordScore[word]
		}
		keywordCandidates[phrase] = candiateScore
	}

	return keywordCandidates, phraseCounts
}

func MapToKeywordScores(keywordScoreMap map[string]float64, keywordCountMap map[string]int) []KeywordScore {
	keywordScores := []KeywordScore{}
	for key, value := range keywordScoreMap {
		keywordScore := KeywordScore{
			Keyword: key,
			Score:   value,
		}
		count, _ := keywordCountMap[key]
		keywordScore.Count = count
		keywordScores = append(keywordScores, keywordScore)
	}
	return keywordScores
}

func loadStopWords(stopWordFile string) []string {
	stopWords := []string{}
	contents, err := ioutil.ReadFile(stopWordFile)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	lines := strings.Split(string(contents), "\n")
	for _, line := range lines {
		line = strings.ToLower(strings.TrimSpace(line))
		if strings.Index(line, "#") != 0 {
			// in case more than one per line
			for _, word := range strings.Fields(line) {
				stopWords = append(stopWords, word)
			}

		}
	}
	return stopWords
}

func BuildStopWordMap(stopWordList []string) map[string]bool {
	stopWordMap := make(map[string]bool, len(stopWordList))
	for _, word := range stopWordList {
		stopWordMap[strings.ToLower(word)] = true
	}
	return stopWordMap
}

// ByScore implements sort.Interface for []KeywordScore based on
// the Score field. Sorts in reverse order.
type ByScore []KeywordScore

func (a ByScore) Len() int           { return len(a) }
func (a ByScore) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByScore) Less(i, j int) bool { return a[i].Score > a[j].Score }

type Rake struct {
	stopWordsPath       string
	stopWords           map[string]bool
	minCharLength       int
	maxWordsLength      int
	minKeywordFrequency int
}

type KeywordScore struct {
	Keyword string
	Score   float64
	Count   int
}

func NewRake(stopWordsPath string, minCharLength int, maxWordsLength int, minKeywordFrequency int) *Rake {
	rake := &Rake{
		minCharLength:       minCharLength,
		maxWordsLength:      maxWordsLength,
		minKeywordFrequency: minKeywordFrequency,
	}

	if stopWordsPath != "" {
		rake.stopWords = BuildStopWordMap(loadStopWords(stopWordsPath))
	}

	return rake
}

func (rake *Rake) SetStopWords(stopWordsList []string) {
	rake.stopWords = BuildStopWordMap(stopWordsList)
}

func (rake *Rake) Run(text string) []KeywordScore {
	sentenceList := strings.FieldsFunc(text, sentenceSplitter)
	phraseList := GenerateCandidateKeywords(sentenceList, rake.stopWords, rake.minCharLength, rake.maxWordsLength)
	wordScores := CalculateWordScores(phraseList)
	keywordCandidates, keywordCounts := GenerateCandidateKeywordScores(phraseList, wordScores, rake.minKeywordFrequency)
	keywordScores := MapToKeywordScores(keywordCandidates, keywordCounts)
	sort.Sort(ByScore(keywordScores))
	return keywordScores
}
<|endoftext|>


<commit_before>package node

import (
	"encoding/binary"
	"erlang/dist"
	"erlang/epmd"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"
)

type Node struct {
	epmd.NodeInfo
	Cookie string
	port   int32
}

func NewNode(name string, cookie string) (n *Node) {
	log.Printf("Start with name '%s' and cookie '%s'", name, cookie)
	// TODO: add fqdn support
	ns := strings.Split(name, "@")
	nodeInfo := epmd.NodeInfo{
		FullName: name,
		Name:     ns[0],
		Domain:   ns[1],
		Port:     0,
		Type:     77, // or 72 if hidden
		Protocol: 0,
		HighVsn:  5,
		LowVsn:   5,
		Creation: 0,
	}

	return &Node{
		NodeInfo: nodeInfo,
		Cookie:   cookie,
	}
}

func (n *Node) Connect(remote string) {

}

func (n *Node) Publish(port int) (err error) {
	log.Printf("Publish ENode at %d", port)
	l, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(port)))
	if err != nil {
		return
	}
	n.Port = uint16(port)
	aliveResp := make(chan uint16)
	go epmdC(n, aliveResp)
	creation := <-aliveResp
	switch creation {
	case 99:
		return fmt.Errorf("Duplicate name '%s'", n.Name)
	case 100:
		return fmt.Errorf("Cannot connect to EPMD")
	default:
		n.Creation = creation
	}

	go func() {
		for {
			conn, err := l.Accept()
			log.Printf("Accept new at ENode")
			if err != nil {
				log.Printf(err.Error())
			} else {
				go n.mLoop(conn)
			}
		}
	}()
	return nil
}

func (currNode *Node) mLoop(c net.Conn) {

	currNd := dist.NewNodeDesc(currNode.FullName, currNode.Cookie, false)

	for {
		err := currNd.ReadMessage(c)
		if err != nil {
			log.Printf("Enode error: %s", err.Error())
			break
		}

	}
	c.Close()
}

func epmdC(n *Node, resp chan uint16) {
	conn, err := net.Dial("tcp", ":4369")
	defer conn.Close()
	if err != nil {
		resp <- 100
		return
	}

	epmdFROM := make(chan []byte)
	go epmdREADER(conn, epmdFROM)

	epmdTO := make(chan []byte)
	go epmdWRITER(conn, epmdFROM, epmdTO)

	epmdTO <- epmd.Compose_ALIVE2_REQ(&n.NodeInfo)

	for {

		select {
		case reply := <-epmdFROM:
			log.Printf("From EPMD: %v", reply)

			switch epmd.MessageId(reply[0]) {
			case epmd.ALIVE2_RESP:
				if creation, ok := epmd.Read_ALIVE2_RESP(reply); ok {
					resp <- creation
				} else {
					resp <- 99
				}
			}

		}

	}
}

func epmdREADER(conn net.Conn, in chan []byte) {
	for {
		buf := make([]byte, 1024)
		n, err := conn.Read(buf)
		if err != nil {
			in <- buf[0:n]
			in <- []byte{}
			return
		}
		log.Printf("Read from EPMD %d: %v", n, buf[:n])
		in <- buf[:n]
	}
}

func epmdWRITER(conn net.Conn, in chan []byte, out chan []byte) {
	for {
		select {
		case data := <-out:
			buf := make([]byte, 2)
			binary.BigEndian.PutUint16(buf[0:2], uint16(len(data)))
			buf = append(buf, data...)
			_, err := conn.Write(buf)
			if err != nil {
				in <- []byte{}
			}
		}
	}
}
<commit_msg>Add flags to trace erlang libraries<commit_after>package node

import (
	"encoding/binary"
	"erlang/dist"
	"erlang/epmd"
	"erlang/term"
	"flag"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"
)

var nTrace bool

func init() {
	flag.BoolVar(&nTrace, "erlang.node.trace", false, "trace erlang node")
}

func nLog(f string, a ...interface{}) {
	if nTrace {
		log.Printf(f, a...)
	}
}

type Node struct {
	epmd.NodeInfo
	Cookie string
	port   int32
}

func NewNode(name string, cookie string) (n *Node) {
	nLog("Start with name '%s' and cookie '%s'", name, cookie)
	// TODO: add fqdn support
	ns := strings.Split(name, "@")
	nodeInfo := epmd.NodeInfo{
		FullName: name,
		Name:     ns[0],
		Domain:   ns[1],
		Port:     0,
		Type:     77, // or 72 if hidden
		Protocol: 0,
		HighVsn:  5,
		LowVsn:   5,
		Creation: 0,
	}

	return &Node{
		NodeInfo: nodeInfo,
		Cookie:   cookie,
	}
}

func (n *Node) Connect(remote string) {

}

func (n *Node) Publish(port int) (err error) {
	nLog("Publish ENode at %d", port)
	l, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(port)))
	if err != nil {
		return
	}
	n.Port = uint16(port)
	aliveResp := make(chan uint16)
	go epmdC(n, aliveResp)
	creation := <-aliveResp
	switch creation {
	case 99:
		return fmt.Errorf("Duplicate name '%s'", n.Name)
	case 100:
		return fmt.Errorf("Cannot connect to EPMD")
	default:
		n.Creation = creation
	}

	go func() {
		for {
			conn, err := l.Accept()
			nLog("Accept new at ENode")
			if err != nil {
				nLog(err.Error())
			} else {
				go n.mLoop(conn)
			}
		}
	}()
	return nil
}

func (currNode *Node) mLoop(c net.Conn) {

	currNd := dist.NewNodeDesc(currNode.FullName, currNode.Cookie, false)

	for {
		terms, err := currNd.ReadMessage(c)
		if err != nil {
			nLog("Enode error: %s", err.Error())
			break
		}
		handleTerms(c, terms)
	}
	c.Close()
}

func handleTerms(c net.Conn, terms []term.Term) {
	nLog("Node terms: %#v", terms)
}

func epmdC(n *Node, resp chan uint16) {
	conn, err := net.Dial("tcp", ":4369")
	defer conn.Close()
	if err != nil {
		resp <- 100
		return
	}

	epmdFROM := make(chan []byte)
	go epmdREADER(conn, epmdFROM)

	epmdTO := make(chan []byte)
	go epmdWRITER(conn, epmdFROM, epmdTO)

	epmdTO <- epmd.Compose_ALIVE2_REQ(&n.NodeInfo)

	for {

		select {
		case reply := <-epmdFROM:
			nLog("From EPMD: %v", reply)

			switch epmd.MessageId(reply[0]) {
			case epmd.ALIVE2_RESP:
				if creation, ok := epmd.Read_ALIVE2_RESP(reply); ok {
					resp <- creation
				} else {
					resp <- 99
				}
			}

		}

	}
}

func epmdREADER(conn net.Conn, in chan []byte) {
	for {
		buf := make([]byte, 1024)
		n, err := conn.Read(buf)
		if err != nil {
			in <- buf[0:n]
			in <- []byte{}
			return
		}
		nLog("Read from EPMD %d: %v", n, buf[:n])
		in <- buf[:n]
	}
}

func epmdWRITER(conn net.Conn, in chan []byte, out chan []byte) {
	for {
		select {
		case data := <-out:
			buf := make([]byte, 2)
			binary.BigEndian.PutUint16(buf[0:2], uint16(len(data)))
			buf = append(buf, data...)
			_, err := conn.Write(buf)
			if err != nil {
				in <- []byte{}
			}
		}
	}
}
<|endoftext|>


<commit_before>package rpc // import "collectd.org/rpc"

import (
	"io"

	pb "collectd.org/rpc/proto"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

// RegisterServer registers the implementation srv with the gRPC instance s.
func RegisterServer(s *grpc.Server, srv Interface) {
	pb.RegisterCollectdServer(s, &server{
		Interface: srv,
	})
}

// server implements pb.CollectdServer using srv.
type server struct {
	Interface
}

// DispatchValues reads ValueLists from stream and calls the Write()
// implementation on each one.
func (s *server) DispatchValues(stream pb.Collectd_DispatchValuesServer) error {
	for {
		req, err := stream.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		vl, err := UnmarshalValueList(req.GetValueList())
		if err != nil {
			return err
		}

		// TODO(octo): pass stream.Context() to srv.Write() once the interface allows that.
		if err := s.Write(*vl); err != nil {
			return grpc.Errorf(codes.Internal, "Write(%v): %v", vl, err)
		}
	}

	return stream.SendAndClose(&pb.DispatchValuesResponse{})
}

// QueryValues calls the Query() implementation and streams all ValueLists from
// the channel back to the client.
func (s *server) QueryValues(req *pb.QueryValuesRequest, stream pb.Collectd_QueryValuesServer) error {
	id := UnmarshalIdentifier(req.GetIdentifier())

	ch, err := s.Query(stream.Context(), id)
	if err != nil {
		return grpc.Errorf(codes.Internal, "Query(%v): %v", id, err)
	}

	for vl := range ch {
		pbVL, err := MarshalValueList(vl)
		if err != nil {
			return err
		}

		res := &pb.QueryValuesResponse{
			ValueList: pbVL,
		}
		if err := stream.Send(res); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Package rpc: Cancel the context passed to the Query function.<commit_after>package rpc // import "collectd.org/rpc"

import (
	"context"
	"io"

	pb "collectd.org/rpc/proto"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

// RegisterServer registers the implementation srv with the gRPC instance s.
func RegisterServer(s *grpc.Server, srv Interface) {
	pb.RegisterCollectdServer(s, &server{
		Interface: srv,
	})
}

// server implements pb.CollectdServer using srv.
type server struct {
	Interface
}

// DispatchValues reads ValueLists from stream and calls the Write()
// implementation on each one.
func (s *server) DispatchValues(stream pb.Collectd_DispatchValuesServer) error {
	for {
		req, err := stream.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		vl, err := UnmarshalValueList(req.GetValueList())
		if err != nil {
			return err
		}

		// TODO(octo): pass stream.Context() to srv.Write() once the interface allows that.
		if err := s.Write(*vl); err != nil {
			return grpc.Errorf(codes.Internal, "Write(%v): %v", vl, err)
		}
	}

	return stream.SendAndClose(&pb.DispatchValuesResponse{})
}

// QueryValues calls the Query() implementation and streams all ValueLists from
// the channel back to the client.
func (s *server) QueryValues(req *pb.QueryValuesRequest, stream pb.Collectd_QueryValuesServer) error {
	id := UnmarshalIdentifier(req.GetIdentifier())

	ctx, cancel := context.WithCancel(stream.Context())
	defer cancel()

	ch, err := s.Query(ctx, id)
	if err != nil {
		return grpc.Errorf(codes.Internal, "Query(%v): %v", id, err)
	}

	for vl := range ch {
		pbVL, err := MarshalValueList(vl)
		if err != nil {
			return err
		}

		res := &pb.QueryValuesResponse{
			ValueList: pbVL,
		}
		if err := stream.Send(res); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package sqlx

// Named Query Support
//
//  * BindMap - bind query bindvars to map/struct args
//	* NamedExec, NamedQuery - named query w/ struct or map
//  * NamedStmt - a pre-compiled named query which is a prepared statement
//
// Internal Interfaces:
//
//  * compileNamedQuery - rebind a named query, returning a query and list of names
//  * bindArgs, bindMapArgs, bindAnyArgs - given a list of names, return an arglist
//
import (
	"database/sql"
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"unicode"

	"github.com/jmoiron/sqlx/reflectx"
)

// NamedStmt is a prepared statement that executes named queries.  Prepare it
// how you would execute a NamedQuery, but pass in a struct or map when executing.
type NamedStmt struct {
	Params      []string
	QueryString string
	Stmt        *Stmt
}

// Close closes the named statement.
func (n *NamedStmt) Close() error {
	return n.Stmt.Close()
}

// Exec executes a named statement using the struct passed.
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Exec(arg interface{}) (sql.Result, error) {
	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
	if err != nil {
		return *new(sql.Result), err
	}
	return n.Stmt.Exec(args...)
}

// Query executes a named statement using the struct argument, returning rows.
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Query(arg interface{}) (*sql.Rows, error) {
	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
	if err != nil {
		return nil, err
	}
	return n.Stmt.Query(args...)
}

// QueryRow executes a named statement against the database.  Because sqlx cannot
// create a *sql.Row with an error condition pre-set for binding errors, sqlx
// returns a *sqlx.Row instead.
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) QueryRow(arg interface{}) *Row {
	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
	if err != nil {
		return &Row{err: err}
	}
	return n.Stmt.QueryRowx(args...)
}

// MustExec execs a NamedStmt, panicing on error
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) MustExec(arg interface{}) sql.Result {
	res, err := n.Exec(arg)
	if err != nil {
		panic(err)
	}
	return res
}

// Queryx using this NamedStmt
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Queryx(arg interface{}) (*Rows, error) {
	r, err := n.Query(arg)
	if err != nil {
		return nil, err
	}
	return &Rows{Rows: r, Mapper: n.Stmt.Mapper, unsafe: isUnsafe(n)}, err
}

// QueryRowx this NamedStmt.  Because of limitations with QueryRow, this is
// an alias for QueryRow.
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) QueryRowx(arg interface{}) *Row {
	return n.QueryRow(arg)
}

// Select using this NamedStmt
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Select(dest interface{}, arg interface{}) error {
	rows, err := n.Queryx(arg)
	if err != nil {
		return err
	}
	// if something happens here, we want to make sure the rows are Closed
	defer rows.Close()
	return scanAll(rows, dest, false)
}

// Get using this NamedStmt
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Get(dest interface{}, arg interface{}) error {
	r := n.QueryRowx(arg)
	return r.scanAny(dest, false)
}

// Unsafe creates an unsafe version of the NamedStmt
func (n *NamedStmt) Unsafe() *NamedStmt {
	r := &NamedStmt{Params: n.Params, Stmt: n.Stmt, QueryString: n.QueryString}
	r.Stmt.unsafe = true
	return r
}

// A union interface of preparer and binder, required to be able to prepare
// named statements (as the bindtype must be determined).
type namedPreparer interface {
	Preparer
	binder
}

func prepareNamed(p namedPreparer, query string) (*NamedStmt, error) {
	bindType := BindType(p.DriverName())
	q, args, err := compileNamedQuery([]byte(query), bindType)
	if err != nil {
		return nil, err
	}
	stmt, err := Preparex(p, q)
	if err != nil {
		return nil, err
	}
	return &NamedStmt{
		QueryString: q,
		Params:      args,
		Stmt:        stmt,
	}, nil
}

func bindAnyArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) {
	if maparg, ok := arg.(map[string]interface{}); ok {
		return bindMapArgs(names, maparg)
	}
	return bindArgs(names, arg, m)
}

// private interface to generate a list of interfaces from a given struct
// type, given a list of names to pull out of the struct.  Used by public
// BindStruct interface.
func bindArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) {
	arglist := make([]interface{}, 0, len(names))

	// grab the indirected value of arg
	v := reflect.ValueOf(arg)
	for v = reflect.ValueOf(arg); v.Kind() == reflect.Ptr; {
		v = v.Elem()
	}

	fields := m.TraversalsByName(v.Type(), names)
	for i, t := range fields {
		if len(t) == 0 {
			return arglist, fmt.Errorf("could not find name %s in %#v", names[i], arg)
		}
		val := reflectx.FieldByIndexesReadOnly(v, t)
		arglist = append(arglist, val.Interface())
	}

	return arglist, nil
}

// like bindArgs, but for maps.
func bindMapArgs(names []string, arg map[string]interface{}) ([]interface{}, error) {
	arglist := make([]interface{}, 0, len(names))

	for _, name := range names {
		val, ok := arg[name]
		if !ok {
			return arglist, fmt.Errorf("could not find name %s in %#v", name, arg)
		}
		arglist = append(arglist, val)
	}
	return arglist, nil
}

// bindStruct binds a named parameter query with fields from a struct argument.
// The rules for binding field names to parameter names follow the same
// conventions as for StructScan, including obeying the `db` struct tags.
func bindStruct(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) {
	bound, names, err := compileNamedQuery([]byte(query), bindType)
	if err != nil {
		return "", []interface{}{}, err
	}

	arglist, err := bindArgs(names, arg, m)
	if err != nil {
		return "", []interface{}{}, err
	}

	return bound, arglist, nil
}

// bindMap binds a named parameter query with a map of arguments.
func bindMap(bindType int, query string, args map[string]interface{}) (string, []interface{}, error) {
	bound, names, err := compileNamedQuery([]byte(query), bindType)
	if err != nil {
		return "", []interface{}{}, err
	}

	arglist, err := bindMapArgs(names, args)
	return bound, arglist, err
}

// -- Compilation of Named Queries

// Allow digits and letters in bind params;  additionally runes are
// checked against underscores, meaning that bind params can have be
// alphanumeric with underscores.  Mind the difference between unicode
// digits and numbers, where '5' is a digit but '五' is not.
var allowedBindRunes = []*unicode.RangeTable{unicode.Letter, unicode.Digit}

// FIXME: this function isn't safe for unicode named params, as a failing test
// can testify.  This is not a regression but a failure of the original code
// as well.  It should be modified to range over runes in a string rather than
// bytes, even though this is less convenient and slower.  Hopefully the
// addition of the prepared NamedStmt (which will only do this once) will make
// up for the slightly slower ad-hoc NamedExec/NamedQuery.

// compile a NamedQuery into an unbound query (using the '?' bindvar) and
// a list of names.
func compileNamedQuery(qs []byte, bindType int) (query string, names []string, err error) {
	names = make([]string, 0, 10)
	rebound := make([]byte, 0, len(qs))

	inName := false
	last := len(qs) - 1
	currentVar := 1
	name := make([]byte, 0, 10)

	for i, b := range qs {
		// a ':' while we're in a name is an error
		if b == ':' {
			// if this is the second ':' in a '::' escape sequence, append a ':'
			if inName && i > 0 && qs[i-1] == ':' {
				rebound = append(rebound, ':')
				inName = false
				continue
			} else if inName {
				err = errors.New("unexpected `:` while reading named param at " + strconv.Itoa(i))
				return query, names, err
			}
			inName = true
			name = []byte{}
			// if we're in a name, and this is an allowed character, continue
		} else if inName && (unicode.IsOneOf(allowedBindRunes, rune(b)) || b == '_' || b == '.') && i != last {
			// append the byte to the name if we are in a name and not on the last byte
			name = append(name, b)
			// if we're in a name and it's not an allowed character, the name is done
		} else if inName {
			inName = false
			// if this is the final byte of the string and it is part of the name, then
			// make sure to add it to the name
			if i == last && unicode.IsOneOf(allowedBindRunes, rune(b)) {
				name = append(name, b)
			}
			// add the string representation to the names list
			names = append(names, string(name))
			// add a proper bindvar for the bindType
			switch bindType {
			// oracle only supports named type bind vars even for positional
			case NAMED:
				rebound = append(rebound, ':')
				rebound = append(rebound, name...)
			case QUESTION, UNKNOWN:
				rebound = append(rebound, '?')
			case DOLLAR:
				rebound = append(rebound, '$')
				for _, b := range strconv.Itoa(currentVar) {
					rebound = append(rebound, byte(b))
				}
				currentVar++
			}
			// add this byte to string unless it was not part of the name
			if i != last {
				rebound = append(rebound, b)
			} else if !unicode.IsOneOf(allowedBindRunes, rune(b)) {
				rebound = append(rebound, b)
			}
		} else {
			// this is a normal byte and should just go onto the rebound query
			rebound = append(rebound, b)
		}
	}

	return string(rebound), names, err
}

// BindNamed binds a struct or a map to a query with named parameters.
// DEPRECATED: use sqlx.Named` instead of this, it may be removed in future.
func BindNamed(bindType int, query string, arg interface{}) (string, []interface{}, error) {
	return bindNamedMapper(bindType, query, arg, mapper())
}

// Named takes a query using named parameters and an argument and
// returns a new query with a list of args that can be executed by
// a database.  The return value uses the `?` bindvar.
func Named(query string, arg interface{}) (string, []interface{}, error) {
	return bindNamedMapper(QUESTION, query, arg, mapper())
}

func bindNamedMapper(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) {
	if maparg, ok := arg.(map[string]interface{}); ok {
		return bindMap(bindType, query, maparg)
	}
	return bindStruct(bindType, query, arg, m)
}

// NamedQuery binds a named query and then runs Query on the result using the
// provided Ext (sqlx.Tx, sqlx.Db).  It works with both structs and with
// map[string]interface{} types.
func NamedQuery(e Ext, query string, arg interface{}) (*Rows, error) {
	q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e))
	if err != nil {
		return nil, err
	}
	return e.Queryx(q, args...)
}

// NamedExec uses BindStruct to get a query executable by the driver and
// then runs Exec on the result.  Returns an error from the binding
// or the query excution itself.
func NamedExec(e Ext, query string, arg interface{}) (sql.Result, error) {
	q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e))
	if err != nil {
		return nil, err
	}
	return e.Exec(q, args...)
}
<commit_msg>Use TraversalsByNameFunc for binding named arguments<commit_after>package sqlx

// Named Query Support
//
//  * BindMap - bind query bindvars to map/struct args
//	* NamedExec, NamedQuery - named query w/ struct or map
//  * NamedStmt - a pre-compiled named query which is a prepared statement
//
// Internal Interfaces:
//
//  * compileNamedQuery - rebind a named query, returning a query and list of names
//  * bindArgs, bindMapArgs, bindAnyArgs - given a list of names, return an arglist
//
import (
	"database/sql"
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"unicode"

	"github.com/jmoiron/sqlx/reflectx"
)

// NamedStmt is a prepared statement that executes named queries.  Prepare it
// how you would execute a NamedQuery, but pass in a struct or map when executing.
type NamedStmt struct {
	Params      []string
	QueryString string
	Stmt        *Stmt
}

// Close closes the named statement.
func (n *NamedStmt) Close() error {
	return n.Stmt.Close()
}

// Exec executes a named statement using the struct passed.
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Exec(arg interface{}) (sql.Result, error) {
	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
	if err != nil {
		return *new(sql.Result), err
	}
	return n.Stmt.Exec(args...)
}

// Query executes a named statement using the struct argument, returning rows.
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Query(arg interface{}) (*sql.Rows, error) {
	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
	if err != nil {
		return nil, err
	}
	return n.Stmt.Query(args...)
}

// QueryRow executes a named statement against the database.  Because sqlx cannot
// create a *sql.Row with an error condition pre-set for binding errors, sqlx
// returns a *sqlx.Row instead.
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) QueryRow(arg interface{}) *Row {
	args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper)
	if err != nil {
		return &Row{err: err}
	}
	return n.Stmt.QueryRowx(args...)
}

// MustExec execs a NamedStmt, panicing on error
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) MustExec(arg interface{}) sql.Result {
	res, err := n.Exec(arg)
	if err != nil {
		panic(err)
	}
	return res
}

// Queryx using this NamedStmt
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Queryx(arg interface{}) (*Rows, error) {
	r, err := n.Query(arg)
	if err != nil {
		return nil, err
	}
	return &Rows{Rows: r, Mapper: n.Stmt.Mapper, unsafe: isUnsafe(n)}, err
}

// QueryRowx this NamedStmt.  Because of limitations with QueryRow, this is
// an alias for QueryRow.
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) QueryRowx(arg interface{}) *Row {
	return n.QueryRow(arg)
}

// Select using this NamedStmt
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Select(dest interface{}, arg interface{}) error {
	rows, err := n.Queryx(arg)
	if err != nil {
		return err
	}
	// if something happens here, we want to make sure the rows are Closed
	defer rows.Close()
	return scanAll(rows, dest, false)
}

// Get using this NamedStmt
// Any named placeholder parameters are replaced with fields from arg.
func (n *NamedStmt) Get(dest interface{}, arg interface{}) error {
	r := n.QueryRowx(arg)
	return r.scanAny(dest, false)
}

// Unsafe creates an unsafe version of the NamedStmt
func (n *NamedStmt) Unsafe() *NamedStmt {
	r := &NamedStmt{Params: n.Params, Stmt: n.Stmt, QueryString: n.QueryString}
	r.Stmt.unsafe = true
	return r
}

// A union interface of preparer and binder, required to be able to prepare
// named statements (as the bindtype must be determined).
type namedPreparer interface {
	Preparer
	binder
}

func prepareNamed(p namedPreparer, query string) (*NamedStmt, error) {
	bindType := BindType(p.DriverName())
	q, args, err := compileNamedQuery([]byte(query), bindType)
	if err != nil {
		return nil, err
	}
	stmt, err := Preparex(p, q)
	if err != nil {
		return nil, err
	}
	return &NamedStmt{
		QueryString: q,
		Params:      args,
		Stmt:        stmt,
	}, nil
}

func bindAnyArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) {
	if maparg, ok := arg.(map[string]interface{}); ok {
		return bindMapArgs(names, maparg)
	}
	return bindArgs(names, arg, m)
}

// private interface to generate a list of interfaces from a given struct
// type, given a list of names to pull out of the struct.  Used by public
// BindStruct interface.
func bindArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) {
	arglist := make([]interface{}, 0, len(names))

	// grab the indirected value of arg
	v := reflect.ValueOf(arg)
	for v = reflect.ValueOf(arg); v.Kind() == reflect.Ptr; {
		v = v.Elem()
	}

	err := m.TraversalsByNameFunc(v.Type(), names, func(i int, t []int) error {
		if len(t) == 0 {
			return fmt.Errorf("could not find name %s in %#v", names[i], arg)
		}

		val := reflectx.FieldByIndexesReadOnly(v, t)
		arglist = append(arglist, val.Interface())

		return nil
	})

	return arglist, err
}

// like bindArgs, but for maps.
func bindMapArgs(names []string, arg map[string]interface{}) ([]interface{}, error) {
	arglist := make([]interface{}, 0, len(names))

	for _, name := range names {
		val, ok := arg[name]
		if !ok {
			return arglist, fmt.Errorf("could not find name %s in %#v", name, arg)
		}
		arglist = append(arglist, val)
	}
	return arglist, nil
}

// bindStruct binds a named parameter query with fields from a struct argument.
// The rules for binding field names to parameter names follow the same
// conventions as for StructScan, including obeying the `db` struct tags.
func bindStruct(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) {
	bound, names, err := compileNamedQuery([]byte(query), bindType)
	if err != nil {
		return "", []interface{}{}, err
	}

	arglist, err := bindArgs(names, arg, m)
	if err != nil {
		return "", []interface{}{}, err
	}

	return bound, arglist, nil
}

// bindMap binds a named parameter query with a map of arguments.
func bindMap(bindType int, query string, args map[string]interface{}) (string, []interface{}, error) {
	bound, names, err := compileNamedQuery([]byte(query), bindType)
	if err != nil {
		return "", []interface{}{}, err
	}

	arglist, err := bindMapArgs(names, args)
	return bound, arglist, err
}

// -- Compilation of Named Queries

// Allow digits and letters in bind params;  additionally runes are
// checked against underscores, meaning that bind params can have be
// alphanumeric with underscores.  Mind the difference between unicode
// digits and numbers, where '5' is a digit but '五' is not.
var allowedBindRunes = []*unicode.RangeTable{unicode.Letter, unicode.Digit}

// FIXME: this function isn't safe for unicode named params, as a failing test
// can testify.  This is not a regression but a failure of the original code
// as well.  It should be modified to range over runes in a string rather than
// bytes, even though this is less convenient and slower.  Hopefully the
// addition of the prepared NamedStmt (which will only do this once) will make
// up for the slightly slower ad-hoc NamedExec/NamedQuery.

// compile a NamedQuery into an unbound query (using the '?' bindvar) and
// a list of names.
func compileNamedQuery(qs []byte, bindType int) (query string, names []string, err error) {
	names = make([]string, 0, 10)
	rebound := make([]byte, 0, len(qs))

	inName := false
	last := len(qs) - 1
	currentVar := 1
	name := make([]byte, 0, 10)

	for i, b := range qs {
		// a ':' while we're in a name is an error
		if b == ':' {
			// if this is the second ':' in a '::' escape sequence, append a ':'
			if inName && i > 0 && qs[i-1] == ':' {
				rebound = append(rebound, ':')
				inName = false
				continue
			} else if inName {
				err = errors.New("unexpected `:` while reading named param at " + strconv.Itoa(i))
				return query, names, err
			}
			inName = true
			name = []byte{}
			// if we're in a name, and this is an allowed character, continue
		} else if inName && (unicode.IsOneOf(allowedBindRunes, rune(b)) || b == '_' || b == '.') && i != last {
			// append the byte to the name if we are in a name and not on the last byte
			name = append(name, b)
			// if we're in a name and it's not an allowed character, the name is done
		} else if inName {
			inName = false
			// if this is the final byte of the string and it is part of the name, then
			// make sure to add it to the name
			if i == last && unicode.IsOneOf(allowedBindRunes, rune(b)) {
				name = append(name, b)
			}
			// add the string representation to the names list
			names = append(names, string(name))
			// add a proper bindvar for the bindType
			switch bindType {
			// oracle only supports named type bind vars even for positional
			case NAMED:
				rebound = append(rebound, ':')
				rebound = append(rebound, name...)
			case QUESTION, UNKNOWN:
				rebound = append(rebound, '?')
			case DOLLAR:
				rebound = append(rebound, '$')
				for _, b := range strconv.Itoa(currentVar) {
					rebound = append(rebound, byte(b))
				}
				currentVar++
			}
			// add this byte to string unless it was not part of the name
			if i != last {
				rebound = append(rebound, b)
			} else if !unicode.IsOneOf(allowedBindRunes, rune(b)) {
				rebound = append(rebound, b)
			}
		} else {
			// this is a normal byte and should just go onto the rebound query
			rebound = append(rebound, b)
		}
	}

	return string(rebound), names, err
}

// BindNamed binds a struct or a map to a query with named parameters.
// DEPRECATED: use sqlx.Named` instead of this, it may be removed in future.
func BindNamed(bindType int, query string, arg interface{}) (string, []interface{}, error) {
	return bindNamedMapper(bindType, query, arg, mapper())
}

// Named takes a query using named parameters and an argument and
// returns a new query with a list of args that can be executed by
// a database.  The return value uses the `?` bindvar.
func Named(query string, arg interface{}) (string, []interface{}, error) {
	return bindNamedMapper(QUESTION, query, arg, mapper())
}

func bindNamedMapper(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) {
	if maparg, ok := arg.(map[string]interface{}); ok {
		return bindMap(bindType, query, maparg)
	}
	return bindStruct(bindType, query, arg, m)
}

// NamedQuery binds a named query and then runs Query on the result using the
// provided Ext (sqlx.Tx, sqlx.Db).  It works with both structs and with
// map[string]interface{} types.
func NamedQuery(e Ext, query string, arg interface{}) (*Rows, error) {
	q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e))
	if err != nil {
		return nil, err
	}
	return e.Queryx(q, args...)
}

// NamedExec uses BindStruct to get a query executable by the driver and
// then runs Exec on the result.  Returns an error from the binding
// or the query excution itself.
func NamedExec(e Ext, query string, arg interface{}) (sql.Result, error) {
	q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e))
	if err != nil {
		return nil, err
	}
	return e.Exec(q, args...)
}
<|endoftext|>


<commit_before>package reporting

import (
	"runtime"
	"strings"
)

type Report struct {
	File       string
	Line       int
	Failure    string
	Error      interface{}
	stackTrace string
}

func NewFailureReport(failure string) *Report {
	file, line := caller()
	stack := stackTrace()
	report := Report{file, line, failure, nil, stack}
	return &report
}
func NewErrorReport(err interface{}) *Report {
	file, line := caller()
	stack := fullStackTrace()
	report := Report{file, line, "", err, stack}
	return &report
}
func NewSuccessReport() *Report {
	file, line := caller()
	stack := stackTrace()
	report := Report{file, line, "", nil, stack}
	return &report
}

func caller() (file string, line int) {
	// TODO: what if they have extracted the So() call into a helper method?
	//       (runtime.Caller(3) will not yield the correct stack entry!)
	_, file, line, _ = runtime.Caller(3)
	return
}
func stackTrace() string {
	buffer := make([]byte, 1024*64)
	runtime.Stack(buffer, false)
	formatted := strings.Trim(string(buffer), string([]byte{0}))
	return filterStack(formatted)
}
func fullStackTrace() string {
	buffer := make([]byte, 1024*64)
	runtime.Stack(buffer, true)
	formatted := strings.Trim(string(buffer), string([]byte{0}))
	return filterStack(formatted)
}
func filterStack(stack string) string {
	lines := strings.Split(stack, newline)
	filtered := []string{}
	for _, line := range lines {
		// if isExternal(line) {
		filtered = append(filtered, line)
		// }
	}
	return strings.Join(filtered, newline)
}
func isExternal(line string) bool {
	if strings.Contains(line, "goconvey/convey") {
		return false
	} else if strings.Contains(line, "goconvey/execution") {
		return false
	} else if strings.Contains(line, "goconvey/gotest") {
		return false
	} else if strings.Contains(line, "goconvey/printing") {
		return false
	} else if strings.Contains(line, "goconvey/reporting") {
		return false
	}
	return true
}

const newline = "\n"
const success = "✓"
const failure = "✗"
const error_ = "🔥"
const errorTemplate = "* %s \n* Line: %d - %v \n%s"
const failureTemplate = "* %s \n* Line %d: %s"
<commit_msg>This TODO was actually not needed.<commit_after>package reporting

import (
	"runtime"
	"strings"
)

type Report struct {
	File       string
	Line       int
	Failure    string
	Error      interface{}
	stackTrace string
}

func NewFailureReport(failure string) *Report {
	file, line := caller()
	stack := stackTrace()
	report := Report{file, line, failure, nil, stack}
	return &report
}
func NewErrorReport(err interface{}) *Report {
	file, line := caller()
	stack := fullStackTrace()
	report := Report{file, line, "", err, stack}
	return &report
}
func NewSuccessReport() *Report {
	file, line := caller()
	stack := stackTrace()
	report := Report{file, line, "", nil, stack}
	return &report
}

func caller() (file string, line int) {
	_, file, line, _ = runtime.Caller(3)
	return
}
func stackTrace() string {
	buffer := make([]byte, 1024*64)
	runtime.Stack(buffer, false)
	formatted := strings.Trim(string(buffer), string([]byte{0}))
	return filterStack(formatted)
}
func fullStackTrace() string {
	buffer := make([]byte, 1024*64)
	runtime.Stack(buffer, true)
	formatted := strings.Trim(string(buffer), string([]byte{0}))
	return filterStack(formatted)
}
func filterStack(stack string) string {
	lines := strings.Split(stack, newline)
	filtered := []string{}
	for _, line := range lines {
		// if isExternal(line) {
		filtered = append(filtered, line)
		// }
	}
	return strings.Join(filtered, newline)
}
func isExternal(line string) bool {
	if strings.Contains(line, "goconvey/convey") {
		return false
	} else if strings.Contains(line, "goconvey/execution") {
		return false
	} else if strings.Contains(line, "goconvey/gotest") {
		return false
	} else if strings.Contains(line, "goconvey/printing") {
		return false
	} else if strings.Contains(line, "goconvey/reporting") {
		return false
	}
	return true
}

const newline = "\n"
const success = "✓"
const failure = "✗"
const error_ = "🔥"
const errorTemplate = "* %s \n* Line: %d - %v \n%s"
const failureTemplate = "* %s \n* Line %d: %s"
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package cni

import (
	"fmt"
	"os"
	"sort"
	"strings"

	cnilibrary "github.com/containernetworking/cni/libcni"
	"github.com/containernetworking/cni/pkg/invoke"
	"github.com/containernetworking/cni/pkg/version"
)

// Opt sets options for a CNI instance
type Opt func(c *libcni) error

// WithInterfacePrefix sets the prefix for network interfaces
// e.g. eth or wlan
func WithInterfacePrefix(prefix string) Opt {
	return func(c *libcni) error {
		c.prefix = prefix
		return nil
	}
}

// WithPluginDir can be used to set the locations of
// the cni plugin binaries
func WithPluginDir(dirs []string) Opt {
	return func(c *libcni) error {
		c.pluginDirs = dirs
		c.cniConfig = cnilibrary.NewCNIConfig(
			dirs,
			&invoke.DefaultExec{
				RawExec:       &invoke.RawExec{Stderr: os.Stderr},
				PluginDecoder: version.PluginDecoder{},
			},
		)
		return nil
	}
}

// WithPluginConfDir can be used to configure the
// cni configuration directory.
func WithPluginConfDir(dir string) Opt {
	return func(c *libcni) error {
		c.pluginConfDir = dir
		return nil
	}
}

// WithPluginMaxConfNum can be used to configure the
// max cni plugin config file num.
func WithPluginMaxConfNum(max int) Opt {
	return func(c *libcni) error {
		c.pluginMaxConfNum = max
		return nil
	}
}

// WithMinNetworkCount can be used to configure the
// minimum networks to be configured and initialized
// for the status to report success. By default its 1.
func WithMinNetworkCount(count int) Opt {
	return func(c *libcni) error {
		c.networkCount = count
		return nil
	}
}

// WithLoNetwork can be used to load the loopback
// network config.
func WithLoNetwork(c *libcni) error {
	loConfig, _ := cnilibrary.ConfListFromBytes([]byte(`{
"cniVersion": "1.0.0",
"name": "cni-loopback",
"plugins": [{
  "type": "loopback"
}]
}`))

	c.networks = append(c.networks, &Network{
		cni:    c.cniConfig,
		config: loConfig,
		ifName: "lo",
	})
	return nil
}

// WithConf can be used to load config directly
// from byte.
func WithConf(bytes []byte) Opt {
	return WithConfIndex(bytes, 0)
}

// WithConfIndex can be used to load config directly
// from byte and set the interface name's index.
func WithConfIndex(bytes []byte, index int) Opt {
	return func(c *libcni) error {
		conf, err := cnilibrary.ConfFromBytes(bytes)
		if err != nil {
			return err
		}
		confList, err := cnilibrary.ConfListFromConf(conf)
		if err != nil {
			return err
		}
		c.networks = append(c.networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, index),
		})
		return nil
	}
}

// WithConfFile can be used to load network config
// from an .conf file. Supported with absolute fileName
// with path only.
func WithConfFile(fileName string) Opt {
	return func(c *libcni) error {
		conf, err := cnilibrary.ConfFromFile(fileName)
		if err != nil {
			return err
		}
		// upconvert to conf list
		confList, err := cnilibrary.ConfListFromConf(conf)
		if err != nil {
			return err
		}
		c.networks = append(c.networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, 0),
		})
		return nil
	}
}

// WithConfListBytes can be used to load network config list directly
// from byte
func WithConfListBytes(bytes []byte) Opt {
	return func(c *libcni) error {
		confList, err := cnilibrary.ConfListFromBytes(bytes)
		if err != nil {
			return err
		}
		i := len(c.networks)
		c.networks = append(c.networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, i),
		})
		return nil
	}
}

// WithConfListFile can be used to load network config
// from an .conflist file. Supported with absolute fileName
// with path only.
func WithConfListFile(fileName string) Opt {
	return func(c *libcni) error {
		confList, err := cnilibrary.ConfListFromFile(fileName)
		if err != nil {
			return err
		}
		i := len(c.networks)
		c.networks = append(c.networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, i),
		})
		return nil
	}
}

// WithDefaultConf can be used to detect the default network
// config file from the configured cni config directory and load
// it.
// Since the CNI spec does not specify a way to detect default networks,
// the convention chosen is - the first network configuration in the sorted
// list of network conf files as the default network.
func WithDefaultConf(c *libcni) error {
	return loadFromConfDir(c, c.pluginMaxConfNum)
}

// WithAllConf can be used to detect all network config
// files from the configured cni config directory and load
// them.
func WithAllConf(c *libcni) error {
	return loadFromConfDir(c, 0)
}

// loadFromConfDir detects network config files from the
// configured cni config directory and load them. max is
// the maximum network config to load (max i<= 0 means no limit).
func loadFromConfDir(c *libcni, max int) error {
	files, err := cnilibrary.ConfFiles(c.pluginConfDir, []string{".conf", ".conflist", ".json"})
	switch {
	case err != nil:
		return fmt.Errorf("failed to read config file: %v: %w", err, ErrRead)
	case len(files) == 0:
		return fmt.Errorf("no network config found in %s: %w", c.pluginConfDir, ErrCNINotInitialized)
	}

	// files contains the network config files associated with cni network.
	// Use lexicographical way as a defined order for network config files.
	sort.Strings(files)
	// Since the CNI spec does not specify a way to detect default networks,
	// the convention chosen is - the first network configuration in the sorted
	// list of network conf files as the default network and choose the default
	// interface provided during init as the network interface for this default
	// network. For every other network use a generated interface id.
	i := 0
	var networks []*Network
	for _, confFile := range files {
		var confList *cnilibrary.NetworkConfigList
		if strings.HasSuffix(confFile, ".conflist") {
			confList, err = cnilibrary.ConfListFromFile(confFile)
			if err != nil {
				return fmt.Errorf("failed to load CNI config list file %s: %v: %w", confFile, err, ErrInvalidConfig)
			}
		} else {
			conf, err := cnilibrary.ConfFromFile(confFile)
			if err != nil {
				return fmt.Errorf("failed to load CNI config file %s: %v: %w", confFile, err, ErrInvalidConfig)
			}
			// Ensure the config has a "type" so we know what plugin to run.
			// Also catches the case where somebody put a conflist into a conf file.
			if conf.Network.Type == "" {
				return fmt.Errorf("network type not found in %s: %w", confFile, ErrInvalidConfig)
			}

			confList, err = cnilibrary.ConfListFromConf(conf)
			if err != nil {
				return fmt.Errorf("failed to convert CNI config file %s to CNI config list: %v: %w", confFile, err, ErrInvalidConfig)
			}
		}
		if len(confList.Plugins) == 0 {
			return fmt.Errorf("CNI config list in config file %s has no networks, skipping: %w", confFile, ErrInvalidConfig)

		}
		networks = append(networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, i),
		})
		i++
		if i == max {
			break
		}
	}
	if len(networks) == 0 {
		return fmt.Errorf("no valid networks found in %s: %w", c.pluginDirs, ErrCNINotInitialized)
	}
	c.networks = append(c.networks, networks...)
	return nil
}
<commit_msg>Revert "Update loopback version to support check"<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package cni

import (
	"fmt"
	"os"
	"sort"
	"strings"

	cnilibrary "github.com/containernetworking/cni/libcni"
	"github.com/containernetworking/cni/pkg/invoke"
	"github.com/containernetworking/cni/pkg/version"
)

// Opt sets options for a CNI instance
type Opt func(c *libcni) error

// WithInterfacePrefix sets the prefix for network interfaces
// e.g. eth or wlan
func WithInterfacePrefix(prefix string) Opt {
	return func(c *libcni) error {
		c.prefix = prefix
		return nil
	}
}

// WithPluginDir can be used to set the locations of
// the cni plugin binaries
func WithPluginDir(dirs []string) Opt {
	return func(c *libcni) error {
		c.pluginDirs = dirs
		c.cniConfig = cnilibrary.NewCNIConfig(
			dirs,
			&invoke.DefaultExec{
				RawExec:       &invoke.RawExec{Stderr: os.Stderr},
				PluginDecoder: version.PluginDecoder{},
			},
		)
		return nil
	}
}

// WithPluginConfDir can be used to configure the
// cni configuration directory.
func WithPluginConfDir(dir string) Opt {
	return func(c *libcni) error {
		c.pluginConfDir = dir
		return nil
	}
}

// WithPluginMaxConfNum can be used to configure the
// max cni plugin config file num.
func WithPluginMaxConfNum(max int) Opt {
	return func(c *libcni) error {
		c.pluginMaxConfNum = max
		return nil
	}
}

// WithMinNetworkCount can be used to configure the
// minimum networks to be configured and initialized
// for the status to report success. By default its 1.
func WithMinNetworkCount(count int) Opt {
	return func(c *libcni) error {
		c.networkCount = count
		return nil
	}
}

// WithLoNetwork can be used to load the loopback
// network config.
func WithLoNetwork(c *libcni) error {
	loConfig, _ := cnilibrary.ConfListFromBytes([]byte(`{
"cniVersion": "0.3.1",
"name": "cni-loopback",
"plugins": [{
  "type": "loopback"
}]
}`))

	c.networks = append(c.networks, &Network{
		cni:    c.cniConfig,
		config: loConfig,
		ifName: "lo",
	})
	return nil
}

// WithConf can be used to load config directly
// from byte.
func WithConf(bytes []byte) Opt {
	return WithConfIndex(bytes, 0)
}

// WithConfIndex can be used to load config directly
// from byte and set the interface name's index.
func WithConfIndex(bytes []byte, index int) Opt {
	return func(c *libcni) error {
		conf, err := cnilibrary.ConfFromBytes(bytes)
		if err != nil {
			return err
		}
		confList, err := cnilibrary.ConfListFromConf(conf)
		if err != nil {
			return err
		}
		c.networks = append(c.networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, index),
		})
		return nil
	}
}

// WithConfFile can be used to load network config
// from an .conf file. Supported with absolute fileName
// with path only.
func WithConfFile(fileName string) Opt {
	return func(c *libcni) error {
		conf, err := cnilibrary.ConfFromFile(fileName)
		if err != nil {
			return err
		}
		// upconvert to conf list
		confList, err := cnilibrary.ConfListFromConf(conf)
		if err != nil {
			return err
		}
		c.networks = append(c.networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, 0),
		})
		return nil
	}
}

// WithConfListBytes can be used to load network config list directly
// from byte
func WithConfListBytes(bytes []byte) Opt {
	return func(c *libcni) error {
		confList, err := cnilibrary.ConfListFromBytes(bytes)
		if err != nil {
			return err
		}
		i := len(c.networks)
		c.networks = append(c.networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, i),
		})
		return nil
	}
}

// WithConfListFile can be used to load network config
// from an .conflist file. Supported with absolute fileName
// with path only.
func WithConfListFile(fileName string) Opt {
	return func(c *libcni) error {
		confList, err := cnilibrary.ConfListFromFile(fileName)
		if err != nil {
			return err
		}
		i := len(c.networks)
		c.networks = append(c.networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, i),
		})
		return nil
	}
}

// WithDefaultConf can be used to detect the default network
// config file from the configured cni config directory and load
// it.
// Since the CNI spec does not specify a way to detect default networks,
// the convention chosen is - the first network configuration in the sorted
// list of network conf files as the default network.
func WithDefaultConf(c *libcni) error {
	return loadFromConfDir(c, c.pluginMaxConfNum)
}

// WithAllConf can be used to detect all network config
// files from the configured cni config directory and load
// them.
func WithAllConf(c *libcni) error {
	return loadFromConfDir(c, 0)
}

// loadFromConfDir detects network config files from the
// configured cni config directory and load them. max is
// the maximum network config to load (max i<= 0 means no limit).
func loadFromConfDir(c *libcni, max int) error {
	files, err := cnilibrary.ConfFiles(c.pluginConfDir, []string{".conf", ".conflist", ".json"})
	switch {
	case err != nil:
		return fmt.Errorf("failed to read config file: %v: %w", err, ErrRead)
	case len(files) == 0:
		return fmt.Errorf("no network config found in %s: %w", c.pluginConfDir, ErrCNINotInitialized)
	}

	// files contains the network config files associated with cni network.
	// Use lexicographical way as a defined order for network config files.
	sort.Strings(files)
	// Since the CNI spec does not specify a way to detect default networks,
	// the convention chosen is - the first network configuration in the sorted
	// list of network conf files as the default network and choose the default
	// interface provided during init as the network interface for this default
	// network. For every other network use a generated interface id.
	i := 0
	var networks []*Network
	for _, confFile := range files {
		var confList *cnilibrary.NetworkConfigList
		if strings.HasSuffix(confFile, ".conflist") {
			confList, err = cnilibrary.ConfListFromFile(confFile)
			if err != nil {
				return fmt.Errorf("failed to load CNI config list file %s: %v: %w", confFile, err, ErrInvalidConfig)
			}
		} else {
			conf, err := cnilibrary.ConfFromFile(confFile)
			if err != nil {
				return fmt.Errorf("failed to load CNI config file %s: %v: %w", confFile, err, ErrInvalidConfig)
			}
			// Ensure the config has a "type" so we know what plugin to run.
			// Also catches the case where somebody put a conflist into a conf file.
			if conf.Network.Type == "" {
				return fmt.Errorf("network type not found in %s: %w", confFile, ErrInvalidConfig)
			}

			confList, err = cnilibrary.ConfListFromConf(conf)
			if err != nil {
				return fmt.Errorf("failed to convert CNI config file %s to CNI config list: %v: %w", confFile, err, ErrInvalidConfig)
			}
		}
		if len(confList.Plugins) == 0 {
			return fmt.Errorf("CNI config list in config file %s has no networks, skipping: %w", confFile, ErrInvalidConfig)

		}
		networks = append(networks, &Network{
			cni:    c.cniConfig,
			config: confList,
			ifName: getIfName(c.prefix, i),
		})
		i++
		if i == max {
			break
		}
	}
	if len(networks) == 0 {
		return fmt.Errorf("no valid networks found in %s: %w", c.pluginDirs, ErrCNINotInitialized)
	}
	c.networks = append(c.networks, networks...)
	return nil
}
<|endoftext|>


<commit_before>// Radix -- Asynchronous Redis client for Go.
//
// Simple client for accessing the Redis database. After establishing a connection
// with NewRedisDatabase() commands can be executed with Command(). So every command
// is possible. The method returns a ResultSet with different methods for success
// testing and access to the retrieved values. The method MultiCommand() can be used
// for transactions. The passed function gets a MultiCommand instance as argument for
// calling the inner Command() methods.
package redis
<commit_msg>doc update<commit_after>// Package redis implements an asynchronous Redis client.
//
// Simple client for accessing the Redis database. After establishing a connection
// with NewRedisDatabase() commands can be executed with Command(). So every command
// is possible. The method returns a ResultSet with different methods for success
// testing and access to the retrieved values. The method MultiCommand() can be used
// for transactions. The passed function gets a MultiCommand instance as argument for
// calling the inner Command() methods.
package redis
<|endoftext|>


<commit_before>package ngorm

// DB contains information for current db connection
type DB struct {
}
<commit_msg>[godoc] Add package description<commit_after>//Package ngorm is a Go Object relation mapper that foucs on performance,
//maintainability, modularity,	battle testing, extensibility , safety and
//developer frindliness.
//
// To achieve all of the goals, the project is divided into many components. The
// components are desined in a functional style API, whereby objects are
// explicitly passed around as arguments to functions that operate on them.
//
// This tries to avoid defining methods on structs. This comes at a cost of
// limiting chaining, this cost is intential. I intend to work really hard on
// improving perfomance and thus avoiding spaghetti is not an option.
//
// The package is divided into two phases, Query building and Query execution
// phase.
//
// Query Building
//
// The subpackage engine exposes a structure named Engine. This structure has
// everything necessary to build a query. Most of the functions defined in this
// package subpackages operate on this struct by accepting it as the first
// argument.
//
// Having this as a separate layer helps fine tuning the generated querries and
// also it make easy to test and very that the ORM is doing the right thing. So,
// the generated query can be easily optimised without adding a lot of overhead.
//
// Query execution
//
// This is where the generated sql qery is executed. This phase is as generic as
// possible in a way that you can easily implement adoptes for non SQL database
// and still reap all the benefits of this package.
package ngorm

// DB contains information for current db connection
type DB struct {
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package repr contains functions useful for serializing and deserializing
// directory listing structs.
package repr

import (
	"code.google.com/p/goprotobuf/proto"
	"fmt"
	"github.com/jacobsa/comeback/fs"
)

func convertType(t fs.EntryType) DirectoryEntryProto_Type {
	switch t {
	case fs.TypeFile:
		return DirectoryEntryProto_TYPE_FILE
	case fs.TypeDirectory:
		return DirectoryEntryProto_TYPE_DIRECTORY
	case fs.TypeSymlink:
		return DirectoryEntryProto_TYPE_SYMLINK
	}

	panic(fmt.Sprintf("Unrecognized EntryType: %v", t))
}

func makeEntryProto(entry fs.DirectoryEntry) *DirectoryEntryProto {
	blobs := []*BlobInfoProto{}
	for _, score := range entry.Scores {
		proto := &BlobInfoProto{Hash: score.Sha1Hash()}
		blobs = append(blobs, proto)
	}

	return &DirectoryEntryProto{
		Type:        convertType(entry.Type).Enum(),
		Permissions: proto.Uint32(entry.Permissions),
		Name:        proto.String(entry.Name),
		Mtime: &TimeProto{
			Second:     proto.Int64(entry.MTime.Unix()),
			Nanosecond: proto.Uint32(uint32(entry.MTime.Nanosecond())),
		},
		Blob: blobs,
	}
}

// Marshal turns a list of directory entries into bytes that can later be used
// with Unmarshal.
func Marshal(entries []fs.DirectoryEntry) (d []byte, err error) {
	return nil, fmt.Errorf("TODO")
}

// Unmarshal recovers a list of directory entries from bytes previously
// returned by Marshal.
func Unmarshal(d []byte) (entries []fs.DirectoryEntry, err error) {
	return nil, fmt.Errorf("TODO")
}
<commit_msg>Implemented Marshal.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package repr contains functions useful for serializing and deserializing
// directory listing structs.
package repr

import (
	"code.google.com/p/goprotobuf/proto"
	"fmt"
	"github.com/jacobsa/comeback/fs"
)

func convertType(t fs.EntryType) DirectoryEntryProto_Type {
	switch t {
	case fs.TypeFile:
		return DirectoryEntryProto_TYPE_FILE
	case fs.TypeDirectory:
		return DirectoryEntryProto_TYPE_DIRECTORY
	case fs.TypeSymlink:
		return DirectoryEntryProto_TYPE_SYMLINK
	}

	panic(fmt.Sprintf("Unrecognized EntryType: %v", t))
}

func makeEntryProto(entry fs.DirectoryEntry) *DirectoryEntryProto {
	blobs := []*BlobInfoProto{}
	for _, score := range entry.Scores {
		proto := &BlobInfoProto{Hash: score.Sha1Hash()}
		blobs = append(blobs, proto)
	}

	return &DirectoryEntryProto{
		Type:        convertType(entry.Type).Enum(),
		Permissions: proto.Uint32(entry.Permissions),
		Name:        proto.String(entry.Name),
		Mtime: &TimeProto{
			Second:     proto.Int64(entry.MTime.Unix()),
			Nanosecond: proto.Uint32(uint32(entry.MTime.Nanosecond())),
		},
		Blob: blobs,
	}
}

// Marshal turns a list of directory entries into bytes that can later be used
// with Unmarshal.
func Marshal(entries []fs.DirectoryEntry) (d []byte, err error) {
	entryProtos := []*DirectoryEntryProto{}
	for _, entry := range entries {
		entryProtos = append(entryProtos, makeEntryProto(entry))
	}

	listingProto := &DirectoryListingProto{Entry: entryProtos}
	return proto.Marshal(listingProto)
}

// Unmarshal recovers a list of directory entries from bytes previously
// returned by Marshal.
func Unmarshal(d []byte) (entries []fs.DirectoryEntry, err error) {
	return nil, fmt.Errorf("TODO")
}
<|endoftext|>