<commit_before>package main

import (
	"bitbucket.org/ant512/gobble/akismet"
	"errors"
	"fmt"
	"github.com/russross/blackfriday"
	"html"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

type FileRepository struct {
	directory string
	posts     BlogPosts
	tags      map[string]int
	mutex     sync.RWMutex
}

func NewFileRepository(directory string) *FileRepository {

	f := new(FileRepository)
	f.directory = directory

	go f.update()

	return f
}

func (f *FileRepository) AllTags() map[string]int {
	return f.tags
}

func (f *FileRepository) AllPosts() BlogPosts {
	return f.posts
}

func (f *FileRepository) PostWithUrl(url string) (*BlogPost, error) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	for i := range f.posts {
		if f.posts[i].Url() == url {
			return f.posts[i], nil
		}
	}

	err := errors.New("Could not find post")

	return nil, err
}

func (f *FileRepository) PostWithId(id int) (*BlogPost, error) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	for i := range f.posts {
		if f.posts[i].Id == id {
			return f.posts[i], nil
		}
	}

	err := errors.New("Could not find post")

	return nil, err
}

func (f *FileRepository) PostsWithTag(tag string, start int, count int) (BlogPosts, int) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	filteredPosts := BlogPosts{}

	for i := range f.posts {
		if f.posts[i].ContainsTag(tag) {
			filteredPosts = append(filteredPosts, f.posts[i])
		}
	}

	if start > len(filteredPosts) {
		return BlogPosts{}, 0
	}

	if start+count > len(filteredPosts) {
		count = len(filteredPosts) - start
	}

	return filteredPosts[start : start+count], len(filteredPosts)
}

func (f *FileRepository) SearchPosts(term string, start int, count int) (BlogPosts, int) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	filteredPosts := BlogPosts{}

	if len(term) > 0 {
		for i := range f.posts {
			if f.posts[i].ContainsTerm(term) {
				filteredPosts = append(filteredPosts, f.posts[i])
			}
		}
	} else {
		filteredPosts = f.posts
	}

	if start > len(filteredPosts) {
		return BlogPosts{}, 0
	}

	if start+count > len(filteredPosts) {
		count = len(filteredPosts) - start
	}

	return filteredPosts[start : start+count], len(filteredPosts)
}

func (f *FileRepository) SaveComment(post *BlogPost, akismetAPIKey, serverAddress, remoteAddress, userAgent, referer, author, email, body string) {

	// TODO: Ensure file name is unique
	isSpam, _ := akismet.IsSpamComment(body, serverAddress, remoteAddress, userAgent, referer, author, email, akismetAPIKey)

	comment := new(Comment)

	comment.Author = html.EscapeString(author)
	comment.Email = html.EscapeString(email)
	comment.Date = time.Now()
	comment.Body = html.EscapeString(body)
	comment.IsSpam = isSpam

	f.mutex.Lock()
	post.Comments = append(post.Comments, comment)
	f.mutex.Unlock()

	postPath := post.FilePath[:len(post.FilePath)-3]

	dirname := postPath + string(filepath.Separator) + "comments" + string(filepath.Separator)

	filename := timeToFilename(comment.Date)

	log.Println(dirname + filename)
	os.MkdirAll(dirname, 0775)

	content := "Author: " + comment.Author + "\n"
	content += "Email: " + comment.Email + "\n"
	content += "Date: " + timeToString(comment.Date) + "\n"

	if isSpam {
		content += "Spam: true\n"
	}

	content += "\n"

	content += comment.Body

	err := ioutil.WriteFile(dirname+filename, []byte(content), 0644)

	if err != nil {
		log.Println(err)
	}
}

func (f *FileRepository) update() {
	for {
		start := time.Now()

		f.fetchAllPosts()
		f.fetchAllTags()

		end := time.Now()
		log.Printf("Cached %v posts in %v", len(f.posts), end.Sub(start))

		time.Sleep(10 * time.Minute)
	}
}

func (f *FileRepository) fetchAllPosts() error {

	dirname := f.directory + string(filepath.Separator)

	files, err := ioutil.ReadDir(dirname)

	posts := BlogPosts{}

	for i := range files {

		if files[i].IsDir() {
			continue
		}

		if filepath.Ext(files[i].Name()) != ".md" {
			continue
		}

		post, err := f.fetchPost(dirname + files[i].Name())

		if err != nil {
			return err
		}

		posts = append(posts, post)
	}

	sort.Sort(posts)

	f.mutex.Lock()
	f.posts = posts
	f.mutex.Unlock()

	return err
}

func (f *FileRepository) fetchAllTags() {

	// We're using a map to simulate a set
	tags := make(map[string]int)

	f.mutex.RLock()

	for i := range f.posts {
		for j := range f.posts[i].Tags {

			value := tags[strings.ToLower(f.posts[i].Tags[j])] + 1
			tags[strings.ToLower(f.posts[i].Tags[j])] = value
		}
	}

	f.mutex.RUnlock()

	f.mutex.Lock()
	f.tags = tags
	f.mutex.Unlock()
}

func (f *FileRepository) fetchPost(filename string) (*BlogPost, error) {

	post := new(BlogPost)
	post.FilePath = filename

	file, err := ioutil.ReadFile(filename)

	if err != nil {
		return post, err
	}

	file = []byte(extractPostHeader(string(file), post))

	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_HARD_LINE_BREAK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS

	renderer := blackfriday.HtmlRenderer(htmlFlags, post.Title, "")

	output := blackfriday.Markdown(file, renderer, extensions)

	post.Body = string(output)

	f.fetchCommentsForPost(post, filename)

	return post, nil
}

func (f *FileRepository) fetchCommentsForPost(post *BlogPost, filename string) {

	dirname := filename[:len(filename)-3] + string(filepath.Separator) + "comments" + string(filepath.Separator)

	files, err := ioutil.ReadDir(dirname)

	if err != nil {
		return
	}

	post.Comments = Comments{}

	for i := range files {

		if files[i].IsDir() {
			continue
		}

		if filepath.Ext(files[i].Name()) != ".md" {
			continue
		}

		comment, err := f.fetchComment(dirname + files[i].Name())

		if err != nil {
			log.Fatal(err)
			return
		}

		post.Comments = append(post.Comments, comment)
	}
}

func (f *FileRepository) fetchComment(filename string) (*Comment, error) {
	comment := new(Comment)

	file, err := ioutil.ReadFile(filename)

	if err != nil {
		return comment, err
	}

	file = []byte(extractCommentHeader(string(file), comment))

	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_HARD_LINE_BREAK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	output := blackfriday.Markdown(file, renderer, extensions)

	comment.Body = string(output)

	return comment, nil
}

func extractCommentHeader(text string, comment *Comment) string {

	lines := strings.Split(text, "\n")

	headerSize := 0

	for i := range lines {
		if strings.Contains(lines[i], ":") {
			components := strings.Split(lines[i], ":")

			header := strings.ToLower(strings.Trim(components[0], " "))
			separatorIndex := strings.Index(lines[i], ":") + 1
			data := strings.Trim(lines[i][separatorIndex:], " ")

			switch header {
			case "author":
				comment.Author = data
			case "email":
				comment.Email = data
			case "date":
				comment.Date = stringToTime(data)
			case "spam":
				comment.IsSpam = data == "true"
			default:
				continue
			}

			headerSize += len(lines[i]) + 1
		} else {
			break
		}
	}

	return text[headerSize:]
}

func extractPostHeader(text string, post *BlogPost) string {

	lines := strings.Split(text, "\n")

	headerSize := 0

	for i := range lines {
		if strings.Contains(lines[i], ":") {
			components := strings.Split(lines[i], ":")

			header := strings.ToLower(strings.Trim(components[0], " "))
			separatorIndex := strings.Index(lines[i], ":") + 1
			data := strings.Trim(lines[i][separatorIndex:], " ")

			switch header {
			case "title":
				post.Title = data
			case "id":
				post.Id, _ = strconv.Atoi(data)
			case "tags":

				tags := strings.Split(data, ",")

				formattedTags := []string{}

				for j := range tags {
					tags[j] = strings.Trim(tags[j], " ")
					tags[j] = strings.Replace(tags[j], " ", "-", -1)
					tags[j] = strings.Replace(tags[j], "/", "-", -1)
					tags[j] = strings.ToLower(tags[j])

					if tags[j] != "" {
						formattedTags = append(formattedTags, tags[j])
					}
				}

				post.Tags = formattedTags
			case "date":
				post.PublishDate = stringToTime(data)
			case "disallowcomments":
				post.DisallowComments = data == "true"
			default:
				continue
			}

			headerSize += len(lines[i]) + 1
		} else {
			break
		}
	}

	return text[headerSize:]
}

func timeToFilename(t time.Time) string {
	return fmt.Sprintf("%04d-%02d-%02d_%02d-%02d-%02d.md", t.Year(), int(t.Month()), t.Day(), t.Hour(), t.Minute(), t.Second())
}

func timeToString(t time.Time) string {
	return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", t.Year(), int(t.Month()), t.Day(), t.Hour(), t.Minute(), t.Second())
}

func stringToTime(s string) time.Time {

	year, err := strconv.Atoi(s[:4])

	if err != nil {
		return time.Unix(0, 0)
	}

	month, err := strconv.Atoi(s[5:7])

	if err != nil {
		return time.Unix(0, 0)
	}

	day, err := strconv.Atoi(s[8:10])

	if err != nil {
		return time.Unix(0, 0)
	}

	hour, err := strconv.Atoi(s[11:13])

	if err != nil {
		return time.Unix(0, 0)
	}

	minute, err := strconv.Atoi(s[14:16])

	if err != nil {
		return time.Unix(0, 0)
	}

	seconds, err := strconv.Atoi(s[17:19])

	if err != nil {
		return time.Unix(0, 0)
	}

	location, err := time.LoadLocation("UTC")

	return time.Date(year, time.Month(month), day, hour, minute, seconds, 0, location)
}
<commit_msg>Added Akismet logging.<commit_after>package main

import (
	"bitbucket.org/ant512/gobble/akismet"
	"errors"
	"fmt"
	"github.com/russross/blackfriday"
	"html"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

type FileRepository struct {
	directory string
	posts     BlogPosts
	tags      map[string]int
	mutex     sync.RWMutex
}

func NewFileRepository(directory string) *FileRepository {

	f := new(FileRepository)
	f.directory = directory

	go f.update()

	return f
}

func (f *FileRepository) AllTags() map[string]int {
	return f.tags
}

func (f *FileRepository) AllPosts() BlogPosts {
	return f.posts
}

func (f *FileRepository) PostWithUrl(url string) (*BlogPost, error) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	for i := range f.posts {
		if f.posts[i].Url() == url {
			return f.posts[i], nil
		}
	}

	err := errors.New("Could not find post")

	return nil, err
}

func (f *FileRepository) PostWithId(id int) (*BlogPost, error) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	for i := range f.posts {
		if f.posts[i].Id == id {
			return f.posts[i], nil
		}
	}

	err := errors.New("Could not find post")

	return nil, err
}

func (f *FileRepository) PostsWithTag(tag string, start int, count int) (BlogPosts, int) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	filteredPosts := BlogPosts{}

	for i := range f.posts {
		if f.posts[i].ContainsTag(tag) {
			filteredPosts = append(filteredPosts, f.posts[i])
		}
	}

	if start > len(filteredPosts) {
		return BlogPosts{}, 0
	}

	if start+count > len(filteredPosts) {
		count = len(filteredPosts) - start
	}

	return filteredPosts[start : start+count], len(filteredPosts)
}

func (f *FileRepository) SearchPosts(term string, start int, count int) (BlogPosts, int) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	filteredPosts := BlogPosts{}

	if len(term) > 0 {
		for i := range f.posts {
			if f.posts[i].ContainsTerm(term) {
				filteredPosts = append(filteredPosts, f.posts[i])
			}
		}
	} else {
		filteredPosts = f.posts
	}

	if start > len(filteredPosts) {
		return BlogPosts{}, 0
	}

	if start+count > len(filteredPosts) {
		count = len(filteredPosts) - start
	}

	return filteredPosts[start : start+count], len(filteredPosts)
}

func (f *FileRepository) SaveComment(post *BlogPost, akismetAPIKey, serverAddress, remoteAddress, userAgent, referer, author, email, body string) {

	// TODO: Ensure file name is unique
	isSpam, _ := akismet.IsSpamComment(body, serverAddress, remoteAddress, userAgent, referer, author, email, akismetAPIKey)

	log.Println(isSpam)
	log.Println(serverAddress)
	log.Printf("%s", remoteAddress)
	log.Println(userAgent)
	log.Println(body)
	log.Println(referer)
	log.Println(author)
	log.Println(email)
	log.Println(akismetAPIKey)

	comment := new(Comment)

	comment.Author = html.EscapeString(author)
	comment.Email = html.EscapeString(email)
	comment.Date = time.Now()
	comment.Body = html.EscapeString(body)
	comment.IsSpam = isSpam

	f.mutex.Lock()
	post.Comments = append(post.Comments, comment)
	f.mutex.Unlock()

	postPath := post.FilePath[:len(post.FilePath)-3]

	dirname := postPath + string(filepath.Separator) + "comments" + string(filepath.Separator)

	filename := timeToFilename(comment.Date)

	log.Println(dirname + filename)
	os.MkdirAll(dirname, 0775)

	content := "Author: " + comment.Author + "\n"
	content += "Email: " + comment.Email + "\n"
	content += "Date: " + timeToString(comment.Date) + "\n"

	if isSpam {
		content += "Spam: true\n"
	}

	content += "\n"

	content += comment.Body

	err := ioutil.WriteFile(dirname+filename, []byte(content), 0644)

	if err != nil {
		log.Println(err)
	}
}

func (f *FileRepository) update() {
	for {
		start := time.Now()

		f.fetchAllPosts()
		f.fetchAllTags()

		end := time.Now()
		log.Printf("Cached %v posts in %v", len(f.posts), end.Sub(start))

		time.Sleep(10 * time.Minute)
	}
}

func (f *FileRepository) fetchAllPosts() error {

	dirname := f.directory + string(filepath.Separator)

	files, err := ioutil.ReadDir(dirname)

	posts := BlogPosts{}

	for i := range files {

		if files[i].IsDir() {
			continue
		}

		if filepath.Ext(files[i].Name()) != ".md" {
			continue
		}

		post, err := f.fetchPost(dirname + files[i].Name())

		if err != nil {
			return err
		}

		posts = append(posts, post)
	}

	sort.Sort(posts)

	f.mutex.Lock()
	f.posts = posts
	f.mutex.Unlock()

	return err
}

func (f *FileRepository) fetchAllTags() {

	// We're using a map to simulate a set
	tags := make(map[string]int)

	f.mutex.RLock()

	for i := range f.posts {
		for j := range f.posts[i].Tags {

			value := tags[strings.ToLower(f.posts[i].Tags[j])] + 1
			tags[strings.ToLower(f.posts[i].Tags[j])] = value
		}
	}

	f.mutex.RUnlock()

	f.mutex.Lock()
	f.tags = tags
	f.mutex.Unlock()
}

func (f *FileRepository) fetchPost(filename string) (*BlogPost, error) {

	post := new(BlogPost)
	post.FilePath = filename

	file, err := ioutil.ReadFile(filename)

	if err != nil {
		return post, err
	}

	file = []byte(extractPostHeader(string(file), post))

	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_HARD_LINE_BREAK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS

	renderer := blackfriday.HtmlRenderer(htmlFlags, post.Title, "")

	output := blackfriday.Markdown(file, renderer, extensions)

	post.Body = string(output)

	f.fetchCommentsForPost(post, filename)

	return post, nil
}

func (f *FileRepository) fetchCommentsForPost(post *BlogPost, filename string) {

	dirname := filename[:len(filename)-3] + string(filepath.Separator) + "comments" + string(filepath.Separator)

	files, err := ioutil.ReadDir(dirname)

	if err != nil {
		return
	}

	post.Comments = Comments{}

	for i := range files {

		if files[i].IsDir() {
			continue
		}

		if filepath.Ext(files[i].Name()) != ".md" {
			continue
		}

		comment, err := f.fetchComment(dirname + files[i].Name())

		if err != nil {
			log.Fatal(err)
			return
		}

		post.Comments = append(post.Comments, comment)
	}
}

func (f *FileRepository) fetchComment(filename string) (*Comment, error) {
	comment := new(Comment)

	file, err := ioutil.ReadFile(filename)

	if err != nil {
		return comment, err
	}

	file = []byte(extractCommentHeader(string(file), comment))

	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_HARD_LINE_BREAK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	output := blackfriday.Markdown(file, renderer, extensions)

	comment.Body = string(output)

	return comment, nil
}

func extractCommentHeader(text string, comment *Comment) string {

	lines := strings.Split(text, "\n")

	headerSize := 0

	for i := range lines {
		if strings.Contains(lines[i], ":") {
			components := strings.Split(lines[i], ":")

			header := strings.ToLower(strings.Trim(components[0], " "))
			separatorIndex := strings.Index(lines[i], ":") + 1
			data := strings.Trim(lines[i][separatorIndex:], " ")

			switch header {
			case "author":
				comment.Author = data
			case "email":
				comment.Email = data
			case "date":
				comment.Date = stringToTime(data)
			case "spam":
				comment.IsSpam = data == "true"
			default:
				continue
			}

			headerSize += len(lines[i]) + 1
		} else {
			break
		}
	}

	return text[headerSize:]
}

func extractPostHeader(text string, post *BlogPost) string {

	lines := strings.Split(text, "\n")

	headerSize := 0

	for i := range lines {
		if strings.Contains(lines[i], ":") {
			components := strings.Split(lines[i], ":")

			header := strings.ToLower(strings.Trim(components[0], " "))
			separatorIndex := strings.Index(lines[i], ":") + 1
			data := strings.Trim(lines[i][separatorIndex:], " ")

			switch header {
			case "title":
				post.Title = data
			case "id":
				post.Id, _ = strconv.Atoi(data)
			case "tags":

				tags := strings.Split(data, ",")

				formattedTags := []string{}

				for j := range tags {
					tags[j] = strings.Trim(tags[j], " ")
					tags[j] = strings.Replace(tags[j], " ", "-", -1)
					tags[j] = strings.Replace(tags[j], "/", "-", -1)
					tags[j] = strings.ToLower(tags[j])

					if tags[j] != "" {
						formattedTags = append(formattedTags, tags[j])
					}
				}

				post.Tags = formattedTags
			case "date":
				post.PublishDate = stringToTime(data)
			case "disallowcomments":
				post.DisallowComments = data == "true"
			default:
				continue
			}

			headerSize += len(lines[i]) + 1
		} else {
			break
		}
	}

	return text[headerSize:]
}

func timeToFilename(t time.Time) string {
	return fmt.Sprintf("%04d-%02d-%02d_%02d-%02d-%02d.md", t.Year(), int(t.Month()), t.Day(), t.Hour(), t.Minute(), t.Second())
}

func timeToString(t time.Time) string {
	return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", t.Year(), int(t.Month()), t.Day(), t.Hour(), t.Minute(), t.Second())
}

func stringToTime(s string) time.Time {

	year, err := strconv.Atoi(s[:4])

	if err != nil {
		return time.Unix(0, 0)
	}

	month, err := strconv.Atoi(s[5:7])

	if err != nil {
		return time.Unix(0, 0)
	}

	day, err := strconv.Atoi(s[8:10])

	if err != nil {
		return time.Unix(0, 0)
	}

	hour, err := strconv.Atoi(s[11:13])

	if err != nil {
		return time.Unix(0, 0)
	}

	minute, err := strconv.Atoi(s[14:16])

	if err != nil {
		return time.Unix(0, 0)
	}

	seconds, err := strconv.Atoi(s[17:19])

	if err != nil {
		return time.Unix(0, 0)
	}

	location, err := time.LoadLocation("UTC")

	return time.Date(year, time.Month(month), day, hour, minute, seconds, 0, location)
}
<|endoftext|>


<commit_before>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package mux

import (
	"errors"
	"net/http"
	"strconv"
)

type contextKey int

const contextKeyParams contextKey = 0

// ErrParamNotExists 表示地址参数中并不存在该名称的值。
var ErrParamNotExists = errors.New("不存在该值")

// Params 用以保存请求地址中的参数内容
type Params map[string]string

// GetParams 从 r 中获取路由参数。
//
// 以下情况两个参数都会返回 nil：
//  非正则路由；
//  正则路由，但是所有匹配参数都是未命名的；
func GetParams(r *http.Request) (Params, error) {
	params := r.Context().Value(contextKeyParams)
	if params == nil {
		return nil, nil
	}

	if ret, ok := params.(Params); ok {
		return ret, nil
	}
	return nil, errors.New("无法转换成 Params 类型")
}

// Exists 查找指定名称的参数是否存在。
// 可选参数，可能会不存在。
func (p Params) Exists(key string) bool {
	_, found := p[key]
	return found
}

// String 获取地址参数中的名为 key 的变量，并将其转换成 string
//
// 当参数不存在时，返回 ErrParamNotExists 错误。
func (p Params) String(key string) (string, error) {
	v, found := p[key]
	if !found {
		return "", ErrParamNotExists
	}

	return v, nil
}

// MustString 获取地址参数中的名为 key 的变量，并将其转换成 string，
// 若不存在或是无法转换则返回 def。
func (p Params) MustString(key, def string) string {
	v, found := p[key]
	if !found {
		return def
	}

	return v
}

// Int 获取地址参数中的名为 key 的变量，并将其转换成 int64
//
// 当参数不存在时，返回 ErrParamNotExists 错误。
func (p Params) Int(key string) (int64, error) {
	str, found := p[key]
	if !found {
		return 0, ErrParamNotExists
	}

	return strconv.ParseInt(str, 10, 64)
}

// MustInt 获取地址参数中的名为 key 的变量，并将其转换成 int64，
// 若不存在或是无法转换则返回 def。
func (p Params) MustInt(key string, def int64) int64 {
	str, found := p[key]
	if !found {
		return def
	}

	if val, err := strconv.ParseInt(str, 10, 64); err == nil {
		return val
	}

	return def
}

// Uint 获取地址参数中的名为 key 的变量，并将其转换成 uint64
//
// 当参数不存在时，返回 ErrParamNotExists 错误。
func (p Params) Uint(key string) (uint64, error) {
	str, found := p[key]
	if !found {
		return 0, ErrParamNotExists
	}

	return strconv.ParseUint(str, 10, 64)
}

// MustUint 获取地址参数中的名为 key 的变量，并将其转换成 uint64，
// 若不存在或是无法转换则返回 def。
func (p Params) MustUint(key string, def uint64) uint64 {
	str, found := p[key]
	if !found {
		return def
	}

	if val, err := strconv.ParseUint(str, 10, 64); err == nil {
		return val
	}

	return def
}

// Bool 获取地址参数中的名为 key 的变量，并将其转换成 bool
//
// 当参数不存在时，返回 ErrParamNotExists 错误。
func (p Params) Bool(key string) (bool, error) {
	str, found := p[key]
	if !found {
		return false, ErrParamNotExists
	}

	return strconv.ParseBool(str)
}

// MustBool 获取地址参数中的名为 key 的变量，并将其转换成 bool，
// 若不存在或是无法转换则返回 def。
func (p Params) MustBool(key string, def bool) bool {
	str, found := p[key]
	if !found {
		return def
	}

	if val, err := strconv.ParseBool(str); err == nil {
		return val
	}

	return def
}
<commit_msg>添加 Params.Float 等函数<commit_after>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package mux

import (
	"errors"
	"net/http"
	"strconv"
)

type contextKey int

const contextKeyParams contextKey = 0

// ErrParamNotExists 表示地址参数中并不存在该名称的值。
var ErrParamNotExists = errors.New("不存在该值")

// Params 用以保存请求地址中的参数内容
type Params map[string]string

// GetParams 从 r 中获取路由参数。
//
// 以下情况两个参数都会返回 nil：
//  非正则路由；
//  正则路由，但是所有匹配参数都是未命名的；
func GetParams(r *http.Request) (Params, error) {
	params := r.Context().Value(contextKeyParams)
	if params == nil {
		return nil, nil
	}

	if ret, ok := params.(Params); ok {
		return ret, nil
	}
	return nil, errors.New("无法转换成 Params 类型")
}

// Exists 查找指定名称的参数是否存在。
// 可选参数，可能会不存在。
func (p Params) Exists(key string) bool {
	_, found := p[key]
	return found
}

// String 获取地址参数中的名为 key 的变量，并将其转换成 string
//
// 当参数不存在时，返回 ErrParamNotExists 错误。
func (p Params) String(key string) (string, error) {
	v, found := p[key]
	if !found {
		return "", ErrParamNotExists
	}

	return v, nil
}

// MustString 获取地址参数中的名为 key 的变量，并将其转换成 string，
// 若不存在或是无法转换则返回 def。
func (p Params) MustString(key, def string) string {
	v, found := p[key]
	if !found {
		return def
	}

	return v
}

// Int 获取地址参数中的名为 key 的变量，并将其转换成 int64
//
// 当参数不存在时，返回 ErrParamNotExists 错误。
func (p Params) Int(key string) (int64, error) {
	str, found := p[key]
	if !found {
		return 0, ErrParamNotExists
	}

	return strconv.ParseInt(str, 10, 64)
}

// MustInt 获取地址参数中的名为 key 的变量，并将其转换成 int64，
// 若不存在或是无法转换则返回 def。
func (p Params) MustInt(key string, def int64) int64 {
	str, found := p[key]
	if !found {
		return def
	}

	if val, err := strconv.ParseInt(str, 10, 64); err == nil {
		return val
	}

	return def
}

// Uint 获取地址参数中的名为 key 的变量，并将其转换成 uint64
//
// 当参数不存在时，返回 ErrParamNotExists 错误。
func (p Params) Uint(key string) (uint64, error) {
	str, found := p[key]
	if !found {
		return 0, ErrParamNotExists
	}

	return strconv.ParseUint(str, 10, 64)
}

// MustUint 获取地址参数中的名为 key 的变量，并将其转换成 uint64，
// 若不存在或是无法转换则返回 def。
func (p Params) MustUint(key string, def uint64) uint64 {
	str, found := p[key]
	if !found {
		return def
	}

	if val, err := strconv.ParseUint(str, 10, 64); err == nil {
		return val
	}

	return def
}

// Bool 获取地址参数中的名为 key 的变量，并将其转换成 bool
//
// 当参数不存在时，返回 ErrParamNotExists 错误。
func (p Params) Bool(key string) (bool, error) {
	str, found := p[key]
	if !found {
		return false, ErrParamNotExists
	}

	return strconv.ParseBool(str)
}

// MustBool 获取地址参数中的名为 key 的变量，并将其转换成 bool，
// 若不存在或是无法转换则返回 def。
func (p Params) MustBool(key string, def bool) bool {
	str, found := p[key]
	if !found {
		return def
	}

	if val, err := strconv.ParseBool(str); err == nil {
		return val
	}

	return def
}

// Float 获取地址参数中的名为 key 的变量，并将其转换成 Float64
//
// 当参数不存在时，返回 ErrParamNotExists 错误。
func (p Params) Float(key string) (float64, error) {
	str, found := p[key]
	if !found {
		return 0, ErrParamNotExists
	}

	return strconv.ParseFloat(str, 64)
}

// MustFloat 获取地址参数中的名为 key 的变量，并将其转换成 float64，
// 若不存在或是无法转换则返回 def。
func (p Params) MustFloat(key string, def float64) float64 {
	str, found := p[key]
	if !found {
		return def
	}

	if val, err := strconv.ParseFloat(str, 64); err == nil {
		return val
	}

	return def
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcchain

import (
	"github.com/conformal/btcwire"
	"math/big"
)

// Params houses parameters unique to various bitcoin networks such as the main
// network and test networks.  See ChainParams.
type Params struct {
	// GenesisBlock is the genesis block for the specific network.
	GenesisBlock *btcwire.MsgBlock

	// GenesisHash is the genesis block hash for the specific network.
	GenesisHash *btcwire.ShaHash

	// PowLimit is the highest proof of work value a bitcoin block can have
	// for the specific network.
	PowLimit *big.Int

	// PowLimitBits is the highest proof of work value a bitcoin block can
	// have represented in compact form.  See CompactToBig for more details
	// on compact form.
	PowLimitBits uint32
}

// mainPowLimit is the highest proof of work value a bitcoin block can have for
// the main network.  It is the value 2^224 - 1.
var mainPowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 224), bigOne)

// mainNetParams contains parameters specific to the main network
// (btcwire.MainNet).
var mainNetParams = Params{
	GenesisBlock: &btcwire.GenesisBlock,
	GenesisHash:  &btcwire.GenesisHash,
	PowLimit:     mainPowLimit,
	PowLimitBits: BigToCompact(mainPowLimit),
}

// regressionPowLimit is the highest proof of work value a bitcoin block can
// have for the regression test network.  It is the value 2^256 - 1.
var regressionPowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 255), bigOne)

// regressionParams contains parameters specific to the regression test network
// (btcwire.TestNet).
var regressionParams = Params{
	GenesisBlock: &btcwire.TestNetGenesisBlock,
	GenesisHash:  &btcwire.TestNetGenesisHash,
	PowLimit:     regressionPowLimit,
	PowLimitBits: BigToCompact(regressionPowLimit),
}

// testNetPowLimit is the highest proof of work value a bitcoin block can have
// for the test network (version 3).  It is the value 2^224 - 1.
var testNetPowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 224), bigOne)

// testNet3Params contains parameters specific to the test network (version 3)
// (btcwire.TestNet3).
var testNet3Params = Params{
	GenesisBlock: &btcwire.TestNet3GenesisBlock,
	GenesisHash:  &btcwire.TestNet3GenesisHash,
	PowLimit:     testNetPowLimit,
	PowLimitBits: BigToCompact(testNetPowLimit),
}

// chainParams returns chain parameters specific to the bitcoin network
// associated with the BlockChain instance.
func (b *BlockChain) chainParams() *Params {
	return ChainParams(b.btcnet)
}

// ChainParams returns chain parameters specific to the passed bitcoin network.
// It returns the parameters for btcwire.MainNet if the passed network is not
// supported.
func ChainParams(btcnet btcwire.BitcoinNet) *Params {
	switch btcnet {
	case btcwire.TestNet:
		return &regressionParams

	case btcwire.TestNet3:
		return &testNet3Params

	// Return main net by default.
	case btcwire.MainNet:
		fallthrough
	default:
		return &mainNetParams
	}
}
<commit_msg>Correct reg test proof of work limit comment.<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcchain

import (
	"github.com/conformal/btcwire"
	"math/big"
)

// Params houses parameters unique to various bitcoin networks such as the main
// network and test networks.  See ChainParams.
type Params struct {
	// GenesisBlock is the genesis block for the specific network.
	GenesisBlock *btcwire.MsgBlock

	// GenesisHash is the genesis block hash for the specific network.
	GenesisHash *btcwire.ShaHash

	// PowLimit is the highest proof of work value a bitcoin block can have
	// for the specific network.
	PowLimit *big.Int

	// PowLimitBits is the highest proof of work value a bitcoin block can
	// have represented in compact form.  See CompactToBig for more details
	// on compact form.
	PowLimitBits uint32
}

// mainPowLimit is the highest proof of work value a bitcoin block can have for
// the main network.  It is the value 2^224 - 1.
var mainPowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 224), bigOne)

// mainNetParams contains parameters specific to the main network
// (btcwire.MainNet).
var mainNetParams = Params{
	GenesisBlock: &btcwire.GenesisBlock,
	GenesisHash:  &btcwire.GenesisHash,
	PowLimit:     mainPowLimit,
	PowLimitBits: BigToCompact(mainPowLimit),
}

// regressionPowLimit is the highest proof of work value a bitcoin block can
// have for the regression test network.  It is the value 2^255 - 1.
var regressionPowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 255), bigOne)

// regressionParams contains parameters specific to the regression test network
// (btcwire.TestNet).
var regressionParams = Params{
	GenesisBlock: &btcwire.TestNetGenesisBlock,
	GenesisHash:  &btcwire.TestNetGenesisHash,
	PowLimit:     regressionPowLimit,
	PowLimitBits: BigToCompact(regressionPowLimit),
}

// testNetPowLimit is the highest proof of work value a bitcoin block can have
// for the test network (version 3).  It is the value 2^224 - 1.
var testNetPowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 224), bigOne)

// testNet3Params contains parameters specific to the test network (version 3)
// (btcwire.TestNet3).
var testNet3Params = Params{
	GenesisBlock: &btcwire.TestNet3GenesisBlock,
	GenesisHash:  &btcwire.TestNet3GenesisHash,
	PowLimit:     testNetPowLimit,
	PowLimitBits: BigToCompact(testNetPowLimit),
}

// chainParams returns chain parameters specific to the bitcoin network
// associated with the BlockChain instance.
func (b *BlockChain) chainParams() *Params {
	return ChainParams(b.btcnet)
}

// ChainParams returns chain parameters specific to the passed bitcoin network.
// It returns the parameters for btcwire.MainNet if the passed network is not
// supported.
func ChainParams(btcnet btcwire.BitcoinNet) *Params {
	switch btcnet {
	case btcwire.TestNet:
		return &regressionParams

	case btcwire.TestNet3:
		return &testNet3Params

	// Return main net by default.
	case btcwire.MainNet:
		fallthrough
	default:
		return &mainNetParams
	}
}
<|endoftext|>


<commit_before>package filters

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/getlantern/errors"
)

// Filter supports intercepting and modifying http requests and responses
type Filter interface {
	Apply(ctx context.Context, req *http.Request, next Next) (*http.Response, error)
}

// FilterFunc adapts a function to a Filter
type FilterFunc func(ctx context.Context, req *http.Request, next Next) (*http.Response, error)

// Apply implements the interface Filter
func (ff FilterFunc) Apply(ctx context.Context, req *http.Request, next Next) (*http.Response, error) {
	return ff(ctx, req, next)
}

// Next is a function that's used to indicate that request processing should
// continue as usual.
type Next func(ctx context.Context, req *http.Request) (*http.Response, error)

// ShortCircuit is a convenience method for creating short-circuiting responses.
func ShortCircuit(req *http.Request, resp *http.Response) (*http.Response, error) {
	resp.Request = req
	if resp.Header == nil {
		resp.Header = make(http.Header)
	}
	return resp, nil
}

// Discard discards the given request. Make sure to use this when discarding
// requests in order to make sure that the request body is read.
func Discard(req *http.Request) (*http.Response, error) {
	if req.Body != nil {
		io.Copy(ioutil.Discard, req.Body)
		req.Body.Close()
	}
	return nil, nil
}

// Chain is a chain of Filters that acts as an http.Handler.
type Chain []Filter

// Fail fails execution of the current chain
func Fail(msg string, args ...interface{}) error {
	return errors.New(msg, args...)
}

// Stop stops execution of the current chain
func Stop() error {
	return nil
}

// Join constructs a new chain of filters that executes the filters in order
// until it encounters a filter that returns false.
func Join(filters ...Filter) Chain {
	return Chain(filters)
}

// Append creates a new Chain by appending the given filters.
func (c Chain) Append(post ...Filter) Chain {
	return append(c, post...)
}

// Prepend creates a new chain by prepending the given filter.
func (c Chain) Prepend(pre Filter) Chain {
	result := make(Chain, len(c)+1)
	result[0] = pre
	copy(result[1:], c)
	return result
}

// Apply implements the interface Filter
func (c Chain) Apply(ctx context.Context, req *http.Request, next Next) (*http.Response, error) {
	return c.apply(ctx, req, next, 0)
}

func (c Chain) apply(ctx context.Context, req *http.Request, next Next, idx int) (*http.Response, error) {
	_next := next
	if idx < len(c)-1 {
		_next = func(ctx context.Context, req *http.Request) (*http.Response, error) {
			return c.apply(ctx, req, next, idx+1)
		}
	}
	return c[idx].Apply(ctx, req, _next)
}
<commit_msg>Added filters.Fail<commit_after>package filters

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

// Filter supports intercepting and modifying http requests and responses
type Filter interface {
	Apply(ctx context.Context, req *http.Request, next Next) (*http.Response, error)
}

// FilterFunc adapts a function to a Filter
type FilterFunc func(ctx context.Context, req *http.Request, next Next) (*http.Response, error)

// Apply implements the interface Filter
func (ff FilterFunc) Apply(ctx context.Context, req *http.Request, next Next) (*http.Response, error) {
	return ff(ctx, req, next)
}

// Next is a function that's used to indicate that request processing should
// continue as usual.
type Next func(ctx context.Context, req *http.Request) (*http.Response, error)

// ShortCircuit is a convenience method for creating short-circuiting responses.
func ShortCircuit(req *http.Request, resp *http.Response) (*http.Response, error) {
	resp.Request = req
	if resp.Header == nil {
		resp.Header = make(http.Header)
	}
	return resp, nil
}

// Fail fails processing, returning a response with the given status code and
// description populated from error.
func Fail(req *http.Request, statusCode int, err error) (*http.Response, error) {
	resp := &http.Response{
		Request:    req,
		StatusCode: statusCode,
		Body:       ioutil.NopCloser(strings.NewReader(err.Error())),
	}
	return resp, err
}

// Discard discards the given request. Make sure to use this when discarding
// requests in order to make sure that the request body is read.
func Discard(req *http.Request) (*http.Response, error) {
	if req.Body != nil {
		io.Copy(ioutil.Discard, req.Body)
		req.Body.Close()
	}
	return nil, nil
}

// Chain is a chain of Filters that acts as an http.Handler.
type Chain []Filter

// Join constructs a new chain of filters that executes the filters in order
// until it encounters a filter that returns false.
func Join(filters ...Filter) Chain {
	return Chain(filters)
}

// Append creates a new Chain by appending the given filters.
func (c Chain) Append(post ...Filter) Chain {
	return append(c, post...)
}

// Prepend creates a new chain by prepending the given filter.
func (c Chain) Prepend(pre Filter) Chain {
	result := make(Chain, len(c)+1)
	result[0] = pre
	copy(result[1:], c)
	return result
}

// Apply implements the interface Filter
func (c Chain) Apply(ctx context.Context, req *http.Request, next Next) (*http.Response, error) {
	return c.apply(ctx, req, next, 0)
}

func (c Chain) apply(ctx context.Context, req *http.Request, next Next, idx int) (*http.Response, error) {
	_next := next
	if idx < len(c)-1 {
		_next = func(ctx context.Context, req *http.Request) (*http.Response, error) {
			return c.apply(ctx, req, next, idx+1)
		}
	}
	return c[idx].Apply(ctx, req, _next)
}
<|endoftext|>


<commit_before>package kingpin

import (
	"bufio"
	"fmt"
	"os"
	"strings"
	"unicode/utf8"
)

type TokenType int

// Token types.
const (
	TokenShort TokenType = iota
	TokenLong
	TokenArg
	TokenError
	TokenEOL
)

func (t TokenType) String() string {
	switch t {
	case TokenShort:
		return "short flag"
	case TokenLong:
		return "long flag"
	case TokenArg:
		return "argument"
	case TokenError:
		return "error"
	case TokenEOL:
		return "<EOL>"
	}
	return "?"
}

var (
	TokenEOLMarker = Token{-1, TokenEOL, ""}
)

type Token struct {
	Index int
	Type  TokenType
	Value string
}

func (t *Token) Equal(o *Token) bool {
	return t.Index == o.Index
}

func (t *Token) IsFlag() bool {
	return t.Type == TokenShort || t.Type == TokenLong
}

func (t *Token) IsEOF() bool {
	return t.Type == TokenEOL
}

func (t *Token) String() string {
	switch t.Type {
	case TokenShort:
		return "-" + t.Value
	case TokenLong:
		return "--" + t.Value
	case TokenArg:
		return t.Value
	case TokenError:
		return "error: " + t.Value
	case TokenEOL:
		return "<EOL>"
	default:
		panic("unhandled type")
	}
}

// A union of possible elements in a parse stack.
type ParseElement struct {
	// Clause is either *CmdClause, *ArgClause or *FlagClause.
	Clause interface{}
	// Value is corresponding value for an ArgClause or FlagClause (if any).
	Value *string
}

// ParseContext holds the current context of the parser. When passed to
// Action() callbacks Elements will be fully populated with *FlagClause,
// *ArgClause and *CmdClause values and their corresponding arguments (if
// any).
type ParseContext struct {
	SelectedCommand *CmdClause
	ignoreDefault   bool
	argsOnly        bool
	peek            []*Token
	argi            int // Index of current command-line arg we're processing.
	args            []string
	rawArgs         []string
	flags           *flagGroup
	arguments       *argGroup
	argumenti       int // Cursor into arguments
	// Flags, arguments and commands encountered and collected during parse.
	Elements []*ParseElement
}

func (p *ParseContext) nextArg() *ArgClause {
	if p.argumenti >= len(p.arguments.args) {
		return nil
	}
	arg := p.arguments.args[p.argumenti]
	if !arg.consumesRemainder() {
		p.argumenti++
	}
	return arg
}

func (p *ParseContext) next() {
	p.argi++
	p.args = p.args[1:]
}

// HasTrailingArgs returns true if there are unparsed command-line arguments.
// This can occur if the parser can not match remaining arguments.
func (p *ParseContext) HasTrailingArgs() bool {
	return len(p.args) > 0
}

func tokenize(args []string, ignoreDefault bool) *ParseContext {
	return &ParseContext{
		ignoreDefault: ignoreDefault,
		args:          args,
		rawArgs:       args,
		flags:         newFlagGroup(),
		arguments:     newArgGroup(),
	}
}

func (p *ParseContext) mergeFlags(flags *flagGroup) {
	for _, flag := range flags.flagOrder {
		if flag.shorthand != 0 {
			p.flags.short[string(flag.shorthand)] = flag
		}
		p.flags.long[flag.name] = flag
		p.flags.flagOrder = append(p.flags.flagOrder, flag)
	}
}

func (p *ParseContext) mergeArgs(args *argGroup) {
	for _, arg := range args.args {
		p.arguments.args = append(p.arguments.args, arg)
	}
}

func (p *ParseContext) EOL() bool {
	return p.Peek().Type == TokenEOL
}

// Next token in the parse context.
func (p *ParseContext) Next() *Token {
	if len(p.peek) > 0 {
		return p.pop()
	}

	// End of tokens.
	if len(p.args) == 0 {
		return &Token{Index: p.argi, Type: TokenEOL}
	}

	arg := p.args[0]
	p.next()

	if p.argsOnly {
		return &Token{p.argi, TokenArg, arg}
	}

	// All remaining args are passed directly.
	if arg == "--" {
		p.argsOnly = true
		return p.Next()
	}

	if strings.HasPrefix(arg, "--") {
		parts := strings.SplitN(arg[2:], "=", 2)
		token := &Token{p.argi, TokenLong, parts[0]}
		if len(parts) == 2 {
			p.Push(&Token{p.argi, TokenArg, parts[1]})
		}
		return token
	}

	if strings.HasPrefix(arg, "-") {
		if len(arg) == 1 {
			return &Token{Index: p.argi, Type: TokenShort}
		}
		shortRune, size := utf8.DecodeRuneInString(arg[1:])
		short := string(shortRune)
		flag, ok := p.flags.short[short]
		// Not a known short flag, we'll just return it anyway.
		if !ok {
		} else if fb, ok := flag.value.(boolFlag); ok && fb.IsBoolFlag() {
			// Bool short flag.
		} else {
			// Short flag with combined argument: -fARG
			token := &Token{p.argi, TokenShort, short}
			if len(arg) > size+1 {
				p.Push(&Token{p.argi, TokenArg, arg[size+1:]})
			}
			return token
		}

		if len(arg) > size+1 {
			p.args = append([]string{"-" + arg[size+1:]}, p.args...)
		}
		return &Token{p.argi, TokenShort, short}
	} else if strings.HasPrefix(arg, "@") {
		expanded, err := ExpandArgsFromFile(arg[1:])
		if err != nil {
			return &Token{p.argi, TokenError, err.Error()}
		}
		if len(p.args) == 0 {
			p.args = expanded
		} else {
			p.args = append(expanded, p.args...)
		}
		return p.Next()
	}

	return &Token{p.argi, TokenArg, arg}
}

func (p *ParseContext) Peek() *Token {
	if len(p.peek) == 0 {
		return p.Push(p.Next())
	}
	return p.peek[len(p.peek)-1]
}

func (p *ParseContext) Push(token *Token) *Token {
	p.peek = append(p.peek, token)
	return token
}

func (p *ParseContext) pop() *Token {
	end := len(p.peek) - 1
	token := p.peek[end]
	p.peek = p.peek[0:end]
	return token
}

func (p *ParseContext) String() string {
	return p.SelectedCommand.FullCommand()
}

func (p *ParseContext) matchedFlag(flag *FlagClause, value string) {
	p.Elements = append(p.Elements, &ParseElement{Clause: flag, Value: &value})
}

func (p *ParseContext) matchedArg(arg *ArgClause, value string) {
	p.Elements = append(p.Elements, &ParseElement{Clause: arg, Value: &value})
}

func (p *ParseContext) matchedCmd(cmd *CmdClause) {
	p.Elements = append(p.Elements, &ParseElement{Clause: cmd})
	p.mergeFlags(cmd.flagGroup)
	p.mergeArgs(cmd.argGroup)
	p.SelectedCommand = cmd
}

// Expand arguments from a file. Lines starting with # will be treated as comments.
func ExpandArgsFromFile(filename string) (out []string, err error) {
	r, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "#") {
			continue
		}
		out = append(out, line)
	}
	err = scanner.Err()
	return
}

func parse(context *ParseContext, app *Application) (err error) {
	context.mergeFlags(app.flagGroup)
	context.mergeArgs(app.argGroup)

	cmds := app.cmdGroup
	ignoreDefault := context.ignoreDefault

loop:
	for !context.EOL() {
		token := context.Peek()

		switch token.Type {
		case TokenLong, TokenShort:
			if flag, err := context.flags.parse(context); err != nil {
				if !ignoreDefault {
					if cmd := cmds.defaultSubcommand(); cmd != nil {
						cmd.completionAlts = cmds.cmdNames()
						context.matchedCmd(cmd)
						cmds = cmd.cmdGroup
						break
					}
				}
				return err
			} else if flag == HelpFlag {
				ignoreDefault = true
			}

		case TokenArg:
			if cmds.have() {
				selectedDefault := false
				cmd, ok := cmds.commands[token.String()]
				if !ok {
					if !ignoreDefault {
						if cmd = cmds.defaultSubcommand(); cmd != nil {
							cmd.completionAlts = cmds.cmdNames()
							selectedDefault = true
						}
					}
					if cmd == nil {
						return fmt.Errorf("expected command but got %q", token)
					}
				}
				if cmd == HelpCommand {
					ignoreDefault = true
				}
				cmd.completionAlts = nil
				context.matchedCmd(cmd)
				cmds = cmd.cmdGroup
				if !selectedDefault {
					context.Next()
				}
			} else if context.arguments.have() {
				if app.noInterspersed {
					// no more flags
					context.argsOnly = true
				}
				arg := context.nextArg()
				if arg == nil {
					break loop
				}
				context.matchedArg(arg, token.String())
				context.Next()
			} else {
				break loop
			}

		case TokenEOL:
			break loop
		}
	}

	// Move to innermost default command.
	for !ignoreDefault {
		if cmd := cmds.defaultSubcommand(); cmd != nil {
			cmd.completionAlts = cmds.cmdNames()
			context.matchedCmd(cmd)
			cmds = cmd.cmdGroup
		} else {
			break
		}
	}

	if !context.EOL() {
		return fmt.Errorf("unexpected %s", context.Peek())
	}

	// Set defaults for all remaining args.
	for arg := context.nextArg(); arg != nil && !arg.consumesRemainder(); arg = context.nextArg() {
		for _, defaultValue := range arg.defaultValues {
			if err := arg.value.Set(defaultValue); err != nil {
				return fmt.Errorf("invalid default value '%s' for argument '%s'", defaultValue, arg.name)
			}
		}
	}

	return
}
<commit_msg>Fix error propagation from @ failures.<commit_after>package kingpin

import (
	"bufio"
	"fmt"
	"os"
	"strings"
	"unicode/utf8"
)

type TokenType int

// Token types.
const (
	TokenShort TokenType = iota
	TokenLong
	TokenArg
	TokenError
	TokenEOL
)

func (t TokenType) String() string {
	switch t {
	case TokenShort:
		return "short flag"
	case TokenLong:
		return "long flag"
	case TokenArg:
		return "argument"
	case TokenError:
		return "error"
	case TokenEOL:
		return "<EOL>"
	}
	return "?"
}

var (
	TokenEOLMarker = Token{-1, TokenEOL, ""}
)

type Token struct {
	Index int
	Type  TokenType
	Value string
}

func (t *Token) Equal(o *Token) bool {
	return t.Index == o.Index
}

func (t *Token) IsFlag() bool {
	return t.Type == TokenShort || t.Type == TokenLong
}

func (t *Token) IsEOF() bool {
	return t.Type == TokenEOL
}

func (t *Token) String() string {
	switch t.Type {
	case TokenShort:
		return "-" + t.Value
	case TokenLong:
		return "--" + t.Value
	case TokenArg:
		return t.Value
	case TokenError:
		return "error: " + t.Value
	case TokenEOL:
		return "<EOL>"
	default:
		panic("unhandled type")
	}
}

// A union of possible elements in a parse stack.
type ParseElement struct {
	// Clause is either *CmdClause, *ArgClause or *FlagClause.
	Clause interface{}
	// Value is corresponding value for an ArgClause or FlagClause (if any).
	Value *string
}

// ParseContext holds the current context of the parser. When passed to
// Action() callbacks Elements will be fully populated with *FlagClause,
// *ArgClause and *CmdClause values and their corresponding arguments (if
// any).
type ParseContext struct {
	SelectedCommand *CmdClause
	ignoreDefault   bool
	argsOnly        bool
	peek            []*Token
	argi            int // Index of current command-line arg we're processing.
	args            []string
	rawArgs         []string
	flags           *flagGroup
	arguments       *argGroup
	argumenti       int // Cursor into arguments
	// Flags, arguments and commands encountered and collected during parse.
	Elements []*ParseElement
}

func (p *ParseContext) nextArg() *ArgClause {
	if p.argumenti >= len(p.arguments.args) {
		return nil
	}
	arg := p.arguments.args[p.argumenti]
	if !arg.consumesRemainder() {
		p.argumenti++
	}
	return arg
}

func (p *ParseContext) next() {
	p.argi++
	p.args = p.args[1:]
}

// HasTrailingArgs returns true if there are unparsed command-line arguments.
// This can occur if the parser can not match remaining arguments.
func (p *ParseContext) HasTrailingArgs() bool {
	return len(p.args) > 0
}

func tokenize(args []string, ignoreDefault bool) *ParseContext {
	return &ParseContext{
		ignoreDefault: ignoreDefault,
		args:          args,
		rawArgs:       args,
		flags:         newFlagGroup(),
		arguments:     newArgGroup(),
	}
}

func (p *ParseContext) mergeFlags(flags *flagGroup) {
	for _, flag := range flags.flagOrder {
		if flag.shorthand != 0 {
			p.flags.short[string(flag.shorthand)] = flag
		}
		p.flags.long[flag.name] = flag
		p.flags.flagOrder = append(p.flags.flagOrder, flag)
	}
}

func (p *ParseContext) mergeArgs(args *argGroup) {
	for _, arg := range args.args {
		p.arguments.args = append(p.arguments.args, arg)
	}
}

func (p *ParseContext) EOL() bool {
	return p.Peek().Type == TokenEOL
}

func (p *ParseContext) Error() bool {
	return p.Peek().Type == TokenError
}

// Next token in the parse context.
func (p *ParseContext) Next() *Token {
	if len(p.peek) > 0 {
		return p.pop()
	}

	// End of tokens.
	if len(p.args) == 0 {
		return &Token{Index: p.argi, Type: TokenEOL}
	}

	arg := p.args[0]
	p.next()

	if p.argsOnly {
		return &Token{p.argi, TokenArg, arg}
	}

	// All remaining args are passed directly.
	if arg == "--" {
		p.argsOnly = true
		return p.Next()
	}

	if strings.HasPrefix(arg, "--") {
		parts := strings.SplitN(arg[2:], "=", 2)
		token := &Token{p.argi, TokenLong, parts[0]}
		if len(parts) == 2 {
			p.Push(&Token{p.argi, TokenArg, parts[1]})
		}
		return token
	}

	if strings.HasPrefix(arg, "-") {
		if len(arg) == 1 {
			return &Token{Index: p.argi, Type: TokenShort}
		}
		shortRune, size := utf8.DecodeRuneInString(arg[1:])
		short := string(shortRune)
		flag, ok := p.flags.short[short]
		// Not a known short flag, we'll just return it anyway.
		if !ok {
		} else if fb, ok := flag.value.(boolFlag); ok && fb.IsBoolFlag() {
			// Bool short flag.
		} else {
			// Short flag with combined argument: -fARG
			token := &Token{p.argi, TokenShort, short}
			if len(arg) > size+1 {
				p.Push(&Token{p.argi, TokenArg, arg[size+1:]})
			}
			return token
		}

		if len(arg) > size+1 {
			p.args = append([]string{"-" + arg[size+1:]}, p.args...)
		}
		return &Token{p.argi, TokenShort, short}
	} else if strings.HasPrefix(arg, "@") {
		expanded, err := ExpandArgsFromFile(arg[1:])
		if err != nil {
			return &Token{p.argi, TokenError, err.Error()}
		}
		if len(p.args) == 0 {
			p.args = expanded
		} else {
			p.args = append(expanded, p.args...)
		}
		return p.Next()
	}

	return &Token{p.argi, TokenArg, arg}
}

func (p *ParseContext) Peek() *Token {
	if len(p.peek) == 0 {
		return p.Push(p.Next())
	}
	return p.peek[len(p.peek)-1]
}

func (p *ParseContext) Push(token *Token) *Token {
	p.peek = append(p.peek, token)
	return token
}

func (p *ParseContext) pop() *Token {
	end := len(p.peek) - 1
	token := p.peek[end]
	p.peek = p.peek[0:end]
	return token
}

func (p *ParseContext) String() string {
	return p.SelectedCommand.FullCommand()
}

func (p *ParseContext) matchedFlag(flag *FlagClause, value string) {
	p.Elements = append(p.Elements, &ParseElement{Clause: flag, Value: &value})
}

func (p *ParseContext) matchedArg(arg *ArgClause, value string) {
	p.Elements = append(p.Elements, &ParseElement{Clause: arg, Value: &value})
}

func (p *ParseContext) matchedCmd(cmd *CmdClause) {
	p.Elements = append(p.Elements, &ParseElement{Clause: cmd})
	p.mergeFlags(cmd.flagGroup)
	p.mergeArgs(cmd.argGroup)
	p.SelectedCommand = cmd
}

// Expand arguments from a file. Lines starting with # will be treated as comments.
func ExpandArgsFromFile(filename string) (out []string, err error) {
	if filename == "" {
		return nil, fmt.Errorf("expected @ file to expand arguments from")
	}
	r, err := os.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("failed to open arguments file %q: %s", filename, err)
	}
	defer r.Close()
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "#") {
			continue
		}
		out = append(out, line)
	}
	err = scanner.Err()
	if err != nil {
		return nil, fmt.Errorf("failed to read arguments from %q: %s", filename, err)
	}
	return
}

func parse(context *ParseContext, app *Application) (err error) {
	context.mergeFlags(app.flagGroup)
	context.mergeArgs(app.argGroup)

	cmds := app.cmdGroup
	ignoreDefault := context.ignoreDefault

loop:
	for !context.EOL() && !context.Error() {
		token := context.Peek()

		switch token.Type {
		case TokenLong, TokenShort:
			if flag, err := context.flags.parse(context); err != nil {
				if !ignoreDefault {
					if cmd := cmds.defaultSubcommand(); cmd != nil {
						cmd.completionAlts = cmds.cmdNames()
						context.matchedCmd(cmd)
						cmds = cmd.cmdGroup
						break
					}
				}
				return err
			} else if flag == HelpFlag {
				ignoreDefault = true
			}

		case TokenArg:
			if cmds.have() {
				selectedDefault := false
				cmd, ok := cmds.commands[token.String()]
				if !ok {
					if !ignoreDefault {
						if cmd = cmds.defaultSubcommand(); cmd != nil {
							cmd.completionAlts = cmds.cmdNames()
							selectedDefault = true
						}
					}
					if cmd == nil {
						return fmt.Errorf("expected command but got %q", token)
					}
				}
				if cmd == HelpCommand {
					ignoreDefault = true
				}
				cmd.completionAlts = nil
				context.matchedCmd(cmd)
				cmds = cmd.cmdGroup
				if !selectedDefault {
					context.Next()
				}
			} else if context.arguments.have() {
				if app.noInterspersed {
					// no more flags
					context.argsOnly = true
				}
				arg := context.nextArg()
				if arg == nil {
					break loop
				}
				context.matchedArg(arg, token.String())
				context.Next()
			} else {
				break loop
			}

		case TokenEOL:
			break loop
		}
	}

	// Move to innermost default command.
	for !ignoreDefault {
		if cmd := cmds.defaultSubcommand(); cmd != nil {
			cmd.completionAlts = cmds.cmdNames()
			context.matchedCmd(cmd)
			cmds = cmd.cmdGroup
		} else {
			break
		}
	}

	if context.Error() {
		return fmt.Errorf("%s", context.Peek().Value)
	}

	if !context.EOL() {
		return fmt.Errorf("unexpected %s", context.Peek())
	}

	// Set defaults for all remaining args.
	for arg := context.nextArg(); arg != nil && !arg.consumesRemainder(); arg = context.nextArg() {
		for _, defaultValue := range arg.defaultValues {
			if err := arg.value.Set(defaultValue); err != nil {
				return fmt.Errorf("invalid default value '%s' for argument '%s'", defaultValue, arg.name)
			}
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vertigo

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

const (
	channelChunkSize          = 250000 // changing the value affects performance (10k...300k ~ 15%)
	logProgressEachNthDefault = 1000000
	LineTypeToken             = "token"
	LineTypeStruct            = "struct"
	LineTypeIgnored           = "ignored"

	AccumulatorTypeStack = "stack"
	AccumulatorTypeComb  = "comb"
	AccumulatorTypeNil   = "nil"

	CharsetISO8859_1   = "iso-8859-1"
	CharsetISO8859_2   = "iso-8859-2"
	CharsetISO8859_3   = "iso-8859-3"
	CharsetISO8859_4   = "iso-8859-4"
	CharsetISO8859_5   = "iso-8859-5"
	CharsetISO8859_6   = "iso-8859-6"
	CharsetISO8859_7   = "iso-8859-7"
	CharsetISO8859_8   = "iso-8859-8"
	CharsetWindows1250 = "windows-1250"
	CharsetWindows1251 = "windows-1251"
	CharsetWindows1252 = "windows-1252"
	CharsetWindows1253 = "windows-1253"
	CharsetWindows1254 = "windows-1254"
	CharsetWindows1255 = "windows-1255"
	CharsetWindows1256 = "windows-1256"
	CharsetWindows1257 = "windows-1257"
	CharsetWindows1258 = "windows-1258"
	CharsetUTF_8       = "utf-8"
)

var (
	tagSrchRegexp  = regexp.MustCompile("^<([\\w\\d\\p{Po}]+)(\\s+.*?|)/?>$")
	attrValRegexp  = regexp.MustCompile("(\\w+)=\"([^\"]+)\"")
	closeTagRegexp = regexp.MustCompile("</([^>]+)\\s*>")
)

// --------------------------------------------------------

// ParserConf contains configuration parameters for
// vertical file parser
type ParserConf struct {

	// Source vertical file (either a plain text file or a gzip one)
	InputFilePath string `json:"inputFilePath"`

	Encoding string `json:"encoding"`

	FilterArgs [][][]string `json:"filterArgs"`

	StructAttrAccumulator string `json:"structAttrAccumulator"`

	LogProgressEachNth int `json:"logProgressEachNth"`
}

// LoadConfig loads the configuration from a JSON file.
// In case of an error the program exits with panic.
func LoadConfig(path string) *ParserConf {
	rawData, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	var conf ParserConf
	err = json.Unmarshal(rawData, &conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

// --------------------------------------------------------

// Token is a representation of
// a parsed line. It connects both, positional attributes
// and currently accumulated structural attributes.
type Token struct {
	Idx         int
	Word        string
	Attrs       []string
	StructAttrs map[string]string
}

func (v *Token) WordLC() string {
	return strings.ToLower(v.Word)
}

// --------------------------------------------------------

type Structure struct {
	Name  string
	Attrs map[string]string
}

// --------------------------------------------------------

type StructureClose struct {
	Name string
}

// --------------------------------------------------------

type LineProcessor interface {
	ProcToken(token *Token)
	ProcStruct(strc *Structure)
	ProcStructClose(strc *StructureClose)
}

// --------------------------------------------------------

// this is quite simplified but it should work for our purposes
func isElement(tagSrc string) bool {
	return strings.HasPrefix(tagSrc, "<") && strings.HasSuffix(tagSrc, ">")
}

func isOpenElement(tagSrc string) bool {
	return isElement(tagSrc) && !strings.HasPrefix(tagSrc, "</") &&
		!strings.HasSuffix(tagSrc, "/>")
}

func isCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasPrefix(tagSrc, "</")
}

func isSelfCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasSuffix(tagSrc, "/>")
}

func parseAttrVal(src string) map[string]string {
	ans := make(map[string]string)
	srch := attrValRegexp.FindAllStringSubmatch(src, -1)
	for i := 0; i < len(srch); i++ {
		ans[srch[i][1]] = srch[i][2]
	}
	return ans
}

func parseLine(line string, elmStack structAttrAccumulator) interface{} {
	switch {
	case isOpenElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		meta := &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
		elmStack.Begin(meta)
		return meta
	case isCloseElement(line):
		srch := closeTagRegexp.FindStringSubmatch(line)
		elmStack.End(srch[1])
		return &StructureClose{Name: srch[1]}
	case isSelfCloseElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		return &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
	default:
		items := strings.Split(line, "\t")
		return &Token{
			Word:        items[0],
			Attrs:       items[1:],
			StructAttrs: elmStack.GetAttrs(),
		}
	}
}

// TokenMatchesFilter tests whether a provided token matches
// a filter in Conjunctive normal form encoded as a 3-d list
// E.g.:
// div.author = 'John Doe' AND (div.title = 'Unknown' OR div.title = 'Superunknown')
// encodes as:
// { {{"div.author" "John Doe"}} {{"div.title" "Unknown"} {"div.title" "Superunknown"}} }
func TokenMatchesFilter(token *Token, filterCNF [][][]string) bool {
	var sub bool
	for _, item := range filterCNF {
		sub = false
		for _, v := range item {
			if v[1] == token.StructAttrs[v[0]] {
				sub = true
				break
			}
		}
		if sub == false {
			return false
		}
	}
	return true
}

func createStructAttrAccumulator(ident string) (structAttrAccumulator, error) {
	switch ident {
	case AccumulatorTypeStack:
		return newStack(), nil
	case AccumulatorTypeComb:
		return newStructAttrs(), nil
	case AccumulatorTypeNil:
		return newNilStructAttrs(), nil
	default:
		return nil, fmt.Errorf("Unknown accumulator type \"%s\"", ident)
	}
}

// SupportedCharsets returns a list of names of
// character sets.
func SupportedCharsets() []string {
	return []string{CharsetISO8859_2, CharsetUTF_8, CharsetWindows1250}
}

// GetCharmapByName returns a proper Charmap instance based
// on provided encoding name. The name detection is case
// insensitive (e.g. utf-8 is the same as UTF-8). The number
// of supported charsets is
func GetCharmapByName(name string) (*charmap.Charmap, error) {
	switch strings.ToLower(name) {
	case CharsetISO8859_1:
		return charmap.ISO8859_1, nil
	case CharsetISO8859_2:
		return charmap.ISO8859_2, nil
	case CharsetISO8859_3:
		return charmap.ISO8859_3, nil
	case CharsetISO8859_4:
		return charmap.ISO8859_4, nil
	case CharsetISO8859_5:
		return charmap.ISO8859_5, nil
	case CharsetISO8859_6:
		return charmap.ISO8859_6, nil
	case CharsetISO8859_7:
		return charmap.ISO8859_7, nil
	case CharsetISO8859_8:
		return charmap.ISO8859_8, nil
	case CharsetWindows1250:
		return charmap.Windows1250, nil
	case CharsetWindows1251:
		return charmap.Windows1251, nil
	case CharsetWindows1252:
		return charmap.Windows1252, nil
	case CharsetWindows1253:
		return charmap.Windows1253, nil
	case CharsetWindows1254:
		return charmap.Windows1254, nil
	case CharsetWindows1255:
		return charmap.Windows1255, nil
	case CharsetWindows1256:
		return charmap.Windows1256, nil
	case CharsetWindows1257:
		return charmap.Windows1257, nil
	case CharsetWindows1258:
		return charmap.Windows1258, nil
	case CharsetUTF_8:
		return nil, nil
	case "":
		log.Printf("No charset specified, assuming utf-8")
		return nil, nil
	default:
		return nil, fmt.Errorf("Unsupported charset '%s'", name)
	}
}

func importString(s string, ch *charmap.Charmap) string {
	if ch == nil {
		return s
	}
	ans, _, _ := transform.String(ch.NewDecoder(), s)
	// TODO handle error
	return ans
}

func openInputFile(path string) (io.Reader, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	finfo, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if !finfo.Mode().IsRegular() {
		return nil, fmt.Errorf("Path %s is not a regular file", path)
	}

	var rd io.Reader
	if strings.HasSuffix(path, ".gz") {
		rd, err = gzip.NewReader(f)
		if err != nil {
			return nil, err
		}

	} else {
		rd = f
	}
	return rd, nil
}

// ParseVerticalFile processes a corpus vertical file
// line by line and applies a custom LineProcessor on
// them. The processing is parallelized in the sense
// that reading a file into lines and processing of
// the lines runs in different goroutines. To reduce
// overhead, the data are passed between goroutines
// in chunks.
func ParseVerticalFile(conf *ParserConf, lproc LineProcessor) error {
	logProgressEachNth := logProgressEachNthDefault
	if conf.LogProgressEachNth > 0 {
		logProgressEachNth = conf.LogProgressEachNth
	}
	rd, err := openInputFile(conf.InputFilePath)
	if err != nil {
		return err
	}
	brd := bufio.NewScanner(rd)

	stack, err := createStructAttrAccumulator(conf.StructAttrAccumulator)
	if err != nil {
		return err
	}

	chm, chErr := GetCharmapByName(conf.Encoding)
	if chErr != nil {
		return chErr
	}
	log.Printf("Configured conversion from charset %s", chm)
	ch := make(chan []interface{})
	chunk := make([]interface{}, channelChunkSize)
	go func() {
		i := 0
		progress := 0
		tokenNum := 0
		for brd.Scan() {
			line := parseLine(importString(brd.Text(), chm), stack)
			tok, isTok := line.(Token)
			if isTok {
				tok.Idx = tokenNum
				tokenNum++
			}
			chunk[i] = line
			i++
			if i == channelChunkSize {
				i = 0
				ch <- chunk
				chunk = make([]interface{}, channelChunkSize)
			}
			progress++
			if progress%logProgressEachNth == 0 {
				log.Printf("...processed %d lines.\n", progress)
			}
		}
		if i > 0 {
			ch <- chunk[:i]
		}
		close(ch)
	}()

	for tokens := range ch {
		for _, token := range tokens {
			switch token.(type) {
			case *Token:
				tk := token.(*Token)
				if TokenMatchesFilter(tk, conf.FilterArgs) {
					lproc.ProcToken(tk)
				}
			case *Structure:
				lproc.ProcStruct(token.(*Structure))
			case *StructureClose:
				lproc.ProcStructClose(token.(*StructureClose))
			}
		}
	}
	log.Println("Parsing done. Metadata stack size: ", stack.Size())
	return nil
}

//ParseVerticalFileNoGoRo is just for benchmarking purposes
func ParseVerticalFileNoGoRo(conf *ParserConf, lproc LineProcessor) {
	f, err := os.Open(conf.InputFilePath)
	if err != nil {
		panic(err)
	}
	rd := bufio.NewScanner(f)
	stack := newStack()

	for rd.Scan() {
		token := parseLine(rd.Text(), stack)
		switch token.(type) {
		case *Token:
			lproc.ProcToken(token.(*Token))
		}
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
}
<commit_msg>Fix type assertion<commit_after>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vertigo

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

const (
	channelChunkSize          = 250000 // changing the value affects performance (10k...300k ~ 15%)
	logProgressEachNthDefault = 1000000
	LineTypeToken             = "token"
	LineTypeStruct            = "struct"
	LineTypeIgnored           = "ignored"

	AccumulatorTypeStack = "stack"
	AccumulatorTypeComb  = "comb"
	AccumulatorTypeNil   = "nil"

	CharsetISO8859_1   = "iso-8859-1"
	CharsetISO8859_2   = "iso-8859-2"
	CharsetISO8859_3   = "iso-8859-3"
	CharsetISO8859_4   = "iso-8859-4"
	CharsetISO8859_5   = "iso-8859-5"
	CharsetISO8859_6   = "iso-8859-6"
	CharsetISO8859_7   = "iso-8859-7"
	CharsetISO8859_8   = "iso-8859-8"
	CharsetWindows1250 = "windows-1250"
	CharsetWindows1251 = "windows-1251"
	CharsetWindows1252 = "windows-1252"
	CharsetWindows1253 = "windows-1253"
	CharsetWindows1254 = "windows-1254"
	CharsetWindows1255 = "windows-1255"
	CharsetWindows1256 = "windows-1256"
	CharsetWindows1257 = "windows-1257"
	CharsetWindows1258 = "windows-1258"
	CharsetUTF_8       = "utf-8"
)

var (
	tagSrchRegexp  = regexp.MustCompile("^<([\\w\\d\\p{Po}]+)(\\s+.*?|)/?>$")
	attrValRegexp  = regexp.MustCompile("(\\w+)=\"([^\"]+)\"")
	closeTagRegexp = regexp.MustCompile("</([^>]+)\\s*>")
)

// --------------------------------------------------------

// ParserConf contains configuration parameters for
// vertical file parser
type ParserConf struct {

	// Source vertical file (either a plain text file or a gzip one)
	InputFilePath string `json:"inputFilePath"`

	Encoding string `json:"encoding"`

	FilterArgs [][][]string `json:"filterArgs"`

	StructAttrAccumulator string `json:"structAttrAccumulator"`

	LogProgressEachNth int `json:"logProgressEachNth"`
}

// LoadConfig loads the configuration from a JSON file.
// In case of an error the program exits with panic.
func LoadConfig(path string) *ParserConf {
	rawData, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	var conf ParserConf
	err = json.Unmarshal(rawData, &conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

// --------------------------------------------------------

// Token is a representation of
// a parsed line. It connects both, positional attributes
// and currently accumulated structural attributes.
type Token struct {
	Idx         int
	Word        string
	Attrs       []string
	StructAttrs map[string]string
}

func (v *Token) WordLC() string {
	return strings.ToLower(v.Word)
}

// --------------------------------------------------------

type Structure struct {
	Name  string
	Attrs map[string]string
}

// --------------------------------------------------------

type StructureClose struct {
	Name string
}

// --------------------------------------------------------

type LineProcessor interface {
	ProcToken(token *Token)
	ProcStruct(strc *Structure)
	ProcStructClose(strc *StructureClose)
}

// --------------------------------------------------------

// this is quite simplified but it should work for our purposes
func isElement(tagSrc string) bool {
	return strings.HasPrefix(tagSrc, "<") && strings.HasSuffix(tagSrc, ">")
}

func isOpenElement(tagSrc string) bool {
	return isElement(tagSrc) && !strings.HasPrefix(tagSrc, "</") &&
		!strings.HasSuffix(tagSrc, "/>")
}

func isCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasPrefix(tagSrc, "</")
}

func isSelfCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasSuffix(tagSrc, "/>")
}

func parseAttrVal(src string) map[string]string {
	ans := make(map[string]string)
	srch := attrValRegexp.FindAllStringSubmatch(src, -1)
	for i := 0; i < len(srch); i++ {
		ans[srch[i][1]] = srch[i][2]
	}
	return ans
}

func parseLine(line string, elmStack structAttrAccumulator) interface{} {
	switch {
	case isOpenElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		meta := &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
		elmStack.Begin(meta)
		return meta
	case isCloseElement(line):
		srch := closeTagRegexp.FindStringSubmatch(line)
		elmStack.End(srch[1])
		return &StructureClose{Name: srch[1]}
	case isSelfCloseElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		return &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
	default:
		items := strings.Split(line, "\t")
		return &Token{
			Word:        items[0],
			Attrs:       items[1:],
			StructAttrs: elmStack.GetAttrs(),
		}
	}
}

// TokenMatchesFilter tests whether a provided token matches
// a filter in Conjunctive normal form encoded as a 3-d list
// E.g.:
// div.author = 'John Doe' AND (div.title = 'Unknown' OR div.title = 'Superunknown')
// encodes as:
// { {{"div.author" "John Doe"}} {{"div.title" "Unknown"} {"div.title" "Superunknown"}} }
func TokenMatchesFilter(token *Token, filterCNF [][][]string) bool {
	var sub bool
	for _, item := range filterCNF {
		sub = false
		for _, v := range item {
			if v[1] == token.StructAttrs[v[0]] {
				sub = true
				break
			}
		}
		if sub == false {
			return false
		}
	}
	return true
}

func createStructAttrAccumulator(ident string) (structAttrAccumulator, error) {
	switch ident {
	case AccumulatorTypeStack:
		return newStack(), nil
	case AccumulatorTypeComb:
		return newStructAttrs(), nil
	case AccumulatorTypeNil:
		return newNilStructAttrs(), nil
	default:
		return nil, fmt.Errorf("Unknown accumulator type \"%s\"", ident)
	}
}

// SupportedCharsets returns a list of names of
// character sets.
func SupportedCharsets() []string {
	return []string{CharsetISO8859_2, CharsetUTF_8, CharsetWindows1250}
}

// GetCharmapByName returns a proper Charmap instance based
// on provided encoding name. The name detection is case
// insensitive (e.g. utf-8 is the same as UTF-8). The number
// of supported charsets is
func GetCharmapByName(name string) (*charmap.Charmap, error) {
	switch strings.ToLower(name) {
	case CharsetISO8859_1:
		return charmap.ISO8859_1, nil
	case CharsetISO8859_2:
		return charmap.ISO8859_2, nil
	case CharsetISO8859_3:
		return charmap.ISO8859_3, nil
	case CharsetISO8859_4:
		return charmap.ISO8859_4, nil
	case CharsetISO8859_5:
		return charmap.ISO8859_5, nil
	case CharsetISO8859_6:
		return charmap.ISO8859_6, nil
	case CharsetISO8859_7:
		return charmap.ISO8859_7, nil
	case CharsetISO8859_8:
		return charmap.ISO8859_8, nil
	case CharsetWindows1250:
		return charmap.Windows1250, nil
	case CharsetWindows1251:
		return charmap.Windows1251, nil
	case CharsetWindows1252:
		return charmap.Windows1252, nil
	case CharsetWindows1253:
		return charmap.Windows1253, nil
	case CharsetWindows1254:
		return charmap.Windows1254, nil
	case CharsetWindows1255:
		return charmap.Windows1255, nil
	case CharsetWindows1256:
		return charmap.Windows1256, nil
	case CharsetWindows1257:
		return charmap.Windows1257, nil
	case CharsetWindows1258:
		return charmap.Windows1258, nil
	case CharsetUTF_8:
		return nil, nil
	case "":
		log.Printf("No charset specified, assuming utf-8")
		return nil, nil
	default:
		return nil, fmt.Errorf("Unsupported charset '%s'", name)
	}
}

func importString(s string, ch *charmap.Charmap) string {
	if ch == nil {
		return s
	}
	ans, _, _ := transform.String(ch.NewDecoder(), s)
	// TODO handle error
	return ans
}

func openInputFile(path string) (io.Reader, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	finfo, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if !finfo.Mode().IsRegular() {
		return nil, fmt.Errorf("Path %s is not a regular file", path)
	}

	var rd io.Reader
	if strings.HasSuffix(path, ".gz") {
		rd, err = gzip.NewReader(f)
		if err != nil {
			return nil, err
		}

	} else {
		rd = f
	}
	return rd, nil
}

// ParseVerticalFile processes a corpus vertical file
// line by line and applies a custom LineProcessor on
// them. The processing is parallelized in the sense
// that reading a file into lines and processing of
// the lines runs in different goroutines. To reduce
// overhead, the data are passed between goroutines
// in chunks.
func ParseVerticalFile(conf *ParserConf, lproc LineProcessor) error {
	logProgressEachNth := logProgressEachNthDefault
	if conf.LogProgressEachNth > 0 {
		logProgressEachNth = conf.LogProgressEachNth
	}
	rd, err := openInputFile(conf.InputFilePath)
	if err != nil {
		return err
	}
	brd := bufio.NewScanner(rd)

	stack, err := createStructAttrAccumulator(conf.StructAttrAccumulator)
	if err != nil {
		return err
	}

	chm, chErr := GetCharmapByName(conf.Encoding)
	if chErr != nil {
		return chErr
	}
	log.Printf("Configured conversion from charset %s", chm)
	ch := make(chan []interface{})
	chunk := make([]interface{}, channelChunkSize)
	go func() {
		i := 0
		progress := 0
		tokenNum := 0
		for brd.Scan() {
			line := parseLine(importString(brd.Text(), chm), stack)
			tok, isTok := line.(*Token)
			if isTok {
				tok.Idx = tokenNum
				tokenNum++
			}
			chunk[i] = line
			i++
			if i == channelChunkSize {
				i = 0
				ch <- chunk
				chunk = make([]interface{}, channelChunkSize)
			}
			progress++
			if progress%logProgressEachNth == 0 {
				log.Printf("...processed %d lines.\n", progress)
			}
		}
		if i > 0 {
			ch <- chunk[:i]
		}
		close(ch)
	}()

	for tokens := range ch {
		for _, token := range tokens {
			switch token.(type) {
			case *Token:
				tk := token.(*Token)
				if TokenMatchesFilter(tk, conf.FilterArgs) {
					lproc.ProcToken(tk)
				}
			case *Structure:
				lproc.ProcStruct(token.(*Structure))
			case *StructureClose:
				lproc.ProcStructClose(token.(*StructureClose))
			}
		}
	}
	log.Println("Parsing done. Metadata stack size: ", stack.Size())
	return nil
}

//ParseVerticalFileNoGoRo is just for benchmarking purposes
func ParseVerticalFileNoGoRo(conf *ParserConf, lproc LineProcessor) {
	f, err := os.Open(conf.InputFilePath)
	if err != nil {
		panic(err)
	}
	rd := bufio.NewScanner(f)
	stack := newStack()

	for rd.Scan() {
		token := parseLine(rd.Text(), stack)
		switch token.(type) {
		case *Token:
			lproc.ProcToken(token.(*Token))
		}
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
}
<|endoftext|>


<commit_before>/* The MIT License (MIT)

Portions of this code Copyright (c) 2013-2015 Errplane Inc.
Copyright (c) 2015 Jesse Nelson

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package metadata

import (
	"fmt"
	"io"

	"github.com/hashicorp/go-version"
)

// Metadata represents a coookbook metadata file
type Metadata struct {
	Depends []Dependency
	Name    string
	Version version.Version
}

type Dependency struct {
	Name       string
	Constraint version.Constraints // ... maybe look at hashi version.Constraint
}

// Parser represents a parser.
type Parser struct {
	s *bufScanner
}

// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
	return &Parser{s: newBufScanner(r)}
}

// Parse parses a metadata file
func (p *Parser) Parse() (md *Metadata, err error) {
	// return decl inits to nil, we need this to be something we can use
	md = &Metadata{}
	for {
		tok, _, _ := p.scanIgnoreWhitespace()

		switch tok {
		case EOF:
			return md, nil
		case NAME:
			md.Name, err = p.parseName()
			if err != nil {
				return md, err
			}
			p.unscan()
		case DEPENDS:
			d, err := p.parseDepends()
			if err != nil {
				return md, err
			}
			md.Depends = append(md.Depends, d)
		case VERSION:
			v, err := p.parseVersion()
			if err != nil {
				return md, err
			}
			md.Version = v
		}
	}
}

func (p *Parser) parseVersion() (version.Version, error) {
	tok, pos, lit := p.scanIgnoreWhitespace()
	if tok != STRING {
		return version.Version{}, fmt.Errorf("found %s expected STRING at %d, %s", tok, pos, lit)
	}

	v, err := version.NewVersion(lit)
	if err != nil {
		return *v, fmt.Errorf("error parsing version at %d, %s : %s", pos, lit, err)
	}

	return *v, nil
}

func (p *Parser) parseName() (string, error) {
	tok, pos, lit := p.scanIgnoreWhitespace()
	if tok != STRING {
		return "", fmt.Errorf("found %s expected STRING at %d, %s", tok, pos, lit)
	}

	return lit, nil
}

func (p *Parser) parseDepends() (Dependency, error) {
	tok, pos, lit := p.scanIgnoreWhitespace()
	if tok != STRING {
		return Dependency{}, fmt.Errorf("found %s expected STRING at %d, %s", tok, pos, lit)
	}

	d := Dependency{}
	d.Name = lit
	// scan forward for the expected comma if there is a dep, if no comma no dep!
	tok, pos, lit = p.scanIgnoreWhitespace()
	if tok != COMMA {
		return d, nil
	}

	// scan forward for the dep
	tok, pos, lit = p.scanIgnoreWhitespace()
	if tok != STRING {
		return d, fmt.Errorf("found %s expected constraint STRING at %d, %s", tok, pos, lit)
	}

	// we have a dependency here
	c, err := version.NewConstraint(lit)
	if err != nil {
		return d, fmt.Errorf("error parsing constriant at %d, %s : %s", pos, lit, err)
	}
	d.Constraint = c

	return d, nil
}

// scan returns the next token from the underlying scanner.
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.s.Scan() }

// scanIgnoreWhitespace scans the next non-whitespace token.
func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) {
	tok, pos, lit = p.scan()
	if tok == WS {
		tok, pos, lit = p.scan()
	}
	return
}

// unscan pushes the previously read token back onto the buffer.
func (p *Parser) unscan() { p.s.Unscan() }
<commit_msg>for version segment length<commit_after>/* The MIT License (MIT)

Portions of this code Copyright (c) 2013-2015 Errplane Inc.
Copyright (c) 2015 Jesse Nelson

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package metadata

import (
	"fmt"
	"io"

	"github.com/hashicorp/go-version"
)

// Metadata represents a coookbook metadata file
type Metadata struct {
	Depends []Dependency
	Name    string
	Version version.Version
}

type Dependency struct {
	Name       string
	Constraint version.Constraints // ... maybe look at hashi version.Constraint
}

// Parser represents a parser.
type Parser struct {
	s *bufScanner
}

// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
	return &Parser{s: newBufScanner(r)}
}

// Parse parses a metadata file
func (p *Parser) Parse() (md *Metadata, err error) {
	// return decl inits to nil, we need this to be something we can use
	md = &Metadata{}
	for {
		tok, _, _ := p.scanIgnoreWhitespace()

		switch tok {
		case EOF:
			return md, nil
		case NAME:
			md.Name, err = p.parseName()
			if err != nil {
				return md, err
			}
			p.unscan()
		case DEPENDS:
			d, err := p.parseDepends()
			if err != nil {
				return md, err
			}
			md.Depends = append(md.Depends, d)
		case VERSION:
			v, err := p.parseVersion()
			if err != nil {
				return nil, err
			}
			md.Version = v
		}
	}
}

func (p *Parser) parseVersion() (version.Version, error) {
	tok, pos, lit := p.scanIgnoreWhitespace()
	if tok != STRING {
		return version.Version{}, fmt.Errorf("found %s expected STRING at %d, %s", tok, pos, lit)
	}

	v, err := version.NewVersion(lit)
	if err != nil {
		return *v, fmt.Errorf("error parsing version at %d, %s : %s", pos, lit, err)
	}

	if len(v.Segments()) < 3 {
		return *v, fmt.Errorf("error parsing version,not enough segments at %d, %s : %s", pos, lit, err)
	}

	return *v, nil
}

func (p *Parser) parseName() (string, error) {
	tok, pos, lit := p.scanIgnoreWhitespace()
	if tok != STRING {
		return "", fmt.Errorf("found %s expected STRING at %d, %s", tok, pos, lit)
	}

	return lit, nil
}

func (p *Parser) parseDepends() (Dependency, error) {
	tok, pos, lit := p.scanIgnoreWhitespace()
	if tok != STRING {
		return Dependency{}, fmt.Errorf("found %s expected STRING at %d, %s", tok, pos, lit)
	}

	d := Dependency{}
	d.Name = lit
	// scan forward for the expected comma if there is a dep, if no comma no dep!
	tok, pos, lit = p.scanIgnoreWhitespace()
	if tok != COMMA {
		return d, nil
	}

	// scan forward for the dep
	tok, pos, lit = p.scanIgnoreWhitespace()
	if tok != STRING {
		return d, fmt.Errorf("found %s expected constraint STRING at %d, %s", tok, pos, lit)
	}

	// we have a dependency here
	c, err := version.NewConstraint(lit)
	if err != nil {
		return d, fmt.Errorf("error parsing constriant at %d, %s : %s", pos, lit, err)
	}
	d.Constraint = c

	return d, nil
}

// scan returns the next token from the underlying scanner.
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.s.Scan() }

// scanIgnoreWhitespace scans the next non-whitespace token.
func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) {
	tok, pos, lit = p.scan()
	if tok == WS {
		tok, pos, lit = p.scan()
	}
	return
}

// unscan pushes the previously read token back onto the buffer.
func (p *Parser) unscan() { p.s.Unscan() }
<|endoftext|>


<commit_before>package yaml

import (
	"fmt"
	"io"
)

type Parser struct {
	scanner    *Scanner
	directives *Directives
}

func NewParser(reader io.Reader) *Parser {
	return &Parser{scanner: NewScanner(reader), directives: NewDirectives()}
}

func (p *Parser) IsValid() bool {
	return !p.scanner.Empty()
}

func (p *Parser) Load(reader io.Reader) {
	p.scanner = NewScanner(reader)
	p.directives = NewDirectives()
}

func (p *Parser) HandleNextDocument(evtHandler EventHandler) bool {
	if p.scanner == nil {
		return false
	}

	p.parseDirectives()
	if p.scanner.Empty() {
		return false
	}

	sdp := &singleDocParser{p.scanner, p.directives}
	sdp.handleDocument(evtHandler)
	return true
}

func (p *Parser) PrintTokens() (output string) {
	if p.scanner == nil {
		return
	}

	for !p.scanner.Empty() {
		output += p.scanner.Peek().String() + "\n"
		p.scanner.Pop()
	}
	return
}

func (p *Parser) parseDirectives() {
	readDirective := false

	for !p.scanner.Empty() {
		token := p.scanner.Peek()
		if token.Type != DIRECTIVE {
			break
		} else if !readDirective {
			// we keep the directives from the last document if none are specified
			// but if any directives are specific, then we reset them
			p.directives = NewDirectives()
		}

		readDirective = true
		p.handleDirective(token)
		p.scanner.Pop()
	}
}

func (p *Parser) handleDirective(token *Token) {
	switch token.Value {
	case "YAML":
		p.handleYamlDirective(token)
	case "TAG":
		p.handleTagDirective(token)
	}
}

func (p *Parser) handleYamlDirective(token *Token) {
	if len(token.Params) != 1 {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_YAML_DIRECTIVE_ARGS))
	} else if !p.directives.Version.IsDefault {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_REPEATED_YAML_DIRECTIVE))
	} else if c, err := fmt.Sscanf(token.Params[0], "%d.%d", &p.directives.Version.Major, &p.directives.Version.Minor); c != 2 || err != nil {
		panic(fmt.Sprintf("%v - %v %v", token.Mark, ERR_YAML_VERSION, token.Params[0]))
	} else if p.directives.Version.Major > 1 {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_YAML_MAJOR_VERSION))
	}

	p.directives.Version.IsDefault = false
}

func (p *Parser) handleTagDirective(token *Token) {
	if len(token.Params) != 2 {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_TAG_DIRECTIVE_ARGS))
	}

	handle := token.Params[0]
	if _, ok := p.directives.Tags[handle]; !ok {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_REPEATED_TAG_DIRECTIVE))
	} else {
		p.directives.Tags[handle] = token.Params[1] // token.Params[1] == prefix
	}
}

/********************************/
/* Single document parsing code */
/********************************/

type singleDocParser struct {
	scanner    *Scanner
	directives *Directives
}

func (s *singleDocParser) handleDocument(evtHandler EventHandler) {

}
<commit_msg>Add single doc parsing placeholder functions.<commit_after>package yaml

import (
	"fmt"
	"io"
)

type Parser struct {
	scanner    *Scanner
	directives *Directives
}

func NewParser(reader io.Reader) *Parser {
	return &Parser{scanner: NewScanner(reader), directives: NewDirectives()}
}

func (p *Parser) IsValid() bool {
	return !p.scanner.Empty()
}

func (p *Parser) Load(reader io.Reader) {
	p.scanner = NewScanner(reader)
	p.directives = NewDirectives()
}

func (p *Parser) HandleNextDocument(evtHandler EventHandler) bool {
	if p.scanner == nil {
		return false
	}

	p.parseDirectives()
	if p.scanner.Empty() {
		return false
	}

	sdp := &singleDocParser{p.scanner, p.directives}
	sdp.handleDocument(evtHandler)
	return true
}

func (p *Parser) PrintTokens() (output string) {
	if p.scanner == nil {
		return
	}

	for !p.scanner.Empty() {
		output += p.scanner.Peek().String() + "\n"
		p.scanner.Pop()
	}
	return
}

func (p *Parser) parseDirectives() {
	readDirective := false

	for !p.scanner.Empty() {
		token := p.scanner.Peek()
		if token.Type != DIRECTIVE {
			break
		} else if !readDirective {
			// we keep the directives from the last document if none are specified
			// but if any directives are specific, then we reset them
			p.directives = NewDirectives()
		}

		readDirective = true
		p.handleDirective(token)
		p.scanner.Pop()
	}
}

func (p *Parser) handleDirective(token *Token) {
	switch token.Value {
	case "YAML":
		p.handleYamlDirective(token)
	case "TAG":
		p.handleTagDirective(token)
	}
}

func (p *Parser) handleYamlDirective(token *Token) {
	if len(token.Params) != 1 {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_YAML_DIRECTIVE_ARGS))
	} else if !p.directives.Version.IsDefault {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_REPEATED_YAML_DIRECTIVE))
	} else if c, err := fmt.Sscanf(token.Params[0], "%d.%d", &p.directives.Version.Major, &p.directives.Version.Minor); c != 2 || err != nil {
		panic(fmt.Sprintf("%v - %v %v", token.Mark, ERR_YAML_VERSION, token.Params[0]))
	} else if p.directives.Version.Major > 1 {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_YAML_MAJOR_VERSION))
	}

	p.directives.Version.IsDefault = false
}

func (p *Parser) handleTagDirective(token *Token) {
	if len(token.Params) != 2 {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_TAG_DIRECTIVE_ARGS))
	}

	handle := token.Params[0]
	if _, ok := p.directives.Tags[handle]; !ok {
		panic(fmt.Sprintf("%v - %v", token.Mark, ERR_REPEATED_TAG_DIRECTIVE))
	} else {
		p.directives.Tags[handle] = token.Params[1] // token.Params[1] == prefix
	}
}

/********************************/
/* Single document parsing code */
/********************************/

type singleDocParser struct {
	scanner    *Scanner
	directives *Directives
}

func (s *singleDocParser) handleDocument(evtHandler EventHandler) {
}

func (s *singleDocParser) handleNode(evtHandler EventHandler) {
}

func (s *singleDocParser) handleSequence(evtHandler EventHandler) {
}

func (s *singleDocParser) handleBlockSequence(evtHandler EventHandler) {
}

func (s *singleDocParser) handleFlowSequence(evtHandler EventHandler) {
}

func (s *singleDocParser) handleMap(evtHandler EventHandler) {
}

func (s *singleDocParser) handleBlockMap(evtHandler EventHandler) {
}

func (s *singleDocParser) handleFlowMap(evtHandler EventHandler) {
}

func (s *singleDocParser) handleCompactMap(evtHandler EventHandler) {
}

func (s *singleDocParser) handleCompactMapWithNoKey(evtHandler EventHandler) {
}

func (s *singleDocParser) parseProperties(tag string, anchor Anchor) {
}

func (s *singleDocParser) parseTag(tag string) {
}

func (s *singleDocParser) parseAnchor(anchor Anchor) {
}

func (s *singleDocParser) registerAnchor(name string) Anchor {
	return NullAnchor
}

func (s *singleDocParser) lookupAnchor(mark Mark, name string) Anchor {
	return NullAnchor
}
<|endoftext|>


<commit_before><commit_msg>Added troll plugin :)<commit_after><|endoftext|>