<commit_before>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	stdpath "path"
	"strings"

	"github.com/pkg/errors"
	"gonum.org/v1/plot/vg"

	"go-hep.org/x/hep/groot/riofs"
	"go-hep.org/x/hep/groot/root"
	"go-hep.org/x/hep/groot/rsrv"
	"go-hep.org/x/hep/groot/rtree"
)

type jsNode struct {
	ID    string `json:"id,omitempty"`
	URI   string `json:"uri,omitempty"`
	Dir   string `json:"dir,omitempty"`
	Obj   string `json:"obj,omitempty"`
	Text  string `json:"text,omitempty"`
	Icon  string `json:"icon,omitempty"`
	State struct {
		Opened   bool `json:"opened,omitempty"`
		Disabled bool `json:"disabled,omitempty"`
		Selected bool `json:"selected,omitempty"`
	} `json:"state,omitempty"`
	Children []jsNode `json:"children,omitempty"`
	LiAttr   jsAttr   `json:"li_attr,omitempty"`
	Attr     jsAttr   `json:"a_attr,omitempty"`
}

type jsAttr map[string]interface{}

type brancher interface {
	Branches() []rtree.Branch
}

type jsNodes []jsNode

func (p jsNodes) Len() int           { return len(p) }
func (p jsNodes) Less(i, j int) bool { return p[i].ID < p[j].ID }
func (p jsNodes) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func newJsNodes(bres brancher, parent jsNode) ([]jsNode, error) {
	var err error
	branches := bres.Branches()
	if len(branches) <= 0 {
		return nil, err
	}
	var nodes []jsNode
	for _, b := range branches {
		id := parent.ID
		bid := strings.Join([]string{id, b.Name()}, "/")
		node := jsNode{
			ID:   bid,
			URI:  parent.URI,
			Dir:  stdpath.Join(parent.Dir, parent.Obj),
			Obj:  b.Name(),
			Text: b.Name(),
			Icon: "fa fa-leaf",
		}
		node.Attr, err = attrFor(b.(root.Object), node)
		if err != nil {
			return nil, err
		}
		node.Children, err = newJsNodes(b, node)
		if err != nil {
			return nil, err
		}
		nodes = append(nodes, node)
	}
	return nodes, nil
}

func fileJsTree(f *riofs.File, fname string) ([]jsNode, error) {
	root := jsNode{
		ID:   f.Name(),
		URI:  fname,
		Dir:  "/",
		Text: fmt.Sprintf("%s (version=%v)", fname, f.Version()),
		Icon: "fa fa-file",
	}
	root.State.Opened = true
	return dirTree(f, fname, root)
}

func dirTree(dir riofs.Directory, path string, root jsNode) ([]jsNode, error) {
	var nodes []jsNode
	for _, k := range dir.Keys() {
		obj, err := k.Object()
		if err != nil {
			return nil, fmt.Errorf("failed to extract key %q: %v", k.Name(), err)
		}
		switch obj := obj.(type) {
		case rtree.Tree:
			tree := obj
			node := jsNode{
				ID:   strings.Join([]string{path, k.Name()}, "/"),
				URI:  root.URI,
				Dir:  stdpath.Join(root.Dir, root.Obj),
				Obj:  k.Name(),
				Text: fmt.Sprintf("%s (entries=%d)", k.Name(), tree.Entries()),
				Icon: "fa fa-tree",
			}
			node.Children, err = newJsNodes(tree, node)
			if err != nil {
				return nil, err
			}
			nodes = append(nodes, node)
		case riofs.Directory:
			dir := obj
			node := jsNode{
				ID:   strings.Join([]string{path, k.Name()}, "/"),
				URI:  root.URI,
				Dir:  stdpath.Join(root.Dir, root.Obj),
				Obj:  k.Name(),
				Text: k.Name(),
				Icon: "fa fa-folder",
			}
			node.Children, err = dirTree(dir, path+"/"+k.Name(), node)
			if err != nil {
				return nil, err
			}
			node.Children = node.Children[0].Children
			nodes = append(nodes, node)
		default:
			id := strings.Join([]string{path, k.Name() + fmt.Sprintf(";%d", k.Cycle())}, "/")
			node := jsNode{
				ID:   id,
				URI:  root.URI,
				Dir:  stdpath.Join(root.Dir, root.Obj),
				Obj:  k.Name(),
				Text: fmt.Sprintf("%s;%d", k.Name(), k.Cycle()),
				Icon: iconFor(obj),
			}
			node.Attr, err = attrFor(obj, node)
			if err != nil {
				return nil, err
			}
			nodes = append(nodes, node)
		}
	}
	root.Children = nodes
	return []jsNode{root}, nil
}

func iconFor(obj root.Object) string {
	cls := obj.Class()
	switch {
	case strings.HasPrefix(cls, "TH1"):
		return "fa fa-bar-chart-o"
	case strings.HasPrefix(cls, "TH2"):
		return "fa fa-bar-chart-o"
	case strings.HasPrefix(cls, "TGraph"):
		return "fa fa-bar-chart-o"
	}
	return "fa fa-cube"
}

func attrFor(obj root.Object, node jsNode) (jsAttr, error) {
	cmd := new(bytes.Buffer)
	cls := obj.Class()
	switch {
	case strings.HasPrefix(cls, "TH1"):
		req := rsrv.PlotH1Request{
			URI: node.URI,
			Dir: node.Dir,
			Obj: node.Obj,
			Options: rsrv.PlotOptions{
				Title:  node.Obj,
				Type:   "svg",
				Height: -1,
				Width:  20 * vg.Centimeter,
			},
		}
		err := json.NewEncoder(cmd).Encode(req)
		if err != nil {
			return nil, err
		}
		return jsAttr{
			"plot": true,
			"href": "/plot-h1",
			"cmd":  cmd.String(),
		}, nil
	case strings.HasPrefix(cls, "TH2"):
		req := rsrv.PlotH2Request{
			URI: node.URI,
			Dir: node.Dir,
			Obj: node.Obj,
			Options: rsrv.PlotOptions{
				Title:  node.Obj,
				Type:   "svg",
				Height: -1,
				Width:  20 * vg.Centimeter,
			},
		}
		err := json.NewEncoder(cmd).Encode(req)
		if err != nil {
			return nil, err
		}
		return jsAttr{
			"plot": true,
			"href": "/plot-h2",
			"cmd":  cmd.String(),
		}, nil
	case strings.HasPrefix(cls, "TGraph"):
		req := rsrv.PlotS2Request{
			URI: node.URI,
			Dir: node.Dir,
			Obj: node.Obj,
			Options: rsrv.PlotOptions{
				Title:  node.Obj,
				Type:   "svg",
				Height: -1,
				Width:  20 * vg.Centimeter,
			},
		}
		err := json.NewEncoder(cmd).Encode(req)
		if err != nil {
			return nil, err
		}
		return jsAttr{
			"plot": true,
			"href": "/plot-s2",
			"cmd":  cmd.String(),
		}, nil
	case strings.HasPrefix(cls, "TBranch"):
		req := rsrv.PlotTreeRequest{
			URI:  node.URI,
			Dir:  stdpath.Dir(node.Dir),
			Obj:  stdpath.Base(node.Dir),
			Vars: []string{node.Obj},
			Options: rsrv.PlotOptions{
				Title:  node.Obj,
				Type:   "svg",
				Height: -1,
				Width:  20 * vg.Centimeter,
			},
		}
		err := json.NewEncoder(cmd).Encode(req)
		if err != nil {
			return nil, err
		}
		return jsAttr{
			"plot": true,
			"href": "/plot-branch",
			"cmd":  cmd.String(),
		}, nil
	}
	return nil, errors.Errorf("unknown node type %q", cls)
}
<commit_msg>groot/cmd/root-srv: ignore types we cant plot<commit_after>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	stdpath "path"
	"strings"

	"gonum.org/v1/plot/vg"

	"go-hep.org/x/hep/groot/riofs"
	"go-hep.org/x/hep/groot/root"
	"go-hep.org/x/hep/groot/rsrv"
	"go-hep.org/x/hep/groot/rtree"
)

type jsNode struct {
	ID    string `json:"id,omitempty"`
	URI   string `json:"uri,omitempty"`
	Dir   string `json:"dir,omitempty"`
	Obj   string `json:"obj,omitempty"`
	Text  string `json:"text,omitempty"`
	Icon  string `json:"icon,omitempty"`
	State struct {
		Opened   bool `json:"opened,omitempty"`
		Disabled bool `json:"disabled,omitempty"`
		Selected bool `json:"selected,omitempty"`
	} `json:"state,omitempty"`
	Children []jsNode `json:"children,omitempty"`
	LiAttr   jsAttr   `json:"li_attr,omitempty"`
	Attr     jsAttr   `json:"a_attr,omitempty"`
}

type jsAttr map[string]interface{}

type brancher interface {
	Branches() []rtree.Branch
}

type jsNodes []jsNode

func (p jsNodes) Len() int           { return len(p) }
func (p jsNodes) Less(i, j int) bool { return p[i].ID < p[j].ID }
func (p jsNodes) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func newJsNodes(bres brancher, parent jsNode) ([]jsNode, error) {
	var err error
	branches := bres.Branches()
	if len(branches) <= 0 {
		return nil, err
	}
	var nodes []jsNode
	for _, b := range branches {
		id := parent.ID
		bid := strings.Join([]string{id, b.Name()}, "/")
		node := jsNode{
			ID:   bid,
			URI:  parent.URI,
			Dir:  stdpath.Join(parent.Dir, parent.Obj),
			Obj:  b.Name(),
			Text: b.Name(),
			Icon: "fa fa-leaf",
		}
		node.Attr, err = attrFor(b.(root.Object), node)
		if err != nil {
			return nil, err
		}
		node.Children, err = newJsNodes(b, node)
		if err != nil {
			return nil, err
		}
		nodes = append(nodes, node)
	}
	return nodes, nil
}

func fileJsTree(f *riofs.File, fname string) ([]jsNode, error) {
	root := jsNode{
		ID:   f.Name(),
		URI:  fname,
		Dir:  "/",
		Text: fmt.Sprintf("%s (version=%v)", fname, f.Version()),
		Icon: "fa fa-file",
	}
	root.State.Opened = true
	return dirTree(f, fname, root)
}

func dirTree(dir riofs.Directory, path string, root jsNode) ([]jsNode, error) {
	var nodes []jsNode
	for _, k := range dir.Keys() {
		obj, err := k.Object()
		if err != nil {
			return nil, fmt.Errorf("failed to extract key %q: %v", k.Name(), err)
		}
		switch obj := obj.(type) {
		case rtree.Tree:
			tree := obj
			node := jsNode{
				ID:   strings.Join([]string{path, k.Name()}, "/"),
				URI:  root.URI,
				Dir:  stdpath.Join(root.Dir, root.Obj),
				Obj:  k.Name(),
				Text: fmt.Sprintf("%s (entries=%d)", k.Name(), tree.Entries()),
				Icon: "fa fa-tree",
			}
			node.Children, err = newJsNodes(tree, node)
			if err != nil {
				return nil, err
			}
			nodes = append(nodes, node)
		case riofs.Directory:
			dir := obj
			node := jsNode{
				ID:   strings.Join([]string{path, k.Name()}, "/"),
				URI:  root.URI,
				Dir:  stdpath.Join(root.Dir, root.Obj),
				Obj:  k.Name(),
				Text: k.Name(),
				Icon: "fa fa-folder",
			}
			node.Children, err = dirTree(dir, path+"/"+k.Name(), node)
			if err != nil {
				return nil, err
			}
			node.Children = node.Children[0].Children
			nodes = append(nodes, node)
		default:
			id := strings.Join([]string{path, k.Name() + fmt.Sprintf(";%d", k.Cycle())}, "/")
			node := jsNode{
				ID:   id,
				URI:  root.URI,
				Dir:  stdpath.Join(root.Dir, root.Obj),
				Obj:  k.Name(),
				Text: fmt.Sprintf("%s;%d", k.Name(), k.Cycle()),
				Icon: iconFor(obj),
			}
			node.Attr, err = attrFor(obj, node)
			if err != nil {
				return nil, err
			}
			nodes = append(nodes, node)
		}
	}
	root.Children = nodes
	return []jsNode{root}, nil
}

func iconFor(obj root.Object) string {
	cls := obj.Class()
	switch {
	case strings.HasPrefix(cls, "TH1"):
		return "fa fa-bar-chart-o"
	case strings.HasPrefix(cls, "TH2"):
		return "fa fa-bar-chart-o"
	case strings.HasPrefix(cls, "TGraph"):
		return "fa fa-bar-chart-o"
	}
	return "fa fa-cube"
}

func attrFor(obj root.Object, node jsNode) (jsAttr, error) {
	cmd := new(bytes.Buffer)
	cls := obj.Class()
	switch {
	case strings.HasPrefix(cls, "TH1"):
		req := rsrv.PlotH1Request{
			URI: node.URI,
			Dir: node.Dir,
			Obj: node.Obj,
			Options: rsrv.PlotOptions{
				Title:  node.Obj,
				Type:   "svg",
				Height: -1,
				Width:  20 * vg.Centimeter,
			},
		}
		err := json.NewEncoder(cmd).Encode(req)
		if err != nil {
			return nil, err
		}
		return jsAttr{
			"plot": true,
			"href": "/plot-h1",
			"cmd":  cmd.String(),
		}, nil
	case strings.HasPrefix(cls, "TH2"):
		req := rsrv.PlotH2Request{
			URI: node.URI,
			Dir: node.Dir,
			Obj: node.Obj,
			Options: rsrv.PlotOptions{
				Title:  node.Obj,
				Type:   "svg",
				Height: -1,
				Width:  20 * vg.Centimeter,
			},
		}
		err := json.NewEncoder(cmd).Encode(req)
		if err != nil {
			return nil, err
		}
		return jsAttr{
			"plot": true,
			"href": "/plot-h2",
			"cmd":  cmd.String(),
		}, nil
	case strings.HasPrefix(cls, "TGraph"):
		req := rsrv.PlotS2Request{
			URI: node.URI,
			Dir: node.Dir,
			Obj: node.Obj,
			Options: rsrv.PlotOptions{
				Title:  node.Obj,
				Type:   "svg",
				Height: -1,
				Width:  20 * vg.Centimeter,
			},
		}
		err := json.NewEncoder(cmd).Encode(req)
		if err != nil {
			return nil, err
		}
		return jsAttr{
			"plot": true,
			"href": "/plot-s2",
			"cmd":  cmd.String(),
		}, nil
	case strings.HasPrefix(cls, "TBranch"):
		req := rsrv.PlotTreeRequest{
			URI:  node.URI,
			Dir:  stdpath.Dir(node.Dir),
			Obj:  stdpath.Base(node.Dir),
			Vars: []string{node.Obj},
			Options: rsrv.PlotOptions{
				Title:  node.Obj,
				Type:   "svg",
				Height: -1,
				Width:  20 * vg.Centimeter,
			},
		}
		err := json.NewEncoder(cmd).Encode(req)
		if err != nil {
			return nil, err
		}
		return jsAttr{
			"plot": true,
			"href": "/plot-branch",
			"cmd":  cmd.String(),
		}, nil
	}
	// TODO(sbinet) do something clever with things we don't know how to handle?
	return nil, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package base

import (
	"bytes"
	"fmt"
	"net/http"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/russross/blackfriday"

	"github.com/gogits/gogs/modules/setting"
)

func isletter(c byte) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

func isalnum(c byte) bool {
	return (c >= '0' && c <= '9') || isletter(c)
}

var validLinks = [][]byte{[]byte("http://"), []byte("https://"), []byte("ftp://"), []byte("mailto://")}

func isLink(link []byte) bool {
	for _, prefix := range validLinks {
		if len(link) > len(prefix) && bytes.Equal(bytes.ToLower(link[:len(prefix)]), prefix) && isalnum(link[len(prefix)]) {
			return true
		}
	}

	return false
}

func IsMarkdownFile(name string) bool {
	name = strings.ToLower(name)
	switch filepath.Ext(name) {
	case ".md", ".markdown", ".mdown", ".mkd":
		return true
	}
	return false
}

func IsTextFile(data []byte) (string, bool) {
	contentType := http.DetectContentType(data)
	if strings.Index(contentType, "text/") != -1 {
		return contentType, true
	}
	return contentType, false
}

func IsImageFile(data []byte) (string, bool) {
	contentType := http.DetectContentType(data)
	if strings.Index(contentType, "image/") != -1 {
		return contentType, true
	}
	return contentType, false
}

// IsReadmeFile returns true if given file name suppose to be a README file.
func IsReadmeFile(name string) bool {
	name = strings.ToLower(name)
	if len(name) < 6 {
		return false
	} else if len(name) == 6 {
		if name == "readme" {
			return true
		}
		return false
	}
	if name[:7] == "readme." {
		return true
	}
	return false
}

type CustomRender struct {
	blackfriday.Renderer
	urlPrefix string
}

func (options *CustomRender) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
	if len(link) > 0 && !isLink(link) {
		if link[0] == '#' {
			// link = append([]byte(options.urlPrefix), link...)
		} else {
			link = []byte(path.Join(options.urlPrefix, string(link)))
		}
	}

	options.Renderer.Link(out, link, title, content)
}

func (options *CustomRender) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {
	if len(link) > 0 && !isLink(link) {
		link = []byte(path.Join(strings.Replace(options.urlPrefix, "/src/", "/raw/", 1), string(link)))
	}

	options.Renderer.Image(out, link, title, alt)
}

var (
	MentionPattern     = regexp.MustCompile(`(\s|^)@[0-9a-zA-Z_]+`)
	commitPattern      = regexp.MustCompile(`(\s|^)https?.*commit/[0-9a-zA-Z]+(#+[0-9a-zA-Z-]*)?`)
	issueFullPattern   = regexp.MustCompile(`(\s|^)https?.*issues/[0-9]+(#+[0-9a-zA-Z-]*)?`)
	issueIndexPattern  = regexp.MustCompile(`( |^)#[0-9]+`)
	sha1CurrentPattern = regexp.MustCompile(`\b[0-9a-f]{40}\b`)
)

func RenderSpecialLink(rawBytes []byte, urlPrefix string) []byte {
	buf := bytes.NewBufferString("")
	inCodeBlock := false
	codeBlockPrefix := []byte("```")
	lineBreak := []byte("\n")
	tab := []byte("\t")
	lines := bytes.Split(rawBytes, lineBreak)
	for _, line := range lines {
		if bytes.HasPrefix(line, codeBlockPrefix) {
			inCodeBlock = !inCodeBlock
		}

		if !inCodeBlock && !bytes.HasPrefix(line, tab) {
			ms := MentionPattern.FindAll(line, -1)
			for _, m := range ms {
				m = bytes.TrimSpace(m)
				line = bytes.Replace(line, m,
					[]byte(fmt.Sprintf(`<a href="%s/%s">%s</a>`, setting.AppSubUrl, m[1:], m)), -1)
			}
		}

		buf.Write(line)
		buf.Write(lineBreak)
	}

	rawBytes = buf.Bytes()
	ms := commitPattern.FindAll(rawBytes, -1)
	for _, m := range ms {
		m = bytes.TrimSpace(m)
		i := strings.Index(string(m), "commit/")
		j := strings.Index(string(m), "#")
		if j == -1 {
			j = len(m)
		}
		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(
			` <code><a href="%s">%s</a></code>`, m, ShortSha(string(m[i+7:j])))), -1)
	}
	ms = issueFullPattern.FindAll(rawBytes, -1)
	for _, m := range ms {
		m = bytes.TrimSpace(m)
		i := strings.Index(string(m), "issues/")
		j := strings.Index(string(m), "#")
		if j == -1 {
			j = len(m)
		}
		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(
			` <a href="%s">#%s</a>`, m, ShortSha(string(m[i+7:j])))), -1)
	}
	rawBytes = RenderIssueIndexPattern(rawBytes, urlPrefix)
	rawBytes = RenderSha1CurrentPattern(rawBytes, urlPrefix)
	return rawBytes
}

func RenderSha1CurrentPattern(rawBytes []byte, urlPrefix string) []byte {
	ms := sha1CurrentPattern.FindAll(rawBytes, -1)
	for _, m := range ms {
		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(
			`<a href="%s/commit/%s"><code>%s</code></a>`, urlPrefix, m, ShortSha(string(m)))), -1)
	}
	return rawBytes
}

func RenderIssueIndexPattern(rawBytes []byte, urlPrefix string) []byte {
	ms := issueIndexPattern.FindAll(rawBytes, -1)
	for _, m := range ms {
		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(`<a href="%s/issues/%s">%s</a>`,
			urlPrefix, strings.TrimPrefix(string(m[1:]), "#"), m)), -1)
	}
	return rawBytes
}

func RenderRawMarkdown(body []byte, urlPrefix string) []byte {
	htmlFlags := 0
	// htmlFlags |= blackfriday.HTML_USE_XHTML
	// htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
	// htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
	// htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
	// htmlFlags |= blackfriday.HTML_SKIP_HTML
	htmlFlags |= blackfriday.HTML_SKIP_STYLE
	// htmlFlags |= blackfriday.HTML_SKIP_SCRIPT
	// htmlFlags |= blackfriday.HTML_GITHUB_BLOCKCODE
	htmlFlags |= blackfriday.HTML_OMIT_CONTENTS
	// htmlFlags |= blackfriday.HTML_COMPLETE_PAGE
	renderer := &CustomRender{
		Renderer:  blackfriday.HtmlRenderer(htmlFlags, "", ""),
		urlPrefix: urlPrefix,
	}

	// set up the parser
	extensions := 0
	extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
	extensions |= blackfriday.EXTENSION_TABLES
	extensions |= blackfriday.EXTENSION_FENCED_CODE
	extensions |= blackfriday.EXTENSION_AUTOLINK
	extensions |= blackfriday.EXTENSION_STRIKETHROUGH
	extensions |= blackfriday.EXTENSION_HARD_LINE_BREAK
	extensions |= blackfriday.EXTENSION_SPACE_HEADERS
	extensions |= blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK

	body = blackfriday.Markdown(body, renderer, extensions)
	return body
}

func RenderMarkdown(rawBytes []byte, urlPrefix string) []byte {
	body := RenderSpecialLink(rawBytes, urlPrefix)
	body = RenderRawMarkdown(body, urlPrefix)
	body = Sanitizer.SanitizeBytes(body)
	return body
}

func RenderMarkdownString(raw, urlPrefix string) string {
	return string(RenderMarkdown([]byte(raw), urlPrefix))
}
<commit_msg>Exclude HTML tags from Markdown post-processing<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package base

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/russross/blackfriday"

	"github.com/gogits/gogs/modules/setting"

	"golang.org/x/net/html"
)

func isletter(c byte) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

func isalnum(c byte) bool {
	return (c >= '0' && c <= '9') || isletter(c)
}

var validLinks = [][]byte{[]byte("http://"), []byte("https://"), []byte("ftp://"), []byte("mailto://")}

func isLink(link []byte) bool {
	for _, prefix := range validLinks {
		if len(link) > len(prefix) && bytes.Equal(bytes.ToLower(link[:len(prefix)]), prefix) && isalnum(link[len(prefix)]) {
			return true
		}
	}

	return false
}

func IsMarkdownFile(name string) bool {
	name = strings.ToLower(name)
	switch filepath.Ext(name) {
	case ".md", ".markdown", ".mdown", ".mkd":
		return true
	}
	return false
}

func IsTextFile(data []byte) (string, bool) {
	contentType := http.DetectContentType(data)
	if strings.Index(contentType, "text/") != -1 {
		return contentType, true
	}
	return contentType, false
}

func IsImageFile(data []byte) (string, bool) {
	contentType := http.DetectContentType(data)
	if strings.Index(contentType, "image/") != -1 {
		return contentType, true
	}
	return contentType, false
}

// IsReadmeFile returns true if given file name suppose to be a README file.
func IsReadmeFile(name string) bool {
	name = strings.ToLower(name)
	if len(name) < 6 {
		return false
	} else if len(name) == 6 {
		if name == "readme" {
			return true
		}
		return false
	}
	if name[:7] == "readme." {
		return true
	}
	return false
}

type CustomRender struct {
	blackfriday.Renderer
	urlPrefix string
}

func (options *CustomRender) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
	if len(link) > 0 && !isLink(link) {
		if link[0] == '#' {
			// link = append([]byte(options.urlPrefix), link...)
		} else {
			link = []byte(path.Join(options.urlPrefix, string(link)))
		}
	}

	options.Renderer.Link(out, link, title, content)
}

func (options *CustomRender) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {
	if len(link) > 0 && !isLink(link) {
		link = []byte(path.Join(strings.Replace(options.urlPrefix, "/src/", "/raw/", 1), string(link)))
	}

	options.Renderer.Image(out, link, title, alt)
}

var (
	MentionPattern     = regexp.MustCompile(`(\s|^)@[0-9a-zA-Z_]+`)
	commitPattern      = regexp.MustCompile(`(\s|^)https?.*commit/[0-9a-zA-Z]+(#+[0-9a-zA-Z-]*)?`)
	issueFullPattern   = regexp.MustCompile(`(\s|^)https?.*issues/[0-9]+(#+[0-9a-zA-Z-]*)?`)
	issueIndexPattern  = regexp.MustCompile(`( |^)#[0-9]+`)
	sha1CurrentPattern = regexp.MustCompile(`\b[0-9a-f]{40}\b`)
)

func RenderSpecialLink(rawBytes []byte, urlPrefix string) []byte {
	buf := bytes.NewBufferString("")
	inCodeBlock := false
	codeBlockPrefix := []byte("```")
	lineBreak := []byte("\n")
	tab := []byte("\t")
	lines := bytes.Split(rawBytes, lineBreak)
	for _, line := range lines {
		if bytes.HasPrefix(line, codeBlockPrefix) {
			inCodeBlock = !inCodeBlock
		}

		if !inCodeBlock && !bytes.HasPrefix(line, tab) {
			ms := MentionPattern.FindAll(line, -1)
			for _, m := range ms {
				m = bytes.TrimSpace(m)
				line = bytes.Replace(line, m,
					[]byte(fmt.Sprintf(`<a href="%s/%s">%s</a>`, setting.AppSubUrl, m[1:], m)), -1)
			}
		}

		buf.Write(line)
		buf.Write(lineBreak)
	}

	rawBytes = buf.Bytes()
	ms := commitPattern.FindAll(rawBytes, -1)
	for _, m := range ms {
		m = bytes.TrimSpace(m)
		i := strings.Index(string(m), "commit/")
		j := strings.Index(string(m), "#")
		if j == -1 {
			j = len(m)
		}
		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(
			` <code><a href="%s">%s</a></code>`, m, ShortSha(string(m[i+7:j])))), -1)
	}
	ms = issueFullPattern.FindAll(rawBytes, -1)
	for _, m := range ms {
		m = bytes.TrimSpace(m)
		i := strings.Index(string(m), "issues/")
		j := strings.Index(string(m), "#")
		if j == -1 {
			j = len(m)
		}
		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(
			` <a href="%s">#%s</a>`, m, ShortSha(string(m[i+7:j])))), -1)
	}
	rawBytes = RenderIssueIndexPattern(rawBytes, urlPrefix)
	rawBytes = RenderSha1CurrentPattern(rawBytes, urlPrefix)
	return rawBytes
}

func RenderSha1CurrentPattern(rawBytes []byte, urlPrefix string) []byte {
	ms := sha1CurrentPattern.FindAll(rawBytes, -1)
	for _, m := range ms {
		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(
			`<a href="%s/commit/%s"><code>%s</code></a>`, urlPrefix, m, ShortSha(string(m)))), -1)
	}
	return rawBytes
}

func RenderIssueIndexPattern(rawBytes []byte, urlPrefix string) []byte {
	ms := issueIndexPattern.FindAll(rawBytes, -1)
	for _, m := range ms {
		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(`<a href="%s/issues/%s">%s</a>`,
			urlPrefix, strings.TrimPrefix(string(m[1:]), "#"), m)), -1)
	}
	return rawBytes
}

func RenderRawMarkdown(body []byte, urlPrefix string) []byte {
	htmlFlags := 0
	// htmlFlags |= blackfriday.HTML_USE_XHTML
	// htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
	// htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
	// htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
	// htmlFlags |= blackfriday.HTML_SKIP_HTML
	htmlFlags |= blackfriday.HTML_SKIP_STYLE
	// htmlFlags |= blackfriday.HTML_SKIP_SCRIPT
	// htmlFlags |= blackfriday.HTML_GITHUB_BLOCKCODE
	htmlFlags |= blackfriday.HTML_OMIT_CONTENTS
	// htmlFlags |= blackfriday.HTML_COMPLETE_PAGE
	renderer := &CustomRender{
		Renderer:  blackfriday.HtmlRenderer(htmlFlags, "", ""),
		urlPrefix: urlPrefix,
	}

	// set up the parser
	extensions := 0
	extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
	extensions |= blackfriday.EXTENSION_TABLES
	extensions |= blackfriday.EXTENSION_FENCED_CODE
	extensions |= blackfriday.EXTENSION_AUTOLINK
	extensions |= blackfriday.EXTENSION_STRIKETHROUGH
	extensions |= blackfriday.EXTENSION_HARD_LINE_BREAK
	extensions |= blackfriday.EXTENSION_SPACE_HEADERS
	extensions |= blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK

	body = blackfriday.Markdown(body, renderer, extensions)
	return body
}

func RenderMarkdown(rawBytes []byte, urlPrefix string) []byte {
	result := RenderRawMarkdown(rawBytes, urlPrefix)
	result = PostProcessMarkdown(result, urlPrefix)
	result = Sanitizer.SanitizeBytes(result)
	return result
}

func RenderMarkdownString(raw, urlPrefix string) string {
	return string(RenderMarkdown([]byte(raw), urlPrefix))
}

func PostProcessMarkdown(rawHtml []byte, urlPrefix string) []byte {
	var buf bytes.Buffer
	tokenizer := html.NewTokenizer(bytes.NewReader(rawHtml))
	for html.ErrorToken != tokenizer.Next() {

			// A parse error has occurred, so return the original input unmodified
			return rawHtml

		token := tokenizer.Token()
		switch token.Type {
			case html.TextToken:
				text := []byte(token.String())
				text = RenderSpecialLink(text, urlPrefix)

				buf.Write(text)

			case html.StartTagToken:
				buf.WriteString(token.String())

				tagName := token.Data
				// If this is an excluded tag, we skip processing all output until a close tag is encountered
				if strings.EqualFold("a", tagName) || strings.EqualFold("code", tagName) || strings.EqualFold("pre", tagName) {
					for html.ErrorToken != tokenizer.Next() {
						token = tokenizer.Token()
						// Copy the token to the output verbatim
						buf.WriteString(token.String())
						// If this is the close tag, we are done
						if html.EndTagToken == token.Type && strings.EqualFold(tagName, token.Data) { break }
					}
				}

			default:
				buf.WriteString(token.String())
		}
	}

	if io.EOF == tokenizer.Err() {
		return buf.Bytes()
	}

	// If we are not at the end of the input, then some other parsing error has occurred, so return
	// the input verbatim.
	return rawHtml
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package expvar provides a standardized interface to public variables, such
// as operation counters in servers. It exposes these variables via HTTP at
// /debug/vars in JSON format.
//
// Operations to set or modify these public variables are atomic.
//
// In addition to adding the HTTP handler, this package registers the
// following variables:
//
//	cmdline   os.Args
//	memstats  runtime.Memstats
//
// The package is sometimes only imported for the side effect of
// registering its HTTP handler and the above variables.  To use it
// this way, link this package into your program:
//	import _ "expvar"
//
package expvar

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"sort"
	"strconv"
	"sync"
)

// Var is an abstract type for all exported variables.
type Var interface {
	String() string
}

// Int is a 64-bit integer variable that satisfies the Var interface.
type Int struct {
	mu sync.RWMutex
	i  int64
}

func (v *Int) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return strconv.FormatInt(v.i, 10)
}

func (v *Int) Add(delta int64) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.i += delta
}

func (v *Int) Set(value int64) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.i = value
}

// Float is a 64-bit float variable that satisfies the Var interface.
type Float struct {
	mu sync.RWMutex
	f  float64
}

func (v *Float) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return strconv.FormatFloat(v.f, 'g', -1, 64)
}

// Add adds delta to v.
func (v *Float) Add(delta float64) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.f += delta
}

// Set sets v to value.
func (v *Float) Set(value float64) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.f = value
}

// Map is a string-to-Var map variable that satisfies the Var interface.
type Map struct {
	mu   sync.RWMutex
	m    map[string]Var
	keys []string // sorted
}

// KeyValue represents a single entry in a Map.
type KeyValue struct {
	Key   string
	Value Var
}

func (v *Map) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	var b bytes.Buffer
	fmt.Fprintf(&b, "{")
	first := true
	v.Do(func(kv KeyValue) {
		if !first {
			fmt.Fprintf(&b, ", ")
		}
		fmt.Fprintf(&b, "\"%s\": %v", kv.Key, kv.Value)
		first = false
	})
	fmt.Fprintf(&b, "}")
	return b.String()
}

func (v *Map) Init() *Map {
	v.m = make(map[string]Var)
	return v
}

// updateKeys updates the sorted list of keys in v.keys.
// must be called with v.mu held.
func (v *Map) updateKeys() {
	if len(v.m) == len(v.keys) {
		// No new key.
		return
	}
	v.keys = v.keys[:0]
	for k := range v.m {
		v.keys = append(v.keys, k)
	}
	sort.Strings(v.keys)
}

func (v *Map) Get(key string) Var {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.m[key]
}

func (v *Map) Set(key string, av Var) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.m[key] = av
	v.updateKeys()
}

func (v *Map) Add(key string, delta int64) {
	v.mu.RLock()
	av, ok := v.m[key]
	v.mu.RUnlock()
	if !ok {
		// check again under the write lock
		v.mu.Lock()
		av, ok = v.m[key]
		if !ok {
			av = new(Int)
			v.m[key] = av
			v.updateKeys()
		}
		v.mu.Unlock()
	}

	// Add to Int; ignore otherwise.
	if iv, ok := av.(*Int); ok {
		iv.Add(delta)
	}
}

// AddFloat adds delta to the *Float value stored under the given map key.
func (v *Map) AddFloat(key string, delta float64) {
	v.mu.RLock()
	av, ok := v.m[key]
	v.mu.RUnlock()
	if !ok {
		// check again under the write lock
		v.mu.Lock()
		av, ok = v.m[key]
		if !ok {
			av = new(Float)
			v.m[key] = av
			v.updateKeys()
		}
		v.mu.Unlock()
	}

	// Add to Float; ignore otherwise.
	if iv, ok := av.(*Float); ok {
		iv.Add(delta)
	}
}

// Do calls f for each entry in the map.
// The map is locked during the iteration,
// but existing entries may be concurrently updated.
func (v *Map) Do(f func(KeyValue)) {
	v.mu.RLock()
	defer v.mu.RUnlock()
	for _, k := range v.keys {
		f(KeyValue{k, v.m[k]})
	}
}

// String is a string variable, and satisfies the Var interface.
type String struct {
	mu sync.RWMutex
	s  string
}

func (v *String) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return strconv.Quote(v.s)
}

func (v *String) Set(value string) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.s = value
}

// Func implements Var by calling the function
// and formatting the returned value using JSON.
type Func func() interface{}

func (f Func) String() string {
	v, _ := json.Marshal(f())
	return string(v)
}

// All published variables.
var (
	mutex   sync.RWMutex
	vars    = make(map[string]Var)
	varKeys []string // sorted
)

// Publish declares a named exported variable. This should be called from a
// package's init function when it creates its Vars. If the name is already
// registered then this will log.Panic.
func Publish(name string, v Var) {
	mutex.Lock()
	defer mutex.Unlock()
	if _, existing := vars[name]; existing {
		log.Panicln("Reuse of exported var name:", name)
	}
	vars[name] = v
	varKeys = append(varKeys, name)
	sort.Strings(varKeys)
}

// Get retrieves a named exported variable.
func Get(name string) Var {
	mutex.RLock()
	defer mutex.RUnlock()
	return vars[name]
}

// Convenience functions for creating new exported variables.

func NewInt(name string) *Int {
	v := new(Int)
	Publish(name, v)
	return v
}

func NewFloat(name string) *Float {
	v := new(Float)
	Publish(name, v)
	return v
}

func NewMap(name string) *Map {
	v := new(Map).Init()
	Publish(name, v)
	return v
}

func NewString(name string) *String {
	v := new(String)
	Publish(name, v)
	return v
}

// Do calls f for each exported variable.
// The global variable map is locked during the iteration,
// but existing entries may be concurrently updated.
func Do(f func(KeyValue)) {
	mutex.RLock()
	defer mutex.RUnlock()
	for _, k := range varKeys {
		f(KeyValue{k, vars[k]})
	}
}

func expvarHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	fmt.Fprintf(w, "{\n")
	first := true
	Do(func(kv KeyValue) {
		if !first {
			fmt.Fprintf(w, ",\n")
		}
		first = false
		fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "\n}\n")
}

func cmdline() interface{} {
	return os.Args
}

func memstats() interface{} {
	stats := new(runtime.MemStats)
	runtime.ReadMemStats(stats)
	return *stats
}

func init() {
	http.HandleFunc("/debug/vars", expvarHandler)
	Publish("cmdline", Func(cmdline))
	Publish("memstats", Func(memstats))
}
<commit_msg>expvar: don't recursively acquire Map.RLock<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package expvar provides a standardized interface to public variables, such
// as operation counters in servers. It exposes these variables via HTTP at
// /debug/vars in JSON format.
//
// Operations to set or modify these public variables are atomic.
//
// In addition to adding the HTTP handler, this package registers the
// following variables:
//
//	cmdline   os.Args
//	memstats  runtime.Memstats
//
// The package is sometimes only imported for the side effect of
// registering its HTTP handler and the above variables.  To use it
// this way, link this package into your program:
//	import _ "expvar"
//
package expvar

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"sort"
	"strconv"
	"sync"
)

// Var is an abstract type for all exported variables.
type Var interface {
	String() string
}

// Int is a 64-bit integer variable that satisfies the Var interface.
type Int struct {
	mu sync.RWMutex
	i  int64
}

func (v *Int) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return strconv.FormatInt(v.i, 10)
}

func (v *Int) Add(delta int64) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.i += delta
}

func (v *Int) Set(value int64) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.i = value
}

// Float is a 64-bit float variable that satisfies the Var interface.
type Float struct {
	mu sync.RWMutex
	f  float64
}

func (v *Float) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return strconv.FormatFloat(v.f, 'g', -1, 64)
}

// Add adds delta to v.
func (v *Float) Add(delta float64) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.f += delta
}

// Set sets v to value.
func (v *Float) Set(value float64) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.f = value
}

// Map is a string-to-Var map variable that satisfies the Var interface.
type Map struct {
	mu   sync.RWMutex
	m    map[string]Var
	keys []string // sorted
}

// KeyValue represents a single entry in a Map.
type KeyValue struct {
	Key   string
	Value Var
}

func (v *Map) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	var b bytes.Buffer
	fmt.Fprintf(&b, "{")
	first := true
	v.doLocked(func(kv KeyValue) {
		if !first {
			fmt.Fprintf(&b, ", ")
		}
		fmt.Fprintf(&b, "\"%s\": %v", kv.Key, kv.Value)
		first = false
	})
	fmt.Fprintf(&b, "}")
	return b.String()
}

func (v *Map) Init() *Map {
	v.m = make(map[string]Var)
	return v
}

// updateKeys updates the sorted list of keys in v.keys.
// must be called with v.mu held.
func (v *Map) updateKeys() {
	if len(v.m) == len(v.keys) {
		// No new key.
		return
	}
	v.keys = v.keys[:0]
	for k := range v.m {
		v.keys = append(v.keys, k)
	}
	sort.Strings(v.keys)
}

func (v *Map) Get(key string) Var {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.m[key]
}

func (v *Map) Set(key string, av Var) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.m[key] = av
	v.updateKeys()
}

func (v *Map) Add(key string, delta int64) {
	v.mu.RLock()
	av, ok := v.m[key]
	v.mu.RUnlock()
	if !ok {
		// check again under the write lock
		v.mu.Lock()
		av, ok = v.m[key]
		if !ok {
			av = new(Int)
			v.m[key] = av
			v.updateKeys()
		}
		v.mu.Unlock()
	}

	// Add to Int; ignore otherwise.
	if iv, ok := av.(*Int); ok {
		iv.Add(delta)
	}
}

// AddFloat adds delta to the *Float value stored under the given map key.
func (v *Map) AddFloat(key string, delta float64) {
	v.mu.RLock()
	av, ok := v.m[key]
	v.mu.RUnlock()
	if !ok {
		// check again under the write lock
		v.mu.Lock()
		av, ok = v.m[key]
		if !ok {
			av = new(Float)
			v.m[key] = av
			v.updateKeys()
		}
		v.mu.Unlock()
	}

	// Add to Float; ignore otherwise.
	if iv, ok := av.(*Float); ok {
		iv.Add(delta)
	}
}

// Do calls f for each entry in the map.
// The map is locked during the iteration,
// but existing entries may be concurrently updated.
func (v *Map) Do(f func(KeyValue)) {
	v.mu.RLock()
	defer v.mu.RUnlock()
	v.doLocked(f)
}

// doRLocked calls f for each entry in the map.
// v.mu must be held for reads.
func (v *Map) doLocked(f func(KeyValue)) {
	for _, k := range v.keys {
		f(KeyValue{k, v.m[k]})
	}
}

// String is a string variable, and satisfies the Var interface.
type String struct {
	mu sync.RWMutex
	s  string
}

func (v *String) String() string {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return strconv.Quote(v.s)
}

func (v *String) Set(value string) {
	v.mu.Lock()
	defer v.mu.Unlock()
	v.s = value
}

// Func implements Var by calling the function
// and formatting the returned value using JSON.
type Func func() interface{}

func (f Func) String() string {
	v, _ := json.Marshal(f())
	return string(v)
}

// All published variables.
var (
	mutex   sync.RWMutex
	vars    = make(map[string]Var)
	varKeys []string // sorted
)

// Publish declares a named exported variable. This should be called from a
// package's init function when it creates its Vars. If the name is already
// registered then this will log.Panic.
func Publish(name string, v Var) {
	mutex.Lock()
	defer mutex.Unlock()
	if _, existing := vars[name]; existing {
		log.Panicln("Reuse of exported var name:", name)
	}
	vars[name] = v
	varKeys = append(varKeys, name)
	sort.Strings(varKeys)
}

// Get retrieves a named exported variable.
func Get(name string) Var {
	mutex.RLock()
	defer mutex.RUnlock()
	return vars[name]
}

// Convenience functions for creating new exported variables.

func NewInt(name string) *Int {
	v := new(Int)
	Publish(name, v)
	return v
}

func NewFloat(name string) *Float {
	v := new(Float)
	Publish(name, v)
	return v
}

func NewMap(name string) *Map {
	v := new(Map).Init()
	Publish(name, v)
	return v
}

func NewString(name string) *String {
	v := new(String)
	Publish(name, v)
	return v
}

// Do calls f for each exported variable.
// The global variable map is locked during the iteration,
// but existing entries may be concurrently updated.
func Do(f func(KeyValue)) {
	mutex.RLock()
	defer mutex.RUnlock()
	for _, k := range varKeys {
		f(KeyValue{k, vars[k]})
	}
}

func expvarHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	fmt.Fprintf(w, "{\n")
	first := true
	Do(func(kv KeyValue) {
		if !first {
			fmt.Fprintf(w, ",\n")
		}
		first = false
		fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "\n}\n")
}

func cmdline() interface{} {
	return os.Args
}

func memstats() interface{} {
	stats := new(runtime.MemStats)
	runtime.ReadMemStats(stats)
	return *stats
}

func init() {
	http.HandleFunc("/debug/vars", expvarHandler)
	Publish("cmdline", Func(cmdline))
	Publish("memstats", Func(memstats))
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ast

import (
	"go/token"
	"sort"
)

// ----------------------------------------------------------------------------
// Export filtering

// exportFilter is a special filter function to extract exported nodes.
func exportFilter(name string) bool {
	return IsExported(name)
}

// FileExports trims the AST for a Go source file in place such that
// only exported nodes remain: all top-level identifiers which are not exported
// and their associated information (such as type, initial value, or function
// body) are removed. Non-exported fields and methods of exported types are
// stripped. The File.Comments list is not changed.
//
// FileExports returns true if there are exported declarationa;
// it returns false otherwise.
//
func FileExports(src *File) bool {
	return filterFile(src, exportFilter, true)
}

// PackageExports trims the AST for a Go package in place such that
// only exported nodes remain. The pkg.Files list is not changed, so that
// file names and top-level package comments don't get lost.
//
// PackageExports returns true if there are exported declarations;
// it returns false otherwise.
//
func PackageExports(pkg *Package) bool {
	return filterPackage(pkg, exportFilter, true)
}

// ----------------------------------------------------------------------------
// General filtering

type Filter func(string) bool

func filterIdentList(list []*Ident, f Filter) []*Ident {
	j := 0
	for _, x := range list {
		if f(x.Name) {
			list[j] = x
			j++
		}
	}
	return list[0:j]
}

// fieldName assumes that x is the type of an anonymous field and
// returns the corresponding field name. If x is not an acceptable
// anonymous field, the result is nil.
//
func fieldName(x Expr) *Ident {
	switch t := x.(type) {
	case *Ident:
		return t
	case *SelectorExpr:
		if _, ok := t.X.(*Ident); ok {
			return t.Sel
		}
	case *StarExpr:
		return fieldName(t.X)
	}
	return nil
}

func filterFieldList(fields *FieldList, filter Filter, export bool) (removedFields bool) {
	if fields == nil {
		return false
	}
	list := fields.List
	j := 0
	for _, f := range list {
		keepField := false
		if len(f.Names) == 0 {
			// anonymous field
			name := fieldName(f.Type)
			keepField = name != nil && filter(name.Name)
		} else {
			n := len(f.Names)
			f.Names = filterIdentList(f.Names, filter)
			if len(f.Names) < n {
				removedFields = true
			}
			keepField = len(f.Names) > 0
		}
		if keepField {
			if export {
				filterType(f.Type, filter, export)
			}
			list[j] = f
			j++
		}
	}
	if j < len(list) {
		removedFields = true
	}
	fields.List = list[0:j]
	return
}

func filterParamList(fields *FieldList, filter Filter, export bool) bool {
	if fields == nil {
		return false
	}
	var b bool
	for _, f := range fields.List {
		if filterType(f.Type, filter, export) {
			b = true
		}
	}
	return b
}

func filterType(typ Expr, f Filter, export bool) bool {
	switch t := typ.(type) {
	case *Ident:
		return f(t.Name)
	case *ParenExpr:
		return filterType(t.X, f, export)
	case *ArrayType:
		return filterType(t.Elt, f, export)
	case *StructType:
		if filterFieldList(t.Fields, f, export) {
			t.Incomplete = true
		}
		return len(t.Fields.List) > 0
	case *FuncType:
		b1 := filterParamList(t.Params, f, export)
		b2 := filterParamList(t.Results, f, export)
		return b1 || b2
	case *InterfaceType:
		if filterFieldList(t.Methods, f, export) {
			t.Incomplete = true
		}
		return len(t.Methods.List) > 0
	case *MapType:
		b1 := filterType(t.Key, f, export)
		b2 := filterType(t.Value, f, export)
		return b1 || b2
	case *ChanType:
		return filterType(t.Value, f, export)
	}
	return false
}

func filterSpec(spec Spec, f Filter, export bool) bool {
	switch s := spec.(type) {
	case *ValueSpec:
		s.Names = filterIdentList(s.Names, f)
		if len(s.Names) > 0 {
			if export {
				filterType(s.Type, f, export)
			}
			return true
		}
	case *TypeSpec:
		if f(s.Name.Name) {
			if export {
				filterType(s.Type, f, export)
			}
			return true
		}
		if !export {
			// For general filtering (not just exports),
			// filter type even if name is not filtered
			// out.
			// If the type contains filtered elements,
			// keep the declaration.
			return filterType(s.Type, f, export)
		}
	}
	return false
}

func filterSpecList(list []Spec, f Filter, export bool) []Spec {
	j := 0
	for _, s := range list {
		if filterSpec(s, f, export) {
			list[j] = s
			j++
		}
	}
	return list[0:j]
}

// FilterDecl trims the AST for a Go declaration in place by removing
// all names (including struct field and interface method names, but
// not from parameter lists) that don't pass through the filter f.
//
// FilterDecl returns true if there are any declared names left after
// filtering; it returns false otherwise.
//
func FilterDecl(decl Decl, f Filter) bool {
	return filterDecl(decl, f, false)
}

func filterDecl(decl Decl, f Filter, export bool) bool {
	switch d := decl.(type) {
	case *GenDecl:
		d.Specs = filterSpecList(d.Specs, f, export)
		return len(d.Specs) > 0
	case *FuncDecl:
		return f(d.Name.Name)
	}
	return false
}

// FilterFile trims the AST for a Go file in place by removing all
// names from top-level declarations (including struct field and
// interface method names, but not from parameter lists) that don't
// pass through the filter f. If the declaration is empty afterwards,
// the declaration is removed from the AST. The File.Comments list
// is not changed.
//
// FilterFile returns true if there are any top-level declarations
// left after filtering; it returns false otherwise.
//
func FilterFile(src *File, f Filter) bool {
	return filterFile(src, f, false)
}

func filterFile(src *File, f Filter, export bool) bool {
	j := 0
	for _, d := range src.Decls {
		if filterDecl(d, f, export) {
			src.Decls[j] = d
			j++
		}
	}
	src.Decls = src.Decls[0:j]
	return j > 0
}

// FilterPackage trims the AST for a Go package in place by removing
// all names from top-level declarations (including struct field and
// interface method names, but not from parameter lists) that don't
// pass through the filter f. If the declaration is empty afterwards,
// the declaration is removed from the AST. The pkg.Files list is not
// changed, so that file names and top-level package comments don't get
// lost.
//
// FilterPackage returns true if there are any top-level declarations
// left after filtering; it returns false otherwise.
//
func FilterPackage(pkg *Package, f Filter) bool {
	return filterPackage(pkg, f, false)
}

func filterPackage(pkg *Package, f Filter, export bool) bool {
	hasDecls := false
	for _, src := range pkg.Files {
		if filterFile(src, f, export) {
			hasDecls = true
		}
	}
	return hasDecls
}

// ----------------------------------------------------------------------------
// Merging of package files

// The MergeMode flags control the behavior of MergePackageFiles.
type MergeMode uint

const (
	// If set, duplicate function declarations are excluded.
	FilterFuncDuplicates MergeMode = 1 << iota
	// If set, comments that are not associated with a specific
	// AST node (as Doc or Comment) are excluded.
	FilterUnassociatedComments
	// If set, duplicate import declarations are excluded.
	FilterImportDuplicates
)

// separator is an empty //-style comment that is interspersed between
// different comment groups when they are concatenated into a single group
//
var separator = &Comment{noPos, "//"}

// MergePackageFiles creates a file AST by merging the ASTs of the
// files belonging to a package. The mode flags control merging behavior.
//
func MergePackageFiles(pkg *Package, mode MergeMode) *File {
	// Count the number of package docs, comments and declarations across
	// all package files. Also, compute sorted list of filenames, so that
	// subsequent iterations can always iterate in the same order.
	ndocs := 0
	ncomments := 0
	ndecls := 0
	filenames := make([]string, len(pkg.Files))
	i := 0
	for filename, f := range pkg.Files {
		filenames[i] = filename
		i++
		if f.Doc != nil {
			ndocs += len(f.Doc.List) + 1 // +1 for separator
		}
		ncomments += len(f.Comments)
		ndecls += len(f.Decls)
	}
	sort.Strings(filenames)

	// Collect package comments from all package files into a single
	// CommentGroup - the collected package documentation. In general
	// there should be only one file with a package comment; but it's
	// better to collect extra comments than drop them on the floor.
	var doc *CommentGroup
	var pos token.Pos
	if ndocs > 0 {
		list := make([]*Comment, ndocs-1) // -1: no separator before first group
		i := 0
		for _, filename := range filenames {
			f := pkg.Files[filename]
			if f.Doc != nil {
				if i > 0 {
					// not the first group - add separator
					list[i] = separator
					i++
				}
				for _, c := range f.Doc.List {
					list[i] = c
					i++
				}
				if f.Package > pos {
					// Keep the maximum package clause position as
					// position for the package clause of the merged
					// files.
					pos = f.Package
				}
			}
		}
		doc = &CommentGroup{list}
	}

	// Collect declarations from all package files.
	var decls []Decl
	if ndecls > 0 {
		decls = make([]Decl, ndecls)
		funcs := make(map[string]int) // map of global function name -> decls index
		i := 0                        // current index
		n := 0                        // number of filtered entries
		for _, filename := range filenames {
			f := pkg.Files[filename]
			for _, d := range f.Decls {
				if mode&FilterFuncDuplicates != 0 {
					// A language entity may be declared multiple
					// times in different package files; only at
					// build time declarations must be unique.
					// For now, exclude multiple declarations of
					// functions - keep the one with documentation.
					//
					// TODO(gri): Expand this filtering to other
					//            entities (const, type, vars) if
					//            multiple declarations are common.
					if f, isFun := d.(*FuncDecl); isFun {
						name := f.Name.Name
						if j, exists := funcs[name]; exists {
							// function declared already
							if decls[j] != nil && decls[j].(*FuncDecl).Doc == nil {
								// existing declaration has no documentation;
								// ignore the existing declaration
								decls[j] = nil
							} else {
								// ignore the new declaration
								d = nil
							}
							n++ // filtered an entry
						} else {
							funcs[name] = i
						}
					}
				}
				decls[i] = d
				i++
			}
		}

		// Eliminate nil entries from the decls list if entries were
		// filtered. We do this using a 2nd pass in order to not disturb
		// the original declaration order in the source (otherwise, this
		// would also invalidate the monotonically increasing position
		// info within a single file).
		if n > 0 {
			i = 0
			for _, d := range decls {
				if d != nil {
					decls[i] = d
					i++
				}
			}
			decls = decls[0:i]
		}
	}

	// Collect import specs from all package files.
	var imports []*ImportSpec
	if mode&FilterImportDuplicates != 0 {
		seen := make(map[string]bool)
		for _, filename := range filenames {
			f := pkg.Files[filename]
			for _, imp := range f.Imports {
				if path := imp.Path.Value; !seen[path] {
					// TODO: consider handling cases where:
					// - 2 imports exist with the same import path but
					//   have different local names (one should probably
					//   keep both of them)
					// - 2 imports exist but only one has a comment
					// - 2 imports exist and they both have (possibly
					//   different) comments
					imports = append(imports, imp)
					seen[path] = true
				}
			}
		}
	} else {
		for _, f := range pkg.Files {
			imports = append(imports, f.Imports...)
		}
	}

	// Collect comments from all package files.
	var comments []*CommentGroup
	if mode&FilterUnassociatedComments == 0 {
		comments = make([]*CommentGroup, ncomments)
		i := 0
		for _, f := range pkg.Files {
			i += copy(comments[i:], f.Comments)
		}
	}

	// TODO(gri) need to compute unresolved identifiers!
	return &File{doc, pos, NewIdent(pkg.Name), decls, pkg.Scope, imports, nil, comments}
}
<commit_msg>go/ast: fix typo<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ast

import (
	"go/token"
	"sort"
)

// ----------------------------------------------------------------------------
// Export filtering

// exportFilter is a special filter function to extract exported nodes.
func exportFilter(name string) bool {
	return IsExported(name)
}

// FileExports trims the AST for a Go source file in place such that
// only exported nodes remain: all top-level identifiers which are not exported
// and their associated information (such as type, initial value, or function
// body) are removed. Non-exported fields and methods of exported types are
// stripped. The File.Comments list is not changed.
//
// FileExports returns true if there are exported declarations;
// it returns false otherwise.
//
func FileExports(src *File) bool {
	return filterFile(src, exportFilter, true)
}

// PackageExports trims the AST for a Go package in place such that
// only exported nodes remain. The pkg.Files list is not changed, so that
// file names and top-level package comments don't get lost.
//
// PackageExports returns true if there are exported declarations;
// it returns false otherwise.
//
func PackageExports(pkg *Package) bool {
	return filterPackage(pkg, exportFilter, true)
}

// ----------------------------------------------------------------------------
// General filtering

type Filter func(string) bool

func filterIdentList(list []*Ident, f Filter) []*Ident {
	j := 0
	for _, x := range list {
		if f(x.Name) {
			list[j] = x
			j++
		}
	}
	return list[0:j]
}

// fieldName assumes that x is the type of an anonymous field and
// returns the corresponding field name. If x is not an acceptable
// anonymous field, the result is nil.
//
func fieldName(x Expr) *Ident {
	switch t := x.(type) {
	case *Ident:
		return t
	case *SelectorExpr:
		if _, ok := t.X.(*Ident); ok {
			return t.Sel
		}
	case *StarExpr:
		return fieldName(t.X)
	}
	return nil
}

func filterFieldList(fields *FieldList, filter Filter, export bool) (removedFields bool) {
	if fields == nil {
		return false
	}
	list := fields.List
	j := 0
	for _, f := range list {
		keepField := false
		if len(f.Names) == 0 {
			// anonymous field
			name := fieldName(f.Type)
			keepField = name != nil && filter(name.Name)
		} else {
			n := len(f.Names)
			f.Names = filterIdentList(f.Names, filter)
			if len(f.Names) < n {
				removedFields = true
			}
			keepField = len(f.Names) > 0
		}
		if keepField {
			if export {
				filterType(f.Type, filter, export)
			}
			list[j] = f
			j++
		}
	}
	if j < len(list) {
		removedFields = true
	}
	fields.List = list[0:j]
	return
}

func filterParamList(fields *FieldList, filter Filter, export bool) bool {
	if fields == nil {
		return false
	}
	var b bool
	for _, f := range fields.List {
		if filterType(f.Type, filter, export) {
			b = true
		}
	}
	return b
}

func filterType(typ Expr, f Filter, export bool) bool {
	switch t := typ.(type) {
	case *Ident:
		return f(t.Name)
	case *ParenExpr:
		return filterType(t.X, f, export)
	case *ArrayType:
		return filterType(t.Elt, f, export)
	case *StructType:
		if filterFieldList(t.Fields, f, export) {
			t.Incomplete = true
		}
		return len(t.Fields.List) > 0
	case *FuncType:
		b1 := filterParamList(t.Params, f, export)
		b2 := filterParamList(t.Results, f, export)
		return b1 || b2
	case *InterfaceType:
		if filterFieldList(t.Methods, f, export) {
			t.Incomplete = true
		}
		return len(t.Methods.List) > 0
	case *MapType:
		b1 := filterType(t.Key, f, export)
		b2 := filterType(t.Value, f, export)
		return b1 || b2
	case *ChanType:
		return filterType(t.Value, f, export)
	}
	return false
}

func filterSpec(spec Spec, f Filter, export bool) bool {
	switch s := spec.(type) {
	case *ValueSpec:
		s.Names = filterIdentList(s.Names, f)
		if len(s.Names) > 0 {
			if export {
				filterType(s.Type, f, export)
			}
			return true
		}
	case *TypeSpec:
		if f(s.Name.Name) {
			if export {
				filterType(s.Type, f, export)
			}
			return true
		}
		if !export {
			// For general filtering (not just exports),
			// filter type even if name is not filtered
			// out.
			// If the type contains filtered elements,
			// keep the declaration.
			return filterType(s.Type, f, export)
		}
	}
	return false
}

func filterSpecList(list []Spec, f Filter, export bool) []Spec {
	j := 0
	for _, s := range list {
		if filterSpec(s, f, export) {
			list[j] = s
			j++
		}
	}
	return list[0:j]
}

// FilterDecl trims the AST for a Go declaration in place by removing
// all names (including struct field and interface method names, but
// not from parameter lists) that don't pass through the filter f.
//
// FilterDecl returns true if there are any declared names left after
// filtering; it returns false otherwise.
//
func FilterDecl(decl Decl, f Filter) bool {
	return filterDecl(decl, f, false)
}

func filterDecl(decl Decl, f Filter, export bool) bool {
	switch d := decl.(type) {
	case *GenDecl:
		d.Specs = filterSpecList(d.Specs, f, export)
		return len(d.Specs) > 0
	case *FuncDecl:
		return f(d.Name.Name)
	}
	return false
}

// FilterFile trims the AST for a Go file in place by removing all
// names from top-level declarations (including struct field and
// interface method names, but not from parameter lists) that don't
// pass through the filter f. If the declaration is empty afterwards,
// the declaration is removed from the AST. The File.Comments list
// is not changed.
//
// FilterFile returns true if there are any top-level declarations
// left after filtering; it returns false otherwise.
//
func FilterFile(src *File, f Filter) bool {
	return filterFile(src, f, false)
}

func filterFile(src *File, f Filter, export bool) bool {
	j := 0
	for _, d := range src.Decls {
		if filterDecl(d, f, export) {
			src.Decls[j] = d
			j++
		}
	}
	src.Decls = src.Decls[0:j]
	return j > 0
}

// FilterPackage trims the AST for a Go package in place by removing
// all names from top-level declarations (including struct field and
// interface method names, but not from parameter lists) that don't
// pass through the filter f. If the declaration is empty afterwards,
// the declaration is removed from the AST. The pkg.Files list is not
// changed, so that file names and top-level package comments don't get
// lost.
//
// FilterPackage returns true if there are any top-level declarations
// left after filtering; it returns false otherwise.
//
func FilterPackage(pkg *Package, f Filter) bool {
	return filterPackage(pkg, f, false)
}

func filterPackage(pkg *Package, f Filter, export bool) bool {
	hasDecls := false
	for _, src := range pkg.Files {
		if filterFile(src, f, export) {
			hasDecls = true
		}
	}
	return hasDecls
}

// ----------------------------------------------------------------------------
// Merging of package files

// The MergeMode flags control the behavior of MergePackageFiles.
type MergeMode uint

const (
	// If set, duplicate function declarations are excluded.
	FilterFuncDuplicates MergeMode = 1 << iota
	// If set, comments that are not associated with a specific
	// AST node (as Doc or Comment) are excluded.
	FilterUnassociatedComments
	// If set, duplicate import declarations are excluded.
	FilterImportDuplicates
)

// separator is an empty //-style comment that is interspersed between
// different comment groups when they are concatenated into a single group
//
var separator = &Comment{noPos, "//"}

// MergePackageFiles creates a file AST by merging the ASTs of the
// files belonging to a package. The mode flags control merging behavior.
//
func MergePackageFiles(pkg *Package, mode MergeMode) *File {
	// Count the number of package docs, comments and declarations across
	// all package files. Also, compute sorted list of filenames, so that
	// subsequent iterations can always iterate in the same order.
	ndocs := 0
	ncomments := 0
	ndecls := 0
	filenames := make([]string, len(pkg.Files))
	i := 0
	for filename, f := range pkg.Files {
		filenames[i] = filename
		i++
		if f.Doc != nil {
			ndocs += len(f.Doc.List) + 1 // +1 for separator
		}
		ncomments += len(f.Comments)
		ndecls += len(f.Decls)
	}
	sort.Strings(filenames)

	// Collect package comments from all package files into a single
	// CommentGroup - the collected package documentation. In general
	// there should be only one file with a package comment; but it's
	// better to collect extra comments than drop them on the floor.
	var doc *CommentGroup
	var pos token.Pos
	if ndocs > 0 {
		list := make([]*Comment, ndocs-1) // -1: no separator before first group
		i := 0
		for _, filename := range filenames {
			f := pkg.Files[filename]
			if f.Doc != nil {
				if i > 0 {
					// not the first group - add separator
					list[i] = separator
					i++
				}
				for _, c := range f.Doc.List {
					list[i] = c
					i++
				}
				if f.Package > pos {
					// Keep the maximum package clause position as
					// position for the package clause of the merged
					// files.
					pos = f.Package
				}
			}
		}
		doc = &CommentGroup{list}
	}

	// Collect declarations from all package files.
	var decls []Decl
	if ndecls > 0 {
		decls = make([]Decl, ndecls)
		funcs := make(map[string]int) // map of global function name -> decls index
		i := 0                        // current index
		n := 0                        // number of filtered entries
		for _, filename := range filenames {
			f := pkg.Files[filename]
			for _, d := range f.Decls {
				if mode&FilterFuncDuplicates != 0 {
					// A language entity may be declared multiple
					// times in different package files; only at
					// build time declarations must be unique.
					// For now, exclude multiple declarations of
					// functions - keep the one with documentation.
					//
					// TODO(gri): Expand this filtering to other
					//            entities (const, type, vars) if
					//            multiple declarations are common.
					if f, isFun := d.(*FuncDecl); isFun {
						name := f.Name.Name
						if j, exists := funcs[name]; exists {
							// function declared already
							if decls[j] != nil && decls[j].(*FuncDecl).Doc == nil {
								// existing declaration has no documentation;
								// ignore the existing declaration
								decls[j] = nil
							} else {
								// ignore the new declaration
								d = nil
							}
							n++ // filtered an entry
						} else {
							funcs[name] = i
						}
					}
				}
				decls[i] = d
				i++
			}
		}

		// Eliminate nil entries from the decls list if entries were
		// filtered. We do this using a 2nd pass in order to not disturb
		// the original declaration order in the source (otherwise, this
		// would also invalidate the monotonically increasing position
		// info within a single file).
		if n > 0 {
			i = 0
			for _, d := range decls {
				if d != nil {
					decls[i] = d
					i++
				}
			}
			decls = decls[0:i]
		}
	}

	// Collect import specs from all package files.
	var imports []*ImportSpec
	if mode&FilterImportDuplicates != 0 {
		seen := make(map[string]bool)
		for _, filename := range filenames {
			f := pkg.Files[filename]
			for _, imp := range f.Imports {
				if path := imp.Path.Value; !seen[path] {
					// TODO: consider handling cases where:
					// - 2 imports exist with the same import path but
					//   have different local names (one should probably
					//   keep both of them)
					// - 2 imports exist but only one has a comment
					// - 2 imports exist and they both have (possibly
					//   different) comments
					imports = append(imports, imp)
					seen[path] = true
				}
			}
		}
	} else {
		for _, f := range pkg.Files {
			imports = append(imports, f.Imports...)
		}
	}

	// Collect comments from all package files.
	var comments []*CommentGroup
	if mode&FilterUnassociatedComments == 0 {
		comments = make([]*CommentGroup, ncomments)
		i := 0
		for _, f := range pkg.Files {
			i += copy(comments[i:], f.Comments)
		}
	}

	// TODO(gri) need to compute unresolved identifiers!
	return &File{doc, pos, NewIdent(pkg.Name), decls, pkg.Scope, imports, nil, comments}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"runtime"
	"testing"
	"time"
)

func newLocalListener(t *testing.T) Listener {
	ln, err := Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ln, err = Listen("tcp6", "[::1]:0")
	}
	if err != nil {
		t.Fatal(err)
	}
	return ln
}

func TestDialTimeout(t *testing.T) {
	ln := newLocalListener(t)
	defer ln.Close()

	errc := make(chan error)

	numConns := listenerBacklog + 10

	// TODO(bradfitz): It's hard to test this in a portable
	// way. This is unfortunate, but works for now.
	switch runtime.GOOS {
	case "linux":
		// The kernel will start accepting TCP connections before userspace
		// gets a chance to not accept them, so fire off a bunch to fill up
		// the kernel's backlog.  Then we test we get a failure after that.
		for i := 0; i < numConns; i++ {
			go func() {
				_, err := DialTimeout("tcp", ln.Addr().String(), 200*time.Millisecond)
				errc <- err
			}()
		}
	case "darwin", "windows":
		// At least OS X 10.7 seems to accept any number of
		// connections, ignoring listen's backlog, so resort
		// to connecting to a hopefully-dead 127/8 address.
		// Same for windows.
		//
		// Use an IANA reserved port (49151) instead of 80, because
		// on our 386 builder, this Dial succeeds, connecting
		// to an IIS web server somewhere.  The data center
		// or VM or firewall must be stealing the TCP connection.
		//
		// IANA Service Name and Transport Protocol Port Number Registry
		// <http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xml>
		go func() {
			c, err := DialTimeout("tcp", "127.0.71.111:49151", 200*time.Millisecond)
			if err == nil {
				err = fmt.Errorf("unexpected: connected to %s!", c.RemoteAddr())
				c.Close()
			}
			errc <- err
		}()
	default:
		// TODO(bradfitz):
		// OpenBSD may have a reject route to 127/8 except 127.0.0.1/32
		// by default. FreeBSD likely works, but is untested.
		// TODO(rsc):
		// The timeout never happens on Windows.  Why?  Issue 3016.
		t.Logf("skipping test on %q; untested.", runtime.GOOS)
		return
	}

	connected := 0
	for {
		select {
		case <-time.After(15 * time.Second):
			t.Fatal("too slow")
		case err := <-errc:
			if err == nil {
				connected++
				if connected == numConns {
					t.Fatal("all connections connected; expected some to time out")
				}
			} else {
				terr, ok := err.(timeout)
				if !ok {
					t.Fatalf("got error %q; want error with timeout interface", err)
				}
				if !terr.Timeout() {
					t.Fatalf("got error %q; not a timeout", err)
				}
				// Pass. We saw a timeout error.
				return
			}
		}
	}
}

func TestSelfConnect(t *testing.T) {
	if runtime.GOOS == "windows" {
		// TODO(brainman): do not know why it hangs.
		t.Logf("skipping known-broken test on windows")
		return
	}
	// Test that Dial does not honor self-connects.
	// See the comment in DialTCP.

	// Find a port that would be used as a local address.
	l, err := Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	c, err := Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	addr := c.LocalAddr().String()
	c.Close()
	l.Close()

	// Try to connect to that address repeatedly.
	n := 100000
	if testing.Short() {
		n = 1000
	}
	switch runtime.GOOS {
	case "darwin", "freebsd", "netbsd", "openbsd", "plan9", "windows":
		// Non-Linux systems take a long time to figure
		// out that there is nothing listening on localhost.
		n = 100
	}
	for i := 0; i < n; i++ {
		c, err := Dial("tcp", addr)
		if err == nil {
			c.Close()
			t.Errorf("#%d: Dial %q succeeded", i, addr)
		}
	}
}

var runErrorTest = flag.Bool("run_error_test", false, "let TestDialError check for dns errors")

type DialErrorTest struct {
	Net     string
	Raddr   string
	Pattern string
}

var dialErrorTests = []DialErrorTest{
	{
		"datakit", "mh/astro/r70",
		"dial datakit mh/astro/r70: unknown network datakit",
	},
	{
		"tcp", "127.0.0.1:☺",
		"dial tcp 127.0.0.1:☺: unknown port tcp/☺",
	},
	{
		"tcp", "no-such-name.google.com.:80",
		"dial tcp no-such-name.google.com.:80: lookup no-such-name.google.com.( on .*)?: no (.*)",
	},
	{
		"tcp", "no-such-name.no-such-top-level-domain.:80",
		"dial tcp no-such-name.no-such-top-level-domain.:80: lookup no-such-name.no-such-top-level-domain.( on .*)?: no (.*)",
	},
	{
		"tcp", "no-such-name:80",
		`dial tcp no-such-name:80: lookup no-such-name\.(.*\.)?( on .*)?: no (.*)`,
	},
	{
		"tcp", "mh/astro/r70:http",
		"dial tcp mh/astro/r70:http: lookup mh/astro/r70: invalid domain name",
	},
	{
		"unix", "/etc/file-not-found",
		"dial unix /etc/file-not-found: no such file or directory",
	},
	{
		"unix", "/etc/",
		"dial unix /etc/: (permission denied|socket operation on non-socket|connection refused)",
	},
	{
		"unixpacket", "/etc/file-not-found",
		"dial unixpacket /etc/file-not-found: no such file or directory",
	},
	{
		"unixpacket", "/etc/",
		"dial unixpacket /etc/: (permission denied|socket operation on non-socket|connection refused)",
	},
}

var duplicateErrorPattern = `dial (.*) dial (.*)`

func TestDialError(t *testing.T) {
	if !*runErrorTest {
		t.Logf("test disabled; use -run_error_test to enable")
		return
	}
	for i, tt := range dialErrorTests {
		c, err := Dial(tt.Net, tt.Raddr)
		if c != nil {
			c.Close()
		}
		if err == nil {
			t.Errorf("#%d: nil error, want match for %#q", i, tt.Pattern)
			continue
		}
		s := err.Error()
		match, _ := regexp.MatchString(tt.Pattern, s)
		if !match {
			t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern)
		}
		match, _ = regexp.MatchString(duplicateErrorPattern, s)
		if match {
			t.Errorf("#%d: %q, duplicate error return from Dial", i, s)
		}
	}
}

func TestDialTimeoutFDLeak(t *testing.T) {
	if runtime.GOOS != "linux" {
		// TODO(bradfitz): test on other platforms
		t.Logf("skipping test on %s", runtime.GOOS)
		return
	}

	ln := newLocalListener(t)
	defer ln.Close()

	type connErr struct {
		conn Conn
		err  error
	}
	dials := listenerBacklog + 100
	maxGoodConnect := listenerBacklog + 5 // empirically 131 good ones (of 128). who knows?
	resc := make(chan connErr)
	for i := 0; i < dials; i++ {
		go func() {
			conn, err := DialTimeout("tcp", ln.Addr().String(), 500*time.Millisecond)
			resc <- connErr{conn, err}
		}()
	}

	var firstErr string
	var ngood int
	var toClose []io.Closer
	for i := 0; i < dials; i++ {
		ce := <-resc
		if ce.err == nil {
			ngood++
			if ngood > maxGoodConnect {
				t.Errorf("%d good connects; expected at most %d", ngood, maxGoodConnect)
			}
			toClose = append(toClose, ce.conn)
			continue
		}
		err := ce.err
		if firstErr == "" {
			firstErr = err.Error()
		} else if err.Error() != firstErr {
			t.Fatalf("inconsistent error messages: first was %q, then later %q", firstErr, err)
		}
	}
	for _, c := range toClose {
		c.Close()
	}
	for i := 0; i < 100; i++ {
		if got := numFD(); got < dials {
			// Test passes.
			return
		}
		time.Sleep(10 * time.Millisecond)
	}
	if got := numFD(); got >= dials {
		t.Errorf("num fds after %d timeouts = %d; want <%d", dials, got, dials)
	}
}

func numFD() int {
	if runtime.GOOS == "linux" {
		f, err := os.Open("/proc/self/fd")
		if err != nil {
			panic(err)
		}
		defer f.Close()
		names, err := f.Readdirnames(0)
		if err != nil {
			panic(err)
		}
		return len(names)
	}
	// All tests using this should be skipped anyway, but:
	panic("numFDs not implemented on " + runtime.GOOS)
}
<commit_msg>net: TestDialTimeoutFDLeak failure<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"runtime"
	"testing"
	"time"
)

func newLocalListener(t *testing.T) Listener {
	ln, err := Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ln, err = Listen("tcp6", "[::1]:0")
	}
	if err != nil {
		t.Fatal(err)
	}
	return ln
}

func TestDialTimeout(t *testing.T) {
	ln := newLocalListener(t)
	defer ln.Close()

	errc := make(chan error)

	numConns := listenerBacklog + 10

	// TODO(bradfitz): It's hard to test this in a portable
	// way. This is unfortunate, but works for now.
	switch runtime.GOOS {
	case "linux":
		// The kernel will start accepting TCP connections before userspace
		// gets a chance to not accept them, so fire off a bunch to fill up
		// the kernel's backlog.  Then we test we get a failure after that.
		for i := 0; i < numConns; i++ {
			go func() {
				_, err := DialTimeout("tcp", ln.Addr().String(), 200*time.Millisecond)
				errc <- err
			}()
		}
	case "darwin", "windows":
		// At least OS X 10.7 seems to accept any number of
		// connections, ignoring listen's backlog, so resort
		// to connecting to a hopefully-dead 127/8 address.
		// Same for windows.
		//
		// Use an IANA reserved port (49151) instead of 80, because
		// on our 386 builder, this Dial succeeds, connecting
		// to an IIS web server somewhere.  The data center
		// or VM or firewall must be stealing the TCP connection.
		//
		// IANA Service Name and Transport Protocol Port Number Registry
		// <http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xml>
		go func() {
			c, err := DialTimeout("tcp", "127.0.71.111:49151", 200*time.Millisecond)
			if err == nil {
				err = fmt.Errorf("unexpected: connected to %s!", c.RemoteAddr())
				c.Close()
			}
			errc <- err
		}()
	default:
		// TODO(bradfitz):
		// OpenBSD may have a reject route to 127/8 except 127.0.0.1/32
		// by default. FreeBSD likely works, but is untested.
		// TODO(rsc):
		// The timeout never happens on Windows.  Why?  Issue 3016.
		t.Logf("skipping test on %q; untested.", runtime.GOOS)
		return
	}

	connected := 0
	for {
		select {
		case <-time.After(15 * time.Second):
			t.Fatal("too slow")
		case err := <-errc:
			if err == nil {
				connected++
				if connected == numConns {
					t.Fatal("all connections connected; expected some to time out")
				}
			} else {
				terr, ok := err.(timeout)
				if !ok {
					t.Fatalf("got error %q; want error with timeout interface", err)
				}
				if !terr.Timeout() {
					t.Fatalf("got error %q; not a timeout", err)
				}
				// Pass. We saw a timeout error.
				return
			}
		}
	}
}

func TestSelfConnect(t *testing.T) {
	if runtime.GOOS == "windows" {
		// TODO(brainman): do not know why it hangs.
		t.Logf("skipping known-broken test on windows")
		return
	}
	// Test that Dial does not honor self-connects.
	// See the comment in DialTCP.

	// Find a port that would be used as a local address.
	l, err := Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	c, err := Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	addr := c.LocalAddr().String()
	c.Close()
	l.Close()

	// Try to connect to that address repeatedly.
	n := 100000
	if testing.Short() {
		n = 1000
	}
	switch runtime.GOOS {
	case "darwin", "freebsd", "netbsd", "openbsd", "plan9", "windows":
		// Non-Linux systems take a long time to figure
		// out that there is nothing listening on localhost.
		n = 100
	}
	for i := 0; i < n; i++ {
		c, err := Dial("tcp", addr)
		if err == nil {
			c.Close()
			t.Errorf("#%d: Dial %q succeeded", i, addr)
		}
	}
}

var runErrorTest = flag.Bool("run_error_test", false, "let TestDialError check for dns errors")

type DialErrorTest struct {
	Net     string
	Raddr   string
	Pattern string
}

var dialErrorTests = []DialErrorTest{
	{
		"datakit", "mh/astro/r70",
		"dial datakit mh/astro/r70: unknown network datakit",
	},
	{
		"tcp", "127.0.0.1:☺",
		"dial tcp 127.0.0.1:☺: unknown port tcp/☺",
	},
	{
		"tcp", "no-such-name.google.com.:80",
		"dial tcp no-such-name.google.com.:80: lookup no-such-name.google.com.( on .*)?: no (.*)",
	},
	{
		"tcp", "no-such-name.no-such-top-level-domain.:80",
		"dial tcp no-such-name.no-such-top-level-domain.:80: lookup no-such-name.no-such-top-level-domain.( on .*)?: no (.*)",
	},
	{
		"tcp", "no-such-name:80",
		`dial tcp no-such-name:80: lookup no-such-name\.(.*\.)?( on .*)?: no (.*)`,
	},
	{
		"tcp", "mh/astro/r70:http",
		"dial tcp mh/astro/r70:http: lookup mh/astro/r70: invalid domain name",
	},
	{
		"unix", "/etc/file-not-found",
		"dial unix /etc/file-not-found: no such file or directory",
	},
	{
		"unix", "/etc/",
		"dial unix /etc/: (permission denied|socket operation on non-socket|connection refused)",
	},
	{
		"unixpacket", "/etc/file-not-found",
		"dial unixpacket /etc/file-not-found: no such file or directory",
	},
	{
		"unixpacket", "/etc/",
		"dial unixpacket /etc/: (permission denied|socket operation on non-socket|connection refused)",
	},
}

var duplicateErrorPattern = `dial (.*) dial (.*)`

func TestDialError(t *testing.T) {
	if !*runErrorTest {
		t.Logf("test disabled; use -run_error_test to enable")
		return
	}
	for i, tt := range dialErrorTests {
		c, err := Dial(tt.Net, tt.Raddr)
		if c != nil {
			c.Close()
		}
		if err == nil {
			t.Errorf("#%d: nil error, want match for %#q", i, tt.Pattern)
			continue
		}
		s := err.Error()
		match, _ := regexp.MatchString(tt.Pattern, s)
		if !match {
			t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern)
		}
		match, _ = regexp.MatchString(duplicateErrorPattern, s)
		if match {
			t.Errorf("#%d: %q, duplicate error return from Dial", i, s)
		}
	}
}

func TestDialTimeoutFDLeak(t *testing.T) {
	if runtime.GOOS != "linux" {
		// TODO(bradfitz): test on other platforms
		t.Logf("skipping test on %s", runtime.GOOS)
		return
	}

	ln := newLocalListener(t)
	defer ln.Close()

	type connErr struct {
		conn Conn
		err  error
	}
	dials := listenerBacklog + 100
	// used to be listenerBacklog + 5, but was found to be unreliable, issue 4384.
	maxGoodConnect := 150
	resc := make(chan connErr)
	for i := 0; i < dials; i++ {
		go func() {
			conn, err := DialTimeout("tcp", ln.Addr().String(), 500*time.Millisecond)
			resc <- connErr{conn, err}
		}()
	}

	var firstErr string
	var ngood int
	var toClose []io.Closer
	for i := 0; i < dials; i++ {
		ce := <-resc
		if ce.err == nil {
			ngood++
			if ngood > maxGoodConnect {
				t.Errorf("%d good connects; expected at most %d", ngood, maxGoodConnect)
			}
			toClose = append(toClose, ce.conn)
			continue
		}
		err := ce.err
		if firstErr == "" {
			firstErr = err.Error()
		} else if err.Error() != firstErr {
			t.Fatalf("inconsistent error messages: first was %q, then later %q", firstErr, err)
		}
	}
	for _, c := range toClose {
		c.Close()
	}
	for i := 0; i < 100; i++ {
		if got := numFD(); got < dials {
			// Test passes.
			return
		}
		time.Sleep(10 * time.Millisecond)
	}
	if got := numFD(); got >= dials {
		t.Errorf("num fds after %d timeouts = %d; want <%d", dials, got, dials)
	}
}

func numFD() int {
	if runtime.GOOS == "linux" {
		f, err := os.Open("/proc/self/fd")
		if err != nil {
			panic(err)
		}
		defer f.Close()
		names, err := f.Readdirnames(0)
		if err != nil {
			panic(err)
		}
		return len(names)
	}
	// All tests using this should be skipped anyway, but:
	panic("numFDs not implemented on " + runtime.GOOS)
}
<|endoftext|>


<commit_before>package server

import (
	dnet "doozer/net"
	"doozer/paxos"
	"doozer/proto"
	"doozer/store"
	"doozer/util"
	"net"
	"os"
	"rand"
	"reflect"
	"strconv"
	"time"
)

import "log"

const packetSize = 3000

const lease = 3e9 // ns == 3s

var (
	ErrNoWrite = os.NewError("no known writeable address")
	responded  = os.NewError("already responded")
)

const (
	Ok = proto.Line("OK")
)

var slots = store.MustCompileGlob("/doozer/slot/*")

type conn struct {
	*proto.Conn
	c   net.Conn
	s   *Server
	cal bool
}

type Manager interface {
	paxos.Proposer
	ProposeOnce(v string) store.Event
	PutFrom(string, paxos.Msg)
	Alpha() int
}

type Server struct {
	Conn net.PacketConn
	Addr string
	St   *store.Store
	Mg   Manager
	Self string
}

func (sv *Server) ServeUdp(outs chan paxos.Packet) {
	r := dnet.Ackify(sv.Conn, outs)

	for p := range r {
		sv.Mg.PutFrom(p.Addr, p.Msg)
	}
}

var clg = util.NewLogger("cal")

func (s *Server) Serve(l net.Listener, cal chan int) os.Error {
	for {
		rw, err := l.Accept()
		if err != nil {
			log.Printf("%#v", err)
			if e, ok := err.(*net.OpError); ok && e.Error == os.EINVAL {
				return nil
			}
			return err
		}
		c := &conn{proto.NewConn(rw), rw, s, closed(cal)}
		go c.serve()
	}

	panic("unreachable")
}

func (sv *Server) cals() []string {
	cals := make([]string, 0)
	_, g := sv.St.Snap()
	store.Walk(g, slots, func(_, body, _ string) {
		if len(body) > 0 {
			cals = append(cals, body)
		}
	})
	return cals
}

// Repeatedly propose nop values until a successful read from `done`.
func (sv *Server) AdvanceUntil(done chan int) {
	for _, ok := <-done; !ok; _, ok = <-done {
		sv.Mg.Propose(store.Nop)
	}
}

func (c *conn) redirect(rid uint) {
	cals := c.s.cals()
	if len(cals) < 1 {
		c.SendResponse(rid, proto.Valid|proto.Done, ErrNoWrite)
		return
	}
	cal := cals[rand.Intn(len(cals))]
	parts, cas := c.s.St.Get("/doozer/info/" + cal + "/public-addr")
	if cas == store.Dir && cas == store.Missing {
		c.SendResponse(rid, proto.Valid|proto.Done, ErrNoWrite)
		return
	}
	c.SendResponse(rid, proto.Valid|proto.Done, proto.Redirect(parts[0]))
}

func get(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqGet)
	v, cas := c.s.St.Get(r.Path)
	return proto.ResGet{v, cas}
}

func sget(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqGet)
	g, err := c.s.St.SyncPath(r.Path)
	if err != nil {
		return err
	}
	return store.GetString(g, r.Path)
}

func set(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqSet)
	_, cas, err := paxos.Set(c.s.Mg, r.Path, r.Body, r.Cas)
	if err != nil {
		return err
	}
	return cas
}

func del(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqDel)
	err := paxos.Del(c.s.Mg, r.Path, r.Cas)
	if err != nil {
		return err
	}

	return Ok
}

func noop(c *conn, _ uint, data interface{}) interface{} {
	c.s.Mg.ProposeOnce(store.Nop)
	return Ok
}

func join(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqJoin)
	key := "/doozer/members/" + r.Who
	seqn, _, err := paxos.Set(c.s.Mg, key, r.Addr, store.Missing)
	if err != nil {
		return err
	}

	done := make(chan int)
	go c.s.AdvanceUntil(done)
	c.s.St.Sync(seqn + uint64(c.s.Mg.Alpha()))
	close(done)
	seqn, snap := c.s.St.Snapshot()
	return proto.ResJoin{seqn, snap}
}

func sett(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqSett)
	t := time.Nanoseconds() + r.Interval
	_, cas, err := paxos.Set(c.s.Mg, r.Path, strconv.Itoa64(t), r.Cas)
	if err != nil {
		return err
	}
	return proto.ResSett{t, cas}
}

func checkin(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqCheckin)
	t := time.Nanoseconds() + lease
	_, cas, err := paxos.Set(c.s.Mg, "/session/"+r.Sid, strconv.Itoa64(t), r.Cas)
	if err != nil {
		return err
	}
	return proto.ResCheckin{t, cas}
}

func closeOp(c *conn, _ uint, data interface{}) interface{} {
	err := c.CloseResponse(data.(uint))
	if err != nil {
		return err
	}
	return Ok
}

func watch(c *conn, id uint, data interface{}) interface{} {
	glob, err := store.CompileGlob(data.(string))
	if err != nil {
		return err
	}

	w := store.NewWatch(c.s.St, glob)

	// TODO buffer (and possibly discard) events
	for ev := range w.C {
		var r proto.ResWatch
		r.Path = ev.Path
		r.Body = ev.Body
		r.Cas = ev.Cas
		err := c.SendResponse(id, proto.Valid, r)
		if err == proto.ErrClosed {
			w.Stop()
			close(w.C)
			err = nil
		}
		if err != nil {
			// TODO log error
			break
		}
	}
	return responded
}

func indirect(x interface{}) interface{} {
	return reflect.Indirect(reflect.NewValue(x)).Interface()
}

type handler func(*conn, uint, interface{}) interface{}

type op struct {
	p interface{}
	f handler

	redirect bool
}

var ops = map[string]op{
	// new stuff, see doc/proto.md
	"CLOSE": {p: new(uint), f: closeOp},
	"DEL":   {p: new(*proto.ReqDel), f: del, redirect: true},
	"NOOP":  {p: new(interface{}), f: noop, redirect: true},
	"GET":   {p: new(*proto.ReqGet), f: get},
	"SET":   {p: new(*proto.ReqSet), f: set, redirect: true},
	"SETT":  {p: new(*proto.ReqSett), f: sett, redirect: true},
	"WATCH": {p: new(string), f: watch},

	// former stuff
	"sget":    {p: new(*proto.ReqGet), f: sget},
	"join":    {p: new(*proto.ReqJoin), f: join, redirect: true},
	"checkin": {p: new(*proto.ReqCheckin), f: checkin, redirect: true},
}

func (c *conn) handle(rid uint, f handler, data interface{}) {
	res := f(c, rid, data)
	if res == responded {
		return
	}

	c.SendResponse(rid, proto.Valid|proto.Done, res)
}

func (c *conn) serve() {
	logger := util.NewLogger("%v", c.c.RemoteAddr())
	logger.Println("accepted connection")
	for {
		rid, verb, data, err := c.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Println("connection closed by peer")
			} else {
				logger.Println(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.c.RemoteAddr(), rid)

		if o, ok := ops[verb]; ok {
			err := proto.Fit(data, o.p)
			if err != nil {
				c.SendResponse(rid, proto.Valid|proto.Done, err)
				continue
			}

			if o.redirect && !c.cal {
				c.redirect(rid)
				continue
			}

			go c.handle(rid, o.f, indirect(o.p))
			continue
		}

		rlogger.Printf("unknown command <%s>", verb)
		c.SendResponse(rid, proto.Valid|proto.Done, os.ErrorString(proto.InvalidCommand+" "+verb))
	}
}
<commit_msg>remove noisy debugging output<commit_after>package server

import (
	dnet "doozer/net"
	"doozer/paxos"
	"doozer/proto"
	"doozer/store"
	"doozer/util"
	"net"
	"os"
	"rand"
	"reflect"
	"strconv"
	"time"
)

const packetSize = 3000

const lease = 3e9 // ns == 3s

var (
	ErrNoWrite = os.NewError("no known writeable address")
	responded  = os.NewError("already responded")
)

const (
	Ok = proto.Line("OK")
)

var slots = store.MustCompileGlob("/doozer/slot/*")

type conn struct {
	*proto.Conn
	c   net.Conn
	s   *Server
	cal bool
}

type Manager interface {
	paxos.Proposer
	ProposeOnce(v string) store.Event
	PutFrom(string, paxos.Msg)
	Alpha() int
}

type Server struct {
	Conn net.PacketConn
	Addr string
	St   *store.Store
	Mg   Manager
	Self string
}

func (sv *Server) ServeUdp(outs chan paxos.Packet) {
	r := dnet.Ackify(sv.Conn, outs)

	for p := range r {
		sv.Mg.PutFrom(p.Addr, p.Msg)
	}
}

var clg = util.NewLogger("cal")

func (s *Server) Serve(l net.Listener, cal chan int) os.Error {
	for {
		rw, err := l.Accept()
		if err != nil {
			if e, ok := err.(*net.OpError); ok && e.Error == os.EINVAL {
				return nil
			}
			return err
		}
		c := &conn{proto.NewConn(rw), rw, s, closed(cal)}
		go c.serve()
	}

	panic("unreachable")
}

func (sv *Server) cals() []string {
	cals := make([]string, 0)
	_, g := sv.St.Snap()
	store.Walk(g, slots, func(_, body, _ string) {
		if len(body) > 0 {
			cals = append(cals, body)
		}
	})
	return cals
}

// Repeatedly propose nop values until a successful read from `done`.
func (sv *Server) AdvanceUntil(done chan int) {
	for _, ok := <-done; !ok; _, ok = <-done {
		sv.Mg.Propose(store.Nop)
	}
}

func (c *conn) redirect(rid uint) {
	cals := c.s.cals()
	if len(cals) < 1 {
		c.SendResponse(rid, proto.Valid|proto.Done, ErrNoWrite)
		return
	}
	cal := cals[rand.Intn(len(cals))]
	parts, cas := c.s.St.Get("/doozer/info/" + cal + "/public-addr")
	if cas == store.Dir && cas == store.Missing {
		c.SendResponse(rid, proto.Valid|proto.Done, ErrNoWrite)
		return
	}
	c.SendResponse(rid, proto.Valid|proto.Done, proto.Redirect(parts[0]))
}

func get(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqGet)
	v, cas := c.s.St.Get(r.Path)
	return proto.ResGet{v, cas}
}

func sget(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqGet)
	g, err := c.s.St.SyncPath(r.Path)
	if err != nil {
		return err
	}
	return store.GetString(g, r.Path)
}

func set(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqSet)
	_, cas, err := paxos.Set(c.s.Mg, r.Path, r.Body, r.Cas)
	if err != nil {
		return err
	}
	return cas
}

func del(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqDel)
	err := paxos.Del(c.s.Mg, r.Path, r.Cas)
	if err != nil {
		return err
	}

	return Ok
}

func noop(c *conn, _ uint, data interface{}) interface{} {
	c.s.Mg.ProposeOnce(store.Nop)
	return Ok
}

func join(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqJoin)
	key := "/doozer/members/" + r.Who
	seqn, _, err := paxos.Set(c.s.Mg, key, r.Addr, store.Missing)
	if err != nil {
		return err
	}

	done := make(chan int)
	go c.s.AdvanceUntil(done)
	c.s.St.Sync(seqn + uint64(c.s.Mg.Alpha()))
	close(done)
	seqn, snap := c.s.St.Snapshot()
	return proto.ResJoin{seqn, snap}
}

func sett(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqSett)
	t := time.Nanoseconds() + r.Interval
	_, cas, err := paxos.Set(c.s.Mg, r.Path, strconv.Itoa64(t), r.Cas)
	if err != nil {
		return err
	}
	return proto.ResSett{t, cas}
}

func checkin(c *conn, _ uint, data interface{}) interface{} {
	r := data.(*proto.ReqCheckin)
	t := time.Nanoseconds() + lease
	_, cas, err := paxos.Set(c.s.Mg, "/session/"+r.Sid, strconv.Itoa64(t), r.Cas)
	if err != nil {
		return err
	}
	return proto.ResCheckin{t, cas}
}

func closeOp(c *conn, _ uint, data interface{}) interface{} {
	err := c.CloseResponse(data.(uint))
	if err != nil {
		return err
	}
	return Ok
}

func watch(c *conn, id uint, data interface{}) interface{} {
	glob, err := store.CompileGlob(data.(string))
	if err != nil {
		return err
	}

	w := store.NewWatch(c.s.St, glob)

	// TODO buffer (and possibly discard) events
	for ev := range w.C {
		var r proto.ResWatch
		r.Path = ev.Path
		r.Body = ev.Body
		r.Cas = ev.Cas
		err := c.SendResponse(id, proto.Valid, r)
		if err == proto.ErrClosed {
			w.Stop()
			close(w.C)
			err = nil
		}
		if err != nil {
			// TODO log error
			break
		}
	}
	return responded
}

func indirect(x interface{}) interface{} {
	return reflect.Indirect(reflect.NewValue(x)).Interface()
}

type handler func(*conn, uint, interface{}) interface{}

type op struct {
	p interface{}
	f handler

	redirect bool
}

var ops = map[string]op{
	// new stuff, see doc/proto.md
	"CLOSE": {p: new(uint), f: closeOp},
	"DEL":   {p: new(*proto.ReqDel), f: del, redirect: true},
	"NOOP":  {p: new(interface{}), f: noop, redirect: true},
	"GET":   {p: new(*proto.ReqGet), f: get},
	"SET":   {p: new(*proto.ReqSet), f: set, redirect: true},
	"SETT":  {p: new(*proto.ReqSett), f: sett, redirect: true},
	"WATCH": {p: new(string), f: watch},

	// former stuff
	"sget":    {p: new(*proto.ReqGet), f: sget},
	"join":    {p: new(*proto.ReqJoin), f: join, redirect: true},
	"checkin": {p: new(*proto.ReqCheckin), f: checkin, redirect: true},
}

func (c *conn) handle(rid uint, f handler, data interface{}) {
	res := f(c, rid, data)
	if res == responded {
		return
	}

	c.SendResponse(rid, proto.Valid|proto.Done, res)
}

func (c *conn) serve() {
	logger := util.NewLogger("%v", c.c.RemoteAddr())
	logger.Println("accepted connection")
	for {
		rid, verb, data, err := c.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Println("connection closed by peer")
			} else {
				logger.Println(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.c.RemoteAddr(), rid)

		if o, ok := ops[verb]; ok {
			err := proto.Fit(data, o.p)
			if err != nil {
				c.SendResponse(rid, proto.Valid|proto.Done, err)
				continue
			}

			if o.redirect && !c.cal {
				c.redirect(rid)
				continue
			}

			go c.handle(rid, o.f, indirect(o.p))
			continue
		}

		rlogger.Printf("unknown command <%s>", verb)
		c.SendResponse(rid, proto.Valid|proto.Done, os.ErrorString(proto.InvalidCommand+" "+verb))
	}
}
<|endoftext|>


<commit_before>package ogdat

import (
	"encoding/csv"
	"io"
	"log"
	"os"
	"strconv"
)

/*	Version10 = "OGD Austria Metadata 1.0" // Version 1.0: 24.10.2011
	Version11 = "OGD Austria Metadata 1.1" // Version 1.1: 12.03.2012
	Version20 = "OGD Austria Metadata 2.0" // Version 2.0: 10.10.2012
	Version21 = "OGD Austria Metadata 2.1" // Version 2.1: 15.10.2012
*/
type OGDSet []*Beschreibung

var specification = make(map[string]OGDSet)

type Occurrence int

const (
	OccUndef Occurrence = iota
	OccRequired
	OccOptional
)

func (desc *Beschreibung) Version() string {
	return desc.version
}

func (desc *Beschreibung) Occurrence() Occurrence {
	return desc.occurrence
}

type Beschreibung struct {
	ID            int
	Bezeichner    string
	OGD_Kurzname  string
	CKAN_Feld     string
	Anzahl        byte
	Definition_DE string
	Erlauterung   string
	Beispiel      string
	ONA2270       string
	ISO19115      string
	RDFProperty   string
	Definition_EN string
	occurrence    Occurrence
	version       string
}

func NewBeschreibung(ID int, occur Occurrence, ver string) *Beschreibung {
	return &Beschreibung{ID: ID, occurrence: occur, version: ver}
}

func (set OGDSet) GetSpecForID(id int) *Beschreibung {
	if set != nil {
		for idx, elm := range set {
			if elm.ID == id {
				return set[idx]
			}
		}
	}
	return nil
}

func Register(version, specfile string) {
	specmap, _ := Loadogdatspec(version, specfile)
	specification[version] = specmap
}

func Loadogdatspec(version, filename string) (OGDSet, error) {
	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	spec := make(OGDSet, 0)
	csvreader := csv.NewReader(reader)
	csvreader.Comma = '|'
	csvreader.LazyQuotes = true

	// skip the first line as it contains the field description
	record, err := csvreader.Read()

	for record, err = csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
		id, _ := strconv.Atoi(record[0])
		var occ Occurrence
		switch record[12][0] {
		case 'R':
			occ = OccRequired
		case 'O':
			occ = OccOptional
		}
		descrecord := NewBeschreibung(id, occ, version)

		descrecord.Bezeichner = record[1]
		descrecord.OGD_Kurzname = record[2]
		descrecord.CKAN_Feld = record[3]
		descrecord.Anzahl = byte(record[4][0])
		descrecord.Definition_DE = record[5]
		descrecord.Erlauterung = record[6]
		descrecord.Beispiel = record[7]
		descrecord.ONA2270 = record[8]
		descrecord.ISO19115 = record[9]
		descrecord.RDFProperty = record[10]
		descrecord.Definition_EN = record[11]

		spec = append(spec, descrecord)
	}
	log.Printf("Info: Read %d %s specifiaction records", len(spec), version)

	return spec, nil
}
<commit_msg>add labels of specification as a description of the set<commit_after>package ogdat

import (
	"encoding/csv"
	"io"
	"log"
	"os"
	"strconv"
)

/*	Version10 = "OGD Austria Metadata 1.0" // Version 1.0: 24.10.2011
	Version11 = "OGD Austria Metadata 1.1" // Version 1.1: 12.03.2012
	Version20 = "OGD Austria Metadata 2.0" // Version 2.0: 10.10.2012
	Version21 = "OGD Austria Metadata 2.1" // Version 2.1: 15.10.2012
*/
var specification = make(map[string]*OGDSet)

type Occurrence int

const (
	OccUndef Occurrence = iota
	OccRequired
	OccOptional
)

type Beschreibung struct {
	ID            int
	Bezeichner    string
	OGD_Kurzname  string
	CKAN_Feld     string
	Anzahl        string
	Definition_DE string
	Erlauterung   string
	Beispiel      string
	ONA2270       string
	ISO19115      string
	RDFProperty   string
	Definition_EN string
	occurrence    Occurrence
	version       string
}

func NewBeschreibung(ID int, occur Occurrence, ver string) *Beschreibung {
	return &Beschreibung{ID: ID, occurrence: occur, version: ver}
}

func (desc *Beschreibung) Version() string {
	return desc.version
}

func (desc *Beschreibung) Occurrence() Occurrence {
	return desc.occurrence
}

func (desc *Beschreibung) IsRequired() bool {
	return desc.occurrence == OccRequired
}

type OGDSet struct {
	Label        []string
	Beschreibung []*Beschreibung
}

func (set *OGDSet) GetSpecForID(id int) *Beschreibung {
	if set != nil {
		for idx, elm := range set.Beschreibung {
			if elm.ID == id {
				return set.Beschreibung[idx]
			}
		}
	}
	return nil
}

func Register(version, specfile string) {
	specmap, _ := Loadogdatspec(version, specfile)
	specification[version] = specmap
}

func Loadogdatspec(version, filename string) (*OGDSet, error) {
	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	csvreader := csv.NewReader(reader)
	csvreader.Comma = '|'
	csvreader.LazyQuotes = true

	// Read the first line and use it as the labels for the items to load
	record, err := csvreader.Read()
	if err != nil {
		return nil, err
	}

	set := &OGDSet{}
	set.Label = record

	spec := make([]*Beschreibung, 0)
	for record, err = csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
		id, _ := strconv.Atoi(record[0])
		var occ Occurrence
		switch record[12][0] {
		case 'R':
			occ = OccRequired
		case 'O':
			occ = OccOptional
		}
		descrecord := NewBeschreibung(id, occ, version)

		descrecord.Bezeichner = record[1]
		descrecord.OGD_Kurzname = record[2]
		descrecord.CKAN_Feld = record[3]
		descrecord.Anzahl = record[4]
		descrecord.Definition_DE = record[5]
		descrecord.Erlauterung = record[6]
		descrecord.Beispiel = record[7]
		descrecord.ONA2270 = record[8]
		descrecord.ISO19115 = record[9]
		descrecord.RDFProperty = record[10]
		descrecord.Definition_EN = record[11]

		spec = append(spec, descrecord)
	}
	set.Beschreibung = spec
	log.Printf("Info: Read %d %s specifiaction records", len(spec), version)

	return set, nil
}
<|endoftext|>