<commit_before>package source

import (
	"bytes"
	"io"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

// CommentToMarkdown converts comment text to formatted markdown.
// The comment was prepared by DocReader,
// so it is known not to have leading, trailing blank lines
// nor to have trailing spaces at the end of lines.
// The comment markers have already been removed.
//
// Each line is converted into a markdown line and empty lines are just converted to
// newlines. Heading are prefixed with `### ` to make it a markdown heading.
//
// A span of indented lines retains a 4 space prefix block, with the common indent
// prefix removed unless empty, in which case it will be converted to a newline.
//
// URLs in the comment text are converted into links.
func CommentToMarkdown(text string) string {
	buf := &bytes.Buffer{}
	commentToMarkdown(buf, text)
	return buf.String()
}

var (
	mdNewline   = []byte("\n")
	mdHeader    = []byte("### ")
	mdIndent    = []byte("    ")
	mdLinkStart = []byte("[")
	mdLinkDiv   = []byte("](")
	mdLinkEnd   = []byte(")")
)

func commentToMarkdown(w io.Writer, text string) {
	isFirstLine := true
	for _, b := range blocks(text) {
		switch b.op {
		case opPara:
			if !isFirstLine {
				w.Write(mdNewline)
			}

			for _, line := range b.lines {
				emphasize(w, line, true)
			}
		case opHead:
			if !isFirstLine {
				w.Write(mdNewline)
			}
			w.Write(mdNewline)

			for _, line := range b.lines {
				w.Write(mdHeader)
				commentEscape(w, line, true)
				w.Write(mdNewline)
			}
		case opPre:
			if !isFirstLine {
				w.Write(mdNewline)
			}
			w.Write(mdNewline)

			for _, line := range b.lines {
				if isBlank(line) {
					w.Write(mdNewline)
				} else {
					w.Write(mdIndent)
					w.Write([]byte(line))
					w.Write(mdNewline)
				}
			}
		}
		isFirstLine = false
	}
}

const (
	ulquo = "“"
	urquo = "”"
)

var (
	markdownEscape = regexp.MustCompile(`([\\\x60*{}[\]()#+\-.!_>~|"$%&'\/:;<=?@^])`)

	unicodeQuoteReplacer = strings.NewReplacer("``", ulquo, "''", urquo)
)

// commentEscape escapes comment text for markdown. If nice is set,
// also turn `` into “; and '' into ”;.
func commentEscape(w io.Writer, text string, nice bool) {
	if nice {
		text = convertQuotes(text)
	}
	text = escapeRegex(text)
	w.Write([]byte(text))
}

func convertQuotes(text string) string {
	return unicodeQuoteReplacer.Replace(text)
}

func escapeRegex(text string) string {
	return markdownEscape.ReplaceAllString(text, `\$1`)
}

func emphasize(w io.Writer, line string, nice bool) {
	for {
		m := matchRx.FindStringSubmatchIndex(line)
		if m == nil {
			break
		}
		// m >= 6 (two parenthesized sub-regexps in matchRx, 1st one is urlRx)

		// write text before match
		commentEscape(w, line[0:m[0]], nice)

		// adjust match for URLs
		match := line[m[0]:m[1]]
		if strings.Contains(match, "://") {
			m0, m1 := m[0], m[1]
			for _, s := range []string{"()", "{}", "[]"} {
				open, close := s[:1], s[1:] // E.g., "(" and ")"
				// require opening parentheses before closing parentheses (#22285)
				if i := strings.Index(match, close); i >= 0 && i < strings.Index(match, open) {
					m1 = m0 + i
					match = line[m0:m1]
				}
				// require balanced pairs of parentheses (#5043)
				for i := 0; strings.Count(match, open) != strings.Count(match, close) && i < 10; i++ {
					m1 = strings.LastIndexAny(line[:m1], s)
					match = line[m0:m1]
				}
			}
			if m1 != m[1] {
				// redo matching with shortened line for correct indices
				m = matchRx.FindStringSubmatchIndex(line[:m[0]+len(match)])
			}
		}

		// Following code has been modified from go/doc since words is always
		// nil. All html formatting has also been transformed into markdown formatting

		// analyze match
		url := ""
		if m[2] >= 0 {
			url = match
		}

		// write match
		if len(url) > 0 {
			w.Write(mdLinkStart)
		}

		commentEscape(w, match, nice)

		if len(url) > 0 {
			w.Write(mdLinkDiv)
			w.Write([]byte(urlReplacer.Replace(url)))
			w.Write(mdLinkEnd)
		}

		// advance
		line = line[m[1]:]
	}
	commentEscape(w, line, nice)
}

// Everything from here on is a copy of go/doc/comment.go

const (
	// Regexp for Go identifiers
	identRx = `[\pL_][\pL_0-9]*`

	// Regexp for URLs
	// Match parens, and check later for balance - see #5043, #22285
	// Match .,:;?! within path, but not at end - see #18139, #16565
	// This excludes some rare yet valid urls ending in common punctuation
	// in order to allow sentences ending in URLs.

	// protocol (required) e.g. http
	protoPart = `(https?|ftp|file|gopher|mailto|nntp)`
	// host (required) e.g. www.example.com or [::1]:8080
	hostPart = `([a-zA-Z0-9_@\-.\[\]:]+)`
	// path+query+fragment (optional) e.g. /path/index.html?q=foo#bar
	pathPart = `([.,:;?!]*[a-zA-Z0-9$'()*+&#=@~_/\-\[\]%])*`

	urlRx = protoPart + `://` + hostPart + pathPart
)

var (
	matchRx     = regexp.MustCompile(`(` + urlRx + `)|(` + identRx + `)`)
	urlReplacer = strings.NewReplacer(`(`, `\(`, `)`, `\)`)
)

func indentLen(s string) int {
	i := 0
	for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
		i++
	}
	return i
}

func isBlank(s string) bool {
	return len(s) == 0 || (len(s) == 1 && s[0] == '\n')
}

func commonPrefix(a, b string) string {
	i := 0
	for i < len(a) && i < len(b) && a[i] == b[i] {
		i++
	}
	return a[0:i]
}

func unindent(block []string) {
	if len(block) == 0 {
		return
	}

	// compute maximum common white prefix
	prefix := block[0][0:indentLen(block[0])]
	for _, line := range block {
		if !isBlank(line) {
			prefix = commonPrefix(prefix, line[0:indentLen(line)])
		}
	}
	n := len(prefix)

	// remove
	for i, line := range block {
		if !isBlank(line) {
			block[i] = line[n:]
		}
	}
}

// heading returns the trimmed line if it passes as a section heading;
// otherwise it returns the empty string.
func heading(line string) string {
	line = strings.TrimSpace(line)
	if len(line) == 0 {
		return ""
	}

	// a heading must start with an uppercase letter
	r, _ := utf8.DecodeRuneInString(line)
	if !unicode.IsLetter(r) || !unicode.IsUpper(r) {
		return ""
	}

	// it must end in a letter or digit:
	r, _ = utf8.DecodeLastRuneInString(line)
	if !unicode.IsLetter(r) && !unicode.IsDigit(r) {
		return ""
	}

	// exclude lines with illegal characters. we allow "(),"
	if strings.ContainsAny(line, ";:!?+*/=[]{}_^°&§~%#@<\">\\") {
		return ""
	}

	// allow "'" for possessive "'s" only
	for b := line; ; {
		i := strings.IndexRune(b, '\'')
		if i < 0 {
			break
		}
		if i+1 >= len(b) || b[i+1] != 's' || (i+2 < len(b) && b[i+2] != ' ') {
			return "" // not followed by "s "
		}
		b = b[i+2:]
	}

	// allow "." when followed by non-space
	for b := line; ; {
		i := strings.IndexRune(b, '.')
		if i < 0 {
			break
		}
		if i+1 >= len(b) || b[i+1] == ' ' {
			return "" // not followed by non-space
		}
		b = b[i+1:]
	}

	return line
}

type op int

const (
	opPara op = iota
	opHead
	opPre
)

type block struct {
	op    op
	lines []string
}

func blocks(text string) []block {
	var (
		out  []block
		para []string

		lastWasBlank   = false
		lastWasHeading = false
	)

	close := func() {
		if para != nil {
			out = append(out, block{opPara, para})
			para = nil
		}
	}

	lines := strings.SplitAfter(text, "\n")
	unindent(lines)
	for i := 0; i < len(lines); {
		line := lines[i]
		if isBlank(line) {
			// close paragraph
			close()
			i++
			lastWasBlank = true
			continue
		}
		if indentLen(line) > 0 {
			// close paragraph
			close()

			// count indented or blank lines
			j := i + 1
			for j < len(lines) && (isBlank(lines[j]) || indentLen(lines[j]) > 0) {
				j++
			}
			// but not trailing blank lines
			for j > i && isBlank(lines[j-1]) {
				j--
			}
			pre := lines[i:j]
			i = j

			unindent(pre)

			// put those lines in a pre block
			out = append(out, block{opPre, pre})
			lastWasHeading = false
			continue
		}

		if lastWasBlank && !lastWasHeading && i+2 < len(lines) &&
			isBlank(lines[i+1]) && !isBlank(lines[i+2]) && indentLen(lines[i+2]) == 0 {
			// current line is non-blank, surrounded by blank lines
			// and the next non-blank line is not indented: this
			// might be a heading.
			if head := heading(line); head != "" {
				close()
				out = append(out, block{opHead, []string{head}})
				i += 2
				lastWasHeading = true
				continue
			}
		}

		// open paragraph
		lastWasBlank = false
		lastWasHeading = false
		para = append(para, lines[i])
		i++
	}
	close()

	return out
}
<commit_msg>internal/lsp/source: use space character in markdown formatting<commit_after>package source

import (
	"bytes"
	"io"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

// CommentToMarkdown converts comment text to formatted markdown.
// The comment was prepared by DocReader,
// so it is known not to have leading, trailing blank lines
// nor to have trailing spaces at the end of lines.
// The comment markers have already been removed.
//
// Each line is converted into a markdown line and empty lines are just converted to
// newlines. Heading are prefixed with `### ` to make it a markdown heading.
//
// A span of indented lines retains a 4 space prefix block, with the common indent
// prefix removed unless empty, in which case it will be converted to a newline.
//
// URLs in the comment text are converted into links.
func CommentToMarkdown(text string) string {
	buf := &bytes.Buffer{}
	commentToMarkdown(buf, text)
	return buf.String()
}

var (
	mdNewline   = []byte("\n")
	mdHeader    = []byte("### ")
	mdIndent    = []byte("    ")
	mdLinkStart = []byte("[")
	mdLinkDiv   = []byte("](")
	mdLinkEnd   = []byte(")")
)

func commentToMarkdown(w io.Writer, text string) {
	isFirstLine := true
	for _, b := range blocks(text) {
		switch b.op {
		case opPara:
			if !isFirstLine {
				w.Write(mdNewline)
			}

			for _, line := range b.lines {
				emphasize(w, line, true)
			}
		case opHead:
			if !isFirstLine {
				w.Write(mdNewline)
			}
			w.Write(mdNewline)

			for _, line := range b.lines {
				w.Write(mdHeader)
				commentEscape(w, line, true)
				w.Write(mdNewline)
			}
		case opPre:
			if !isFirstLine {
				w.Write(mdNewline)
			}
			w.Write(mdNewline)

			for _, line := range b.lines {
				if isBlank(line) {
					w.Write(mdNewline)
				} else {
					w.Write(mdIndent)
					w.Write([]byte(line))
					w.Write(mdNewline)
				}
			}
		}
		isFirstLine = false
	}
}

const (
	ulquo = "“"
	urquo = "”"
)

var (
	markdownEscape = regexp.MustCompile(`([\\\x60*{}[\]()#+\-.!_>~|"$%&'\/:;<=?@^])`)

	unicodeQuoteReplacer = strings.NewReplacer("``", ulquo, "''", urquo)
)

// commentEscape escapes comment text for markdown. If nice is set,
// also turn `` into “; and '' into ”;.
func commentEscape(w io.Writer, text string, nice bool) {
	if nice {
		text = convertQuotes(text)
	}
	text = escapeRegex(text)
	w.Write([]byte(text))
}

func convertQuotes(text string) string {
	return unicodeQuoteReplacer.Replace(text)
}

func escapeRegex(text string) string {
	return markdownEscape.ReplaceAllString(text, `\$1`)
}

func emphasize(w io.Writer, line string, nice bool) {
	for {
		m := matchRx.FindStringSubmatchIndex(line)
		if m == nil {
			break
		}
		// m >= 6 (two parenthesized sub-regexps in matchRx, 1st one is urlRx)

		// write text before match
		commentEscape(w, line[0:m[0]], nice)

		// adjust match for URLs
		match := line[m[0]:m[1]]
		if strings.Contains(match, "://") {
			m0, m1 := m[0], m[1]
			for _, s := range []string{"()", "{}", "[]"} {
				open, close := s[:1], s[1:] // E.g., "(" and ")"
				// require opening parentheses before closing parentheses (#22285)
				if i := strings.Index(match, close); i >= 0 && i < strings.Index(match, open) {
					m1 = m0 + i
					match = line[m0:m1]
				}
				// require balanced pairs of parentheses (#5043)
				for i := 0; strings.Count(match, open) != strings.Count(match, close) && i < 10; i++ {
					m1 = strings.LastIndexAny(line[:m1], s)
					match = line[m0:m1]
				}
			}
			if m1 != m[1] {
				// redo matching with shortened line for correct indices
				m = matchRx.FindStringSubmatchIndex(line[:m[0]+len(match)])
			}
		}

		// Following code has been modified from go/doc since words is always
		// nil. All html formatting has also been transformed into markdown formatting

		// analyze match
		url := ""
		if m[2] >= 0 {
			url = match
		}

		// write match
		if len(url) > 0 {
			w.Write(mdLinkStart)
		}

		commentEscape(w, match, nice)

		if len(url) > 0 {
			w.Write(mdLinkDiv)
			w.Write([]byte(urlReplacer.Replace(url)))
			w.Write(mdLinkEnd)
		}

		// advance
		line = line[m[1]:]
	}
	commentEscape(w, line, nice)
}

// Everything from here on is a copy of go/doc/comment.go

const (
	// Regexp for Go identifiers
	identRx = `[\pL_][\pL_0-9]*`

	// Regexp for URLs
	// Match parens, and check later for balance - see #5043, #22285
	// Match .,:;?! within path, but not at end - see #18139, #16565
	// This excludes some rare yet valid urls ending in common punctuation
	// in order to allow sentences ending in URLs.

	// protocol (required) e.g. http
	protoPart = `(https?|ftp|file|gopher|mailto|nntp)`
	// host (required) e.g. www.example.com or [::1]:8080
	hostPart = `([a-zA-Z0-9_@\-.\[\]:]+)`
	// path+query+fragment (optional) e.g. /path/index.html?q=foo#bar
	pathPart = `([.,:;?!]*[a-zA-Z0-9$'()*+&#=@~_/\-\[\]%])*`

	urlRx = protoPart + `://` + hostPart + pathPart
)

var (
	matchRx     = regexp.MustCompile(`(` + urlRx + `)|(` + identRx + `)`)
	urlReplacer = strings.NewReplacer(`(`, `\(`, `)`, `\)`)
)

func indentLen(s string) int {
	i := 0
	for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
		i++
	}
	return i
}

func isBlank(s string) bool {
	return len(s) == 0 || (len(s) == 1 && s[0] == '\n')
}

func commonPrefix(a, b string) string {
	i := 0
	for i < len(a) && i < len(b) && a[i] == b[i] {
		i++
	}
	return a[0:i]
}

func unindent(block []string) {
	if len(block) == 0 {
		return
	}

	// compute maximum common white prefix
	prefix := block[0][0:indentLen(block[0])]
	for _, line := range block {
		if !isBlank(line) {
			prefix = commonPrefix(prefix, line[0:indentLen(line)])
		}
	}
	n := len(prefix)

	// remove
	for i, line := range block {
		if !isBlank(line) {
			block[i] = line[n:]
		}
	}
}

// heading returns the trimmed line if it passes as a section heading;
// otherwise it returns the empty string.
func heading(line string) string {
	line = strings.TrimSpace(line)
	if len(line) == 0 {
		return ""
	}

	// a heading must start with an uppercase letter
	r, _ := utf8.DecodeRuneInString(line)
	if !unicode.IsLetter(r) || !unicode.IsUpper(r) {
		return ""
	}

	// it must end in a letter or digit:
	r, _ = utf8.DecodeLastRuneInString(line)
	if !unicode.IsLetter(r) && !unicode.IsDigit(r) {
		return ""
	}

	// exclude lines with illegal characters. we allow "(),"
	if strings.ContainsAny(line, ";:!?+*/=[]{}_^°&§~%#@<\">\\") {
		return ""
	}

	// allow "'" for possessive "'s" only
	for b := line; ; {
		i := strings.IndexRune(b, '\'')
		if i < 0 {
			break
		}
		if i+1 >= len(b) || b[i+1] != 's' || (i+2 < len(b) && b[i+2] != ' ') {
			return "" // not followed by "s "
		}
		b = b[i+2:]
	}

	// allow "." when followed by non-space
	for b := line; ; {
		i := strings.IndexRune(b, '.')
		if i < 0 {
			break
		}
		if i+1 >= len(b) || b[i+1] == ' ' {
			return "" // not followed by non-space
		}
		b = b[i+1:]
	}

	return line
}

type op int

const (
	opPara op = iota
	opHead
	opPre
)

type block struct {
	op    op
	lines []string
}

func blocks(text string) []block {
	var (
		out  []block
		para []string

		lastWasBlank   = false
		lastWasHeading = false
	)

	close := func() {
		if para != nil {
			out = append(out, block{opPara, para})
			para = nil
		}
	}

	lines := strings.SplitAfter(text, "\n")
	unindent(lines)
	for i := 0; i < len(lines); {
		line := lines[i]
		if isBlank(line) {
			// close paragraph
			close()
			i++
			lastWasBlank = true
			continue
		}
		if indentLen(line) > 0 {
			// close paragraph
			close()

			// count indented or blank lines
			j := i + 1
			for j < len(lines) && (isBlank(lines[j]) || indentLen(lines[j]) > 0) {
				j++
			}
			// but not trailing blank lines
			for j > i && isBlank(lines[j-1]) {
				j--
			}
			pre := lines[i:j]
			i = j

			unindent(pre)

			// put those lines in a pre block
			out = append(out, block{opPre, pre})
			lastWasHeading = false
			continue
		}

		if lastWasBlank && !lastWasHeading && i+2 < len(lines) &&
			isBlank(lines[i+1]) && !isBlank(lines[i+2]) && indentLen(lines[i+2]) == 0 {
			// current line is non-blank, surrounded by blank lines
			// and the next non-blank line is not indented: this
			// might be a heading.
			if head := heading(line); head != "" {
				close()
				out = append(out, block{opHead, []string{head}})
				i += 2
				lastWasHeading = true
				continue
			}
		}

		// open paragraph
		lastWasBlank = false
		lastWasHeading = false
		para = append(para, lines[i])
		i++
	}
	close()

	return out
}
<|endoftext|>


<commit_before>package restic

import (
	"context"
	"errors"
	"sync"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/ui/progress"
	"golang.org/x/sync/errgroup"
)

const streamTreeParallelism = 5

// TreeItem is used to return either an error or the tree for a tree id
type TreeItem struct {
	ID
	Error error
	*Tree
}

type trackedTreeItem struct {
	TreeItem
	rootIdx int
}

type trackedID struct {
	ID
	rootIdx int
}

// loadTreeWorker loads trees from repo and sends them to out.
func loadTreeWorker(ctx context.Context, repo TreeLoader,
	in <-chan trackedID, out chan<- trackedTreeItem) {

	for treeID := range in {
		tree, err := repo.LoadTree(ctx, treeID.ID)
		debug.Log("load tree %v (%v) returned err: %v", tree, treeID, err)
		job := trackedTreeItem{TreeItem: TreeItem{ID: treeID.ID, Error: err, Tree: tree}, rootIdx: treeID.rootIdx}

		select {
		case <-ctx.Done():
			return
		case out <- job:
		}
	}
}

func filterTrees(ctx context.Context, trees IDs, loaderChan chan<- trackedID,
	in <-chan trackedTreeItem, out chan<- TreeItem, skip func(tree ID) bool, p *progress.Counter) {

	var (
		inCh                    = in
		outCh                   chan<- TreeItem
		loadCh                  chan<- trackedID
		job                     TreeItem
		nextTreeID              trackedID
		outstandingLoadTreeJobs = 0
	)
	rootCounter := make([]int, len(trees))
	backlog := make([]trackedID, 0, len(trees))
	for idx, id := range trees {
		backlog = append(backlog, trackedID{ID: id, rootIdx: idx})
		rootCounter[idx] = 1
	}

	for {
		if loadCh == nil && len(backlog) > 0 {
			// process last added ids first, that is traverse the tree in depth-first order
			ln := len(backlog) - 1
			nextTreeID, backlog = backlog[ln], backlog[:ln]

			if skip(nextTreeID.ID) {
				rootCounter[nextTreeID.rootIdx]--
				if p != nil && rootCounter[nextTreeID.rootIdx] == 0 {
					p.Add(1)
				}
				continue
			}

			loadCh = loaderChan
		}

		if loadCh == nil && outCh == nil && outstandingLoadTreeJobs == 0 {
			debug.Log("backlog is empty, all channels nil, exiting")
			return
		}

		select {
		case <-ctx.Done():
			return

		case loadCh <- nextTreeID:
			outstandingLoadTreeJobs++
			loadCh = nil

		case j, ok := <-inCh:
			if !ok {
				debug.Log("input channel closed")
				inCh = nil
				in = nil
				continue
			}

			outstandingLoadTreeJobs--
			rootCounter[j.rootIdx]--

			debug.Log("input job tree %v", j.ID)

			if j.Error != nil {
				debug.Log("received job with error: %v (tree %v, ID %v)", j.Error, j.Tree, j.ID)
			} else if j.Tree == nil {
				debug.Log("received job with nil tree pointer: %v (ID %v)", j.Error, j.ID)
				// send a new job with the new error instead of the old one
				j = trackedTreeItem{TreeItem: TreeItem{ID: j.ID, Error: errors.New("tree is nil and error is nil")}, rootIdx: j.rootIdx}
			} else {
				subtrees := j.Tree.Subtrees()
				debug.Log("subtrees for tree %v: %v", j.ID, subtrees)
				// iterate backwards over subtree to compensate backwards traversal order of nextTreeID selection
				for i := len(subtrees) - 1; i >= 0; i-- {
					id := subtrees[i]
					if id.IsNull() {
						// We do not need to raise this error here, it is
						// checked when the tree is checked. Just make sure
						// that we do not add any null IDs to the backlog.
						debug.Log("tree %v has nil subtree", j.ID)
						continue
					}
					backlog = append(backlog, trackedID{ID: id, rootIdx: j.rootIdx})
					rootCounter[j.rootIdx]++
				}
			}
			if p != nil && rootCounter[j.rootIdx] == 0 {
				p.Add(1)
			}

			job = j.TreeItem
			outCh = out
			inCh = nil

		case outCh <- job:
			debug.Log("tree sent to process: %v", job.ID)
			outCh = nil
			inCh = in
		}
	}
}

// StreamTrees iteratively loads the given trees and their subtrees. The skip method
// is guaranteed to always be called from the same goroutine.
func StreamTrees(ctx context.Context, wg *errgroup.Group, repo TreeLoader, trees IDs, skip func(tree ID) bool, p *progress.Counter) <-chan TreeItem {
	loaderChan := make(chan trackedID)
	loadedTreeChan := make(chan trackedTreeItem)
	treeStream := make(chan TreeItem)

	var loadTreeWg sync.WaitGroup

	for i := 0; i < streamTreeParallelism; i++ {
		loadTreeWg.Add(1)
		wg.Go(func() error {
			defer loadTreeWg.Done()
			loadTreeWorker(ctx, repo, loaderChan, loadedTreeChan)
			return nil
		})
	}

	// close once all loadTreeWorkers have completed
	wg.Go(func() error {
		loadTreeWg.Wait()
		close(loadedTreeChan)
		return nil
	})

	wg.Go(func() error {
		defer close(loaderChan)
		defer close(treeStream)
		filterTrees(ctx, trees, loaderChan, loadedTreeChan, treeStream, skip, p)
		return nil
	})
	return treeStream
}
<commit_msg>restic: add comment about StreamTrees shutdown<commit_after>package restic

import (
	"context"
	"errors"
	"sync"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/ui/progress"
	"golang.org/x/sync/errgroup"
)

const streamTreeParallelism = 5

// TreeItem is used to return either an error or the tree for a tree id
type TreeItem struct {
	ID
	Error error
	*Tree
}

type trackedTreeItem struct {
	TreeItem
	rootIdx int
}

type trackedID struct {
	ID
	rootIdx int
}

// loadTreeWorker loads trees from repo and sends them to out.
func loadTreeWorker(ctx context.Context, repo TreeLoader,
	in <-chan trackedID, out chan<- trackedTreeItem) {

	for treeID := range in {
		tree, err := repo.LoadTree(ctx, treeID.ID)
		debug.Log("load tree %v (%v) returned err: %v", tree, treeID, err)
		job := trackedTreeItem{TreeItem: TreeItem{ID: treeID.ID, Error: err, Tree: tree}, rootIdx: treeID.rootIdx}

		select {
		case <-ctx.Done():
			return
		case out <- job:
		}
	}
}

func filterTrees(ctx context.Context, trees IDs, loaderChan chan<- trackedID,
	in <-chan trackedTreeItem, out chan<- TreeItem, skip func(tree ID) bool, p *progress.Counter) {

	var (
		inCh                    = in
		outCh                   chan<- TreeItem
		loadCh                  chan<- trackedID
		job                     TreeItem
		nextTreeID              trackedID
		outstandingLoadTreeJobs = 0
	)
	rootCounter := make([]int, len(trees))
	backlog := make([]trackedID, 0, len(trees))
	for idx, id := range trees {
		backlog = append(backlog, trackedID{ID: id, rootIdx: idx})
		rootCounter[idx] = 1
	}

	for {
		if loadCh == nil && len(backlog) > 0 {
			// process last added ids first, that is traverse the tree in depth-first order
			ln := len(backlog) - 1
			nextTreeID, backlog = backlog[ln], backlog[:ln]

			if skip(nextTreeID.ID) {
				rootCounter[nextTreeID.rootIdx]--
				if p != nil && rootCounter[nextTreeID.rootIdx] == 0 {
					p.Add(1)
				}
				continue
			}

			loadCh = loaderChan
		}

		if loadCh == nil && outCh == nil && outstandingLoadTreeJobs == 0 {
			debug.Log("backlog is empty, all channels nil, exiting")
			return
		}

		select {
		case <-ctx.Done():
			return

		case loadCh <- nextTreeID:
			outstandingLoadTreeJobs++
			loadCh = nil

		case j, ok := <-inCh:
			if !ok {
				debug.Log("input channel closed")
				inCh = nil
				in = nil
				continue
			}

			outstandingLoadTreeJobs--
			rootCounter[j.rootIdx]--

			debug.Log("input job tree %v", j.ID)

			if j.Error != nil {
				debug.Log("received job with error: %v (tree %v, ID %v)", j.Error, j.Tree, j.ID)
			} else if j.Tree == nil {
				debug.Log("received job with nil tree pointer: %v (ID %v)", j.Error, j.ID)
				// send a new job with the new error instead of the old one
				j = trackedTreeItem{TreeItem: TreeItem{ID: j.ID, Error: errors.New("tree is nil and error is nil")}, rootIdx: j.rootIdx}
			} else {
				subtrees := j.Tree.Subtrees()
				debug.Log("subtrees for tree %v: %v", j.ID, subtrees)
				// iterate backwards over subtree to compensate backwards traversal order of nextTreeID selection
				for i := len(subtrees) - 1; i >= 0; i-- {
					id := subtrees[i]
					if id.IsNull() {
						// We do not need to raise this error here, it is
						// checked when the tree is checked. Just make sure
						// that we do not add any null IDs to the backlog.
						debug.Log("tree %v has nil subtree", j.ID)
						continue
					}
					backlog = append(backlog, trackedID{ID: id, rootIdx: j.rootIdx})
					rootCounter[j.rootIdx]++
				}
			}
			if p != nil && rootCounter[j.rootIdx] == 0 {
				p.Add(1)
			}

			job = j.TreeItem
			outCh = out
			inCh = nil

		case outCh <- job:
			debug.Log("tree sent to process: %v", job.ID)
			outCh = nil
			inCh = in
		}
	}
}

// StreamTrees iteratively loads the given trees and their subtrees. The skip method
// is guaranteed to always be called from the same goroutine. To shutdown the started
// goroutines, either read all items from the channel or cancel the context. Then `Wait()`
// on the errgroup until all goroutines were stopped.
func StreamTrees(ctx context.Context, wg *errgroup.Group, repo TreeLoader, trees IDs, skip func(tree ID) bool, p *progress.Counter) <-chan TreeItem {
	loaderChan := make(chan trackedID)
	loadedTreeChan := make(chan trackedTreeItem)
	treeStream := make(chan TreeItem)

	var loadTreeWg sync.WaitGroup

	for i := 0; i < streamTreeParallelism; i++ {
		loadTreeWg.Add(1)
		wg.Go(func() error {
			defer loadTreeWg.Done()
			loadTreeWorker(ctx, repo, loaderChan, loadedTreeChan)
			return nil
		})
	}

	// close once all loadTreeWorkers have completed
	wg.Go(func() error {
		loadTreeWg.Wait()
		close(loadedTreeChan)
		return nil
	})

	wg.Go(func() error {
		defer close(loaderChan)
		defer close(treeStream)
		filterTrees(ctx, trees, loaderChan, loadedTreeChan, treeStream, skip, p)
		return nil
	})
	return treeStream
}
<|endoftext|>


<commit_before>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package reverse

import (
	"crypto/tls"
	"fmt"
	"math/rand"
	"net"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// startReverse manages the actual reverse connection to the Circonus broker
func (c *Connection) startReverse() error {
	for {
		conn, cerr := c.connect()
		if cerr != nil {
			if cerr.fatal {
				c.logger.Fatal().Err(cerr.err).Msg("unable to establish reverse connection to broker")
			}
			c.logger.Warn().Err(cerr.err).Msg("retrying")
			continue
		}

		if c.shutdown() {
			return nil
		}

		done := make(chan interface{})
		commandReader := c.newCommandReader(done, conn)
		commandProcessor := c.newCommandProcessor(done, commandReader)
		for result := range commandProcessor {
			if c.shutdown() {
				close(done)
				conn.Close()
				return nil
			}
			if result.ignore {
				// c.logger.Debug().Err(result.err).Int("timeouts", c.commTimeouts).Msg("ignored")
				continue
			}
			if result.err != nil {
				if result.reset {
					c.logger.Warn().Err(result.err).Int("timeouts", c.commTimeouts).Msg("resetting connection")
					close(done)
					break
				} else if result.fatal {
					c.logger.Error().Err(result.err).Interface("result", result).Msg("fatal error, exiting")
					conn.Close()
					close(done)
					return result.err
				} else {
					c.logger.Error().Err(result.err).Interface("result", result).Msg("unhandled error state...")
					continue
				}
			}

			// send metrics to broker
			if err := c.sendMetricData(conn, result.channelID, result.metrics); err != nil {
				c.logger.Warn().Err(err).Msg("sending metric data, resetting connection")
				close(done)
				break
			}

			c.resetConnectionAttempts()
		}

		conn.Close()
		if c.shutdown() {
			return nil
		}

	}
}

// connect to broker w/tls and send initial introduction
// NOTE: all reverse connections require tls
func (c *Connection) connect() (*tls.Conn, *connError) {
	c.Lock()
	if c.connAttempts > 0 {
		if c.maxConnRetry != -1 && c.connAttempts >= c.maxConnRetry {
			return nil, &connError{fatal: true, err: errors.Errorf("max broker connection attempts reached (%d of %d)", c.connAttempts, c.maxConnRetry)}
		}

		c.logger.Info().
			Str("delay", c.delay.String()).
			Int("attempt", c.connAttempts).
			Msg("connect retry")

		time.Sleep(c.delay)
		c.delay = c.getNextDelay(c.delay)

		// Under normal circumstances the configuration for reverse is
		// non-volatile. There are, however, some situations where the
		// configuration must be rebuilt. (e.g. ip of broker changed,
		// check changed to use a different broker, broker certificate
		// changes, etc.) The majority of configuration based errors are
		// fatal, no attempt is made to resolve.
		if c.connAttempts%c.configRetryLimit == 0 {
			c.logger.Info().Int("attempts", c.connAttempts).Msg("reconfig triggered")
			c.logger.Debug().Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).Msg("refreshing check")
			if err := c.check.RefreshCheckConfig(); err != nil {
				return nil, &connError{fatal: true, err: errors.Wrap(err, "refreshing check configuration")}
			}
			c.logger.Debug().Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).Msg("setting reverse config")
			rc, err := c.check.GetReverseConfig()
			if err != nil {
				return nil, &connError{fatal: true, err: errors.Wrap(err, "reconfiguring reverse connection")}
			}
			if rc == nil {
				return nil, &connError{fatal: true, err: errors.Wrap(err, "invalid reverse configuration (nil)")}
			}
			c.revConfig = *rc
			c.logger = log.With().Str("pkg", "reverse").Str("cid", viper.GetString(config.KeyCheckBundleID)).Logger()
			c.logger.Info().
				Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).
				Str("rev_host", c.revConfig.ReverseURL.Hostname()).
				Str("rev_port", c.revConfig.ReverseURL.Port()).
				Str("rev_path", c.revConfig.ReverseURL.Path).
				Str("agent", c.agentAddress).
				Msg("reverse configuration")
		}
	}
	c.Unlock()

	revHost := c.revConfig.ReverseURL.Host
	c.logger.Debug().Str("host", revHost).Msg("connecting")
	c.Lock()
	c.connAttempts++
	c.Unlock()
	dialer := &net.Dialer{Timeout: c.dialerTimeout}
	conn, err := tls.DialWithDialer(dialer, "tcp", c.revConfig.BrokerAddr.String(), c.revConfig.TLSConfig)
	if err != nil {
		return nil, &connError{fatal: false, err: errors.Wrapf(err, "connecting to %s", revHost)}
	}
	c.logger.Info().Str("host", revHost).Msg("connected")

	conn.SetDeadline(time.Now().Add(c.commTimeout))
	introReq := "REVERSE " + c.revConfig.ReverseURL.Path
	if c.revConfig.ReverseURL.Fragment != "" {
		introReq += "#" + c.revConfig.ReverseURL.Fragment // reverse secret is placed here when reverse url is parsed
	}
	c.logger.Debug().Msg(fmt.Sprintf("sending intro '%s'", introReq))
	if _, err := fmt.Fprintf(conn, "%s HTTP/1.1\r\n\r\n", introReq); err != nil {
		if err != nil {
			c.logger.Error().Err(err).Msg("sending intro")
			return nil, &connError{fatal: false, err: errors.Wrapf(err, "unable to write intro to %s", revHost)}
		}
	}

	c.Lock()
	// reset timeouts after successful (re)connection
	c.commTimeouts = 0
	c.Unlock()

	return conn, nil
}

// getNextDelay for failed connection attempts
func (c *Connection) getNextDelay(currDelay time.Duration) time.Duration {
	if currDelay == c.maxDelay {
		return currDelay
	}

	delay := currDelay

	if delay < c.maxDelay {
		drift := rand.Intn(c.maxDelayStep-c.minDelayStep) + c.minDelayStep
		delay += time.Duration(drift) * time.Second
	}

	if delay > c.maxDelay {
		delay = c.maxDelay
	}

	return delay
}

// resetConnectionAttempts on successful send/receive
func (c *Connection) resetConnectionAttempts() {
	c.Lock()
	if c.connAttempts > 0 {
		c.delay = 1 * time.Second
		c.connAttempts = 0
	}
	c.Unlock()
}

// Error returns string representation of a connError
func (e *connError) Error() string {
	return e.err.Error()
}
<commit_msg>upd: retry reverse check refresh failures for unstable infrastructures<commit_after>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package reverse

import (
	"crypto/tls"
	"fmt"
	"math/rand"
	"net"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// startReverse manages the actual reverse connection to the Circonus broker
func (c *Connection) startReverse() error {
	for {
		conn, cerr := c.connect()
		if cerr != nil {
			if cerr.fatal {
				c.logger.Fatal().Err(cerr.err).Msg("unable to establish reverse connection to broker")
			}
			c.logger.Warn().Err(cerr.err).Msg("retrying")
			continue
		}

		if c.shutdown() {
			return nil
		}

		done := make(chan interface{})
		commandReader := c.newCommandReader(done, conn)
		commandProcessor := c.newCommandProcessor(done, commandReader)
		for result := range commandProcessor {
			if c.shutdown() {
				close(done)
				conn.Close()
				return nil
			}
			if result.ignore {
				// c.logger.Debug().Err(result.err).Int("timeouts", c.commTimeouts).Msg("ignored")
				continue
			}
			if result.err != nil {
				if result.reset {
					c.logger.Warn().Err(result.err).Int("timeouts", c.commTimeouts).Msg("resetting connection")
					close(done)
					break
				} else if result.fatal {
					c.logger.Error().Err(result.err).Interface("result", result).Msg("fatal error, exiting")
					conn.Close()
					close(done)
					return result.err
				} else {
					c.logger.Error().Err(result.err).Interface("result", result).Msg("unhandled error state...")
					continue
				}
			}

			// send metrics to broker
			if err := c.sendMetricData(conn, result.channelID, result.metrics); err != nil {
				c.logger.Warn().Err(err).Msg("sending metric data, resetting connection")
				close(done)
				break
			}

			c.resetConnectionAttempts()
		}

		conn.Close()
		if c.shutdown() {
			return nil
		}

	}
}

// connect to broker w/tls and send initial introduction
// NOTE: all reverse connections require tls
func (c *Connection) connect() (*tls.Conn, *connError) {
	c.Lock()
	if c.connAttempts > 0 {
		if c.maxConnRetry != -1 && c.connAttempts >= c.maxConnRetry {
			return nil, &connError{fatal: true, err: errors.Errorf("max broker connection attempts reached (%d of %d)", c.connAttempts, c.maxConnRetry)}
		}

		c.logger.Info().
			Str("delay", c.delay.String()).
			Int("attempt", c.connAttempts).
			Msg("connect retry")

		time.Sleep(c.delay)
		c.delay = c.getNextDelay(c.delay)

		// Under normal circumstances the configuration for reverse is
		// non-volatile. There are, however, some situations where the
		// configuration must be rebuilt. (e.g. ip of broker changed,
		// check changed to use a different broker, broker certificate
		// changes, etc.) The majority of configuration based errors are
		// fatal, no attempt is made to resolve.
		if c.connAttempts%c.configRetryLimit == 0 {
			c.logger.Info().Int("attempts", c.connAttempts).Msg("reconfig triggered")
			if err := c.check.RefreshCheckConfig(); err != nil {
				return nil, &connError{fatal: false, err: errors.Wrap(err, "refreshing check configuration")}
			}
			c.logger.Debug().Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).Msg("setting reverse config")
			rc, err := c.check.GetReverseConfig()
			if err != nil {
				return nil, &connError{fatal: true, err: errors.Wrap(err, "reconfiguring reverse connection")}
			}
			if rc == nil {
				return nil, &connError{fatal: true, err: errors.Wrap(err, "invalid reverse configuration (nil)")}
			}
			c.revConfig = *rc
			c.logger = log.With().Str("pkg", "reverse").Str("cid", viper.GetString(config.KeyCheckBundleID)).Logger()
			c.logger.Info().
				Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).
				Str("rev_host", c.revConfig.ReverseURL.Hostname()).
				Str("rev_port", c.revConfig.ReverseURL.Port()).
				Str("rev_path", c.revConfig.ReverseURL.Path).
				Str("agent", c.agentAddress).
				Msg("reverse configuration")
		}
	}
	c.Unlock()

	revHost := c.revConfig.ReverseURL.Host
	c.logger.Debug().Str("host", revHost).Msg("connecting")
	c.Lock()
	c.connAttempts++
	c.Unlock()
	dialer := &net.Dialer{Timeout: c.dialerTimeout}
	conn, err := tls.DialWithDialer(dialer, "tcp", c.revConfig.BrokerAddr.String(), c.revConfig.TLSConfig)
	if err != nil {
		return nil, &connError{fatal: false, err: errors.Wrapf(err, "connecting to %s", revHost)}
	}
	c.logger.Info().Str("host", revHost).Msg("connected")

	conn.SetDeadline(time.Now().Add(c.commTimeout))
	introReq := "REVERSE " + c.revConfig.ReverseURL.Path
	if c.revConfig.ReverseURL.Fragment != "" {
		introReq += "#" + c.revConfig.ReverseURL.Fragment // reverse secret is placed here when reverse url is parsed
	}
	c.logger.Debug().Msg(fmt.Sprintf("sending intro '%s'", introReq))
	if _, err := fmt.Fprintf(conn, "%s HTTP/1.1\r\n\r\n", introReq); err != nil {
		if err != nil {
			c.logger.Error().Err(err).Msg("sending intro")
			return nil, &connError{fatal: false, err: errors.Wrapf(err, "unable to write intro to %s", revHost)}
		}
	}

	c.Lock()
	// reset timeouts after successful (re)connection
	c.commTimeouts = 0
	c.Unlock()

	return conn, nil
}

// getNextDelay for failed connection attempts
func (c *Connection) getNextDelay(currDelay time.Duration) time.Duration {
	if currDelay == c.maxDelay {
		return currDelay
	}

	delay := currDelay

	if delay < c.maxDelay {
		drift := rand.Intn(c.maxDelayStep-c.minDelayStep) + c.minDelayStep
		delay += time.Duration(drift) * time.Second
	}

	if delay > c.maxDelay {
		delay = c.maxDelay
	}

	return delay
}

// resetConnectionAttempts on successful send/receive
func (c *Connection) resetConnectionAttempts() {
	c.Lock()
	if c.connAttempts > 0 {
		c.delay = 1 * time.Second
		c.connAttempts = 0
	}
	c.Unlock()
}

// Error returns string representation of a connError
func (e *connError) Error() string {
	return e.err.Error()
}
<|endoftext|>


<commit_before>// Generated automatically by make.
package build

const goosList = "darwin freebsd linux netbsd openbsd plan9 windows "
const goarchList = "386 amd64 arm "
<commit_msg>go/build: update syslist.go package comment<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package build

const goosList = "darwin freebsd linux netbsd openbsd plan9 windows "
const goarchList = "386 amd64 arm "
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Utility functions.

package ioutil

import (
	"bytes"
	"io"
	"os"
	"sort"
)

// ReadAll reads from r until an error or EOF and returns the data it read.
func ReadAll(r io.Reader) ([]byte, os.Error) {
	var buf bytes.Buffer
	_, err := io.Copy(&buf, r)
	return buf.Bytes(), err
}

// ReadFile reads the file named by filename and returns the contents.
func ReadFile(filename string) ([]byte, os.Error) {
	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	// It's a good but not certain bet that Stat will tell us exactly how much to
	// read, so let's try it but be prepared for the answer to be wrong.
	dir, err := f.Stat()
	var n uint64
	if err != nil && dir.Size < 2e9 { // Don't preallocate a huge buffer, just in case.
		n = dir.Size
	}
	// Add a little extra in case Size is zero, and to avoid another allocation after
	// Read has filled the buffer.
	n += bytes.MinRead
	// Pre-allocate the correct size of buffer, then set its size to zero.  The
	// Buffer will read into the allocated space cheaply.  If the size was wrong,
	// we'll either waste some space off the end or reallocate as needed, but
	// in the overwhelmingly common case we'll get it just right.
	buf := bytes.NewBuffer(make([]byte, 0, n))
	_, err = buf.ReadFrom(f)
	return buf.Bytes(), err
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm;
// otherwise WriteFile truncates it before writing.
func WriteFile(filename string, data []byte, perm int) os.Error {
	f, err := os.Open(filename, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, perm)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	f.Close()
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	return err
}

// A dirList implements sort.Interface.
type dirList []*os.Dir

func (d dirList) Len() int           { return len(d) }
func (d dirList) Less(i, j int) bool { return d[i].Name < d[j].Name }
func (d dirList) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }

// ReadDir reads the directory named by dirname and returns
// a list of sorted directory entries.
func ReadDir(dirname string) ([]*os.Dir, os.Error) {
	f, err := os.Open(dirname, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	list, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	dirs := make(dirList, len(list))
	for i := range list {
		dirs[i] = &list[i]
	}
	sort.Sort(dirs)
	return dirs, nil
}
<commit_msg>io/ioutil: fix bug in ReadFile when Open succeeds but Stat fails<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Utility functions.

package ioutil

import (
	"bytes"
	"io"
	"os"
	"sort"
)

// ReadAll reads from r until an error or EOF and returns the data it read.
func ReadAll(r io.Reader) ([]byte, os.Error) {
	var buf bytes.Buffer
	_, err := io.Copy(&buf, r)
	return buf.Bytes(), err
}

// ReadFile reads the file named by filename and returns the contents.
func ReadFile(filename string) ([]byte, os.Error) {
	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	// It's a good but not certain bet that Stat will tell us exactly how much to
	// read, so let's try it but be prepared for the answer to be wrong.
	dir, err := f.Stat()
	var n uint64
	if err == nil && dir.Size < 2e9 { // Don't preallocate a huge buffer, just in case.
		n = dir.Size
	}
	// Add a little extra in case Size is zero, and to avoid another allocation after
	// Read has filled the buffer.
	n += bytes.MinRead
	// Pre-allocate the correct size of buffer, then set its size to zero.  The
	// Buffer will read into the allocated space cheaply.  If the size was wrong,
	// we'll either waste some space off the end or reallocate as needed, but
	// in the overwhelmingly common case we'll get it just right.
	buf := bytes.NewBuffer(make([]byte, 0, n))
	_, err = buf.ReadFrom(f)
	return buf.Bytes(), err
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm;
// otherwise WriteFile truncates it before writing.
func WriteFile(filename string, data []byte, perm int) os.Error {
	f, err := os.Open(filename, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, perm)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	f.Close()
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	return err
}

// A dirList implements sort.Interface.
type dirList []*os.Dir

func (d dirList) Len() int           { return len(d) }
func (d dirList) Less(i, j int) bool { return d[i].Name < d[j].Name }
func (d dirList) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }

// ReadDir reads the directory named by dirname and returns
// a list of sorted directory entries.
func ReadDir(dirname string) ([]*os.Dir, os.Error) {
	f, err := os.Open(dirname, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	list, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	dirs := make(dirList, len(list))
	for i := range list {
		dirs[i] = &list[i]
	}
	sort.Sort(dirs)
	return dirs, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ioutil implements some I/O utility functions.
package ioutil

import (
	"bytes"
	"io"
	"os"
	"sort"
)

// readAll reads from r until an error or EOF and returns the data it read
// from the internal buffer allocated with a specified capacity.
func readAll(r io.Reader, capacity int64) ([]byte, os.Error) {
	buf := bytes.NewBuffer(make([]byte, 0, capacity))
	_, err := buf.ReadFrom(r)
	return buf.Bytes(), err
}

// ReadAll reads from r until an error or EOF and returns the data it read.
func ReadAll(r io.Reader) ([]byte, os.Error) {
	return readAll(r, bytes.MinRead)
}

// ReadFile reads the file named by filename and returns the contents.
func ReadFile(filename string) ([]byte, os.Error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	// It's a good but not certain bet that FileInfo will tell us exactly how much to
	// read, so let's try it but be prepared for the answer to be wrong.
	fi, err := f.Stat()
	var n int64
	if err == nil && fi.Size < 2e9 { // Don't preallocate a huge buffer, just in case.
		n = fi.Size
	}
	// As initial capacity for readAll, use n + a little extra in case Size is zero,
	// and to avoid another allocation after Read has filled the buffer.  The readAll
	// call will read into its allocated internal buffer cheaply.  If the size was
	// wrong, we'll either waste some space off the end or reallocate as needed, but
	// in the overwhelmingly common case we'll get it just right.
	return readAll(f, n+bytes.MinRead)
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm;
// otherwise WriteFile truncates it before writing.
func WriteFile(filename string, data []byte, perm uint32) os.Error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	f.Close()
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	return err
}

// A dirList implements sort.Interface.
type fileInfoList []*os.FileInfo

func (f fileInfoList) Len() int           { return len(f) }
func (f fileInfoList) Less(i, j int) bool { return f[i].Name < f[j].Name }
func (f fileInfoList) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// ReadDir reads the directory named by dirname and returns
// a list of sorted directory entries.
func ReadDir(dirname string) ([]*os.FileInfo, os.Error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	list, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	fi := make(fileInfoList, len(list))
	for i := range list {
		fi[i] = &list[i]
	}
	sort.Sort(fi)
	return fi, nil
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

// NopCloser returns a ReadCloser with a no-op Close method wrapping
// the provided Reader r.
func NopCloser(r io.Reader) io.ReadCloser {
	return nopCloser{r}
}

type devNull int

func (devNull) Write(p []byte) (int, os.Error) {
	return len(p), nil
}

// Discard is an io.Writer on which all Write calls succeed
// without doing anything.
var Discard io.Writer = devNull(0)
<commit_msg>io/ioutil: fix typo in comment<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ioutil implements some I/O utility functions.
package ioutil

import (
	"bytes"
	"io"
	"os"
	"sort"
)

// readAll reads from r until an error or EOF and returns the data it read
// from the internal buffer allocated with a specified capacity.
func readAll(r io.Reader, capacity int64) ([]byte, os.Error) {
	buf := bytes.NewBuffer(make([]byte, 0, capacity))
	_, err := buf.ReadFrom(r)
	return buf.Bytes(), err
}

// ReadAll reads from r until an error or EOF and returns the data it read.
func ReadAll(r io.Reader) ([]byte, os.Error) {
	return readAll(r, bytes.MinRead)
}

// ReadFile reads the file named by filename and returns the contents.
func ReadFile(filename string) ([]byte, os.Error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	// It's a good but not certain bet that FileInfo will tell us exactly how much to
	// read, so let's try it but be prepared for the answer to be wrong.
	fi, err := f.Stat()
	var n int64
	if err == nil && fi.Size < 2e9 { // Don't preallocate a huge buffer, just in case.
		n = fi.Size
	}
	// As initial capacity for readAll, use n + a little extra in case Size is zero,
	// and to avoid another allocation after Read has filled the buffer.  The readAll
	// call will read into its allocated internal buffer cheaply.  If the size was
	// wrong, we'll either waste some space off the end or reallocate as needed, but
	// in the overwhelmingly common case we'll get it just right.
	return readAll(f, n+bytes.MinRead)
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm;
// otherwise WriteFile truncates it before writing.
func WriteFile(filename string, data []byte, perm uint32) os.Error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	f.Close()
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	return err
}

// A fileInfoList implements sort.Interface.
type fileInfoList []*os.FileInfo

func (f fileInfoList) Len() int           { return len(f) }
func (f fileInfoList) Less(i, j int) bool { return f[i].Name < f[j].Name }
func (f fileInfoList) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// ReadDir reads the directory named by dirname and returns
// a list of sorted directory entries.
func ReadDir(dirname string) ([]*os.FileInfo, os.Error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	list, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	fi := make(fileInfoList, len(list))
	for i := range list {
		fi[i] = &list[i]
	}
	sort.Sort(fi)
	return fi, nil
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

// NopCloser returns a ReadCloser with a no-op Close method wrapping
// the provided Reader r.
func NopCloser(r io.Reader) io.ReadCloser {
	return nopCloser{r}
}

type devNull int

func (devNull) Write(p []byte) (int, os.Error) {
	return len(p), nil
}

// Discard is an io.Writer on which all Write calls succeed
// without doing anything.
var Discard io.Writer = devNull(0)
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Sockets for Windows

package net

import (
	"syscall"
)

func setKernelSpecificSockopt(s syscall.Handle, f int) {
	// Allow broadcast.
	syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)

	if f == syscall.AF_INET6 {
		// using ip, tcp, udp, etc.
		// allow both protocols even if the OS default is otherwise.
		syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0)
	}
}
<commit_msg>net: document why we do not use SO_REUSEADDR on windows<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Sockets for Windows

package net

import (
	"syscall"
)

func setKernelSpecificSockopt(s syscall.Handle, f int) {
	// Windows will reuse recently-used addresses by default.
	// SO_REUSEADDR should not be used here, as it allows
	// a socket to forcibly bind to a port in use by another socket.
	// This could lead to a non-deterministic behavior, where
	// connection requests over the port cannot be guaranteed
	// to be handled by the correct socket.

	// Allow broadcast.
	syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)

	if f == syscall.AF_INET6 {
		// using ip, tcp, udp, etc.
		// allow both protocols even if the OS default is otherwise.
		syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0)
	}
}
<|endoftext|>


<commit_before>/*
Package re combines regular expression matching with fmt.Scan like
extraction of sub-matches into caller-supplied objects. Pointers to
variables can be passed as extra arguments to re.Scan.  These
variables are filled in with regular expression sub-matches.  The
sub-matches are parsed appropriately based on the type of the
variable.  E.g., if a *int is passed in, the sub-match is parsed as a
number (and overflow is detected).

For example, the host and port portions of a URL can be extracted as
follows:

	var host string
	var port int
	reg := regexp.MustCompile(`^https?://([^/:]+):(\d+)/`)
	if err := re.Scan(reg, url, &host, &port); err == nil {
		Process(host, port)
	}

A "func([]byte) error" can also be passed in as an extra argument to provide
custom parsing.
*/
package re

import (
	"fmt"
	"reflect"
	"regexp"
	"strconv"
)

// Scan returns nil if regular expression re matches somewhere in
// input, and for every non-nil entry in output, the corresponding
// regular expression sub-match is succesfully parsed and stored into
// *output[i].
//
// The following can be passed as output arguments to Scan:
//
// nil: The corresponding sub-match is discarded without being saved.
//
// Pointer to string or []byte: The corresponding sub-match is
// stored in the pointed-to object.  When storing into a []byte, no
// copying is done, and the stored slice is an alias of the input.
//
// Pointer to some built-in numeric types (int, int8, int16, int32,
// int64, uint, uintptr, uint8, uint16, uint32, uint64, float32,
// float64): The corresponding sub-match will be parsed as a literal
// of the numeric type and the result stored into *output[i].  Scan
// will return an error if the sub-match cannot be parsed
// successfully, or the parse result is out of range for the type.
//
// Pointer to a rune or a byte: rune is an alias of uint32 and byte is
// an alias of uint8, so the preceding rule applies; i.e., Scan treats
// the input as a string of digits to be parsed into the rune or
// byte. Therefore Scan cannot be used to directly extract a single
// rune or byte from the input. For that, parse into a string or
// []byte and use the first element, or pass in a custom parsing
// function (see below).
//
// func([]byte) error: The function is passed the corresponding
// sub-match.  If the result is a non-nil error, the Scan call fails
// with that error. Pass in such a function to provide custom parsing:
// e.g., treating a number as decimal even if it starts with "0"
// (normally Scan would treat such as a number as octal); or parsing
// an otherwise unsupported type like time.Duration.
//
// An error is returned if output[i] does not have one of the preceding
// types.  Caveat: the set of supported types might be extended in the
// future.
//
// Extra sub-matches (ones with no corresponding output) are discarded silently.
func Scan(re *regexp.Regexp, input []byte, output ...interface{}) error {
	matches := re.FindSubmatchIndex(input)
	if matches == nil {
		return fmt.Errorf(`re.Scan: could not find "%s" in "%s"`,
			re, input)
	}
	if len(matches) < 2+2*len(output) {
		return fmt.Errorf(`re.Scan: only got %d matches from "%s"; need at least %d`,
			len(matches)/2-1, re, len(output))
	}
	for i, r := range output {
		start, limit := matches[2+2*i], matches[2+2*i+1]
		if start < 0 || limit < 0 {
			// Sub-expression is missing; treat as empty.
			start = 0
			limit = 0
		}
		if err := assign(r, input[start:limit]); err != nil {
			return err
		}
	}
	return nil
}

func assign(r interface{}, b []byte) error {
	switch v := r.(type) {
	case nil:
		// Discard the match.
	case func([]byte) error:
		if err := v(b); err != nil {
			return err
		}
	case *string:
		*v = string(b)
	case *[]byte:
		*v = b
	case *int:
		if i, err := strconv.ParseInt(string(b), 0, 64); err != nil {
			return err
		} else {
			if int64(int(i)) != i {
				return parseError("out of range for int", b)
			}
			*v = int(i)
		}
	case *int8:
		if i, err := strconv.ParseInt(string(b), 0, 8); err != nil {
			return err
		} else {
			*v = int8(i)
		}
	case *int16:
		if i, err := strconv.ParseInt(string(b), 0, 16); err != nil {
			return err
		} else {
			*v = int16(i)
		}
	case *int32:
		if i, err := strconv.ParseInt(string(b), 0, 32); err != nil {
			return err
		} else {
			*v = int32(i)
		}
	case *int64:
		if i, err := strconv.ParseInt(string(b), 0, 64); err != nil {
			return err
		} else {
			*v = i
		}
	case *uint:
		if u, err := strconv.ParseUint(string(b), 0, 64); err != nil {
			return err
		} else {
			if uint64(uint(u)) != u {
				return parseError("out of range for uint", b)
			}
			*v = uint(u)
		}
	case *uintptr:
		if u, err := strconv.ParseUint(string(b), 0, 64); err != nil {
			return err
		} else {
			if uint64(uintptr(u)) != u {
				return parseError("out of range for uintptr", b)
			}
			*v = uintptr(u)
		}
	case *uint8:
		if u, err := strconv.ParseUint(string(b), 0, 8); err != nil {
			return err
		} else {
			*v = uint8(u)
		}
	case *uint16:
		if u, err := strconv.ParseUint(string(b), 0, 16); err != nil {
			return err
		} else {
			*v = uint16(u)
		}
	case *uint32:
		if u, err := strconv.ParseUint(string(b), 0, 32); err != nil {
			return err
		} else {
			*v = uint32(u)
		}
	case *uint64:
		if u, err := strconv.ParseUint(string(b), 0, 64); err != nil {
			return err
		} else {
			*v = u
		}
	case *float32:
		if f, err := strconv.ParseFloat(string(b), 32); err != nil {
			return err
		} else {
			*v = float32(f)
		}
	case *float64:
		if f, err := strconv.ParseFloat(string(b), 64); err != nil {
			return err
		} else {
			*v = f
		}
	default:
		t := reflect.ValueOf(r).Type()
		return parseError(fmt.Sprintf("unsupported type %s", t), b)
	}
	return nil
}

func parseError(explanation string, b []byte) error {
	return fmt.Errorf(`re.Scan: parsing "%s": %s`, b, explanation)
}
<commit_msg>fixed lint errors<commit_after>/*
Package re combines regular expression matching with fmt.Scan like
extraction of sub-matches into caller-supplied objects. Pointers to
variables can be passed as extra arguments to re.Scan.  These
variables are filled in with regular expression sub-matches.  The
sub-matches are parsed appropriately based on the type of the
variable.  E.g., if a *int is passed in, the sub-match is parsed as a
number (and overflow is detected).

For example, the host and port portions of a URL can be extracted as
follows:

	var host string
	var port int
	reg := regexp.MustCompile(`^https?://([^/:]+):(\d+)/`)
	if err := re.Scan(reg, url, &host, &port); err == nil {
		Process(host, port)
	}

A "func([]byte) error" can also be passed in as an extra argument to provide
custom parsing.
*/
package re

import (
	"fmt"
	"reflect"
	"regexp"
	"strconv"
)

// Scan returns nil if regular expression re matches somewhere in
// input, and for every non-nil entry in output, the corresponding
// regular expression sub-match is succesfully parsed and stored into
// *output[i].
//
// The following can be passed as output arguments to Scan:
//
// nil: The corresponding sub-match is discarded without being saved.
//
// Pointer to string or []byte: The corresponding sub-match is
// stored in the pointed-to object.  When storing into a []byte, no
// copying is done, and the stored slice is an alias of the input.
//
// Pointer to some built-in numeric types (int, int8, int16, int32,
// int64, uint, uintptr, uint8, uint16, uint32, uint64, float32,
// float64): The corresponding sub-match will be parsed as a literal
// of the numeric type and the result stored into *output[i].  Scan
// will return an error if the sub-match cannot be parsed
// successfully, or the parse result is out of range for the type.
//
// Pointer to a rune or a byte: rune is an alias of uint32 and byte is
// an alias of uint8, so the preceding rule applies; i.e., Scan treats
// the input as a string of digits to be parsed into the rune or
// byte. Therefore Scan cannot be used to directly extract a single
// rune or byte from the input. For that, parse into a string or
// []byte and use the first element, or pass in a custom parsing
// function (see below).
//
// func([]byte) error: The function is passed the corresponding
// sub-match.  If the result is a non-nil error, the Scan call fails
// with that error. Pass in such a function to provide custom parsing:
// e.g., treating a number as decimal even if it starts with "0"
// (normally Scan would treat such as a number as octal); or parsing
// an otherwise unsupported type like time.Duration.
//
// An error is returned if output[i] does not have one of the preceding
// types.  Caveat: the set of supported types might be extended in the
// future.
//
// Extra sub-matches (ones with no corresponding output) are discarded silently.
func Scan(re *regexp.Regexp, input []byte, output ...interface{}) error {
	matches := re.FindSubmatchIndex(input)
	if matches == nil {
		return fmt.Errorf(`re.Scan: could not find "%s" in "%s"`,
			re, input)
	}
	if len(matches) < 2+2*len(output) {
		return fmt.Errorf(`re.Scan: only got %d matches from "%s"; need at least %d`,
			len(matches)/2-1, re, len(output))
	}
	for i, r := range output {
		start, limit := matches[2+2*i], matches[2+2*i+1]
		if start < 0 || limit < 0 {
			// Sub-expression is missing; treat as empty.
			start = 0
			limit = 0
		}
		if err := assign(r, input[start:limit]); err != nil {
			return err
		}
	}
	return nil
}

func assign(r interface{}, b []byte) error {
	switch v := r.(type) {
	case nil:
		// Discard the match.
	case func([]byte) error:
		if err := v(b); err != nil {
			return err
		}
	case *string:
		*v = string(b)
	case *[]byte:
		*v = b
	case *int:
		i, err := strconv.ParseInt(string(b), 0, 64)
		if err != nil {
			return err
		}
		if int64(int(i)) != i {
			return parseError("out of range for int", b)
		}
		*v = int(i)
	case *int8:
		i, err := strconv.ParseInt(string(b), 0, 8)
		if err != nil {
			return err
		}
		*v = int8(i)
	case *int16:
		i, err := strconv.ParseInt(string(b), 0, 16)
		if err != nil {
			return err
		}
		*v = int16(i)
	case *int32:
		i, err := strconv.ParseInt(string(b), 0, 32)
		if err != nil {
			return err
		}
		*v = int32(i)
	case *int64:
		i, err := strconv.ParseInt(string(b), 0, 64)
		if err != nil {
			return err
		}
		*v = i
	case *uint:
		u, err := strconv.ParseUint(string(b), 0, 64)
		if err != nil {
			return err
		}
		if uint64(uint(u)) != u {
			return parseError("out of range for uint", b)
		}
		*v = uint(u)
	case *uintptr:
		u, err := strconv.ParseUint(string(b), 0, 64)
		if err != nil {
			return err
		}
		if uint64(uintptr(u)) != u {
			return parseError("out of range for uintptr", b)
		}
		*v = uintptr(u)
	case *uint8:
		u, err := strconv.ParseUint(string(b), 0, 8)
		if err != nil {
			return err
		}
		*v = uint8(u)
	case *uint16:
		u, err := strconv.ParseUint(string(b), 0, 16)
		if err != nil {
			return err
		}
		*v = uint16(u)
	case *uint32:
		u, err := strconv.ParseUint(string(b), 0, 32)
		if err != nil {
			return err
		}
		*v = uint32(u)
	case *uint64:
		u, err := strconv.ParseUint(string(b), 0, 64)
		if err != nil {
			return err
		}
		*v = u
	case *float32:
		f, err := strconv.ParseFloat(string(b), 32)
		if err != nil {
			return err
		}
		*v = float32(f)
	case *float64:
		f, err := strconv.ParseFloat(string(b), 64)
		if err != nil {
			return err
		}
		*v = f
	default:
		t := reflect.ValueOf(r).Type()
		return parseError(fmt.Sprintf("unsupported type %s", t), b)
	}
	return nil
}

func parseError(explanation string, b []byte) error {
	return fmt.Errorf(`re.Scan: parsing "%s": %s`, b, explanation)
}
<|endoftext|>


<commit_before>package sh

import (
	"bytes"
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/codegangsta/inject"
)

type Return struct {
	Stdout string
	Stderr string
}

func (r *Return) String() string {
	return r.Stdout
}

func (r *Return) Trim() string {
	return strings.TrimSpace(r.Stdout)
}

func Capture(a ...interface{}) (ret *Return, err error) {
	s := NewSession()
	return s.Capture(a...)
}

type Dir string

type Session struct {
	inj     inject.Injector
	alias   map[string][]string
	cmds    []*exec.Cmd
	started bool
	Env     map[string]string
	Stdout  io.Writer
	Stderr  io.Writer
}

func NewSession(a ...interface{}) *Session {
	env := map[string]string{
		"PATH": "/bin:/usr/bin:/usr/local/bin",
	}
	s := &Session{
		inj:    inject.New(),
		alias:  make(map[string][]string),
		Stdout: os.Stdout,
		Stderr: os.Stderr,
		Env:    env,
	}
	dir := Dir("")
	args := []string{}
	s.inj.Map(env).Map(dir).Map(args)
	for _, v := range a {
		s.inj.Map(v)
	}
	return s
}

func (s *Session) Alias(alias, cmd string, args ...string) {
	v := []string{cmd}
	v = append(v, args...)
	s.alias[alias] = v
}

func (s *Session) Command(a ...interface{}) *Session {
	for _, v := range a {
		s.inj.Map(v)
	}
	s.inj.Invoke(s.appendCmd)
	return s
}

func (s *Session) Call(a ...interface{}) error {
	return s.Command(a...).Run()
}

/*
func (s *Session) Exec(cmd string, args ...string) error {
	return s.Call(cmd, args)
}
*/

func (s *Session) Capture(a ...interface{}) (ret *Return, err error) {
	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	oldout, olderr := s.Stdout, s.Stderr
	s.Stdout, s.Stderr = stdout, stderr
	err = s.Call(a...)
	s.Stdout, s.Stderr = oldout, olderr

	ret = new(Return)
	ret.Stdout = string(stdout.Bytes())
	ret.Stderr = string(stderr.Bytes())
	return
}

func (s *Session) Set(a ...interface{}) *Session {
	for _, v := range a {
		s.inj.Map(v)
	}
	return s
}

func (s *Session) appendCmd(cmd string, args []string, cwd Dir) {
	if s.started {
		s.started = false
		s.cmds = make([]*exec.Cmd, 0)
	}
	envs := make([]string, 0, len(s.Env))
	for k, v := range s.Env {
		envs = append(envs, k+"="+v)
	}
	v, ok := s.alias[cmd]
	if ok {
		cmd = v[0]
		args = append(v[1:], args...)
	}
	c := exec.Command(cmd, args...)
	c.Env = envs
	c.Dir = string(cwd)
	s.cmds = append(s.cmds, c)
}
<commit_msg>fix for windows<commit_after>package sh

import (
	"bytes"
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/codegangsta/inject"
)

type Return struct {
	Stdout string
	Stderr string
}

func (r *Return) String() string {
	return r.Stdout
}

func (r *Return) Trim() string {
	return strings.TrimSpace(r.Stdout)
}

func Capture(a ...interface{}) (ret *Return, err error) {
	s := NewSession()
	return s.Capture(a...)
}

type Dir string

type Session struct {
	inj     inject.Injector
	alias   map[string][]string
	cmds    []*exec.Cmd
	started bool
	Env     map[string]string
	Stdout  io.Writer
	Stderr  io.Writer
}

func NewSession(a ...interface{}) *Session {
	env := make(map[string]string)
	for _, key := range []string{"PATH"} {
		env[key] = os.Getenv(key)
	}
	s := &Session{
		inj:    inject.New(),
		alias:  make(map[string][]string),
		Stdout: os.Stdout,
		Stderr: os.Stderr,
		Env:    env,
	}
	dir := Dir("")
	args := []string{}
	s.inj.Map(env).Map(dir).Map(args)
	for _, v := range a {
		s.inj.Map(v)
	}
	return s
}

func (s *Session) Alias(alias, cmd string, args ...string) {
	v := []string{cmd}
	v = append(v, args...)
	s.alias[alias] = v
}

func (s *Session) Command(a ...interface{}) *Session {
	for _, v := range a {
		s.inj.Map(v)
	}
	s.inj.Invoke(s.appendCmd)
	return s
}

func (s *Session) Call(a ...interface{}) error {
	return s.Command(a...).Run()
}

/*
func (s *Session) Exec(cmd string, args ...string) error {
	return s.Call(cmd, args)
}
*/

func (s *Session) Capture(a ...interface{}) (ret *Return, err error) {
	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	oldout, olderr := s.Stdout, s.Stderr
	s.Stdout, s.Stderr = stdout, stderr
	err = s.Call(a...)
	s.Stdout, s.Stderr = oldout, olderr

	ret = new(Return)
	ret.Stdout = string(stdout.Bytes())
	ret.Stderr = string(stderr.Bytes())
	return
}

func (s *Session) Set(a ...interface{}) *Session {
	for _, v := range a {
		s.inj.Map(v)
	}
	return s
}

func (s *Session) appendCmd(cmd string, args []string, cwd Dir) {
	if s.started {
		s.started = false
		s.cmds = make([]*exec.Cmd, 0)
	}
	envs := make([]string, 0, len(s.Env))
	for k, v := range s.Env {
		envs = append(envs, k+"="+v)
	}
	v, ok := s.alias[cmd]
	if ok {
		cmd = v[0]
		args = append(v[1:], args...)
	}
	c := exec.Command(cmd, args...)
	c.Env = envs
	c.Dir = string(cwd)
	s.cmds = append(s.cmds, c)
}
<|endoftext|>


<commit_before>// Copyright 2015 Cocoon Alarm Ltd.
//
// See LICENSE file for terms and conditions.

package libflac

import (
	"io"
	"os"
	"testing"

	"github.com/jbert/testify/assert"
)

func TestDecode(t *testing.T) {
	a := assert.New(t)

	d, err := NewDecoder("data/nonexistent.flac")

	a.Equal(d, (*Decoder)(nil), "decoder is nil")

	d, err = NewDecoder("data/sine24-00.flac")

	a.Equal(err, nil, "err is nil")
	a.Equal(d.Channels, 1, "channels is 1")
	a.Equal(d.Depth, 24, "depth is 24")
	a.Equal(d.Rate, 48000, "depth is 48000")

	samples := 0

	f, err := d.ReadFrame()

	a.Equal(err, nil, "err is nil")
	a.Equal(f.Channels, 1, "channels is 1")
	a.Equal(f.Depth, 24, "depth is 24")
	a.Equal(f.Rate, 48000, "depth is 48000")

	samples = samples + len(f.Buffer)

	for {
		f, err := d.ReadFrame()

		if err == nil || err == io.EOF {
			if f != nil {
				samples = samples + len(f.Buffer)
			}
		} else {
			a.Equal(err, nil, "error reported")
			break
		}

		if err == io.EOF {
			break
		}
	}

	a.Equal(samples, 200000, "all samples read")
	d.Close()
}

func TestEncode(t *testing.T) {
	a := assert.New(t)

	fileName := "data/test.flac"

	e, err := NewEncoder(fileName, 2, 24, 48000)

	a.Equal(err, nil, "err is nil")

	f := Frame{Channels: 1, Depth: 24, Rate: 48000}

	err = e.WriteFrame(f)

	a.Error(err, "channels mismatch")

	f.Channels = 2
	f.Buffer = make([]int32, 2*100)

	err = e.WriteFrame(f)

	a.Equal(err, nil, "frame encoded")

	e.Close()

	os.Remove(fileName)
}

func TestRoundTrip(t *testing.T) {
	a := assert.New(t)

	inputFile := "data/sine24-00.flac"
	outputFile := "data/test.flac"

	d, err := NewDecoder(inputFile)

	a.Equal(err, nil, "err is nil")

	e, err := NewEncoder(outputFile, d.Channels, d.Depth, d.Rate)

	samples := 0

	for {
		f, err := d.ReadFrame()
		if err == nil || err == io.EOF {
			if f != nil {
				_ = e.WriteFrame(*f)
				samples = samples + len(f.Buffer)
			}
		} else {
			a.Equal(err, nil, "error reported")
			break
		}

		if err == io.EOF {
			break
		}
	}

	a.Equal(samples, 200000, "all samples read")
	d.Close()
	e.Close()

	os.Remove(outputFile)
}

func TestRoundTripStereo(t *testing.T) {
	a := assert.New(t)

	inputFile := "data/sine16-12.flac"
	outputFile := "data/test.flac"

	d, err := NewDecoder(inputFile)

	a.Equal(err, nil, "err is nil")

	e, err := NewEncoder(outputFile, d.Channels, d.Depth, d.Rate)

	samples := 0

	for {
		f, err := d.ReadFrame()
		if err == nil || err == io.EOF {
			if f != nil {
				_ = e.WriteFrame(*f)
				samples = samples + len(f.Buffer)
			}
		} else {
			a.Equal(err, nil, "error reported")
			break
		}

		if err == io.EOF {
			break
		}
	}

	a.Equal(samples, 400000, "all samples read")
	d.Close()
	e.Close()

	os.Remove(outputFile)
}
<commit_msg>add test for nil encoder if we can't create the file, check err (not nil) on failed decoder open<commit_after>// Copyright 2015 Cocoon Alarm Ltd.
//
// See LICENSE file for terms and conditions.

package libflac

import (
	"io"
	"os"
	"testing"

	"github.com/jbert/testify/assert"
)

func TestDecode(t *testing.T) {
	a := assert.New(t)

	d, err := NewDecoder("data/nonexistent.flac")
	a.Equal(d, (*Decoder)(nil), "decoder is nil")
	a.NotNil(err, "err is not nil")

	d, err = NewDecoder("data/sine24-00.flac")

	a.Equal(err, nil, "err is nil")
	a.Equal(d.Channels, 1, "channels is 1")
	a.Equal(d.Depth, 24, "depth is 24")
	a.Equal(d.Rate, 48000, "depth is 48000")

	samples := 0

	f, err := d.ReadFrame()

	a.Equal(err, nil, "err is nil")
	a.Equal(f.Channels, 1, "channels is 1")
	a.Equal(f.Depth, 24, "depth is 24")
	a.Equal(f.Rate, 48000, "depth is 48000")

	samples = samples + len(f.Buffer)

	for {
		f, err := d.ReadFrame()

		if err == nil || err == io.EOF {
			if f != nil {
				samples = samples + len(f.Buffer)
			}
		} else {
			a.Equal(err, nil, "error reported")
			break
		}

		if err == io.EOF {
			break
		}
	}

	a.Equal(samples, 200000, "all samples read")
	d.Close()
}

func TestEncode(t *testing.T) {
	a := assert.New(t)

	e, err := NewEncoder("not-existdir/foo.flac", 2, 24, 48000)
	a.Equal(e, (*Encoder)(nil), "encoder is nil")
	a.NotNil(err, "err is not nil")

	fileName := "data/test.flac"

	e, err = NewEncoder(fileName, 2, 24, 48000)

	a.Equal(err, nil, "err is nil")

	f := Frame{Channels: 1, Depth: 24, Rate: 48000}

	err = e.WriteFrame(f)

	a.Error(err, "channels mismatch")

	f.Channels = 2
	f.Buffer = make([]int32, 2*100)

	err = e.WriteFrame(f)

	a.Equal(err, nil, "frame encoded")

	e.Close()

	os.Remove(fileName)
}

func TestRoundTrip(t *testing.T) {
	a := assert.New(t)

	inputFile := "data/sine24-00.flac"
	outputFile := "data/test.flac"

	d, err := NewDecoder(inputFile)

	a.Equal(err, nil, "err is nil")

	e, err := NewEncoder(outputFile, d.Channels, d.Depth, d.Rate)

	samples := 0

	for {
		f, err := d.ReadFrame()
		if err == nil || err == io.EOF {
			if f != nil {
				_ = e.WriteFrame(*f)
				samples = samples + len(f.Buffer)
			}
		} else {
			a.Equal(err, nil, "error reported")
			break
		}

		if err == io.EOF {
			break
		}
	}

	a.Equal(samples, 200000, "all samples read")
	d.Close()
	e.Close()

	os.Remove(outputFile)
}

func TestRoundTripStereo(t *testing.T) {
	a := assert.New(t)

	inputFile := "data/sine16-12.flac"
	outputFile := "data/test.flac"

	d, err := NewDecoder(inputFile)

	a.Equal(err, nil, "err is nil")

	e, err := NewEncoder(outputFile, d.Channels, d.Depth, d.Rate)

	samples := 0

	for {
		f, err := d.ReadFrame()
		if err == nil || err == io.EOF {
			if f != nil {
				_ = e.WriteFrame(*f)
				samples = samples + len(f.Buffer)
			}
		} else {
			a.Equal(err, nil, "error reported")
			break
		}

		if err == io.EOF {
			break
		}
	}

	a.Equal(samples, 400000, "all samples read")
	d.Close()
	e.Close()

	os.Remove(outputFile)
}
<|endoftext|>


<commit_before>package keys

func MinStep(n int) int {
	if n <= 1 {
		return 0
	}
	dp := make([]int, n+1)
	for i := 1; i <= n; i++ {
		dp[i] = i
		for j := 2; j < i; j++ {
			if i%j == 0 {
				dp[i] = j + dp[i/j]
				break
			}
		}
	}
	return dp[n]
}

func MinStep2(n int) int {
	if n <= 1 {
		return 0
	}
	var ret int
	for i := 2; i <= n; i++ {
		for n%i == 0 {
			ret, n = ret+i, n/i
		}
	}
	return ret
}
<commit_msg>with comments<commit_after>package keys

func MinStep(n int) int {
	if n <= 1 {
		return 0
	}
	dp := make([]int, n+1)
	// dp[k] represents the **min steps** to get k 'A's
	// so to get min steps, the action must be ended with paste
	// suppose the last copy-all pressed when there are j 'A' s
	// to get k 'A's we have to (k-j)/j steps with paste
	// so the total steps to get k 'A's from j 'A's is 1 (copy-all) + (k-j)/j (paste)
	// so we have this formular dp[k] = j + dp[k/j]
	for i := 1; i <= n; i++ {
		dp[i] = i
		for j := 2; j < i; j++ {
			if i%j == 0 {
				dp[i] = j + dp[i/j]
				break
			}
		}
	}
	return dp[n]
}

func MinStep2(n int) int {
	if n <= 1 {
		return 0
	}
	var ret int
	for i := 2; i <= n; i++ {
		for n%i == 0 {
			ret, n = ret+i, n/i
		}
	}
	return ret
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"testing"

	"fmt"
	"github.com/tg123/sshpiper/sshpiperd/auditor"
	"github.com/tg123/sshpiper/sshpiperd/challenger"
	"github.com/tg123/sshpiper/sshpiperd/registry"
	"github.com/tg123/sshpiper/sshpiperd/upstream"
	"golang.org/x/crypto/ssh"
	"net"
	"time"
)

type testplugin struct {
	name string
	init func(logger *log.Logger) error
}

func (p *testplugin) GetName() string {
	return p.name
}

func (p *testplugin) GetOpts() interface{} {
	return nil
}

func (p *testplugin) Init(logger *log.Logger) error {
	if p.init == nil {
		return nil
	}
	return p.init(logger)
}

func Test_getAndInstall(t *testing.T) {

	// ignore empty
	getAndInstall("", func(n string) registry.Plugin {
		t.Errorf("should not call get")
		return nil
	}, func(plugin registry.Plugin) error {
		t.Errorf("should not call install")
		return nil
	}, nil)

	// fail when not found
	err := getAndInstall("test", func(n string) registry.Plugin {
		if n != "test" {
			t.Errorf("plugin name changed")
		}
		return nil
	}, func(plugin registry.Plugin) error {
		t.Errorf("should not call install")
		return nil
	}, nil)

	if err == nil {
		t.Errorf("should err when not found")
	}

	// init err
	err = getAndInstall("test", func(n string) registry.Plugin {
		return &testplugin{
			init: func(logger *log.Logger) error {
				return fmt.Errorf("init failed")
			},
		}
	}, func(plugin registry.Plugin) error {
		return fmt.Errorf("test")
	}, nil)

	if err == nil {
		t.Errorf("should err when not found")
	}

	// call init
	inited := false
	installed := false
	err = getAndInstall("test", func(n string) registry.Plugin {
		if n != "test" {
			t.Errorf("plugin name changed")
		}
		return &testplugin{
			init: func(logger *log.Logger) error {
				inited = true
				return nil
			},
		}
	}, func(plugin registry.Plugin) error {
		if !inited {
			t.Errorf("not inited")
		}

		installed = true
		return nil
	}, nil)

	if !installed {
		t.Errorf("not installed")
	}

	if err != nil {
		t.Errorf("should err when not found")
	}
}

type testupstream struct {
	testplugin

	h upstream.Handler
}

func (t *testupstream) GetHandler() upstream.Handler {
	return t.h
}

type testchallenger struct {
	testplugin

	h challenger.Handler
}

func (t *testchallenger) GetHandler() challenger.Handler {
	return t.h
}

type testauditorprovider struct {
	testplugin

	a auditor.Auditor
}

func (t *testauditorprovider) Create(ssh.ConnMetadata) (auditor.Auditor, error) {
	return t.a, nil
}

type testauditor struct {
	up   auditor.Hook
	down auditor.Hook
}

func (t *testauditor) GetUpstreamHook() auditor.Hook {
	return t.up
}

func (t *testauditor) GetDownstreamHook() auditor.Hook {
	return t.down
}

func (t *testauditor) Close() error {
	return nil
}

func Test_installDriver(t *testing.T) {
	var (
		upstreamName    = fmt.Sprintf("u_%v", time.Now().UTC().UnixNano())
		challengerName  = fmt.Sprintf("c_%v", time.Now().UTC().UnixNano())
		auditorName     = fmt.Sprintf("a_%v", time.Now().UTC().UnixNano())
		upstreamErrName = fmt.Sprintf("ue_%v", time.Now().UTC().UnixNano())
		upstreamNilName = fmt.Sprintf("un_%v", time.Now().UTC().UnixNano())
	)

	findUpstream := func(conn ssh.ConnMetadata) (net.Conn, *ssh.SSHPiperAuthPipe, error) {
		return nil, nil, nil
	}

	upstream.Register(upstreamName, &testupstream{
		testplugin: testplugin{
			name: upstreamName,
		},
		h: findUpstream,
	})

	upstream.Register(upstreamErrName, &testupstream{
		testplugin: testplugin{
			name: upstreamErrName,
			init: func(logger *log.Logger) error {
				return fmt.Errorf("test err")
			},
		},
		h: findUpstream,
	})

	upstream.Register(upstreamNilName, &testupstream{
		testplugin: testplugin{
			name: upstreamNilName,
		},
		h: nil,
	})

	challenger.Register(challengerName, &testchallenger{
		testplugin: testplugin{
			name: upstreamName,
		},
		h: func(conn ssh.ConnMetadata, client ssh.KeyboardInteractiveChallenge) (bool, error) {
			return true, nil
		},
	})

	auditor.Register(auditorName, &testauditorprovider{})

	// empty driver name
	{
		piper := &ssh.SSHPiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: "",
		}, nil)

		if err == nil {
			t.Errorf("should fail when empty driver")
		}
	}

	// install upstream
	{
		piper := &ssh.SSHPiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: upstreamName,
		}, nil)

		if err != nil {
			t.Errorf("install failed %v", err)
		}

		if _, _, err := piper.FindUpstream(nil); err != nil {
			t.Errorf("install wrong func")
		}

		if piper.AdditionalChallenge != nil {
			t.Errorf("should not install challenger")
		}
	}

	// install upstream with failed init
	{
		piper := &ssh.SSHPiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: upstreamErrName,
		}, nil)

		if err == nil {
			t.Errorf("install should fail")
		}

		if piper.FindUpstream != nil {
			t.Errorf("should not install upstream provider")
		}

		if piper.AdditionalChallenge != nil {
			t.Errorf("should not install challenger")
		}
	}

	// install upstream with nil handler
	{
		piper := &ssh.SSHPiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: upstreamNilName,
		}, nil)

		if err == nil {
			t.Errorf("install should fail")
		}

		if piper.FindUpstream != nil {
			t.Errorf("should not install upstream provider")
		}

		if piper.AdditionalChallenge != nil {
			t.Errorf("should not install challenger")
		}
	}

	// install challenger
	{
		piper := &ssh.SSHPiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver:   upstreamName,
			ChallengerDriver: challengerName,
		}, nil)

		if err != nil {
			t.Errorf("install failed %v", err)
		}

		if ok, err := piper.AdditionalChallenge(nil, nil); err != nil || !ok {
			t.Errorf("should install challenger")
		}
	}

	// install auditor
	{
		piper := &ssh.SSHPiperConfig{}
		ap, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: upstreamName,
			AuditorDriver:  auditorName,
		}, nil)

		if err != nil {
			t.Errorf("install failed %v", err)
		}

		if ap == nil {
			t.Errorf("nil auditor provider")
		}

		ap0 := ap.(*testauditorprovider)

		ap0.a = &testauditor{
			up: func(conn ssh.ConnMetadata, msg []byte) ([]byte, error) {
				msg[0] = 42
				return msg, nil
			},
			down: func(conn ssh.ConnMetadata, msg []byte) ([]byte, error) {
				msg[0] = 100
				return msg, nil
			},
		}

		a, err := ap.Create(nil)
		if err != nil {
			t.Errorf("install failed %v", err)
		}

		m := []byte{0}

		a.GetUpstreamHook()(nil, m)
		if m[0] != 42 {
			t.Errorf("upstream not handled")
		}

		a.GetDownstreamHook()(nil, m)
		if m[0] != 100 {
			t.Errorf("downstream not handled")
		}
	}
}
<commit_msg>fix missing test sym<commit_after>package main

import (
	"log"
	"testing"

	"fmt"
	"github.com/tg123/sshpiper/sshpiperd/auditor"
	"github.com/tg123/sshpiper/sshpiperd/challenger"
	"github.com/tg123/sshpiper/sshpiperd/registry"
	"github.com/tg123/sshpiper/sshpiperd/upstream"
	"golang.org/x/crypto/ssh"
	"net"
	"time"
)

type testplugin struct {
	name string
	init func(logger *log.Logger) error
}

func (p *testplugin) GetName() string {
	return p.name
}

func (p *testplugin) GetOpts() interface{} {
	return nil
}

func (p *testplugin) Init(logger *log.Logger) error {
	if p.init == nil {
		return nil
	}
	return p.init(logger)
}

func Test_getAndInstall(t *testing.T) {

	// ignore empty
	getAndInstall("", func(n string) registry.Plugin {
		t.Errorf("should not call get")
		return nil
	}, func(plugin registry.Plugin) error {
		t.Errorf("should not call install")
		return nil
	}, nil)

	// fail when not found
	err := getAndInstall("test", func(n string) registry.Plugin {
		if n != "test" {
			t.Errorf("plugin name changed")
		}
		return nil
	}, func(plugin registry.Plugin) error {
		t.Errorf("should not call install")
		return nil
	}, nil)

	if err == nil {
		t.Errorf("should err when not found")
	}

	// init err
	err = getAndInstall("test", func(n string) registry.Plugin {
		return &testplugin{
			init: func(logger *log.Logger) error {
				return fmt.Errorf("init failed")
			},
		}
	}, func(plugin registry.Plugin) error {
		return fmt.Errorf("test")
	}, nil)

	if err == nil {
		t.Errorf("should err when not found")
	}

	// call init
	inited := false
	installed := false
	err = getAndInstall("test", func(n string) registry.Plugin {
		if n != "test" {
			t.Errorf("plugin name changed")
		}
		return &testplugin{
			init: func(logger *log.Logger) error {
				inited = true
				return nil
			},
		}
	}, func(plugin registry.Plugin) error {
		if !inited {
			t.Errorf("not inited")
		}

		installed = true
		return nil
	}, nil)

	if !installed {
		t.Errorf("not installed")
	}

	if err != nil {
		t.Errorf("should err when not found")
	}
}

type testupstream struct {
	testplugin

	h upstream.Handler
}

func (t *testupstream) GetHandler() upstream.Handler {
	return t.h
}

type testchallenger struct {
	testplugin

	h challenger.Handler
}

func (t *testchallenger) GetHandler() challenger.Handler {
	return t.h
}

type testauditorprovider struct {
	testplugin

	a auditor.Auditor
}

func (t *testauditorprovider) Create(ssh.ConnMetadata) (auditor.Auditor, error) {
	return t.a, nil
}

type testauditor struct {
	up   auditor.Hook
	down auditor.Hook
}

func (t *testauditor) GetUpstreamHook() auditor.Hook {
	return t.up
}

func (t *testauditor) GetDownstreamHook() auditor.Hook {
	return t.down
}

func (t *testauditor) Close() error {
	return nil
}

func Test_installDriver(t *testing.T) {
	var (
		upstreamName    = fmt.Sprintf("u_%v", time.Now().UTC().UnixNano())
		challengerName  = fmt.Sprintf("c_%v", time.Now().UTC().UnixNano())
		auditorName     = fmt.Sprintf("a_%v", time.Now().UTC().UnixNano())
		upstreamErrName = fmt.Sprintf("ue_%v", time.Now().UTC().UnixNano())
		upstreamNilName = fmt.Sprintf("un_%v", time.Now().UTC().UnixNano())
	)

	findUpstream := func(conn ssh.ConnMetadata) (net.Conn, *ssh.AuthPipe, error) {
		return nil, nil, nil
	}

	upstream.Register(upstreamName, &testupstream{
		testplugin: testplugin{
			name: upstreamName,
		},
		h: findUpstream,
	})

	upstream.Register(upstreamErrName, &testupstream{
		testplugin: testplugin{
			name: upstreamErrName,
			init: func(logger *log.Logger) error {
				return fmt.Errorf("test err")
			},
		},
		h: findUpstream,
	})

	upstream.Register(upstreamNilName, &testupstream{
		testplugin: testplugin{
			name: upstreamNilName,
		},
		h: nil,
	})

	challenger.Register(challengerName, &testchallenger{
		testplugin: testplugin{
			name: upstreamName,
		},
		h: func(conn ssh.ConnMetadata, client ssh.KeyboardInteractiveChallenge) (bool, error) {
			return true, nil
		},
	})

	auditor.Register(auditorName, &testauditorprovider{})

	// empty driver name
	{
		piper := &ssh.PiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: "",
		}, nil)

		if err == nil {
			t.Errorf("should fail when empty driver")
		}
	}

	// install upstream
	{
		piper := &ssh.PiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: upstreamName,
		}, nil)

		if err != nil {
			t.Errorf("install failed %v", err)
		}

		if _, _, err := piper.FindUpstream(nil); err != nil {
			t.Errorf("install wrong func")
		}

		if piper.AdditionalChallenge != nil {
			t.Errorf("should not install challenger")
		}
	}

	// install upstream with failed init
	{
		piper := &ssh.PiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: upstreamErrName,
		}, nil)

		if err == nil {
			t.Errorf("install should fail")
		}

		if piper.FindUpstream != nil {
			t.Errorf("should not install upstream provider")
		}

		if piper.AdditionalChallenge != nil {
			t.Errorf("should not install challenger")
		}
	}

	// install upstream with nil handler
	{
		piper := &ssh.PiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: upstreamNilName,
		}, nil)

		if err == nil {
			t.Errorf("install should fail")
		}

		if piper.FindUpstream != nil {
			t.Errorf("should not install upstream provider")
		}

		if piper.AdditionalChallenge != nil {
			t.Errorf("should not install challenger")
		}
	}

	// install challenger
	{
		piper := &ssh.PiperConfig{}
		_, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver:   upstreamName,
			ChallengerDriver: challengerName,
		}, nil)

		if err != nil {
			t.Errorf("install failed %v", err)
		}

		if ok, err := piper.AdditionalChallenge(nil, nil); err != nil || !ok {
			t.Errorf("should install challenger")
		}
	}

	// install auditor
	{
		piper := &ssh.PiperConfig{}
		ap, err := installDrivers(piper, &piperdConfig{
			UpstreamDriver: upstreamName,
			AuditorDriver:  auditorName,
		}, nil)

		if err != nil {
			t.Errorf("install failed %v", err)
		}

		if ap == nil {
			t.Errorf("nil auditor provider")
		}

		ap0 := ap.(*testauditorprovider)

		ap0.a = &testauditor{
			up: func(conn ssh.ConnMetadata, msg []byte) ([]byte, error) {
				msg[0] = 42
				return msg, nil
			},
			down: func(conn ssh.ConnMetadata, msg []byte) ([]byte, error) {
				msg[0] = 100
				return msg, nil
			},
		}

		a, err := ap.Create(nil)
		if err != nil {
			t.Errorf("install failed %v", err)
		}

		m := []byte{0}

		a.GetUpstreamHook()(nil, m)
		if m[0] != 42 {
			t.Errorf("upstream not handled")
		}

		a.GetDownstreamHook()(nil, m)
		if m[0] != 100 {
			t.Errorf("downstream not handled")
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package api_test

import (
	"io"
	"net"

	gc "launchpad.net/gocheck"

	jujutesting "launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/utils/parallel"
)

type apiclientSuite struct {
	jujutesting.JujuConnSuite
}

var _ = gc.Suite(&apiclientSuite{})

func (s *apiclientSuite) TestOpenMultiple(c *gc.C) {
	// Create a socket that proxies to the API server.
	info := s.APIInfo(c)
	serverAddr := info.Addrs[0]
	server, err := net.Dial("tcp", serverAddr)
	c.Assert(err, gc.IsNil)
	defer server.Close()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	c.Assert(err, gc.IsNil)
	defer listener.Close()
	go func() {
		for {
			client, err := listener.Accept()
			if err != nil {
				return
			}
			go io.Copy(client, server)
			go io.Copy(server, client)
		}
	}()

	// Check that we can use the proxy to connect.
	proxyAddr := listener.Addr().String()
	info.Addrs = []string{proxyAddr}
	st, err := api.Open(info, api.DialOpts{})
	c.Assert(err, gc.IsNil)
	defer st.Close()
	c.Assert(st.Addr(), gc.Equals, proxyAddr)

	// Now break Addrs[0], and ensure that Addrs[1]
	// is successfully connected to.
	info.Addrs = []string{proxyAddr, serverAddr}
	listener.Close()
	st, err = api.Open(info, api.DialOpts{})
	c.Assert(err, gc.IsNil)
	defer st.Close()
	c.Assert(st.Addr(), gc.Equals, serverAddr)
}

func (s *apiclientSuite) TestOpenMultipleError(c *gc.C) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	c.Assert(err, gc.IsNil)
	defer listener.Close()
	go func() {
		for {
			client, err := listener.Accept()
			if err != nil {
				return
			}
			client.Close()
		}
	}()
	info := s.APIInfo(c)
	addr := listener.Addr().String()
	c.Logf("addr: %q", addr)
	info.Addrs = []string{addr, addr, addr}
	_, err = api.Open(info, api.DialOpts{})
	c.Assert(err, gc.ErrorMatches, `timed out connecting to "wss://.*/"`)
}

func (s *apiclientSuite) TestDialWebsocketStopped(c *gc.C) {
	stopped := make(chan struct{})
	f := api.NewWebsocketDialer(nil, api.DialOpts{})
	close(stopped)
	result, err := f(stopped)
	c.Assert(err, gc.Equals, parallel.ErrStopped)
	c.Assert(result, gc.IsNil)
}
<commit_msg>delete log statement<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package api_test

import (
	"io"
	"net"

	gc "launchpad.net/gocheck"

	jujutesting "launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/utils/parallel"
)

type apiclientSuite struct {
	jujutesting.JujuConnSuite
}

var _ = gc.Suite(&apiclientSuite{})

func (s *apiclientSuite) TestOpenMultiple(c *gc.C) {
	// Create a socket that proxies to the API server.
	info := s.APIInfo(c)
	serverAddr := info.Addrs[0]
	server, err := net.Dial("tcp", serverAddr)
	c.Assert(err, gc.IsNil)
	defer server.Close()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	c.Assert(err, gc.IsNil)
	defer listener.Close()
	go func() {
		for {
			client, err := listener.Accept()
			if err != nil {
				return
			}
			go io.Copy(client, server)
			go io.Copy(server, client)
		}
	}()

	// Check that we can use the proxy to connect.
	proxyAddr := listener.Addr().String()
	info.Addrs = []string{proxyAddr}
	st, err := api.Open(info, api.DialOpts{})
	c.Assert(err, gc.IsNil)
	defer st.Close()
	c.Assert(st.Addr(), gc.Equals, proxyAddr)

	// Now break Addrs[0], and ensure that Addrs[1]
	// is successfully connected to.
	info.Addrs = []string{proxyAddr, serverAddr}
	listener.Close()
	st, err = api.Open(info, api.DialOpts{})
	c.Assert(err, gc.IsNil)
	defer st.Close()
	c.Assert(st.Addr(), gc.Equals, serverAddr)
}

func (s *apiclientSuite) TestOpenMultipleError(c *gc.C) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	c.Assert(err, gc.IsNil)
	defer listener.Close()
	go func() {
		for {
			client, err := listener.Accept()
			if err != nil {
				return
			}
			client.Close()
		}
	}()
	info := s.APIInfo(c)
	addr := listener.Addr().String()
	info.Addrs = []string{addr, addr, addr}
	_, err = api.Open(info, api.DialOpts{})
	c.Assert(err, gc.ErrorMatches, `timed out connecting to "wss://.*/"`)
}

func (s *apiclientSuite) TestDialWebsocketStopped(c *gc.C) {
	stopped := make(chan struct{})
	f := api.NewWebsocketDialer(nil, api.DialOpts{})
	close(stopped)
	result, err := f(stopped)
	c.Assert(err, gc.Equals, parallel.ErrStopped)
	c.Assert(result, gc.IsNil)
}
<|endoftext|>