<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
	"unicode"
	"unicode/utf8"
	"strings"
)

// itemElement are the types that are emitted by the lexer.
type itemElement int

const (
	itemEOF itemElement = iota
	itemError
	itemTitle
	itemSectionAdornment
	itemParagraph
	itemBlockquote
	itemLiteralBlock
	itemSystemMessage
	itemSpace
	itemBlankLine
)

var elements = [...]string{
	"itemEOF",
	"itemError",
	"itemTitle",
	"itemSectionAdornment",
	"itemParagraph",
	"itemBlockquote",
	"itemLiteralBlock",
	"itemSystemMessage",
	"itemSpace",
	"itemBlankLine",
}

// String implements the Stringer interface for printing itemElement types.
func (t itemElement) String() string { return elements[t] }

// Valid section adornment runes
var sectionAdornments = []rune{'!', '"', '#', '$', '\'', '%', '&', '(', ')', '*',
	'+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\',
	']', '^', '_', '`', '{', '|', '}', '~'}

// Emitted by the lexer on End of File
const eof rune = -1

// Function prototype for scanner functions
type stateFn func(*lexer) stateFn

// Struct for tokens emitted by the scanning process
type item struct {
	ElementName string      `json:"element-name"`
	ElementType itemElement `json: "-"`
	Position    Pos         `json: "position"`
	Line        int         `json: "line"`
	Value       interface{} `json: "value"`
}

// The lexer struct tracks the state of the lexer
type lexer struct {
	name     string    // The name of the current lexer
	input    string    // The input text
	state    stateFn   // The current state of the lexer
	pos      Pos       // Position in input
	start    Pos       // The start of the current token
	width    Pos       // The width of the current position
	items    chan item // The channel items are emitted to
	lastItem *item     // The last item emitted to the channel
	lastPos  Pos
}

// lex is the entry point of the lexer
func lex(name, input string) *lexer {
	log.Debugln("Start")
	l := &lexer{
		name:  name,
		input: input,
		items: make(chan item),
	}
	go l.run()
	log.Debugln("End")
	return l
}

// run is the engine of the lexing process.
func (l *lexer) run() {
	log.Debugln("Start")
	for l.state = lexStart; l.state != nil; {
		l.state = l.state(l)
	}
	log.Debugln("End")
}

// emit passes an item back to the client.
func (l *lexer) emit(t itemElement) {
	if l.start == l.pos && int(l.pos) < len(l.input) {
		l.pos += 1
	}
	log.Debugf("#### %s: %q start: %d pos: %d line: %d\n", t,
		l.input[l.start:l.pos], l.start, l.pos, l.lineNumber())
	nItem := item{ElementType: t, ElementName: fmt.Sprint(t), Position: l.start + 1, Line:
		l.lineNumber(), Value: l.input[l.start:l.pos]}
	l.items <- nItem
	l.lastItem = &nItem
	l.start = l.pos
}

// backup backs up the lexer by one position using the width of the last rune retrieved from the
// input.
func (l *lexer) backup() {
	l.pos -= l.width
}

// current returns the rune at the current position in the input.
func (l *lexer) current() rune {
	r, _ := utf8.DecodeRuneInString(l.input[l.pos:])
	return r
}

// peek looks ahead in the input by one position and returns the rune.
func (l *lexer) peek() rune {
	r := l.next()
	l.backup()
	return r
}

// advance fast forwards the lexer to the next rune in the input specified by "to".
func (l *lexer) advance(to rune) {
	for {
		if l.next() == eof || to == l.current() {
			break
		}
	}
}

// next advances the position of the lexer by one rune and returns that rune.
func (l *lexer) next() rune {
	if int(l.pos) >= len(l.input) {
		log.Debugln("Reached eof!")
		l.width = 0
		return eof
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = Pos(w)
	l.pos += l.width
	log.Debugf("cur: %q start: %d pos: %d\n", r, l.start, l.pos)
	return r
}

// nextItem returns the next item from the input.
func (l *lexer) nextItem() item {
	item := <-l.items
	l.lastPos = item.Position
	return item

}

func (l *lexer) lineNumber() int {
	return 1 + strings.Count(l.input[:l.pos - 1], "\n")
}

// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
	return r == ' ' || r == '\t' || r == '\n' || r == '\r'
}

// isEndOfLine reports whether r is an end-of-line character.
func isEndOfLine(r rune) bool {
	return r == '\r' || r == '\n'
}

// isSection compares a number of positions (skipping whitespace) to
// determine if the runes are sectionAdornments and returns a true if the
// positions match each other. Rune comparison begins at the current lexer
// position. isSection returns false if there is a blank line between the
// positions or if there is a rune mismatch between positions.
func isSection(l *lexer) bool {
	log.Debugln("Start")
	var lookPositions = 2
	var lastAdornment rune
	var newLineNum int
	var runePositions []rune
	var matchCount int
	cPos := l.pos

	exit := func(value bool) bool {
		l.pos = cPos
		log.Debugln("Returning", value)
		return value
	}

	log.Debugln("Looking ahead", lookPositions, "position(s) for sectionAdornments...")

	// Check two runes to see if they are section adornments, if not, we will check the next
	// line (after whitespace).
	if isSectionAdornment(l.current()) && isSectionAdornment(l.peek()) &&
		l.current() == l.peek() {
		return true
	}

	// Advance to the end of the line
	l.advance('\n')
	if l.current() == eof {
		return false
	}

	for j := 0; j < lookPositions; j++ {
		for isSpace(l.current()) {
			if isEndOfLine(l.current()) {
				if newLineNum == 1 {
					log.Debugln("Too many newlines!")
					return exit(false)
				}
				newLineNum += 1
				log.Debugln("newLineNum:", newLineNum)
			}
			l.next()
		}

		if isSectionAdornment(l.current()) {
			log.Debugf("Found adornment: \"%s\" pos: %d\n", string(l.current()), l.pos)
			if lastAdornment != 0 && l.current() != lastAdornment {
				log.Debugf("Adornment mismatch, last: %s current: %s\n",
					string(lastAdornment), string(l.current()))
				return exit(false)
			}
			runePositions = append(runePositions, l.current())
			lastAdornment = l.current()
			matchCount += 1
		}
		l.next()
	}

	if len(runePositions) == 0 || matchCount != lookPositions {
		return exit(false)
	}

	return exit(true)
}

// isSectionAdornment returns true if r matches a section adornment.
func isSectionAdornment(r rune) bool {
	for _, a := range sectionAdornments {
		if a == r {
			return true
		}
	}
	return false
}

// lexStart is the first stateFn called by run(). From here other stateFn's are called depending on
// the input. When this function returns nil, the lexing is finished and run() will exit.
func lexStart(l *lexer) stateFn {
	log.Debugln("Start")
	for {
		var tokenLength = l.pos - l.start
		switch r := l.current(); {
		case tokenLength == 1:
			log.Debugln("tokenLength == 1; Start of new token")
			if isEndOfLine(r) {
				l.emit(itemBlankLine)
				l.start += 1
			} else if isSpace(r) {
				return lexSpace
			} else if isSection(l) {
				return lexSection
			}
		case isEndOfLine(r):
			log.Debugln("isEndOfLine == true")
			if l.pos > l.start {
				l.emit(itemParagraph)
				l.start += 1 // Skip the new line
			} else if l.start == l.pos {
				l.emit(itemBlankLine)
			}

		}
		if l.next() == eof {
			break
		}
	}

	// Correctly reached eof.
	if l.pos > l.start {
		l.emit(itemParagraph)
	}

	l.emit(itemEOF)
	log.Debugln("End")
	return nil
}

// lexSpace consumes space characters (space and tab) in the input and emits a itemSpace token.
func lexSpace(l *lexer) stateFn {
	log.Debugln("Start")
	for isSpace(l.current()) {
		l.next()
	}
	if l.start < l.pos {
		l.emit(itemSpace)
		l.next()
	}
	log.Debugln("End")
	return lexStart
}

// lexSection is used after isSection() has determined that the next runes of input are section.
// From here, the lexTitle() and lexSectionAdornment() are called based on the input.
func lexSection(l *lexer) stateFn {
	log.Debugln("Start")
	// The order of the case statement matter here
	switch r := l.next(); {
	case isSectionAdornment(r):
		if l.lastItem.ElementType != itemTitle {
			return lexSectionAdornment
		}
		lexSectionAdornment(l)
	case isSpace(r):
		return lexSpace
	case r <= unicode.MaxASCII && unicode.IsPrint(r):
		return lexTitle
	case isEndOfLine(r):
		l.start += 1
	}
	log.Debugln("Exit")
	return lexStart
}

// lexTitle consumes input until newline and emits an itemTitle token. Once the token is emitted,
// control is returned to lexSection().
func lexTitle(l *lexer) stateFn {
	log.Debugln("Start")
	for {
		l.next()
		if l.peek() == '\n' {
			l.emit(itemTitle)
			l.start += 1
			l.pos += 1
			break
		}
	}
	log.Debugln("End")
	return lexSection
}

// lexSectionAdornment advances the lexer until a newline is encountered and emits a
// itemSectionAdornment token. Control is returned to lexSection() on completion.
func lexSectionAdornment(l *lexer) stateFn {
	log.Debugln("Start")
	for {
		//TODO: Add adornment rune check
		l.next()
		if l.peek() == '\n' {
			l.emit(itemSectionAdornment)
			l.start += 1
			l.pos += 1
			break
		}
	}
	log.Debugln("End")
	return lexSection
}
<commit_msg>lex.go: Add Length to item struct<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
	"unicode"
	"unicode/utf8"
	"strings"
)

// itemElement are the types that are emitted by the lexer.
type itemElement int

const (
	itemEOF itemElement = iota
	itemError
	itemTitle
	itemSectionAdornment
	itemParagraph
	itemBlockquote
	itemLiteralBlock
	itemSystemMessage
	itemSpace
	itemBlankLine
)

var elements = [...]string{
	"itemEOF",
	"itemError",
	"itemTitle",
	"itemSectionAdornment",
	"itemParagraph",
	"itemBlockquote",
	"itemLiteralBlock",
	"itemSystemMessage",
	"itemSpace",
	"itemBlankLine",
}

// String implements the Stringer interface for printing itemElement types.
func (t itemElement) String() string { return elements[t] }

// Valid section adornment runes
var sectionAdornments = []rune{'!', '"', '#', '$', '\'', '%', '&', '(', ')', '*',
	'+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\',
	']', '^', '_', '`', '{', '|', '}', '~'}

// Emitted by the lexer on End of File
const eof rune = -1

// Function prototype for scanner functions
type stateFn func(*lexer) stateFn

// Struct for tokens emitted by the scanning process
type item struct {
	ElementName string      `json:"element-name"`
	ElementType itemElement `json: "-"`
	Position    Pos         `json: "position"`
	Line        int         `json: "line"`
	Length	    int		`json: "length"`
	Value       interface{} `json: "value"`
}

// The lexer struct tracks the state of the lexer
type lexer struct {
	name     string    // The name of the current lexer
	input    string    // The input text
	state    stateFn   // The current state of the lexer
	pos      Pos       // Position in input
	start    Pos       // The start of the current token
	width    Pos       // The width of the current position
	items    chan item // The channel items are emitted to
	lastItem *item     // The last item emitted to the channel
	lastPos  Pos
}

// lex is the entry point of the lexer
func lex(name, input string) *lexer {
	log.Debugln("Start")
	l := &lexer{
		name:  name,
		input: input,
		items: make(chan item),
	}
	go l.run()
	log.Debugln("End")
	return l
}

// run is the engine of the lexing process.
func (l *lexer) run() {
	log.Debugln("Start")
	for l.state = lexStart; l.state != nil; {
		l.state = l.state(l)
	}
	log.Debugln("End")
}

// emit passes an item back to the client.
func (l *lexer) emit(t itemElement) {
	if l.start == l.pos && int(l.pos) < len(l.input) {
		l.pos += 1
	}
	log.Debugf("#### %s: %q start: %d pos: %d line: %d\n", t,
		l.input[l.start:l.pos], l.start, l.pos, l.lineNumber())
	nItem := item{ElementType: t, ElementName: fmt.Sprint(t), Position: l.start + 1, Line:
		l.lineNumber(), Value: l.input[l.start:l.pos]}
	l.items <- nItem
	l.lastItem = &nItem
	l.start = l.pos
}

// backup backs up the lexer by one position using the width of the last rune retrieved from the
// input.
func (l *lexer) backup() {
	l.pos -= l.width
}

// current returns the rune at the current position in the input.
func (l *lexer) current() rune {
	r, _ := utf8.DecodeRuneInString(l.input[l.pos:])
	return r
}

// peek looks ahead in the input by one position and returns the rune.
func (l *lexer) peek() rune {
	r := l.next()
	l.backup()
	return r
}

// advance fast forwards the lexer to the next rune in the input specified by "to".
func (l *lexer) advance(to rune) {
	for {
		if l.next() == eof || to == l.current() {
			break
		}
	}
}

// next advances the position of the lexer by one rune and returns that rune.
func (l *lexer) next() rune {
	if int(l.pos) >= len(l.input) {
		log.Debugln("Reached eof!")
		l.width = 0
		return eof
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = Pos(w)
	l.pos += l.width
	log.Debugf("cur: %q start: %d pos: %d\n", r, l.start, l.pos)
	return r
}

// nextItem returns the next item from the input.
func (l *lexer) nextItem() item {
	item := <-l.items
	l.lastPos = item.Position
	return item

}

func (l *lexer) lineNumber() int {
	return 1 + strings.Count(l.input[:l.pos - 1], "\n")
}

// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
	return r == ' ' || r == '\t' || r == '\n' || r == '\r'
}

// isEndOfLine reports whether r is an end-of-line character.
func isEndOfLine(r rune) bool {
	return r == '\r' || r == '\n'
}

// isSection compares a number of positions (skipping whitespace) to
// determine if the runes are sectionAdornments and returns a true if the
// positions match each other. Rune comparison begins at the current lexer
// position. isSection returns false if there is a blank line between the
// positions or if there is a rune mismatch between positions.
func isSection(l *lexer) bool {
	log.Debugln("Start")
	var lookPositions = 2
	var lastAdornment rune
	var newLineNum int
	var runePositions []rune
	var matchCount int
	cPos := l.pos

	exit := func(value bool) bool {
		l.pos = cPos
		log.Debugln("Returning", value)
		return value
	}

	log.Debugln("Looking ahead", lookPositions, "position(s) for sectionAdornments...")

	// Check two runes to see if they are section adornments, if not, we will check the next
	// line (after whitespace).
	if isSectionAdornment(l.current()) && isSectionAdornment(l.peek()) &&
		l.current() == l.peek() {
		return true
	}

	// Advance to the end of the line
	l.advance('\n')
	if l.current() == eof {
		return false
	}

	for j := 0; j < lookPositions; j++ {
		for isSpace(l.current()) {
			if isEndOfLine(l.current()) {
				if newLineNum == 1 {
					log.Debugln("Too many newlines!")
					return exit(false)
				}
				newLineNum += 1
				log.Debugln("newLineNum:", newLineNum)
			}
			l.next()
		}

		if isSectionAdornment(l.current()) {
			log.Debugf("Found adornment: \"%s\" pos: %d\n", string(l.current()), l.pos)
			if lastAdornment != 0 && l.current() != lastAdornment {
				log.Debugf("Adornment mismatch, last: %s current: %s\n",
					string(lastAdornment), string(l.current()))
				return exit(false)
			}
			runePositions = append(runePositions, l.current())
			lastAdornment = l.current()
			matchCount += 1
		}
		l.next()
	}

	if len(runePositions) == 0 || matchCount != lookPositions {
		return exit(false)
	}

	return exit(true)
}

// isSectionAdornment returns true if r matches a section adornment.
func isSectionAdornment(r rune) bool {
	for _, a := range sectionAdornments {
		if a == r {
			return true
		}
	}
	return false
}

// lexStart is the first stateFn called by run(). From here other stateFn's are called depending on
// the input. When this function returns nil, the lexing is finished and run() will exit.
func lexStart(l *lexer) stateFn {
	log.Debugln("Start")
	for {
		var tokenLength = l.pos - l.start
		switch r := l.current(); {
		case tokenLength == 1:
			log.Debugln("tokenLength == 1; Start of new token")
			if isEndOfLine(r) {
				l.emit(itemBlankLine)
				l.start += 1
			} else if isSpace(r) {
				return lexSpace
			} else if isSection(l) {
				return lexSection
			}
		case isEndOfLine(r):
			log.Debugln("isEndOfLine == true")
			if l.pos > l.start {
				l.emit(itemParagraph)
				l.start += 1 // Skip the new line
			} else if l.start == l.pos {
				l.emit(itemBlankLine)
			}

		}
		if l.next() == eof {
			break
		}
	}

	// Correctly reached eof.
	if l.pos > l.start {
		l.emit(itemParagraph)
	}

	l.emit(itemEOF)
	log.Debugln("End")
	return nil
}

// lexSpace consumes space characters (space and tab) in the input and emits a itemSpace token.
func lexSpace(l *lexer) stateFn {
	log.Debugln("Start")
	for isSpace(l.current()) {
		l.next()
	}
	if l.start < l.pos {
		l.emit(itemSpace)
		l.next()
	}
	log.Debugln("End")
	return lexStart
}

// lexSection is used after isSection() has determined that the next runes of input are section.
// From here, the lexTitle() and lexSectionAdornment() are called based on the input.
func lexSection(l *lexer) stateFn {
	log.Debugln("Start")
	// The order of the case statement matter here
	switch r := l.next(); {
	case isSectionAdornment(r):
		if l.lastItem.ElementType != itemTitle {
			return lexSectionAdornment
		}
		lexSectionAdornment(l)
	case isSpace(r):
		return lexSpace
	case r <= unicode.MaxASCII && unicode.IsPrint(r):
		return lexTitle
	case isEndOfLine(r):
		l.start += 1
	}
	log.Debugln("Exit")
	return lexStart
}

// lexTitle consumes input until newline and emits an itemTitle token. Once the token is emitted,
// control is returned to lexSection().
func lexTitle(l *lexer) stateFn {
	log.Debugln("Start")
	for {
		l.next()
		if l.peek() == '\n' {
			l.emit(itemTitle)
			l.start += 1
			l.pos += 1
			break
		}
	}
	log.Debugln("End")
	return lexSection
}

// lexSectionAdornment advances the lexer until a newline is encountered and emits a
// itemSectionAdornment token. Control is returned to lexSection() on completion.
func lexSectionAdornment(l *lexer) stateFn {
	log.Debugln("Start")
	for {
		//TODO: Add adornment rune check
		l.next()
		if l.peek() == '\n' {
			l.emit(itemSectionAdornment)
			l.start += 1
			l.pos += 1
			break
		}
	}
	log.Debugln("End")
	return lexSection
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package memcache provides a client for the memcached cache server.
package memcache

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"
)

const testServer = "localhost:11211"

func setup(t *testing.T) bool {
	c, err := net.Dial("tcp", testServer)
	if err != nil {
		t.Logf("skipping test; no server running at %s", testServer)
		return false
	}
	c.Write([]byte("flush_all\r\n"))
	c.Close()
	return true
}

func TestLocalhost(t *testing.T) {
	if !setup(t) {
		return
	}
	testWithClient(t, New(testServer))
}

// Run the memcached binary as a child process and connect to its unix socket.
func TestUnixSocket(t *testing.T) {
	sock := fmt.Sprintf("/tmp/test-gomemcache-%d.sock", os.Getpid())
	cmd := exec.Command("memcached", "-s", sock)
	if err := cmd.Start(); err != nil {
		t.Logf("skipping test; couldn't find memcached")
		return
	}
	defer cmd.Wait()
	defer cmd.Process.Kill()

	// Wait a bit for the socket to appear.
	for i := 0; i < 10; i++ {
		if _, err := os.Stat(sock); err == nil {
			break
		}
		time.Sleep(time.Duration(25*i) * time.Millisecond)
	}

	testWithClient(t, New(sock))
}

func testWithClient(t *testing.T, c *Client) {
	checkErr := func(err error, format string, args ...interface{}) {
		if err != nil {
			t.Fatalf(format, args...)
		}
	}

	mustSet := func(it *Item) {
		if err := c.Set(it); err != nil {
			t.Fatalf("failed to Set %#v: %v", *it, err)
		}
	}

	// Set
	foo := &Item{Key: "foo", Value: []byte("fooval"), Flags: 123}
	err := c.Set(foo)
	checkErr(err, "first set(foo): %v", err)
	err = c.Set(foo)
	checkErr(err, "second set(foo): %v", err)

	// Get
	it, err := c.Get("foo")
	checkErr(err, "get(foo): %v", err)
	if it.Key != "foo" {
		t.Errorf("get(foo) Key = %q, want foo", it.Key)
	}
	if string(it.Value) != "fooval" {
		t.Errorf("get(foo) Value = %q, want fooval", string(it.Value))
	}
	if it.Flags != 123 {
		t.Errorf("get(foo) Flags = %v, want 123", it.Flags)
	}

	// Add
	bar := &Item{Key: "bar", Value: []byte("barval")}
	err = c.Add(bar)
	checkErr(err, "first add(bar): %v", err)
	if err := c.Add(bar); err != ErrNotStored {
		t.Fatalf("second add(bar) want ErrNotStored, got %v", err)
	}

	// GetMulti
	m, err := c.GetMulti([]string{"foo", "bar"})
	checkErr(err, "GetMulti: %v", err)
	if g, e := len(m), 2; g != e {
		t.Errorf("GetMulti: got len(map) = %d, want = %d", g, e)
	}
	if _, ok := m["foo"]; !ok {
		t.Fatalf("GetMulti: didn't get key 'foo'")
	}
	if _, ok := m["bar"]; !ok {
		t.Fatalf("GetMulti: didn't get key 'bar'")
	}
	if g, e := string(m["foo"].Value), "fooval"; g != e {
		t.Errorf("GetMulti: foo: got %q, want %q", g, e)
	}
	if g, e := string(m["bar"].Value), "barval"; g != e {
		t.Errorf("GetMulti: bar: got %q, want %q", g, e)
	}

	// Delete
	err = c.Delete("foo")
	checkErr(err, "Delete: %v", err)
	_, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Errorf("post-Delete want ErrCacheMiss, got %v", err)
	}

	// Incr/Decr
	mustSet(&Item{Key: "num", Value: []byte("42")})
	n, err := c.Increment("num", 8)
	checkErr(err, "Increment num + 8: %v", err)
	if n != 50 {
		t.Fatalf("Increment num + 8: want=50, got=%d", n)
	}
	n, err = c.Decrement("num", 49)
	checkErr(err, "Decrement: %v", err)
	if n != 1 {
		t.Fatalf("Decrement 49: want=1, got=%d", n)
	}
	err = c.Delete("num")
	checkErr(err, "delete num: %v", err)
	n, err = c.Increment("num", 1)
	if err != ErrCacheMiss {
		t.Fatalf("increment post-delete: want ErrCacheMiss, got %v", err)
	}
	mustSet(&Item{Key: "num", Value: []byte("not-numeric")})
	n, err = c.Increment("num", 1)
	if err == nil || !strings.Contains(err.Error(), "client error") {
		t.Fatalf("increment non-number: want client error, got %v", err)
	}

}
<commit_msg>Move code to start a local memcached server to its own function<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package memcache provides a client for the memcached cache server.
package memcache

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"
)

const testServer = "localhost:11211"

func setup(t *testing.T) bool {
	c, err := net.Dial("tcp", testServer)
	if err != nil {
		t.Logf("skipping test; no server running at %s", testServer)
		return false
	}
	c.Write([]byte("flush_all\r\n"))
	c.Close()
	return true
}

func TestLocalhost(t *testing.T) {
	if !setup(t) {
		return
	}
	testWithClient(t, New(testServer))
}

func newUnixServer(tb testing.TB) (*exec.Cmd, *Client) {
	sock := fmt.Sprintf("/tmp/test-gomemcache-%d.sock", os.Getpid())
	os.Remove(sock)
	cmd := exec.Command("memcached", "-s", sock)
	if err := cmd.Start(); err != nil {
		tb.Skip("skipping test; couldn't find memcached")
		return nil, nil
	}

	// Wait a bit for the socket to appear.
	for i := 0; i < 10; i++ {
		if _, err := os.Stat(sock); err == nil {
			break
		}
		time.Sleep(time.Duration(25*i) * time.Millisecond)
	}
	return cmd, New(sock)
}

// Run the memcached binary as a child process and connect to its unix socket.
func TestUnixSocket(t *testing.T) {
	cmd, c := newUnixServer(t)
	defer cmd.Wait()
	defer cmd.Process.Kill()
	testWithClient(t, c)
}

func testWithClient(t *testing.T, c *Client) {
	checkErr := func(err error, format string, args ...interface{}) {
		if err != nil {
			t.Fatalf(format, args...)
		}
	}

	mustSet := func(it *Item) {
		if err := c.Set(it); err != nil {
			t.Fatalf("failed to Set %#v: %v", *it, err)
		}
	}

	// Set
	foo := &Item{Key: "foo", Value: []byte("fooval"), Flags: 123}
	err := c.Set(foo)
	checkErr(err, "first set(foo): %v", err)
	err = c.Set(foo)
	checkErr(err, "second set(foo): %v", err)

	// Get
	it, err := c.Get("foo")
	checkErr(err, "get(foo): %v", err)
	if it.Key != "foo" {
		t.Errorf("get(foo) Key = %q, want foo", it.Key)
	}
	if string(it.Value) != "fooval" {
		t.Errorf("get(foo) Value = %q, want fooval", string(it.Value))
	}
	if it.Flags != 123 {
		t.Errorf("get(foo) Flags = %v, want 123", it.Flags)
	}

	// Add
	bar := &Item{Key: "bar", Value: []byte("barval")}
	err = c.Add(bar)
	checkErr(err, "first add(bar): %v", err)
	if err := c.Add(bar); err != ErrNotStored {
		t.Fatalf("second add(bar) want ErrNotStored, got %v", err)
	}

	// GetMulti
	m, err := c.GetMulti([]string{"foo", "bar"})
	checkErr(err, "GetMulti: %v", err)
	if g, e := len(m), 2; g != e {
		t.Errorf("GetMulti: got len(map) = %d, want = %d", g, e)
	}
	if _, ok := m["foo"]; !ok {
		t.Fatalf("GetMulti: didn't get key 'foo'")
	}
	if _, ok := m["bar"]; !ok {
		t.Fatalf("GetMulti: didn't get key 'bar'")
	}
	if g, e := string(m["foo"].Value), "fooval"; g != e {
		t.Errorf("GetMulti: foo: got %q, want %q", g, e)
	}
	if g, e := string(m["bar"].Value), "barval"; g != e {
		t.Errorf("GetMulti: bar: got %q, want %q", g, e)
	}

	// Delete
	err = c.Delete("foo")
	checkErr(err, "Delete: %v", err)
	_, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Errorf("post-Delete want ErrCacheMiss, got %v", err)
	}

	// Incr/Decr
	mustSet(&Item{Key: "num", Value: []byte("42")})
	n, err := c.Increment("num", 8)
	checkErr(err, "Increment num + 8: %v", err)
	if n != 50 {
		t.Fatalf("Increment num + 8: want=50, got=%d", n)
	}
	n, err = c.Decrement("num", 49)
	checkErr(err, "Decrement: %v", err)
	if n != 1 {
		t.Fatalf("Decrement 49: want=1, got=%d", n)
	}
	err = c.Delete("num")
	checkErr(err, "delete num: %v", err)
	n, err = c.Increment("num", 1)
	if err != ErrCacheMiss {
		t.Fatalf("increment post-delete: want ErrCacheMiss, got %v", err)
	}
	mustSet(&Item{Key: "num", Value: []byte("not-numeric")})
	n, err = c.Increment("num", 1)
	if err == nil || !strings.Contains(err.Error(), "client error") {
		t.Fatalf("increment non-number: want client error, got %v", err)
	}

}
<|endoftext|>


<commit_before>package main

import "strings"
import "errors"

type Member struct {
	Name   string
	Addr   string
	Role   string
	Status string
	Tags   map[string]string
}

type Members struct {
	Index map[string]int
	Data  []Member
}

// check if member already exists
func (mems *Members) CheckMember(name string) bool {
	index := mems.Index[name]
	if index >= 0 {
		if mems.Data[index].Name == name {
			return true
		}
	}
	return false
}

func (mems *Members) AddMember(name string, addr string, role string, status string, tags string) error {
	// process tags
	tag_list := strings.Split(tags, ",")
	len_tag_list := len(tag_list)
	// check if member already exists
	if mems.CheckMember(name) {
		// update new member info
		index := mems.Index[name]
		mems.Data[index].Name = name
		mems.Data[index].Addr = addr
		mems.Data[index].Status = status
		mems.Data[index].Role = role
		for i := 0; i < len_tag_list; i++ {
			tag := strings.Split(tag_list[i], "=")
			mems.Data[index].Tags[tag[0]] = tag[1]
		}
		return nil
	} else {
		// create new member
		new_member := Member{
			Name:   name,
			Addr:   addr,
			Role:   role,
			Status: status,
			Tags:   make(map[string]string),
		}
		for i := 0; i < len_tag_list; i++ {
			tag := strings.Split(tag_list[i], "=")
			new_member.Tags[tag[0]] = tag[1]
		}
		// register new member
		new_member_index := len(mems.Data) + 1
		mems.Index[new_member.Name] = new_member_index
		mems.Data = append(mems.Data, new_member)
		return nil
	}
}

func (mems *Members) RemoveMember(name string) error {
	index := mems.Index[name]
	if mems.CheckMember(name) {
		mems.Data = append(mems.Data[:index], mems.Data[index+1:]...)
		delete(mems.Index, name)
		return nil
	} else {
		return errors.New("No such member: " + name)
	}
}
<commit_msg>rewrite member structure<commit_after>package main

import "strings"
import "errors"

type Member struct {
	Name   string
	Addr   string
	Port int
	Tags   map[string]string
	Status string
	Protocol map[string]int
}

type SerfOutput string {
	Members []Member
}
<|endoftext|>


<commit_before>/* Copyright 2013 Steve Leonard <sleonard76@gmail.com>. All rights reserved.
Use of this source code is governed by the MIT license that can be found
in the LICENSE file.
*/

/* A fixed merkle tree implementation

Example use:

    package main

    import (
        "crypto/md5"
        "fmt"
        "github.com/xsleonard/go-merkle"
        "io/ioutil"
    )

    func splitData(data []byte, size int) [][]byte {
        // Splits data into an array of slices of len(size)
        count := len(data) / size
        blocks := make([][]byte, 0, count)
        for i := 0; i < count; i++ {
            block := data[i*size : (i+1)*size]
            blocks = append(blocks, block)
        }
        if len(data)%size != 0 {
            blocks = append(blocks, data[len(blocks)*size:])
        }
        return blocks
    }

    func main() {
        // Grab some data to make the tree out of, and partition
        data, err := ioutil.ReadFile("testdata") // assume testdata exists
        if err != nil {
            fmt.Println(err)
            return
        }
        blocks := splitData(data, 32)

        // Create & generate the tree
        tree := merkle.NewTree()
				// Create & generate the tree with sorted hashes
    		//tree := merkle.NewTreeWithOpts(TreeOptions{EnableHashSorting: true})
        err = tree.Generate(blocks, md5.New())
        if err != nil {
            fmt.Println(err)
            return
        }

        fmt.Printf("Height: %d\n", tree.Height())
        fmt.Printf("Root: %v\n", tree.Root())
        fmt.Printf("N Leaves: %v\n", len(tree.Leaves()))
        fmt.Printf("Height 2: %v\n", tree.GetNodesAtHeight(2))
    }
*/
package merkle

import (
	"errors"
	"hash"
	"bytes"
)

// Options that holds behavior changes
type TreeOptions struct {
	EnableHashSorting bool
}

// A node in the merkle tree
type Node struct {
	Hash  []byte
	Left  *Node
	Right *Node
}

// Creates a node given a hash function and data to hash
func NewNode(h hash.Hash, block []byte) (Node, error) {
	if h == nil || block == nil {
		return Node{}, nil
	}
	defer h.Reset()
	_, err := h.Write(block[:])
	if err != nil {
		return Node{}, err
	}
	return Node{Hash: h.Sum(nil)}, nil
}

// Contains all nodes
type Tree struct {
	// All nodes, linear
	Nodes []Node
	// Points to each level in the node. The first level contains the root node
	Levels [][]Node
	// Any particular behavior changing option
	Options TreeOptions
}

func NewTreeWithOpts(options TreeOptions) Tree {
	tree := NewTree()
	tree.Options = options
	return tree
}

func NewTree() Tree {
	return Tree{Nodes: nil, Levels: nil}
}

// Returns a slice of the leaf nodes in the tree, if available, else nil
func (self *Tree) Leaves() []Node {
	if self.Levels == nil {
		return nil
	} else {
		return self.Levels[len(self.Levels)-1]
	}
}

// Returns the root node of the tree, if available, else nil
func (self *Tree) Root() *Node {
	if self.Nodes == nil {
		return nil
	} else {
		return &self.Levels[0][0]
	}
}

// Returns all nodes at a given height, where height 1 returns a 1-element
// slice containing the root node, and a height of tree.Height() returns
// the leaves
func (self *Tree) GetNodesAtHeight(h uint64) []Node {
	if self.Levels == nil || h == 0 || h > uint64(len(self.Levels)) {
		return nil
	} else {
		return self.Levels[h-1]
	}
}

// Returns the height of this tree
func (self *Tree) Height() uint64 {
	return uint64(len(self.Levels))
}

// Generates the tree nodes
func (self *Tree) Generate(blocks [][]byte, hashf hash.Hash) error {
	blockCount := uint64(len(blocks))
	if blockCount == 0 {
		return errors.New("Empty tree")
	}
	height, nodeCount := CalculateHeightAndNodeCount(blockCount)
	levels := make([][]Node, height)
	nodes := make([]Node, nodeCount)

	// Create the leaf nodes
	for i, block := range blocks {
		node, err := NewNode(hashf, block)
		if err != nil {
			return err
		}
		nodes[i] = node
	}
	levels[height-1] = nodes[:len(blocks)]

	// Create each node level
	current := nodes[len(blocks):]
	h := height - 1
	for ; h > 0; h-- {
		below := levels[h]
		wrote, err := self.generateNodeLevel(below, current, hashf)
		if err != nil {
			return err
		}
		levels[h-1] = current[:wrote]
		current = current[wrote:]
	}

	self.Nodes = nodes
	self.Levels = levels
	return nil
}

// Creates all the non-leaf nodes for a certain height. The number of nodes
// is calculated to be 1/2 the number of nodes in the lower rung.  The newly
// created nodes will reference their Left and Right children.
// Returns the number of nodes added to current
func (self *Tree) generateNodeLevel(below []Node, current []Node,
	h hash.Hash) (uint64, error) {
	h.Reset()

	end := (len(below) + (len(below) % 2)) / 2
	for i := 0; i < end; i++ {
		// Concatenate the two children hashes and hash them, if both are
		// available, otherwise reuse the hash from the lone left node
		node := Node{}
		ileft := 2 * i
		iright := 2*i + 1
		left := &below[ileft]
		var right *Node = nil
		var rightHash []byte
		if len(below) > iright {
			right = &below[iright]
			rightHash = right.Hash
		}
		node, err := self.generateNode(below[ileft].Hash, rightHash, h)
		if err != nil {
			return 0, err
		}
		// Point the new node to its children and save
		node.Left = left
		node.Right = right
		current[i] = node

	}
	return uint64(end), nil
}

func (self *Tree) generateNode(left, right []byte, h hash.Hash) (Node, error) {
	data := make([]byte, h.Size()*2)
	bottomHash := left
	topHash := right
	if right == nil {
		b := data[:h.Size()]
		copy(b, left)
		return Node{Hash: b}, nil
	}
	if self.Options.EnableHashSorting {
		if bytes.Compare(left, right) > 0 {
			bottomHash = right
			topHash = left
		}
	}
	copy(data[:h.Size()], bottomHash)
	copy(data[h.Size():], topHash)

	return NewNode(h, data)
}

// Returns the height and number of nodes in an unbalanced binary tree given
// number of leaves
func CalculateHeightAndNodeCount(leaves uint64) (height, nodeCount uint64) {
	height = calculateTreeHeight(leaves)
	nodeCount = calculateNodeCount(height, leaves)
	return
}

// Calculates the number of nodes in a binary tree unbalanced strictly on
// the right side.  Height is assumed to be equal to
// calculateTreeHeight(size)
func calculateNodeCount(height, size uint64) uint64 {
	if isPowerOfTwo(size) {
		return 2*size - 1
	}
	count := size
	prev := size
	i := uint64(1)
	for ; i < height; i++ {
		next := (prev + (prev % 2)) / 2
		count += next
		prev = next
	}
	return count
}

// Returns the height of a full, complete binary tree given nodeCount nodes
func calculateTreeHeight(nodeCount uint64) uint64 {
	if nodeCount == 0 {
		return 0
	} else if nodeCount == 1 {
		return 2
	} else {
		return logBaseTwo(nextPowerOfTwo(nodeCount)) + 1
	}
}

// Returns true if n is a power of 2
func isPowerOfTwo(n uint64) bool {
	// http://graphics.stanford.edu/~seander/bithacks.html#DetermineIfPowerOf2
	return n != 0 && (n&(n-1)) == 0
}

// Returns the next highest power of 2 above n, if n is not already a
// power of 2
func nextPowerOfTwo(n uint64) uint64 {
	if n == 0 {
		return 1
	}
	// http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++
	return n
}

// Lookup table for integer log2 implementation
var log2lookup []uint64 = []uint64{
	0xFFFFFFFF00000000,
	0x00000000FFFF0000,
	0x000000000000FF00,
	0x00000000000000F0,
	0x000000000000000C,
	0x0000000000000002,
}

// Returns log2(n) assuming n is a power of 2
func logBaseTwo(x uint64) uint64 {
	if x == 0 {
		return 0
	}
	ct := uint64(0)
	for x != 0 {
		x >>= 1
		ct += 1
	}
	return ct - 1
}

// Returns the ceil'd log2 value of n
// See: http://stackoverflow.com/a/15327567
func ceilLogBaseTwo(x uint64) uint64 {
	y := uint64(1)
	if (x & (x - 1)) == 0 {
		y = 0
	}
	j := uint64(32)
	i := uint64(0)

	for ; i < 6; i++ {
		k := j
		if (x & log2lookup[i]) == 0 {
			k = 0
		}
		y += k
		x >>= k
		j >>= 1
	}

	return y
}
<commit_msg>Removing unused var<commit_after>/* Copyright 2013 Steve Leonard <sleonard76@gmail.com>. All rights reserved.
Use of this source code is governed by the MIT license that can be found
in the LICENSE file.
*/

/* A fixed merkle tree implementation

Example use:

    package main

    import (
        "crypto/md5"
        "fmt"
        "github.com/xsleonard/go-merkle"
        "io/ioutil"
    )

    func splitData(data []byte, size int) [][]byte {
        // Splits data into an array of slices of len(size)
        count := len(data) / size
        blocks := make([][]byte, 0, count)
        for i := 0; i < count; i++ {
            block := data[i*size : (i+1)*size]
            blocks = append(blocks, block)
        }
        if len(data)%size != 0 {
            blocks = append(blocks, data[len(blocks)*size:])
        }
        return blocks
    }

    func main() {
        // Grab some data to make the tree out of, and partition
        data, err := ioutil.ReadFile("testdata") // assume testdata exists
        if err != nil {
            fmt.Println(err)
            return
        }
        blocks := splitData(data, 32)

        // Create & generate the tree
        tree := merkle.NewTree()
				// Create & generate the tree with sorted hashes
    		//tree := merkle.NewTreeWithOpts(TreeOptions{EnableHashSorting: true})
        err = tree.Generate(blocks, md5.New())
        if err != nil {
            fmt.Println(err)
            return
        }

        fmt.Printf("Height: %d\n", tree.Height())
        fmt.Printf("Root: %v\n", tree.Root())
        fmt.Printf("N Leaves: %v\n", len(tree.Leaves()))
        fmt.Printf("Height 2: %v\n", tree.GetNodesAtHeight(2))
    }
*/
package merkle

import (
	"errors"
	"hash"
	"bytes"
)

// Options that holds behavior changes
type TreeOptions struct {
	EnableHashSorting bool
}

// A node in the merkle tree
type Node struct {
	Hash  []byte
	Left  *Node
	Right *Node
}

// Creates a node given a hash function and data to hash
func NewNode(h hash.Hash, block []byte) (Node, error) {
	if h == nil || block == nil {
		return Node{}, nil
	}
	defer h.Reset()
	_, err := h.Write(block[:])
	if err != nil {
		return Node{}, err
	}
	return Node{Hash: h.Sum(nil)}, nil
}

// Contains all nodes
type Tree struct {
	// All nodes, linear
	Nodes []Node
	// Points to each level in the node. The first level contains the root node
	Levels [][]Node
	// Any particular behavior changing option
	Options TreeOptions
}

func NewTreeWithOpts(options TreeOptions) Tree {
	tree := NewTree()
	tree.Options = options
	return tree
}

func NewTree() Tree {
	return Tree{Nodes: nil, Levels: nil}
}

// Returns a slice of the leaf nodes in the tree, if available, else nil
func (self *Tree) Leaves() []Node {
	if self.Levels == nil {
		return nil
	} else {
		return self.Levels[len(self.Levels)-1]
	}
}

// Returns the root node of the tree, if available, else nil
func (self *Tree) Root() *Node {
	if self.Nodes == nil {
		return nil
	} else {
		return &self.Levels[0][0]
	}
}

// Returns all nodes at a given height, where height 1 returns a 1-element
// slice containing the root node, and a height of tree.Height() returns
// the leaves
func (self *Tree) GetNodesAtHeight(h uint64) []Node {
	if self.Levels == nil || h == 0 || h > uint64(len(self.Levels)) {
		return nil
	} else {
		return self.Levels[h-1]
	}
}

// Returns the height of this tree
func (self *Tree) Height() uint64 {
	return uint64(len(self.Levels))
}

// Generates the tree nodes
func (self *Tree) Generate(blocks [][]byte, hashf hash.Hash) error {
	blockCount := uint64(len(blocks))
	if blockCount == 0 {
		return errors.New("Empty tree")
	}
	height, nodeCount := CalculateHeightAndNodeCount(blockCount)
	levels := make([][]Node, height)
	nodes := make([]Node, nodeCount)

	// Create the leaf nodes
	for i, block := range blocks {
		node, err := NewNode(hashf, block)
		if err != nil {
			return err
		}
		nodes[i] = node
	}
	levels[height-1] = nodes[:len(blocks)]

	// Create each node level
	current := nodes[len(blocks):]
	h := height - 1
	for ; h > 0; h-- {
		below := levels[h]
		wrote, err := self.generateNodeLevel(below, current, hashf)
		if err != nil {
			return err
		}
		levels[h-1] = current[:wrote]
		current = current[wrote:]
	}

	self.Nodes = nodes
	self.Levels = levels
	return nil
}

// Creates all the non-leaf nodes for a certain height. The number of nodes
// is calculated to be 1/2 the number of nodes in the lower rung.  The newly
// created nodes will reference their Left and Right children.
// Returns the number of nodes added to current
func (self *Tree) generateNodeLevel(below []Node, current []Node,
	h hash.Hash) (uint64, error) {
	h.Reset()

	end := (len(below) + (len(below) % 2)) / 2
	for i := 0; i < end; i++ {
		// Concatenate the two children hashes and hash them, if both are
		// available, otherwise reuse the hash from the lone left node
		ileft := 2 * i
		iright := 2*i + 1
		left := &below[ileft]
		var right *Node = nil
		var rightHash []byte
		if len(below) > iright {
			right = &below[iright]
			rightHash = right.Hash
		}
		node, err := self.generateNode(below[ileft].Hash, rightHash, h)
		if err != nil {
			return 0, err
		}
		// Point the new node to its children and save
		node.Left = left
		node.Right = right
		current[i] = node

	}
	return uint64(end), nil
}

func (self *Tree) generateNode(left, right []byte, h hash.Hash) (Node, error) {
	data := make([]byte, h.Size()*2)
	bottomHash := left
	topHash := right
	if right == nil {
		b := data[:h.Size()]
		copy(b, left)
		return Node{Hash: b}, nil
	}
	if self.Options.EnableHashSorting {
		if bytes.Compare(left, right) > 0 {
			bottomHash = right
			topHash = left
		}
	}
	copy(data[:h.Size()], bottomHash)
	copy(data[h.Size():], topHash)

	return NewNode(h, data)
}

// Returns the height and number of nodes in an unbalanced binary tree given
// number of leaves
func CalculateHeightAndNodeCount(leaves uint64) (height, nodeCount uint64) {
	height = calculateTreeHeight(leaves)
	nodeCount = calculateNodeCount(height, leaves)
	return
}

// Calculates the number of nodes in a binary tree unbalanced strictly on
// the right side.  Height is assumed to be equal to
// calculateTreeHeight(size)
func calculateNodeCount(height, size uint64) uint64 {
	if isPowerOfTwo(size) {
		return 2*size - 1
	}
	count := size
	prev := size
	i := uint64(1)
	for ; i < height; i++ {
		next := (prev + (prev % 2)) / 2
		count += next
		prev = next
	}
	return count
}

// Returns the height of a full, complete binary tree given nodeCount nodes
func calculateTreeHeight(nodeCount uint64) uint64 {
	if nodeCount == 0 {
		return 0
	} else if nodeCount == 1 {
		return 2
	} else {
		return logBaseTwo(nextPowerOfTwo(nodeCount)) + 1
	}
}

// Returns true if n is a power of 2
func isPowerOfTwo(n uint64) bool {
	// http://graphics.stanford.edu/~seander/bithacks.html#DetermineIfPowerOf2
	return n != 0 && (n&(n-1)) == 0
}

// Returns the next highest power of 2 above n, if n is not already a
// power of 2
func nextPowerOfTwo(n uint64) uint64 {
	if n == 0 {
		return 1
	}
	// http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++
	return n
}

// Lookup table for integer log2 implementation
var log2lookup []uint64 = []uint64{
	0xFFFFFFFF00000000,
	0x00000000FFFF0000,
	0x000000000000FF00,
	0x00000000000000F0,
	0x000000000000000C,
	0x0000000000000002,
}

// Returns log2(n) assuming n is a power of 2
func logBaseTwo(x uint64) uint64 {
	if x == 0 {
		return 0
	}
	ct := uint64(0)
	for x != 0 {
		x >>= 1
		ct += 1
	}
	return ct - 1
}

// Returns the ceil'd log2 value of n
// See: http://stackoverflow.com/a/15327567
func ceilLogBaseTwo(x uint64) uint64 {
	y := uint64(1)
	if (x & (x - 1)) == 0 {
		y = 0
	}
	j := uint64(32)
	i := uint64(0)

	for ; i < 6; i++ {
		k := j
		if (x & log2lookup[i]) == 0 {
			k = 0
		}
		y += k
		x >>= k
		j >>= 1
	}

	return y
}
<|endoftext|>


<commit_before>// Copyright 2014 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package mux

import (
	"errors"
	"fmt"
	"net/http"
	"strings"
	"sync"

	"github.com/issue9/context"
)

// 用于处理特定method的handler，定义了六组函数：
//  Add()    / AddFunc()
//  Get()    / GetFunc()
//  Post()   / PostFunc()
//  Delete() / DeleteFunc()
//  Put()    / PutFunc()
//  Any()    / AnyFunc()
//
//
//  m := mux.NewMethod()
//  m.MustGet(h1).
//    MustPost(h2).
//    MustAdd(h3, "GET", "POST")
//  http.ListenAndServe(m)
type Method struct {
	mu      sync.Mutex
	entries map[string]*methodEntries
}

type methodEntries struct {
	list  []*entry
	named map[string]*entry
}

// 声明一个新的Method
func NewMethod() *Method {
	return &Method{
		// 至少5个：get/post/delete/put/*
		entries: make(map[string]*methodEntries, 5),
	}
}

// 添加一条路由数据。
//
// pattern为路由匹配模式，可以是正则匹配也可以是字符串匹配，
// 以第一个字符来确定是否为一个正则匹配，若第一个字符为'?'，
// 则将'?'之后的所有字符当作一个正则表达式来匹配路由；
// 否则为一个普通的字符串匹配；若pattern以'\?'开头，则'\'仅当转换字符。
// methods参数应该只能为http.Request.Method中合法的字符串以及代表所有方法的"*"，
// 其它任何字符串都是无效的，但不会提示错误。
//
// 当methods或是h为空时，将返回错误信息。
func (m *Method) Add(pattern string, h http.Handler, methods ...string) error {
	if len(methods) == 0 {
		return errors.New("Add:请至少指定一个methods参数")
	}

	if h == nil {
		return errors.New("Add:参数h不能为空")
	}

	m.mu.Lock()
	defer m.mu.Unlock()

	for _, method := range methods {
		method = strings.ToUpper(method)

		entries, found := m.entries[method]
		if !found {
			entries = &methodEntries{
				list:  []*entry{},
				named: map[string]*entry{},
			}
			m.entries[method] = entries
		}

		if _, found = entries.named[pattern]; found {
			return fmt.Errorf("Add:该表达式[%v]已经存在", pattern)
		}

		entry, err := newEntry(pattern, h)
		if err != nil {
			return err
		}
		entries.list = append(entries.list, entry)
		entries.named[pattern] = entry
	} // end for methods

	return nil
}

// Get相当于m.Add(h, "GET")的简易写法
func (m *Method) Get(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "GET")
}

// Post相当于m.Add(h, "POST")的简易写法
func (m *Method) Post(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "POST")
}

// Delete相当于m.Add(h, "DELETE")的简易写法
func (m *Method) Delete(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "DELETE")
}

// Put相当于m.Add(h, "PUT")的简易写法
func (m *Method) Put(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "PUT")
}

// Any相当于m.Add(h, "*")的简易写法
func (m *Method) Any(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "*")
}

// 功能同Add()，但是将第二个参数从http.Handler换成了func(http.ResponseWriter, *http.Request)
func (m *Method) AddFunc(pattern string, fun func(http.ResponseWriter, *http.Request), methods ...string) error {
	return m.Add(pattern, http.HandlerFunc(fun), methods...)
}

// GetFunc相当于m.Add(h, "GET")的简易写法
func (m *Method) GetFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "GET")
}

// PutFunc相当于m.Add(h, "PUT")的简易写法
func (m *Method) PutFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "PUT")
}

// PostFunc相当于m.Add(h, "POST")的简易写法
func (m *Method) PostFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "POST")
}

// DeleteFunc相当于m.Add(h, "DELETE")的简易写法
func (m *Method) DeleteFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "DELETE")
}

// AnyFunc相当于m.Add(h, "*")的简易写法
func (m *Method) AnyFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "*")
}

// implement http.Handler.ServerHTTP()
func (m *Method) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	m.mu.Lock()
	defer m.mu.Unlock()

	// 找到与req.Method对应的map，若不存在，则尝试找*
	entries, found := m.entries[req.Method]
	if !found {
		entries, found = m.entries["*"]
		if !found { // 也不存在*
			panic("没有找到与之匹配的方法：" + req.Method)
		}
	}

	for _, entry := range entries.list {
		if ok, mapped := entry.match(req.URL.Path); ok {
			ctx := context.Get(req)
			ctx.Set("params", mapped)
			entry.handler.ServeHTTP(w, req)
			return
		}
	}
	panic("没有找到与之前匹配的路径：" + req.URL.Path)
}
<commit_msg>修正部分示例代码错误<commit_after>// Copyright 2014 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package mux

import (
	"errors"
	"fmt"
	"net/http"
	"strings"
	"sync"

	"github.com/issue9/context"
)

// 用于处理特定method的handler，定义了六组函数：
//  Add()    / AddFunc()
//  Get()    / GetFunc()
//  Post()   / PostFunc()
//  Delete() / DeleteFunc()
//  Put()    / PutFunc()
//  Any()    / AnyFunc()
//
//
//  m := mux.NewMethod()
//  m.Get(h1).
//    Post(h2).
//    Add(h3, "GET", "POST")
//  http.ListenAndServe(m)
type Method struct {
	mu      sync.Mutex
	entries map[string]*methodEntries
}

type methodEntries struct {
	list  []*entry
	named map[string]*entry
}

// 声明一个新的Method
func NewMethod() *Method {
	return &Method{
		// 至少5个：get/post/delete/put/*
		entries: make(map[string]*methodEntries, 5),
	}
}

// 添加一条路由数据。
//
// pattern为路由匹配模式，可以是正则匹配也可以是字符串匹配，
// 以第一个字符来确定是否为一个正则匹配，若第一个字符为'?'，
// 则将'?'之后的所有字符当作一个正则表达式来匹配路由；
// 否则为一个普通的字符串匹配；若pattern以'\?'开头，则'\'仅当转换字符。
// methods参数应该只能为http.Request.Method中合法的字符串以及代表所有方法的"*"，
// 其它任何字符串都是无效的，但不会提示错误。
//
// 当methods或是h为空时，将返回错误信息。
func (m *Method) Add(pattern string, h http.Handler, methods ...string) error {
	if len(methods) == 0 {
		return errors.New("Add:请至少指定一个methods参数")
	}

	if h == nil {
		return errors.New("Add:参数h不能为空")
	}

	m.mu.Lock()
	defer m.mu.Unlock()

	for _, method := range methods {
		method = strings.ToUpper(method)

		entries, found := m.entries[method]
		if !found {
			entries = &methodEntries{
				list:  []*entry{},
				named: map[string]*entry{},
			}
			m.entries[method] = entries
		}

		if _, found = entries.named[pattern]; found {
			return fmt.Errorf("Add:该表达式[%v]已经存在", pattern)
		}

		entry, err := newEntry(pattern, h)
		if err != nil {
			return err
		}
		entries.list = append(entries.list, entry)
		entries.named[pattern] = entry
	} // end for methods

	return nil
}

// Get相当于m.Add(h, "GET")的简易写法
func (m *Method) Get(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "GET")
}

// Post相当于m.Add(h, "POST")的简易写法
func (m *Method) Post(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "POST")
}

// Delete相当于m.Add(h, "DELETE")的简易写法
func (m *Method) Delete(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "DELETE")
}

// Put相当于m.Add(h, "PUT")的简易写法
func (m *Method) Put(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "PUT")
}

// Any相当于m.Add(h, "*")的简易写法
func (m *Method) Any(pattern string, h http.Handler) error {
	return m.Add(pattern, h, "*")
}

// 功能同Add()，但是将第二个参数从http.Handler换成了func(http.ResponseWriter, *http.Request)
func (m *Method) AddFunc(pattern string, fun func(http.ResponseWriter, *http.Request), methods ...string) error {
	return m.Add(pattern, http.HandlerFunc(fun), methods...)
}

// GetFunc相当于m.Add(h, "GET")的简易写法
func (m *Method) GetFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "GET")
}

// PutFunc相当于m.Add(h, "PUT")的简易写法
func (m *Method) PutFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "PUT")
}

// PostFunc相当于m.Add(h, "POST")的简易写法
func (m *Method) PostFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "POST")
}

// DeleteFunc相当于m.Add(h, "DELETE")的简易写法
func (m *Method) DeleteFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "DELETE")
}

// AnyFunc相当于m.Add(h, "*")的简易写法
func (m *Method) AnyFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) error {
	return m.AddFunc(pattern, fun, "*")
}

// implement http.Handler.ServerHTTP()
func (m *Method) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	m.mu.Lock()
	defer m.mu.Unlock()

	// 找到与req.Method对应的map，若不存在，则尝试找*
	entries, found := m.entries[req.Method]
	if !found {
		entries, found = m.entries["*"]
		if !found { // 也不存在*
			panic("没有找到与之匹配的方法：" + req.Method)
		}
	}

	for _, entry := range entries.list {
		if ok, mapped := entry.match(req.URL.Path); ok {
			ctx := context.Get(req)
			ctx.Set("params", mapped)
			entry.handler.ServeHTTP(w, req)
			return
		}
	}
	panic("没有找到与之前匹配的路径：" + req.URL.Path)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strings"
)

type Region struct {
	Situation   string `json:"situation"`
	Observation string `json:"observation"`
	WindAndSea  string `json:"ventEtMer"`
	Waves       string `json:"houle"`
	Visibility  string `json:"visi"`
	Confidence  string `json:"indice"`
}

type Echeance struct {
	Title string    `json:"titreEcheance"`
	Areas []*Region `json:"region"`
}

type Bulletin struct {
	Title     string      `json:"titreBulletin"`
	Special   string      `json:"bulletinSpecial"`
	Echeances []*Echeance `json:"echeance"`
}

func formatReport(b *Bulletin) string {
	buf := &bytes.Buffer{}
	w := func(format string, args ...interface{}) {
		fmt.Fprintf(buf, format, args...)
	}
	wif := func(s string) {
		if s != "" {
			s = strings.Replace(s, "
", "\n", -1)
			w("%s\n", s)
		}
	}
	w("%s\n", b.Title)
	w("Bulletin spécial: %s\n", b.Special)
	w("\n")
	for _, e := range b.Echeances {
		w("%s\n", e.Title)
		for _, r := range e.Areas {
			wif(r.Situation)
			wif(r.Observation)
			parts := strings.Split(r.WindAndSea, "MER :")
			if len(parts) == 2 {
				wif(parts[0])
				wif("MER :" + parts[1])
			} else {
				wif(r.WindAndSea)
			}
			wif(r.Waves)
			wif(r.Visibility)
			wif(r.Confidence)
		}
		w("\n")
	}
	return buf.String()
}

func fetchWeather() (string, error) {
	url := "http://www.meteofrance.com/mf3-rpc-portlet/rest/bulletins/cote/3/bulletinsMarineMetropole"
	rsp, err := http.Get(url)
	if err != nil {
		return "", fmt.Errorf("could not fetch weather: %s", err)
	}
	defer rsp.Body.Close()
	if rsp.StatusCode != 200 {
		return "", fmt.Errorf("failed to fetch weather: %s", rsp.Status)
	}
	reports := []*Bulletin{}
	err = json.NewDecoder(rsp.Body).Decode(&reports)
	if err != nil {
		return "", fmt.Errorf("could not decode json response: %s", err)
	}
	if len(reports) <= 0 {
		return "", fmt.Errorf("no report retrieved")
	}
	return formatReport(reports[0]), nil
}

func formatJsonWeather(w http.ResponseWriter, req *http.Request) {
	report, err := fetchWeather()
	w.Header().Set("Content-Type", "text/plain;charset=utf-8")
	if err != nil {
		w.WriteHeader(500)
		fmt.Fprintf(w, "error: %s\n", err)
		return
	}
	fmt.Fprintf(w, "%s", report)
}

func metmar(args []string) error {
	if len(args) != 1 {
		return fmt.Errorf("web server address expected")
	}
	addr := args[0]
	http.HandleFunc("/", formatJsonWeather)
	return http.ListenAndServe(addr, nil)
}

func main() {
	err := metmar(os.Args[1:])
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		os.Exit(1)
	}
}
<commit_msg>metmar: implement ETag<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strings"
)

type Region struct {
	Situation   string `json:"situation"`
	Observation string `json:"observation"`
	WindAndSea  string `json:"ventEtMer"`
	Waves       string `json:"houle"`
	Visibility  string `json:"visi"`
	Confidence  string `json:"indice"`
}

type Echeance struct {
	Title string    `json:"titreEcheance"`
	Areas []*Region `json:"region"`
}

type Bulletin struct {
	Title     string      `json:"titreBulletin"`
	Special   string      `json:"bulletinSpecial"`
	Echeances []*Echeance `json:"echeance"`
}

func formatReport(b *Bulletin) string {
	buf := &bytes.Buffer{}
	w := func(format string, args ...interface{}) {
		fmt.Fprintf(buf, format, args...)
	}
	wif := func(s string) {
		if s != "" {
			s = strings.Replace(s, "
", "\n", -1)
			w("%s\n", s)
		}
	}
	w("%s\n", b.Title)
	w("Bulletin spécial: %s\n", b.Special)
	w("\n")
	for _, e := range b.Echeances {
		w("%s\n", e.Title)
		for _, r := range e.Areas {
			wif(r.Situation)
			wif(r.Observation)
			parts := strings.Split(r.WindAndSea, "MER :")
			if len(parts) == 2 {
				wif(parts[0])
				wif("MER :" + parts[1])
			} else {
				wif(r.WindAndSea)
			}
			wif(r.Waves)
			wif(r.Visibility)
			wif(r.Confidence)
		}
		w("\n")
	}
	return buf.String()
}

func hashReport(report string) string {
	h := sha256.Sum256([]byte(report))
	return hex.EncodeToString(h[:])
}

func fetchWeather() (string, error) {
	url := "http://www.meteofrance.com/mf3-rpc-portlet/rest/bulletins/cote/3/bulletinsMarineMetropole"
	rsp, err := http.Get(url)
	if err != nil {
		return "", fmt.Errorf("could not fetch weather: %s", err)
	}
	defer rsp.Body.Close()
	if rsp.StatusCode != 200 {
		return "", fmt.Errorf("failed to fetch weather: %s", rsp.Status)
	}
	reports := []*Bulletin{}
	err = json.NewDecoder(rsp.Body).Decode(&reports)
	if err != nil {
		return "", fmt.Errorf("could not decode json response: %s", err)
	}
	if len(reports) <= 0 {
		return "", fmt.Errorf("no report retrieved")
	}
	return formatReport(reports[0]), nil
}

func formatJsonWeather(w http.ResponseWriter, req *http.Request) {
	report, err := fetchWeather()
	w.Header().Set("Content-Type", "text/plain;charset=utf-8")
	if err != nil {
		w.WriteHeader(500)
		fmt.Fprintf(w, "error: %s\n", err)
		return
	}
	h := hashReport(report)
	w.Header().Set("ETag", h)
	etag := req.Header.Get("If-None-Match")
	if etag == h {
		w.WriteHeader(304)
		return
	}
	fmt.Fprintf(w, "%s", report)
}

func metmar(args []string) error {
	if len(args) != 1 {
		return fmt.Errorf("web server address expected")
	}
	addr := args[0]
	http.HandleFunc("/", formatJsonWeather)
	return http.ListenAndServe(addr, nil)
}

func main() {
	err := metmar(os.Args[1:])
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Package minify relates MIME type to minifiers. Several minifiers are provided in the subpackages.
package minify // import "github.com/tdewolff/minify"

import (
	"errors"
	"io"
	"mime"
	"net/http"
	"net/url"
	"os/exec"
	"path"
	"regexp"
	"sync"

	"github.com/tdewolff/parse/v2"
	"github.com/tdewolff/parse/v2/buffer"
)

// ErrNotExist is returned when no minifier exists for a given mimetype.
var ErrNotExist = errors.New("minifier does not exist for mimetype")

////////////////////////////////////////////////////////////////

// MinifierFunc is a function that implements Minifer.
type MinifierFunc func(*M, io.Writer, io.Reader, map[string]string) error

// Minify calls f(m, w, r, params)
func (f MinifierFunc) Minify(m *M, w io.Writer, r io.Reader, params map[string]string) error {
	return f(m, w, r, params)
}

// Minifier is the interface for minifiers.
// The *M parameter is used for minifying embedded resources, such as JS within HTML.
type Minifier interface {
	Minify(*M, io.Writer, io.Reader, map[string]string) error
}

////////////////////////////////////////////////////////////////

type patternMinifier struct {
	pattern *regexp.Regexp
	Minifier
}

type cmdMinifier struct {
	cmd *exec.Cmd
}

func (c *cmdMinifier) Minify(_ *M, w io.Writer, r io.Reader, _ map[string]string) error {
	cmd := &exec.Cmd{}
	*cmd = *c.cmd // concurrency safety
	cmd.Stdout = w
	cmd.Stdin = r
	return cmd.Run()
}

////////////////////////////////////////////////////////////////

// M holds a map of mimetype => function to allow recursive minifier calls of the minifier functions.
type M struct {
	literal map[string]Minifier
	pattern []patternMinifier

	URL *url.URL
}

// New returns a new M.
func New() *M {
	return &M{
		map[string]Minifier{},
		[]patternMinifier{},
		nil,
	}
}

// Add adds a minifier to the mimetype => function map (unsafe for concurrent use).
func (m *M) Add(mimetype string, minifier Minifier) {
	m.literal[mimetype] = minifier
}

// AddFunc adds a minify function to the mimetype => function map (unsafe for concurrent use).
func (m *M) AddFunc(mimetype string, minifier MinifierFunc) {
	m.literal[mimetype] = minifier
}

// AddRegexp adds a minifier to the mimetype => function map (unsafe for concurrent use).
func (m *M) AddRegexp(pattern *regexp.Regexp, minifier Minifier) {
	m.pattern = append(m.pattern, patternMinifier{pattern, minifier})
}

// AddFuncRegexp adds a minify function to the mimetype => function map (unsafe for concurrent use).
func (m *M) AddFuncRegexp(pattern *regexp.Regexp, minifier MinifierFunc) {
	m.pattern = append(m.pattern, patternMinifier{pattern, minifier})
}

// AddCmd adds a minify function to the mimetype => function map (unsafe for concurrent use) that executes a command to process the minification.
// It allows the use of external tools like ClosureCompiler, UglifyCSS, etc. for a specific mimetype.
func (m *M) AddCmd(mimetype string, cmd *exec.Cmd) {
	m.literal[mimetype] = &cmdMinifier{cmd}
}

// AddCmdRegexp adds a minify function to the mimetype => function map (unsafe for concurrent use) that executes a command to process the minification.
// It allows the use of external tools like ClosureCompiler, UglifyCSS, etc. for a specific mimetype regular expression.
func (m *M) AddCmdRegexp(pattern *regexp.Regexp, cmd *exec.Cmd) {
	m.pattern = append(m.pattern, patternMinifier{pattern, &cmdMinifier{cmd}})
}

// Match returns the pattern and minifier that gets matched with the mediatype.
// It returns nil when no matching minifier exists.
// It has the same matching algorithm as Minify.
func (m *M) Match(mediatype string) (string, map[string]string, MinifierFunc) {
	mimetype, params := parse.Mediatype([]byte(mediatype))
	if minifier, ok := m.literal[string(mimetype)]; ok { // string conversion is optimized away
		return string(mimetype), params, minifier.Minify
	}

	for _, minifier := range m.pattern {
		if minifier.pattern.Match(mimetype) {
			return minifier.pattern.String(), params, minifier.Minify
		}
	}
	return string(mimetype), params, nil
}

// Minify minifies the content of a Reader and writes it to a Writer (safe for concurrent use).
// An error is returned when no such mimetype exists (ErrNotExist) or when an error occurred in the minifier function.
// Mediatype may take the form of 'text/plain', 'text/*', '*/*' or 'text/plain; charset=UTF-8; version=2.0'.
func (m *M) Minify(mediatype string, w io.Writer, r io.Reader) error {
	mimetype, params := parse.Mediatype([]byte(mediatype))
	return m.MinifyMimetype(mimetype, w, r, params)
}

// MinifyMimetype minifies the content of a Reader and writes it to a Writer (safe for concurrent use).
// It is a lower level version of Minify and requires the mediatype to be split up into mimetype and parameters.
// It is mostly used internally by minifiers because it is faster (no need to convert a byte-slice to string and vice versa).
func (m *M) MinifyMimetype(mimetype []byte, w io.Writer, r io.Reader, params map[string]string) error {
	err := ErrNotExist
	if minifier, ok := m.literal[string(mimetype)]; ok { // string conversion is optimized away
		err = minifier.Minify(m, w, r, params)
	} else {
		for _, minifier := range m.pattern {
			if minifier.pattern.Match(mimetype) {
				err = minifier.Minify(m, w, r, params)
				break
			}
		}
	}
	return err
}

// Bytes minifies an array of bytes (safe for concurrent use). When an error occurs it return the original array and the error.
// It returns an error when no such mimetype exists (ErrNotExist) or any error occurred in the minifier function.
func (m *M) Bytes(mediatype string, v []byte) ([]byte, error) {
	out := buffer.NewWriter(make([]byte, 0, len(v)))
	if err := m.Minify(mediatype, out, buffer.NewReader(v)); err != nil {
		return v, err
	}
	return out.Bytes(), nil
}

// String minifies a string (safe for concurrent use). When an error occurs it return the original string and the error.
// It returns an error when no such mimetype exists (ErrNotExist) or any error occurred in the minifier function.
func (m *M) String(mediatype string, v string) (string, error) {
	out := buffer.NewWriter(make([]byte, 0, len(v)))
	if err := m.Minify(mediatype, out, buffer.NewReader([]byte(v))); err != nil {
		return v, err
	}
	return string(out.Bytes()), nil
}

// Reader wraps a Reader interface and minifies the stream.
// Errors from the minifier are returned by the reader.
func (m *M) Reader(mediatype string, r io.Reader) io.Reader {
	pr, pw := io.Pipe()
	go func() {
		if err := m.Minify(mediatype, pw, r); err != nil {
			pw.CloseWithError(err)
		} else {
			pw.Close()
		}
	}()
	return pr
}

// minifyWriter makes sure that errors from the minifier are passed down through Close (can be blocking).
type minifyWriter struct {
	pw  *io.PipeWriter
	wg  sync.WaitGroup
	err error
}

// Write intercepts any writes to the writer.
func (w *minifyWriter) Write(b []byte) (int, error) {
	return w.pw.Write(b)
}

// Close must be called when writing has finished. It returns the error from the minifier.
func (w *minifyWriter) Close() error {
	w.pw.Close()
	w.wg.Wait()
	return w.err
}

// Writer wraps a Writer interface and minifies the stream.
// Errors from the minifier are returned by Close on the writer.
// The writer must be closed explicitly.
func (m *M) Writer(mediatype string, w io.Writer) *minifyWriter {
	pr, pw := io.Pipe()
	mw := &minifyWriter{pw, sync.WaitGroup{}, nil}
	mw.wg.Add(1)
	go func() {
		defer mw.wg.Done()

		if err := m.Minify(mediatype, w, pr); err != nil {
			io.Copy(w, pr)
			mw.err = err
		}
		pr.Close()
	}()
	return mw
}

// minifyResponseWriter wraps an http.ResponseWriter and makes sure that errors from the minifier are passed down through Close (can be blocking).
// All writes to the response writer are intercepted and minified on the fly.
// http.ResponseWriter loses all functionality such as Pusher, Hijacker, Flusher, ...
type minifyResponseWriter struct {
	http.ResponseWriter

	writer    *minifyWriter
	m         *M
	mediatype string
}

// WriteHeader intercepts any header writes and removes the Content-Length header.
func (w *minifyResponseWriter) WriteHeader(status int) {
	w.ResponseWriter.Header().Del("Content-Length")
	w.ResponseWriter.WriteHeader(status)
}

// Write intercepts any writes to the response writer.
// The first write will extract the Content-Type as the mediatype. Otherwise it falls back to the RequestURI extension.
func (w *minifyResponseWriter) Write(b []byte) (int, error) {
	if w.writer == nil {
		// first write
		if mediatype := w.ResponseWriter.Header().Get("Content-Type"); mediatype != "" {
			w.mediatype = mediatype
		}
		w.writer = w.m.Writer(w.mediatype, w.ResponseWriter)
	}
	return w.writer.Write(b)
}

// Close must be called when writing has finished. It returns the error from the minifier.
func (w *minifyResponseWriter) Close() error {
	if w.writer != nil {
		return w.writer.Close()
	}
	return nil
}

// ResponseWriter minifies any writes to the http.ResponseWriter.
// http.ResponseWriter loses all functionality such as Pusher, Hijacker, Flusher, ...
// Minification might be slower than just sending the original file! Caching is advised.
func (m *M) ResponseWriter(w http.ResponseWriter, r *http.Request) *minifyResponseWriter {
	mediatype := mime.TypeByExtension(path.Ext(r.RequestURI))
	return &minifyResponseWriter{w, nil, m, mediatype}
}

// Middleware provides a middleware function that minifies content on the fly by intercepting writes to http.ResponseWriter.
// http.ResponseWriter loses all functionality such as Pusher, Hijacker, Flusher, ...
// Minification might be slower than just sending the original file! Caching is advised.
func (m *M) Middleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mw := m.ResponseWriter(w, r)
		defer mw.Close()

		next.ServeHTTP(mw, r)
	})
}
<commit_msg>Make M concurrent-safe, fixes #249<commit_after>// Package minify relates MIME type to minifiers. Several minifiers are provided in the subpackages.
package minify // import "github.com/tdewolff/minify"

import (
	"errors"
	"io"
	"mime"
	"net/http"
	"net/url"
	"os/exec"
	"path"
	"regexp"
	"sync"

	"github.com/tdewolff/parse/v2"
	"github.com/tdewolff/parse/v2/buffer"
)

// ErrNotExist is returned when no minifier exists for a given mimetype.
var ErrNotExist = errors.New("minifier does not exist for mimetype")

////////////////////////////////////////////////////////////////

// MinifierFunc is a function that implements Minifer.
type MinifierFunc func(*M, io.Writer, io.Reader, map[string]string) error

// Minify calls f(m, w, r, params)
func (f MinifierFunc) Minify(m *M, w io.Writer, r io.Reader, params map[string]string) error {
	return f(m, w, r, params)
}

// Minifier is the interface for minifiers.
// The *M parameter is used for minifying embedded resources, such as JS within HTML.
type Minifier interface {
	Minify(*M, io.Writer, io.Reader, map[string]string) error
}

////////////////////////////////////////////////////////////////

type patternMinifier struct {
	pattern *regexp.Regexp
	Minifier
}

type cmdMinifier struct {
	cmd *exec.Cmd
}

func (c *cmdMinifier) Minify(_ *M, w io.Writer, r io.Reader, _ map[string]string) error {
	cmd := &exec.Cmd{}
	*cmd = *c.cmd // concurrency safety
	cmd.Stdout = w
	cmd.Stdin = r
	return cmd.Run()
}

////////////////////////////////////////////////////////////////

// M holds a map of mimetype => function to allow recursive minifier calls of the minifier functions.
type M struct {
	mutex   sync.RWMutex
	literal map[string]Minifier
	pattern []patternMinifier

	URL *url.URL
}

// New returns a new M.
func New() *M {
	return &M{
		sync.RWMutex{},
		map[string]Minifier{},
		[]patternMinifier{},
		nil,
	}
}

// Add adds a minifier to the mimetype => function map (unsafe for concurrent use).
func (m *M) Add(mimetype string, minifier Minifier) {
	m.mutex.Lock()
	m.literal[mimetype] = minifier
	m.mutex.Unlock()
}

// AddFunc adds a minify function to the mimetype => function map (unsafe for concurrent use).
func (m *M) AddFunc(mimetype string, minifier MinifierFunc) {
	m.mutex.Lock()
	m.literal[mimetype] = minifier
	m.mutex.Unlock()
}

// AddRegexp adds a minifier to the mimetype => function map (unsafe for concurrent use).
func (m *M) AddRegexp(pattern *regexp.Regexp, minifier Minifier) {
	m.mutex.Lock()
	m.pattern = append(m.pattern, patternMinifier{pattern, minifier})
	m.mutex.Unlock()
}

// AddFuncRegexp adds a minify function to the mimetype => function map (unsafe for concurrent use).
func (m *M) AddFuncRegexp(pattern *regexp.Regexp, minifier MinifierFunc) {
	m.mutex.Lock()
	m.pattern = append(m.pattern, patternMinifier{pattern, minifier})
	m.mutex.Unlock()
}

// AddCmd adds a minify function to the mimetype => function map (unsafe for concurrent use) that executes a command to process the minification.
// It allows the use of external tools like ClosureCompiler, UglifyCSS, etc. for a specific mimetype.
func (m *M) AddCmd(mimetype string, cmd *exec.Cmd) {
	m.mutex.Lock()
	m.literal[mimetype] = &cmdMinifier{cmd}
	m.mutex.Unlock()
}

// AddCmdRegexp adds a minify function to the mimetype => function map (unsafe for concurrent use) that executes a command to process the minification.
// It allows the use of external tools like ClosureCompiler, UglifyCSS, etc. for a specific mimetype regular expression.
func (m *M) AddCmdRegexp(pattern *regexp.Regexp, cmd *exec.Cmd) {
	m.mutex.Lock()
	m.pattern = append(m.pattern, patternMinifier{pattern, &cmdMinifier{cmd}})
	m.mutex.Unlock()
}

// Match returns the pattern and minifier that gets matched with the mediatype.
// It returns nil when no matching minifier exists.
// It has the same matching algorithm as Minify.
func (m *M) Match(mediatype string) (string, map[string]string, MinifierFunc) {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	mimetype, params := parse.Mediatype([]byte(mediatype))
	if minifier, ok := m.literal[string(mimetype)]; ok { // string conversion is optimized away
		return string(mimetype), params, minifier.Minify
	}

	for _, minifier := range m.pattern {
		if minifier.pattern.Match(mimetype) {
			return minifier.pattern.String(), params, minifier.Minify
		}
	}
	return string(mimetype), params, nil
}

// Minify minifies the content of a Reader and writes it to a Writer (safe for concurrent use).
// An error is returned when no such mimetype exists (ErrNotExist) or when an error occurred in the minifier function.
// Mediatype may take the form of 'text/plain', 'text/*', '*/*' or 'text/plain; charset=UTF-8; version=2.0'.
func (m *M) Minify(mediatype string, w io.Writer, r io.Reader) error {
	mimetype, params := parse.Mediatype([]byte(mediatype))
	return m.MinifyMimetype(mimetype, w, r, params)
}

// MinifyMimetype minifies the content of a Reader and writes it to a Writer (safe for concurrent use).
// It is a lower level version of Minify and requires the mediatype to be split up into mimetype and parameters.
// It is mostly used internally by minifiers because it is faster (no need to convert a byte-slice to string and vice versa).
func (m *M) MinifyMimetype(mimetype []byte, w io.Writer, r io.Reader, params map[string]string) error {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	err := ErrNotExist
	if minifier, ok := m.literal[string(mimetype)]; ok { // string conversion is optimized away
		err = minifier.Minify(m, w, r, params)
	} else {
		for _, minifier := range m.pattern {
			if minifier.pattern.Match(mimetype) {
				err = minifier.Minify(m, w, r, params)
				break
			}
		}
	}
	return err
}

// Bytes minifies an array of bytes (safe for concurrent use). When an error occurs it return the original array and the error.
// It returns an error when no such mimetype exists (ErrNotExist) or any error occurred in the minifier function.
func (m *M) Bytes(mediatype string, v []byte) ([]byte, error) {
	out := buffer.NewWriter(make([]byte, 0, len(v)))
	if err := m.Minify(mediatype, out, buffer.NewReader(v)); err != nil {
		return v, err
	}
	return out.Bytes(), nil
}

// String minifies a string (safe for concurrent use). When an error occurs it return the original string and the error.
// It returns an error when no such mimetype exists (ErrNotExist) or any error occurred in the minifier function.
func (m *M) String(mediatype string, v string) (string, error) {
	out := buffer.NewWriter(make([]byte, 0, len(v)))
	if err := m.Minify(mediatype, out, buffer.NewReader([]byte(v))); err != nil {
		return v, err
	}
	return string(out.Bytes()), nil
}

// Reader wraps a Reader interface and minifies the stream.
// Errors from the minifier are returned by the reader.
func (m *M) Reader(mediatype string, r io.Reader) io.Reader {
	pr, pw := io.Pipe()
	go func() {
		if err := m.Minify(mediatype, pw, r); err != nil {
			pw.CloseWithError(err)
		} else {
			pw.Close()
		}
	}()
	return pr
}

// minifyWriter makes sure that errors from the minifier are passed down through Close (can be blocking).
type minifyWriter struct {
	pw  *io.PipeWriter
	wg  sync.WaitGroup
	err error
}

// Write intercepts any writes to the writer.
func (w *minifyWriter) Write(b []byte) (int, error) {
	return w.pw.Write(b)
}

// Close must be called when writing has finished. It returns the error from the minifier.
func (w *minifyWriter) Close() error {
	w.pw.Close()
	w.wg.Wait()
	return w.err
}

// Writer wraps a Writer interface and minifies the stream.
// Errors from the minifier are returned by Close on the writer.
// The writer must be closed explicitly.
func (m *M) Writer(mediatype string, w io.Writer) *minifyWriter {
	pr, pw := io.Pipe()
	mw := &minifyWriter{pw, sync.WaitGroup{}, nil}
	mw.wg.Add(1)
	go func() {
		defer mw.wg.Done()

		if err := m.Minify(mediatype, w, pr); err != nil {
			io.Copy(w, pr)
			mw.err = err
		}
		pr.Close()
	}()
	return mw
}

// minifyResponseWriter wraps an http.ResponseWriter and makes sure that errors from the minifier are passed down through Close (can be blocking).
// All writes to the response writer are intercepted and minified on the fly.
// http.ResponseWriter loses all functionality such as Pusher, Hijacker, Flusher, ...
type minifyResponseWriter struct {
	http.ResponseWriter

	writer    *minifyWriter
	m         *M
	mediatype string
}

// WriteHeader intercepts any header writes and removes the Content-Length header.
func (w *minifyResponseWriter) WriteHeader(status int) {
	w.ResponseWriter.Header().Del("Content-Length")
	w.ResponseWriter.WriteHeader(status)
}

// Write intercepts any writes to the response writer.
// The first write will extract the Content-Type as the mediatype. Otherwise it falls back to the RequestURI extension.
func (w *minifyResponseWriter) Write(b []byte) (int, error) {
	if w.writer == nil {
		// first write
		if mediatype := w.ResponseWriter.Header().Get("Content-Type"); mediatype != "" {
			w.mediatype = mediatype
		}
		w.writer = w.m.Writer(w.mediatype, w.ResponseWriter)
	}
	return w.writer.Write(b)
}

// Close must be called when writing has finished. It returns the error from the minifier.
func (w *minifyResponseWriter) Close() error {
	if w.writer != nil {
		return w.writer.Close()
	}
	return nil
}

// ResponseWriter minifies any writes to the http.ResponseWriter.
// http.ResponseWriter loses all functionality such as Pusher, Hijacker, Flusher, ...
// Minification might be slower than just sending the original file! Caching is advised.
func (m *M) ResponseWriter(w http.ResponseWriter, r *http.Request) *minifyResponseWriter {
	mediatype := mime.TypeByExtension(path.Ext(r.RequestURI))
	return &minifyResponseWriter{w, nil, m, mediatype}
}

// Middleware provides a middleware function that minifies content on the fly by intercepting writes to http.ResponseWriter.
// http.ResponseWriter loses all functionality such as Pusher, Hijacker, Flusher, ...
// Minification might be slower than just sending the original file! Caching is advised.
func (m *M) Middleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mw := m.ResponseWriter(w, r)
		defer mw.Close()

		next.ServeHTTP(mw, r)
	})
}
<|endoftext|>


<commit_before>// Copyright 2012 The golibpcap Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !safe,!appengine

// The pcap package provides support for using the C pcap library from within a
// Go program.  Not all of the functions exported by <pacp.h> have been included
// in this package and more will be added on request.
//
package pcap

/*
#cgo LDFLAGS: -lpcap
#include <stdlib.h>
#include <pcap.h>
#include "libpcap.h"
*/
import "C"
import (
	"errors"
	"fmt"
	"sync"
	"time"
	"unsafe"

	"code.google.com/p/golibpcap/pcap/pkt"
	"code.google.com/p/golibpcap/pcap/stat"
	"code.google.com/p/golibpcap/trace"
)

var (
	ChanBuffSize    = 5000         // How many packets can we buffer in our channel.
	OptimizeFilters = 1            // Tells the bpf compiler to optimize filters.
	DefaultSnaplen  = int32(65535) // number of bytes to capture per packet.
	DefaultPromisc  = int32(1)     // 0->false, 1->true
	DefaultTimeout  = int32(0)     // ms 0->no timeout
)

//export goCallBack
func goCallBack(user *C.u_char, pkthdr_ptr *C.struct_pcap_pkthdr, buf_ptr *C.u_char) {
	p := (*Pcap)(unsafe.Pointer(user))
	p.m.Lock()
	packet := pkt.NewPacket(unsafe.Pointer(pkthdr_ptr), unsafe.Pointer(buf_ptr))
	p.m.Unlock()
	p.Pchan <- packet
	p.pktCnt++
}

// LibVersion returns information about the version of libpcap being used.
// Note that it contains more information than just a version number.
func LibVersion() string {
	return C.GoString(C.pcap_lib_version())
}

// Statustostr returns error strings for PCAP_ERROR_ and PCAP_WARNING_ values.
func Statustostr(errnum int32) string {
	return C.GoString(C.pcap_statustostr(C.int(errnum)))
}

// Pcap is the wrapper for the pcap_t struct in <pcap.h>.
type Pcap struct {
	FileName string           // Used for pcap_open_offline
	Device   string           // Used for pcap_open_live
	Snaplen  int32            // Specifies the maximum number of bytes to capture
	Promisc  int32            // 0->false, 1->true
	Timeout  int32            // ms
	Filters  []string         // track filters applied to the capture
	Pchan    chan *pkt.Packet // Channel for passing Packet pointers
	cptr     *C.pcap_t        // C Pointer to pcap_t
	pktCnt   uint32           // the number of packets captured
	m        *sync.Mutex      // Mutex to protect the packet memory for decode
}

// OpenOffline returns a *Pcap and opens it to read pcap packets from a save file.
func OpenOffline(file string) (*Pcap, error) {
	p := &Pcap{
		FileName: file,
		Pchan:    make(chan *pkt.Packet, ChanBuffSize),
		m:        &sync.Mutex{},
	}

	return p, p.OpenFile()
}

// OpenFile opens a savefile for reading.
func (p *Pcap) OpenFile() error {
	var buf *C.char
	buf = (*C.char)(C.calloc(C.PCAP_ERRBUF_SIZE, 1))
	defer C.free(unsafe.Pointer(buf))

	cf := C.CString(p.FileName)
	defer C.free(unsafe.Pointer(cf))

	p.cptr = C.pcap_open_offline(cf, buf)
	if p.cptr == nil {
		return errors.New(C.GoString(buf))
	}
	p.Snaplen = int32(C.pcap_snapshot(p.cptr))
	return nil
}

// Create will construct a pcap that can be used to set custom settings like a
// larger buffer.  The resulting Pcap must then be started with a call to
// Activate.  See Open for an example list of calls that should be made.
func Create(device string) (*Pcap, error) {
	p := &Pcap{
		Device: device,
		Pchan:  make(chan *pkt.Packet, ChanBuffSize),
		m:      &sync.Mutex{},
	}

	buf := (*C.char)(C.calloc(C.PCAP_ERRBUF_SIZE, 1))
	defer C.free(unsafe.Pointer(buf))

	dev := C.CString(p.Device)
	defer C.free(unsafe.Pointer(dev))

	p.cptr = C.pcap_create(dev, buf)
	if p.cptr == nil {
		return p, errors.New(C.GoString(buf))
	}
	return p, nil
}

// OpenLive returns a *Pcap and opens it to listen to live network traffic.
func OpenLive(device string, snaplen int32, promisc bool, timeout_ms int32) (*Pcap, error) {
	p := &Pcap{
		Device:  device,
		Snaplen: snaplen,
		Timeout: timeout_ms,
		Pchan:   make(chan *pkt.Packet, ChanBuffSize),
		m:       &sync.Mutex{},
	}
	if promisc {
		p.Promisc = 1
	}
	return p, p.Open()
}

// Open creates a packet capture descriptor to look at packets on the network.
// Calling  C.pcap_open_live is the equivalent of calling:
//	C.pcap_create
//	C.pcap_set_snaplen
//	C.pcap_set_promisc
//	C.pcap_set_timeout
//	C.pcap_activate
//
// In that order.  So if you want to use custom values for any thing that has
// to be set before pcap is active you should use Create instead of Open or
// OpenLive.
func (p *Pcap) Open() error {
	buf := (*C.char)(C.calloc(C.PCAP_ERRBUF_SIZE, 1))
	defer C.free(unsafe.Pointer(buf))

	dev := C.CString(p.Device)
	defer C.free(unsafe.Pointer(dev))

	p.cptr = C.pcap_open_live(dev, C.int(p.Snaplen), C.int(p.Promisc),
		C.int(p.Timeout), buf)
	if p.cptr == nil {
		return errors.New(C.GoString(buf))
	}
	return nil
}

// Close closes the files associated with p and deallocates C resources.
func (p *Pcap) Close() {
	if p.cptr != nil {
		p.m.Lock()
		C.pcap_close(p.cptr)
		p.m.Unlock()
	}
}

// Datalink returns the link layer type.
// For a list of possible DLT values see <pcap/bpf.h>.
func (p *Pcap) Datalink() int32 {
	return int32(C.pcap_datalink(p.cptr))
}

// GetErr returns an error based on the error text returned by pcap_geterr().
func (p *Pcap) GetErr() error {
	return errors.New(C.GoString(C.pcap_geterr(p.cptr)))
}

// SetSnaplen should only be called on a Pcap that was obtained through Create.
func (p *Pcap) SetSnaplen(snaplen int32) error {
	res := int32(C.pcap_set_snaplen(p.cptr, C.int(snaplen)))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	p.Snaplen = snaplen
	return nil
}

// SetPromisc should only be called on a Pcap that was obtained through Create.
func (p *Pcap) SetPromisc(promisc bool) error {
	pro := int32(0)
	if promisc {
		pro = int32(1)
	}
	res := int32(C.pcap_set_promisc(p.cptr, C.int(pro)))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	p.Promisc = pro
	return nil
}

// SetTimeout should only be called on a Pcap that was obtained through Create.
func (p *Pcap) SetTimeout(timeout_ms int32) error {
	res := int32(C.pcap_set_timeout(p.cptr, C.int(timeout_ms)))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	p.Timeout = timeout_ms
	return nil
}

// SetBufferSize should only be called on a Pcap that was obtained through
// Create.  If the buffer size is too small then Activate will fail with a
// generic PCAP_ERROR.
func (p *Pcap) SetBufferSize(bufferSize int32) error {
	// If you try to set a negative buffer size
	// you are going to have a bad time.
	if bufferSize < 0 {
		return fmt.Errorf("negative buffer size")
	}
	res := int32(C.pcap_set_buffer_size(p.cptr, C.int(bufferSize)))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	return nil
}

// Activate should only be called on a Pcap that was obtained through Create.
func (p *Pcap) Activate() error {
	res := int32(C.pcap_activate(p.cptr))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	return nil
}

// Loop keeps reading packets until cnt packets are processed or an error occurs.
func (p *Pcap) Loop(cnt int) {
	C.pcap_loop(p.cptr, C.int(cnt), C.getCallback(), (*C.u_char)(unsafe.Pointer(p)))
	p.Pchan <- nil
}

// Listen will accumulate packets until it is stopped by a nil packet pointer.
func (p *Pcap) Listen(r chan *[]*pkt.Packet) {
	var b []*pkt.Packet
	for {
		packet := <-p.Pchan
		if packet == nil {
			break
		}
		b = append(b, packet)
	}
	r <- &b
}

// BreakLoop stops the reading of packets.
func (p *Pcap) BreakLoop() {
	C.pcap_breakloop(p.cptr)
}

// DelayBreakLoop stops the reading of packets after t seconds.
func (p *Pcap) DelayBreakLoop(t int) {
	time.Sleep(time.Duration(t) * time.Second)
	p.BreakLoop()
}

// Next is a wrapper for NextEx().
func (p *Pcap) Next() *pkt.Packet {
	rv, _ := p.NextEx()
	return rv
}

// NextEx reads the next packet and returns a success/failure indication:
//
//  1	the packet was read without problems
//  0	packets are being read from a live capture, and the timeout expired
// -1	an error occurred while reading the packet
// -2	packets are being read from a file, and there are no more packets to read
func (p *Pcap) NextEx() (*pkt.Packet, int32) {
	var pkthdr_ptr *C.struct_pcap_pkthdr
	var buf_ptr *C.u_char
	res := int32(C.pcap_next_ex(p.cptr, &pkthdr_ptr, &buf_ptr))
	if res == 1 {
		p.m.Lock()
		packet := pkt.NewPacket(unsafe.Pointer(pkthdr_ptr), unsafe.Pointer(buf_ptr))
		p.m.Unlock()
		p.pktCnt++
		return packet, res
	}
	return nil, res
}

// Getstats returns a filled in Stat struct.
func (p *Pcap) Getstats() (*stat.Stat, error) {
	var cs C.struct_pcap_stat
	res := C.pcap_stats(p.cptr, &cs)
	if res == C.PCAP_ERROR {
		return nil, p.GetErr()
	}

	s := &stat.Stat{
		Captured:  p.pktCnt,
		Received:  uint32(cs.ps_recv),
		Dropped:   uint32(cs.ps_drop),
		IfDropped: uint32(cs.ps_ifdrop),
	}
	return s, nil
}

// Setfilter compiles a filter string into a bpf program and sets the filter.
func (p *Pcap) Setfilter(expr string) error {
	cexpr := C.CString(expr)
	defer C.free(unsafe.Pointer(cexpr))

	var bpf C.struct_bpf_program
	res := C.pcap_compile(p.cptr, &bpf, cexpr, C.int(OptimizeFilters),
		C.PCAP_NETMASK_UNKNOWN)
	if res == C.PCAP_ERROR {
		return p.GetErr()
	}

	res = C.pcap_setfilter(p.cptr, &bpf)
	if res == C.PCAP_ERROR {
		C.pcap_freecode(&bpf)
		return p.GetErr()
	}

	p.Filters = append(p.Filters, expr)

	C.pcap_freecode(&bpf)
	return nil
}

// NewPktTrace is a beta function and should be treated as such.
func (p *Pcap) NewPktTrace(data *[]*pkt.Packet) (*trace.PktTrace, error) {
	t := &trace.PktTrace{
		Version:    trace.Version,
		LibVersion: LibVersion(),
		Date:       time.Now(),
		MetaPcap: &trace.MetaPcap{
			Device:   p.Device,
			FileName: p.FileName,
			Snaplen:  p.Snaplen,
			Promisc:  p.Promisc,
			Timeout:  p.Timeout,
			Filters:  make([]string, len(p.Filters)),
		},
		Data: data,
	}
	for i := range p.Filters {
		t.MetaPcap.Filters[i] = p.Filters[i]
	}
	var err error
	t.Stats, err = p.Getstats()
	return t, err
}
<commit_msg>Making default/safe configuration values constant.<commit_after>// Copyright 2012 The golibpcap Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !safe,!appengine

// The pcap package provides support for using the C pcap library from within a
// Go program.  Not all of the functions exported by <pacp.h> have been included
// in this package and more will be added on request.
//
package pcap

/*
#cgo LDFLAGS: -lpcap
#include <stdlib.h>
#include <pcap.h>
#include "libpcap.h"
*/
import "C"
import (
	"errors"
	"fmt"
	"sync"
	"time"
	"unsafe"

	"code.google.com/p/golibpcap/pcap/pkt"
	"code.google.com/p/golibpcap/pcap/stat"
	"code.google.com/p/golibpcap/trace"
)

// Safe default values.  These are probably far from optimal for most systems.
const (
	DefaultBuffer  = int32(102400) // buffer limit
	DefaultPromisc = int32(1)      // 0->false, 1->true
	DefaultSnaplen = int32(65535)  // number of bytes to capture per packet.
	DefaultTimeout = int32(0)      // ms 0->no timeout
)

var (
	ChanBuffSize    = 5000 // How many packets can we buffer in our channel.
	OptimizeFilters = 1    // Tells the bpf compiler to optimize filters.
)

//export goCallBack
func goCallBack(user *C.u_char, pkthdr_ptr *C.struct_pcap_pkthdr, buf_ptr *C.u_char) {
	p := (*Pcap)(unsafe.Pointer(user))
	p.m.Lock()
	packet := pkt.NewPacket(unsafe.Pointer(pkthdr_ptr), unsafe.Pointer(buf_ptr))
	p.m.Unlock()
	p.Pchan <- packet
	p.pktCnt++
}

// LibVersion returns information about the version of libpcap being used.
// Note that it contains more information than just a version number.
func LibVersion() string {
	return C.GoString(C.pcap_lib_version())
}

// Statustostr returns error strings for PCAP_ERROR_ and PCAP_WARNING_ values.
func Statustostr(errnum int32) string {
	return C.GoString(C.pcap_statustostr(C.int(errnum)))
}

// Pcap is the wrapper for the pcap_t struct in <pcap.h>.
type Pcap struct {
	FileName string           // Used for pcap_open_offline
	Device   string           // Used for pcap_open_live
	Snaplen  int32            // Specifies the maximum number of bytes to capture
	Promisc  int32            // 0->false, 1->true
	Timeout  int32            // ms
	Filters  []string         // track filters applied to the capture
	Pchan    chan *pkt.Packet // Channel for passing Packet pointers
	cptr     *C.pcap_t        // C Pointer to pcap_t
	pktCnt   uint32           // the number of packets captured
	m        *sync.Mutex      // Mutex to protect the packet memory for decode
}

// OpenOffline returns a *Pcap and opens it to read pcap packets from a save file.
func OpenOffline(file string) (*Pcap, error) {
	p := &Pcap{
		FileName: file,
		Pchan:    make(chan *pkt.Packet, ChanBuffSize),
		m:        &sync.Mutex{},
	}

	return p, p.OpenFile()
}

// OpenFile opens a savefile for reading.
func (p *Pcap) OpenFile() error {
	var buf *C.char
	buf = (*C.char)(C.calloc(C.PCAP_ERRBUF_SIZE, 1))
	defer C.free(unsafe.Pointer(buf))

	cf := C.CString(p.FileName)
	defer C.free(unsafe.Pointer(cf))

	p.cptr = C.pcap_open_offline(cf, buf)
	if p.cptr == nil {
		return errors.New(C.GoString(buf))
	}
	p.Snaplen = int32(C.pcap_snapshot(p.cptr))
	return nil
}

// Create will construct a pcap that can be used to set custom settings like a
// larger buffer.  The resulting Pcap must then be started with a call to
// Activate.  See Open for an example list of calls that should be made.
func Create(device string) (*Pcap, error) {
	p := &Pcap{
		Device: device,
		Pchan:  make(chan *pkt.Packet, ChanBuffSize),
		m:      &sync.Mutex{},
	}

	buf := (*C.char)(C.calloc(C.PCAP_ERRBUF_SIZE, 1))
	defer C.free(unsafe.Pointer(buf))

	dev := C.CString(p.Device)
	defer C.free(unsafe.Pointer(dev))

	p.cptr = C.pcap_create(dev, buf)
	if p.cptr == nil {
		return p, errors.New(C.GoString(buf))
	}
	return p, nil
}

// OpenLive returns a *Pcap and opens it to listen to live network traffic.
func OpenLive(device string, snaplen int32, promisc bool, timeout_ms int32) (*Pcap, error) {
	p := &Pcap{
		Device:  device,
		Snaplen: snaplen,
		Timeout: timeout_ms,
		Pchan:   make(chan *pkt.Packet, ChanBuffSize),
		m:       &sync.Mutex{},
	}
	if promisc {
		p.Promisc = 1
	}
	return p, p.Open()
}

// Open creates a packet capture descriptor to look at packets on the network.
// Calling  C.pcap_open_live is the equivalent of calling:
//	C.pcap_create
//	C.pcap_set_snaplen
//	C.pcap_set_promisc
//	C.pcap_set_timeout
//	C.pcap_activate
//
// In that order.  So if you want to use custom values for any thing that has
// to be set before pcap is active you should use Create instead of Open or
// OpenLive.
func (p *Pcap) Open() error {
	buf := (*C.char)(C.calloc(C.PCAP_ERRBUF_SIZE, 1))
	defer C.free(unsafe.Pointer(buf))

	dev := C.CString(p.Device)
	defer C.free(unsafe.Pointer(dev))

	p.cptr = C.pcap_open_live(dev, C.int(p.Snaplen), C.int(p.Promisc),
		C.int(p.Timeout), buf)
	if p.cptr == nil {
		return errors.New(C.GoString(buf))
	}
	return nil
}

// Close closes the files associated with p and deallocates C resources.
func (p *Pcap) Close() {
	if p.cptr != nil {
		p.m.Lock()
		C.pcap_close(p.cptr)
		p.m.Unlock()
	}
}

// Datalink returns the link layer type.
// For a list of possible DLT values see <pcap/bpf.h>.
func (p *Pcap) Datalink() int32 {
	return int32(C.pcap_datalink(p.cptr))
}

// GetErr returns an error based on the error text returned by pcap_geterr().
func (p *Pcap) GetErr() error {
	return errors.New(C.GoString(C.pcap_geterr(p.cptr)))
}

// SetSnaplen should only be called on a Pcap that was obtained through Create.
func (p *Pcap) SetSnaplen(snaplen int32) error {
	res := int32(C.pcap_set_snaplen(p.cptr, C.int(snaplen)))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	p.Snaplen = snaplen
	return nil
}

// SetPromisc should only be called on a Pcap that was obtained through Create.
func (p *Pcap) SetPromisc(promisc bool) error {
	pro := int32(0)
	if promisc {
		pro = int32(1)
	}
	res := int32(C.pcap_set_promisc(p.cptr, C.int(pro)))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	p.Promisc = pro
	return nil
}

// SetTimeout should only be called on a Pcap that was obtained through Create.
func (p *Pcap) SetTimeout(timeout_ms int32) error {
	res := int32(C.pcap_set_timeout(p.cptr, C.int(timeout_ms)))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	p.Timeout = timeout_ms
	return nil
}

// SetBufferSize should only be called on a Pcap that was obtained through
// Create.  If the buffer size is too small then Activate will fail with a
// generic PCAP_ERROR.
func (p *Pcap) SetBufferSize(bufferSize int32) error {
	// If you try to set a negative buffer size
	// you are going to have a bad time.
	if bufferSize < 0 {
		return fmt.Errorf("negative buffer size")
	}
	res := int32(C.pcap_set_buffer_size(p.cptr, C.int(bufferSize)))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	return nil
}

// Activate should only be called on a Pcap that was obtained through Create.
func (p *Pcap) Activate() error {
	res := int32(C.pcap_activate(p.cptr))
	if res < 0 {
		return fmt.Errorf("%s(errnum=%d)", Statustostr(res), res)
	}
	return nil
}

// Loop keeps reading packets until cnt packets are processed or an error occurs.
func (p *Pcap) Loop(cnt int) {
	C.pcap_loop(p.cptr, C.int(cnt), C.getCallback(), (*C.u_char)(unsafe.Pointer(p)))
	p.Pchan <- nil
}

// Listen will accumulate packets until it is stopped by a nil packet pointer.
func (p *Pcap) Listen(r chan *[]*pkt.Packet) {
	var b []*pkt.Packet
	for {
		packet := <-p.Pchan
		if packet == nil {
			break
		}
		b = append(b, packet)
	}
	r <- &b
}

// BreakLoop stops the reading of packets.
func (p *Pcap) BreakLoop() {
	C.pcap_breakloop(p.cptr)
}

// DelayBreakLoop stops the reading of packets after t seconds.
func (p *Pcap) DelayBreakLoop(t int) {
	time.Sleep(time.Duration(t) * time.Second)
	p.BreakLoop()
}

// Next is a wrapper for NextEx().
func (p *Pcap) Next() *pkt.Packet {
	rv, _ := p.NextEx()
	return rv
}

// NextEx reads the next packet and returns a success/failure indication:
//
//  1	the packet was read without problems
//  0	packets are being read from a live capture, and the timeout expired
// -1	an error occurred while reading the packet
// -2	packets are being read from a file, and there are no more packets to read
func (p *Pcap) NextEx() (*pkt.Packet, int32) {
	var pkthdr_ptr *C.struct_pcap_pkthdr
	var buf_ptr *C.u_char
	res := int32(C.pcap_next_ex(p.cptr, &pkthdr_ptr, &buf_ptr))
	if res == 1 {
		p.m.Lock()
		packet := pkt.NewPacket(unsafe.Pointer(pkthdr_ptr), unsafe.Pointer(buf_ptr))
		p.m.Unlock()
		p.pktCnt++
		return packet, res
	}
	return nil, res
}

// Getstats returns a filled in Stat struct.
func (p *Pcap) Getstats() (*stat.Stat, error) {
	var cs C.struct_pcap_stat
	res := C.pcap_stats(p.cptr, &cs)
	if res == C.PCAP_ERROR {
		return nil, p.GetErr()
	}

	s := &stat.Stat{
		Captured:  p.pktCnt,
		Received:  uint32(cs.ps_recv),
		Dropped:   uint32(cs.ps_drop),
		IfDropped: uint32(cs.ps_ifdrop),
	}
	return s, nil
}

// Setfilter compiles a filter string into a bpf program and sets the filter.
func (p *Pcap) Setfilter(expr string) error {
	cexpr := C.CString(expr)
	defer C.free(unsafe.Pointer(cexpr))

	var bpf C.struct_bpf_program
	res := C.pcap_compile(p.cptr, &bpf, cexpr, C.int(OptimizeFilters),
		C.PCAP_NETMASK_UNKNOWN)
	if res == C.PCAP_ERROR {
		return p.GetErr()
	}

	res = C.pcap_setfilter(p.cptr, &bpf)
	if res == C.PCAP_ERROR {
		C.pcap_freecode(&bpf)
		return p.GetErr()
	}

	p.Filters = append(p.Filters, expr)

	C.pcap_freecode(&bpf)
	return nil
}

// NewPktTrace is a beta function and should be treated as such.
func (p *Pcap) NewPktTrace(data *[]*pkt.Packet) (*trace.PktTrace, error) {
	t := &trace.PktTrace{
		Version:    trace.Version,
		LibVersion: LibVersion(),
		Date:       time.Now(),
		MetaPcap: &trace.MetaPcap{
			Device:   p.Device,
			FileName: p.FileName,
			Snaplen:  p.Snaplen,
			Promisc:  p.Promisc,
			Timeout:  p.Timeout,
			Filters:  make([]string, len(p.Filters)),
		},
		Data: data,
	}
	for i := range p.Filters {
		t.MetaPcap.Filters[i] = p.Filters[i]
	}
	var err error
	t.Stats, err = p.Getstats()
	return t, err
}
<|endoftext|>