<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package sensor provides sensor events from various movement sensors.
package sensor

import (
	"errors"
	"time"
)

// Type represents a sensor type.
type Type int

var sensorNames = map[Type]string{
	Accelerometer: "Accelerometer",
	Gyroscope:     "Gyrsocope",
	Magnetometer:  "Magnetometer",
}

// String returns the string representation of the sensor type.
func (t Type) String() string {
	if n, ok := sensorNames[t]; ok {
		return n
	}
	return "Unknown sensor"
}

var (
	Accelerometer = Type(0)
	Gyroscope     = Type(1)
	Magnetometer  = Type(2)
)

// Event represents a sensor event.
type Event struct {
	// Sensor is the type of the sensor the event is coming from.
	Sensor Type

	// Timestamp is a device specific event time in nanoseconds.
	// Timestamps are not Unix times, they represent a time that is
	// only valid for the device's default sensor.
	Timestamp int64

	// Data is the event data.
	//
	// If the event source is Accelerometer,
	//  - Data[0]: acceleration force in x axis in m/s^2
	//  - Data[1]: acceleration force in y axis in m/s^2
	//  - Data[2]: acceleration force in z axis in m/s^2
	//
	// If the event source is Gyroscope,
	//  - Data[0]: rate of rotation around the x axis in rad/s
	//  - Data[1]: rate of rotation around the y axis in rad/s
	//  - Data[2]: rate of rotation around the z axis in rad/s
	//
	// If the event source is Magnetometer,
	//  - Data[0]: force of gravity along the x axis in m/s^2
	//  - Data[1]: force of gravity along the y axis in m/s^2
	//  - Data[2]: force of gravity along the z axis in m/s^2
	//
	Data []float64
}

// Manager multiplexes sensor event data from various sensor sources.
type Manager struct {
	m *manager // platform-specific implementation of the underlying manager
}

// Enable enables a sensor with the specified delay rate.
// If there are multiple sensors of type t on the device, Enable uses
// the default one.
// If there is no default sensor of type t on the device, an error returned.
// Valid sensor types supported by this package are Accelerometer,
// Gyroscope, Magnetometer and Altimeter.
func (m *Manager) Enable(t Type, delay time.Duration) error {
	if m.m == nil {
		m.m = new(manager)
		m.m.initialize()
	}
	if t < 0 || int(t) >= len(sensorNames) {
		return errors.New("sensor: unknown sensor type")
	}
	return m.m.enable(t, delay)
}

// Disable disables to feed the manager with the specified sensor.
func (m *Manager) Disable(t Type) error {
	if m.m == nil {
		m.m = new(manager)
		m.m.initialize()
	}
	if t < 0 || int(t) >= len(sensorNames) {
		return errors.New("sensor: unknown sensor type")
	}
	return m.m.disable(t)
}

// Read reads a series of events from the manager.
// It may read up to len(e) number of events, but will return
// less events if timeout occurs.
func (m *Manager) Read(e []Event) (n int, err error) {
	if m.m == nil {
		m.m = new(manager)
		m.m.initialize()
	}
	return m.m.read(e)
}

// Close stops the manager and frees the related resources.
// Once Close is called, Manager becomes invalid to use.
func (m *Manager) Close() error {
	if m.m == nil {
		return nil
	}
	return m.m.close()
}
<commit_msg>sensor: make the global Type values consts instead of vars.<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package sensor provides sensor events from various movement sensors.
package sensor

import (
	"errors"
	"time"
)

// Type represents a sensor type.
type Type int

var sensorNames = map[Type]string{
	Accelerometer: "Accelerometer",
	Gyroscope:     "Gyrsocope",
	Magnetometer:  "Magnetometer",
}

// String returns the string representation of the sensor type.
func (t Type) String() string {
	if n, ok := sensorNames[t]; ok {
		return n
	}
	return "Unknown sensor"
}

const (
	Accelerometer = Type(0)
	Gyroscope     = Type(1)
	Magnetometer  = Type(2)
)

// Event represents a sensor event.
type Event struct {
	// Sensor is the type of the sensor the event is coming from.
	Sensor Type

	// Timestamp is a device specific event time in nanoseconds.
	// Timestamps are not Unix times, they represent a time that is
	// only valid for the device's default sensor.
	Timestamp int64

	// Data is the event data.
	//
	// If the event source is Accelerometer,
	//  - Data[0]: acceleration force in x axis in m/s^2
	//  - Data[1]: acceleration force in y axis in m/s^2
	//  - Data[2]: acceleration force in z axis in m/s^2
	//
	// If the event source is Gyroscope,
	//  - Data[0]: rate of rotation around the x axis in rad/s
	//  - Data[1]: rate of rotation around the y axis in rad/s
	//  - Data[2]: rate of rotation around the z axis in rad/s
	//
	// If the event source is Magnetometer,
	//  - Data[0]: force of gravity along the x axis in m/s^2
	//  - Data[1]: force of gravity along the y axis in m/s^2
	//  - Data[2]: force of gravity along the z axis in m/s^2
	//
	Data []float64
}

// Manager multiplexes sensor event data from various sensor sources.
type Manager struct {
	m *manager // platform-specific implementation of the underlying manager
}

// Enable enables a sensor with the specified delay rate.
// If there are multiple sensors of type t on the device, Enable uses
// the default one.
// If there is no default sensor of type t on the device, an error returned.
// Valid sensor types supported by this package are Accelerometer,
// Gyroscope, Magnetometer and Altimeter.
func (m *Manager) Enable(t Type, delay time.Duration) error {
	if m.m == nil {
		m.m = new(manager)
		m.m.initialize()
	}
	if t < 0 || int(t) >= len(sensorNames) {
		return errors.New("sensor: unknown sensor type")
	}
	return m.m.enable(t, delay)
}

// Disable disables to feed the manager with the specified sensor.
func (m *Manager) Disable(t Type) error {
	if m.m == nil {
		m.m = new(manager)
		m.m.initialize()
	}
	if t < 0 || int(t) >= len(sensorNames) {
		return errors.New("sensor: unknown sensor type")
	}
	return m.m.disable(t)
}

// Read reads a series of events from the manager.
// It may read up to len(e) number of events, but will return
// less events if timeout occurs.
func (m *Manager) Read(e []Event) (n int, err error) {
	if m.m == nil {
		m.m = new(manager)
		m.m.initialize()
	}
	return m.m.read(e)
}

// Close stops the manager and frees the related resources.
// Once Close is called, Manager becomes invalid to use.
func (m *Manager) Close() error {
	if m.m == nil {
		return nil
	}
	return m.m.close()
}
<|endoftext|>


<commit_before>package sgr

import (
	"fmt"
	"strconv"
	"strings"
)

func MustParse(format string) string {
	str, err := parse(true, false, format)
	if err != nil {
		panic(err)
	}
	return str
}

func MustParseln(format string) string {
	str, err := parse(true, true, format)
	if err != nil {
		panic(err)
	}
	return str
}

func MustParseWithoutReset(format string) string {
	str, err := parse(false, false, format)
	if err != nil {
		panic(err)
	}
	return str
}

func Parse(format string) (string, error) {
	return parse(true, false, format)
}

func Parseln(format string) (string, error) {
	return parse(true, true, format)
}

func ParseWithoutReset(format string) (string, error) {
	return parse(false, false, format)
}

func parse(reset bool, newline bool, format string) (string, error) {
	// Builder used to build the colored string.
	sb := new(sgrBuilder)

	// position in the parsing process
	pos := 0

	// index of the currenctly processing square bracket start
	idxStart := 0
	idxEnd := 0

	for {
		// Find next square bracket, break loop when none was found.
		relBlockOpen := strings.IndexRune(format[pos:], '[')
		if relBlockOpen == -1 {
			sb.appendString(format[pos:])
			break
		}
		idxStart = pos + relBlockOpen

		// Test for escaped square bracket
		if format[idxStart+1] == '[' {
			sb.appendString(format[pos : idxStart+1])
			pos = idxStart + 2
			continue
		}

		// Add skipped string (if any)
		if idxStart > pos { //idxStart > pos+1 ???
			sb.appendString(format[pos:idxStart])
		}

		// Find square bracket end
		relBlockClose := strings.IndexRune(format[idxStart:], ']')
		if relBlockClose == -1 {
			return "", fmt.Errorf("Opened square bracket never closed at pos %d. If you want a literal bracket escape it: [[", idxStart)
		}
		idxEnd = idxStart + relBlockClose

		// found a block
		block := format[idxStart+1 : idxEnd]
		fields := strings.Fields(block)
		for _, field := range fields {

			switch field {
			// Options
			case "reset":
				sb.appendSgr(Reset)
			case "fg-reset":
				sb.appendSgr(ResetForegroundColor)
			case "bg-reset":
				sb.appendSgr(ResetBackgroundColor)
			case "bold":
				sb.appendSgr(Bold)
			case "boldOff":
				sb.appendSgr(BoldOff)
			case "underline":
				sb.appendSgr(Underline)
			case "underlineOff":
				sb.appendSgr(UnderlineOff)
			case "blink":
				sb.appendSgr(Blink)
			case "blinkOff":
				sb.appendSgr(BlinkOff)
			case "imageNegative":
				sb.appendSgr(ImageNegative)
			case "imagePositive":
				sb.appendSgr(ImagePositive)
			case "framed":
				sb.appendSgr(Framed)
			case "encircled":
				sb.appendSgr(Encircled)
			case "framedEncircledOff":
				sb.appendSgr(FramedEncircledOff)
			case "overlined":
				sb.appendSgr(Overlined)
			case "overlinedOff":
				sb.appendSgr(OverlinedOff)

			// Foreground Colors
			case "fg-black":
				sb.appendSgr(FgBlack)
			case "fg-red":
				sb.appendSgr(FgRed)
			case "fg-green":
				sb.appendSgr(FgGreen)
			case "fg-yellow":
				sb.appendSgr(FgYellow)
			case "fg-blue":
				sb.appendSgr(FgBlue)
			case "fg-magenta":
				sb.appendSgr(FgMagenta)
			case "fg-cyan":
				sb.appendSgr(FgCyan)
			case "fg-grey":
				sb.appendSgr(FgGrey)
			case "fg-white":
				sb.appendSgr(FgWhite)

			// Background Colors
			case "bg-black":
				sb.appendSgr(BgBlack)
			case "bg-red":
				sb.appendSgr(BgRed)
			case "bg-green":
				sb.appendSgr(BgGreen)
			case "bg-yellow":
				sb.appendSgr(BgYellow)
			case "bg-blue":
				sb.appendSgr(BgBlue)
			case "bg-magenta":
				sb.appendSgr(BgMagenta)
			case "bg-cyan":
				sb.appendSgr(BgCyan)
			case "bg-grey":
				sb.appendSgr(BgGrey)
			case "bg-white":
				sb.appendSgr(BgWhite)

			// Not Found
			default:
				isFgColor := strings.HasPrefix(field, "fg-")
				isBgColor := strings.HasPrefix(field, "bg-")
				if isFgColor || isBgColor {
					// Get the color number from the fg- or bg- code
					clr, err := strconv.Atoi(field[3:])
					if err != nil {
						goto invalidBlockCode
					}
					if clr < 0 || clr > 255 {
						return "", fmt.Errorf("Invalid color code %s. Expecting 0-255 or a defined color.", field[3:])
					}

					// Add actual fg or bg color to sgrBuilder
					if isFgColor {
						sb.appendSgr(FgColor(clr))
					} else {
						sb.appendSgr(BgColor(clr))
					}
					continue // next field
				} else {
					// not valid
					goto invalidBlockCode
				}
			}

			continue
		invalidBlockCode:
			return "", fmt.Errorf("Invalid block code '%s' in block at position %d.", field, idxStart)

		}

		// Change starting position for next iteration.
		pos = idxEnd + 1
	}

	if reset {
		sb.appendSgr(Reset)
	}

	if newline {
		sb.appendString("\n")
	}

	return sb.string(), nil
}
<commit_msg>s/blockcode/code<commit_after>package sgr

import (
	"fmt"
	"strconv"
	"strings"
)

func MustParse(format string) string {
	str, err := parse(true, false, format)
	if err != nil {
		panic(err)
	}
	return str
}

func MustParseln(format string) string {
	str, err := parse(true, true, format)
	if err != nil {
		panic(err)
	}
	return str
}

func MustParseWithoutReset(format string) string {
	str, err := parse(false, false, format)
	if err != nil {
		panic(err)
	}
	return str
}

func Parse(format string) (string, error) {
	return parse(true, false, format)
}

func Parseln(format string) (string, error) {
	return parse(true, true, format)
}

func ParseWithoutReset(format string) (string, error) {
	return parse(false, false, format)
}

func parse(reset bool, newline bool, format string) (string, error) {
	// Builder used to build the colored string.
	sb := new(sgrBuilder)

	// position in the parsing process
	pos := 0

	// index of the currenctly processing square bracket start
	idxStart := 0
	idxEnd := 0

	for {
		// Find next square bracket, break loop when none was found.
		relBlockOpen := strings.IndexRune(format[pos:], '[')
		if relBlockOpen == -1 {
			sb.appendString(format[pos:])
			break
		}
		idxStart = pos + relBlockOpen

		// Test for escaped square bracket
		if format[idxStart+1] == '[' {
			sb.appendString(format[pos : idxStart+1])
			pos = idxStart + 2
			continue
		}

		// Add skipped string (if any)
		if idxStart > pos { //idxStart > pos+1 ???
			sb.appendString(format[pos:idxStart])
		}

		// Find square bracket end
		relBlockClose := strings.IndexRune(format[idxStart:], ']')
		if relBlockClose == -1 {
			return "", fmt.Errorf("Opened square bracket never closed at pos %d. If you want a literal bracket escape it: [[", idxStart)
		}
		idxEnd = idxStart + relBlockClose

		// found a block
		block := format[idxStart+1 : idxEnd]
		fields := strings.Fields(block)
		for _, field := range fields {

			switch field {
			// Options
			case "reset":
				sb.appendSgr(Reset)
			case "fg-reset":
				sb.appendSgr(ResetForegroundColor)
			case "bg-reset":
				sb.appendSgr(ResetBackgroundColor)
			case "bold":
				sb.appendSgr(Bold)
			case "boldOff":
				sb.appendSgr(BoldOff)
			case "underline":
				sb.appendSgr(Underline)
			case "underlineOff":
				sb.appendSgr(UnderlineOff)
			case "blink":
				sb.appendSgr(Blink)
			case "blinkOff":
				sb.appendSgr(BlinkOff)
			case "imageNegative":
				sb.appendSgr(ImageNegative)
			case "imagePositive":
				sb.appendSgr(ImagePositive)
			case "framed":
				sb.appendSgr(Framed)
			case "encircled":
				sb.appendSgr(Encircled)
			case "framedEncircledOff":
				sb.appendSgr(FramedEncircledOff)
			case "overlined":
				sb.appendSgr(Overlined)
			case "overlinedOff":
				sb.appendSgr(OverlinedOff)

			// Foreground Colors
			case "fg-black":
				sb.appendSgr(FgBlack)
			case "fg-red":
				sb.appendSgr(FgRed)
			case "fg-green":
				sb.appendSgr(FgGreen)
			case "fg-yellow":
				sb.appendSgr(FgYellow)
			case "fg-blue":
				sb.appendSgr(FgBlue)
			case "fg-magenta":
				sb.appendSgr(FgMagenta)
			case "fg-cyan":
				sb.appendSgr(FgCyan)
			case "fg-grey":
				sb.appendSgr(FgGrey)
			case "fg-white":
				sb.appendSgr(FgWhite)

			// Background Colors
			case "bg-black":
				sb.appendSgr(BgBlack)
			case "bg-red":
				sb.appendSgr(BgRed)
			case "bg-green":
				sb.appendSgr(BgGreen)
			case "bg-yellow":
				sb.appendSgr(BgYellow)
			case "bg-blue":
				sb.appendSgr(BgBlue)
			case "bg-magenta":
				sb.appendSgr(BgMagenta)
			case "bg-cyan":
				sb.appendSgr(BgCyan)
			case "bg-grey":
				sb.appendSgr(BgGrey)
			case "bg-white":
				sb.appendSgr(BgWhite)

			// Not Found
			default:
				isFgColor := strings.HasPrefix(field, "fg-")
				isBgColor := strings.HasPrefix(field, "bg-")
				if isFgColor || isBgColor {
					// Get the color number from the fg- or bg- code
					clr, err := strconv.Atoi(field[3:])
					if err != nil {
						goto invalidCode
					}
					if clr < 0 || clr > 255 {
						return "", fmt.Errorf("Invalid color code %s. Expecting 0-255 or a defined color.", field[3:])
					}

					// Add actual fg or bg color to sgrBuilder
					if isFgColor {
						sb.appendSgr(FgColor(clr))
					} else {
						sb.appendSgr(BgColor(clr))
					}
					continue // next field
				} else {
					// not valid
					goto invalidCode
				}
			}

			continue
		invalidCode:
			return "", fmt.Errorf("Invalid code '%s' in block at position %d.", field, idxStart)

		}

		// Change starting position for next iteration.
		pos = idxEnd + 1
	}

	if reset {
		sb.appendSgr(Reset)
	}

	if newline {
		sb.appendString("\n")
	}

	return sb.string(), nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"bufio"
	"fmt"
	"io"
	"regexp"
	"unicode/utf8"
)

func Parse(r io.Reader, name string) (Prog, error) {
	p := &parser{
		r:    bufio.NewReader(r),
		name: name,
		npos: position{
			line: 1,
			col:  1,
		},
	}
	p.push(&p.prog.Stmts)
	p.next()
	p.program()
	return p.prog, p.err
}

type parser struct {
	r    *bufio.Reader
	name string

	err error

	ltok Token
	tok  Token
	lval string
	val  string

	lpos position
	pos  position
	npos position

	prog  Prog
	stack []interface{}
}

type position struct {
	line int
	col  int
}

var reserved = map[rune]bool{
	'\n': true,
	'#':  true,
	'&':  true,
	'>':  true,
	'<':  true,
	'|':  true,
	';':  true,
	'(':  true,
	')':  true,
}

// like reserved, but these can appear in the middle of a word
var starters = map[rune]bool{
	'{': true,
	'}': true,
}

var space = map[rune]bool{
	' ':  true,
	'\t': true,
}

var identRe = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)

func (p *parser) readRune() (rune, error) {
	r, _, err := p.r.ReadRune()
	if err != nil {
		if err == io.EOF {
			p.eof()
		} else {
			p.errPass(err)
		}
		return 0, err
	}
	p.moveWith(r)
	return r, nil
}

func (p *parser) moveWith(r rune) {
	if r == '\n' {
		p.npos.line++
		p.npos.col = 1
	} else {
		p.npos.col++
	}
}

func (p *parser) unreadRune() {
	if err := p.r.UnreadRune(); err != nil {
		panic(err)
	}
}

func (p *parser) readOnly(wanted rune) bool {
	r, _, err := p.r.ReadRune()
	if r == wanted {
		p.moveWith(r)
		return true
	}
	if err == nil {
		p.unreadRune()
	}
	return false
}

func (p *parser) next() {
	p.lpos = p.pos
	r := ' '
	var err error
	for space[r] {
		r, err = p.readRune()
		if err != nil {
			return
		}
	}
	p.pos = p.npos
	p.pos.col--
	if r == '\\' && p.readOnly('\n') {
		p.next()
		return
	}
	p.lval = p.val
	if reserved[r] || starters[r] {
		switch r {
		case '#':
			p.advance(COMMENT, p.readUpTo('\n'))
		case '\n':
			p.advance('\n', "")
		default:
			p.setTok(p.doToken(r))
		}
		return
	}
	rs := []rune{r}
	q := rune(0)
	if r == '"' || r == '\'' {
		q = r
	}
	for {
		r, err = p.readRune()
		if err != nil {
			if q != 0 {
				p.errWanted(Token(q))
			}
			break
		}
		if q != 0 {
			if q == '"' && r == '\\' {
				rs = append(rs, r)
				r, _ = p.readRune()
			} else if r == q {
				q = 0
			}
		} else if r == '"' || r == '\'' {
			q = r
		} else if reserved[r] || space[r] {
			p.npos.col--
			p.unreadRune()
			break
		}
		rs = append(rs, r)
	}
	p.setTok(WORD)
	p.val = string(rs)
}

func (p *parser) doToken(r rune) Token {
	switch r {
	case '&':
		if p.readOnly('&') {
			return LAND
		}
		return AND
	case '|':
		if p.readOnly('|') {
			return LOR
		}
		return OR
	case '(':
		return LPAREN
	case '{':
		return LBRACE
	case ')':
		return RPAREN
	case '}':
		return RBRACE
	case ';':
		if p.readOnly(';') {
			return DSEMICOLON
		}
		return SEMICOLON
	case '<':
		return LSS
	case '>':
		if p.readOnly('>') {
			return SHR
		}
		return GTR
	default:
		return Token(r)
	}
}

func (p *parser) advance(tok Token, val string) {
	p.setTok(tok)
	p.lval = p.val
	p.val = val
}

func (p *parser) setTok(tok Token) {
	p.ltok = p.tok
	p.tok = tok
}

func (p *parser) eof() {
	p.advance(EOF, "EOF")
}

func (p *parser) readUpTo(delim byte) string {
	b, err := p.r.ReadBytes(delim)
	cont := b
	if err == io.EOF {
	} else if err != nil {
		p.errPass(err)
	} else {
		cont = cont[:len(b)-1]
	}
	p.npos.col += utf8.RuneCount(b)
	return string(cont)
}

// We can't simply have these as tokens as they can sometimes be valid
// words, e.g. `echo if`.
var reservedWords = map[Token]string{
	IF:    "if",
	THEN:  "then",
	ELIF:  "elif",
	ELSE:  "else",
	FI:    "fi",
	WHILE: "while",
	DO:    "do",
	DONE:  "done",
}

func (p *parser) peek(tok Token) bool {
	return p.tok == tok || (p.tok == WORD && p.val == reservedWords[tok])
}

func (p *parser) got(tok Token) bool {
	if p.peek(tok) {
		p.next()
		return true
	}
	return false
}

func (p *parser) want(tok Token) {
	if !p.peek(tok) {
		p.errWanted(tok)
		return
	}
	p.next()
}

func (p *parser) errPass(err error) {
	if p.err == nil {
		p.err = err
	}
	p.eof()
}

func (p *parser) posErr(pos position, format string, v ...interface{}) {
	prefix := fmt.Sprintf("%s:%d:%d: ", p.name, pos.line, pos.col)
	p.errPass(fmt.Errorf(prefix+format, v...))
}

func (p *parser) curErr(format string, v ...interface{}) {
	p.posErr(p.pos, format, v...)
}

func (p *parser) errWantedStr(s string) {
	if p.tok == EOF {
		p.pos = p.npos
	}
	p.curErr("unexpected token %s - wanted %s", p.tok, s)
}

func (p *parser) errWanted(tok Token) {
	p.errWantedStr(tok.String())
}

func (p *parser) errAfterStr(s string) {
	p.curErr("unexpected token %s after %s", p.tok, s)
}

func (p *parser) add(n Node) {
	cur := p.stack[len(p.stack)-1]
	switch x := cur.(type) {
	case *[]Node:
		*x = append(*x, n)
	case *Node:
		if *x != nil {
			panic("single node set twice")
		}
		*x = n
	default:
		panic("unknown type in the stack")
	}
}

func (p *parser) pop() {
	p.stack = p.stack[:len(p.stack)-1]
}

func (p *parser) push(v interface{}) {
	p.stack = append(p.stack, v)
}

func (p *parser) popAdd(n Node) {
	p.pop()
	p.add(n)
}

func (p *parser) program() {
	p.commands()
}

func (p *parser) commands(stop ...Token) (count int) {
	for p.tok != EOF {
		for _, tok := range stop {
			if p.peek(tok) {
				return
			}
		}
		p.command()
		count++
	}
	return
}

func (p *parser) command() {
	switch {
	case p.got(COMMENT):
		com := Comment{
			Text: p.lval,
		}
		p.add(com)
	case p.got('\n'), p.got(COMMENT):
		if p.tok != EOF {
			p.command()
		}
	case p.got(LPAREN):
		var sub Subshell
		p.push(&sub.Stmts)
		if p.commands(RPAREN) == 0 {
			p.errWantedStr("command")
		}
		p.want(RPAREN)
		p.popAdd(sub)
	case p.got(LBRACE):
		var bl Block
		p.push(&bl.Stmts)
		if p.commands(RBRACE) == 0 {
			p.errWantedStr("command")
		}
		p.want(RBRACE)
		p.popAdd(bl)
	case p.got(IF):
		var ifs IfStmt
		p.push(&ifs.Cond)
		p.command()
		p.pop()
		p.want(THEN)
		p.push(&ifs.ThenStmts)
		p.commands(FI, ELIF, ELSE)
		p.pop()
		p.push(&ifs.Elifs)
		for p.got(ELIF) {
			var elf Elif
			p.push(&elf.Cond)
			p.command()
			p.pop()
			p.want(THEN)
			p.push(&elf.ThenStmts)
			p.commands(FI, ELIF, ELSE)
			p.popAdd(elf)
		}
		if p.got(ELSE) {
			p.pop()
			p.push(&ifs.ElseStmts)
			p.commands(FI)
		}
		p.want(FI)
		p.popAdd(ifs)
	case p.got(WHILE):
		var whl WhileStmt
		p.push(&whl.Cond)
		p.command()
		p.pop()
		p.want(DO)
		p.push(&whl.DoStmts)
		p.commands(DONE)
		p.want(DONE)
		p.popAdd(whl)
	case p.got(WORD):
		var cmd Command
		p.push(&cmd.Args)
		p.add(Lit{Val: p.lval})
		first := p.lpos
	args:
		for p.tok != EOF {
			switch {
			case p.got(WORD):
				p.add(Lit{Val: p.lval})
			case p.got(AND):
				cmd.Background = true
				break args
			case p.got(LAND):
				p.binaryExpr(LAND, cmd)
				return
			case p.got(OR):
				p.binaryExpr(OR, cmd)
				return
			case p.got(LOR):
				p.binaryExpr(LOR, cmd)
				return
			case p.got(LPAREN):
				if !identRe.MatchString(p.lval) {
					p.posErr(first, "invalid func name %q", p.lval)
					break args
				}
				fun := FuncDecl{
					Name: Lit{Val: p.lval},
				}
				p.want(RPAREN)
				p.push(&fun.Body)
				p.command()
				p.pop()
				p.popAdd(fun)
				return
			case p.gotRedirect():
			case p.got(SEMICOLON):
				break args
			case p.got('\n'):
				break args
			default:
				p.errAfterStr("command")
			}
		}
		p.popAdd(cmd)
	default:
		p.errWantedStr("command")
	}
}

func (p *parser) binaryExpr(op Token, left Node) {
	b := BinaryExpr{Op: op}
	p.push(&b.Y)
	p.command()
	p.pop()
	b.X = left
	p.popAdd(b)
}

func (p *parser) gotRedirect() bool {
	var r Redirect
	switch {
	case p.got(GTR):
		r.Op = GTR
		if p.got(AND) {
			p.want(WORD)
			r.Obj = Lit{Val: "&" + p.lval}
		} else {
			p.want(WORD)
			r.Obj = Lit{Val: p.lval}
		}
	case p.got(SHR):
		r.Op = SHR
		p.want(WORD)
		r.Obj = Lit{Val: p.lval}
	case p.got(LSS):
		r.Op = LSS
		p.want(WORD)
		r.Obj = Lit{Val: p.lval}
	default:
		return false
	}
	p.add(r)
	return true
}
<commit_msg>Rename eof() to setEOF() for clarity<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"bufio"
	"fmt"
	"io"
	"regexp"
	"unicode/utf8"
)

func Parse(r io.Reader, name string) (Prog, error) {
	p := &parser{
		r:    bufio.NewReader(r),
		name: name,
		npos: position{
			line: 1,
			col:  1,
		},
	}
	p.push(&p.prog.Stmts)
	p.next()
	p.program()
	return p.prog, p.err
}

type parser struct {
	r    *bufio.Reader
	name string

	err error

	ltok Token
	tok  Token
	lval string
	val  string

	lpos position
	pos  position
	npos position

	prog  Prog
	stack []interface{}
}

type position struct {
	line int
	col  int
}

var reserved = map[rune]bool{
	'\n': true,
	'#':  true,
	'&':  true,
	'>':  true,
	'<':  true,
	'|':  true,
	';':  true,
	'(':  true,
	')':  true,
}

// like reserved, but these can appear in the middle of a word
var starters = map[rune]bool{
	'{': true,
	'}': true,
}

var space = map[rune]bool{
	' ':  true,
	'\t': true,
}

var identRe = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)

func (p *parser) readRune() (rune, error) {
	r, _, err := p.r.ReadRune()
	if err != nil {
		if err == io.EOF {
			p.setEOF()
		} else {
			p.errPass(err)
		}
		return 0, err
	}
	p.moveWith(r)
	return r, nil
}

func (p *parser) moveWith(r rune) {
	if r == '\n' {
		p.npos.line++
		p.npos.col = 1
	} else {
		p.npos.col++
	}
}

func (p *parser) unreadRune() {
	if err := p.r.UnreadRune(); err != nil {
		panic(err)
	}
}

func (p *parser) readOnly(wanted rune) bool {
	r, _, err := p.r.ReadRune()
	if r == wanted {
		p.moveWith(r)
		return true
	}
	if err == nil {
		p.unreadRune()
	}
	return false
}

func (p *parser) next() {
	p.lpos = p.pos
	r := ' '
	var err error
	for space[r] {
		r, err = p.readRune()
		if err != nil {
			return
		}
	}
	p.pos = p.npos
	p.pos.col--
	if r == '\\' && p.readOnly('\n') {
		p.next()
		return
	}
	p.lval = p.val
	if reserved[r] || starters[r] {
		switch r {
		case '#':
			p.advance(COMMENT, p.readUpTo('\n'))
		case '\n':
			p.advance('\n', "")
		default:
			p.setTok(p.doToken(r))
		}
		return
	}
	rs := []rune{r}
	q := rune(0)
	if r == '"' || r == '\'' {
		q = r
	}
	for {
		r, err = p.readRune()
		if err != nil {
			if q != 0 {
				p.errWanted(Token(q))
			}
			break
		}
		if q != 0 {
			if q == '"' && r == '\\' {
				rs = append(rs, r)
				r, _ = p.readRune()
			} else if r == q {
				q = 0
			}
		} else if r == '"' || r == '\'' {
			q = r
		} else if reserved[r] || space[r] {
			p.npos.col--
			p.unreadRune()
			break
		}
		rs = append(rs, r)
	}
	p.setTok(WORD)
	p.val = string(rs)
}

func (p *parser) doToken(r rune) Token {
	switch r {
	case '&':
		if p.readOnly('&') {
			return LAND
		}
		return AND
	case '|':
		if p.readOnly('|') {
			return LOR
		}
		return OR
	case '(':
		return LPAREN
	case '{':
		return LBRACE
	case ')':
		return RPAREN
	case '}':
		return RBRACE
	case ';':
		if p.readOnly(';') {
			return DSEMICOLON
		}
		return SEMICOLON
	case '<':
		return LSS
	case '>':
		if p.readOnly('>') {
			return SHR
		}
		return GTR
	default:
		return Token(r)
	}
}

func (p *parser) advance(tok Token, val string) {
	p.setTok(tok)
	p.lval = p.val
	p.val = val
}

func (p *parser) setTok(tok Token) {
	p.ltok = p.tok
	p.tok = tok
}

func (p *parser) setEOF() {
	p.advance(EOF, "EOF")
}

func (p *parser) readUpTo(delim byte) string {
	b, err := p.r.ReadBytes(delim)
	cont := b
	if err == io.EOF {
	} else if err != nil {
		p.errPass(err)
	} else {
		cont = cont[:len(b)-1]
	}
	p.npos.col += utf8.RuneCount(b)
	return string(cont)
}

// We can't simply have these as tokens as they can sometimes be valid
// words, e.g. `echo if`.
var reservedWords = map[Token]string{
	IF:    "if",
	THEN:  "then",
	ELIF:  "elif",
	ELSE:  "else",
	FI:    "fi",
	WHILE: "while",
	DO:    "do",
	DONE:  "done",
}

func (p *parser) peek(tok Token) bool {
	return p.tok == tok || (p.tok == WORD && p.val == reservedWords[tok])
}

func (p *parser) got(tok Token) bool {
	if p.peek(tok) {
		p.next()
		return true
	}
	return false
}

func (p *parser) want(tok Token) {
	if !p.peek(tok) {
		p.errWanted(tok)
		return
	}
	p.next()
}

func (p *parser) errPass(err error) {
	if p.err == nil {
		p.err = err
	}
	p.setEOF()
}

func (p *parser) posErr(pos position, format string, v ...interface{}) {
	prefix := fmt.Sprintf("%s:%d:%d: ", p.name, pos.line, pos.col)
	p.errPass(fmt.Errorf(prefix+format, v...))
}

func (p *parser) curErr(format string, v ...interface{}) {
	p.posErr(p.pos, format, v...)
}

func (p *parser) errWantedStr(s string) {
	if p.tok == EOF {
		p.pos = p.npos
	}
	p.curErr("unexpected token %s - wanted %s", p.tok, s)
}

func (p *parser) errWanted(tok Token) {
	p.errWantedStr(tok.String())
}

func (p *parser) errAfterStr(s string) {
	p.curErr("unexpected token %s after %s", p.tok, s)
}

func (p *parser) add(n Node) {
	cur := p.stack[len(p.stack)-1]
	switch x := cur.(type) {
	case *[]Node:
		*x = append(*x, n)
	case *Node:
		if *x != nil {
			panic("single node set twice")
		}
		*x = n
	default:
		panic("unknown type in the stack")
	}
}

func (p *parser) pop() {
	p.stack = p.stack[:len(p.stack)-1]
}

func (p *parser) push(v interface{}) {
	p.stack = append(p.stack, v)
}

func (p *parser) popAdd(n Node) {
	p.pop()
	p.add(n)
}

func (p *parser) program() {
	p.commands()
}

func (p *parser) commands(stop ...Token) (count int) {
	for p.tok != EOF {
		for _, tok := range stop {
			if p.peek(tok) {
				return
			}
		}
		p.command()
		count++
	}
	return
}

func (p *parser) command() {
	switch {
	case p.got(COMMENT):
		com := Comment{
			Text: p.lval,
		}
		p.add(com)
	case p.got('\n'), p.got(COMMENT):
		if p.tok != EOF {
			p.command()
		}
	case p.got(LPAREN):
		var sub Subshell
		p.push(&sub.Stmts)
		if p.commands(RPAREN) == 0 {
			p.errWantedStr("command")
		}
		p.want(RPAREN)
		p.popAdd(sub)
	case p.got(LBRACE):
		var bl Block
		p.push(&bl.Stmts)
		if p.commands(RBRACE) == 0 {
			p.errWantedStr("command")
		}
		p.want(RBRACE)
		p.popAdd(bl)
	case p.got(IF):
		var ifs IfStmt
		p.push(&ifs.Cond)
		p.command()
		p.pop()
		p.want(THEN)
		p.push(&ifs.ThenStmts)
		p.commands(FI, ELIF, ELSE)
		p.pop()
		p.push(&ifs.Elifs)
		for p.got(ELIF) {
			var elf Elif
			p.push(&elf.Cond)
			p.command()
			p.pop()
			p.want(THEN)
			p.push(&elf.ThenStmts)
			p.commands(FI, ELIF, ELSE)
			p.popAdd(elf)
		}
		if p.got(ELSE) {
			p.pop()
			p.push(&ifs.ElseStmts)
			p.commands(FI)
		}
		p.want(FI)
		p.popAdd(ifs)
	case p.got(WHILE):
		var whl WhileStmt
		p.push(&whl.Cond)
		p.command()
		p.pop()
		p.want(DO)
		p.push(&whl.DoStmts)
		p.commands(DONE)
		p.want(DONE)
		p.popAdd(whl)
	case p.got(WORD):
		var cmd Command
		p.push(&cmd.Args)
		p.add(Lit{Val: p.lval})
		first := p.lpos
	args:
		for p.tok != EOF {
			switch {
			case p.got(WORD):
				p.add(Lit{Val: p.lval})
			case p.got(AND):
				cmd.Background = true
				break args
			case p.got(LAND):
				p.binaryExpr(LAND, cmd)
				return
			case p.got(OR):
				p.binaryExpr(OR, cmd)
				return
			case p.got(LOR):
				p.binaryExpr(LOR, cmd)
				return
			case p.got(LPAREN):
				if !identRe.MatchString(p.lval) {
					p.posErr(first, "invalid func name %q", p.lval)
					break args
				}
				fun := FuncDecl{
					Name: Lit{Val: p.lval},
				}
				p.want(RPAREN)
				p.push(&fun.Body)
				p.command()
				p.pop()
				p.popAdd(fun)
				return
			case p.gotRedirect():
			case p.got(SEMICOLON):
				break args
			case p.got('\n'):
				break args
			default:
				p.errAfterStr("command")
			}
		}
		p.popAdd(cmd)
	default:
		p.errWantedStr("command")
	}
}

func (p *parser) binaryExpr(op Token, left Node) {
	b := BinaryExpr{Op: op}
	p.push(&b.Y)
	p.command()
	p.pop()
	b.X = left
	p.popAdd(b)
}

func (p *parser) gotRedirect() bool {
	var r Redirect
	switch {
	case p.got(GTR):
		r.Op = GTR
		if p.got(AND) {
			p.want(WORD)
			r.Obj = Lit{Val: "&" + p.lval}
		} else {
			p.want(WORD)
			r.Obj = Lit{Val: p.lval}
		}
	case p.got(SHR):
		r.Op = SHR
		p.want(WORD)
		r.Obj = Lit{Val: p.lval}
	case p.got(LSS):
		r.Op = LSS
		p.want(WORD)
		r.Obj = Lit{Val: p.lval}
	default:
		return false
	}
	p.add(r)
	return true
}
<|endoftext|>


<commit_before>package packer

import (
	"bytes"
	"cgl.tideland.biz/asserts"
	"os"
	"testing"
)

func testEnvironment() *Environment {
	config := &EnvironmentConfig{}
	config.Ui = &ReaderWriterUi{
		new(bytes.Buffer),
		new(bytes.Buffer),
	}

	return NewEnvironment(config)
}

func TestEnvironment_Cli_CallsRun(t *testing.T) {
	//_ := asserts.NewTestingAsserts(t, true)

	// TODO: Test that the call to `Run` is done with
	// proper arguments and such.
}

func TestEnvironment_DefaultCli_Empty(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	defaultEnv := testEnvironment()

	assert.Equal(defaultEnv.Cli([]string{}), 1, "CLI with no args")
}

func TestEnvironment_DefaultCli_Help(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	defaultEnv := testEnvironment()

	// Test the basic version options
	assert.Equal(defaultEnv.Cli([]string{"--help"}), 1, "--help should print")
	assert.Equal(defaultEnv.Cli([]string{"-h"}), 1, "--help should print")
}

func TestEnvironment_DefaultCli_Version(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	defaultEnv := testEnvironment()

	// Test the basic version options
	assert.Equal(defaultEnv.Cli([]string{"version"}), 0, "version should work")
	assert.Equal(defaultEnv.Cli([]string{"--version"}), 0, "--version should work")
	assert.Equal(defaultEnv.Cli([]string{"-v"}), 0, "-v should work")

	// Test the --version and -v can appear anywhere
	assert.Equal(defaultEnv.Cli([]string{"bad", "-v"}), 0, "-v should work anywhere")
	assert.Equal(defaultEnv.Cli([]string{"bad", "--version"}), 0, "--version should work anywhere")

	// Test that "version" can't appear anywhere
	assert.Equal(defaultEnv.Cli([]string{"bad", "version"}), 1, "version should NOT work anywhere")
}

func TestEnvironment_DefaultUi(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	defaultEnv := NewEnvironment(nil)
	assert.NotNil(defaultEnv.Ui(), "default UI should not be nil")

	rwUi, ok := defaultEnv.Ui().(*ReaderWriterUi)
	assert.True(ok, "default UI should be ReaderWriterUi")
	assert.Equal(rwUi.Writer, os.Stdout, "default UI should go to stdout")
	assert.Equal(rwUi.Reader, os.Stdin, "default UI should read from stdin")
}

func TestEnvironment_PrintHelp(t *testing.T) {
	// Just call the function and verify that no panics occur
	testEnvironment().PrintHelp()
}

func TestEnvironment_SettingUi(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	ui := &ReaderWriterUi{new(bytes.Buffer), new(bytes.Buffer)}

	config := &EnvironmentConfig{}
	config.Ui = ui

	env := NewEnvironment(config)

	assert.Equal(env.Ui(), ui, "UIs should be equal")
}
<commit_msg>Test the CLI that Run is called on custom commands<commit_after>package packer

import (
	"bytes"
	"cgl.tideland.biz/asserts"
	"os"
	"testing"
)

type TestCommand struct {
	runCalled bool
}

func (tc *TestCommand) Run(env *Environment, args []string) int {
	tc.runCalled = true
	return 0
}

func (tc *TestCommand) Synopsis() string {
	return ""
}

func testEnvironment() *Environment {
	config := &EnvironmentConfig{}
	config.Ui = &ReaderWriterUi{
		new(bytes.Buffer),
		new(bytes.Buffer),
	}

	return NewEnvironment(config)
}

func TestEnvironment_Cli_CallsRun(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	command := &TestCommand{}

	config := &EnvironmentConfig{}
	config.Command = make(map[string]Command)
	config.Command["foo"] = command

	env := NewEnvironment(config)
	assert.Equal(env.Cli([]string{"foo"}), 0, "runs foo command")
	assert.True(command.runCalled, "run should've been called")
}

func TestEnvironment_DefaultCli_Empty(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	defaultEnv := testEnvironment()

	assert.Equal(defaultEnv.Cli([]string{}), 1, "CLI with no args")
}

func TestEnvironment_DefaultCli_Help(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	defaultEnv := testEnvironment()

	// Test the basic version options
	assert.Equal(defaultEnv.Cli([]string{"--help"}), 1, "--help should print")
	assert.Equal(defaultEnv.Cli([]string{"-h"}), 1, "--help should print")
}

func TestEnvironment_DefaultCli_Version(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	defaultEnv := testEnvironment()

	// Test the basic version options
	assert.Equal(defaultEnv.Cli([]string{"version"}), 0, "version should work")
	assert.Equal(defaultEnv.Cli([]string{"--version"}), 0, "--version should work")
	assert.Equal(defaultEnv.Cli([]string{"-v"}), 0, "-v should work")

	// Test the --version and -v can appear anywhere
	assert.Equal(defaultEnv.Cli([]string{"bad", "-v"}), 0, "-v should work anywhere")
	assert.Equal(defaultEnv.Cli([]string{"bad", "--version"}), 0, "--version should work anywhere")

	// Test that "version" can't appear anywhere
	assert.Equal(defaultEnv.Cli([]string{"bad", "version"}), 1, "version should NOT work anywhere")
}

func TestEnvironment_DefaultUi(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	defaultEnv := NewEnvironment(nil)
	assert.NotNil(defaultEnv.Ui(), "default UI should not be nil")

	rwUi, ok := defaultEnv.Ui().(*ReaderWriterUi)
	assert.True(ok, "default UI should be ReaderWriterUi")
	assert.Equal(rwUi.Writer, os.Stdout, "default UI should go to stdout")
	assert.Equal(rwUi.Reader, os.Stdin, "default UI should read from stdin")
}

func TestEnvironment_PrintHelp(t *testing.T) {
	// Just call the function and verify that no panics occur
	testEnvironment().PrintHelp()
}

func TestEnvironment_SettingUi(t *testing.T) {
	assert := asserts.NewTestingAsserts(t, true)

	ui := &ReaderWriterUi{new(bytes.Buffer), new(bytes.Buffer)}

	config := &EnvironmentConfig{}
	config.Ui = ui

	env := NewEnvironment(config)

	assert.Equal(env.Ui(), ui, "UIs should be equal")
}
<|endoftext|>


<commit_before><commit_msg>add CreateDirForFileMust()<commit_after><|endoftext|>


<commit_before>package server

import (
	"net/http"

	"github.com/heroku/busl/assets"
	"github.com/heroku/busl/broker"
	"github.com/heroku/busl/storage"
	"github.com/heroku/busl/util"
)

func handleError(w http.ResponseWriter, r *http.Request, err error) {
	if err == broker.ErrNotRegistered || err == storage.ErrNoStorage {
		message := "Channel is not registered."
		if r.Header.Get("Accept") == "text/ascii; version=feral" {
			message = assets.HttpCatGone
		}

		http.Error(w, message, http.StatusNotFound)

	} else if err != nil {
		util.CountWithData("server.handleError", 1, "error=%s", err.Error())
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
	}
}
<commit_msg>Use errNoContent to signal 204<commit_after>package server

import (
	"errors"
	"net/http"

	"github.com/heroku/busl/assets"
	"github.com/heroku/busl/broker"
	"github.com/heroku/busl/storage"
	"github.com/heroku/busl/util"
)

var errNoContent = errors.New("No Content")

func handleError(w http.ResponseWriter, r *http.Request, err error) {
	if err == broker.ErrNotRegistered || err == storage.ErrNoStorage {
		message := "Channel is not registered."
		if r.Header.Get("Accept") == "text/ascii; version=feral" {
			message = assets.HttpCatGone
		}

		http.Error(w, message, http.StatusNotFound)

	} else if err == errNoContent {
		// As indicated in the w3 spec[1] an SSE stream
		// that's already done should return a `204 No Content`
		// [1]: http://www.w3.org/TR/2012/WD-eventsource-20120426/
		w.WriteHeader(http.StatusNoContent)

	} else if err != nil {
		util.CountWithData("server.handleError", 1, "error=%s", err.Error())
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
	}
}
<|endoftext|>


<commit_before><commit_msg>Fix race updating piece priority when piece being read is unexpectedly unavailable<commit_after><|endoftext|>


<commit_before>package storage

import "testing"

/**
 * ZooKeeper Storage Manager Test
 *
 * @author: Anant Bhardwaj
 * @date: 05/11/2014
 */

func TestBasicLeaderModified(t *testing.T) {
  servers := "localhost:2181"
  sm := MakeStorageManager()
  sm.Open(servers)
  defer sm.Close()
  sm.Create("/test", "")
  sm.Create("/test/_var_tmp_824-1000_px-12568-basic-1", "")
  sm.Create("/test/_var_tmp_824-1000_px-12568-basic-1/store", "")
  sm.Create("/test/_var_tmp_824-1000_px-12568-basic-1/done", "")
  sm.Create("/test/_var_tmp_824-1000_px-12568-basic-1/store/0", "6.824")

  _ = sm.Write("/test/_var_tmp_824-1000_px-12568-basic-1/store/0", "6.824")
  data, _ := sm.Read("/test/_var_tmp_824-1000_px-12568-basic-1/store/0")
  if data != "6.824" {
    t.Fatalf("got=%v wanted=%v", data, "6.824")
  }

  sm.Close(servers)
  sm.Open(servers)
  data, _ := sm.Read("/test/_var_tmp_824-1000_px-12568-basic-1/store/0")
  if data != "6.824" {
    t.Fatalf("got=%v wanted=%v", data, "6.824")
  }
}

<commit_msg>more tests<commit_after>package storage

import "testing"

/**
 * ZooKeeper Storage Manager Test
 *
 * @author: Anant Bhardwaj
 * @date: 05/11/2014
 */

func TestBasicLeaderModified(t *testing.T) {
  servers := "localhost:2181"
  sm := MakeStorageManager()
  sm.Open(servers)
  defer sm.Close()
  sm.Create("/test", "")
  sm.Create("/test/_var_tmp_824-1000_px-12568-basic-1", "")
  sm.Create("/test/_var_tmp_824-1000_px-12568-basic-1/store", "")
  sm.Create("/test/_var_tmp_824-1000_px-12568-basic-1/done", "")
  sm.Create("/test/_var_tmp_824-1000_px-12568-basic-1/store/0", "6.824")

  _ = sm.Write("/test/_var_tmp_824-1000_px-12568-basic-1/store/0", "6.824")
  data, _ := sm.Read("/test/_var_tmp_824-1000_px-12568-basic-1/store/0")
  if data != "6.824" {
    t.Fatalf("got=%v wanted=%v", data, "6.824")
  }

  sm.Close()
  sm.Open(servers)
  data, _ := sm.Read("/test/_var_tmp_824-1000_px-12568-basic-1/store/0")
  if data != "6.824" {
    t.Fatalf("got=%v wanted=%v", data, "6.824")
  }
  sm.Close()
}

<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"context"
	"crypto/tls"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"os/signal"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/golang/glog"
	"github.com/livepeer/lpms/ffmpeg"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/status"

	"github.com/livepeer/go-livepeer/common"
	"github.com/livepeer/go-livepeer/core"
	"github.com/livepeer/go-livepeer/net"
)

const protoVerLPT = "Livepeer-Transcoder-1.0"
const transcodingErrorMimeType = "livepeer/transcoding-error"

var errSecret = errors.New("Invalid secret")
var errZeroCapacity = errors.New("Zero capacity")

// Standalone Transcoder

// RunTranscoder is main routing of standalone transcoder
// Exiting it will terminate executable
func RunTranscoder(n *core.LivepeerNode, orchAddr string, capacity int) {
	expb := backoff.NewExponentialBackOff()
	expb.MaxInterval = time.Minute
	expb.MaxElapsedTime = 0
	backoff.Retry(func() error {
		glog.Info("Registering transcoder to ", orchAddr)
		err := runTranscoder(n, orchAddr, capacity)
		glog.Info("Unregistering transcoder: ", err)
		if _, fatal := err.(core.RemoteTranscoderFatalError); fatal {
			glog.Info("Terminating transcoder because of ", err)
			// Returning nil here will make `backoff` to stop trying to reconnect and exit
			return nil
		}
		// By returning error we tell `backoff` to try to connect again
		return err
	}, expb)
}

func checkTranscoderError(err error) error {
	if err != nil {
		s := status.Convert(err)
		if s.Message() == errSecret.Error() { // consider this unrecoverable
			return core.NewRemoteTranscoderFatalError(errSecret)
		}
		if s.Message() == errZeroCapacity.Error() { // consider this unrecoverable
			return core.NewRemoteTranscoderFatalError(errZeroCapacity)
		}
		if status.Code(err) == codes.Canceled {
			return core.NewRemoteTranscoderFatalError(fmt.Errorf("Execution interrupted"))
		}
	}
	return err
}

func runTranscoder(n *core.LivepeerNode, orchAddr string, capacity int) error {
	tlsConfig := &tls.Config{InsecureSkipVerify: true}
	conn, err := grpc.Dial(orchAddr,
		grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
	if err != nil {
		glog.Error("Did not connect transcoder to orchesrator: ", err)
		return err
	}
	defer conn.Close()

	c := net.NewTranscoderClient(conn)
	ctx := context.Background()
	ctx, cancel := context.WithCancel(ctx)
	// Silence linter
	defer cancel()
	r, err := c.RegisterTranscoder(ctx, &net.RegisterRequest{Secret: n.OrchSecret, Capacity: int64(capacity)})
	if err := checkTranscoderError(err); err != nil {
		glog.Error("Could not register transcoder to orchestrator ", err)
		return err
	}

	// Catch interrupt signal to shut down transcoder
	exitc := make(chan os.Signal)
	signal.Notify(exitc, os.Interrupt, syscall.SIGTERM)
	defer signal.Stop(exitc)
	go func() {
		select {
		case sig := <-exitc:
			glog.Infof("Exiting Livepeer Transcoder: %v", sig)
			// Cancelling context will close connection to orchestrator
			cancel()
			return
		}
	}()

	httpc := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	var wg sync.WaitGroup
	for {
		notify, err := r.Recv()
		if err := checkTranscoderError(err); err != nil {
			glog.Infof(`End of stream receive cycle because of err="%v", waiting for running transcode jobs to complete`, err)
			wg.Wait()
			return err
		}
		wg.Add(1)
		go func() {
			runTranscode(n, orchAddr, httpc, notify)
			wg.Done()
		}()
	}
}

func runTranscode(n *core.LivepeerNode, orchAddr string, httpc *http.Client, notify *net.NotifySegment) {
	profiles := []ffmpeg.VideoProfile{}
	if len(notify.FullProfiles) > 0 {
		profiles = makeFfmpegVideoProfiles(notify.FullProfiles)
	} else if len(notify.Profiles) > 0 {
		prof, err := common.TxDataToVideoProfile(hex.EncodeToString(notify.Profiles))
		profiles = prof
		if err != nil {
			glog.Error("Unable to deserialize profiles ", err)
		}
	}

	glog.Infof("Transcoding taskId=%d url=%s", notify.TaskId, notify.Url)
	var contentType string
	var body bytes.Buffer

	tData, err := n.Transcoder.Transcode(notify.Job, notify.Url, profiles)
	glog.V(common.VERBOSE).Infof("Transcoding done for taskId=%d url=%s err=%v", notify.TaskId, notify.Url, err)
	if err != nil {
		glog.Error("Unable to transcode ", err)
		body.Write([]byte(err.Error()))
		contentType = transcodingErrorMimeType
	} else {
		boundary := common.RandName()
		w := multipart.NewWriter(&body)
		for _, v := range tData.Segments {
			w.SetBoundary(boundary)
			hdrs := textproto.MIMEHeader{
				"Content-Type":   {"video/MP2T"},
				"Content-Length": {strconv.Itoa(len(v.Data))},
				"Pixels":         {strconv.FormatInt(v.Pixels, 10)},
			}
			fw, err := w.CreatePart(hdrs)
			if err != nil {
				glog.Error("Could not create multipart part ", err)
			}
			io.Copy(fw, bytes.NewBuffer(v.Data))
		}
		w.Close()
		contentType = "multipart/mixed; boundary=" + boundary
	}
	req, err := http.NewRequest("POST", "https://"+orchAddr+"/transcodeResults", &body)
	if err != nil {
		glog.Error("Error posting results ", err)
	}
	req.Header.Set("Authorization", protoVerLPT)
	req.Header.Set("Credentials", n.OrchSecret)
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("TaskId", strconv.FormatInt(notify.TaskId, 10))
	if tData != nil {
		req.Header.Set("Pixels", strconv.FormatInt(tData.Pixels, 10))
	}
	resp, err := httpc.Do(req)
	if err != nil {
		glog.Error("Error submitting results ", err)
	} else {
		ioutil.ReadAll(resp.Body)
		resp.Body.Close()
	}
	glog.V(common.VERBOSE).Infof("Transcoding done results sent for taskId=%d url=%s err=%v", notify.TaskId, notify.Url, err)
}

// Orchestrator gRPC

func (h *lphttp) RegisterTranscoder(req *net.RegisterRequest, stream net.Transcoder_RegisterTranscoderServer) error {
	from := common.GetConnectionAddr(stream.Context())
	glog.Infof("Got a RegisterTranscoder request from transcoder=%s capacity=%d", from, req.Capacity)

	if req.Secret != h.orchestrator.TranscoderSecret() {
		glog.Info(errSecret.Error())
		return errSecret
	}
	if req.Capacity <= 0 {
		glog.Info(errZeroCapacity.Error())
		return errZeroCapacity
	}

	// blocks until stream is finished
	h.orchestrator.ServeTranscoder(stream, int(req.Capacity))
	return nil
}

// Orchestrator HTTP

func (h *lphttp) TranscodeResults(w http.ResponseWriter, r *http.Request) {
	orch := h.orchestrator

	authType := r.Header.Get("Authorization")
	creds := r.Header.Get("Credentials")
	if protoVerLPT != authType {
		glog.Error("Invalid auth type ", authType)
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	if creds != orch.TranscoderSecret() {
		glog.Error("Invalid shared secret")
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		glog.Error("Error getting mime type ", err)
		http.Error(w, err.Error(), http.StatusUnsupportedMediaType)
		return
	}

	tid, err := strconv.ParseInt(r.Header.Get("TaskId"), 10, 64)
	if err != nil {
		glog.Error("Could not parse task ID ", err)
		http.Error(w, "Invalid Task ID", http.StatusBadRequest)
		return
	}

	decodedPixels, err := strconv.ParseInt(r.Header.Get("Pixels"), 10, 64)
	if err != nil {
		glog.Error("Could not parse decoded pixels", err)
		http.Error(w, "Invalid Pixels", http.StatusBadRequest)
		return
	}

	var res core.RemoteTranscoderResult
	if transcodingErrorMimeType == mediaType {
		w.Write([]byte("OK"))
		body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			glog.Errorf("Unable to read transcoding error body taskID=%v err=%v", tid, err)
			res.Err = err
		} else {
			res.Err = fmt.Errorf(string(body))
		}
		glog.Errorf("Trascoding error for taskID=%v err=%v", tid, res.Err)
		orch.TranscoderResults(tid, &res)
		return
	}

	var segments []*core.TranscodedSegmentData
	if "multipart/mixed" == mediaType {
		mr := multipart.NewReader(r.Body, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			}
			if err != nil {
				glog.Error("Could not process multipart part ", err)
				res.Err = err
				break
			}
			body, err := ioutil.ReadAll(p)
			if err != nil {
				glog.Error("Error reading body ", err)
				res.Err = err
				break
			}

			encodedPixels, err := strconv.ParseInt(p.Header.Get("Pixels"), 10, 64)
			if err != nil {
				glog.Error("Error getting pixels in header:", err)
				res.Err = err
				break
			}

			segments = append(segments, &core.TranscodedSegmentData{Data: body, Pixels: encodedPixels})
		}
		res.TranscodeData = &core.TranscodeData{
			Segments: segments,
			Pixels:   decodedPixels,
		}
		orch.TranscoderResults(tid, &res)
	}
	if res.Err != nil {
		http.Error(w, res.Err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write([]byte("OK"))
}
<commit_msg>Explicitly use HTTP2 transport in communication between orchestrator and transcoder. Mitigates issue in the golang's implementation of HTTP2 https://github.com/golang/go/issues/32441<commit_after>package server

import (
	"bytes"
	"context"
	"crypto/tls"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"os/signal"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/golang/glog"
	"github.com/livepeer/lpms/ffmpeg"
	"golang.org/x/net/http2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/status"

	"github.com/livepeer/go-livepeer/common"
	"github.com/livepeer/go-livepeer/core"
	"github.com/livepeer/go-livepeer/net"
)

const protoVerLPT = "Livepeer-Transcoder-1.0"
const transcodingErrorMimeType = "livepeer/transcoding-error"

var errSecret = errors.New("Invalid secret")
var errZeroCapacity = errors.New("Zero capacity")

// Standalone Transcoder

// RunTranscoder is main routing of standalone transcoder
// Exiting it will terminate executable
func RunTranscoder(n *core.LivepeerNode, orchAddr string, capacity int) {
	expb := backoff.NewExponentialBackOff()
	expb.MaxInterval = time.Minute
	expb.MaxElapsedTime = 0
	backoff.Retry(func() error {
		glog.Info("Registering transcoder to ", orchAddr)
		err := runTranscoder(n, orchAddr, capacity)
		glog.Info("Unregistering transcoder: ", err)
		if _, fatal := err.(core.RemoteTranscoderFatalError); fatal {
			glog.Info("Terminating transcoder because of ", err)
			// Returning nil here will make `backoff` to stop trying to reconnect and exit
			return nil
		}
		// By returning error we tell `backoff` to try to connect again
		return err
	}, expb)
}

func checkTranscoderError(err error) error {
	if err != nil {
		s := status.Convert(err)
		if s.Message() == errSecret.Error() { // consider this unrecoverable
			return core.NewRemoteTranscoderFatalError(errSecret)
		}
		if s.Message() == errZeroCapacity.Error() { // consider this unrecoverable
			return core.NewRemoteTranscoderFatalError(errZeroCapacity)
		}
		if status.Code(err) == codes.Canceled {
			return core.NewRemoteTranscoderFatalError(fmt.Errorf("Execution interrupted"))
		}
	}
	return err
}

func runTranscoder(n *core.LivepeerNode, orchAddr string, capacity int) error {
	tlsConfig := &tls.Config{InsecureSkipVerify: true}
	conn, err := grpc.Dial(orchAddr,
		grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
	if err != nil {
		glog.Error("Did not connect transcoder to orchesrator: ", err)
		return err
	}
	defer conn.Close()

	c := net.NewTranscoderClient(conn)
	ctx := context.Background()
	ctx, cancel := context.WithCancel(ctx)
	// Silence linter
	defer cancel()
	r, err := c.RegisterTranscoder(ctx, &net.RegisterRequest{Secret: n.OrchSecret, Capacity: int64(capacity)})
	if err := checkTranscoderError(err); err != nil {
		glog.Error("Could not register transcoder to orchestrator ", err)
		return err
	}

	// Catch interrupt signal to shut down transcoder
	exitc := make(chan os.Signal)
	signal.Notify(exitc, os.Interrupt, syscall.SIGTERM)
	defer signal.Stop(exitc)
	go func() {
		select {
		case sig := <-exitc:
			glog.Infof("Exiting Livepeer Transcoder: %v", sig)
			// Cancelling context will close connection to orchestrator
			cancel()
			return
		}
	}()

	httpc := &http.Client{Transport: &http2.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	var wg sync.WaitGroup
	for {
		notify, err := r.Recv()
		if err := checkTranscoderError(err); err != nil {
			glog.Infof(`End of stream receive cycle because of err="%v", waiting for running transcode jobs to complete`, err)
			wg.Wait()
			return err
		}
		wg.Add(1)
		go func() {
			runTranscode(n, orchAddr, httpc, notify)
			wg.Done()
		}()
	}
}

func runTranscode(n *core.LivepeerNode, orchAddr string, httpc *http.Client, notify *net.NotifySegment) {
	profiles := []ffmpeg.VideoProfile{}
	if len(notify.FullProfiles) > 0 {
		profiles = makeFfmpegVideoProfiles(notify.FullProfiles)
	} else if len(notify.Profiles) > 0 {
		prof, err := common.TxDataToVideoProfile(hex.EncodeToString(notify.Profiles))
		profiles = prof
		if err != nil {
			glog.Error("Unable to deserialize profiles ", err)
		}
	}

	glog.Infof("Transcoding taskId=%d url=%s", notify.TaskId, notify.Url)
	var contentType string
	var body bytes.Buffer

	tData, err := n.Transcoder.Transcode(notify.Job, notify.Url, profiles)
	glog.V(common.VERBOSE).Infof("Transcoding done for taskId=%d url=%s err=%v", notify.TaskId, notify.Url, err)
	if err != nil {
		glog.Error("Unable to transcode ", err)
		body.Write([]byte(err.Error()))
		contentType = transcodingErrorMimeType
	} else {
		boundary := common.RandName()
		w := multipart.NewWriter(&body)
		for _, v := range tData.Segments {
			w.SetBoundary(boundary)
			hdrs := textproto.MIMEHeader{
				"Content-Type":   {"video/MP2T"},
				"Content-Length": {strconv.Itoa(len(v.Data))},
				"Pixels":         {strconv.FormatInt(v.Pixels, 10)},
			}
			fw, err := w.CreatePart(hdrs)
			if err != nil {
				glog.Error("Could not create multipart part ", err)
			}
			io.Copy(fw, bytes.NewBuffer(v.Data))
		}
		w.Close()
		contentType = "multipart/mixed; boundary=" + boundary
	}
	req, err := http.NewRequest("POST", "https://"+orchAddr+"/transcodeResults", &body)
	if err != nil {
		glog.Error("Error posting results ", err)
	}
	req.Header.Set("Authorization", protoVerLPT)
	req.Header.Set("Credentials", n.OrchSecret)
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("TaskId", strconv.FormatInt(notify.TaskId, 10))
	if tData != nil {
		req.Header.Set("Pixels", strconv.FormatInt(tData.Pixels, 10))
	}
	resp, err := httpc.Do(req)
	if err != nil {
		glog.Error("Error submitting results ", err)
	} else {
		ioutil.ReadAll(resp.Body)
		resp.Body.Close()
	}
	glog.V(common.VERBOSE).Infof("Transcoding done results sent for taskId=%d url=%s err=%v", notify.TaskId, notify.Url, err)
}

// Orchestrator gRPC

func (h *lphttp) RegisterTranscoder(req *net.RegisterRequest, stream net.Transcoder_RegisterTranscoderServer) error {
	from := common.GetConnectionAddr(stream.Context())
	glog.Infof("Got a RegisterTranscoder request from transcoder=%s capacity=%d", from, req.Capacity)

	if req.Secret != h.orchestrator.TranscoderSecret() {
		glog.Info(errSecret.Error())
		return errSecret
	}
	if req.Capacity <= 0 {
		glog.Info(errZeroCapacity.Error())
		return errZeroCapacity
	}

	// blocks until stream is finished
	h.orchestrator.ServeTranscoder(stream, int(req.Capacity))
	return nil
}

// Orchestrator HTTP

func (h *lphttp) TranscodeResults(w http.ResponseWriter, r *http.Request) {
	orch := h.orchestrator

	authType := r.Header.Get("Authorization")
	creds := r.Header.Get("Credentials")
	if protoVerLPT != authType {
		glog.Error("Invalid auth type ", authType)
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	if creds != orch.TranscoderSecret() {
		glog.Error("Invalid shared secret")
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		glog.Error("Error getting mime type ", err)
		http.Error(w, err.Error(), http.StatusUnsupportedMediaType)
		return
	}

	tid, err := strconv.ParseInt(r.Header.Get("TaskId"), 10, 64)
	if err != nil {
		glog.Error("Could not parse task ID ", err)
		http.Error(w, "Invalid Task ID", http.StatusBadRequest)
		return
	}

	decodedPixels, err := strconv.ParseInt(r.Header.Get("Pixels"), 10, 64)
	if err != nil {
		glog.Error("Could not parse decoded pixels", err)
		http.Error(w, "Invalid Pixels", http.StatusBadRequest)
		return
	}

	var res core.RemoteTranscoderResult
	if transcodingErrorMimeType == mediaType {
		w.Write([]byte("OK"))
		body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			glog.Errorf("Unable to read transcoding error body taskID=%v err=%v", tid, err)
			res.Err = err
		} else {
			res.Err = fmt.Errorf(string(body))
		}
		glog.Errorf("Trascoding error for taskID=%v err=%v", tid, res.Err)
		orch.TranscoderResults(tid, &res)
		return
	}

	var segments []*core.TranscodedSegmentData
	if "multipart/mixed" == mediaType {
		mr := multipart.NewReader(r.Body, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			}
			if err != nil {
				glog.Error("Could not process multipart part ", err)
				res.Err = err
				break
			}
			body, err := ioutil.ReadAll(p)
			if err != nil {
				glog.Error("Error reading body ", err)
				res.Err = err
				break
			}

			encodedPixels, err := strconv.ParseInt(p.Header.Get("Pixels"), 10, 64)
			if err != nil {
				glog.Error("Error getting pixels in header:", err)
				res.Err = err
				break
			}

			segments = append(segments, &core.TranscodedSegmentData{Data: body, Pixels: encodedPixels})
		}
		res.TranscodeData = &core.TranscodeData{
			Segments: segments,
			Pixels:   decodedPixels,
		}
		orch.TranscoderResults(tid, &res)
	}
	if res.Err != nil {
		http.Error(w, res.Err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write([]byte("OK"))
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/NYTimes/gziphandler"
	"github.com/discoviking/website/server/storage"
	"github.com/gorilla/mux"
	"net/http"
)

func createRouter(storageService storage.Service) *mux.Router {
	// Main Router.
	r := mux.NewRouter()

	r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "../app/build/src/index.html")
	})

	r.HandleFunc("/app/{path:.*}", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "../app/build/src/index.html")
	})

	appHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "../app/build/src/app.js")
	})
	r.Handle("/app.js", gziphandler.GzipHandler(appHandler))

	fs := http.FileServer(http.Dir("../app/build/src/assets/"))
	r.Handle("/assets/{assetPath:.*}", http.StripPrefix("/assets/", fs))

	storageHandler := storage.NewHandler(storageService)
	r.Handle("/storage/{key}", http.StripPrefix("/storage", storageHandler))

	return r
}
<commit_msg>Simplify router a bit<commit_after>package main

import (
	"github.com/NYTimes/gziphandler"
	"github.com/discoviking/website/server/storage"
	"github.com/gorilla/mux"
	"net/http"
)

func createRouter(storageService storage.Service) *mux.Router {
	// Main Router.
	r := mux.NewRouter()

	fs := http.FileServer(http.Dir("../app/build/src/assets/"))
	r.Handle("/assets/{assetPath:.*}", http.StripPrefix("/assets/", fs))

	storageHandler := storage.NewHandler(storageService)
	r.Handle("/storage/{key}", http.StripPrefix("/storage", storageHandler))

	appHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "../app/build/src/app.js")
	})
	r.Handle("/app.js", gziphandler.GzipHandler(appHandler))

	// For all other paths just serve the app and defer to the front-end to handle it.
	r.HandleFunc("/{path:.*}", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "../app/build/src/index.html")
	})

	return r
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/rand"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	metrics "github.com/rcrowley/go-metrics"
)

var (
	sync = flag.Bool(
		"sync",
		false,
		"Use a synchronous producer.",
	)
	messageLoad = flag.Int(
		"message-load",
		0,
		"REQUIRED: The number of messages to produce to -topic.",
	)
	messageSize = flag.Int(
		"message-size",
		0,
		"REQUIRED: The approximate size (in bytes) of each message to produce to -topic.",
	)
	brokers = flag.String(
		"brokers",
		"",
		"REQUIRED: A comma separated list of broker addresses.",
	)
	topic = flag.String(
		"topic",
		"",
		"REQUIRED: The topic to run the performance test on.",
	)
	partition = flag.Int(
		"partition",
		-1,
		"The partition of -topic to run the performance test on.",
	)
	throughput = flag.Int(
		"throughput",
		0,
		"The maximum number of messages to send per second (0 for no limit).",
	)
	maxMessageBytes = flag.Int(
		"max-message-bytes",
		1000000,
		"The max permitted size of a message.",
	)
	requiredAcks = flag.Int(
		"required-acks",
		1,
		"The required number of acks needed from the broker (-1: all, 0: none, 1: local).",
	)
	timeout = flag.Duration(
		"timeout",
		10*time.Second,
		"The duration the producer will wait to receive -required-acks.",
	)
	partitioner = flag.String(
		"partitioner",
		"roundrobin",
		"The partitioning scheme to use (hash, manual, random, roundrobin).",
	)
	compression = flag.String(
		"compression",
		"none",
		"The compression method to use (none, gzip, snappy, lz4).",
	)
	flushFrequency = flag.Duration(
		"flush-frequency",
		0,
		"The best-effort frequency of flushes.",
	)
	flushBytes = flag.Int(
		"flush-bytes",
		0,
		"The best-effort number of bytes needed to trigger a flush.",
	)
	flushMessages = flag.Int(
		"flush-messages",
		0,
		"The best-effort number of messages needed to trigger a flush.",
	)
	flushMaxMessages = flag.Int(
		"flush-max-messages",
		0,
		"The maximum number of messages the producer will send in a single request.",
	)
	retryMax = flag.Int(
		"retry-max",
		3,
		"The total number of times to retry sending a message.",
	)
	retryBackoff = flag.Duration(
		"retry-backoff",
		100*time.Millisecond,
		"The duration the producer will wait for the cluster to settle between retries.",
	)
	clientID = flag.String(
		"client-id",
		"sarama",
		"The client ID sent with every request to the brokers.",
	)
	channelBufferSize = flag.Int(
		"channel-buffer-size",
		256,
		"The number of events to buffer in internal and external channels.",
	)
	version = flag.String(
		"version",
		"0.8.2.0",
		"The assumed version of Kafka.",
	)
)

func parseCompression(scheme string) sarama.CompressionCodec {
	switch scheme {
	case "none":
		return sarama.CompressionNone
	case "gzip":
		return sarama.CompressionGZIP
	case "snappy":
		return sarama.CompressionSnappy
	case "lz4":
		return sarama.CompressionLZ4
	default:
		printUsageErrorAndExit(fmt.Sprintf("Unknown -compression: %s", scheme))
	}
	panic("should not happen")
}

func parsePartitioner(scheme string, partition int) sarama.PartitionerConstructor {
	if partition < 0 && scheme == "manual" {
		printUsageErrorAndExit("-partition must not be -1 for -partitioning=manual")
	}
	switch scheme {
	case "manual":
		return sarama.NewManualPartitioner
	case "hash":
		return sarama.NewHashPartitioner
	case "random":
		return sarama.NewRandomPartitioner
	case "roundrobin":
		return sarama.NewRoundRobinPartitioner
	default:
		printUsageErrorAndExit(fmt.Sprintf("Unknown -partitioning: %s", scheme))
	}
	panic("should not happen")
}

func parseVersion(version string) sarama.KafkaVersion {
	result, err := sarama.ParseKafkaVersion(version)
	if err != nil {
		printUsageErrorAndExit(fmt.Sprintf("unknown -version: %s", version))
	}
	return result
}

func main() {
	flag.Parse()

	if *brokers == "" {
		printUsageErrorAndExit("-brokers is required")
	}
	if *topic == "" {
		printUsageErrorAndExit("-topic is required")
	}
	if *messageLoad <= 0 {
		printUsageErrorAndExit("-message-load must be greater than 0")
	}
	if *messageSize <= 0 {
		printUsageErrorAndExit("-message-size must be greater than 0")
	}

	config := sarama.NewConfig()

	config.Producer.MaxMessageBytes = *maxMessageBytes
	config.Producer.RequiredAcks = sarama.RequiredAcks(*requiredAcks)
	config.Producer.Timeout = *timeout
	config.Producer.Partitioner = parsePartitioner(*partitioner, *partition)
	config.Producer.Compression = parseCompression(*compression)
	config.Producer.Flush.Frequency = *flushFrequency
	config.Producer.Flush.Bytes = *flushBytes
	config.Producer.Flush.Messages = *flushMessages
	config.Producer.Flush.MaxMessages = *flushMaxMessages
	config.Producer.Return.Successes = true
	config.ClientID = *clientID
	config.ChannelBufferSize = *channelBufferSize
	config.Version = parseVersion(*version)

	if err := config.Validate(); err != nil {
		printErrorAndExit(69, "Invalid configuration: %s", err)
	}

	// Construct -messageLoad messages of appoximately -messageSize random bytes.
	messages := make([]*sarama.ProducerMessage, *messageLoad)
	for i := 0; i < *messageLoad; i++ {
		payload := make([]byte, *messageSize)
		if _, err := rand.Read(payload); err != nil {
			printErrorAndExit(69, "Failed to generate message payload: %s", err)
		}
		messages[i] = &sarama.ProducerMessage{
			Topic:     *topic,
			Partition: int32(*partition),
			Value:     sarama.ByteEncoder(payload),
		}
	}

	// Print out metrics periodically.
	ctx, cancel := context.WithCancel(context.Background())
	go func(ctx context.Context) {
		t := time.Tick(5 * time.Second)
		for {
			select {
			case <-t:
				printMetrics(os.Stdout, config.MetricRegistry)
			case <-ctx.Done():
				return
			}
		}
	}(ctx)

	brokers := strings.Split(*brokers, ",")
	if *sync {
		runSyncProducer(config, brokers, messages, *throughput)
	} else {
		runAsyncProducer(config, brokers, messages, *throughput)
	}

	cancel()
	<-ctx.Done()

	// Print final metrics.
	printMetrics(os.Stdout, config.MetricRegistry)
}

func runAsyncProducer(config *sarama.Config, brokers []string,
	messages []*sarama.ProducerMessage, throughput int) {
	producer, err := sarama.NewAsyncProducer(brokers, config)
	if err != nil {
		printErrorAndExit(69, "Failed to create producer: %s", err)
	}
	defer func() {
		if err := producer.Close(); err != nil {
			printErrorAndExit(69, "Failed to close producer: %s", err)
		}
	}()

	messagesDone := make(chan struct{})
	go func() {
		for i := 0; i < *messageLoad; i++ {
			select {
			case <-producer.Successes():
			case err = <-producer.Errors():
				printErrorAndExit(69, "%s", err)
			}
		}
		messagesDone <- struct{}{}
	}()

	if throughput > 0 {
		ticker := time.NewTicker(time.Second)
		for _, message := range messages {
			for i := 0; i < throughput; i++ {
				producer.Input() <- message
			}
			<-ticker.C
		}
		ticker.Stop()
	} else {
		for _, message := range messages {
			producer.Input() <- message
		}
	}

	<-messagesDone
	close(messagesDone)
}

func runSyncProducer(config *sarama.Config, brokers []string,
	messages []*sarama.ProducerMessage, throughput int) {
	producer, err := sarama.NewSyncProducer(brokers, config)
	if err != nil {
		printErrorAndExit(69, "Failed to create producer: %s", err)
	}
	defer func() {
		if err := producer.Close(); err != nil {
			printErrorAndExit(69, "Failed to close producer: %s", err)
		}
	}()

	if throughput > 0 {
		ticker := time.NewTicker(time.Second)
		for _, message := range messages {
			for i := 0; i < throughput; i++ {
				_, _, err = producer.SendMessage(message)
				if err != nil {
					printErrorAndExit(69, "Failed to send message: %s", err)
				}
			}
			<-ticker.C
		}
		ticker.Stop()
	} else {
		for _, message := range messages {
			_, _, err = producer.SendMessage(message)
			if err != nil {
				printErrorAndExit(69, "Failed to send message: %s", err)
			}
		}
	}
}

func printMetrics(w io.Writer, r metrics.Registry) {
	recordSendRate := r.Get("record-send-rate").(metrics.Meter).Snapshot()
	requestLatency := r.Get("request-latency-in-ms").(metrics.Histogram).Snapshot()
	requestLatencyPercentiles := requestLatency.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
	fmt.Fprintf(w, "%d records sent, %.1f records/sec (%.2f MB/sec), "+
		"%.1f ms avg latency, %.1f ms stddev, %.1f ms 50th, %.1f ms 75th, "+
		"%.1f ms 95th, %.1f ms 99th, %.1f ms 99.9th\n",
		recordSendRate.Count(),
		recordSendRate.RateMean(),
		recordSendRate.RateMean()*float64(*messageSize)/1024/1024,
		requestLatency.Mean(),
		requestLatency.StdDev(),
		requestLatencyPercentiles[0],
		requestLatencyPercentiles[1],
		requestLatencyPercentiles[2],
		requestLatencyPercentiles[3],
		requestLatencyPercentiles[4],
	)
}

func printUsageErrorAndExit(message string) {
	fmt.Fprintln(os.Stderr, "ERROR:", message)
	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "Available command line options:")
	flag.PrintDefaults()
	os.Exit(64)
}

func printErrorAndExit(code int, format string, values ...interface{}) {
	fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
	fmt.Fprintln(os.Stderr)
	os.Exit(code)
}
<commit_msg>Add routines for sync producer on performance tool<commit_after>package main

import (
	"context"
	"crypto/rand"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
	gosync "sync"
	"time"

	"github.com/Shopify/sarama"
	metrics "github.com/rcrowley/go-metrics"
)

var (
	sync = flag.Bool(
		"sync",
		false,
		"Use a synchronous producer.",
	)
	messageLoad = flag.Int(
		"message-load",
		0,
		"REQUIRED: The number of messages to produce to -topic.",
	)
	messageSize = flag.Int(
		"message-size",
		0,
		"REQUIRED: The approximate size (in bytes) of each message to produce to -topic.",
	)
	brokers = flag.String(
		"brokers",
		"",
		"REQUIRED: A comma separated list of broker addresses.",
	)
	topic = flag.String(
		"topic",
		"",
		"REQUIRED: The topic to run the performance test on.",
	)
	partition = flag.Int(
		"partition",
		-1,
		"The partition of -topic to run the performance test on.",
	)
	throughput = flag.Int(
		"throughput",
		0,
		"The maximum number of messages to send per second (0 for no limit).",
	)
	maxMessageBytes = flag.Int(
		"max-message-bytes",
		1000000,
		"The max permitted size of a message.",
	)
	requiredAcks = flag.Int(
		"required-acks",
		1,
		"The required number of acks needed from the broker (-1: all, 0: none, 1: local).",
	)
	timeout = flag.Duration(
		"timeout",
		10*time.Second,
		"The duration the producer will wait to receive -required-acks.",
	)
	partitioner = flag.String(
		"partitioner",
		"roundrobin",
		"The partitioning scheme to use (hash, manual, random, roundrobin).",
	)
	compression = flag.String(
		"compression",
		"none",
		"The compression method to use (none, gzip, snappy, lz4).",
	)
	flushFrequency = flag.Duration(
		"flush-frequency",
		0,
		"The best-effort frequency of flushes.",
	)
	flushBytes = flag.Int(
		"flush-bytes",
		0,
		"The best-effort number of bytes needed to trigger a flush.",
	)
	flushMessages = flag.Int(
		"flush-messages",
		0,
		"The best-effort number of messages needed to trigger a flush.",
	)
	flushMaxMessages = flag.Int(
		"flush-max-messages",
		0,
		"The maximum number of messages the producer will send in a single request.",
	)
	retryMax = flag.Int(
		"retry-max",
		3,
		"The total number of times to retry sending a message.",
	)
	retryBackoff = flag.Duration(
		"retry-backoff",
		100*time.Millisecond,
		"The duration the producer will wait for the cluster to settle between retries.",
	)
	clientID = flag.String(
		"client-id",
		"sarama",
		"The client ID sent with every request to the brokers.",
	)
	channelBufferSize = flag.Int(
		"channel-buffer-size",
		256,
		"The number of events to buffer in internal and external channels.",
	)
	routines = flag.Int(
		"routines",
		1,
		"The number of routines to send the messages from (-sync only).",
	)
	version = flag.String(
		"version",
		"0.8.2.0",
		"The assumed version of Kafka.",
	)
)

func parseCompression(scheme string) sarama.CompressionCodec {
	switch scheme {
	case "none":
		return sarama.CompressionNone
	case "gzip":
		return sarama.CompressionGZIP
	case "snappy":
		return sarama.CompressionSnappy
	case "lz4":
		return sarama.CompressionLZ4
	default:
		printUsageErrorAndExit(fmt.Sprintf("Unknown -compression: %s", scheme))
	}
	panic("should not happen")
}

func parsePartitioner(scheme string, partition int) sarama.PartitionerConstructor {
	if partition < 0 && scheme == "manual" {
		printUsageErrorAndExit("-partition must not be -1 for -partitioning=manual")
	}
	switch scheme {
	case "manual":
		return sarama.NewManualPartitioner
	case "hash":
		return sarama.NewHashPartitioner
	case "random":
		return sarama.NewRandomPartitioner
	case "roundrobin":
		return sarama.NewRoundRobinPartitioner
	default:
		printUsageErrorAndExit(fmt.Sprintf("Unknown -partitioning: %s", scheme))
	}
	panic("should not happen")
}

func parseVersion(version string) sarama.KafkaVersion {
	result, err := sarama.ParseKafkaVersion(version)
	if err != nil {
		printUsageErrorAndExit(fmt.Sprintf("unknown -version: %s", version))
	}
	return result
}

func generateMessages(topic string, partition, messageLoad, messageSize int) []*sarama.ProducerMessage {
	messages := make([]*sarama.ProducerMessage, messageLoad)
	for i := 0; i < messageLoad; i++ {
		payload := make([]byte, messageSize)
		if _, err := rand.Read(payload); err != nil {
			printErrorAndExit(69, "Failed to generate message payload: %s", err)
		}
		messages[i] = &sarama.ProducerMessage{
			Topic:     topic,
			Partition: int32(partition),
			Value:     sarama.ByteEncoder(payload),
		}
	}
	return messages
}

func main() {
	flag.Parse()

	if *brokers == "" {
		printUsageErrorAndExit("-brokers is required")
	}
	if *topic == "" {
		printUsageErrorAndExit("-topic is required")
	}
	if *messageLoad <= 0 {
		printUsageErrorAndExit("-message-load must be greater than 0")
	}
	if *messageSize <= 0 {
		printUsageErrorAndExit("-message-size must be greater than 0")
	}
	if *routines < 1 || *routines > *messageLoad {
		printUsageErrorAndExit("-routines must be greater than 0 and less than or equal to -message-load")
	}

	config := sarama.NewConfig()

	config.Producer.MaxMessageBytes = *maxMessageBytes
	config.Producer.RequiredAcks = sarama.RequiredAcks(*requiredAcks)
	config.Producer.Timeout = *timeout
	config.Producer.Partitioner = parsePartitioner(*partitioner, *partition)
	config.Producer.Compression = parseCompression(*compression)
	config.Producer.Flush.Frequency = *flushFrequency
	config.Producer.Flush.Bytes = *flushBytes
	config.Producer.Flush.Messages = *flushMessages
	config.Producer.Flush.MaxMessages = *flushMaxMessages
	config.Producer.Return.Successes = true
	config.ClientID = *clientID
	config.ChannelBufferSize = *channelBufferSize
	config.Version = parseVersion(*version)

	if err := config.Validate(); err != nil {
		printErrorAndExit(69, "Invalid configuration: %s", err)
	}

	// Print out metrics periodically.
	ctx, cancel := context.WithCancel(context.Background())
	go func(ctx context.Context) {
		t := time.Tick(5 * time.Second)
		for {
			select {
			case <-t:
				printMetrics(os.Stdout, config.MetricRegistry)
			case <-ctx.Done():
				return
			}
		}
	}(ctx)

	brokers := strings.Split(*brokers, ",")
	if *sync {
		runSyncProducer(*topic, *partition, *messageLoad, *messageSize, *routines,
			config, brokers, *throughput)
	} else {
		runAsyncProducer(*topic, *partition, *messageLoad, *messageSize,
			config, brokers, *throughput)
	}

	cancel()
	<-ctx.Done()

	// Print final metrics.
	printMetrics(os.Stdout, config.MetricRegistry)
}

func runAsyncProducer(topic string, partition, messageLoad, messageSize int,
	config *sarama.Config, brokers []string, throughput int) {
	producer, err := sarama.NewAsyncProducer(brokers, config)
	if err != nil {
		printErrorAndExit(69, "Failed to create producer: %s", err)
	}
	defer func() {
		if err := producer.Close(); err != nil {
			printErrorAndExit(69, "Failed to close producer: %s", err)
		}
	}()

	messages := generateMessages(topic, partition, messageLoad, messageSize)

	messagesDone := make(chan struct{})
	go func() {
		for i := 0; i < messageLoad; i++ {
			select {
			case <-producer.Successes():
			case err = <-producer.Errors():
				printErrorAndExit(69, "%s", err)
			}
		}
		messagesDone <- struct{}{}
	}()

	if throughput > 0 {
		ticker := time.NewTicker(time.Second)
		for _, message := range messages {
			for i := 0; i < throughput; i++ {
				producer.Input() <- message
			}
			<-ticker.C
		}
		ticker.Stop()
	} else {
		for _, message := range messages {
			producer.Input() <- message
		}
	}

	<-messagesDone
	close(messagesDone)
}

func runSyncProducer(topic string, partition, messageLoad, messageSize, routines int,
	config *sarama.Config, brokers []string, throughput int) {
	producer, err := sarama.NewSyncProducer(brokers, config)
	if err != nil {
		printErrorAndExit(69, "Failed to create producer: %s", err)
	}
	defer func() {
		if err := producer.Close(); err != nil {
			printErrorAndExit(69, "Failed to close producer: %s", err)
		}
	}()

	messages := make([][]*sarama.ProducerMessage, routines)
	for i := 0; i < routines; i++ {
		if i == routines-1 {
			messages[i] = generateMessages(topic, partition, messageLoad/routines+messageLoad%routines, messageSize)
		} else {
			messages[i] = generateMessages(topic, partition, messageLoad/routines, messageSize)
		}
	}

	var wg gosync.WaitGroup
	if throughput > 0 {
		for _, messages := range messages {
			messages := messages
			wg.Add(1)
			go func() {
				ticker := time.NewTicker(time.Second)
				for _, message := range messages {
					for i := 0; i < throughput; i++ {
						_, _, err = producer.SendMessage(message)
						if err != nil {
							printErrorAndExit(69, "Failed to send message: %s", err)
						}
					}
					<-ticker.C
				}
				ticker.Stop()
				wg.Done()
			}()
		}
	} else {
		for _, messages := range messages {
			messages := messages
			wg.Add(1)
			go func() {
				for _, message := range messages {
					_, _, err = producer.SendMessage(message)
					if err != nil {
						printErrorAndExit(69, "Failed to send message: %s", err)
					}
				}
				wg.Done()
			}()
		}
	}
	wg.Wait()
}

func printMetrics(w io.Writer, r metrics.Registry) {
	recordSendRate := r.Get("record-send-rate").(metrics.Meter).Snapshot()
	requestLatency := r.Get("request-latency-in-ms").(metrics.Histogram).Snapshot()
	requestLatencyPercentiles := requestLatency.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
	fmt.Fprintf(w, "%d records sent, %.1f records/sec (%.2f MB/sec), "+
		"%.1f ms avg latency, %.1f ms stddev, %.1f ms 50th, %.1f ms 75th, "+
		"%.1f ms 95th, %.1f ms 99th, %.1f ms 99.9th\n",
		recordSendRate.Count(),
		recordSendRate.RateMean(),
		recordSendRate.RateMean()*float64(*messageSize)/1024/1024,
		requestLatency.Mean(),
		requestLatency.StdDev(),
		requestLatencyPercentiles[0],
		requestLatencyPercentiles[1],
		requestLatencyPercentiles[2],
		requestLatencyPercentiles[3],
		requestLatencyPercentiles[4],
	)
}

func printUsageErrorAndExit(message string) {
	fmt.Fprintln(os.Stderr, "ERROR:", message)
	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "Available command line options:")
	flag.PrintDefaults()
	os.Exit(64)
}

func printErrorAndExit(code int, format string, values ...interface{}) {
	fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
	fmt.Fprintln(os.Stderr)
	os.Exit(code)
}
<|endoftext|>