<commit_before>package sftp

import (
	"encoding"
	"fmt"
	"io"
	"reflect"
)

func marshalUint32(b []byte, v uint32) []byte {
	return append(b, byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
}

func marshalUint64(b []byte, v uint64) []byte {
	return marshalUint32(marshalUint32(b, uint32(v>>32)), uint32(v))
}

func marshalString(b []byte, v string) []byte {
	return append(marshalUint32(b, uint32(len(v))), v...)
}

func marshal(b []byte, v interface{}) []byte {
	switch v := v.(type) {
	case uint8:
		return append(b, v)
	case uint32:
		return marshalUint32(b, v)
	case uint64:
		return marshalUint64(b, v)
	case string:
		return marshalString(b, v)
	default:
		switch d := reflect.ValueOf(v); d.Kind() {
		case reflect.Struct:
			for i, n := 0, d.NumField(); i < n; i++ {
				b = append(marshal(b, d.Field(i).Interface()))
			}
			return b
		case reflect.Slice:
			for i, n := 0, d.Len(); i < n; i++ {
				b = append(marshal(b, d.Index(i).Interface()))
			}
			return b
		default:
			panic(fmt.Sprintf("marshal(%#v): cannot handle type %T", v, v))
		}
	}
}

func unmarshalUint32(b []byte) (uint32, []byte) {
	v := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
	return v, b[4:]
}

func unmarshalUint64(b []byte) (uint64, []byte) {
	h, b := unmarshalUint32(b)
	l, b := unmarshalUint32(b)
	return uint64(h)<<32 | uint64(l), b
}

func unmarshalString(b []byte) (string, []byte) {
	n, b := unmarshalUint32(b)
	return string(b[:n]), b[n:]
}

// sendPacket marshals p according to RFC 4234.

func sendPacket(w io.Writer, m encoding.BinaryMarshaler) error {
	bb, err := m.MarshalBinary()
	if err != nil {
		return fmt.Errorf("marshal2(%#v): binary marshaller failed", err)
	}
	l := uint32(len(bb))
	hdr := []byte{byte(l >> 24), byte(l >> 16), byte(l >> 8), byte(l)}
	debug("send packet %T, len: %v", m, l)
	_, err = w.Write(hdr)
	if err != nil {
		return err
	}
	_, err = w.Write(bb)
	return err
}

func recvPacket(r io.Reader) (uint8, []byte, error) {
	var b = []byte{0, 0, 0, 0}
	if _, err := io.ReadFull(r, b); err != nil {
		return 0, nil, err
	}
	l, _ := unmarshalUint32(b)
	b = make([]byte, l)
	if _, err := io.ReadFull(r, b); err != nil {
		return 0, nil, err
	}
	return b[0], b[1:], nil
}

// Here starts the definition of packets along with their MarshalBinary
// implementations.
// Manually writing the marshalling logic wins us a lot of time and
// allocation.

type sshFxInitPacket struct {
	Version    uint32
	Extensions []struct {
		Name, Data string
	}
}

func (p sshFxInitPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 // byte + uint32
	for _, e := range p.Extensions {
		l += 4 + len(e.Name) + 4 + len(e.Data)
	}

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_INIT)
	b = marshalUint32(b, p.Version)
	for _, e := range p.Extensions {
		b = marshalString(b, e.Name)
		b = marshalString(b, e.Data)
	}
	return b, nil
}

func marshalIdString(packetType byte, id uint32, str string) ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(str)

	b := make([]byte, 0, l)
	b = append(b, packetType)
	b = marshalUint32(b, id)
	b = marshalString(b, str)
	return b, nil
}

type sshFxpReaddirPacket struct {
	Id     uint32
	Handle string
}

func (p sshFxpReaddirPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_READDIR, p.Id, p.Handle)
}

type sshFxpOpendirPacket struct {
	Id   uint32
	Path string
}

func (p sshFxpOpendirPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_OPENDIR, p.Id, p.Path)
}

type sshFxpLstatPacket struct {
	Id   uint32
	Path string
}

func (p sshFxpLstatPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_LSTAT, p.Id, p.Path)
}

type sshFxpFstatPacket struct {
	Id     uint32
	Handle string
}

func (p sshFxpFstatPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_FSTAT, p.Id, p.Handle)
}

type sshFxpClosePacket struct {
	Id     uint32
	Handle string
}

func (p sshFxpClosePacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_CLOSE, p.Id, p.Handle)
}

type sshFxpRemovePacket struct {
	Id       uint32
	Filename string
}

func (p sshFxpRemovePacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_REMOVE, p.Id, p.Filename)
}

type sshFxpRmdirPacket struct {
	Id   uint32
	Path string
}

func (p sshFxpRmdirPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_RMDIR, p.Id, p.Path)
}

type sshFxpReadlinkPacket struct {
	Id   uint32
	Path string
}

func (p sshFxpReadlinkPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_READLINK, p.Id, p.Path)
}

type sshFxpOpenPacket struct {
	Id     uint32
	Path   string
	Pflags uint32
	Flags  uint32 // ignored
}

func (p sshFxpOpenPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 +
		4 + len(p.Path) +
		4 + 4

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_OPEN)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Path)
	b = marshalUint32(b, p.Pflags)
	b = marshalUint32(b, p.Flags)
	return b, nil
}

type sshFxpReadPacket struct {
	Id     uint32
	Handle string
	Offset uint64
	Len    uint32
}

func (p sshFxpReadPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(p.Handle) +
		8 + 4 // uint64 + uint32

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_READ)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Handle)
	b = marshalUint64(b, p.Offset)
	b = marshalUint32(b, p.Len)
	return b, nil
}

type sshFxpRenamePacket struct {
	Id      uint32
	Oldpath string
	Newpath string
}

func (p sshFxpRenamePacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(p.Oldpath) +
		4 + len(p.Newpath)

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_RENAME)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Oldpath)
	b = marshalString(b, p.Newpath)
	return b, nil
}

type sshFxpWritePacket struct {
	Id     uint32
	Handle string
	Offset uint64
	Length uint32
	Data   []byte
}

func (s sshFxpWritePacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(s.Handle) +
		8 + 4 + // uint64 + uint32
		len(s.Data)

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_WRITE)
	b = marshalUint32(b, s.Id)
	b = marshalString(b, s.Handle)
	b = marshalUint64(b, s.Offset)
	b = marshalUint32(b, s.Length)
	b = append(b, s.Data...)
	return b, nil
}

type sshFxpMkdirPacket struct {
	Id    uint32
	Path  string
	Flags uint32 // ignored
	Size  uint64 // ignored
}

func (p sshFxpMkdirPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(p.Path) +
		4 + 8 // uint32 + uint64

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_MKDIR)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Path)
	b = marshalUint32(b, p.Flags)
	b = marshalUint64(b, p.Size)
	return b, nil
}

type sshFxpSetstatPacket struct {
	Id    uint32
	Path  string
	Flags uint32
	Attrs interface{}
}

func (p sshFxpSetstatPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(p.Path) +
		4 // uint32 + uint64

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_SETSTAT)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Path)
	b = marshalUint32(b, p.Flags)
	b = marshal(b, p.Attrs)
	return b, nil
}
<commit_msg>removed unused field on fxpMkdir<commit_after>package sftp

import (
	"encoding"
	"fmt"
	"io"
	"reflect"
)

func marshalUint32(b []byte, v uint32) []byte {
	return append(b, byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
}

func marshalUint64(b []byte, v uint64) []byte {
	return marshalUint32(marshalUint32(b, uint32(v>>32)), uint32(v))
}

func marshalString(b []byte, v string) []byte {
	return append(marshalUint32(b, uint32(len(v))), v...)
}

func marshal(b []byte, v interface{}) []byte {
	switch v := v.(type) {
	case uint8:
		return append(b, v)
	case uint32:
		return marshalUint32(b, v)
	case uint64:
		return marshalUint64(b, v)
	case string:
		return marshalString(b, v)
	default:
		switch d := reflect.ValueOf(v); d.Kind() {
		case reflect.Struct:
			for i, n := 0, d.NumField(); i < n; i++ {
				b = append(marshal(b, d.Field(i).Interface()))
			}
			return b
		case reflect.Slice:
			for i, n := 0, d.Len(); i < n; i++ {
				b = append(marshal(b, d.Index(i).Interface()))
			}
			return b
		default:
			panic(fmt.Sprintf("marshal(%#v): cannot handle type %T", v, v))
		}
	}
}

func unmarshalUint32(b []byte) (uint32, []byte) {
	v := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
	return v, b[4:]
}

func unmarshalUint64(b []byte) (uint64, []byte) {
	h, b := unmarshalUint32(b)
	l, b := unmarshalUint32(b)
	return uint64(h)<<32 | uint64(l), b
}

func unmarshalString(b []byte) (string, []byte) {
	n, b := unmarshalUint32(b)
	return string(b[:n]), b[n:]
}

// sendPacket marshals p according to RFC 4234.

func sendPacket(w io.Writer, m encoding.BinaryMarshaler) error {
	bb, err := m.MarshalBinary()
	if err != nil {
		return fmt.Errorf("marshal2(%#v): binary marshaller failed", err)
	}
	l := uint32(len(bb))
	hdr := []byte{byte(l >> 24), byte(l >> 16), byte(l >> 8), byte(l)}
	debug("send packet %T, len: %v", m, l)
	_, err = w.Write(hdr)
	if err != nil {
		return err
	}
	_, err = w.Write(bb)
	return err
}

func recvPacket(r io.Reader) (uint8, []byte, error) {
	var b = []byte{0, 0, 0, 0}
	if _, err := io.ReadFull(r, b); err != nil {
		return 0, nil, err
	}
	l, _ := unmarshalUint32(b)
	b = make([]byte, l)
	if _, err := io.ReadFull(r, b); err != nil {
		return 0, nil, err
	}
	return b[0], b[1:], nil
}

// Here starts the definition of packets along with their MarshalBinary
// implementations.
// Manually writing the marshalling logic wins us a lot of time and
// allocation.

type sshFxInitPacket struct {
	Version    uint32
	Extensions []struct {
		Name, Data string
	}
}

func (p sshFxInitPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 // byte + uint32
	for _, e := range p.Extensions {
		l += 4 + len(e.Name) + 4 + len(e.Data)
	}

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_INIT)
	b = marshalUint32(b, p.Version)
	for _, e := range p.Extensions {
		b = marshalString(b, e.Name)
		b = marshalString(b, e.Data)
	}
	return b, nil
}

func marshalIdString(packetType byte, id uint32, str string) ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(str)

	b := make([]byte, 0, l)
	b = append(b, packetType)
	b = marshalUint32(b, id)
	b = marshalString(b, str)
	return b, nil
}

type sshFxpReaddirPacket struct {
	Id     uint32
	Handle string
}

func (p sshFxpReaddirPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_READDIR, p.Id, p.Handle)
}

type sshFxpOpendirPacket struct {
	Id   uint32
	Path string
}

func (p sshFxpOpendirPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_OPENDIR, p.Id, p.Path)
}

type sshFxpLstatPacket struct {
	Id   uint32
	Path string
}

func (p sshFxpLstatPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_LSTAT, p.Id, p.Path)
}

type sshFxpFstatPacket struct {
	Id     uint32
	Handle string
}

func (p sshFxpFstatPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_FSTAT, p.Id, p.Handle)
}

type sshFxpClosePacket struct {
	Id     uint32
	Handle string
}

func (p sshFxpClosePacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_CLOSE, p.Id, p.Handle)
}

type sshFxpRemovePacket struct {
	Id       uint32
	Filename string
}

func (p sshFxpRemovePacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_REMOVE, p.Id, p.Filename)
}

type sshFxpRmdirPacket struct {
	Id   uint32
	Path string
}

func (p sshFxpRmdirPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_RMDIR, p.Id, p.Path)
}

type sshFxpReadlinkPacket struct {
	Id   uint32
	Path string
}

func (p sshFxpReadlinkPacket) MarshalBinary() ([]byte, error) {
	return marshalIdString(ssh_FXP_READLINK, p.Id, p.Path)
}

type sshFxpOpenPacket struct {
	Id     uint32
	Path   string
	Pflags uint32
	Flags  uint32 // ignored
}

func (p sshFxpOpenPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 +
		4 + len(p.Path) +
		4 + 4

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_OPEN)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Path)
	b = marshalUint32(b, p.Pflags)
	b = marshalUint32(b, p.Flags)
	return b, nil
}

type sshFxpReadPacket struct {
	Id     uint32
	Handle string
	Offset uint64
	Len    uint32
}

func (p sshFxpReadPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(p.Handle) +
		8 + 4 // uint64 + uint32

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_READ)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Handle)
	b = marshalUint64(b, p.Offset)
	b = marshalUint32(b, p.Len)
	return b, nil
}

type sshFxpRenamePacket struct {
	Id      uint32
	Oldpath string
	Newpath string
}

func (p sshFxpRenamePacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(p.Oldpath) +
		4 + len(p.Newpath)

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_RENAME)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Oldpath)
	b = marshalString(b, p.Newpath)
	return b, nil
}

type sshFxpWritePacket struct {
	Id     uint32
	Handle string
	Offset uint64
	Length uint32
	Data   []byte
}

func (s sshFxpWritePacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(s.Handle) +
		8 + 4 + // uint64 + uint32
		len(s.Data)

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_WRITE)
	b = marshalUint32(b, s.Id)
	b = marshalString(b, s.Handle)
	b = marshalUint64(b, s.Offset)
	b = marshalUint32(b, s.Length)
	b = append(b, s.Data...)
	return b, nil
}

type sshFxpMkdirPacket struct {
	Id    uint32
	Path  string
	Flags uint32 // ignored
}

func (p sshFxpMkdirPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(p.Path) +
		4  // uint32

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_MKDIR)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Path)
	b = marshalUint32(b, p.Flags)
	return b, nil
}

type sshFxpSetstatPacket struct {
	Id    uint32
	Path  string
	Flags uint32
	Attrs interface{}
}

func (p sshFxpSetstatPacket) MarshalBinary() ([]byte, error) {
	l := 1 + 4 + // type(byte) + uint32
		4 + len(p.Path) +
		4 // uint32 + uint64

	b := make([]byte, 0, l)
	b = append(b, ssh_FXP_SETSTAT)
	b = marshalUint32(b, p.Id)
	b = marshalString(b, p.Path)
	b = marshalUint32(b, p.Flags)
	b = marshal(b, p.Attrs)
	return b, nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed onder the MIT license that can be found in the LICENSE file.

package ini

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"unicode"
)

const (
	commentStart byte = ';'
	seperator    byte = '='
	sectionStart byte = '['
	sectionEnd   byte = ']'
	doubleQuote  byte = '"'
	singleQuote  byte = '\''
	escape       byte = '\\'
	lineEnd      byte = '\n'
	nilQuote     byte = 0
)

type lineType int

const (
	commentLine lineType = iota
	sectionLine
	keyValuePairLine
)

type parser struct {
	Config            Config
	scanner           *bufio.Scanner
	currentSection    string
	currentLine       []byte
	currentlineNumber int
}

func (p *parser) parse() error {
	for p.scanner.Scan() {
		line := p.scanner.Bytes()

		if err := p.handleLine(line); err != nil {
			return newSynthaxError(p.currentlineNumber, string(p.currentLine),
				err.Error())
		}
	}

	if err := p.scanner.Err(); err != nil {
		return fmt.Errorf("ini: error reading: %s", err.Error())
	}
	return nil
}

func (p *parser) handleLine(line []byte) error {
	line = bytes.TrimSpace(line)
	p.currentLine = line
	p.currentlineNumber++

	if len(line) == 0 {
		return nil
	}

	switch detectLineType(line[0]) {
	case sectionLine:
		sectionName, err := parseSection(line)
		if err != nil {
			return err
		}

		p.updateSection(sectionName)
	case keyValuePairLine:
		key, value, err := parseKeyValue(line)
		if err != nil {
			return err
		}

		if err := p.addKeyValue(key, value); err != nil {
			return err
		}
	}

	return nil
}

func (p *parser) updateSection(sectionName string) {
	p.currentSection = sectionName
	p.Config[sectionName] = map[string]string{}
}

func (p *parser) addKeyValue(key, value string) error {
	sectionName := p.currentSection
	if _, ok := p.Config[sectionName][key]; ok {
		return fmt.Errorf("key %q already used in section %q", key, sectionName)
	}

	p.Config[sectionName][key] = value
	return nil
}

// Parse parses ini formatted input.
//
// *Note the reader already gets buffered, so there is no need to buffer it
// youself.
func Parse(r io.Reader) (Config, error) {
	p := newParser(r)
	if err := p.parse(); err != nil {
		return nil, err
	}

	return p.Config, nil
}

func newParser(r io.Reader) *parser {
	return &parser{
		Config:         Config{Global: {}},
		currentSection: Global,
		scanner:        bufio.NewScanner(r),
	}
}

func detectLineType(b byte) lineType {
	if b == commentStart {
		return commentLine
	} else if b == sectionStart {
		return sectionLine
	}

	return keyValuePairLine
}

func parseSection(line []byte) (string, error) {
	if line[0] != sectionStart {
		return "", fmt.Errorf("section should start with %q", string(sectionStart))
	}

	var sectionName = make([]byte, 0, len(line)-2)
	var sectionEnded bool

	// Skipping the opening bracket.
	for i := 1; i < len(line); i++ {
		b := line[i]

		if b == sectionEnd {
			sectionEnded = true
			continue
		} else if b == commentStart && sectionEnded {
			break
		} else if sectionEnded && !unicode.IsSpace(rune(b)) {
			return "", fmt.Errorf("unexpected %q after section closed", string(b))
		}

		sectionName = append(sectionName, b)
	}

	if !sectionEnded {
		return "", errors.New("unclosed section")
	}

	section := string(bytes.TrimSpace(sectionName))
	return section, nil
}

func parseKeyValue(line []byte) (key, value string, err error) {
	var values [2][]byte
	var hasSeperator bool

	// Keep track of the byte number for both the key and value.
	for i, valueNumber := 0, 0; valueNumber <= 1; valueNumber++ {
		var isQouted, isEscaped, nextShouldBeSeperator bool
		var usedQouted byte

		for ; i < len(line); i++ {
			b := line[i]

			if nextShouldBeSeperator && !unicode.IsSpace(rune(b)) && b != seperator {
				return "", "", fmt.Errorf("unexpected %q, expected the seperator %q",
					string(b), string(seperator))
			} else if (b == doubleQuote || b == singleQuote) && !isEscaped {
				if !isQouted {
					isQouted = true
					usedQouted = b
					continue
				} else if b == usedQouted {
					isQouted = false
					usedQouted = nilQuote
					if valueNumber == 0 {
						nextShouldBeSeperator = true
					}
					continue
				}
			} else if b == commentStart && !isQouted && hasSeperator {
				break
			} else if b == escape && !isEscaped {
				isEscaped = true
				continue
			} else if b == seperator && valueNumber == 0 && !isQouted {
				hasSeperator = true
				i++ // skip the current byte (the separator) in the next loop.
				break
			}

			isEscaped = false
			values[valueNumber] = append(values[valueNumber], b)
		}

		if isQouted {
			return "", "", errors.New("qoute not closed")
		}
	}

	if !hasSeperator {
		return "", "", errors.New("no separator found")
	}

	key = string(bytes.TrimSpace(values[0]))
	value = string(bytes.TrimSpace(values[1]))
	if len(key) == 0 {
		return "", "", errors.New("key can't be empty")
	}
	return key, value, nil
}
<commit_msg>Reorder constants<commit_after>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed onder the MIT license that can be found in the LICENSE file.

package ini

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"unicode"
)

const (
	commentStart byte = ';'
	seperator    byte = '='
	sectionStart byte = '['
	sectionEnd   byte = ']'
	escape       byte = '\\'
	lineEnd      byte = '\n'
	doubleQuote  byte = '"'
	singleQuote  byte = '\''
	nilQuote     byte = 0
)

type lineType int

const (
	commentLine lineType = iota
	sectionLine
	keyValuePairLine
)

type parser struct {
	Config            Config
	scanner           *bufio.Scanner
	currentSection    string
	currentLine       []byte
	currentlineNumber int
}

func (p *parser) parse() error {
	for p.scanner.Scan() {
		line := p.scanner.Bytes()

		if err := p.handleLine(line); err != nil {
			return newSynthaxError(p.currentlineNumber, string(p.currentLine),
				err.Error())
		}
	}

	if err := p.scanner.Err(); err != nil {
		return fmt.Errorf("ini: error reading: %s", err.Error())
	}
	return nil
}

func (p *parser) handleLine(line []byte) error {
	line = bytes.TrimSpace(line)
	p.currentLine = line
	p.currentlineNumber++

	if len(line) == 0 {
		return nil
	}

	switch detectLineType(line[0]) {
	case sectionLine:
		sectionName, err := parseSection(line)
		if err != nil {
			return err
		}

		p.updateSection(sectionName)
	case keyValuePairLine:
		key, value, err := parseKeyValue(line)
		if err != nil {
			return err
		}

		if err := p.addKeyValue(key, value); err != nil {
			return err
		}
	}

	return nil
}

func (p *parser) updateSection(sectionName string) {
	p.currentSection = sectionName
	p.Config[sectionName] = map[string]string{}
}

func (p *parser) addKeyValue(key, value string) error {
	sectionName := p.currentSection
	if _, ok := p.Config[sectionName][key]; ok {
		return fmt.Errorf("key %q already used in section %q", key, sectionName)
	}

	p.Config[sectionName][key] = value
	return nil
}

// Parse parses ini formatted input.
//
// *Note the reader already gets buffered, so there is no need to buffer it
// youself.
func Parse(r io.Reader) (Config, error) {
	p := newParser(r)
	if err := p.parse(); err != nil {
		return nil, err
	}

	return p.Config, nil
}

func newParser(r io.Reader) *parser {
	return &parser{
		Config:         Config{Global: {}},
		currentSection: Global,
		scanner:        bufio.NewScanner(r),
	}
}

func detectLineType(b byte) lineType {
	if b == commentStart {
		return commentLine
	} else if b == sectionStart {
		return sectionLine
	}

	return keyValuePairLine
}

func parseSection(line []byte) (string, error) {
	if line[0] != sectionStart {
		return "", fmt.Errorf("section should start with %q", string(sectionStart))
	}

	var sectionName = make([]byte, 0, len(line)-2)
	var sectionEnded bool

	// Skipping the opening bracket.
	for i := 1; i < len(line); i++ {
		b := line[i]

		if b == sectionEnd {
			sectionEnded = true
			continue
		} else if b == commentStart && sectionEnded {
			break
		} else if sectionEnded && !unicode.IsSpace(rune(b)) {
			return "", fmt.Errorf("unexpected %q after section closed", string(b))
		}

		sectionName = append(sectionName, b)
	}

	if !sectionEnded {
		return "", errors.New("unclosed section")
	}

	section := string(bytes.TrimSpace(sectionName))
	return section, nil
}

func parseKeyValue(line []byte) (key, value string, err error) {
	var values [2][]byte
	var hasSeperator bool

	// Keep track of the byte number for both the key and value.
	for i, valueNumber := 0, 0; valueNumber <= 1; valueNumber++ {
		var isQouted, isEscaped, nextShouldBeSeperator bool
		var usedQouted byte

		for ; i < len(line); i++ {
			b := line[i]

			if nextShouldBeSeperator && !unicode.IsSpace(rune(b)) && b != seperator {
				return "", "", fmt.Errorf("unexpected %q, expected the seperator %q",
					string(b), string(seperator))
			} else if (b == doubleQuote || b == singleQuote) && !isEscaped {
				if !isQouted {
					isQouted = true
					usedQouted = b
					continue
				} else if b == usedQouted {
					isQouted = false
					usedQouted = nilQuote
					if valueNumber == 0 {
						nextShouldBeSeperator = true
					}
					continue
				}
			} else if b == commentStart && !isQouted && hasSeperator {
				break
			} else if b == escape && !isEscaped {
				isEscaped = true
				continue
			} else if b == seperator && valueNumber == 0 && !isQouted {
				hasSeperator = true
				i++ // skip the current byte (the separator) in the next loop.
				break
			}

			isEscaped = false
			values[valueNumber] = append(values[valueNumber], b)
		}

		if isQouted {
			return "", "", errors.New("qoute not closed")
		}
	}

	if !hasSeperator {
		return "", "", errors.New("no separator found")
	}

	key = string(bytes.TrimSpace(values[0]))
	value = string(bytes.TrimSpace(values[1]))
	if len(key) == 0 {
		return "", "", errors.New("key can't be empty")
	}
	return key, value, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/kylelemons/go-gypsy/yaml"
	"strconv"
)

type ClientProfile struct {
	Name         string
	Distribution string
	Parameters   []float64
}

type Profile struct {
	URL      string
	Method   string
	Template string

	Clients []*ClientProfile
}

func parseString(node yaml.Node) (str string, err error) {
	if node == nil {
		str = ""
		return
	}
	if scalar, ok := node.(yaml.Scalar); ok {
		str = string(scalar)
	} else {
		err = fmt.Errorf("not a scalar")
	}
	return
}

func parseFloatList(node yaml.Node) (l []float64, err error) {
	if node == nil {
		l = nil
		return
	}
	if list, ok := node.(yaml.List); ok {
		var scalar string
		var f float64
		l = make([]float64, 0, len(list))
		for i, n := range list {
			scalar, err = parseString(n)
			if err != nil {
				err = fmt.Errorf("element %v is not a scalar", i)
				return
			}
			f, err = strconv.ParseFloat(scalar, 64)
			if err != nil {
				err = fmt.Errorf("element %v is not a float64", i)
				return
			}
			l = append(l, f)
		}
	} else {
		err = fmt.Errorf("not a list")
	}
	return
}

func parseClientProfile(clientName string, node yaml.Node) (*ClientProfile, error) {
	if kv, ok := node.(yaml.Map); ok {
		var err error
		prof := new(ClientProfile)
		if dist, ok := kv["distribution"]; ok {
			prof.Distribution, err = parseString(dist)
			if err != nil {
				return nil, fmt.Errorf("client %v's distribution is %v", clientName, err)
			}
		} else {
			return nil, fmt.Errorf("client %v should have a distribution of inter arrival time", clientName)
		}
		if params, ok := kv["parameters"]; ok {
			prof.Parameters, err = parseFloatList(params)
			if err != nil {
				return nil, fmt.Errorf("client %v's parameter error: %v", clientName, err)
			}
		}
		return prof, nil
	}
	return nil, fmt.Errorf("client %v's profile should be a map", clientName)
}

func parseStringFromMap(kv yaml.Map, key string) (string, error) {
	if str, ok := kv[key]; ok {
		s, err := parseString(str)
		if err != nil {
			return "", fmt.Errorf("%v should be a string", key)
		}
		return s, nil
	}
	return "", fmt.Errorf("cannot find %v", key)
}

func ParseProfile(node yaml.Node) (p *Profile, err error) {
	if kv, ok := node.(yaml.Map); ok {
		var client *ClientProfile
		p = new(Profile)
		p.Clients = make([]*ClientProfile, 0, len(kv)-1)
		p.URL, err = parseStringFromMap(kv, "url")
		if err != nil {
			return
		}

		p.Method, err = parseStringFromMap(kv, "method")
		if err != nil {
			p.Method = "GET"
		}

		p.Template, err = parseStringFromMap(kv, "template")
		if err != nil {
			p.Template = ""
		}
		for k, v := range kv {
			if k == "url" || k == "method" || k == "template" {
				continue
			}
			client, err = parseClientProfile(k, v)
			if err != nil {
				return
			}
			p.Clients = append(p.Clients, client)
		}
	} else {
		return nil, fmt.Errorf("profile should be a map")
	}

	if len(p.Clients) == 0 {
		err = fmt.Errorf("no client")
	}
	return
}
<commit_msg>max nr req and max duration<commit_after>package main

import (
	"fmt"
	"github.com/kylelemons/go-gypsy/yaml"
	"strconv"
	"time"
)

type ClientProfile struct {
	Name         string
	Distribution string
	Parameters   []float64
	MaxNrReq     int
	MaxDuration  time.Duration
}

type Profile struct {
	URL      string
	Method   string
	Template string

	Clients []*ClientProfile
}

func parseDuration(node yaml.Node) (t time.Duration, err error) {
	if scalar, ok := node.(yaml.Scalar); ok {
		t, err = time.ParseDuration(string(scalar))
	} else {
		err = fmt.Errorf("duration should be a scalar")
	}
	return
}

func parseInt(node yaml.Node) (n int, err error) {
	if scalar, ok := node.(yaml.Scalar); ok {
		str := string(scalar)
		n, err = strconv.Atoi(str)
	} else {
		err = fmt.Errorf("Not a scalar")
	}
	return
}

func parseString(node yaml.Node) (str string, err error) {
	if node == nil {
		str = ""
		return
	}
	if scalar, ok := node.(yaml.Scalar); ok {
		str = string(scalar)
	} else {
		err = fmt.Errorf("not a scalar")
	}
	return
}

func parseFloatList(node yaml.Node) (l []float64, err error) {
	if node == nil {
		l = nil
		return
	}
	if list, ok := node.(yaml.List); ok {
		var scalar string
		var f float64
		l = make([]float64, 0, len(list))
		for i, n := range list {
			scalar, err = parseString(n)
			if err != nil {
				err = fmt.Errorf("element %v is not a scalar", i)
				return
			}
			f, err = strconv.ParseFloat(scalar, 64)
			if err != nil {
				err = fmt.Errorf("element %v is not a float64", i)
				return
			}
			l = append(l, f)
		}
	} else {
		err = fmt.Errorf("not a list")
	}
	return
}

func parseClientProfile(clientName string, node yaml.Node) (*ClientProfile, error) {
	if kv, ok := node.(yaml.Map); ok {
		var err error
		prof := new(ClientProfile)
		if dist, ok := kv["distribution"]; ok {
			prof.Distribution, err = parseString(dist)
			if err != nil {
				return nil, fmt.Errorf("client %v's distribution is %v", clientName, err)
			}
		} else {
			return nil, fmt.Errorf("client %v should have a distribution of inter arrival time", clientName)
		}
		if params, ok := kv["parameters"]; ok {
			prof.Parameters, err = parseFloatList(params)
			if err != nil {
				return nil, fmt.Errorf("client %v's parameter error: %v", clientName, err)
			}
		}

		if n, ok := kv["max-number-of-request"]; ok {
			prof.MaxNrReq, err = parseInt(n)
			if err != nil {
				return nil, fmt.Errorf("client %v's max number of request error: %v", clientName, err)
			}
		}

		if d, ok := kv["max-duration"]; ok {
			prof.MaxDuration, err = parseDuration(d)
			if err != nil {
				return nil, fmt.Errorf("client %v's max duration error: %v", clientName, err)
			}
		}
		return prof, nil
	}
	return nil, fmt.Errorf("client %v's profile should be a map", clientName)
}

func parseStringFromMap(kv yaml.Map, key string) (string, error) {
	if str, ok := kv[key]; ok {
		s, err := parseString(str)
		if err != nil {
			return "", fmt.Errorf("%v should be a string", key)
		}
		return s, nil
	}
	return "", fmt.Errorf("cannot find %v", key)
}

func ParseProfile(node yaml.Node) (p *Profile, err error) {
	if kv, ok := node.(yaml.Map); ok {
		var client *ClientProfile
		p = new(Profile)
		p.Clients = make([]*ClientProfile, 0, len(kv)-1)
		p.URL, err = parseStringFromMap(kv, "url")
		if err != nil {
			return
		}

		p.Method, err = parseStringFromMap(kv, "method")
		if err != nil {
			p.Method = "GET"
		}

		p.Template, err = parseStringFromMap(kv, "template")
		if err != nil {
			p.Template = ""
		}
		for k, v := range kv {
			if k == "url" || k == "method" || k == "template" {
				continue
			}
			client, err = parseClientProfile(k, v)
			if err != nil {
				return
			}
			p.Clients = append(p.Clients, client)
		}
	} else {
		return nil, fmt.Errorf("profile should be a map")
	}

	if len(p.Clients) == 0 {
		err = fmt.Errorf("no client")
	}
	return
}

func ParseProfileFile(filename string) (*Profile, error) {
	file, err := yaml.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	root := file.Root
	return ParseProfile(root)
}
<|endoftext|>


<commit_before>package jwthelper

import (
	"fmt"

	"github.com/dgrijalva/jwt-go"
)

// Parser is used to parse JWT token string.
type Parser struct {
	key    interface{}
	parser jwt.Parser
}

// ParserOption represents the option for parsing JWT token string.
// Use option helper functions to set options:
// e.g. ParserUseJSONNumber()
type ParserOption struct {
	f func(p *Parser)
}

var (
	// ErrInvalidParser represents the error of invalid parser.
	ErrInvalidParser = fmt.Errorf("invalid parser")
	// ErrParseClaims represents the error of failed to parse claims.
	ErrParseClaims = fmt.Errorf("failed to parse claims")
	// ErrInvalidToken represents the error of invalid token.
	ErrInvalidToken = fmt.Errorf("invalid token")
)

// ParserUseJSONNumber returns the option for using JSON number.
// It causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
// After calling Parser.Parse(), the type of number stored in the map[string]interface{} is:
// * float64: flag is false.
// * json.Number: flag is true.
// See https://godoc.org/encoding/json#Decoder.UseNumber
func ParserUseJSONNumber(flag bool) ParserOption {
	return ParserOption{func(p *Parser) {
		p.parser.UseJSONNumber = flag
	}}
}

// NewRSASHAParser news a parser with RSASHA alg.
//
//     Params:
//         key: RSA public PEM key.
//         options: variadic options returned by option helper functions.
//                  e.g. ParserUseJSONNumber.
func NewRSASHAParser(key []byte, options ...ParserOption) *Parser {
	p := &Parser{
		nil,
		jwt.Parser{
			// UseJSONNumber will call encoding/json.Decoder.UseNumber().
			// It causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
			// See https://godoc.org/encoding/json#Decoder.UseNumber
			UseJSONNumber: true,
			// If populated, only these methods will be considered valid
			// See https://godoc.org/github.com/dgrijalva/jwt-go#Parser
			ValidMethods: []string{
				jwt.SigningMethodRS256.Alg(),
				jwt.SigningMethodRS384.Alg(),
				jwt.SigningMethodRS512.Alg(),
			},
		},
	}

	// Override customized options.
	for _, op := range options {
		op.f(p)
	}

	publicKey, err := jwt.ParseRSAPublicKeyFromPEM(key)
	if err != nil {
		return &Parser{}
	}

	p.key = publicKey
	return p
}

func NewRSASHAParserFromPEMFile(publicPEM string, options ...ParserOption) *Parser {
	key, err := ReadKey(publicPEM)
	if err != nil {
		return &Parser{}
	}

	return NewRSASHAParser(key, options...)
}

func (p *Parser) Valid() bool {
	if p.key == nil {
		return false
	}
	return true
}

func (p *Parser) Parse(tokenString string) (map[string]interface{}, error) {
	m := map[string]interface{}{}

	if !p.Valid() {
		return m, ErrInvalidParser
	}

	token, err := p.parser.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
		return p.key, nil
	})

	if err != nil {
		return m, err
	}

	claims, ok := token.Claims.(jwt.MapClaims)
	if !ok {
		return m, ErrParseClaims
	}

	if !token.Valid {
		return m, ErrInvalidToken
	}

	return claims, nil
}
<commit_msg>Add comment of NewRSASHAParserFromPEMFile<commit_after>package jwthelper

import (
	"fmt"

	"github.com/dgrijalva/jwt-go"
)

// Parser is used to parse JWT token string.
type Parser struct {
	key    interface{}
	parser jwt.Parser
}

// ParserOption represents the option for parsing JWT token string.
// Use option helper functions to set options:
// e.g. ParserUseJSONNumber()
type ParserOption struct {
	f func(p *Parser)
}

var (
	// ErrInvalidParser represents the error of invalid parser.
	ErrInvalidParser = fmt.Errorf("invalid parser")
	// ErrParseClaims represents the error of failed to parse claims.
	ErrParseClaims = fmt.Errorf("failed to parse claims")
	// ErrInvalidToken represents the error of invalid token.
	ErrInvalidToken = fmt.Errorf("invalid token")
)

// ParserUseJSONNumber returns the option for using JSON number.
// It causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
// After calling Parser.Parse(), the type of number stored in the map[string]interface{} is:
// * float64: flag is false.
// * json.Number: flag is true.
// See https://godoc.org/encoding/json#Decoder.UseNumber
func ParserUseJSONNumber(flag bool) ParserOption {
	return ParserOption{func(p *Parser) {
		p.parser.UseJSONNumber = flag
	}}
}

// NewRSASHAParser news a parser with RSASHA alg.
//
//     Params:
//         key: RSA public PEM key.
//         options: variadic options returned by option helper functions.
//                  e.g. ParserUseJSONNumber.
func NewRSASHAParser(key []byte, options ...ParserOption) *Parser {
	p := &Parser{
		nil,
		jwt.Parser{
			// UseJSONNumber will call encoding/json.Decoder.UseNumber().
			// It causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
			// See https://godoc.org/encoding/json#Decoder.UseNumber
			UseJSONNumber: true,
			// If populated, only these methods will be considered valid
			// See https://godoc.org/github.com/dgrijalva/jwt-go#Parser
			ValidMethods: []string{
				jwt.SigningMethodRS256.Alg(),
				jwt.SigningMethodRS384.Alg(),
				jwt.SigningMethodRS512.Alg(),
			},
		},
	}

	// Override customized options.
	for _, op := range options {
		op.f(p)
	}

	publicKey, err := jwt.ParseRSAPublicKeyFromPEM(key)
	if err != nil {
		return &Parser{}
	}

	p.key = publicKey
	return p
}

// NewRSASHAParserFromPEMFile news a parser with RSASHA alg from the RSA public PEM file.
//
//     Params:
//         key: RSA public PEM file path.
//         options: variadic options returned by option helper functions.
//                  e.g. ParserUseJSONNumber.
func NewRSASHAParserFromPEMFile(publicPEM string, options ...ParserOption) *Parser {
	key, err := ReadKey(publicPEM)
	if err != nil {
		return &Parser{}
	}

	return NewRSASHAParser(key, options...)
}

func (p *Parser) Valid() bool {
	if p.key == nil {
		return false
	}
	return true
}

func (p *Parser) Parse(tokenString string) (map[string]interface{}, error) {
	m := map[string]interface{}{}

	if !p.Valid() {
		return m, ErrInvalidParser
	}

	token, err := p.parser.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
		return p.key, nil
	})

	if err != nil {
		return m, err
	}

	claims, ok := token.Claims.(jwt.MapClaims)
	if !ok {
		return m, ErrParseClaims
	}

	if !token.Valid {
		return m, ErrInvalidToken
	}

	return claims, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vertigo

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

const (
	channelChunkSize          = 250000 // changing the value affects performance (10k...300k ~ 15%)
	logProgressEachNthDefault = 1000000
	LineTypeToken             = "token"
	LineTypeStruct            = "struct"
	LineTypeIgnored           = "ignored"

	AccumulatorTypeStack = "stack"
	AccumulatorTypeComb  = "comb"
	AccumulatorTypeNil   = "nil"

	CharsetISO8859_1   = "iso-8859-1"
	CharsetISO8859_2   = "iso-8859-2"
	CharsetISO8859_3   = "iso-8859-3"
	CharsetISO8859_4   = "iso-8859-4"
	CharsetISO8859_5   = "iso-8859-5"
	CharsetISO8859_6   = "iso-8859-6"
	CharsetISO8859_7   = "iso-8859-7"
	CharsetISO8859_8   = "iso-8859-8"
	CharsetWindows1250 = "windows-1250"
	CharsetWindows1251 = "windows-1251"
	CharsetWindows1252 = "windows-1252"
	CharsetWindows1253 = "windows-1253"
	CharsetWindows1254 = "windows-1254"
	CharsetWindows1255 = "windows-1255"
	CharsetWindows1256 = "windows-1256"
	CharsetWindows1257 = "windows-1257"
	CharsetWindows1258 = "windows-1258"
	CharsetUTF_8       = "utf-8"
)

// --------------------------------------------------------

// ParserConf contains configuration parameters for
// vertical file parser
type ParserConf struct {

	// Source vertical file (either a plain text file or a gzip one)
	InputFilePath string `json:"inputFilePath"`

	Encoding string `json:"encoding"`

	FilterArgs [][][]string `json:"filterArgs"`

	StructAttrAccumulator string `json:"structAttrAccumulator"`

	LogProgressEachNth int `json:"logProgressEachNth"`
}

// LoadConfig loads the configuration from a JSON file.
// In case of an error the program exits with panic.
func LoadConfig(path string) *ParserConf {
	rawData, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	var conf ParserConf
	err = json.Unmarshal(rawData, &conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

// ------

type structAttrAccumulator interface {
	Begin(value *Structure) error
	End(name string) (*Structure, error)
	GetAttrs() map[string]string
	Size() int
}

// --------------------------------------------------------

// LineProcessor describes an object able to handle
// Vertigo's parsing events.
type LineProcessor interface {

	// ProcToken is called each time the parser encounters a positional
	// attribute. In case parsing produces an error, it is passed to the
	// function without stopping the whole process.
	// In case the function returns an error, the parser stops
	// (in the simplest case it can be even the error it recieves)
	ProcToken(token *Token, line int, err error) error

	// ProcStruct is called each time parser encounters a structure opening
	// element (e.g. <doc>). In case parsing produces an error, it is passed
	// to the function without stopping the whole process.
	// In case the function returns an error, the parser stops.
	ProcStruct(strc *Structure, line int, err error) error

	// ProcStructClose is called each time parser encouters a structure
	// closing element (e.g. </doc>). In case parsing produces an error,
	// it is passed to the function without stopping the whole process.
	// In case the function returns an error, the parser stops.
	ProcStructClose(strc *StructureClose, line int, err error) error
}

// ----

type procItem struct {
	idx   int
	value interface{}
	err   error
}

// --------------------------------------------------------

func createStructAttrAccumulator(ident string) (structAttrAccumulator, error) {
	switch ident {
	case AccumulatorTypeStack:
		return newStack(), nil
	case AccumulatorTypeComb:
		return newStructAttrs(), nil
	case AccumulatorTypeNil:
		return newNilStructAttrs(), nil
	default:
		return nil, fmt.Errorf("Unknown accumulator type \"%s\"", ident)
	}
}

// SupportedCharsets returns a list of names of
// character sets.
func SupportedCharsets() []string {
	return []string{CharsetISO8859_2, CharsetUTF_8, CharsetWindows1250}
}

// GetCharmapByName returns a proper Charmap instance based
// on provided encoding name. The name detection is case
// insensitive (e.g. utf-8 is the same as UTF-8). The number
// of supported charsets is
func GetCharmapByName(name string) (*charmap.Charmap, error) {
	switch strings.ToLower(name) {
	case CharsetISO8859_1:
		return charmap.ISO8859_1, nil
	case CharsetISO8859_2:
		return charmap.ISO8859_2, nil
	case CharsetISO8859_3:
		return charmap.ISO8859_3, nil
	case CharsetISO8859_4:
		return charmap.ISO8859_4, nil
	case CharsetISO8859_5:
		return charmap.ISO8859_5, nil
	case CharsetISO8859_6:
		return charmap.ISO8859_6, nil
	case CharsetISO8859_7:
		return charmap.ISO8859_7, nil
	case CharsetISO8859_8:
		return charmap.ISO8859_8, nil
	case CharsetWindows1250:
		return charmap.Windows1250, nil
	case CharsetWindows1251:
		return charmap.Windows1251, nil
	case CharsetWindows1252:
		return charmap.Windows1252, nil
	case CharsetWindows1253:
		return charmap.Windows1253, nil
	case CharsetWindows1254:
		return charmap.Windows1254, nil
	case CharsetWindows1255:
		return charmap.Windows1255, nil
	case CharsetWindows1256:
		return charmap.Windows1256, nil
	case CharsetWindows1257:
		return charmap.Windows1257, nil
	case CharsetWindows1258:
		return charmap.Windows1258, nil
	case CharsetUTF_8:
		return nil, nil
	case "":
		log.Printf("No charset specified, assuming utf-8")
		return nil, nil
	default:
		return nil, fmt.Errorf("Unsupported charset '%s'", name)
	}
}

func importString(s string, ch *charmap.Charmap) string {
	if ch == nil {
		return s
	}
	ans, _, _ := transform.String(ch.NewDecoder(), s)
	// TODO handle error
	return ans
}

func openInputFile(path string) (io.Reader, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	finfo, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if !finfo.Mode().IsRegular() {
		return nil, fmt.Errorf("Path %s is not a regular file", path)
	}

	var rd io.Reader
	if strings.HasSuffix(path, ".gz") {
		rd, err = gzip.NewReader(f)
		if err != nil {
			return nil, err
		}

	} else {
		rd = f
	}
	return rd, nil
}

// ParseVerticalFile processes a corpus vertical file
// line by line and applies a custom LineProcessor on
// them. The processing is parallelized in the sense
// that reading a file into lines and processing of
// the lines runs in different goroutines. But the
// function as a whole behaves synchronously - i.e.
// once it returns a value, the processing is finished.
func ParseVerticalFile(conf *ParserConf, lproc LineProcessor) error {
	logProgressEachNth := logProgressEachNthDefault
	if conf.LogProgressEachNth > 0 {
		logProgressEachNth = conf.LogProgressEachNth
	}
	rd, err := openInputFile(conf.InputFilePath)
	if err != nil {
		return err
	}
	brd := bufio.NewScanner(rd)

	stack, err := createStructAttrAccumulator(conf.StructAttrAccumulator)
	if err != nil {
		return err
	}

	chm, chErr := GetCharmapByName(conf.Encoding)
	if chErr != nil {
		return chErr
	}
	log.Printf("Configured conversion from charset %s", chm)
	ch := make(chan []procItem)
	chunk := make([]procItem, channelChunkSize)
	stop := make(chan struct{})
	defer close(stop)

	go func() {
		defer close(ch)
		i := 0
		lineNum := 0
		tokenNum := 0
		for brd.Scan() {
			line, parseErr := parseLine(importString(brd.Text(), chm), stack)
			tok, isTok := line.(*Token)
			if isTok {
				tok.Idx = tokenNum
				tokenNum++
			}
			chunk[i] = procItem{idx: lineNum, value: line, err: parseErr}
			i++
			if i == channelChunkSize {
				i = 0
				ch <- chunk
			}
			if lineNum%logProgressEachNth == 0 {
				log.Printf("...processed %d lines.\n", lineNum)
			}
			lineNum++
			select {
			case <-stop:
				fmt.Println("STOPPING PARSING")
				return
			default:
			}
		}
		if i > 0 {
			ch <- chunk[:i]
		}
	}()

	var procErr error
	for items := range ch {
		for _, item := range items {
			switch item.value.(type) {
			case *Token:
				tk := item.value.(*Token)
				if tk.MatchesFilter(conf.FilterArgs) {
					procErr = lproc.ProcToken(tk, item.idx, err)
				}
			case *Structure:
				procErr = lproc.ProcStruct(item.value.(*Structure), item.idx, item.err)
			case *StructureClose:
				procErr = lproc.ProcStructClose(item.value.(*StructureClose), item.idx, item.err)
			}
			if procErr != nil {
				return procErr
			}
		}
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
	return nil
}

//ParseVerticalFileNoGoRo is just for benchmarking purposes
func ParseVerticalFileNoGoRo(conf *ParserConf, lproc LineProcessor) {
	f, err := os.Open(conf.InputFilePath)
	if err != nil {
		panic(err)
	}
	rd := bufio.NewScanner(f)
	stack := newStack()
	i := 0
	for rd.Scan() {
		token, err := parseLine(rd.Text(), stack)
		switch token.(type) {
		case *Token:
			lproc.ProcToken(token.(*Token), i, err)
		}
		i++
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
}
<commit_msg>Cannot share single chunk slice between channel sending actions<commit_after>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vertigo

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

const (
	channelChunkSize          = 250000 // changing the value affects performance (10k...300k ~ 15%)
	logProgressEachNthDefault = 1000000
	LineTypeToken             = "token"
	LineTypeStruct            = "struct"
	LineTypeIgnored           = "ignored"

	AccumulatorTypeStack = "stack"
	AccumulatorTypeComb  = "comb"
	AccumulatorTypeNil   = "nil"

	CharsetISO8859_1   = "iso-8859-1"
	CharsetISO8859_2   = "iso-8859-2"
	CharsetISO8859_3   = "iso-8859-3"
	CharsetISO8859_4   = "iso-8859-4"
	CharsetISO8859_5   = "iso-8859-5"
	CharsetISO8859_6   = "iso-8859-6"
	CharsetISO8859_7   = "iso-8859-7"
	CharsetISO8859_8   = "iso-8859-8"
	CharsetWindows1250 = "windows-1250"
	CharsetWindows1251 = "windows-1251"
	CharsetWindows1252 = "windows-1252"
	CharsetWindows1253 = "windows-1253"
	CharsetWindows1254 = "windows-1254"
	CharsetWindows1255 = "windows-1255"
	CharsetWindows1256 = "windows-1256"
	CharsetWindows1257 = "windows-1257"
	CharsetWindows1258 = "windows-1258"
	CharsetUTF_8       = "utf-8"
)

// --------------------------------------------------------

// ParserConf contains configuration parameters for
// vertical file parser
type ParserConf struct {

	// Source vertical file (either a plain text file or a gzip one)
	InputFilePath string `json:"inputFilePath"`

	Encoding string `json:"encoding"`

	FilterArgs [][][]string `json:"filterArgs"`

	StructAttrAccumulator string `json:"structAttrAccumulator"`

	LogProgressEachNth int `json:"logProgressEachNth"`
}

// LoadConfig loads the configuration from a JSON file.
// In case of an error the program exits with panic.
func LoadConfig(path string) *ParserConf {
	rawData, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	var conf ParserConf
	err = json.Unmarshal(rawData, &conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

// ------

type structAttrAccumulator interface {
	Begin(value *Structure) error
	End(name string) (*Structure, error)
	GetAttrs() map[string]string
	Size() int
}

// --------------------------------------------------------

// LineProcessor describes an object able to handle
// Vertigo's parsing events.
type LineProcessor interface {

	// ProcToken is called each time the parser encounters a positional
	// attribute. In case parsing produces an error, it is passed to the
	// function without stopping the whole process.
	// In case the function returns an error, the parser stops
	// (in the simplest case it can be even the error it recieves)
	ProcToken(token *Token, line int, err error) error

	// ProcStruct is called each time parser encounters a structure opening
	// element (e.g. <doc>). In case parsing produces an error, it is passed
	// to the function without stopping the whole process.
	// In case the function returns an error, the parser stops.
	ProcStruct(strc *Structure, line int, err error) error

	// ProcStructClose is called each time parser encouters a structure
	// closing element (e.g. </doc>). In case parsing produces an error,
	// it is passed to the function without stopping the whole process.
	// In case the function returns an error, the parser stops.
	ProcStructClose(strc *StructureClose, line int, err error) error
}

// ----

type procItem struct {
	idx   int
	value interface{}
	err   error
}

// --------------------------------------------------------

func createStructAttrAccumulator(ident string) (structAttrAccumulator, error) {
	switch ident {
	case AccumulatorTypeStack:
		return newStack(), nil
	case AccumulatorTypeComb:
		return newStructAttrs(), nil
	case AccumulatorTypeNil:
		return newNilStructAttrs(), nil
	default:
		return nil, fmt.Errorf("Unknown accumulator type \"%s\"", ident)
	}
}

// SupportedCharsets returns a list of names of
// character sets.
func SupportedCharsets() []string {
	return []string{CharsetISO8859_2, CharsetUTF_8, CharsetWindows1250}
}

// GetCharmapByName returns a proper Charmap instance based
// on provided encoding name. The name detection is case
// insensitive (e.g. utf-8 is the same as UTF-8). The number
// of supported charsets is
func GetCharmapByName(name string) (*charmap.Charmap, error) {
	switch strings.ToLower(name) {
	case CharsetISO8859_1:
		return charmap.ISO8859_1, nil
	case CharsetISO8859_2:
		return charmap.ISO8859_2, nil
	case CharsetISO8859_3:
		return charmap.ISO8859_3, nil
	case CharsetISO8859_4:
		return charmap.ISO8859_4, nil
	case CharsetISO8859_5:
		return charmap.ISO8859_5, nil
	case CharsetISO8859_6:
		return charmap.ISO8859_6, nil
	case CharsetISO8859_7:
		return charmap.ISO8859_7, nil
	case CharsetISO8859_8:
		return charmap.ISO8859_8, nil
	case CharsetWindows1250:
		return charmap.Windows1250, nil
	case CharsetWindows1251:
		return charmap.Windows1251, nil
	case CharsetWindows1252:
		return charmap.Windows1252, nil
	case CharsetWindows1253:
		return charmap.Windows1253, nil
	case CharsetWindows1254:
		return charmap.Windows1254, nil
	case CharsetWindows1255:
		return charmap.Windows1255, nil
	case CharsetWindows1256:
		return charmap.Windows1256, nil
	case CharsetWindows1257:
		return charmap.Windows1257, nil
	case CharsetWindows1258:
		return charmap.Windows1258, nil
	case CharsetUTF_8:
		return nil, nil
	case "":
		log.Printf("No charset specified, assuming utf-8")
		return nil, nil
	default:
		return nil, fmt.Errorf("Unsupported charset '%s'", name)
	}
}

func importString(s string, ch *charmap.Charmap) string {
	if ch == nil {
		return s
	}
	ans, _, _ := transform.String(ch.NewDecoder(), s)
	// TODO handle error
	return ans
}

func openInputFile(path string) (io.Reader, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	finfo, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if !finfo.Mode().IsRegular() {
		return nil, fmt.Errorf("Path %s is not a regular file", path)
	}

	var rd io.Reader
	if strings.HasSuffix(path, ".gz") {
		rd, err = gzip.NewReader(f)
		if err != nil {
			return nil, err
		}

	} else {
		rd = f
	}
	return rd, nil
}

// ParseVerticalFile processes a corpus vertical file
// line by line and applies a custom LineProcessor on
// them. The processing is parallelized in the sense
// that reading a file into lines and processing of
// the lines runs in different goroutines. But the
// function as a whole behaves synchronously - i.e.
// once it returns a value, the processing is finished.
func ParseVerticalFile(conf *ParserConf, lproc LineProcessor) error {
	logProgressEachNth := logProgressEachNthDefault
	if conf.LogProgressEachNth > 0 {
		logProgressEachNth = conf.LogProgressEachNth
	}
	rd, err := openInputFile(conf.InputFilePath)
	if err != nil {
		return err
	}
	brd := bufio.NewScanner(rd)

	stack, err := createStructAttrAccumulator(conf.StructAttrAccumulator)
	if err != nil {
		return err
	}

	chm, chErr := GetCharmapByName(conf.Encoding)
	if chErr != nil {
		return chErr
	}
	log.Printf("Configured conversion from charset %s", chm)
	ch := make(chan []procItem)
	chunk := make([]procItem, channelChunkSize)
	stop := make(chan struct{})
	defer close(stop)

	go func() {
		defer close(ch)
		i := 0
		lineNum := 0
		tokenNum := 0
		for brd.Scan() {
			line, parseErr := parseLine(importString(brd.Text(), chm), stack)
			tok, isTok := line.(*Token)
			if isTok {
				tok.Idx = tokenNum
				tokenNum++
			}
			chunk[i] = procItem{idx: lineNum, value: line, err: parseErr}
			i++
			if i == channelChunkSize {
				i = 0
				ch <- chunk
				chunk = make([]procItem, channelChunkSize)
			}
			if lineNum%logProgressEachNth == 0 {
				log.Printf("...processed %d lines.\n", lineNum)
			}
			lineNum++
			select {
			case <-stop:
				fmt.Println("STOPPING PARSING")
				return
			default:
			}
		}
		if i > 0 {
			ch <- chunk[:i]
		}
	}()

	var procErr error
	for items := range ch {
		for _, item := range items {
			switch item.value.(type) {
			case *Token:
				tk := item.value.(*Token)
				if tk.MatchesFilter(conf.FilterArgs) {
					procErr = lproc.ProcToken(tk, item.idx, err)
				}
			case *Structure:
				procErr = lproc.ProcStruct(item.value.(*Structure), item.idx, item.err)
			case *StructureClose:
				procErr = lproc.ProcStructClose(item.value.(*StructureClose), item.idx, item.err)
			}
			if procErr != nil {
				return procErr
			}
		}
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
	return nil
}

//ParseVerticalFileNoGoRo is just for benchmarking purposes
func ParseVerticalFileNoGoRo(conf *ParserConf, lproc LineProcessor) {
	f, err := os.Open(conf.InputFilePath)
	if err != nil {
		panic(err)
	}
	rd := bufio.NewScanner(f)
	stack := newStack()
	i := 0
	for rd.Scan() {
		token, err := parseLine(rd.Text(), stack)
		switch token.(type) {
		case *Token:
			lproc.ProcToken(token.(*Token), i, err)
		}
		i++
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
}
<|endoftext|>


<commit_before>package jo

const (
	_StateValue = iota
	_StateDone

	_StateObjectKeyOrBrace   // {
	_StateObjectColon        // {"foo"
	_StateObjectCommaOrBrace // {"foo":"bar"
	_StateObjectKey          // {"foo":"bar",

	_StateArrayValueOrBracket // [
	_StateArrayCommaOrBracket // ["any value"

	// leading spaces must be be consumed from all states listed above
	_IgnoreSpace

	_StateStringUnicode  // "\u
	_StateStringUnicode2 // "\u1
	_StateStringUnicode3 // "\u12
	_StateStringUnicode4 // "\u123
	_StateString         // "
	_StateStringEscaped  // "\

	_StateKeyUnicode  // "\u
	_StateKeyUnicode2 // "\u1
	_StateKeyUnicode3 // "\u12
	_StateKeyUnicode4 // "\u123
	_StateKey         // "
	_StateKeyEscaped  // "\

	_StateNumberNegative      // -
	_StateNumberZero          // 0
	_StateNumber              // 123
	_StateNumberDotFirstDigit // 123.
	_StateNumberDotDigit      // 123.4
	_StateNumberExpSign       // 123e
	_StateNumberExpFirstDigit // 123e+
	_StateNumberExpDigit      // 123e+1

	_StateTrue  // t
	_StateTrue2 // tr
	_StateTrue3 // tru

	_StateFalse  // f
	_StateFalse2 // fa
	_StateFalse3 // fal
	_StateFalse4 // fals

	_StateNull  // n
	_StateNull2 // nu
	_StateNull3 // nul
)

// Parser state machine.
type Parser struct {
	state int
	queue []int
	err   error
}

// Our own little implementation of the `error` interface.
type err string

func (e err) Error() string {
	return string(e)
}

// Returns the syntax error stored in the parser, if any.
func (p *Parser) LastError() error {
	return p.err
}

// Parses a byte slice containing JSON data. Returns the number of bytes
// read and an appropriate Event.
func (p *Parser) Parse(input []byte) (int, Event) {
	for i := 0; i < len(input); i++ {
		var event Event = Continue
		var b byte = input[i]

		if p.state < _IgnoreSpace && isSpace(b) {
			continue
		}

		switch p.state {
		case _StateValue:
			if b == '{' {
				event = ObjectStart
				p.state = _StateObjectKeyOrBrace
			} else if b == '[' {
				event = ArrayStart
				p.state = _StateArrayValueOrBracket
			} else if b == '"' {
				event = StringStart
				p.state = _StateString
			} else if b == '-' {
				event = NumberStart
				p.state = _StateNumberNegative
			} else if b == '0' {
				event = NumberStart
				p.state = _StateNumberZero
			} else if '1' <= b && b <= '9' {
				event = NumberStart
				p.state = _StateNumber
			} else if b == 't' {
				event = BoolStart
				p.state = _StateTrue
			} else if b == 'f' {
				event = BoolStart
				p.state = _StateFalse
			} else if b == 'n' {
				event = NullStart
				p.state = _StateNull
			} else {
				event = p.error(`expecting beginning of JSON value`)
			}

		case _StateObjectKeyOrBrace:
			if b == '}' {
				event = ObjectEnd
				p.state = p.next()
				break
			}

			// if it's not a brace, it must be a key
			p.state = _StateObjectKey
			fallthrough

		case _StateObjectKey:
			if b == '"' {
				event = KeyStart
				p.state = _StateKey
			} else {
				event = p.error(`expected object key`)
			}

		case _StateObjectColon:
			if b == ':' {
				p.state = _StateValue
				p.push(_StateObjectCommaOrBrace)
			} else {
				event = p.error(`expected ':' after object key`)
			}

		case _StateObjectCommaOrBrace:
			if b == ',' {
				p.state = _StateObjectKey
			} else if b == '}' {
				event = ObjectEnd
				p.state = p.next()
			} else {
				event = p.error(`expected ',' or '}' after object value`)
			}

		case _StateArrayValueOrBracket:
			if b == ']' {
				event = ArrayEnd
				p.state = p.next()
			} else {
				p.state = _StateValue
				p.push(_StateArrayCommaOrBracket)

				// rewind and let _StateValue parse this byte for us
				i--
			}

		case _StateArrayCommaOrBracket:
			if b == ',' {
				p.state = _StateValue
				p.push(_StateArrayCommaOrBracket)
			} else if b == ']' {
				event = ArrayEnd
				p.state = p.next()
			} else {
				event = p.error(`expected ',' or ']' after array value`)
			}

		case _StateStringUnicode, _StateKeyUnicode,
			_StateStringUnicode2, _StateKeyUnicode2,
			_StateStringUnicode3, _StateKeyUnicode3,
			_StateStringUnicode4, _StateKeyUnicode4:
			if isHex(b) {
				// move on to the next unicode byte state, or back to
				// `_State{String/Key}` if this was the fourth hexadecimal
				// character after "\u"
				p.state++
			} else {
				event = p.error(`expected four hexadecimal chars after "\u"`)
			}

		case _StateString, _StateKey:
			if b == '"' {
				if p.state == _StateKey {
					event = KeyEnd
					p.state = _StateObjectColon
				} else {
					event = StringEnd
					p.state = p.next()
				}
			} else if b == '\\' {
				// continue with `_State{String,Key}Escaped`
				p.state++
			} else if b < 0x20 {
				event = p.error(`expected valid string character`)
			}

		case _StateStringEscaped, _StateKeyEscaped:
			switch b {
			case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
				// jump back to _State{String,Key}
				p.state--
			case 'u':
				// continue to `_State{String,Key}Unicode`
				p.state -= 5
			default:
				event = p.error(`expected valid escape sequence after '\'`)
			}

		case _StateNumberNegative:
			if b == '0' {
				p.state = _StateNumberZero
			} else if '1' <= b && b <= '9' {
				p.state = _StateNumber
			} else {
				event = p.error(`digit after '-'`)
			}

		case _StateNumber:
			if isDigit(b) {
				break
			}

			// the same limits apply here as in _StateNumberZero
			fallthrough

		case _StateNumberZero:
			if b == '.' {
				p.state = _StateNumberDotFirstDigit
			} else if b == 'e' || b == 'E' {
				p.state = _StateNumberExpSign
			} else {
				event = NumberEnd
				p.state = p.next()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case _StateNumberDotFirstDigit:
			if isDigit(b) {
				p.state = _StateNumberDotDigit
			} else {
				event = p.error(`expected digit after dot in number`)
			}

		case _StateNumberDotDigit:
			if b == 'e' || b == 'E' {
				p.state = _StateNumberExpSign
			} else if !isDigit(b) {
				event = NumberEnd
				p.state = p.next()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case _StateNumberExpSign:
			p.state = _StateNumberExpFirstDigit
			if b == '+' || b == '-' {
				break
			}
			fallthrough

		case _StateNumberExpFirstDigit:
			if !isDigit(b) {
				event = p.error(`expected digit after exponent in number`)
			} else {
				p.state++
			}

		case _StateNumberExpDigit:
			if !isDigit(b) {
				event = NumberEnd
				p.state = p.next()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case _StateTrue:
			if b == 'r' {
				p.state = _StateTrue2
			} else {
				event = p.error(`expected 'r' in literal true`)
			}

		case _StateTrue2:
			if b == 'u' {
				p.state = _StateTrue3
			} else {
				return i, p.error(`expected 'u' in literal true`)
			}

		case _StateTrue3:
			if b == 'e' {
				event = BoolEnd
				p.state = p.next()
			} else {
				return i, p.error(`expected 'e' in literal true`)
			}

		case _StateFalse:
			if b == 'a' {
				p.state = _StateFalse2
			} else {
				event = p.error(`expected 'a' in literal false`)
			}

		case _StateFalse2:
			if b == 'l' {
				p.state = _StateFalse3
			} else {
				event = p.error(`expected 'l' in literal false`)
			}

		case _StateFalse3:
			if b == 's' {
				p.state = _StateFalse4
			} else {
				event = p.error(`expected 's' in literal false`)
			}

		case _StateFalse4:
			if b == 'e' {
				event = BoolEnd
				p.state = p.next()
			} else {
				event = p.error(`expected 'e' in literal false`)
			}

		case _StateNull:
			if b == 'u' {
				p.state = _StateNull2
			} else {
				event = p.error(`expected 'u' in literal false`)
			}

		case _StateNull2:
			if b == 'l' {
				p.state = _StateNull3
			} else {
				event = p.error(`expected 'l' in literal false`)
			}

		case _StateNull3:
			if b == 'l' {
				event = NullEnd
				p.state = p.next()
			} else {
				event = p.error(`expected 'l' in literal false`)
			}

		case _StateDone:
			return i, p.error(`expected nothing after top-level value`)

		default:
			panic(`invalid state`)
		}

		if event == SyntaxError {
			// don't consume the byte that caused the error
			return i, SyntaxError
		}

		if event != Continue {
			return i + 1, event
		}
	}

	return len(input), Continue
}

// Informs the parser not to expect any further input. Will return pending
// NumberEnd events (if there are any), or a SyntaxError event if EOF was
// not expected -- otherwise Done.
func (p *Parser) Eof() Event {
	switch p.state {
	case _StateNumberZero,
		_StateNumber,
		_StateNumberDotDigit,
		_StateNumberExpDigit:
		p.state = _StateDone
		return NumberEnd
	case _StateDone:
		return Done
	}
	return p.error(`.Eof(): @todo`)
}

// Pops the next state off the parser struct's queue.
func (p *Parser) next() int {
	length := len(p.queue)

	// with the "state queue" empty, we can only wait for EOF
	if length == 0 {
		return _StateDone
	}

	state := p.queue[length-1]
	p.queue = p.queue[:length-1]

	return state
}

// Insert a new state at the top of the queue.
func (p *Parser) push(state int) {
	p.queue = append(p.queue, state)
}

// Registers a syntax error. Always returns a SyntaxError event.
func (p *Parser) error(message string) Event {
	p.err = err(message)
	return SyntaxError
}

func isSpace(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

func isHex(b byte) bool {
	return isDigit(b) || ('a' <= b && b <= 'f') || ('A' <= b && b <= 'F')
}

func isDigit(b byte) bool {
	return '0' <= b && b <= '9'
}
<commit_msg>Clarify comment regarding certain states and whitespace<commit_after>package jo

const (
	_StateValue = iota
	_StateDone

	_StateObjectKeyOrBrace   // {
	_StateObjectColon        // {"foo"
	_StateObjectCommaOrBrace // {"foo":"bar"
	_StateObjectKey          // {"foo":"bar",

	_StateArrayValueOrBracket // [
	_StateArrayCommaOrBracket // ["any value"

	// leading whitespace must be be consumed before any of the above
	// states are processed
	_IgnoreSpace

	_StateStringUnicode  // "\u
	_StateStringUnicode2 // "\u1
	_StateStringUnicode3 // "\u12
	_StateStringUnicode4 // "\u123
	_StateString         // "
	_StateStringEscaped  // "\

	_StateKeyUnicode  // "\u
	_StateKeyUnicode2 // "\u1
	_StateKeyUnicode3 // "\u12
	_StateKeyUnicode4 // "\u123
	_StateKey         // "
	_StateKeyEscaped  // "\

	_StateNumberNegative      // -
	_StateNumberZero          // 0
	_StateNumber              // 123
	_StateNumberDotFirstDigit // 123.
	_StateNumberDotDigit      // 123.4
	_StateNumberExpSign       // 123e
	_StateNumberExpFirstDigit // 123e+
	_StateNumberExpDigit      // 123e+1

	_StateTrue  // t
	_StateTrue2 // tr
	_StateTrue3 // tru

	_StateFalse  // f
	_StateFalse2 // fa
	_StateFalse3 // fal
	_StateFalse4 // fals

	_StateNull  // n
	_StateNull2 // nu
	_StateNull3 // nul
)

// Parser state machine.
type Parser struct {
	state int
	queue []int
	err   error
}

// Our own little implementation of the `error` interface.
type err string

func (e err) Error() string {
	return string(e)
}

// Returns the syntax error stored in the parser, if any.
func (p *Parser) LastError() error {
	return p.err
}

// Parses a byte slice containing JSON data. Returns the number of bytes
// read and an appropriate Event.
func (p *Parser) Parse(input []byte) (int, Event) {
	for i := 0; i < len(input); i++ {
		var event Event = Continue
		var b byte = input[i]

		if p.state < _IgnoreSpace && isSpace(b) {
			continue
		}

		switch p.state {
		case _StateValue:
			if b == '{' {
				event = ObjectStart
				p.state = _StateObjectKeyOrBrace
			} else if b == '[' {
				event = ArrayStart
				p.state = _StateArrayValueOrBracket
			} else if b == '"' {
				event = StringStart
				p.state = _StateString
			} else if b == '-' {
				event = NumberStart
				p.state = _StateNumberNegative
			} else if b == '0' {
				event = NumberStart
				p.state = _StateNumberZero
			} else if '1' <= b && b <= '9' {
				event = NumberStart
				p.state = _StateNumber
			} else if b == 't' {
				event = BoolStart
				p.state = _StateTrue
			} else if b == 'f' {
				event = BoolStart
				p.state = _StateFalse
			} else if b == 'n' {
				event = NullStart
				p.state = _StateNull
			} else {
				event = p.error(`expecting beginning of JSON value`)
			}

		case _StateObjectKeyOrBrace:
			if b == '}' {
				event = ObjectEnd
				p.state = p.next()
				break
			}

			// if it's not a brace, it must be a key
			p.state = _StateObjectKey
			fallthrough

		case _StateObjectKey:
			if b == '"' {
				event = KeyStart
				p.state = _StateKey
			} else {
				event = p.error(`expected object key`)
			}

		case _StateObjectColon:
			if b == ':' {
				p.state = _StateValue
				p.push(_StateObjectCommaOrBrace)
			} else {
				event = p.error(`expected ':' after object key`)
			}

		case _StateObjectCommaOrBrace:
			if b == ',' {
				p.state = _StateObjectKey
			} else if b == '}' {
				event = ObjectEnd
				p.state = p.next()
			} else {
				event = p.error(`expected ',' or '}' after object value`)
			}

		case _StateArrayValueOrBracket:
			if b == ']' {
				event = ArrayEnd
				p.state = p.next()
			} else {
				p.state = _StateValue
				p.push(_StateArrayCommaOrBracket)

				// rewind and let _StateValue parse this byte for us
				i--
			}

		case _StateArrayCommaOrBracket:
			if b == ',' {
				p.state = _StateValue
				p.push(_StateArrayCommaOrBracket)
			} else if b == ']' {
				event = ArrayEnd
				p.state = p.next()
			} else {
				event = p.error(`expected ',' or ']' after array value`)
			}

		case _StateStringUnicode, _StateKeyUnicode,
			_StateStringUnicode2, _StateKeyUnicode2,
			_StateStringUnicode3, _StateKeyUnicode3,
			_StateStringUnicode4, _StateKeyUnicode4:
			if isHex(b) {
				// move on to the next unicode byte state, or back to
				// `_State{String/Key}` if this was the fourth hexadecimal
				// character after "\u"
				p.state++
			} else {
				event = p.error(`expected four hexadecimal chars after "\u"`)
			}

		case _StateString, _StateKey:
			if b == '"' {
				if p.state == _StateKey {
					event = KeyEnd
					p.state = _StateObjectColon
				} else {
					event = StringEnd
					p.state = p.next()
				}
			} else if b == '\\' {
				// continue with `_State{String,Key}Escaped`
				p.state++
			} else if b < 0x20 {
				event = p.error(`expected valid string character`)
			}

		case _StateStringEscaped, _StateKeyEscaped:
			switch b {
			case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
				// jump back to _State{String,Key}
				p.state--
			case 'u':
				// continue to `_State{String,Key}Unicode`
				p.state -= 5
			default:
				event = p.error(`expected valid escape sequence after '\'`)
			}

		case _StateNumberNegative:
			if b == '0' {
				p.state = _StateNumberZero
			} else if '1' <= b && b <= '9' {
				p.state = _StateNumber
			} else {
				event = p.error(`digit after '-'`)
			}

		case _StateNumber:
			if isDigit(b) {
				break
			}

			// the same limits apply here as in _StateNumberZero
			fallthrough

		case _StateNumberZero:
			if b == '.' {
				p.state = _StateNumberDotFirstDigit
			} else if b == 'e' || b == 'E' {
				p.state = _StateNumberExpSign
			} else {
				event = NumberEnd
				p.state = p.next()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case _StateNumberDotFirstDigit:
			if isDigit(b) {
				p.state = _StateNumberDotDigit
			} else {
				event = p.error(`expected digit after dot in number`)
			}

		case _StateNumberDotDigit:
			if b == 'e' || b == 'E' {
				p.state = _StateNumberExpSign
			} else if !isDigit(b) {
				event = NumberEnd
				p.state = p.next()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case _StateNumberExpSign:
			p.state = _StateNumberExpFirstDigit
			if b == '+' || b == '-' {
				break
			}
			fallthrough

		case _StateNumberExpFirstDigit:
			if !isDigit(b) {
				event = p.error(`expected digit after exponent in number`)
			} else {
				p.state++
			}

		case _StateNumberExpDigit:
			if !isDigit(b) {
				event = NumberEnd
				p.state = p.next()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case _StateTrue:
			if b == 'r' {
				p.state = _StateTrue2
			} else {
				event = p.error(`expected 'r' in literal true`)
			}

		case _StateTrue2:
			if b == 'u' {
				p.state = _StateTrue3
			} else {
				return i, p.error(`expected 'u' in literal true`)
			}

		case _StateTrue3:
			if b == 'e' {
				event = BoolEnd
				p.state = p.next()
			} else {
				return i, p.error(`expected 'e' in literal true`)
			}

		case _StateFalse:
			if b == 'a' {
				p.state = _StateFalse2
			} else {
				event = p.error(`expected 'a' in literal false`)
			}

		case _StateFalse2:
			if b == 'l' {
				p.state = _StateFalse3
			} else {
				event = p.error(`expected 'l' in literal false`)
			}

		case _StateFalse3:
			if b == 's' {
				p.state = _StateFalse4
			} else {
				event = p.error(`expected 's' in literal false`)
			}

		case _StateFalse4:
			if b == 'e' {
				event = BoolEnd
				p.state = p.next()
			} else {
				event = p.error(`expected 'e' in literal false`)
			}

		case _StateNull:
			if b == 'u' {
				p.state = _StateNull2
			} else {
				event = p.error(`expected 'u' in literal false`)
			}

		case _StateNull2:
			if b == 'l' {
				p.state = _StateNull3
			} else {
				event = p.error(`expected 'l' in literal false`)
			}

		case _StateNull3:
			if b == 'l' {
				event = NullEnd
				p.state = p.next()
			} else {
				event = p.error(`expected 'l' in literal false`)
			}

		case _StateDone:
			return i, p.error(`expected nothing after top-level value`)

		default:
			panic(`invalid state`)
		}

		if event == SyntaxError {
			// don't consume the byte that caused the error
			return i, SyntaxError
		}

		if event != Continue {
			return i + 1, event
		}
	}

	return len(input), Continue
}

// Informs the parser not to expect any further input. Will return pending
// NumberEnd events (if there are any), or a SyntaxError event if EOF was
// not expected -- otherwise Done.
func (p *Parser) Eof() Event {
	switch p.state {
	case _StateNumberZero,
		_StateNumber,
		_StateNumberDotDigit,
		_StateNumberExpDigit:
		p.state = _StateDone
		return NumberEnd
	case _StateDone:
		return Done
	}
	return p.error(`.Eof(): @todo`)
}

// Pops the next state off the parser struct's queue.
func (p *Parser) next() int {
	length := len(p.queue)

	// with the "state queue" empty, we can only wait for EOF
	if length == 0 {
		return _StateDone
	}

	state := p.queue[length-1]
	p.queue = p.queue[:length-1]

	return state
}

// Insert a new state at the top of the queue.
func (p *Parser) push(state int) {
	p.queue = append(p.queue, state)
}

// Registers a syntax error. Always returns a SyntaxError event.
func (p *Parser) error(message string) Event {
	p.err = err(message)
	return SyntaxError
}

func isSpace(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

func isHex(b byte) bool {
	return isDigit(b) || ('a' <= b && b <= 'f') || ('A' <= b && b <= 'F')
}

func isDigit(b byte) bool {
	return '0' <= b && b <= '9'
}
<|endoftext|>


<commit_before>// pather is a command-line tool to make working with Unix paths easier.
package main

import (
	"bufio"
	"fmt"
	"github.com/kr/fs"
	"github.com/ogier/pflag"
	"os"
	"runtime"
	"strconv"
	"strings"
)

// getLinuxSearchSources will return a list of known locations for PATH
// segments in Ubuntu Linux.
func getLinuxSearchSources(home string) []string {
	// TODO: add official support for other distributions.
	bashrc := home + "/.bashrc"
	bashprofile := home + "/.bash_profile"
	profile := home + "/.profile"
	env := "/etc/environment"

	return []string{bashrc, bashprofile, profile, env}
}

// getDarwinSearchSources will return a list of known locations for PATH
// segments in OS X.
func getDarwinSearchSources(home string) []string {
	bashprofile := home + "/.bash_profile"
	paths := "/etc/paths"

	searchSources := []string{bashprofile, paths}

	// Lastly, grab all the files under paths.d.
	walker := fs.Walk("/etc/paths.d")
	for walker.Step() {
		if err := walker.Err(); err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		// We want to exclude the top-level directory.
		if walker.Path() != "/etc/paths.d" {
			searchSources = append(searchSources, walker.Path())
		}
	}

	return searchSources
}

// getSearchSources will return a list of search locations that typically set
// path elements. The list depends on the user's OS.
func getSearchSources() []string {
	home := os.Getenv("HOME")
	var searchSources []string

	switch runtime.GOOS {
	case "linux":
		// Ubuntu
		searchSources = getLinuxSearchSources(home)
	case "darwin":
		// OS X
		searchSources = getDarwinSearchSources(home)
	}

	return searchSources
}

// appendSource will search through a list of possible locations, provided by
// getSearchSources(), where the path may have been set and append that data to
// the path string. If it can't be located, "unknown" will be returned.
func appendSource(path string, pathChan chan string) {
	pathSetBy := path + " set by: "

	for _, source := range getSearchSources() {
		f, err := os.Open(source)
		if err != nil {
			// Allow execution to continue as some files are optional.
			//panic(err)
		}
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for i := 1; scanner.Scan(); i++ {
			if runtime.GOOS == "darwin" {
				if strings.Contains(scanner.Text(), path) {
					p := pathSetBy + source + " (line " + strconv.Itoa(i) + ")"
					pathChan <- p
					return
				}
			} else {
				if strings.Contains(scanner.Text(), "PATH=") {
					if strings.Contains(scanner.Text(), path) {
						p := pathSetBy + source + " (line " + strconv.Itoa(i) + ")"
						pathChan <- p
						return
					}
				}
			}
		}
	}

	pathChan <- pathSetBy + "unknown"
	return
}

// returnPathList returns a slice of path strings with or without extra details.
// The strings should be printed to stdout.
func returnPathList(detailedList bool) []string {
	pathList := strings.Split(os.Getenv("PATH"), ":")
	if !detailedList {
		return pathList
	}

	pathChan := make(chan string, len(pathList))
	for _, path := range pathList {
		go appendSource(path, pathChan)
	}

	var appendedPathList []string
	for i := 0; i < len(pathList); i++ {
		appendedPathList = append(appendedPathList, <-pathChan)
	}

	return appendedPathList
}

func main() {
	const listUsage = "use a long listing format"
	useList := pflag.BoolP("list", "l", false, listUsage)

	const detailedUsage = "use a (detailed) long listing format"
	detailedList := pflag.BoolP("detailed-list", "d", false, detailedUsage)

	pflag.Parse()

	if !(*useList || *detailedList) {
		fmt.Println(os.Getenv("PATH"))
		return
	}

	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
		fmt.Println("Sorry, detailed list only supports Linux and OS X for now.")
	}

	for _, p := range returnPathList(*detailedList) {
		fmt.Println(p)
	}
}
<commit_msg>Added more detail to the header comment.<commit_after>/*
pather is a command-line tool to make working with Unix paths easier.

Usage of pather:
  -d, --detailed-list=false: use a (detailed) long listing format
  -l, --list=false: use a long listing format
*/
package main

import (
	"bufio"
	"fmt"
	"github.com/kr/fs"
	"github.com/ogier/pflag"
	"os"
	"runtime"
	"strconv"
	"strings"
)

// getLinuxSearchSources will return a list of known locations for PATH
// segments in Ubuntu Linux.
func getLinuxSearchSources(home string) []string {
	// TODO: add official support for other distributions.
	bashrc := home + "/.bashrc"
	bashprofile := home + "/.bash_profile"
	profile := home + "/.profile"
	env := "/etc/environment"

	return []string{bashrc, bashprofile, profile, env}
}

// getDarwinSearchSources will return a list of known locations for PATH
// segments in OS X.
func getDarwinSearchSources(home string) []string {
	bashprofile := home + "/.bash_profile"
	paths := "/etc/paths"

	searchSources := []string{bashprofile, paths}

	// Lastly, grab all the files under paths.d.
	walker := fs.Walk("/etc/paths.d")
	for walker.Step() {
		if err := walker.Err(); err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		// We want to exclude the top-level directory.
		if walker.Path() != "/etc/paths.d" {
			searchSources = append(searchSources, walker.Path())
		}
	}

	return searchSources
}

// getSearchSources will return a list of search locations that typically set
// path elements. The list depends on the user's OS.
func getSearchSources() []string {
	home := os.Getenv("HOME")
	var searchSources []string

	switch runtime.GOOS {
	case "linux":
		// Ubuntu
		searchSources = getLinuxSearchSources(home)
	case "darwin":
		// OS X
		searchSources = getDarwinSearchSources(home)
	}

	return searchSources
}

// appendSource will search through a list of possible locations, provided by
// getSearchSources(), where the path may have been set and append that data to
// the path string. If it can't be located, "unknown" will be returned.
func appendSource(path string, pathChan chan string) {
	pathSetBy := path + " set by: "

	for _, source := range getSearchSources() {
		f, err := os.Open(source)
		if err != nil {
			// Allow execution to continue as some files are optional.
			//panic(err)
		}
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for i := 1; scanner.Scan(); i++ {
			if runtime.GOOS == "darwin" {
				if strings.Contains(scanner.Text(), path) {
					p := pathSetBy + source + " (line " + strconv.Itoa(i) + ")"
					pathChan <- p
					return
				}
			} else {
				if strings.Contains(scanner.Text(), "PATH=") {
					if strings.Contains(scanner.Text(), path) {
						p := pathSetBy + source + " (line " + strconv.Itoa(i) + ")"
						pathChan <- p
						return
					}
				}
			}
		}
	}

	pathChan <- pathSetBy + "unknown"
	return
}

// returnPathList returns a slice of path strings with or without extra details.
// The strings should be printed to stdout.
func returnPathList(detailedList bool) []string {
	pathList := strings.Split(os.Getenv("PATH"), ":")
	if !detailedList {
		return pathList
	}

	pathChan := make(chan string, len(pathList))
	for _, path := range pathList {
		go appendSource(path, pathChan)
	}

	var appendedPathList []string
	for i := 0; i < len(pathList); i++ {
		appendedPathList = append(appendedPathList, <-pathChan)
	}

	return appendedPathList
}

func main() {
	const listUsage = "use a long listing format"
	useList := pflag.BoolP("list", "l", false, listUsage)

	const detailedUsage = "use a (detailed) long listing format"
	detailedList := pflag.BoolP("detailed-list", "d", false, detailedUsage)

	pflag.Parse()

	if !(*useList || *detailedList) {
		fmt.Println(os.Getenv("PATH"))
		return
	}

	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
		fmt.Println("Sorry, detailed list only supports Linux and OS X for now.")
	}

	for _, p := range returnPathList(*detailedList) {
		fmt.Println(p)
	}
}
<|endoftext|>


<commit_before>package ruler

import (
	"flag"
	"fmt"
	"net/url"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/rules"
	"golang.org/x/net/context"

	"github.com/weaveworks/cortex/chunk"
	"github.com/weaveworks/cortex/distributor"
	"github.com/weaveworks/cortex/querier"
	"github.com/weaveworks/cortex/user"
	"github.com/weaveworks/cortex/util"
)

var (
	evalDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "group_evaluation_duration_seconds",
		Help:      "The duration for a rule group to execute.",
	})
	rulesProcessed = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "rules_processed_total",
		Help:      "How many rules have been processed.",
	})
	blockedWorkers = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "cortex",
		Name:      "blocked_workers",
		Help:      "How many workers are waiting on an item to be ready.",
	})
)

func init() {
	prometheus.MustRegister(evalDuration)
	prometheus.MustRegister(rulesProcessed)
	prometheus.MustRegister(blockedWorkers)
}

// Config is the configuration for the recording rules server.
type Config struct {
	ConfigsAPIURL util.URLValue

	// HTTP timeout duration for requests made to the Weave Cloud configs
	// service.
	ClientTimeout time.Duration

	// This is used for template expansion in alerts. Because we don't support
	// alerts yet, this value doesn't matter. However, it must be a valid URL
	// in order to navigate Prometheus's code paths.
	ExternalURL util.URLValue

	// How frequently to evaluate rules by default.
	EvaluationInterval time.Duration
	NumWorkers         int
}

// RegisterFlags adds the flags required to config this to the given FlagSet
func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
	f.Var(&cfg.ConfigsAPIURL, "ruler.configs.url", "URL of configs API server.")
	f.Var(&cfg.ExternalURL, "ruler.external.url", "URL of alerts return path.")
	f.DurationVar(&cfg.EvaluationInterval, "ruler.evaluation-interval", 15*time.Second, "How frequently to evaluate rules")
	f.DurationVar(&cfg.ClientTimeout, "ruler.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.")
	f.IntVar(&cfg.NumWorkers, "ruler.num-workers", 1, "Number of rule evaluator worker routines in this process")
}

// Ruler evaluates rules.
type Ruler struct {
	engine   *promql.Engine
	pusher   Pusher
	alertURL *url.URL
}

// NewRuler creates a new ruler from a distributor and chunk store.
func NewRuler(cfg Config, d *distributor.Distributor, c chunk.Store) Ruler {
	return Ruler{querier.NewEngine(d, c), d, cfg.ExternalURL.URL}
}

func (r *Ruler) newGroup(ctx context.Context, rs []rules.Rule) *rules.Group {
	appender := appenderAdapter{pusher: r.pusher, ctx: ctx}
	opts := &rules.ManagerOptions{
		SampleAppender: appender,
		QueryEngine:    r.engine,
		Context:        ctx,
		ExternalURL:    r.alertURL,
	}
	delay := 0 * time.Second // Unused, so 0 value is fine.
	return rules.NewGroup("default", delay, rs, opts)
}

// Evaluate a list of rules in the given context.
func (r *Ruler) Evaluate(ctx context.Context, rs []rules.Rule) {
	log.Debugf("Evaluating %d rules...", len(rs))
	start := time.Now()
	g := r.newGroup(ctx, rs)
	g.Eval()
	// The prometheus routines we're calling have their own instrumentation
	// but, a) it's rule-based, not group-based, b) it's a summary, not a
	// histogram, so we can't reliably aggregate.
	evalDuration.Observe(time.Since(start).Seconds())
	rulesProcessed.Add(float64(len(rs)))
}

// Server is a rules server.
type Server struct {
	scheduler *scheduler
	workers   []worker
}

// NewServer makes a new rule processing server.
func NewServer(cfg Config, ruler Ruler) (*Server, error) {
	c := configsAPI{cfg.ConfigsAPIURL.URL, cfg.ClientTimeout}
	// TODO: Separate configuration for polling interval.
	s := newScheduler(c, cfg.EvaluationInterval, cfg.EvaluationInterval)
	if cfg.NumWorkers <= 0 {
		return nil, fmt.Errorf("must have at least 1 worker, got %d", cfg.NumWorkers)
	}
	workers := make([]worker, cfg.NumWorkers)
	for i := 0; i < cfg.NumWorkers; i++ {
		workers[i] = newWorker(&s, ruler)
	}
	return &Server{
		scheduler: &s,
		workers:   workers,
	}, nil
}

// Run the server.
func (s *Server) Run() {
	go s.scheduler.Run()
	for _, w := range s.workers {
		go w.Run()
	}
	log.Infof("Ruler up and running")
}

// Stop the server.
func (s *Server) Stop() {
	for _, w := range s.workers {
		w.Stop()
	}
	s.scheduler.Stop()
}

// Worker does a thing until it's told to stop.
type Worker interface {
	Run()
	Stop()
}

type worker struct {
	scheduler *scheduler
	ruler     Ruler

	done       chan struct{}
	terminated chan struct{}
}

func newWorker(scheduler *scheduler, ruler Ruler) worker {
	return worker{
		scheduler:  scheduler,
		ruler:      ruler,
		done:       make(chan struct{}),
		terminated: make(chan struct{}),
	}
}

func (w *worker) Run() {
	defer close(w.terminated)
	for {
		select {
		case <-w.done:
			return
		default:
		}
		blockedWorkers.Inc()
		log.Debugf("Waiting for next work item")
		item := w.scheduler.nextWorkItem()
		blockedWorkers.Dec()
		if item == nil {
			log.Debugf("Queue closed and empty. Terminating worker.")
			return
		}
		log.Debugf("Processing %v", item)
		ctx := user.WithID(context.Background(), item.userID)
		w.ruler.Evaluate(ctx, item.rules)
		w.scheduler.workItemDone(*item)
		log.Debugf("%v handed back to queue", item)
	}
}

func (w *worker) Stop() {
	close(w.done)
	<-w.terminated
}
<commit_msg>Initialise ruler's cfg.ExternalURL to a valid, empty URL. (#232)<commit_after>package ruler

import (
	"flag"
	"fmt"
	"net/url"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/rules"
	"golang.org/x/net/context"

	"github.com/weaveworks/cortex/chunk"
	"github.com/weaveworks/cortex/distributor"
	"github.com/weaveworks/cortex/querier"
	"github.com/weaveworks/cortex/user"
	"github.com/weaveworks/cortex/util"
)

var (
	evalDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "group_evaluation_duration_seconds",
		Help:      "The duration for a rule group to execute.",
	})
	rulesProcessed = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "rules_processed_total",
		Help:      "How many rules have been processed.",
	})
	blockedWorkers = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "cortex",
		Name:      "blocked_workers",
		Help:      "How many workers are waiting on an item to be ready.",
	})
)

func init() {
	prometheus.MustRegister(evalDuration)
	prometheus.MustRegister(rulesProcessed)
	prometheus.MustRegister(blockedWorkers)
}

// Config is the configuration for the recording rules server.
type Config struct {
	ConfigsAPIURL util.URLValue

	// HTTP timeout duration for requests made to the Weave Cloud configs
	// service.
	ClientTimeout time.Duration

	// This is used for template expansion in alerts. Because we don't support
	// alerts yet, this value doesn't matter. However, it must be a valid URL
	// in order to navigate Prometheus's code paths.
	ExternalURL util.URLValue

	// How frequently to evaluate rules by default.
	EvaluationInterval time.Duration
	NumWorkers         int
}

// RegisterFlags adds the flags required to config this to the given FlagSet
func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
	cfg.ExternalURL.URL, _ = url.Parse("") // Must be non-nil
	f.Var(&cfg.ConfigsAPIURL, "ruler.configs.url", "URL of configs API server.")
	f.Var(&cfg.ExternalURL, "ruler.external.url", "URL of alerts return path.")
	f.DurationVar(&cfg.EvaluationInterval, "ruler.evaluation-interval", 15*time.Second, "How frequently to evaluate rules")
	f.DurationVar(&cfg.ClientTimeout, "ruler.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.")
	f.IntVar(&cfg.NumWorkers, "ruler.num-workers", 1, "Number of rule evaluator worker routines in this process")
}

// Ruler evaluates rules.
type Ruler struct {
	engine   *promql.Engine
	pusher   Pusher
	alertURL *url.URL
}

// NewRuler creates a new ruler from a distributor and chunk store.
func NewRuler(cfg Config, d *distributor.Distributor, c chunk.Store) Ruler {
	return Ruler{querier.NewEngine(d, c), d, cfg.ExternalURL.URL}
}

func (r *Ruler) newGroup(ctx context.Context, rs []rules.Rule) *rules.Group {
	appender := appenderAdapter{pusher: r.pusher, ctx: ctx}
	opts := &rules.ManagerOptions{
		SampleAppender: appender,
		QueryEngine:    r.engine,
		Context:        ctx,
		ExternalURL:    r.alertURL,
	}
	delay := 0 * time.Second // Unused, so 0 value is fine.
	return rules.NewGroup("default", delay, rs, opts)
}

// Evaluate a list of rules in the given context.
func (r *Ruler) Evaluate(ctx context.Context, rs []rules.Rule) {
	log.Debugf("Evaluating %d rules...", len(rs))
	start := time.Now()
	g := r.newGroup(ctx, rs)
	g.Eval()
	// The prometheus routines we're calling have their own instrumentation
	// but, a) it's rule-based, not group-based, b) it's a summary, not a
	// histogram, so we can't reliably aggregate.
	evalDuration.Observe(time.Since(start).Seconds())
	rulesProcessed.Add(float64(len(rs)))
}

// Server is a rules server.
type Server struct {
	scheduler *scheduler
	workers   []worker
}

// NewServer makes a new rule processing server.
func NewServer(cfg Config, ruler Ruler) (*Server, error) {
	c := configsAPI{cfg.ConfigsAPIURL.URL, cfg.ClientTimeout}
	// TODO: Separate configuration for polling interval.
	s := newScheduler(c, cfg.EvaluationInterval, cfg.EvaluationInterval)
	if cfg.NumWorkers <= 0 {
		return nil, fmt.Errorf("must have at least 1 worker, got %d", cfg.NumWorkers)
	}
	workers := make([]worker, cfg.NumWorkers)
	for i := 0; i < cfg.NumWorkers; i++ {
		workers[i] = newWorker(&s, ruler)
	}
	return &Server{
		scheduler: &s,
		workers:   workers,
	}, nil
}

// Run the server.
func (s *Server) Run() {
	go s.scheduler.Run()
	for _, w := range s.workers {
		go w.Run()
	}
	log.Infof("Ruler up and running")
}

// Stop the server.
func (s *Server) Stop() {
	for _, w := range s.workers {
		w.Stop()
	}
	s.scheduler.Stop()
}

// Worker does a thing until it's told to stop.
type Worker interface {
	Run()
	Stop()
}

type worker struct {
	scheduler *scheduler
	ruler     Ruler

	done       chan struct{}
	terminated chan struct{}
}

func newWorker(scheduler *scheduler, ruler Ruler) worker {
	return worker{
		scheduler:  scheduler,
		ruler:      ruler,
		done:       make(chan struct{}),
		terminated: make(chan struct{}),
	}
}

func (w *worker) Run() {
	defer close(w.terminated)
	for {
		select {
		case <-w.done:
			return
		default:
		}
		blockedWorkers.Inc()
		log.Debugf("Waiting for next work item")
		item := w.scheduler.nextWorkItem()
		blockedWorkers.Dec()
		if item == nil {
			log.Debugf("Queue closed and empty. Terminating worker.")
			return
		}
		log.Debugf("Processing %v", item)
		ctx := user.WithID(context.Background(), item.userID)
		w.ruler.Evaluate(ctx, item.rules)
		w.scheduler.workItemDone(*item)
		log.Debugf("%v handed back to queue", item)
	}
}

func (w *worker) Stop() {
	close(w.done)
	<-w.terminated
}
<|endoftext|>