<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"image"
	"image/draw"
	"image/png"
)

type Size struct {
	x, y int
}
}

type sprite struct {
	name   string
	img    image.Image
	area   int
	offset image.Point
	size   Size
}

type ByArea []sprite

func (a ByArea) Len() int           { return len(a) }
func (a ByArea) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByArea) Less(i, j int) bool { return a[i].area < a[j].area }

type Node struct {
	child [2]*Node
	rect  image.Rectangle
	img   *sprite
}

func (n *Node) size() Size {
	return Size{n.rect.Dx(), n.rect.Dy()}
}

func (n *Node) print() {
	fmt.Println(n)
	if n.child[0] != nil {
		n.child[0].print()
	}
	if n.child[1] != nil {
		n.child[1].print()
	}
}

func (n *Node) isLeaf() bool {
	return n.child[0] == nil || n.child[1] == nil
}

func (n *Node) insert(img *sprite) *Node {
	if !n.isLeaf() {
		node := n.child[0].insert(img)

		if node != nil {
			return node
		} else {
			return n.child[1].insert(img)
		}
	}

	// there is already an image in this node
	if n.img != nil {
		return nil
	}

	// space too small
	if n.rect.Dx() < img.size.x || n.rect.Dy() < img.size.y {
		return nil
	}

	// just right
	if n.rect.Dx() == img.size.x && n.rect.Dy() == img.size.y {
		img.offset = n.rect.Min
		n.img = img
		return n
	}

	// the space that is left will be large enough to split
	n.split(img)

	return n.child[0].insert(img)
}

func (n *Node) split(img *sprite) {
	dx := n.rect.Dx() - img.size.x
	dy := n.rect.Dy() - img.size.y

	if dx > dy {
		n.child[0] = &Node{rect: image.Rect(
			n.rect.Min.X,
			n.rect.Min.Y,
			n.rect.Min.X+img.size.x,
			n.rect.Max.Y)}
		n.child[1] = &Node{rect: image.Rect(
			n.rect.Min.X+img.size.x,
			n.rect.Min.Y,
			n.rect.Max.X,
			n.rect.Max.Y)}
	} else {
		n.child[0] = &Node{rect: image.Rect(
			n.rect.Min.X,
			n.rect.Min.Y,
			n.rect.Max.X,
			n.rect.Min.Y+img.size.y)}
		n.child[1] = &Node{rect: image.Rect(
			n.rect.Min.X,
			n.rect.Min.Y+img.size.y,
			n.rect.Max.X,
			n.rect.Max.Y)}
	}

}

func main() {
	var space int
	var dim string

	flag.IntVar(&space, "space", 1, "space added between images")
	flag.StringVar(&dim, "dimensions", "1024x1024", "atlas size")

	flag.Parse()

	args := flag.Args()

	inputDir := args[0]
	outputName := args[1]

	dimX, dimY, err := parseDimensions(dim)
	if err != nil {
		log.Fatal(err)
	}

	files, _ := ioutil.ReadDir(inputDir)
	sprites := make([]sprite, len(files))

	for i := range sprites {
		s := readSprite(inputDir, files[i].Name(), space)
		sprites[i] = s
	}

	// we want to place the largest sprite first
	sort.Sort(sort.Reverse(ByArea(sprites)))

	// the final image
	dst := image.NewRGBA(image.Rect(0, 0, dimX, dimY))

	n := Node{rect: image.Rect(0, 0, dimX, dimY)}

	for i := range sprites {
		s := &sprites[i]
		node := n.insert(s)
		if node != nil {
			draw.Draw(dst, node.rect, s.img, image.ZP, draw.Src)
		} else {
			log.Fatalf("could not place %s\n", s.name)
		}

	}

	writer, err := os.Create(outputName)
	err = png.Encode(writer, dst)
	if err != nil {
		log.Fatal(err)
	}
}

func parseDimensions(dim string) (dimX, dimY int, err error) {
	dims := strings.Split(dim, "x")
	if len(dims) != 2 {
		err = fmt.Errorf("couldn't parse dimension %s\n", dims)
		return
	}

	dimX, err = strconv.Atoi(dims[0])
	if err != nil {
		return
	}

	dimY, err = strconv.Atoi(dims[1])
	if err != nil {
		return
	}

	return
}

func readSprite(dir, name string, space int) (s sprite) {
	path := path.Join(dir, name)
	reader, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	img, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}

	s.name = strings.TrimSuffix(name, filepath.Ext(name))
	s.img = img
	rect := img.Bounds()
	s.size = Size{rect.Dx() + space, rect.Dy() + space}
	s.area = s.size.x * s.size.y
	return
}
<commit_msg>Export struct fields for future marshalling<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"image"
	"image/draw"
	"image/png"
)

type Size struct {
	X, Y int
}

type sprite struct {
	Name   string
	Offset image.Point
	Size   Size
	img    image.Image
	area   int
}

type ByArea []sprite

func (a ByArea) Len() int           { return len(a) }
func (a ByArea) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByArea) Less(i, j int) bool { return a[i].area < a[j].area }

type Node struct {
	child [2]*Node
	rect  image.Rectangle
	img   *sprite
}

func (n *Node) size() Size {
	return Size{n.rect.Dx(), n.rect.Dy()}
}

func (n *Node) print() {
	fmt.Println(n)
	if n.child[0] != nil {
		n.child[0].print()
	}
	if n.child[1] != nil {
		n.child[1].print()
	}
}

func (n *Node) isLeaf() bool {
	return n.child[0] == nil || n.child[1] == nil
}

func (n *Node) insert(img *sprite) *Node {
	if !n.isLeaf() {
		node := n.child[0].insert(img)

		if node != nil {
			return node
		} else {
			return n.child[1].insert(img)
		}
	}

	// there is already an image in this node
	if n.img != nil {
		return nil
	}

	// space too small
	if n.rect.Dx() < img.Size.X || n.rect.Dy() < img.Size.Y {
		return nil
	}

	// just right
	if n.rect.Dx() == img.Size.X && n.rect.Dy() == img.Size.Y {
		img.Offset = n.rect.Min
		n.img = img
		return n
	}

	// the space that is left will be large enough to split
	n.split(img)

	return n.child[0].insert(img)
}

func (n *Node) split(img *sprite) {
	dx := n.rect.Dx() - img.Size.X
	dy := n.rect.Dy() - img.Size.Y

	if dx > dy {
		n.child[0] = &Node{rect: image.Rect(
			n.rect.Min.X,
			n.rect.Min.Y,
			n.rect.Min.X+img.Size.X,
			n.rect.Max.Y)}
		n.child[1] = &Node{rect: image.Rect(
			n.rect.Min.X+img.Size.X,
			n.rect.Min.Y,
			n.rect.Max.X,
			n.rect.Max.Y)}
	} else {
		n.child[0] = &Node{rect: image.Rect(
			n.rect.Min.X,
			n.rect.Min.Y,
			n.rect.Max.X,
			n.rect.Min.Y+img.Size.Y)}
		n.child[1] = &Node{rect: image.Rect(
			n.rect.Min.X,
			n.rect.Min.Y+img.Size.Y,
			n.rect.Max.X,
			n.rect.Max.Y)}
	}

}

func main() {
	var space int
	var dim string

	flag.IntVar(&space, "space", 1, "space added between images")
	flag.StringVar(&dim, "dimensions", "1024x1024", "atlas size")

	flag.Parse()

	args := flag.Args()

	inputDir := args[0]
	outputName := args[1]

	dimX, dimY, err := parseDimensions(dim)
	if err != nil {
		log.Fatal(err)
	}

	files, _ := ioutil.ReadDir(inputDir)
	sprites := make([]sprite, len(files))

	for i := range sprites {
		s := readSprite(inputDir, files[i].Name(), space)
		sprites[i] = s
	}

	// we want to place the largest sprite first
	sort.Sort(sort.Reverse(ByArea(sprites)))

	// the final image
	dst := image.NewRGBA(image.Rect(0, 0, dimX, dimY))

	n := Node{rect: image.Rect(0, 0, dimX, dimY)}

	for i := range sprites {
		s := &sprites[i]
		node := n.insert(s)
		if node != nil {
			draw.Draw(dst, node.rect, s.img, image.ZP, draw.Src)
		} else {
			log.Fatalf("could not place %s\n", s.Name)
		}

	}

	writer, err := os.Create(outputName)
	err = png.Encode(writer, dst)
	if err != nil {
		log.Fatal(err)
	}
}

func parseDimensions(dim string) (dimX, dimY int, err error) {
	dims := strings.Split(dim, "x")
	if len(dims) != 2 {
		err = fmt.Errorf("couldn't parse dimension %s\n", dims)
		return
	}

	dimX, err = strconv.Atoi(dims[0])
	if err != nil {
		return
	}

	dimY, err = strconv.Atoi(dims[1])
	if err != nil {
		return
	}

	return
}

func readSprite(dir, name string, space int) (s sprite) {
	path := path.Join(dir, name)
	reader, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	img, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}

	s.Name = strings.TrimSuffix(name, filepath.Ext(name))
	s.img = img
	rect := img.Bounds()
	s.Size = Size{rect.Dx() + space, rect.Dy() + space}
	s.area = s.Size.X * s.Size.Y
	return
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"

	"image"
	_ "image/png"
)

type sprite struct {
	name   string
	img    image.Image
	bounds image.Rectangle
	area   int
}

type ByArea []sprite

func (a ByArea) Len() int           { return len(a) }
func (a ByArea) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByArea) Less(i, j int) bool { return a[i].area < a[j].area }

func main() {
	flag.Parse()

	args := flag.Args()

	inputDir := args[0]

	files, _ := ioutil.ReadDir(inputDir)
	sprites := make([]sprite, len(files))

	for i := range sprites {
		s := readSprite(inputDir, files[i].Name())
		sprites[i] = s
	}

	// we want to place the largest sprite first
	sort.Sort(sort.Reverse(ByArea(sprites)))

	for _, s := range sprites {
		fmt.Println(s)
	}
}

func readSprite(dir, name string) (s sprite) {
	path := path.Join(dir, name)
	reader, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	img, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}

	s.img = img
	s.bounds = img.Bounds()
	s.area = s.bounds.Dx() * s.bounds.Dy()
	return
}
<commit_msg>Test write out all images<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"

	"image"
	"image/draw"
	"image/png"
)

type Size struct {
	x, y int
}

type sprite struct {
	name string
	img  image.Image
	rect image.Rectangle
	area int
	size Size
}

type ByArea []sprite

func (a ByArea) Len() int           { return len(a) }
func (a ByArea) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByArea) Less(i, j int) bool { return a[i].area < a[j].area }

func main() {
	flag.Parse()

	args := flag.Args()

	inputDir := args[0]

	files, _ := ioutil.ReadDir(inputDir)
	sprites := make([]sprite, len(files))

	totalX := 0
	totalY := 0

	for i := range sprites {
		s := readSprite(inputDir, files[i].Name())
		sprites[i] = s
		totalX += s.size.x
		totalY += s.size.y
	}

	// we want to place the largest sprite first
	sort.Sort(sort.Reverse(ByArea(sprites)))

	// the final image
	dst := image.NewRGBA(image.Rect(0, 0, 1024, 1024))

	for i := range sprites {
		s := &sprites[i]
		draw.Draw(dst, s.rect, s.img, image.ZP, draw.Src)
	}

	writer, err := os.Create("test.png")
	err = png.Encode(writer, dst)
	if err != nil {
		log.Fatal(err)
	}
}

func readSprite(dir, name string) (s sprite) {
	path := path.Join(dir, name)
	reader, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	img, err := png.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}

	s.name = name
	s.img = img
	s.rect = img.Bounds()
	s.size = Size{s.rect.Dx(), s.rect.Dy()}
	s.area = s.size.x * s.size.y
	return
}
<|endoftext|>


<commit_before>package gosnmp

import (
	"bytes"
	"fmt"
	l "github.com/alouca/gologger"
	"strconv"
	"strings"
)

type SnmpVersion uint8

const (
	Version1  SnmpVersion = 0x0
	Version2c SnmpVersion = 0x1
)

func (s SnmpVersion) String() string {
	if s == Version1 {
		return "1"
	} else if s == Version2c {
		return "2c"
	}
	return "U"
}

type SnmpPacket struct {
	Version     SnmpVersion
	Community   string
	RequestType Asn1BER
	RequestID   uint8
	Error       uint8
	ErrorIndex  uint8
	Variables   []SnmpPDU
}

type SnmpPDU struct {
	Name  string
	Type  Asn1BER
	Value interface{}
}

func Unmarshal(packet []byte) (*SnmpPacket, error) {
	log := l.GetDefaultLogger()

	log.Debug("Begin SNMP Packet unmarshal\n")

	//var err error
	response := new(SnmpPacket)
	response.Variables = make([]SnmpPDU, 0, 5)

	// Start parsing the packet
	var cursor uint64 = 0

	// First bytes should be 0x30
	if Asn1BER(packet[0]) == Sequence {
		// Parse packet length
		ber, err := parseField(packet)
		/*		var length int
				// length of structure is spread over two bytes
				if packet[1] == 0x82 {
					length = int(packet[2])<<8 | int(packet[3])
					length += 4 // account for header + length
					cursor += 4

				} else {
					length = int(packet[1])
					length += 2 // account for header + length
					cursor += 2
				}*/

		if err != nil {
			log.Error("Unable to parse packet header: %s\n", err.Error())
			return nil, err
		}

		log.Debug("Packet sanity verified, we got all the bytes (%d)\n", ber.DataLength)

		cursor += ber.HeaderLength
		// Parse SNMP Version
		rawVersion, err := parseField(packet[cursor:])

		if err != nil {
			return nil, fmt.Errorf("Error parsing SNMP packet version: %s", err.Error())
		}

		cursor += rawVersion.DataLength + rawVersion.HeaderLength
		if version, ok := rawVersion.BERVariable.Value.(int); ok {
			response.Version = SnmpVersion(version)
		}

		// Parse community
		rawCommunity, err := parseField(packet[cursor:])
		cursor += rawCommunity.DataLength + rawCommunity.HeaderLength

		if community, ok := rawCommunity.BERVariable.Value.(string); ok {
			response.Community = community
			log.Debug("Parsed community %s\n", community)
		}

		rawPDU, err := parseField(packet[cursor:])

		if err != nil {
			log.Debug("Unable to parse SNMP PDU: %s\n", err.Error())
		}

		switch rawPDU.Type {
		case GetResponse:
			log.Debug("SNMP Packet is GetResponse\n")
			log.Debug("PDU Size is %d\n", rawPDU.DataLength)
			cursor += rawPDU.HeaderLength

			// Parse Request ID
			rawRequestId, err := parseField(packet[cursor:])

			if err != nil {
				return nil, err
			}

			cursor += rawRequestId.DataLength + rawRequestId.HeaderLength
			if requestid, ok := rawRequestId.BERVariable.Value.(int); ok {
				response.RequestID = uint8(requestid)
				log.Debug("Parsed Request ID: %d\n", requestid)
			}

			// Parse Error
			rawError, err := parseField(packet[cursor:])

			if err != nil {
				return nil, err
			}

			cursor += rawError.DataLength + rawError.HeaderLength
			if errorNo, ok := rawError.BERVariable.Value.(int); ok {
				response.Error = uint8(errorNo)
			}

			// Parse Error Index
			rawErrorIndex, err := parseField(packet[cursor:])

			if err != nil {
				return nil, err
			}

			cursor += rawErrorIndex.DataLength + rawErrorIndex.HeaderLength

			if errorindex, ok := rawErrorIndex.BERVariable.Value.(int); ok {
				response.ErrorIndex = uint8(errorindex)
			}

			log.Debug("Request ID: %d Error: %d Error Index: %d\n", response.RequestID, response.Error, response.ErrorIndex)
			rawResp, err := parseField(packet[cursor:])

			if err != nil {
				return nil, err
			}

			cursor += rawResp.HeaderLength
			// Loop & parse Varbinds
			for cursor < uint64(len(packet)) {
				log.Debug("Parsing var bind response (Cursor at %d/%d)", cursor, len(packet))

				rawVarbind, err := parseField(packet[cursor:])

				if err != nil {
					return nil, err
				}

				cursor += rawVarbind.HeaderLength
				log.Debug("Varbind length: %d/%d\n", rawVarbind.HeaderLength, rawVarbind.DataLength)

				log.Debug("Parsing OID (Cursor at %d)\n", cursor)
				// Parse OID
				rawOid, err := parseField(packet[cursor:])

				if err != nil {
					return nil, err
				}

				cursor += rawOid.HeaderLength + rawOid.DataLength

				log.Debug("OID (%v) Field was %d bytes\n", rawOid, rawOid.DataLength)

				rawValue, err := parseField(packet[cursor:])

				if err != nil {
					return nil, err
				}
				cursor += rawValue.HeaderLength + rawValue.DataLength

				log.Debug("Value field was %d bytes\n", rawValue.DataLength)

				if oid, ok := rawOid.BERVariable.Value.([]int); ok {
					log.Debug("Varbind decoding success\n")
					response.Variables = append(response.Variables, SnmpPDU{oidToString(oid), rawValue.Type, rawValue.BERVariable.Value})
				}
			}

		}
	} else {
		return nil, fmt.Errorf("Invalid packet header\n")
	}

	return response, nil
}

type RawBER struct {
	Type         Asn1BER
	HeaderLength uint64
	DataLength   uint64
	Data         []byte
	BERVariable  *Variable
}

// Parses a given field, return the ASN.1 BER Type, its header length and the data
func parseField(data []byte) (*RawBER, error) {
	log := l.GetDefaultLogger()
	var err error

	if len(data) == 0 {
		return nil, fmt.Errorf("Unable to parse BER: Data length 0")
	}

	ber := new(RawBER)

	ber.Type = Asn1BER(data[0])

	// Parse Length
	length := data[1]

	// Check if this is padded or not
	if length > 0x80 {
		length = length - 0x80
		log.Debug("Field length is padded to %d bytes\n", length)
		ber.DataLength = Uvarint(data[2 : 2+length])
		log.Debug("Decoded final length: %d\n", ber.DataLength)

		ber.HeaderLength = 2 + uint64(length)

	} else {
		ber.HeaderLength = 2
		ber.DataLength = uint64(length)
	}

	// Do sanity checks
	if ber.DataLength > uint64(len(data)) {
		return nil, fmt.Errorf("Unable to parse BER: provided data length is longer than actual data (%d vs %d)", ber.DataLength, len(data))
	}

	ber.Data = data[ber.HeaderLength : ber.HeaderLength+ber.DataLength]

	ber.BERVariable, err = decodeValue(ber.Type, ber.Data)

	if err != nil {
		return nil, fmt.Errorf("Unable to decode value: %s\n", err.Error())
	}

	return ber, nil
}

func (packet *SnmpPacket) marshal() ([]byte, error) {
	// Prepare the buffer to send
	buffer := make([]byte, 0, 1024)
	buf := bytes.NewBuffer(buffer)

	// Write the packet header (Message type 0x30) & Version = 2
	buf.Write([]byte{byte(Sequence), 0, 2, 1, byte(packet.Version)})

	// Write Community
	buf.Write([]byte{4, uint8(len(packet.Community))})
	buf.WriteString(packet.Community)

	// Marshal the SNMP PDU
	snmpPduBuffer := make([]byte, 0, 1024)
	snmpPduBuf := bytes.NewBuffer(snmpPduBuffer)

	snmpPduBuf.Write([]byte{byte(packet.RequestType), 0, 2, 1, packet.RequestID, 2, 1, packet.Error, 2, 1, packet.ErrorIndex, byte(Sequence), 0})

	pduLength := 0
	for _, varlist := range packet.Variables {
		pdu, err := marshalPDU(&varlist)

		if err != nil {
			return nil, err
		}
		pduLength += len(pdu)
		snmpPduBuf.Write(pdu)
	}

	pduBytes := snmpPduBuf.Bytes()
	// Varbind list length
	pduBytes[12] = byte(pduLength)
	// SNMP PDU length (PDU header + varbind list length)
	pduBytes[1] = byte(pduLength + 11)

	buf.Write(pduBytes)

	// Write the
	//buf.Write([]byte{packet.RequestType, uint8(17 + len(mOid)), 2, 1, 1, 2, 1, 0, 2, 1, 0, 0x30, uint8(6 + len(mOid)), 0x30, uint8(4 + len(mOid)), 6, uint8(len(mOid))})
	//buf.Write(mOid)
	//buf.Write([]byte{5, 0})

	ret := buf.Bytes()

	// Set the packet size
	ret[1] = uint8(len(ret) - 2)

	return ret, nil
}

func marshalPDU(pdu *SnmpPDU) ([]byte, error) {
	oid, err := marshalOID(pdu.Name)
	if err != nil {
		return nil, err
	}

	pduBuffer := make([]byte, 0, 1024)
	pduBuf := bytes.NewBuffer(pduBuffer)

	// Mashal the PDU type into the appropriate BER
	switch pdu.Type {
	case Null:
		pduBuf.Write([]byte{byte(Sequence), byte(len(oid) + 4)})
		pduBuf.Write([]byte{byte(ObjectIdentifier), byte(len(oid))})
		pduBuf.Write(oid)
		pduBuf.Write([]byte{Null, 0x00})
	default:
		return nil, fmt.Errorf("Unable to marshal PDU: uknown BER type %d", pdu.Type)
	}

	return pduBuf.Bytes(), nil
}

func oidToString(oid []int) (ret string) {
	for _, i := range oid {
		ret = ret + fmt.Sprintf(".%d", i)
	}
	return
}

func marshalOID(oid string) ([]byte, error) {
	var err error

	// Encode the oid
	oid = strings.Trim(oid, ".")
	oidParts := strings.Split(oid, ".")
	oidBytes := make([]int, len(oidParts))

	// Convert the string OID to an array of integers
	for i := 0; i < len(oidParts); i++ {
		oidBytes[i], err = strconv.Atoi(oidParts[i])
		if err != nil {
			return nil, fmt.Errorf("Unable to parse OID: %s\n", err.Error())
		}
	}

	mOid, err := marshalObjectIdentifier(oidBytes)

	if err != nil {
		return nil, fmt.Errorf("Unable to marshal OID: %s\n", err.Error())
	}

	return mOid, err
}
<commit_msg>Removed deprecated code<commit_after>package gosnmp

import (
	"bytes"
	"fmt"
	l "github.com/alouca/gologger"
	"strconv"
	"strings"
)

type SnmpVersion uint8

const (
	Version1  SnmpVersion = 0x0
	Version2c SnmpVersion = 0x1
)

func (s SnmpVersion) String() string {
	if s == Version1 {
		return "1"
	} else if s == Version2c {
		return "2c"
	}
	return "U"
}

type SnmpPacket struct {
	Version     SnmpVersion
	Community   string
	RequestType Asn1BER
	RequestID   uint8
	Error       uint8
	ErrorIndex  uint8
	Variables   []SnmpPDU
}

type SnmpPDU struct {
	Name  string
	Type  Asn1BER
	Value interface{}
}

func Unmarshal(packet []byte) (*SnmpPacket, error) {
	log := l.GetDefaultLogger()

	log.Debug("Begin SNMP Packet unmarshal\n")

	//var err error
	response := new(SnmpPacket)
	response.Variables = make([]SnmpPDU, 0, 5)

	// Start parsing the packet
	var cursor uint64 = 0

	// First bytes should be 0x30
	if Asn1BER(packet[0]) == Sequence {
		// Parse packet length
		ber, err := parseField(packet)

		if err != nil {
			log.Error("Unable to parse packet header: %s\n", err.Error())
			return nil, err
		}

		log.Debug("Packet sanity verified, we got all the bytes (%d)\n", ber.DataLength)

		cursor += ber.HeaderLength
		// Parse SNMP Version
		rawVersion, err := parseField(packet[cursor:])

		if err != nil {
			return nil, fmt.Errorf("Error parsing SNMP packet version: %s", err.Error())
		}

		cursor += rawVersion.DataLength + rawVersion.HeaderLength
		if version, ok := rawVersion.BERVariable.Value.(int); ok {
			response.Version = SnmpVersion(version)
		}

		// Parse community
		rawCommunity, err := parseField(packet[cursor:])
		cursor += rawCommunity.DataLength + rawCommunity.HeaderLength

		if community, ok := rawCommunity.BERVariable.Value.(string); ok {
			response.Community = community
			log.Debug("Parsed community %s\n", community)
		}

		rawPDU, err := parseField(packet[cursor:])

		if err != nil {
			log.Debug("Unable to parse SNMP PDU: %s\n", err.Error())
		}

		switch rawPDU.Type {
		case GetResponse:
			log.Debug("SNMP Packet is GetResponse\n")
			log.Debug("PDU Size is %d\n", rawPDU.DataLength)
			cursor += rawPDU.HeaderLength

			// Parse Request ID
			rawRequestId, err := parseField(packet[cursor:])

			if err != nil {
				return nil, err
			}

			cursor += rawRequestId.DataLength + rawRequestId.HeaderLength
			if requestid, ok := rawRequestId.BERVariable.Value.(int); ok {
				response.RequestID = uint8(requestid)
				log.Debug("Parsed Request ID: %d\n", requestid)
			}

			// Parse Error
			rawError, err := parseField(packet[cursor:])

			if err != nil {
				return nil, err
			}

			cursor += rawError.DataLength + rawError.HeaderLength
			if errorNo, ok := rawError.BERVariable.Value.(int); ok {
				response.Error = uint8(errorNo)
			}

			// Parse Error Index
			rawErrorIndex, err := parseField(packet[cursor:])

			if err != nil {
				return nil, err
			}

			cursor += rawErrorIndex.DataLength + rawErrorIndex.HeaderLength

			if errorindex, ok := rawErrorIndex.BERVariable.Value.(int); ok {
				response.ErrorIndex = uint8(errorindex)
			}

			log.Debug("Request ID: %d Error: %d Error Index: %d\n", response.RequestID, response.Error, response.ErrorIndex)
			rawResp, err := parseField(packet[cursor:])

			if err != nil {
				return nil, err
			}

			cursor += rawResp.HeaderLength
			// Loop & parse Varbinds
			for cursor < uint64(len(packet)) {
				log.Debug("Parsing var bind response (Cursor at %d/%d)", cursor, len(packet))

				rawVarbind, err := parseField(packet[cursor:])

				if err != nil {
					return nil, err
				}

				cursor += rawVarbind.HeaderLength
				log.Debug("Varbind length: %d/%d\n", rawVarbind.HeaderLength, rawVarbind.DataLength)

				log.Debug("Parsing OID (Cursor at %d)\n", cursor)
				// Parse OID
				rawOid, err := parseField(packet[cursor:])

				if err != nil {
					return nil, err
				}

				cursor += rawOid.HeaderLength + rawOid.DataLength

				log.Debug("OID (%v) Field was %d bytes\n", rawOid, rawOid.DataLength)

				rawValue, err := parseField(packet[cursor:])

				if err != nil {
					return nil, err
				}
				cursor += rawValue.HeaderLength + rawValue.DataLength

				log.Debug("Value field was %d bytes\n", rawValue.DataLength)

				if oid, ok := rawOid.BERVariable.Value.([]int); ok {
					log.Debug("Varbind decoding success\n")
					response.Variables = append(response.Variables, SnmpPDU{oidToString(oid), rawValue.Type, rawValue.BERVariable.Value})
				}
			}

		}
	} else {
		return nil, fmt.Errorf("Invalid packet header\n")
	}

	return response, nil
}

type RawBER struct {
	Type         Asn1BER
	HeaderLength uint64
	DataLength   uint64
	Data         []byte
	BERVariable  *Variable
}

// Parses a given field, return the ASN.1 BER Type, its header length and the data
func parseField(data []byte) (*RawBER, error) {
	log := l.GetDefaultLogger()
	var err error

	if len(data) == 0 {
		return nil, fmt.Errorf("Unable to parse BER: Data length 0")
	}

	ber := new(RawBER)

	ber.Type = Asn1BER(data[0])

	// Parse Length
	length := data[1]

	// Check if this is padded or not
	if length > 0x80 {
		length = length - 0x80
		log.Debug("Field length is padded to %d bytes\n", length)
		ber.DataLength = Uvarint(data[2 : 2+length])
		log.Debug("Decoded final length: %d\n", ber.DataLength)

		ber.HeaderLength = 2 + uint64(length)

	} else {
		ber.HeaderLength = 2
		ber.DataLength = uint64(length)
	}

	// Do sanity checks
	if ber.DataLength > uint64(len(data)) {
		return nil, fmt.Errorf("Unable to parse BER: provided data length is longer than actual data (%d vs %d)", ber.DataLength, len(data))
	}

	ber.Data = data[ber.HeaderLength : ber.HeaderLength+ber.DataLength]

	ber.BERVariable, err = decodeValue(ber.Type, ber.Data)

	if err != nil {
		return nil, fmt.Errorf("Unable to decode value: %s\n", err.Error())
	}

	return ber, nil
}

func (packet *SnmpPacket) marshal() ([]byte, error) {
	// Prepare the buffer to send
	buffer := make([]byte, 0, 1024)
	buf := bytes.NewBuffer(buffer)

	// Write the packet header (Message type 0x30) & Version = 2
	buf.Write([]byte{byte(Sequence), 0, 2, 1, byte(packet.Version)})

	// Write Community
	buf.Write([]byte{4, uint8(len(packet.Community))})
	buf.WriteString(packet.Community)

	// Marshal the SNMP PDU
	snmpPduBuffer := make([]byte, 0, 1024)
	snmpPduBuf := bytes.NewBuffer(snmpPduBuffer)

	snmpPduBuf.Write([]byte{byte(packet.RequestType), 0, 2, 1, packet.RequestID, 2, 1, packet.Error, 2, 1, packet.ErrorIndex, byte(Sequence), 0})

	pduLength := 0
	for _, varlist := range packet.Variables {
		pdu, err := marshalPDU(&varlist)

		if err != nil {
			return nil, err
		}
		pduLength += len(pdu)
		snmpPduBuf.Write(pdu)
	}

	pduBytes := snmpPduBuf.Bytes()
	// Varbind list length
	pduBytes[12] = byte(pduLength)
	// SNMP PDU length (PDU header + varbind list length)
	pduBytes[1] = byte(pduLength + 11)

	buf.Write(pduBytes)

	// Write the
	//buf.Write([]byte{packet.RequestType, uint8(17 + len(mOid)), 2, 1, 1, 2, 1, 0, 2, 1, 0, 0x30, uint8(6 + len(mOid)), 0x30, uint8(4 + len(mOid)), 6, uint8(len(mOid))})
	//buf.Write(mOid)
	//buf.Write([]byte{5, 0})

	ret := buf.Bytes()

	// Set the packet size
	ret[1] = uint8(len(ret) - 2)

	return ret, nil
}

func marshalPDU(pdu *SnmpPDU) ([]byte, error) {
	oid, err := marshalOID(pdu.Name)
	if err != nil {
		return nil, err
	}

	pduBuffer := make([]byte, 0, 1024)
	pduBuf := bytes.NewBuffer(pduBuffer)

	// Mashal the PDU type into the appropriate BER
	switch pdu.Type {
	case Null:
		pduBuf.Write([]byte{byte(Sequence), byte(len(oid) + 4)})
		pduBuf.Write([]byte{byte(ObjectIdentifier), byte(len(oid))})
		pduBuf.Write(oid)
		pduBuf.Write([]byte{Null, 0x00})
	default:
		return nil, fmt.Errorf("Unable to marshal PDU: uknown BER type %d", pdu.Type)
	}

	return pduBuf.Bytes(), nil
}

func oidToString(oid []int) (ret string) {
	for _, i := range oid {
		ret = ret + fmt.Sprintf(".%d", i)
	}
	return
}

func marshalOID(oid string) ([]byte, error) {
	var err error

	// Encode the oid
	oid = strings.Trim(oid, ".")
	oidParts := strings.Split(oid, ".")
	oidBytes := make([]int, len(oidParts))

	// Convert the string OID to an array of integers
	for i := 0; i < len(oidParts); i++ {
		oidBytes[i], err = strconv.Atoi(oidParts[i])
		if err != nil {
			return nil, fmt.Errorf("Unable to parse OID: %s\n", err.Error())
		}
	}

	mOid, err := marshalObjectIdentifier(oidBytes)

	if err != nil {
		return nil, fmt.Errorf("Unable to marshal OID: %s\n", err.Error())
	}

	return mOid, err
}
<|endoftext|>


<commit_before>package api2go

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type sourceAdapter struct {
	findAll func() (interface{}, error)
	findOne func(string) (interface{}, error)
	create  func(interface{}) (string, error)
	delete  func(string) error
	update  func(interface{}) error
}

func (a *sourceAdapter) FindAll() (interface{}, error)          { return a.findAll() }
func (a *sourceAdapter) FindOne(id string) (interface{}, error) { return a.findOne(id) }
func (a *sourceAdapter) Create(obj interface{}) (string, error) { return a.create(obj) }
func (a *sourceAdapter) Delete(id string) error                 { return a.delete(id) }
func (a *sourceAdapter) Update(obj interface{}) error           { return a.update(obj) }

var _ = Describe("RestHandler", func() {
	Context("when handling requests", func() {
		type Post struct {
			ID    int
			Title string
		}

		var (
			post1    Post
			post1Map map[string]interface{}

			api *API
			rec *httptest.ResponseRecorder

			deleted bool
		)

		BeforeEach(func() {
			deleted = false

			post1 = Post{ID: 1, Title: "Hello, World!"}
			post1Map = map[string]interface{}{
				"id":    "1",
				"title": "Hello, World!",
			}

			adapter := &sourceAdapter{
				findAll: func() (interface{}, error) {
					return []Post{post1}, nil
				},
				findOne: func(id string) (interface{}, error) {
					switch id {
					case "1":
						return post1, nil
					case "42":
						return Post{ID: 42, Title: "New Post"}, nil
					default:
						return nil, NewHTTPError(nil, "post not found", http.StatusNotFound)
					}
				},
				create: func(obj interface{}) (string, error) {
					p := obj.(Post)
					Expect(p.Title).To(Equal("New Post"))
					return "42", nil
				},
				delete: func(id string) error {
					if id != "1" {
						panic("unknown id")
					}
					deleted = true
					return nil
				},
				update: func(obj interface{}) error {
					p := obj.(Post)
					if p.ID != 1 {
						panic("unknown id")
					}
					post1.Title = p.Title
					return nil
				},
			}

			api = NewAPI()
			api.AddResource(Post{}, adapter)

			rec = httptest.NewRecorder()
		})

		It("GETs collections", func() {
			req, err := http.NewRequest("GET", "/posts", nil)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusOK))
			var result map[string]interface{}
			Expect(json.Unmarshal(rec.Body.Bytes(), &result)).To(BeNil())
			Expect(result).To(Equal(map[string]interface{}{
				"posts": []interface{}{post1Map},
			}))
		})

		It("GETs single objects", func() {
			req, err := http.NewRequest("GET", "/posts/1", nil)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusOK))
			var result map[string]interface{}
			Expect(json.Unmarshal(rec.Body.Bytes(), &result)).To(BeNil())
			Expect(result).To(Equal(map[string]interface{}{
				"posts": []interface{}{post1Map},
			}))
		})

		It("404s", func() {
			req, err := http.NewRequest("GET", "/posts/23", nil)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusNotFound))
			Expect(rec.Body.String()).To(Equal("post not found\n"))
		})

		It("POSTSs new objects", func() {
			reqBody := strings.NewReader(`{"posts": [{"title": "New Post"}]}`)
			req, err := http.NewRequest("POST", "/posts", reqBody)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusCreated))
			Expect(rec.Header().Get("Location")).To(Equal("/posts/42"))
			var result map[string]interface{}
			Expect(json.Unmarshal(rec.Body.Bytes(), &result)).To(BeNil())
			Expect(result).To(Equal(map[string]interface{}{
				"posts": []interface{}{
					map[string]interface{}{
						"id":    "42",
						"title": "New Post",
					},
				},
			}))
		})

		It("OPTIONS on collection route", func() {
			req, err := http.NewRequest("OPTIONS", "/posts", nil)
			api.Handler().ServeHTTP(rec, req)
			Expect(err).To(BeNil())
			Expect(rec.Code).To(Equal(http.StatusNoContent))
			Expect(rec.Header().Get("Allow")).To(Equal("GET,POST,OPTIONS"))
		})

		It("OPTIONS on element route", func() {
			req, err := http.NewRequest("OPTIONS", "/posts/1", nil)
			api.Handler().ServeHTTP(rec, req)
			Expect(err).To(BeNil())
			Expect(rec.Code).To(Equal(http.StatusNoContent))
			Expect(rec.Header().Get("Allow")).To(Equal("GET,PUT,DELETE,OPTIONS"))
		})

		It("DELETEs", func() {
			req, err := http.NewRequest("DELETE", "/posts/1", nil)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusNoContent))
			Expect(deleted).To(BeTrue())
		})

		It("UPDATEs", func() {
			reqBody := strings.NewReader(`{"posts": [{"id": "1", "title": "New Title"}]}`)
			req, err := http.NewRequest("PUT", "/posts/1", reqBody)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusNoContent))
			Expect(post1.Title).To(Equal("New Title"))
		})
	})
})
<commit_msg>improved api tests to use a fixture SourceAdapter, we can ref it in docs<commit_after>package api2go

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type Post struct {
	ID    string
	Title string
}

type fixtureSource struct {
	posts map[string]*Post
}

func (s *fixtureSource) FindAll() (interface{}, error) {
	postsSlice := make([]Post, len(s.posts))
	i := 0
	for _, p := range s.posts {
		postsSlice[i] = *p
		i++
	}
	return postsSlice, nil
}

func (s *fixtureSource) FindOne(id string) (interface{}, error) {
	if p, ok := s.posts[id]; ok {
		return *p, nil
	}
	return nil, NewHTTPError(nil, "post not found", http.StatusNotFound)
}

func (s *fixtureSource) Create(obj interface{}) (string, error) {
	p := obj.(Post)
	maxID := 0
	for k := range s.posts {
		id, _ := strconv.Atoi(k)
		if id > maxID {
			maxID = id
		}
	}
	newID := strconv.Itoa(maxID + 1)
	p.ID = newID
	s.posts[newID] = &p
	return newID, nil
}

func (s *fixtureSource) Delete(id string) error {
	delete(s.posts, id)
	return nil
}

func (s *fixtureSource) Update(obj interface{}) error {
	p := obj.(Post)
	if oldP, ok := s.posts[p.ID]; ok {
		oldP.Title = p.Title
		return nil
	}
	return NewHTTPError(nil, "post not found", http.StatusNotFound)
}

var _ = Describe("RestHandler", func() {
	Context("when handling requests", func() {

		var (
			source    *fixtureSource
			post1Json map[string]interface{}

			api *API
			rec *httptest.ResponseRecorder
		)

		BeforeEach(func() {
			source = &fixtureSource{map[string]*Post{
				"1": &Post{ID: "1", Title: "Hello, World!"},
			}}

			post1Json = map[string]interface{}{
				"id":    "1",
				"title": "Hello, World!",
			}

			api = NewAPI()
			api.AddResource(Post{}, source)

			rec = httptest.NewRecorder()
		})

		It("GETs collections", func() {
			req, err := http.NewRequest("GET", "/posts", nil)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusOK))
			var result map[string]interface{}
			Expect(json.Unmarshal(rec.Body.Bytes(), &result)).To(BeNil())
			Expect(result).To(Equal(map[string]interface{}{
				"posts": []interface{}{post1Json},
			}))
		})

		It("GETs single objects", func() {
			req, err := http.NewRequest("GET", "/posts/1", nil)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusOK))
			var result map[string]interface{}
			Expect(json.Unmarshal(rec.Body.Bytes(), &result)).To(BeNil())
			Expect(result).To(Equal(map[string]interface{}{
				"posts": []interface{}{post1Json},
			}))
		})

		It("404s", func() {
			req, err := http.NewRequest("GET", "/posts/23", nil)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusNotFound))
			Expect(rec.Body.String()).To(Equal("post not found\n"))
		})

		It("POSTSs new objects", func() {
			reqBody := strings.NewReader(`{"posts": [{"title": "New Post"}]}`)
			req, err := http.NewRequest("POST", "/posts", reqBody)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusCreated))
			Expect(rec.Header().Get("Location")).To(Equal("/posts/2"))
			var result map[string]interface{}
			Expect(json.Unmarshal(rec.Body.Bytes(), &result)).To(BeNil())
			Expect(result).To(Equal(map[string]interface{}{
				"posts": []interface{}{
					map[string]interface{}{
						"id":    "2",
						"title": "New Post",
					},
				},
			}))
		})

		It("OPTIONS on collection route", func() {
			req, err := http.NewRequest("OPTIONS", "/posts", nil)
			api.Handler().ServeHTTP(rec, req)
			Expect(err).To(BeNil())
			Expect(rec.Code).To(Equal(http.StatusNoContent))
			Expect(rec.Header().Get("Allow")).To(Equal("GET,POST,OPTIONS"))
		})

		It("OPTIONS on element route", func() {
			req, err := http.NewRequest("OPTIONS", "/posts/1", nil)
			api.Handler().ServeHTTP(rec, req)
			Expect(err).To(BeNil())
			Expect(rec.Code).To(Equal(http.StatusNoContent))
			Expect(rec.Header().Get("Allow")).To(Equal("GET,PUT,DELETE,OPTIONS"))
		})

		It("DELETEs", func() {
			req, err := http.NewRequest("DELETE", "/posts/1", nil)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusNoContent))
			Expect(len(source.posts)).To(BeZero())
		})

		It("UPDATEs", func() {
			reqBody := strings.NewReader(`{"posts": [{"id": "1", "title": "New Title"}]}`)
			req, err := http.NewRequest("PUT", "/posts/1", reqBody)
			Expect(err).To(BeNil())
			api.Handler().ServeHTTP(rec, req)
			Expect(rec.Code).To(Equal(http.StatusNoContent))
			Expect(source.posts["1"].Title).To(Equal("New Title"))
		})
	})
})
<|endoftext|>


<commit_before>package astgen

import (
	"strconv"
	"fmt"
)

type parser struct {
	file []byte
	line int
}

func Load(data []byte) (def *LangDef, e error) {
	var p parser
	p.line = 0
	p.file = data

	defer func() {
		err := recover()
		if err != nil {
			def = nil
			e = fmt.Errorf("Error on line %d: %s\nNext 20 bytes: %s\n", p.line, err, string(p.file[:20]))
		}
	}()

	def = new(LangDef)
	def.Types = make(map[string]Type)

	for !p.finished() {
		t := p.parse_type()
		def.Types[t.Common().Name] = t
	}

	return def, nil
}

func is_nl(b byte) bool {
	return b == '\n' || b == '\r'
}

func is_space(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

func is_letter(b byte) bool {
	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
}

func (p *parser) skip_space() {
	for len(p.file) > 0 && is_space(p.file[0]) {
		if is_nl(p.file[0]) {
			p.line++
		}

		p.file = p.file[1:]
	}
}

func (p *parser) accept_token(token string) bool {
	p.skip_space()

	tok := []byte(token)

	if len(p.file) < len(tok) {
		return false
	}

	for i := range tok {
		if p.file[i] != tok[i] {
			return false
		}
	}

	if len(p.file) != len(tok) {
		next := p.file[len(tok)]
		if !is_space(next) && is_letter(p.file[len(tok)-1]) == is_letter(next) {
			return false
		}
	}

	p.file = p.file[len(tok):]

	return true
}

func (p *parser) match_token(token string) {
	if !p.accept_token(token) {
		panic("can't match token \"" + token + "\". Current: \"" + p.current_token() + "\"")
	}
}

func (p *parser) consume_line() string {
	p.skip_space()

	for i := range p.file {
		if p.file[i] == '\n' {
			ret := string(p.file[:i])
			p.file = p.file[i+1:]
			p.line++
			return ret
		}
	}

	ret := string(p.file)
	p.file = nil
	return ret
}

func (p *parser) consume_token() string {
	p.skip_space()

	if len(p.file) == 0 {
		panic("bad file")
	}

	ident := is_letter(p.file[0])

	for i := range p.file {
		if ident != is_letter(p.file[i]) || is_space(p.file[i]) {
			ret := string(p.file[:i])
			p.file = p.file[i:]
			return ret
		}
	}

	ret := string(p.file)
	p.file = nil
	return ret
}

func (p *parser) current_token() string {
	p.skip_space()

	if len(p.file) == 0 {
		panic("bad file")
	}

	ident := is_letter(p.file[0])

	for i := range p.file {
		if ident != is_letter(p.file[i]) || is_space(p.file[i]) {
			return string(p.file[:i])
		}
	}

	return string(p.file)
}

func (p *parser) finished() bool {
	p.skip_space()
	return len(p.file) == 0
}

func (p *parser) parse_production() Production {
	var line []byte

	for i := range p.file {
		if p.file[i] == '\n' {
			line = p.file[:i]
			p.file = p.file[i:]
			break
		}
	}

	if line == nil {
		panic("bad file")
	}

	tokens := []Token{}

	for len(line) != 0 {
		for len(line) != 0 && line[0] == ' ' {
			line = line[1:]
		}

		token_len := len(line)

		for i := range line {
			if line[i] == ' ' {
				token_len = i
				break
			}
		}

		var token Token
		if line[0] == '$' {
			token = Token{string(line[1:token_len]), ""}
		} else {
			token = Token{"", string(line[:token_len])}
		}
		tokens = append(tokens, token)
		line = line[token_len:]
	}

	return Production{tokens}
}

func (p *parser) parse_member() (m StructMember) {
	for p.accept_token("//") {
		if p.accept_token("terminator") {
			pr := p.parse_production()
			m.ArrayTerminator = &pr
		} else if p.accept_token("separator") {
			pr := p.parse_production()
			m.ArraySeparator = &pr
		} else if p.accept_token("min_length") {
			var err error
			m.ArrayMinLength, err = strconv.Atoi(string(p.consume_line()))
			if err != nil {
				panic(err)
			}
		} else {
			panic("bad file")
		}
	}

	m.Name = p.consume_token()
	p.match_token(":")

	if p.accept_token("?") {
		m.Nullable = true
	}
	if p.accept_token("[]") {
		m.Array = true
	}

	m.Type = p.consume_token()
	return
}

func (p *parser) parse_struct_type() *StructType {
	typ := new(StructType)

	for p.accept_token("//") {
		typ.Productions = append(typ.Productions, p.parse_production())
	}

	p.match_token("type")

	typ.Name = p.consume_token()

	p.match_token("=")
	p.match_token("struct")
	p.match_token("{")

	for !p.accept_token("}") {
		typ.Members = append(typ.Members, p.parse_member())
	}

	return typ
}

func (p *parser) parse_struct_type2(name string) *StructType {
	typ := new(StructType)
	typ.Name = name

	p.match_token("struct")
	p.match_token("{")

	for !p.accept_token("}") {
		typ.Members = append(typ.Members, p.parse_member())
	}

	return typ
}

func (p *parser) parse_type() Type {
	if p.current_token() == "//" {
		return p.parse_struct_type()
	}

	p.match_token("type")

	name := p.consume_token()

	if !p.accept_token("=") {
		return &LexicalType{TypeBase{name, false}}
	}

	if p.current_token() == "struct" {
		return p.parse_struct_type2(name)
	}

	typ := new(OptionType)
	typ.Name = name
	typ.Options = []string{p.consume_token()}

	for p.accept_token("|") {
		typ.Options = append(typ.Options, p.consume_token())
	}

	return typ
}
<commit_msg>Add enum parsing.<commit_after>package astgen

import (
	"strconv"
	"fmt"
)

// TODO: Part of this file can be extracted into a miniparser package.

type parser struct {
	file []byte
	line int
}

func Load(data []byte) (def *LangDef, e error) {
	var p parser
	p.line = 1
	p.file = data

	defer func() {
		err := recover()
		if err != nil {
			def = nil
			e = fmt.Errorf("Error on line %d: %s\nNext 20 bytes: %s\n", p.line, err, string(p.file[:20]))
		}
	}()

	def = new(LangDef)
	def.Types = make(map[string]Type)

	for !p.finished() {
		t := p.parse_type()
		def.Types[t.Common().Name] = t
	}

	return def, nil
}

func is_nl(b byte) bool {
	return b == '\n' || b == '\r'
}

func is_space(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

func is_letter(b byte) bool {
	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
}

func (p *parser) skip_space() {
	for len(p.file) > 0 && is_space(p.file[0]) {
		if is_nl(p.file[0]) {
			p.line++
		}

		p.file = p.file[1:]
	}
}

func (p *parser) accept_token(token string) bool {
	p.skip_space()

	tok := []byte(token)

	if len(p.file) < len(tok) {
		return false
	}

	for i := range tok {
		if p.file[i] != tok[i] {
			return false
		}
	}

	if len(p.file) != len(tok) {
		next := p.file[len(tok)]
		if !is_space(next) && is_letter(p.file[len(tok)-1]) == is_letter(next) {
			return false
		}
	}

	p.file = p.file[len(tok):]

	return true
}

func (p *parser) match_token(token string) {
	if !p.accept_token(token) {
		panic("can't match token \"" + token + "\". Current: \"" + p.current_token() + "\"")
	}
}

func (p *parser) consume_line() string {
	p.skip_space()

	for i := range p.file {
		if p.file[i] == '\n' {
			ret := string(p.file[:i])
			p.file = p.file[i+1:]
			p.line++
			return ret
		}
	}

	ret := string(p.file)
	p.file = nil
	return ret
}

func (p *parser) consume_token() string {
	p.skip_space()

	if len(p.file) == 0 {
		panic("bad file")
	}

	ident := is_letter(p.file[0])

	for i := range p.file {
		if ident != is_letter(p.file[i]) || is_space(p.file[i]) {
			ret := string(p.file[:i])
			p.file = p.file[i:]
			return ret
		}
	}

	ret := string(p.file)
	p.file = nil
	return ret
}

func (p *parser) current_token() string {
	p.skip_space()

	if len(p.file) == 0 {
		panic("bad file")
	}

	ident := is_letter(p.file[0])

	for i := range p.file {
		if ident != is_letter(p.file[i]) || is_space(p.file[i]) {
			return string(p.file[:i])
		}
	}

	return string(p.file)
}

func (p *parser) finished() bool {
	p.skip_space()
	return len(p.file) == 0
}

func (p *parser) parse_production() Production {
	var line []byte

	for i := range p.file {
		if p.file[i] == '\n' {
			line = p.file[:i]
			p.file = p.file[i:]
			break
		}
	}

	if line == nil {
		panic("bad file")
	}

	tokens := []Token{}

	for len(line) != 0 {
		for len(line) != 0 && line[0] == ' ' {
			line = line[1:]
		}

		token_len := len(line)

		for i := range line {
			if line[i] == ' ' {
				token_len = i
				break
			}
		}

		var token Token
		if line[0] == '$' {
			token = Token{string(line[1:token_len]), ""}
		} else {
			token = Token{"", string(line[:token_len])}
		}
		tokens = append(tokens, token)
		line = line[token_len:]
	}

	return Production{tokens}
}

func (p *parser) parse_member() (m StructMember) {
	for p.accept_token("//") {
		if p.accept_token("terminator") {
			pr := p.parse_production()
			m.ArrayTerminator = &pr
		} else if p.accept_token("separator") {
			pr := p.parse_production()
			m.ArraySeparator = &pr
		} else if p.accept_token("min_length") {
			var err error
			m.ArrayMinLength, err = strconv.Atoi(string(p.consume_line()))
			if err != nil {
				panic(err)
			}
		} else {
			panic("bad file")
		}
	}

	m.Name = p.consume_token()
	p.match_token(":")

	if p.accept_token("?") {
		m.Nullable = true
	}
	if p.accept_token("[]") {
		m.Array = true
	}

	m.Type = p.consume_token()
	return
}

func (p *parser) parse_struct_type() *StructType {
	typ := new(StructType)

	for p.accept_token("//") {
		typ.Productions = append(typ.Productions, p.parse_production())
	}

	p.match_token("type")

	typ.Name = p.consume_token()

	p.match_token("=")
	p.match_token("struct")
	p.match_token("{")

	for !p.accept_token("}") {
		typ.Members = append(typ.Members, p.parse_member())
	}

	return typ
}

func (p *parser) parse_struct_type2(name string) *StructType {
	typ := new(StructType)
	typ.Name = name

	p.match_token("struct")
	p.match_token("{")

	for !p.accept_token("}") {
		typ.Members = append(typ.Members, p.parse_member())
	}

	return typ
}

func (p *parser) parse_enum_type(name string) *EnumType {
	typ := new(EnumType)
	typ.Name = name

	p.match_token("enum")
	p.match_token("{")

	for !p.accept_token("}") {
		token_name := p.consume_token()
		token_string := p.consume_token()
		typ.EnumTokens = append(typ.EnumTokens, EnumToken{token_name, token_string})
	}

	return typ
}

func (p *parser) parse_type() Type {
	if p.current_token() == "//" {
		return p.parse_struct_type()
	}

	p.match_token("type")

	name := p.consume_token()

	if p.accept_token("lexical") || !p.accept_token("=") {
		return &LexicalType{TypeBase{name, false}}
	}

	if p.current_token() == "enum" {
		return p.parse_enum_type(name)
	}

	if p.current_token() == "struct" {
		return p.parse_struct_type2(name)
	}

	typ := new(OptionType)
	typ.Name = name
	typ.Options = []string{p.consume_token()}

	for p.accept_token("|") {
		typ.Options = append(typ.Options, p.consume_token())
	}

	return typ
}
<|endoftext|>


<commit_before>package redis

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/vmihailenco/bufio"
)

//------------------------------------------------------------------------------

// Represents Redis nil reply.
var Nil = errors.New("(nil)")

//------------------------------------------------------------------------------

var (
	errReaderTooSmall = errors.New("redis: reader is too small")
	errValNotSet      = errors.New("redis: value is not set")
)

//------------------------------------------------------------------------------

type parserError struct {
	err error
}

func (e *parserError) Error() string {
	return e.err.Error()
}

//------------------------------------------------------------------------------

func appendReq(buf []byte, args []string) []byte {
	buf = append(buf, '*')
	buf = strconv.AppendUint(buf, uint64(len(args)), 10)
	buf = append(buf, '\r', '\n')
	for _, arg := range args {
		buf = append(buf, '$')
		buf = strconv.AppendUint(buf, uint64(len(arg)), 10)
		buf = append(buf, '\r', '\n')
		buf = append(buf, []byte(arg)...)
		buf = append(buf, '\r', '\n')
	}
	return buf
}

//------------------------------------------------------------------------------

type reader interface {
	ReadLine() ([]byte, bool, error)
	Read([]byte) (int, error)
	ReadN(n int) ([]byte, error)
	Buffered() int
}

func readLine(rd reader) ([]byte, error) {
	line, isPrefix, err := rd.ReadLine()
	if err != nil {
		return line, err
	}
	if isPrefix {
		return line, errReaderTooSmall
	}
	return line, nil
}

func readN(rd reader, n int) ([]byte, error) {
	buf, err := rd.ReadN(n)
	if err == bufio.ErrBufferFull {
		newBuf := make([]byte, n)
		r := copy(newBuf, buf)
		buf = newBuf

		for {
			nn, err := rd.Read(buf[r:])
			r += nn
			if r >= n {
				// Ignore error if we read enough.
				break
			}
			if err != nil {
				return nil, err
			}
		}
	} else if err != nil {
		return nil, err
	}
	return buf, nil
}

//------------------------------------------------------------------------------

func ParseReq(rd reader) ([]string, error) {
	line, err := readLine(rd)
	if err != nil {
		return nil, err
	}

	if line[0] != '*' {
		return []string{string(line)}, nil
	}
	numReplies, err := strconv.ParseInt(string(line[1:]), 10, 64)
	if err != nil {
		return nil, err
	}

	args := make([]string, 0, numReplies)
	for i := int64(0); i < numReplies; i++ {
		line, err = readLine(rd)
		if err != nil {
			return nil, err
		}
		if line[0] != '$' {
			return nil, fmt.Errorf("Expected '$', but got %q", line)
		}

		argLen, err := strconv.ParseInt(string(line[1:]), 10, 32)
		if err != nil {
			return nil, err
		}

		arg, err := readN(rd, int(argLen)+2)
		if err != nil {
			return nil, err
		}
		args = append(args, string(arg[:argLen]))
	}
	return args, nil
}

//------------------------------------------------------------------------------

const (
	ifaceSlice = iota
	stringSlice
	boolSlice
)

func parseReply(rd reader) (interface{}, error) {
	return _parseReply(rd, ifaceSlice)
}

func parseStringSliceReply(rd reader) (interface{}, error) {
	return _parseReply(rd, stringSlice)
}

func parseBoolSliceReply(rd reader) (interface{}, error) {
	return _parseReply(rd, boolSlice)
}

func _parseReply(rd reader, multiBulkType int) (interface{}, error) {
	line, err := readLine(rd)
	if err != nil {
		return 0, &parserError{err}
	}

	switch line[0] {
	case '-':
		return nil, errors.New(string(line[1:]))
	case '+':
		return string(line[1:]), nil
	case ':':
		v, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return 0, &parserError{err}
		}
		return v, nil
	case '$':
		if len(line) == 3 && line[1] == '-' && line[2] == '1' {
			return "", Nil
		}

		replyLenInt32, err := strconv.ParseInt(string(line[1:]), 10, 32)
		if err != nil {
			return "", &parserError{err}
		}
		replyLen := int(replyLenInt32) + 2

		line, err = readN(rd, replyLen)
		if err != nil {
			return "", &parserError{err}
		}
		return string(line[:len(line)-2]), nil
	case '*':
		if len(line) == 3 && line[1] == '-' && line[2] == '1' {
			return nil, Nil
		}

		numReplies, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return nil, &parserError{err}
		}

		switch multiBulkType {
		case stringSlice:
			vals := make([]string, 0, numReplies)
			for i := int64(0); i < numReplies; i++ {
				v, err := parseReply(rd)
				if err != nil {
					return nil, err
				} else {
					vals = append(vals, v.(string))
				}
			}

			return vals, nil
		case boolSlice:
			vals := make([]bool, 0, numReplies)
			for i := int64(0); i < numReplies; i++ {
				v, err := parseReply(rd)
				if err != nil {
					return nil, err
				} else {
					vals = append(vals, v.(int64) == 1)
				}
			}

			return vals, nil
		default:
			vals := make([]interface{}, 0, numReplies)
			for i := int64(0); i < numReplies; i++ {
				v, err := parseReply(rd)
				if err == Nil {
					vals = append(vals, nil)
				} else if err != nil {
					return nil, err
				} else {
					vals = append(vals, v)
				}
			}

			return vals, nil
		}
	default:
		return nil, &parserError{fmt.Errorf("redis: can't parse %q", line)}
	}
	panic("not reachable")
}
<commit_msg>Use string in append directly.<commit_after>package redis

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/vmihailenco/bufio"
)

//------------------------------------------------------------------------------

// Represents Redis nil reply.
var Nil = errors.New("(nil)")

//------------------------------------------------------------------------------

var (
	errReaderTooSmall = errors.New("redis: reader is too small")
	errValNotSet      = errors.New("redis: value is not set")
)

//------------------------------------------------------------------------------

type parserError struct {
	err error
}

func (e *parserError) Error() string {
	return e.err.Error()
}

//------------------------------------------------------------------------------

func appendReq(buf []byte, args []string) []byte {
	buf = append(buf, '*')
	buf = strconv.AppendUint(buf, uint64(len(args)), 10)
	buf = append(buf, '\r', '\n')
	for _, arg := range args {
		buf = append(buf, '$')
		buf = strconv.AppendUint(buf, uint64(len(arg)), 10)
		buf = append(buf, '\r', '\n')
		buf = append(buf, arg...)
		buf = append(buf, '\r', '\n')
	}
	return buf
}

//------------------------------------------------------------------------------

type reader interface {
	ReadLine() ([]byte, bool, error)
	Read([]byte) (int, error)
	ReadN(n int) ([]byte, error)
	Buffered() int
}

func readLine(rd reader) ([]byte, error) {
	line, isPrefix, err := rd.ReadLine()
	if err != nil {
		return line, err
	}
	if isPrefix {
		return line, errReaderTooSmall
	}
	return line, nil
}

func readN(rd reader, n int) ([]byte, error) {
	buf, err := rd.ReadN(n)
	if err == bufio.ErrBufferFull {
		newBuf := make([]byte, n)
		r := copy(newBuf, buf)
		buf = newBuf

		for {
			nn, err := rd.Read(buf[r:])
			r += nn
			if r >= n {
				// Ignore error if we read enough.
				break
			}
			if err != nil {
				return nil, err
			}
		}
	} else if err != nil {
		return nil, err
	}
	return buf, nil
}

//------------------------------------------------------------------------------

func ParseReq(rd reader) ([]string, error) {
	line, err := readLine(rd)
	if err != nil {
		return nil, err
	}

	if line[0] != '*' {
		return []string{string(line)}, nil
	}
	numReplies, err := strconv.ParseInt(string(line[1:]), 10, 64)
	if err != nil {
		return nil, err
	}

	args := make([]string, 0, numReplies)
	for i := int64(0); i < numReplies; i++ {
		line, err = readLine(rd)
		if err != nil {
			return nil, err
		}
		if line[0] != '$' {
			return nil, fmt.Errorf("Expected '$', but got %q", line)
		}

		argLen, err := strconv.ParseInt(string(line[1:]), 10, 32)
		if err != nil {
			return nil, err
		}

		arg, err := readN(rd, int(argLen)+2)
		if err != nil {
			return nil, err
		}
		args = append(args, string(arg[:argLen]))
	}
	return args, nil
}

//------------------------------------------------------------------------------

const (
	ifaceSlice = iota
	stringSlice
	boolSlice
)

func parseReply(rd reader) (interface{}, error) {
	return _parseReply(rd, ifaceSlice)
}

func parseStringSliceReply(rd reader) (interface{}, error) {
	return _parseReply(rd, stringSlice)
}

func parseBoolSliceReply(rd reader) (interface{}, error) {
	return _parseReply(rd, boolSlice)
}

func _parseReply(rd reader, multiBulkType int) (interface{}, error) {
	line, err := readLine(rd)
	if err != nil {
		return 0, &parserError{err}
	}

	switch line[0] {
	case '-':
		return nil, errors.New(string(line[1:]))
	case '+':
		return string(line[1:]), nil
	case ':':
		v, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return 0, &parserError{err}
		}
		return v, nil
	case '$':
		if len(line) == 3 && line[1] == '-' && line[2] == '1' {
			return "", Nil
		}

		replyLenInt32, err := strconv.ParseInt(string(line[1:]), 10, 32)
		if err != nil {
			return "", &parserError{err}
		}
		replyLen := int(replyLenInt32) + 2

		line, err = readN(rd, replyLen)
		if err != nil {
			return "", &parserError{err}
		}
		return string(line[:len(line)-2]), nil
	case '*':
		if len(line) == 3 && line[1] == '-' && line[2] == '1' {
			return nil, Nil
		}

		numReplies, err := strconv.ParseInt(string(line[1:]), 10, 64)
		if err != nil {
			return nil, &parserError{err}
		}

		switch multiBulkType {
		case stringSlice:
			vals := make([]string, 0, numReplies)
			for i := int64(0); i < numReplies; i++ {
				v, err := parseReply(rd)
				if err != nil {
					return nil, err
				} else {
					vals = append(vals, v.(string))
				}
			}

			return vals, nil
		case boolSlice:
			vals := make([]bool, 0, numReplies)
			for i := int64(0); i < numReplies; i++ {
				v, err := parseReply(rd)
				if err != nil {
					return nil, err
				} else {
					vals = append(vals, v.(int64) == 1)
				}
			}

			return vals, nil
		default:
			vals := make([]interface{}, 0, numReplies)
			for i := int64(0); i < numReplies; i++ {
				v, err := parseReply(rd)
				if err == Nil {
					vals = append(vals, nil)
				} else if err != nil {
					return nil, err
				} else {
					vals = append(vals, v)
				}
			}

			return vals, nil
		}
	default:
		return nil, &parserError{fmt.Errorf("redis: can't parse %q", line)}
	}
	panic("not reachable")
}
<|endoftext|>


<commit_before>package rmarsh

import (
	"fmt"
	"io"

	"github.com/pkg/errors"
)

// A Token represents a single distinct value type read from a Parser instance.
type Token uint8

// The valid token types.
const (
	tokenInvalid = iota
	tokenStart
	TokenNil
	TokenTrue
	TokenFalse
	TokenFixnum
	TokenFloat
	TokenBignum
	TokenSymbol
	TokenString
	TokenStartArray
	TokenEndArray
	TokenStartHash
	TokenEndHash
	TokenStartIVar
	TokenIVarProps
	TokenEndIVar
	TokenLink
	TokenUsrMarshal
	TokenEOF
)

var tokenNames = map[Token]string{
	TokenNil:        "TokenNil",
	TokenTrue:       "TokenTrue",
	TokenFalse:      "TokenFalse",
	TokenFixnum:     "TokenFixnum",
	TokenFloat:      "TokenFloat",
	TokenBignum:     "TokenBignum",
	TokenSymbol:     "TokenSymbol",
	TokenString:     "TokenString",
	TokenStartArray: "TokenStartArray",
	TokenEndArray:   "TokenEndArray",
	TokenStartHash:  "TokenStartHash",
	TokenEndHash:    "TokenEndHash",
	TokenStartIVar:  "TokenStartIVar",
	TokenIVarProps:  "TokenIVarProps",
	TokenEndIVar:    "TokenEndIVar",
	TokenLink:       "TokenLink",
	TokenUsrMarshal: "TokenUsrMarshal",
	TokenEOF:        "EOF",
}

func (t Token) String() string {
	if n, ok := tokenNames[t]; ok {
		return n
	}
	return "UNKNOWN"
}

// A ParserError is a description of an error encountered while parsing a Ruby Marshal stream.
type ParserError struct {
	msg    string
	Offset int
}

func (e ParserError) Error() string {
	return e.msg
}

// Parser is a low-level pull-based parser of the Ruby Marshal format.
// A Parser will pull bytes from an underlying io.Reader as needed, but will never buffer past the
// end of the current Marshal stream. Even though effort is made to be as efficient in pulling bytes
// as possible, if the Marshal data is already fully available then it should be wrapped in a bufio.Reader
// before being handed to a Parser.
// Parser is very low level and is mostly intended as a building block for the Decoder. You probably
// want to be using that.
type Parser struct {
	r io.Reader // our byte source.

	buf    []byte // The read buffer contains every byte of data that we've read from the stream.
	bufcap int    // Current capacity of the read buffer.
	buflen int    // The number of bytes we've read into the read buffer.
	pos    int    // Our byte position in the read buffer.

	state parserState
	stack parserStack
}

func NewParser(r io.Reader) *Parser {
	return &Parser{
		r:      r,
		buf:    make([]byte, bufInitSz),
		bufcap: bufInitSz,
		state:  parserStateTopLevel,
	}
}

// Reset reverts the Parser into the identity state, ready to read a new Marshal 4.8 stream from the existing Reader.
// If the provided io.Reader is nil, the existing Reader will continue to be used.
func (p *Parser) Reset(r io.Reader) {
	p.stack = p.stack[0:0]
	// p.cur = tokenInvalid
	p.state = parserStateTopLevel

	// If this a replay Parser, our reset is a little less ... reset-y.
	// if p.lnkID > -1 {
	// 	p.pos = p.lnkTbl[p.lnkID].beg
	// 	p.stack = p.stack[0:0]
	// 	return
	// }

	if r != nil {
		p.r = r
	}
	p.pos = 0
	p.buflen = 0
	// p.symTbl = p.symTbl[0:0]
	// p.lnkTbl = p.lnkTbl[0:0]
}

func (p *Parser) Read() (tok Token, b []byte, num int, err error) {
	// Quick early bailout check here. If parser state is "parserStateEOF" then we can just
	// return an EOF token and exit.
	if p.state == parserStateEOF {
		tok = TokenEOF
		return
	}

	// Gets set to false after we run SM.
	runSM := true

	// READ BYTES IF NECESSARY
	// Running the state machine can bail out back here if there's not enough data in the read buffer
	// to transition to the next state.
	// This code would be WAY less complicated if we just filled the buffer with method calls when needed...
	// But that costs too many precious nanos.
	needed := 0
pullbytes:
	if needed > 0 {
		// TODO: port over the stack-based prefetch here.

		from, to := p.buflen, p.buflen+needed

		if to > p.bufcap {
			// Overflowed our read buffer, allocate a new one double the current size, or the required size if it's larger.
			p.bufcap = p.bufcap * 2
			if p.bufcap < to {
				p.bufcap = to
			}
			buf := make([]byte, p.bufcap)
			copy(buf, p.buf[0:p.buflen])
			p.buf = buf
		}

		p.buflen += needed

		var n int
		for from < to && err == nil {
			n, err = p.r.Read(p.buf[from:to])
			from += n
		}
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
			return
		} else if err != nil {
			err = errors.Wrap(err, "fill")
			return
		}

		needed = 0
	}

	// RUN THE STATE MACHINE
	if runSM {
		switch p.state {
		// the initial state of a Parser expects to read 2-byte magic and then a top level value
		case parserStateTopLevel:
			if p.pos == 0 {
				// We can safely pull up to 3 bytes immediately. 2 bytes for the magic and all top level values
				// will be at least 1 byte large.
				if p.buflen < 3 {
					needed = 3 - p.buflen
					goto pullbytes
				}

				if p.buf[p.pos] != 0x04 || p.buf[p.pos+1] != 0x08 {
					err = p.parserError("Expected magic header 0x0408, got 0x%.4X", int16(p.buf[p.pos])<<8|int16(p.buf[p.pos+1]))
					return
				}
				p.pos = 2
			}

			// Our next state is EOF.
			// Unless we read something interesting below which pushes something onto the stack.
			p.state = parserStateEOF
		}

		// Now that we've run the SM, we don't want to run it again if the stream reads
		// need to go back to pullbytes
		runSM = false
	}

	// READ SOMETHING FROM THE STREAM
	if p.pos == p.buflen {
		// This is the worst possible situation to be in - we have to go to the io.Reader to pull a single byte.
		// This situation shouldn't occur very often on real world streams - as we should usually have enough to context to
		// be doing safe read aheads.
		needed = 1
		goto pullbytes
	}

	typ := p.buf[p.pos]
	rd := 1

	switch typ {
	case typeNil:
		tok = TokenNil

	case typeTrue:
		tok = TokenTrue

	case typeFalse:
		tok = TokenFalse

	case typeFixnum:
		tok = TokenFixnum
		// num, err = p.long()

		if p.pos+rd+1 > p.buflen {
			needed = 1
			goto pullbytes
		}
		rd++

		var sz int
		num, sz = p.readLongByte(p.buf[p.pos+1])
		if sz > 0 && p.pos+rd+sz > p.buflen {
			needed = p.pos + rd + sz - p.buflen
			goto pullbytes
		} else if sz > 0 {
			for i := 0; i < sz; i++ {
				if num < 0 {
					num &= ^(0xff << uint(8*i))
				}

				num |= int(p.buf[p.pos+rd]) << uint(8*i)
				rd++
			}
		}
	}

	p.pos += rd

	return
}

// Given an encoded byte, determines if it represents a concrete long or the number of bytes
// needed to decode the real long.
func (p *Parser) readLongByte(b byte) (n int, sz int) {
	n = int(int8(b))
	if 4 < n && n < 128 {
		n = n - 5
	} else if -129 < n && n < -4 {
		n = n + 5
	} else if n > 0 {
		sz = n
		n = 0
	} else if n != 0 {
		sz = -n
		n = -1
	}

	return
}

// func (p *Parser) long() (n int, err error) {
// 	if p.pos == p.buflen {
// 		err = p.fill(1)
// 		if err != nil {
// 			err = errors.Wrap(err, "error parsing long")
// 			return
// 		}
// 	}

// 	n = int(int8(p.buf[p.pos]))
// 	p.pos++
// 	if n == 0 {
// 		return
// 	} else if 4 < n && n < 128 {
// 		n = n - 5
// 		return
// 	} else if -129 < n && n < -4 {
// 		n = n + 5
// 		return
// 	}

// 	// It's a multibyte positive/negative num.
// 	var sz int
// 	if n > 0 {
// 		sz = n
// 		n = 0
// 	} else {
// 		sz = -n
// 		n = -1
// 	}

// 	if p.pos+sz > p.buflen {
// 		err = p.fill(p.pos + sz - p.buflen)
// 		if err != nil {
// 			err = errors.Wrap(err, "error parsing long")
// 			return
// 		}
// 	}

// 	for i := 0; i < sz; i++ {
// 		if n < 0 {
// 			n &= ^(0xff << uint(8*i))
// 		}

// 		n |= int(p.buf[p.pos]) << uint(8*i)
// 		p.pos++
// 	}

// 	return
// }

// pull bytes from the io.Reader into our read buffer
// func (p *Parser) fill(num int) (err error) {
// 	// We don't do actual reads in sub Parser, the data is already in the buffer.
// 	// if p.lnkID > -1 {
// 	// 	return errors.Errorf("fill() called on replay Parser")
// 	// }

// 	// Whenever we go to pull bytes from the source, we prefetch as much as possible. We do this by examining the current
// 	// stack. For example if we're processing an array with 500 elements and we've currently parsing element 232, then we
// 	// know there's at *least* 267 bytes to come (even if every following element was just a single nil byte).
// 	for i := len(p.stack) - 1; i >= 0; i-- {
// 		if n := p.stack[i].sz - p.stack[i].pos - 1; n > 0 {
// 			num += n
// 		}
// 	}

// 	from, to := p.buflen, p.buflen+num

// 	if to > p.bufcap {
// 		// Overflowed our read buffer, allocate a new one double the current size, or the required size if it's larger.
// 		p.bufcap = p.bufcap * 2
// 		if p.bufcap < to {
// 			p.bufcap = to
// 		}
// 		buf := make([]byte, p.bufcap)
// 		copy(buf, p.buf[0:p.buflen])
// 		p.buf = buf
// 	}

// 	p.buflen += num

// 	var n int
// 	for from < to && err == nil {
// 		n, err = p.r.Read(p.buf[from:to])
// 		from += n
// 	}
// 	if err == io.EOF {
// 		err = io.ErrUnexpectedEOF
// 	} else if err != nil {
// 		err = errors.Wrap(err, "fill")
// 	}
// 	return
// }

// Constructs a ParserError using the current pos of the Parser.
func (p *Parser) parserError(format string, a ...interface{}) ParserError {
	return ParserError{fmt.Sprintf(format, a...), p.pos}
}

const (
	bufInitSz    = 256 // Initial size of our read buffer. We double it each time we overflow available space.
	rngTblInitSz = 8   // Initial size of range table entries
	stackInitSz  = 8   // Initial size of stack
)

type parserState uint8

const (
	parserStateTopLevel = iota
	parserStateArray
	parserStateArrayEnd
	parserStateHashKey
	parserStateHashValue
	parserStateHashEnd
	parserStateIVarInit
	parserStateIVarLen
	parserStateIVarKey
	parserStateIVarValue
	parserStateIVarEnd
	parserStateUsrMarshalInit
	parserStateUsrMarshalVal
	parserStateUsrMarshalEnd
	parserStateEOF
)

// parserCtx tracks the current state we're processing when handling complex values like arrays, hashes, ivars,  etc.
// Multiple contexts can be nested in a stack. For example if we're parsing a hash as the nth element of an array,
// then the top of the stack will be ctxTypeHash and the stack item below that will be ctxTypeArray
type parserCtx struct {
	typ  uint8
	sz   int
	pos  int
	r    *rng        // when this context is finished, r (pointing into lnkTbl) is updated with final location
	next parserState // Next state transition when we're done with this stack item
}

// The valid context types
const (
	ctxTypeArray = iota
	ctxTypeHash
	ctxTypeIVar
	ctxTypeUsrMarshal
	ctxTypeReplay
)

type parserStack []parserCtx

func (stk parserStack) cur() *parserCtx {
	if len(stk) == 0 {
		return nil
	}
	return &stk[len(stk)-1]
}

func (stk *parserStack) push(typ uint8, sz int, next parserState) *parserCtx {
	// We track the current parse sym table by slicing the underlying array.
	// That is, if we've seen one symbol in the stream so far, len(p.symTbl) == 1 && cap(p.symTable) == rngTblInitSz
	// Once we exceed cap, we double size of the tbl.
	l := len(*stk)
	if c := cap(*stk); l == c {
		if c == 0 {
			c = stackInitSz
		} else {
			c = c * 2
		}
		newStk := make([]parserCtx, c)
		copy(newStk, *stk)
		*stk = newStk[0:l]
	}

	*stk = append(*stk, parserCtx{typ: typ, sz: sz, r: nil, next: next})
	return &(*stk)[l]
}

func (stk *parserStack) pop() (next parserState) {
	next = (*stk)[len(*stk)-1].next
	*stk = (*stk)[0 : len(*stk)-1]
	return
}

// A rng encodes a pair of start/end positions, used to mark interesting locations in the read buffer.
type rng struct{ beg, end int }

// Range table
type rngTbl []rng

func (t *rngTbl) add(r rng) (err error) {
	// We track the current parse sym table by slicing the underlying array.
	// That is, if we've seen one symbol in the stream so far, len(p.symTbl) == 1 && cap(p.symTable) == rngTblInitSz
	// Once we exceed cap, we double size of the tbl.
	id := len(*t)
	if c := cap(*t); id == c {
		if c == 0 {
			c = rngTblInitSz
		} else {
			c = c * 2
		}
		newT := make([]rng, c)
		copy(newT, *t)
		*t = newT[0:id]
	}
	*t = append(*t, r)
	return
}
<commit_msg>parser: slightly less insane version of this insane thing I'm trying to do<commit_after>package rmarsh

import (
	"fmt"
	"io"

	"github.com/pkg/errors"
)

// A Token represents a single distinct value type read from a Parser instance.
type Token uint8

// The valid token types.
const (
	tokenInvalid = iota
	tokenStart
	TokenNil
	TokenTrue
	TokenFalse
	TokenFixnum
	TokenFloat
	TokenBignum
	TokenSymbol
	TokenString
	TokenStartArray
	TokenEndArray
	TokenStartHash
	TokenEndHash
	TokenStartIVar
	TokenIVarProps
	TokenEndIVar
	TokenLink
	TokenUsrMarshal
	TokenEOF
)

var tokenNames = map[Token]string{
	TokenNil:        "TokenNil",
	TokenTrue:       "TokenTrue",
	TokenFalse:      "TokenFalse",
	TokenFixnum:     "TokenFixnum",
	TokenFloat:      "TokenFloat",
	TokenBignum:     "TokenBignum",
	TokenSymbol:     "TokenSymbol",
	TokenString:     "TokenString",
	TokenStartArray: "TokenStartArray",
	TokenEndArray:   "TokenEndArray",
	TokenStartHash:  "TokenStartHash",
	TokenEndHash:    "TokenEndHash",
	TokenStartIVar:  "TokenStartIVar",
	TokenIVarProps:  "TokenIVarProps",
	TokenEndIVar:    "TokenEndIVar",
	TokenLink:       "TokenLink",
	TokenUsrMarshal: "TokenUsrMarshal",
	TokenEOF:        "EOF",
}

func (t Token) String() string {
	if n, ok := tokenNames[t]; ok {
		return n
	}
	return "UNKNOWN"
}

// A ParserError is a description of an error encountered while parsing a Ruby Marshal stream.
type ParserError struct {
	msg    string
	Offset int
}

func (e ParserError) Error() string {
	return e.msg
}

// Parser is a low-level pull-based parser of the Ruby Marshal format.
// A Parser will pull bytes from an underlying io.Reader as needed, but will never buffer past the
// end of the current Marshal stream. Even though effort is made to be as efficient in pulling bytes
// as possible, if the Marshal data is already fully available then it should be wrapped in a bufio.Reader
// before being handed to a Parser.
// Parser is very low level and is mostly intended as a building block for the Decoder. You probably
// want to be using that.
type Parser struct {
	r io.Reader // our byte source.

	buf    []byte // The read buffer contains every byte of data that we've read from the stream.
	bufcap int    // Current capacity of the read buffer.
	buflen int    // The number of bytes we've read into the read buffer.
	pos    int    // Our byte position in the read buffer.

	state parserState
	stack parserStack
}

func NewParser(r io.Reader) *Parser {
	return &Parser{
		r:      r,
		buf:    make([]byte, bufInitSz),
		bufcap: bufInitSz,
		state:  parserStateTopLevel,
	}
}

// Reset reverts the Parser into the identity state, ready to read a new Marshal 4.8 stream from the existing Reader.
// If the provided io.Reader is nil, the existing Reader will continue to be used.
func (p *Parser) Reset(r io.Reader) {
	p.stack = p.stack[0:0]
	// p.cur = tokenInvalid
	p.state = parserStateTopLevel

	// If this a replay Parser, our reset is a little less ... reset-y.
	// if p.lnkID > -1 {
	// 	p.pos = p.lnkTbl[p.lnkID].beg
	// 	p.stack = p.stack[0:0]
	// 	return
	// }

	if r != nil {
		p.r = r
	}
	p.pos = 0
	p.buflen = 0
	// p.symTbl = p.symTbl[0:0]
	// p.lnkTbl = p.lnkTbl[0:0]
}

func (p *Parser) Read() (tok Token, b []byte, num int, err error) {
	// Quick early bailout check here. If parser state is "parserStateEOF" then we can just
	// return an EOF token and exit.
	if p.state == parserStateEOF {
		tok = TokenEOF
		return
	}

	// Gets set to false after we run SM.
	runSM := true

	// READ BYTES IF NECESSARY
	// Running the state machine can bail out back here if there's not enough data in the read buffer
	// to transition to the next state.
	// This code would be WAY less complicated if we just filled the buffer with method calls when needed...
	// But that costs too many precious nanos.
	needed := 0
pullbytes:
	if needed > 0 {
		// TODO: port over the stack-based prefetch here.

		from, to := p.buflen, p.buflen+needed

		if to > p.bufcap {
			// Overflowed our read buffer, allocate a new one double the current size, or the required size if it's larger.
			p.bufcap = p.bufcap * 2
			if p.bufcap < to {
				p.bufcap = to
			}
			buf := make([]byte, p.bufcap)
			copy(buf, p.buf[0:p.buflen])
			p.buf = buf
		}

		p.buflen += needed

		var n int
		for from < to && err == nil {
			n, err = p.r.Read(p.buf[from:to])
			from += n
		}
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
			return
		} else if err != nil {
			err = errors.Wrap(err, "fill")
			return
		}

		needed = 0
	}

	// RUN THE STATE MACHINE
	if runSM {
		switch p.state {
		// the initial state of a Parser expects to read 2-byte magic and then a top level value
		case parserStateTopLevel:
			if p.pos == 0 {
				// We can safely pull up to 3 bytes immediately. 2 bytes for the magic and all top level values
				// will be at least 1 byte large.
				if p.buflen < 3 {
					needed = 3 - p.buflen
					goto pullbytes
				}

				if p.buf[p.pos] != 0x04 || p.buf[p.pos+1] != 0x08 {
					err = p.parserError("Expected magic header 0x0408, got 0x%.4X", int16(p.buf[p.pos])<<8|int16(p.buf[p.pos+1]))
					return
				}
				p.pos = 2
			}

			// Our next state is EOF.
			// Unless we read something interesting below which pushes something onto the stack.
			p.state = parserStateEOF
		}

		// Now that we've run the SM, we don't want to run it again if the stream reads
		// need to go back to pullbytes
		runSM = false
	}

	// READ SOMETHING FROM THE STREAM
	if p.pos == p.buflen {
		// This is the worst possible situation to be in - we have to go to the io.Reader to pull a single byte.
		// This situation shouldn't occur very often on real world streams - as we should usually have enough to context to
		// be doing safe read aheads.
		needed = 1
		goto pullbytes
	}

	typ := p.buf[p.pos]
	rd := 1

	switch typ {
	case typeNil:
		tok = TokenNil

	case typeTrue:
		tok = TokenTrue

	case typeFalse:
		tok = TokenFalse

	case typeFixnum:
		tok = TokenFixnum

		var sz int
		num, sz, needed = p.decodeLong(p.pos + rd)
		if needed > 0 {
			goto pullbytes
		}
		rd += sz
	}

	p.pos += rd

	return
}

// decodeLong looks at a long in the read buffer at given pos and decodes it.
// It will return either the decoded num, or the number of extra bytes it needs available
// in the read buffer to complete decoding.
func (p *Parser) decodeLong(pos int) (n, sz, need int) {
	if pos == p.buflen {
		// A pretty shitty situation to end up in, unless we happen to be reading a Marshal stream
		// that only contains a single fixnum.
		need = 1
		return
	}

	// Can finish early if the num is 0.
	if p.buf[pos] == 0 {
		return
	}

	n = int(int8(p.buf[pos]))

	// Easy ones first: single byte longs.
	if 4 < n && n < 128 {
		n = n - 5
		return
	} else if -129 < n && n < -4 {
		n = n + 5
		return
	} else if n > 0 {
		sz = n
		n = 0
	} else {
		sz = -n
		n = -1
	}

	if pos+1+sz > p.buflen {
		need = pos + sz + 1 - p.buflen
		return
	}

	for i := 0; i < sz; i++ {
		if n < 0 {
			n &= ^(0xff << uint(8*i))
		}

		n |= int(p.buf[pos+1+i]) << uint(8*i)
	}

	return
}

// Constructs a ParserError using the current pos of the Parser.
func (p *Parser) parserError(format string, a ...interface{}) ParserError {
	return ParserError{fmt.Sprintf(format, a...), p.pos}
}

const (
	bufInitSz    = 256 // Initial size of our read buffer. We double it each time we overflow available space.
	rngTblInitSz = 8   // Initial size of range table entries
	stackInitSz  = 8   // Initial size of stack
)

type parserState uint8

const (
	parserStateTopLevel = iota
	parserStateArray
	parserStateArrayEnd
	parserStateHashKey
	parserStateHashValue
	parserStateHashEnd
	parserStateIVarInit
	parserStateIVarLen
	parserStateIVarKey
	parserStateIVarValue
	parserStateIVarEnd
	parserStateUsrMarshalInit
	parserStateUsrMarshalVal
	parserStateUsrMarshalEnd
	parserStateEOF
)

// parserCtx tracks the current state we're processing when handling complex values like arrays, hashes, ivars,  etc.
// Multiple contexts can be nested in a stack. For example if we're parsing a hash as the nth element of an array,
// then the top of the stack will be ctxTypeHash and the stack item below that will be ctxTypeArray
type parserCtx struct {
	typ  uint8
	sz   int
	pos  int
	r    *rng        // when this context is finished, r (pointing into lnkTbl) is updated with final location
	next parserState // Next state transition when we're done with this stack item
}

// The valid context types
const (
	ctxTypeArray = iota
	ctxTypeHash
	ctxTypeIVar
	ctxTypeUsrMarshal
	ctxTypeReplay
)

type parserStack []parserCtx

func (stk parserStack) cur() *parserCtx {
	if len(stk) == 0 {
		return nil
	}
	return &stk[len(stk)-1]
}

func (stk *parserStack) push(typ uint8, sz int, next parserState) *parserCtx {
	// We track the current parse sym table by slicing the underlying array.
	// That is, if we've seen one symbol in the stream so far, len(p.symTbl) == 1 && cap(p.symTable) == rngTblInitSz
	// Once we exceed cap, we double size of the tbl.
	l := len(*stk)
	if c := cap(*stk); l == c {
		if c == 0 {
			c = stackInitSz
		} else {
			c = c * 2
		}
		newStk := make([]parserCtx, c)
		copy(newStk, *stk)
		*stk = newStk[0:l]
	}

	*stk = append(*stk, parserCtx{typ: typ, sz: sz, r: nil, next: next})
	return &(*stk)[l]
}

func (stk *parserStack) pop() (next parserState) {
	next = (*stk)[len(*stk)-1].next
	*stk = (*stk)[0 : len(*stk)-1]
	return
}

// A rng encodes a pair of start/end positions, used to mark interesting locations in the read buffer.
type rng struct{ beg, end int }

// Range table
type rngTbl []rng

func (t *rngTbl) add(r rng) (err error) {
	// We track the current parse sym table by slicing the underlying array.
	// That is, if we've seen one symbol in the stream so far, len(p.symTbl) == 1 && cap(p.symTable) == rngTblInitSz
	// Once we exceed cap, we double size of the tbl.
	id := len(*t)
	if c := cap(*t); id == c {
		if c == 0 {
			c = rngTblInitSz
		} else {
			c = c * 2
		}
		newT := make([]rng, c)
		copy(newT, *t)
		*t = newT[0:id]
	}
	*t = append(*t, r)
	return
}
<|endoftext|>


<commit_before>package confreaks

import (
	"code.google.com/p/cascadia"
	"code.google.com/p/go-uuid/uuid"
	"code.google.com/p/go.net/html"
	"io"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

func ParseEvents(r io.Reader) (events []*Event, err error) {
	doc, err := html.Parse(r)
	if err != nil {
		return
	}

	eventSel := cascadia.MustCompile("li.event-box")
	for _, dom := range eventSel.MatchAll(doc) {
		ev := &Event{}
		ev.UUID = uuid.NewUUID().String()

		linkSelector := cascadia.MustCompile("span.small > strong > a")
		link := linkSelector.MatchFirst(dom)
		ev.Title = strings.TrimSpace(link.LastChild.Data)

		uri, err := resolveURI(attrVal(link, "href"))
		if err == nil {
			ev.URL = uri.String()
		}

		dateSelector := cascadia.MustCompile("span.small > a")
		date := dateSelector.MatchFirst(dom)
		dstr := date.FirstChild.Data

		re := regexp.MustCompile("([a-zA-Z]{3}) ([0-9]{2})( - [0-9]{2})?, ([0-9]{4})")
		dclean := re.ReplaceAllString(dstr, "$1 $2, $4")

		dt, err := time.Parse("Jan 02, 2006", dclean)
		if err != nil {
			dt = time.Time{}
		}

		ev.Date = dt.UTC()

		numSelector := cascadia.MustCompile("span:last-child.small")
		num := numSelector.MatchFirst(dom)
		nstr := strings.Split(strings.TrimSpace(num.FirstChild.Data), "\n")
		if n, err := strconv.Atoi(nstr[0]); err == nil {
			ev.Count = int32(n)
		}

		events = append(events, ev)
	}

	return
}

func ParseEvent(r io.Reader, event *Event) error {
	doc, err := html.Parse(r)
	if err != nil {
		return err
	}

	var presentationsSelector = cascadia.MustCompile("div.video")
	for _, dom := range presentationsSelector.MatchAll(doc) {
		p := &Presentation{}

		recordedSelector := cascadia.MustCompile(".recorded-at")
		recorded := recordedSelector.MatchFirst(dom)
		recordedStr := strings.TrimSpace(recorded.FirstChild.Data)
		recordedAt, err := time.Parse("02-Jan-06 15:04", recordedStr)
		if err == nil {
			p.Recorded = recordedAt
		}

		infoSelector := cascadia.MustCompile(".main-info")
		info := infoSelector.MatchFirst(dom)

		linkSelector := cascadia.MustCompile(".title a")
		link := linkSelector.MatchFirst(info)
		p.Title = strings.TrimSpace(link.LastChild.Data)

		uri, err := resolveURI(attrVal(link, "href"))
		if err == nil {
			p.URL = uri.String()
		}

		presentersSelector := cascadia.MustCompile(".presenters a")
		presenters := []string{}
		for _, presenter := range presentersSelector.MatchAll(info) {
			presenters = append(presenters, presenter.LastChild.Data)
		}

		p.Presenters = presenters
		event.Presentations = append(event.Presentations, p)
	}

	return nil
}

func ParsePresentation(r io.Reader, p *Presentation) error {
	var err error

	doc, err := html.Parse(r)
	if err != nil {
		return err
	}

	var extract func(*html.Node) []string
	var normalize func(s string) string

	normalize = func(s string) string {
		au, err := url.Parse(s)
		if err != nil {
			return ""
		}

		switch au.Host {
		case "www.youtube.com":
			// http://www.youtube.com/embed/sVd4p6oKeUA
			// => http://www.youtube.com/watch?v=sVd4p6oKeUA

			v := url.Values{}
			v.Set("v", strings.Replace(au.Path, "/embed/", "", 1))

			au.Path = "watch"
			au.RawQuery = v.Encode()
		case "player.vimeo.com":
			// http://player.vimeo.com/video/40143060?badge=0
			// => http://vimeo.com/40143060

			au.Host = "vimeo.com"
			au.Path = strings.Replace(au.Path, "/video/", "", 1)
			au.RawQuery = ""
		}

		return au.String()
	}

	extract = func(n *html.Node) []string {
		texts := []string{}

		for ch := n.FirstChild; ch != nil; ch = ch.NextSibling {
			if ch.LastChild != nil {
				texts = append(texts, ch.LastChild.Data)
			}
		}

		return texts
	}

	presentationSelector := cascadia.MustCompile("div#primary-content")
	content := presentationSelector.MatchFirst(doc)

	titleSelector := cascadia.MustCompile(".video-title")
	title := titleSelector.MatchFirst(content)
	p.Title = strings.TrimSpace(title.LastChild.Data)

	descriptionSelector := cascadia.MustCompile(".video-abstract")
	description := descriptionSelector.MatchFirst(content)
	p.Description = strings.Join(extract(description), "\n")

	var videoSelector cascadia.Selector
	var video *html.Node

	videoSelector = cascadia.MustCompile("iframe")
	video = videoSelector.MatchFirst(content)

	if video == nil {
		videoSelector = cascadia.MustCompile("video source")
		video = videoSelector.MatchFirst(content)
	}

	if video != nil {
		p.VideoURL = normalize(attrVal(video, "src"))
	}

	return nil
}

func attrVal(n *html.Node, k string) string {
	for _, a := range n.Attr {
		if a.Key == k {
			return a.Val
		}
	}

	return ""
}
<commit_msg>no more code.google.com<commit_after>package confreaks

import (
	"github.com/andybalholm/cascadia"
	"github.com/subosito/go-uuid/uuid"
	"golang.org/x/net/html"
	"io"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

func ParseEvents(r io.Reader) (events []*Event, err error) {
	doc, err := html.Parse(r)
	if err != nil {
		return
	}

	eventSel := cascadia.MustCompile("li.event-box")
	for _, dom := range eventSel.MatchAll(doc) {
		ev := &Event{}
		ev.UUID = uuid.NewUUID().String()

		linkSelector := cascadia.MustCompile("span.small > strong > a")
		link := linkSelector.MatchFirst(dom)
		ev.Title = strings.TrimSpace(link.LastChild.Data)

		uri, err := resolveURI(attrVal(link, "href"))
		if err == nil {
			ev.URL = uri.String()
		}

		dateSelector := cascadia.MustCompile("span.small > a")
		date := dateSelector.MatchFirst(dom)
		dstr := date.FirstChild.Data

		re := regexp.MustCompile("([a-zA-Z]{3}) ([0-9]{2})( - [0-9]{2})?, ([0-9]{4})")
		dclean := re.ReplaceAllString(dstr, "$1 $2, $4")

		dt, err := time.Parse("Jan 02, 2006", dclean)
		if err != nil {
			dt = time.Time{}
		}

		ev.Date = dt.UTC()

		numSelector := cascadia.MustCompile("span:last-child.small")
		num := numSelector.MatchFirst(dom)
		nstr := strings.Split(strings.TrimSpace(num.FirstChild.Data), "\n")
		if n, err := strconv.Atoi(nstr[0]); err == nil {
			ev.Count = int32(n)
		}

		events = append(events, ev)
	}

	return
}

func ParseEvent(r io.Reader, event *Event) error {
	doc, err := html.Parse(r)
	if err != nil {
		return err
	}

	var presentationsSelector = cascadia.MustCompile("div.video")
	for _, dom := range presentationsSelector.MatchAll(doc) {
		p := &Presentation{}

		recordedSelector := cascadia.MustCompile(".recorded-at")
		recorded := recordedSelector.MatchFirst(dom)
		recordedStr := strings.TrimSpace(recorded.FirstChild.Data)
		recordedAt, err := time.Parse("02-Jan-06 15:04", recordedStr)
		if err == nil {
			p.Recorded = recordedAt
		}

		infoSelector := cascadia.MustCompile(".main-info")
		info := infoSelector.MatchFirst(dom)

		linkSelector := cascadia.MustCompile(".title a")
		link := linkSelector.MatchFirst(info)
		p.Title = strings.TrimSpace(link.LastChild.Data)

		uri, err := resolveURI(attrVal(link, "href"))
		if err == nil {
			p.URL = uri.String()
		}

		presentersSelector := cascadia.MustCompile(".presenters a")
		presenters := []string{}
		for _, presenter := range presentersSelector.MatchAll(info) {
			presenters = append(presenters, presenter.LastChild.Data)
		}

		p.Presenters = presenters
		event.Presentations = append(event.Presentations, p)
	}

	return nil
}

func ParsePresentation(r io.Reader, p *Presentation) error {
	var err error

	doc, err := html.Parse(r)
	if err != nil {
		return err
	}

	var extract func(*html.Node) []string
	var normalize func(s string) string

	normalize = func(s string) string {
		au, err := url.Parse(s)
		if err != nil {
			return ""
		}

		switch au.Host {
		case "www.youtube.com":
			// http://www.youtube.com/embed/sVd4p6oKeUA
			// => http://www.youtube.com/watch?v=sVd4p6oKeUA

			v := url.Values{}
			v.Set("v", strings.Replace(au.Path, "/embed/", "", 1))

			au.Path = "watch"
			au.RawQuery = v.Encode()
		case "player.vimeo.com":
			// http://player.vimeo.com/video/40143060?badge=0
			// => http://vimeo.com/40143060

			au.Host = "vimeo.com"
			au.Path = strings.Replace(au.Path, "/video/", "", 1)
			au.RawQuery = ""
		}

		return au.String()
	}

	extract = func(n *html.Node) []string {
		texts := []string{}

		for ch := n.FirstChild; ch != nil; ch = ch.NextSibling {
			if ch.LastChild != nil {
				texts = append(texts, ch.LastChild.Data)
			}
		}

		return texts
	}

	presentationSelector := cascadia.MustCompile("div#primary-content")
	content := presentationSelector.MatchFirst(doc)

	titleSelector := cascadia.MustCompile(".video-title")
	title := titleSelector.MatchFirst(content)
	p.Title = strings.TrimSpace(title.LastChild.Data)

	descriptionSelector := cascadia.MustCompile(".video-abstract")
	description := descriptionSelector.MatchFirst(content)
	p.Description = strings.Join(extract(description), "\n")

	var videoSelector cascadia.Selector
	var video *html.Node

	videoSelector = cascadia.MustCompile("iframe")
	video = videoSelector.MatchFirst(content)

	if video == nil {
		videoSelector = cascadia.MustCompile("video source")
		video = videoSelector.MatchFirst(content)
	}

	if video != nil {
		p.VideoURL = normalize(attrVal(video, "src"))
	}

	return nil
}

func attrVal(n *html.Node, k string) string {
	for _, a := range n.Attr {
		if a.Key == k {
			return a.Val
		}
	}

	return ""
}
<|endoftext|>