<commit_before>package lz4

import (
	"bytes"
	"io"

	"github.com/pierrec/lz4/v4/internal/lz4block"
	"github.com/pierrec/lz4/v4/internal/lz4errors"
	"github.com/pierrec/lz4/v4/internal/lz4stream"
)

var readerStates = []aState{
	noState:     newState,
	errorState:  newState,
	newState:    readState,
	readState:   closedState,
	closedState: newState,
}

// NewReader returns a new LZ4 frame decoder.
func NewReader(r io.Reader) *Reader {
	return newReader(r, false)
}

func newReader(r io.Reader, legacy bool) *Reader {
	zr := &Reader{frame: lz4stream.NewFrame()}
	zr.state.init(readerStates)
	_ = zr.Apply(DefaultConcurrency, defaultOnBlockDone)
	zr.Reset(r)
	return zr
}

// Reader allows reading an LZ4 stream.
type Reader struct {
	state   _State
	src     io.Reader        // source reader
	num     int              // concurrency level
	frame   *lz4stream.Frame // frame being read
	data    []byte           // block buffer allocated in non concurrent mode
	reads   chan []byte      // pending data
	idx     int              // size of pending data
	handler func(int)
	cum     uint32
	dict    []byte
}

func (*Reader) private() {}

func (r *Reader) Apply(options ...Option) (err error) {
	defer r.state.check(&err)
	switch r.state.state {
	case newState:
	case errorState:
		return r.state.err
	default:
		return lz4errors.ErrOptionClosedOrError
	}
	for _, o := range options {
		if err = o(r); err != nil {
			return
		}
	}
	return
}

// Size returns the size of the underlying uncompressed data, if set in the stream.
func (r *Reader) Size() int {
	switch r.state.state {
	case readState, closedState:
		if r.frame.Descriptor.Flags.Size() {
			return int(r.frame.Descriptor.ContentSize)
		}
	}
	return 0
}

func (r *Reader) isNotConcurrent() bool {
	return r.num == 1
}

func (r *Reader) init() error {
	err := r.frame.ParseHeaders(r.src)
	if err != nil {
		return err
	}
	if !r.frame.Descriptor.Flags.BlockIndependence() {
		// We can't decompress dependent blocks concurrently.
		// Instead of throwing an error to the user, silently drop concurrency
		r.num = 1
	}
	data, err := r.frame.InitR(r.src, r.num)
	if err != nil {
		return err
	}
	r.reads = data
	r.idx = 0
	size := r.frame.Descriptor.Flags.BlockSizeIndex()
	r.data = size.Get()
	r.cum = 0
	return nil
}

func (r *Reader) Read(buf []byte) (n int, err error) {
	defer r.state.check(&err)
	switch r.state.state {
	case readState:
	case closedState, errorState:
		return 0, r.state.err
	case newState:
		// First initialization.
		if err = r.init(); r.state.next(err) {
			return
		}
	default:
		return 0, r.state.fail()
	}
	for len(buf) > 0 {
		var bn int
		if r.idx == 0 {
			if r.isNotConcurrent() {
				bn, err = r.read(buf)
			} else {
				lz4block.Put(r.data)
				r.data = <-r.reads
				if len(r.data) == 0 {
					// No uncompressed data: something went wrong or we are done.
					err = r.frame.Blocks.ErrorR()
				}
			}
			switch err {
			case nil:
			case io.EOF:
				if er := r.frame.CloseR(r.src); er != nil {
					err = er
				}
				lz4block.Put(r.data)
				r.data = nil
				return
			default:
				return
			}
		}
		if bn == 0 {
			// Fill buf with buffered data.
			bn = copy(buf, r.data[r.idx:])
			r.idx += bn
			if r.idx == len(r.data) {
				// All data read, get ready for the next Read.
				r.idx = 0
			}
		}
		buf = buf[bn:]
		n += bn
		r.handler(bn)
	}
	return
}

// read uncompresses the next block as follow:
// - if buf has enough room, the block is uncompressed into it directly
//   and the lenght of used space is returned
// - else, the uncompress data is stored in r.data and 0 is returned
func (r *Reader) read(buf []byte) (int, error) {
	block := r.frame.Blocks.Block
	_, err := block.Read(r.frame, r.src, r.cum)
	if err != nil {
		return 0, err
	}
	var direct bool
	dst := r.data[:cap(r.data)]
	if len(buf) >= len(dst) {
		// Uncompress directly into buf.
		direct = true
		dst = buf
	}
	dst, err = block.Uncompress(r.frame, dst, r.dict, true)
	if err != nil {
		return 0, err
	}
	if !r.frame.Descriptor.Flags.BlockIndependence() {
		if len(r.dict)+len(dst) > 128*1024 {
			preserveSize := 64*1024 - len(dst)
			if preserveSize < 0 {
				preserveSize = 0
			}
			r.dict = r.dict[len(r.dict)-preserveSize:]
		}
		r.dict = append(r.dict, dst...)
	}
	r.cum += uint32(len(dst))
	if direct {
		return len(dst), nil
	}
	r.data = dst
	return 0, nil
}

// Reset clears the state of the Reader r such that it is equivalent to its
// initial state from NewReader, but instead writing to writer.
// No access to reader is performed.
//
// w.Close must be called before Reset.
func (r *Reader) Reset(reader io.Reader) {
	if r.data != nil {
		lz4block.Put(r.data)
		r.data = nil
	}
	r.frame.Reset(r.num)
	r.state.reset()
	r.src = reader
	r.reads = nil
}

// WriteTo efficiently uncompresses the data from the Reader underlying source to w.
func (r *Reader) WriteTo(w io.Writer) (n int64, err error) {
	switch r.state.state {
	case closedState, errorState:
		return 0, r.state.err
	case newState:
		if err = r.init(); r.state.next(err) {
			return
		}
	default:
		return 0, r.state.fail()
	}
	defer r.state.nextd(&err)

	var data []byte
	if r.isNotConcurrent() {
		size := r.frame.Descriptor.Flags.BlockSizeIndex()
		data = size.Get()
		defer lz4block.Put(data)
	}
	for {
		var bn int
		var dst []byte
		if r.isNotConcurrent() {
			bn, err = r.read(data)
			dst = data[:bn]
		} else {
			lz4block.Put(dst)
			dst = <-r.reads
			bn = len(dst)
			if bn == 0 {
				// No uncompressed data: something went wrong or we are done.
				err = r.frame.Blocks.ErrorR()
			}
		}
		switch err {
		case nil:
		case io.EOF:
			err = r.frame.CloseR(r.src)
			return
		default:
			return
		}
		r.handler(bn)
		bn, err = w.Write(dst)
		n += int64(bn)
		if err != nil {
			return
		}
	}
}

// ValidFrameHeader returns a bool indicating if the given bytes slice matches a LZ4 header.
func ValidFrameHeader(in []byte) (bool, error) {
	f := lz4stream.NewFrame()
	err := f.ParseHeaders(bytes.NewReader(in))
	if err == nil {
		return true, nil
	}
	if err == lz4errors.ErrInvalidFrame {
		return false, nil
	}
	return false, err
}
<commit_msg>Fix Reader.Reset() docs<commit_after>package lz4

import (
	"bytes"
	"io"

	"github.com/pierrec/lz4/v4/internal/lz4block"
	"github.com/pierrec/lz4/v4/internal/lz4errors"
	"github.com/pierrec/lz4/v4/internal/lz4stream"
)

var readerStates = []aState{
	noState:     newState,
	errorState:  newState,
	newState:    readState,
	readState:   closedState,
	closedState: newState,
}

// NewReader returns a new LZ4 frame decoder.
func NewReader(r io.Reader) *Reader {
	return newReader(r, false)
}

func newReader(r io.Reader, legacy bool) *Reader {
	zr := &Reader{frame: lz4stream.NewFrame()}
	zr.state.init(readerStates)
	_ = zr.Apply(DefaultConcurrency, defaultOnBlockDone)
	zr.Reset(r)
	return zr
}

// Reader allows reading an LZ4 stream.
type Reader struct {
	state   _State
	src     io.Reader        // source reader
	num     int              // concurrency level
	frame   *lz4stream.Frame // frame being read
	data    []byte           // block buffer allocated in non concurrent mode
	reads   chan []byte      // pending data
	idx     int              // size of pending data
	handler func(int)
	cum     uint32
	dict    []byte
}

func (*Reader) private() {}

func (r *Reader) Apply(options ...Option) (err error) {
	defer r.state.check(&err)
	switch r.state.state {
	case newState:
	case errorState:
		return r.state.err
	default:
		return lz4errors.ErrOptionClosedOrError
	}
	for _, o := range options {
		if err = o(r); err != nil {
			return
		}
	}
	return
}

// Size returns the size of the underlying uncompressed data, if set in the stream.
func (r *Reader) Size() int {
	switch r.state.state {
	case readState, closedState:
		if r.frame.Descriptor.Flags.Size() {
			return int(r.frame.Descriptor.ContentSize)
		}
	}
	return 0
}

func (r *Reader) isNotConcurrent() bool {
	return r.num == 1
}

func (r *Reader) init() error {
	err := r.frame.ParseHeaders(r.src)
	if err != nil {
		return err
	}
	if !r.frame.Descriptor.Flags.BlockIndependence() {
		// We can't decompress dependent blocks concurrently.
		// Instead of throwing an error to the user, silently drop concurrency
		r.num = 1
	}
	data, err := r.frame.InitR(r.src, r.num)
	if err != nil {
		return err
	}
	r.reads = data
	r.idx = 0
	size := r.frame.Descriptor.Flags.BlockSizeIndex()
	r.data = size.Get()
	r.cum = 0
	return nil
}

func (r *Reader) Read(buf []byte) (n int, err error) {
	defer r.state.check(&err)
	switch r.state.state {
	case readState:
	case closedState, errorState:
		return 0, r.state.err
	case newState:
		// First initialization.
		if err = r.init(); r.state.next(err) {
			return
		}
	default:
		return 0, r.state.fail()
	}
	for len(buf) > 0 {
		var bn int
		if r.idx == 0 {
			if r.isNotConcurrent() {
				bn, err = r.read(buf)
			} else {
				lz4block.Put(r.data)
				r.data = <-r.reads
				if len(r.data) == 0 {
					// No uncompressed data: something went wrong or we are done.
					err = r.frame.Blocks.ErrorR()
				}
			}
			switch err {
			case nil:
			case io.EOF:
				if er := r.frame.CloseR(r.src); er != nil {
					err = er
				}
				lz4block.Put(r.data)
				r.data = nil
				return
			default:
				return
			}
		}
		if bn == 0 {
			// Fill buf with buffered data.
			bn = copy(buf, r.data[r.idx:])
			r.idx += bn
			if r.idx == len(r.data) {
				// All data read, get ready for the next Read.
				r.idx = 0
			}
		}
		buf = buf[bn:]
		n += bn
		r.handler(bn)
	}
	return
}

// read uncompresses the next block as follow:
// - if buf has enough room, the block is uncompressed into it directly
//   and the lenght of used space is returned
// - else, the uncompress data is stored in r.data and 0 is returned
func (r *Reader) read(buf []byte) (int, error) {
	block := r.frame.Blocks.Block
	_, err := block.Read(r.frame, r.src, r.cum)
	if err != nil {
		return 0, err
	}
	var direct bool
	dst := r.data[:cap(r.data)]
	if len(buf) >= len(dst) {
		// Uncompress directly into buf.
		direct = true
		dst = buf
	}
	dst, err = block.Uncompress(r.frame, dst, r.dict, true)
	if err != nil {
		return 0, err
	}
	if !r.frame.Descriptor.Flags.BlockIndependence() {
		if len(r.dict)+len(dst) > 128*1024 {
			preserveSize := 64*1024 - len(dst)
			if preserveSize < 0 {
				preserveSize = 0
			}
			r.dict = r.dict[len(r.dict)-preserveSize:]
		}
		r.dict = append(r.dict, dst...)
	}
	r.cum += uint32(len(dst))
	if direct {
		return len(dst), nil
	}
	r.data = dst
	return 0, nil
}

// Reset clears the state of the Reader r such that it is equivalent to its
// initial state from NewReader, but instead reading from reader.
// No access to reader is performed.
func (r *Reader) Reset(reader io.Reader) {
	if r.data != nil {
		lz4block.Put(r.data)
		r.data = nil
	}
	r.frame.Reset(r.num)
	r.state.reset()
	r.src = reader
	r.reads = nil
}

// WriteTo efficiently uncompresses the data from the Reader underlying source to w.
func (r *Reader) WriteTo(w io.Writer) (n int64, err error) {
	switch r.state.state {
	case closedState, errorState:
		return 0, r.state.err
	case newState:
		if err = r.init(); r.state.next(err) {
			return
		}
	default:
		return 0, r.state.fail()
	}
	defer r.state.nextd(&err)

	var data []byte
	if r.isNotConcurrent() {
		size := r.frame.Descriptor.Flags.BlockSizeIndex()
		data = size.Get()
		defer lz4block.Put(data)
	}
	for {
		var bn int
		var dst []byte
		if r.isNotConcurrent() {
			bn, err = r.read(data)
			dst = data[:bn]
		} else {
			lz4block.Put(dst)
			dst = <-r.reads
			bn = len(dst)
			if bn == 0 {
				// No uncompressed data: something went wrong or we are done.
				err = r.frame.Blocks.ErrorR()
			}
		}
		switch err {
		case nil:
		case io.EOF:
			err = r.frame.CloseR(r.src)
			return
		default:
			return
		}
		r.handler(bn)
		bn, err = w.Write(dst)
		n += int64(bn)
		if err != nil {
			return
		}
	}
}

// ValidFrameHeader returns a bool indicating if the given bytes slice matches a LZ4 header.
func ValidFrameHeader(in []byte) (bool, error) {
	f := lz4stream.NewFrame()
	err := f.ParseHeaders(bytes.NewReader(in))
	if err == nil {
		return true, nil
	}
	if err == lz4errors.ErrInvalidFrame {
		return false, nil
	}
	return false, err
}
<|endoftext|>


<commit_before>package iso9660

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/c4milo/gotoolkit"
)

var (
	// ErrInvalidImage is returned when an attempt to unpack the image Primary Volume Descriptor failed or
	// when the end of the image was reached without finding a primary volume descriptor
	ErrInvalidImage = func(err error) error { return fmt.Errorf("invalid-iso9660-image: %s", err) }
	// ErrCorruptedImage is returned when a seek operation, on the image, failed.
	ErrCorruptedImage = func(err error) error { return fmt.Errorf("corrupted-image: %s", err) }
)

// Reader defines the state of the ISO9660 image reader. It needs to be instantiated
// from its constructor.
type Reader struct {
	// File descriptor to the opened ISO image
	image io.ReadSeeker
	// Copy of unencoded Primary Volume Descriptor
	pvd PrimaryVolume
	// Queue used to walk through file system iteratively
	queue gotoolkit.Queue
	// Current sector
	sector uint32
	// Current bytes read from current sector.
	read uint32
}

// NewReader creates a new ISO9660 reader.
func NewReader(rs io.ReadSeeker) (*Reader, error) {
	// Starts reading from image data area
	sector := dataAreaSector
	// Iterates over volume descriptors until it finds the primary volume descriptor
	// or an error condition.
	for {
		offset, err := rs.Seek(int64(sector*sectorSize), os.SEEK_SET)
		if err != nil {
			return nil, ErrCorruptedImage(err)
		}

		var volDesc VolumeDescriptor
		if err := binary.Read(rs, binary.BigEndian, &volDesc); err != nil {
			return nil, ErrCorruptedImage(err)
		}

		if volDesc.Type == primaryVol {
			// backs up to the beginning of the sector again in order to unpack
			// the entire primary volume descriptor more easily.
			if _, err := rs.Seek(offset, os.SEEK_SET); err != nil {
				return nil, ErrCorruptedImage(err)
			}

			reader := new(Reader)
			reader.image = rs
			reader.queue = new(gotoolkit.SliceQueue)

			if err := reader.unpackPVD(); err != nil {
				return nil, ErrCorruptedImage(err)
			}

			return reader, nil
		}

		if volDesc.Type == volSetTerminator {
			return nil, ErrInvalidImage(errors.New("Volume Set Terminator reached. A Primary Volume Descriptor was not found."))
		}
		sector++
	}
}

// Next moves onto the next directory record using breadth-first search one step
// at the time. It first moves
func (r *Reader) Next() (os.FileInfo, error) {
	if r == nil {
		panic("missing reader instance. Use the constructor to create a Reader instance.")
	}

	if r.queue.IsEmpty() {
		return nil, io.EOF
	}

	// We only dequeue the directory when it does not contain more children
	// or when it is empty and there is no children to go over.
	item, err := r.queue.Peek()
	if err != nil {
		panic(err)
	}
	fi := item.(FileStat)

	fmt.Println(r.sector)
	// If there is no more entries in the current directory, dequeue it
	// and move on the next directory in the queue.
	if r.read > fi.ExtentLengthBE {
		r.read = 0
		r.queue.Dequeue()
		r.sector = 0
		return &fi, nil
	}

	if r.sector == 0 {
		r.sector = fi.ExtentLocationBE
	}

	var drecord FileStat
	var len byte
	// We need this loop to skip parent and self directories: .. and .
	for {
		// If we are at the end of the sector, move onto the next one.
		if (r.read % sectorSize) == 0 {
			r.sector++
			_, err := r.image.Seek(int64(r.sector*sectorSize), os.SEEK_SET)
			if err != nil {
				return nil, ErrCorruptedImage(err)
			}
		}

		if len, err = r.unpackDRecord(&drecord); err != nil && err != io.EOF {
			return nil, ErrCorruptedImage(err)
		}
		r.read += uint32(len)

		if err == io.EOF {
			// directory record is empty, sector lost, move onto next sector.
			rsize := (sectorSize - (r.read % sectorSize))
			buf := make([]byte, rsize)
			if err := binary.Read(r.image, binary.BigEndian, buf); err != nil {
				return nil, ErrCorruptedImage(err)
			}
			r.read += rsize
		}

		if drecord.fileID != "\x00" && drecord.fileID != "\x01" {
			if drecord.IsDir() {
				r.queue.Enqueue(drecord)
			} else {
				drecord.image = r.image
			}
			break
		}
	}

	return &drecord, nil
}

// unpackDRecord unpacks directory record bits into Go's struct
func (r *Reader) unpackDRecord(fi *FileStat) (byte, error) {
	// Gets the directory record length
	var len byte
	if err := binary.Read(r.image, binary.BigEndian, &len); err != nil {
		return len, ErrCorruptedImage(err)
	}

	if len == 0 {
		return len + 1, io.EOF
	}

	// Reads directory record into Go struct
	var drecord DirectoryRecord
	if err := binary.Read(r.image, binary.BigEndian, &drecord); err != nil {
		return len, ErrCorruptedImage(err)
	}

	fi.DirectoryRecord = drecord
	// Gets the name
	name := make([]byte, drecord.FileIDLength)
	if err := binary.Read(r.image, binary.BigEndian, name); err != nil {
		return len, ErrCorruptedImage(err)
	}
	fi.fileID = string(name)

	// Padding field as per section 9.1.12 in ECMA-119
	if (drecord.FileIDLength % 2) == 0 {
		var zero byte
		if err := binary.Read(r.image, binary.BigEndian, &zero); err != nil {
			return len, ErrCorruptedImage(err)
		}
	}

	// System use field as per section 9.1.13 in ECMA-119
	// Directory record has 34 bytes in addition to the name's
	// variable length and the padding field mentioned in section 9.1.12
	totalLen := 34 + drecord.FileIDLength - (drecord.FileIDLength % 2)
	sysUseLen := int64(len - totalLen)
	if sysUseLen > 0 {
		sysData := make([]byte, sysUseLen)
		if err := binary.Read(r.image, binary.BigEndian, sysData); err != nil {
			return len, ErrCorruptedImage(err)
		}
	}
	return len, nil
}

// unpackPVD unpacks Primary Volume Descriptor in three phases. This is
// because the root directory record is a variable-length record and Go's binary
// package doesn't support unpacking variable-length structs easily.
func (r *Reader) unpackPVD() error {
	// Unpack first half
	var pvd1 PrimaryVolumePart1
	if err := binary.Read(r.image, binary.BigEndian, &pvd1); err != nil {
		return ErrCorruptedImage(err)
	}
	r.pvd.PrimaryVolumePart1 = pvd1

	// Unpack root directory record
	var drecord FileStat
	if _, err := r.unpackDRecord(&drecord); err != nil {
		return ErrCorruptedImage(err)
	}
	r.pvd.DirectoryRecord = drecord.DirectoryRecord
	r.queue.Enqueue(drecord)

	// Unpack second half
	var pvd2 PrimaryVolumePart2
	if err := binary.Read(r.image, binary.BigEndian, &pvd2); err != nil {
		return ErrCorruptedImage(err)
	}
	r.pvd.PrimaryVolumePart2 = pvd2

	return nil
}
<commit_msg>Getting closer...<commit_after>package iso9660

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/c4milo/gotoolkit"
)

var (
	// ErrInvalidImage is returned when an attempt to unpack the image Primary Volume Descriptor failed or
	// when the end of the image was reached without finding a primary volume descriptor
	ErrInvalidImage = func(err error) error { return fmt.Errorf("invalid-iso9660-image: %s", err) }
	// ErrCorruptedImage is returned when a seek operation, on the image, failed.
	ErrCorruptedImage = func(err error) error { return fmt.Errorf("corrupted-image: %s", err) }
)

// Reader defines the state of the ISO9660 image reader. It needs to be instantiated
// from its constructor.
type Reader struct {
	// File descriptor to the opened ISO image
	image io.ReadSeeker
	// Copy of unencoded Primary Volume Descriptor
	pvd PrimaryVolume
	// Queue used to walk through file system iteratively
	queue gotoolkit.Queue
	// Current sector
	sector uint32
	// Current bytes read from current sector.
	read uint32
}

// NewReader creates a new ISO9660 reader.
func NewReader(rs io.ReadSeeker) (*Reader, error) {
	// Starts reading from image data area
	sector := dataAreaSector
	// Iterates over volume descriptors until it finds the primary volume descriptor
	// or an error condition.
	for {
		offset, err := rs.Seek(int64(sector*sectorSize), os.SEEK_SET)
		if err != nil {
			return nil, ErrCorruptedImage(err)
		}

		var volDesc VolumeDescriptor
		if err := binary.Read(rs, binary.BigEndian, &volDesc); err != nil {
			return nil, ErrCorruptedImage(err)
		}

		if volDesc.Type == primaryVol {
			// backs up to the beginning of the sector again in order to unpack
			// the entire primary volume descriptor more easily.
			if _, err := rs.Seek(offset, os.SEEK_SET); err != nil {
				return nil, ErrCorruptedImage(err)
			}

			reader := new(Reader)
			reader.image = rs
			reader.queue = new(gotoolkit.SliceQueue)

			if err := reader.unpackPVD(); err != nil {
				return nil, ErrCorruptedImage(err)
			}

			return reader, nil
		}

		if volDesc.Type == volSetTerminator {
			return nil, ErrInvalidImage(errors.New("Volume Set Terminator reached. A Primary Volume Descriptor was not found."))
		}
		sector++
	}
}

// Next moves onto the next directory record using breadth-first search one step
// at the time. It first moves
func (r *Reader) Next() (os.FileInfo, error) {
	if r == nil {
		panic("missing reader instance. Use the constructor to create a Reader instance.")
	}

	if r.queue.IsEmpty() {
		return nil, io.EOF
	}

	// We only dequeue the directory when it does not contain more children
	// or when it is empty and there is no children to iterate over.
	item, err := r.queue.Peek()
	if err != nil {
		panic(err)
	}
	fi := item.(FileStat)

	if r.sector == 0 {
		r.sector = fi.ExtentLocationBE
	}

	var drecord FileStat
	var len byte
	// This loops exists so we can skip .. and . directories
	for {
		if (r.read % sectorSize) == 0 {
			_, err := r.image.Seek(int64(r.sector*sectorSize), os.SEEK_SET)
			if err != nil {
				return nil, ErrCorruptedImage(err)
			}
			r.sector++
		}

		if len, err = r.unpackDRecord(&drecord); err != nil && err != io.EOF {
			return nil, ErrCorruptedImage(err)
		}

		if err == io.EOF {
			// directory record is empty, sector wasted, move onto next sector.
			rsize := (sectorSize - (r.read % sectorSize))
			buf := make([]byte, rsize)
			if err := binary.Read(r.image, binary.BigEndian, buf); err != nil {
				return nil, ErrCorruptedImage(err)
			}
			r.read += rsize
		}
		r.read += uint32(len)

		if drecord.fileID != "\x00" && drecord.fileID != "\x01" {
			break
		}
	}

	if drecord.IsDir() {
		r.queue.Enqueue(drecord)
	} else {
		drecord.image = r.image
	}

	// If there is no more entries in the current directory, dequeue it
	// and move on the next directory in the queue.
	if r.read > fi.ExtentLengthBE {
		r.read = 0
		r.queue.Dequeue()
		r.sector = 0
	}

	return &drecord, nil
}

// unpackDRecord unpacks directory record bits into Go's struct
func (r *Reader) unpackDRecord(fi *FileStat) (byte, error) {
	// Gets the directory record length
	var len byte
	if err := binary.Read(r.image, binary.BigEndian, &len); err != nil {
		return len, ErrCorruptedImage(err)
	}

	if len == 0 {
		return len + 1, io.EOF
	}

	// Reads directory record into Go struct
	var drecord DirectoryRecord
	if err := binary.Read(r.image, binary.BigEndian, &drecord); err != nil {
		return len, ErrCorruptedImage(err)
	}

	fi.DirectoryRecord = drecord
	// Gets the name
	name := make([]byte, drecord.FileIDLength)
	if err := binary.Read(r.image, binary.BigEndian, name); err != nil {
		return len, ErrCorruptedImage(err)
	}
	fi.fileID = string(name)

	// Padding field as per section 9.1.12 in ECMA-119
	if (drecord.FileIDLength % 2) == 0 {
		var zero byte
		if err := binary.Read(r.image, binary.BigEndian, &zero); err != nil {
			return len, ErrCorruptedImage(err)
		}
	}

	// System use field as per section 9.1.13 in ECMA-119
	// Directory record has 34 bytes in addition to the name's
	// variable length and the padding field mentioned in section 9.1.12
	totalLen := 34 + drecord.FileIDLength - (drecord.FileIDLength % 2)
	sysUseLen := int64(len - totalLen)
	if sysUseLen > 0 {
		sysData := make([]byte, sysUseLen)
		if err := binary.Read(r.image, binary.BigEndian, sysData); err != nil {
			return len, ErrCorruptedImage(err)
		}
	}
	return len, nil
}

// unpackPVD unpacks Primary Volume Descriptor in three phases. This is
// because the root directory record is a variable-length record and Go's binary
// package doesn't support unpacking variable-length structs easily.
func (r *Reader) unpackPVD() error {
	// Unpack first half
	var pvd1 PrimaryVolumePart1
	if err := binary.Read(r.image, binary.BigEndian, &pvd1); err != nil {
		return ErrCorruptedImage(err)
	}
	r.pvd.PrimaryVolumePart1 = pvd1

	// Unpack root directory record
	var drecord FileStat
	if _, err := r.unpackDRecord(&drecord); err != nil {
		return ErrCorruptedImage(err)
	}
	r.pvd.DirectoryRecord = drecord.DirectoryRecord
	r.queue.Enqueue(drecord)

	// Unpack second half
	var pvd2 PrimaryVolumePart2
	if err := binary.Read(r.image, binary.BigEndian, &pvd2); err != nil {
		return ErrCorruptedImage(err)
	}
	r.pvd.PrimaryVolumePart2 = pvd2

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package webhook

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/globalsign/mgo/bson"
	"github.com/pkg/errors"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/event"
	"github.com/tsuru/tsuru/log"
	tsuruNet "github.com/tsuru/tsuru/net"
	"github.com/tsuru/tsuru/storage"
	eventTypes "github.com/tsuru/tsuru/types/event"
)

var (
	_ eventTypes.WebHookService = &webHookService{}

	chanBufferSize   = 1000
	defaultUserAgent = "tsuru-webhook-client/1.0"
)

func WebHookService() (eventTypes.WebHookService, error) {
	dbDriver, err := storage.GetCurrentDbDriver()
	if err != nil {
		dbDriver, err = storage.GetDefaultDbDriver()
		if err != nil {
			return nil, err
		}
	}
	s := &webHookService{
		storage: dbDriver.WebHookStorage,
		evtCh:   make(chan string, chanBufferSize),
		quitCh:  make(chan struct{}),
		doneCh:  make(chan struct{}),
	}
	go s.run()
	shutdown.Register(s)
	return s, nil
}

type webHookService struct {
	storage eventTypes.WebHookStorage
	evtCh   chan string
	quitCh  chan struct{}
	doneCh  chan struct{}
}

func (s *webHookService) Shutdown(ctx context.Context) error {
	doneCtx := ctx.Done()
	close(s.quitCh)
	select {
	case <-s.doneCh:
	case <-doneCtx:
		return ctx.Err()
	}
	return nil
}

func (s *webHookService) Notify(evtID string) {
	select {
	case s.evtCh <- evtID:
	case <-s.quitCh:
	}
}

func (s *webHookService) run() {
	defer close(s.doneCh)
	for {
		select {
		case evtID := <-s.evtCh:
			err := s.handleEvent(evtID)
			if err != nil {
				log.Errorf("[webhooks] error handling webhooks for event %s", evtID)
			}
		case <-s.quitCh:
			return
		}
	}
}

func (s *webHookService) handleEvent(evtID string) error {
	evt, err := event.GetByID(bson.ObjectId(evtID))
	if err != nil {
		return err
	}
	filter := eventTypes.EventFilter{
		TargetTypes:  []string{string(evt.Target.Type)},
		TargetValues: []string{evt.Target.Value},
		KindTypes:    []string{string(evt.Kind.Type)},
		KindNames:    []string{evt.Kind.Name},
	}
	for _, t := range evt.ExtraTargets {
		filter.TargetTypes = append(filter.TargetTypes, string(t.Target.Type))
		filter.TargetValues = append(filter.TargetValues, t.Target.Value)
	}
	hooks, err := s.storage.FindByEvent(filter, evt.Error == "")
	if err != nil {
		return err
	}
	for _, h := range hooks {
		err = s.doHook(h)
		if err != nil {
			log.Errorf("[webhooks] error calling webhook %q: %v", h.Name, err)
		}
	}
	return nil
}

func (s *webHookService) doHook(hook eventTypes.WebHook) error {
	var body io.Reader
	if hook.Body != "" {
		body = strings.NewReader(hook.Body)
	}
	req, err := http.NewRequest(hook.Method, hook.URL.String(), body)
	if err != nil {
		return err
	}
	req.Header = hook.Headers
	if req.UserAgent() == "" {
		req.Header.Set("User-Agent", defaultUserAgent)
	}
	client := tsuruNet.Dial5Full60ClientNoKeepAlive
	if hook.Insecure {
		client = &tsuruNet.Dial5Full60ClientNoKeepAliveInsecure
	}
	rsp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer rsp.Body.Close()
	if rsp.StatusCode < 200 || rsp.StatusCode >= 400 {
		data, _ := ioutil.ReadAll(rsp.Body)
		return errors.Errorf("invalid status code calling hook: %d: %s", rsp.StatusCode, string(data))
	}
	return nil
}
<commit_msg>event/webhook: add prometheus metrics<commit_after>// Copyright 2018 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package webhook

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/globalsign/mgo/bson"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/event"
	"github.com/tsuru/tsuru/log"
	tsuruNet "github.com/tsuru/tsuru/net"
	"github.com/tsuru/tsuru/storage"
	eventTypes "github.com/tsuru/tsuru/types/event"
)

var (
	_ eventTypes.WebHookService = &webHookService{}

	chanBufferSize   = 1000
	defaultUserAgent = "tsuru-webhook-client/1.0"

	webhooksLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name: "tsuru_webhooks_latency_seconds",
		Help: "The latency for webhooks requests in seconds",
	})

	webhooksQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_webhooks_event_queue_current",
		Help: "The current number of queued events waiting for webhooks processing",
	})

	webhooksTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_webhooks_calls_total",
		Help: "The total number of webhooks calls",
	})

	webhooksError = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_webhooks_calls_error",
		Help: "The total number of webhooks calls with error",
	})
)

func WebHookService() (eventTypes.WebHookService, error) {
	dbDriver, err := storage.GetCurrentDbDriver()
	if err != nil {
		dbDriver, err = storage.GetDefaultDbDriver()
		if err != nil {
			return nil, err
		}
	}
	s := &webHookService{
		storage: dbDriver.WebHookStorage,
		evtCh:   make(chan string, chanBufferSize),
		quitCh:  make(chan struct{}),
		doneCh:  make(chan struct{}),
	}
	go s.run()
	shutdown.Register(s)
	return s, nil
}

type webHookService struct {
	storage eventTypes.WebHookStorage
	evtCh   chan string
	quitCh  chan struct{}
	doneCh  chan struct{}
}

func (s *webHookService) Shutdown(ctx context.Context) error {
	doneCtx := ctx.Done()
	close(s.quitCh)
	select {
	case <-s.doneCh:
	case <-doneCtx:
		return ctx.Err()
	}
	return nil
}

func (s *webHookService) Notify(evtID string) {
	select {
	case s.evtCh <- evtID:
	case <-s.quitCh:
	}
	webhooksQueue.Set(float64(len(s.evtCh)))
}

func (s *webHookService) run() {
	defer close(s.doneCh)
	for {
		select {
		case evtID := <-s.evtCh:
			webhooksQueue.Set(float64(len(s.evtCh)))
			err := s.handleEvent(evtID)
			if err != nil {
				log.Errorf("[webhooks] error handling webhooks for event %s", evtID)
			}
		case <-s.quitCh:
			return
		}
	}
}

func (s *webHookService) handleEvent(evtID string) error {
	evt, err := event.GetByID(bson.ObjectId(evtID))
	if err != nil {
		return err
	}
	filter := eventTypes.EventFilter{
		TargetTypes:  []string{string(evt.Target.Type)},
		TargetValues: []string{evt.Target.Value},
		KindTypes:    []string{string(evt.Kind.Type)},
		KindNames:    []string{evt.Kind.Name},
	}
	for _, t := range evt.ExtraTargets {
		filter.TargetTypes = append(filter.TargetTypes, string(t.Target.Type))
		filter.TargetValues = append(filter.TargetValues, t.Target.Value)
	}
	hooks, err := s.storage.FindByEvent(filter, evt.Error == "")
	if err != nil {
		return err
	}
	for _, h := range hooks {
		err = s.doHook(h)
		if err != nil {
			log.Errorf("[webhooks] error calling webhook %q: %v", h.Name, err)
		}
	}
	return nil
}

func (s *webHookService) doHook(hook eventTypes.WebHook) (err error) {
	defer func() {
		webhooksTotal.Inc()
		if err != nil {
			webhooksError.Inc()
		}
	}()
	var body io.Reader
	if hook.Body != "" {
		body = strings.NewReader(hook.Body)
	}
	req, err := http.NewRequest(hook.Method, hook.URL.String(), body)
	if err != nil {
		return err
	}
	req.Header = hook.Headers
	if req.UserAgent() == "" {
		req.Header.Set("User-Agent", defaultUserAgent)
	}
	client := tsuruNet.Dial5Full60ClientNoKeepAlive
	if hook.Insecure {
		client = &tsuruNet.Dial5Full60ClientNoKeepAliveInsecure
	}
	reqStart := time.Now()
	rsp, err := client.Do(req)
	webhooksLatency.Observe(time.Since(reqStart).Seconds())
	if err != nil {
		return err
	}
	defer rsp.Body.Close()
	if rsp.StatusCode < 200 || rsp.StatusCode >= 400 {
		data, _ := ioutil.ReadAll(rsp.Body)
		return errors.Errorf("invalid status code calling hook: %d: %s", rsp.StatusCode, string(data))
	}
	return nil
}
<|endoftext|>


<commit_before>package snappystream

import (
	"bytes"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"

	"code.google.com/p/snappy-go/snappy"
)

// errMssingStreamID is returned from a reader when the source stream does not
// begin with a stream identifier block (4.1 Stream identifier).  Its occurance
// signifies that the source byte stream is not snappy framed.
var errMissingStreamID = fmt.Errorf("missing stream identifier")

type reader struct {
	reader io.Reader

	err error

	seenStreamID   bool
	verifyChecksum bool

	buf bytes.Buffer
	hdr []byte
	src []byte
	dst []byte
}

// NewReader returns an io.Reader interface to the snappy framed stream format.
//
// It transparently handles reading the stream identifier (but does not proxy this
// to the caller), decompresses blocks, and (optionally) validates checksums.
//
// Internally, three buffers are maintained.  The first two are for reading
// off the wrapped io.Reader and for holding the decompressed block (both are grown
// automatically and re-used and will never exceed the largest block size, 65536). The
// last buffer contains the *unread* decompressed bytes (and can grow indefinitely).
//
// The second param determines whether or not the reader will verify block
// checksums and can be enabled/disabled with the constants VerifyChecksum and SkipVerifyChecksum
//
// For each Read, the returned length will be up to the lesser of len(b) or 65536
// decompressed bytes, regardless of the length of *compressed* bytes read
// from the wrapped io.Reader.
func NewReader(r io.Reader, verifyChecksum bool) io.Reader {
	return &reader{
		reader: r,

		verifyChecksum: verifyChecksum,

		hdr: make([]byte, 4),
		src: make([]byte, 4096),
		dst: make([]byte, 4096),
	}
}

// WriteTo implements the io.WriterTo interface used by io.Copy.  It writes
// decoded data from the underlying reader to w.  WriteTo returns the number of
// bytes written along with any error encountered.
func (r *reader) WriteTo(w io.Writer) (int64, error) {
	n, err := r.buf.WriteTo(w)
	if err != nil {
		return n, err
	}
	for {
		var m int
		m, err = r.nextFrame(w)
		n += int64(m)
		if err != nil {
			break
		}
	}
	if err == io.EOF {
		err = nil
	}
	return n, err
}

func (r *reader) read(b []byte) (int, error) {
	n, err := r.buf.Read(b)
	r.err = err
	return n, err
}

func (r *reader) Read(b []byte) (int, error) {
	if r.err != nil {
		return 0, r.err
	}

	if r.buf.Len() < len(b) {
		_, r.err = r.nextFrame(&r.buf)
		if r.err == io.EOF {
			// fill b with any remaining bytes in the buffer.
			return r.read(b)
		}
		if r.err != nil {
			return 0, r.err
		}
	}

	return r.read(b)
}

func (r *reader) nextFrame(w io.Writer) (int, error) {
	for {
		// read the 4-byte snappy frame header
		_, err := io.ReadFull(r.reader, r.hdr)
		if err != nil {
			return 0, err
		}

		// a stream identifier may appear anywhere and contains no information.
		// it must appear at the beginning of the stream.  when found, validate
		// it and continue to the next block.
		if r.hdr[0] == blockStreamIdentifier {
			err := r.readStreamID()
			if err != nil {
				return 0, err
			}
			r.seenStreamID = true
			continue
		}
		if !r.seenStreamID {
			return 0, errMissingStreamID
		}

		switch typ := r.hdr[0]; {
		case typ == blockCompressed || typ == blockUncompressed:
			return r.decodeBlock(w)
		case typ == blockPadding || (0x80 <= typ && typ <= 0xfd):
			// skip blocks whose data must not be inspected (4.4 Padding, and 4.6
			// Reserved skippable chunks).
			err := r.discardBlock()
			if err != nil {
				return 0, err
			}
			continue
		default:
			// typ must be unskippable range 0x02-0x7f.  Read the block in full
			// and return an error (4.5 Reserved unskippable chunks).
			err = r.discardBlock()
			if err != nil {
				return 0, err
			}
			return 0, fmt.Errorf("unrecognized unskippable frame %#x", r.hdr[0])
		}
	}
	return nil
}

// decodeDataBlock assumes r.hdr[0] to be either blockCompressed or
// blockUncompressed.
func (r *reader) decodeBlock(w io.Writer) (int, error) {
	// read compressed block data and determine if uncompressed data is too
	// large.
	buf, err := r.readBlock()
	if err != nil {
		return 0, err
	}
	declen := len(buf[4:])
	if r.hdr[0] == blockCompressed {
		declen, err = snappy.DecodedLen(buf[4:])
		if err != nil {
			return 0, err
		}
	}
	if declen > MaxBlockSize {
		return 0, fmt.Errorf("decoded block data too large %d > %d", declen, MaxBlockSize)
	}

	// decode data and verify its integrity using the little-endian crc32
	// preceding encoded data
	crc32le, blockdata := buf[:4], buf[4:]
	if r.hdr[0] == blockCompressed {
		r.dst, err = snappy.Decode(r.dst, blockdata)
		if err != nil {
			return 0, err
		}
		blockdata = r.dst
	}
	if r.verifyChecksum {
		checksum := unmaskChecksum(uint32(crc32le[0]) | uint32(crc32le[1])<<8 | uint32(crc32le[2])<<16 | uint32(crc32le[3])<<24)
		actualChecksum := crc32.Checksum(blockdata, crcTable)
		if checksum != actualChecksum {
			return 0, fmt.Errorf("checksum does not match %x != %x", checksum, actualChecksum)
		}
	}
	return w.Write(blockdata)
}

func (r *reader) readStreamID() error {
	// the length of the block is fixed so don't decode it from the header.
	if !bytes.Equal(r.hdr, streamID[:4]) {
		return fmt.Errorf("invalid stream identifier length")
	}

	// read the identifier block data "sNaPpY"
	block := r.src[:6]
	_, err := noeof(io.ReadFull(r.reader, block))
	if err != nil {
		return err
	}
	if !bytes.Equal(block, streamID[4:]) {
		return fmt.Errorf("invalid stream identifier block")
	}
	return nil
}

func (r *reader) discardBlock() error {
	length := uint64(decodeLength(r.hdr[1:]))
	_, err := noeof64(io.CopyN(ioutil.Discard, r.reader, int64(length)))
	return err
}

func (r *reader) readBlock() ([]byte, error) {
	// check bounds on encoded length (+4 for checksum)
	length := decodeLength(r.hdr[1:])
	if length > (maxEncodedBlockSize + 4) {
		return nil, fmt.Errorf("encoded block data too large %d > %d", length, (maxEncodedBlockSize + 4))
	}

	if int(length) > len(r.src) {
		r.src = make([]byte, length)
	}

	buf := r.src[:length]
	_, err := noeof(io.ReadFull(r.reader, buf))
	if err != nil {
		return nil, err
	}

	return buf, nil
}

// decodeLength decodes a 24-bit (3-byte) little-endian length from b.
func decodeLength(b []byte) uint32 {
	return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16
}

func unmaskChecksum(c uint32) uint32 {
	x := c - 0xa282ead8
	return ((x >> 17) | (x << 15))
}

// noeof is used after reads in situations where EOF signifies invalid
// formatting or corruption.
func noeof(n int, err error) (int, error) {
	if err == io.EOF {
		return n, io.ErrUnexpectedEOF
	}
	return n, err
}

// noeof64 is used after long reads (e.g. io.Copy) in situations where io.EOF
// signifies invalid formatting or corruption.
func noeof64(n int64, err error) (int64, error) {
	if err == io.EOF {
		return n, io.ErrUnexpectedEOF
	}
	return n, err
}
<commit_msg>fix build error: panic instead of (unreachable) return<commit_after>package snappystream

import (
	"bytes"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"

	"code.google.com/p/snappy-go/snappy"
)

// errMssingStreamID is returned from a reader when the source stream does not
// begin with a stream identifier block (4.1 Stream identifier).  Its occurance
// signifies that the source byte stream is not snappy framed.
var errMissingStreamID = fmt.Errorf("missing stream identifier")

type reader struct {
	reader io.Reader

	err error

	seenStreamID   bool
	verifyChecksum bool

	buf bytes.Buffer
	hdr []byte
	src []byte
	dst []byte
}

// NewReader returns an io.Reader interface to the snappy framed stream format.
//
// It transparently handles reading the stream identifier (but does not proxy this
// to the caller), decompresses blocks, and (optionally) validates checksums.
//
// Internally, three buffers are maintained.  The first two are for reading
// off the wrapped io.Reader and for holding the decompressed block (both are grown
// automatically and re-used and will never exceed the largest block size, 65536). The
// last buffer contains the *unread* decompressed bytes (and can grow indefinitely).
//
// The second param determines whether or not the reader will verify block
// checksums and can be enabled/disabled with the constants VerifyChecksum and SkipVerifyChecksum
//
// For each Read, the returned length will be up to the lesser of len(b) or 65536
// decompressed bytes, regardless of the length of *compressed* bytes read
// from the wrapped io.Reader.
func NewReader(r io.Reader, verifyChecksum bool) io.Reader {
	return &reader{
		reader: r,

		verifyChecksum: verifyChecksum,

		hdr: make([]byte, 4),
		src: make([]byte, 4096),
		dst: make([]byte, 4096),
	}
}

// WriteTo implements the io.WriterTo interface used by io.Copy.  It writes
// decoded data from the underlying reader to w.  WriteTo returns the number of
// bytes written along with any error encountered.
func (r *reader) WriteTo(w io.Writer) (int64, error) {
	n, err := r.buf.WriteTo(w)
	if err != nil {
		return n, err
	}
	for {
		var m int
		m, err = r.nextFrame(w)
		n += int64(m)
		if err != nil {
			break
		}
	}
	if err == io.EOF {
		err = nil
	}
	return n, err
}

func (r *reader) read(b []byte) (int, error) {
	n, err := r.buf.Read(b)
	r.err = err
	return n, err
}

func (r *reader) Read(b []byte) (int, error) {
	if r.err != nil {
		return 0, r.err
	}

	if r.buf.Len() < len(b) {
		_, r.err = r.nextFrame(&r.buf)
		if r.err == io.EOF {
			// fill b with any remaining bytes in the buffer.
			return r.read(b)
		}
		if r.err != nil {
			return 0, r.err
		}
	}

	return r.read(b)
}

func (r *reader) nextFrame(w io.Writer) (int, error) {
	for {
		// read the 4-byte snappy frame header
		_, err := io.ReadFull(r.reader, r.hdr)
		if err != nil {
			return 0, err
		}

		// a stream identifier may appear anywhere and contains no information.
		// it must appear at the beginning of the stream.  when found, validate
		// it and continue to the next block.
		if r.hdr[0] == blockStreamIdentifier {
			err := r.readStreamID()
			if err != nil {
				return 0, err
			}
			r.seenStreamID = true
			continue
		}
		if !r.seenStreamID {
			return 0, errMissingStreamID
		}

		switch typ := r.hdr[0]; {
		case typ == blockCompressed || typ == blockUncompressed:
			return r.decodeBlock(w)
		case typ == blockPadding || (0x80 <= typ && typ <= 0xfd):
			// skip blocks whose data must not be inspected (4.4 Padding, and 4.6
			// Reserved skippable chunks).
			err := r.discardBlock()
			if err != nil {
				return 0, err
			}
			continue
		default:
			// typ must be unskippable range 0x02-0x7f.  Read the block in full
			// and return an error (4.5 Reserved unskippable chunks).
			err = r.discardBlock()
			if err != nil {
				return 0, err
			}
			return 0, fmt.Errorf("unrecognized unskippable frame %#x", r.hdr[0])
		}
	}
	panic("unreachable")
}

// decodeDataBlock assumes r.hdr[0] to be either blockCompressed or
// blockUncompressed.
func (r *reader) decodeBlock(w io.Writer) (int, error) {
	// read compressed block data and determine if uncompressed data is too
	// large.
	buf, err := r.readBlock()
	if err != nil {
		return 0, err
	}
	declen := len(buf[4:])
	if r.hdr[0] == blockCompressed {
		declen, err = snappy.DecodedLen(buf[4:])
		if err != nil {
			return 0, err
		}
	}
	if declen > MaxBlockSize {
		return 0, fmt.Errorf("decoded block data too large %d > %d", declen, MaxBlockSize)
	}

	// decode data and verify its integrity using the little-endian crc32
	// preceding encoded data
	crc32le, blockdata := buf[:4], buf[4:]
	if r.hdr[0] == blockCompressed {
		r.dst, err = snappy.Decode(r.dst, blockdata)
		if err != nil {
			return 0, err
		}
		blockdata = r.dst
	}
	if r.verifyChecksum {
		checksum := unmaskChecksum(uint32(crc32le[0]) | uint32(crc32le[1])<<8 | uint32(crc32le[2])<<16 | uint32(crc32le[3])<<24)
		actualChecksum := crc32.Checksum(blockdata, crcTable)
		if checksum != actualChecksum {
			return 0, fmt.Errorf("checksum does not match %x != %x", checksum, actualChecksum)
		}
	}
	return w.Write(blockdata)
}

func (r *reader) readStreamID() error {
	// the length of the block is fixed so don't decode it from the header.
	if !bytes.Equal(r.hdr, streamID[:4]) {
		return fmt.Errorf("invalid stream identifier length")
	}

	// read the identifier block data "sNaPpY"
	block := r.src[:6]
	_, err := noeof(io.ReadFull(r.reader, block))
	if err != nil {
		return err
	}
	if !bytes.Equal(block, streamID[4:]) {
		return fmt.Errorf("invalid stream identifier block")
	}
	return nil
}

func (r *reader) discardBlock() error {
	length := uint64(decodeLength(r.hdr[1:]))
	_, err := noeof64(io.CopyN(ioutil.Discard, r.reader, int64(length)))
	return err
}

func (r *reader) readBlock() ([]byte, error) {
	// check bounds on encoded length (+4 for checksum)
	length := decodeLength(r.hdr[1:])
	if length > (maxEncodedBlockSize + 4) {
		return nil, fmt.Errorf("encoded block data too large %d > %d", length, (maxEncodedBlockSize + 4))
	}

	if int(length) > len(r.src) {
		r.src = make([]byte, length)
	}

	buf := r.src[:length]
	_, err := noeof(io.ReadFull(r.reader, buf))
	if err != nil {
		return nil, err
	}

	return buf, nil
}

// decodeLength decodes a 24-bit (3-byte) little-endian length from b.
func decodeLength(b []byte) uint32 {
	return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16
}

func unmaskChecksum(c uint32) uint32 {
	x := c - 0xa282ead8
	return ((x >> 17) | (x << 15))
}

// noeof is used after reads in situations where EOF signifies invalid
// formatting or corruption.
func noeof(n int, err error) (int, error) {
	if err == io.EOF {
		return n, io.ErrUnexpectedEOF
	}
	return n, err
}

// noeof64 is used after long reads (e.g. io.Copy) in situations where io.EOF
// signifies invalid formatting or corruption.
func noeof64(n int64, err error) (int64, error) {
	if err == io.EOF {
		return n, io.ErrUnexpectedEOF
	}
	return n, err
}
<|endoftext|>


<commit_before>package JsonConfigReader

import (
	"io"
)

type state struct {
	escapeC   int
	inString  bool
	inComment bool
	r         io.Reader
}

// Read acts as a proxy for the underlying reader and cleans p
// of comments and trailing commas preceeding ] and }
// comments are delimitted by // up until the end the line
func (s *state) Read(p []byte) (n int, err error) {
	n, err = s.r.Read(p)
	if err == nil {
		lpcOff := -1
		var lc, lpc byte
		for i, c := range p[0:n] {
			if s.inString {
				if s.escapeC == 0 {
					if c == '"' {
						s.inString = false
					} else if c == '\\' {
						s.escapeC += 1
					}
				} else {
					s.escapeC = 0
				}
			} else {
				if s.inComment {
					if c == '\n' || c == '\r' {
						s.inComment = false
					}
				} else {
					if lpc == ',' && (c == '}' || c == ']') {
						p[lpcOff] = ' '
					}
					if c != ' ' && c != '\t' && c != '\n' && c != '\r' && c != '/' {
						lpc = c
						lpcOff = i
					}

					if c == '"' {
						s.inString = true
					} else if c == '/' && lc == '/' {
						// don't clear a single /, let a syntax error occur
						p[i-1] = ' '
						s.inComment = true
					}
				}

				if s.inComment {
					p[i] = ' '
				}
			}
			lc = c
		}
	}
	return
}

// New returns an io.Reader acting as proxy to r
func New(r io.Reader) io.Reader {
	return &state{r: r}
}
<commit_msg>- rewrite<commit_after>package JsonConfigReader

import (
	"bytes"
	"io"
)

type state struct {
	r  io.Reader
	br *bytes.Reader
}

func isNL(c byte) bool {
	return c == '\n' || c == '\r'
}

func isWS(c byte) bool {
	return c == ' ' || c == '\t' || isNL(c)
}

func consumeComment(s []byte, i int) int {
	if i < len(s) && s[i] == '/' {
		s[i-1] = ' '
		for ; i < len(s) && !isNL(s[i]); i += 1 {
			s[i] = ' '
		}
	}
	return i
}

func prep(r io.Reader) (s []byte, err error) {
	buf := &bytes.Buffer{}
	_, err = io.Copy(buf, r)
	s = buf.Bytes()
	if err != nil {
		return
	}

	i := 0
	for i < len(s) {
		switch s[i] {
		case '"':
			i += 1
			for i < len(s) {
				if s[i] == '"' {
					i += 1
					break
				} else if s[i] == '\\' {
					i += 1
				}
				i += 1
			}
		case '/':
			i = consumeComment(s, i+1)
		case ',':
			j := i
			for {
				i += 1
				if i >= len(s) {
					break
				} else if s[i] == '}' || s[i] == ']' {
					s[j] = ' '
					break
				} else if s[i] == '/' {
					i = consumeComment(s, i+1)
				} else if !isWS(s[i]) {
					break
				}
			}
		default:
			i += 1
		}
	}
	return
}

// Read acts as a proxy for the underlying reader and cleans p
// of comments and trailing commas preceeding ] and }
// comments are delimitted by // up until the end the line
func (st *state) Read(p []byte) (n int, err error) {
	if st.br == nil {
		var s []byte
		if s, err = prep(st.r); err != nil {
			return
		}
		st.br = bytes.NewReader(s)
	}
	return st.br.Read(p)
}

// New returns an io.Reader acting as proxy to r
func New(r io.Reader) io.Reader {
	return &state{r: r}
}
<|endoftext|>


<commit_before>/* Package acm is an ACM-ICPC problems' gallery.
Each problem in putted in one separate file.
Running and the expect result is putted in *_test.go file. */
package acm
<commit_msg>little<commit_after>/*
Package acm is an ACM-ICPC problems' gallery.

Each problem in putted in one separate file.
Running and the expect result is putted in *_test.go file.
*/
package acm
<|endoftext|>


<commit_before>// Copyright 2016 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"image/color"
	"log"
	"math"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/ebitenutil"
	"github.com/hajimehoshi/ebiten/examples/common"
)

const (
	screenWidth  = 320
	screenHeight = 240
	maxAngle     = 256
	maxLean      = 16
)

var (
	skyColor  = color.RGBA{0x66, 0xcc, 0xff, 0xff}
	thePlayer = &player{
		x16:   16 * 100,
		y16:   16 * 200,
		angle: maxAngle * 3 / 4,
	}
	gophersImage *ebiten.Image
	groundImage  *ebiten.Image
)

type player struct {
	x16   int
	y16   int
	angle int
	lean  int
}

func round(x float64) float64 {
	return math.Floor(x + 0.5)
}

func (p *player) MoveForward() {
	w, h := gophersImage.Size()
	mx := w * 16
	my := h * 16
	s, c := math.Sincos(float64(p.angle) * 2 * math.Pi / maxAngle)
	p.x16 += int(round(16 * c))
	p.y16 += int(round(16 * s))
	for mx <= p.x16 {
		p.x16 -= mx
	}
	for my <= p.y16 {
		p.y16 -= my
	}
	for p.x16 < 0 {
		p.x16 += mx
	}
	for p.y16 < 0 {
		p.y16 += my
	}
}

func (p *player) RotateRight() {
	p.angle++
	if maxAngle <= p.angle {
		p.angle -= maxAngle
	}
	p.lean++
	if maxLean < p.lean {
		p.lean = maxLean
	}
}

func (p *player) RotateLeft() {
	p.angle--
	if p.angle < 0 {
		p.angle += maxAngle
	}
	p.lean--
	if p.lean < -maxLean {
		p.lean = -maxLean
	}
}

func (p *player) Stabilize() {
	if 0 < p.lean {
		p.lean--
	}
	if p.lean < 0 {
		p.lean++
	}
}

func (p *player) Position() (int, int) {
	return p.x16, p.y16
}

func (p *player) Angle() int {
	return p.angle
}

func updateGroundImage(ground *ebiten.Image) error {
	x16, y16 := thePlayer.Position()
	a := thePlayer.Angle()
	gw, gh := ground.Size()
	w, h := gophersImage.Size()
	for j := -1; j <= 1; j++ {
		for i := -1; i <= 1; i++ {
			op := &ebiten.DrawImageOptions{}
			op.GeoM.Translate(float64(-x16)/16, float64(-y16)/16)
			op.GeoM.Translate(float64(w*i), float64(h*j))
			op.GeoM.Rotate(float64(-a)*2*math.Pi/maxAngle + math.Pi*3.0/2.0)
			op.GeoM.Translate(float64(gw)/2, float64(gh))
			if err := ground.DrawImage(gophersImage, op); err != nil {
				return err
			}
		}
	}
	return nil
}

type groundParts struct {
	image *ebiten.Image
}

func (g *groundParts) Len() int {
	_, h := g.image.Size()
	return h
}

func (g *groundParts) Src(i int) (int, int, int, int) {
	w, _ := g.image.Size()
	return 0, i, w, i + 1
}

func (g *groundParts) Dst(i int) (int, int, int, int) {
	w, _ := g.image.Size()
	r := i * 2
	j := 1.0
	for idx := 0; idx < i; idx++ {
		j += float64(idx) / 40
	}
	return -r, int(j), w + r, int(float64(j)+float64(i)/40) + 1
}

func drawGroundImage(screen *ebiten.Image, ground *ebiten.Image) error {
	w, _ := ground.Size()
	op := &ebiten.DrawImageOptions{}
	op.GeoM.Translate(-float64(w)/2, 0)
	op.GeoM.Rotate(-1 * float64(thePlayer.lean) / maxLean * math.Pi / 8)
	op.GeoM.Translate(float64(w)/2, 0)
	op.GeoM.Translate(float64(screenWidth-w)/2, screenHeight/3)
	op.ImageParts = &groundParts{ground}
	if err := screen.DrawImage(ground, op); err != nil {
		return err
	}
	return nil
}

func update(screen *ebiten.Image) error {
	if ebiten.IsKeyPressed(ebiten.KeySpace) {
		thePlayer.MoveForward()
	}
	rotated := false
	if ebiten.IsKeyPressed(ebiten.KeyRight) {
		thePlayer.RotateRight()
		rotated = true
	}
	if ebiten.IsKeyPressed(ebiten.KeyLeft) {
		thePlayer.RotateLeft()
		rotated = true
	}
	if !rotated {
		thePlayer.Stabilize()
	}

	if err := screen.Fill(skyColor); err != nil {
		return err
	}
	if err := updateGroundImage(groundImage); err != nil {
		return err
	}
	if err := drawGroundImage(screen, groundImage); err != nil {
		return err
	}

	tutrial := "Space: Move foward\nLeft/Right: Rotate"
	msg := fmt.Sprintf("FPS: %0.2f\n%s", ebiten.CurrentFPS(), tutrial)
	if err := ebitenutil.DebugPrint(screen, msg); err != nil {
		return err
	}
	return nil
}

func main() {
	var err error
	gophersImage, _, err = common.AssetImage("gophers.jpg", ebiten.FilterNearest)
	if err != nil {
		log.Fatal(err)
	}
	groundImage, err = ebiten.NewImage(screenWidth+70, screenHeight*2/3+50, ebiten.FilterNearest)
	if err != nil {
		log.Fatal(err)
	}
	if err := ebiten.Run(update, screenWidth, screenHeight, 2, "Air Ship (Ebiten Demo)"); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>examples/airship: Better scaling<commit_after>// Copyright 2016 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"image/color"
	"log"
	"math"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/ebitenutil"
	"github.com/hajimehoshi/ebiten/examples/common"
)

const (
	screenWidth  = 320
	screenHeight = 240
	maxAngle     = 256
	maxLean      = 16
)

var (
	skyColor  = color.RGBA{0x66, 0xcc, 0xff, 0xff}
	thePlayer = &player{
		x16:   16 * 100,
		y16:   16 * 200,
		angle: maxAngle * 3 / 4,
	}
	gophersImage *ebiten.Image
	groundImage  *ebiten.Image
)

type player struct {
	x16   int
	y16   int
	angle int
	lean  int
}

func round(x float64) float64 {
	return math.Floor(x + 0.5)
}

func (p *player) MoveForward() {
	w, h := gophersImage.Size()
	mx := w * 16
	my := h * 16
	s, c := math.Sincos(float64(p.angle) * 2 * math.Pi / maxAngle)
	p.x16 += int(round(16*c) * 2)
	p.y16 += int(round(16*s) * 2)
	for mx <= p.x16 {
		p.x16 -= mx
	}
	for my <= p.y16 {
		p.y16 -= my
	}
	for p.x16 < 0 {
		p.x16 += mx
	}
	for p.y16 < 0 {
		p.y16 += my
	}
}

func (p *player) RotateRight() {
	p.angle++
	if maxAngle <= p.angle {
		p.angle -= maxAngle
	}
	p.lean++
	if maxLean < p.lean {
		p.lean = maxLean
	}
}

func (p *player) RotateLeft() {
	p.angle--
	if p.angle < 0 {
		p.angle += maxAngle
	}
	p.lean--
	if p.lean < -maxLean {
		p.lean = -maxLean
	}
}

func (p *player) Stabilize() {
	if 0 < p.lean {
		p.lean--
	}
	if p.lean < 0 {
		p.lean++
	}
}

func (p *player) Position() (int, int) {
	return p.x16, p.y16
}

func (p *player) Angle() int {
	return p.angle
}

func updateGroundImage(ground *ebiten.Image) error {
	if err := ground.Clear(); err != nil {
		return err
	}
	x16, y16 := thePlayer.Position()
	a := thePlayer.Angle()
	gw, gh := ground.Size()
	w, h := gophersImage.Size()
	for j := -2; j <= 2; j++ {
		for i := -2; i <= 2; i++ {
			op := &ebiten.DrawImageOptions{}
			op.GeoM.Translate(float64(-x16)/16, float64(-y16)/16)
			op.GeoM.Translate(float64(w*i), float64(h*j))
			op.GeoM.Rotate(float64(-a)*2*math.Pi/maxAngle + math.Pi*3.0/2.0)
			op.GeoM.Translate(float64(gw)/2, float64(gh)-32)
			if err := ground.DrawImage(gophersImage, op); err != nil {
				return err
			}
		}
	}
	return nil
}

type groundParts struct {
	image *ebiten.Image
}

func (g *groundParts) Len() int {
	_, h := g.image.Size()
	return h
}

func (g *groundParts) Src(i int) (int, int, int, int) {
	w, _ := g.image.Size()
	return 0, i, w, i + 1
}

func (g *groundParts) scaleForLine(x float64) float64 {
	_, h := g.image.Size()
	x = float64(h) - x
	return 200*((-x+50)/(x+50)+1) - 200*((-float64(h)+50)/(float64(h)+50)+1)
}

func (g *groundParts) Dst(i int) (int, int, int, int) {
	w, _ := g.image.Size()
	r := g.scaleForLine(float64(i))
	j1 := g.scaleForLine(float64(i))
	j2 := g.scaleForLine(float64(i + 1))
	return -int(r), int(j1), w + int(r), int(math.Ceil(j2))
}

func drawGroundImage(screen *ebiten.Image, ground *ebiten.Image) error {
	w, _ := ground.Size()
	op := &ebiten.DrawImageOptions{}
	op.GeoM.Translate(-float64(w)/2, 0)
	op.GeoM.Rotate(-1 * float64(thePlayer.lean) / maxLean * math.Pi / 8)
	op.GeoM.Translate(float64(w)/2, 0)
	op.GeoM.Translate(float64(screenWidth-w)/2, screenHeight/3)
	op.ImageParts = &groundParts{ground}
	if err := screen.DrawImage(ground, op); err != nil {
		return err
	}
	return nil
}

func update(screen *ebiten.Image) error {
	if ebiten.IsKeyPressed(ebiten.KeySpace) {
		thePlayer.MoveForward()
	}
	rotated := false
	if ebiten.IsKeyPressed(ebiten.KeyRight) {
		thePlayer.RotateRight()
		rotated = true
	}
	if ebiten.IsKeyPressed(ebiten.KeyLeft) {
		thePlayer.RotateLeft()
		rotated = true
	}
	if !rotated {
		thePlayer.Stabilize()
	}

	if err := screen.Fill(skyColor); err != nil {
		return err
	}
	if err := updateGroundImage(groundImage); err != nil {
		return err
	}
	if err := drawGroundImage(screen, groundImage); err != nil {
		return err
	}

	tutrial := "Space: Move foward\nLeft/Right: Rotate"
	msg := fmt.Sprintf("FPS: %0.2f\n%s", ebiten.CurrentFPS(), tutrial)
	if err := ebitenutil.DebugPrint(screen, msg); err != nil {
		return err
	}
	return nil
}

func main() {
	var err error
	gophersImage, _, err = common.AssetImage("gophers.jpg", ebiten.FilterNearest)
	if err != nil {
		log.Fatal(err)
	}
	groundImage, err = ebiten.NewImage(screenWidth+70, screenHeight*2/3+50, ebiten.FilterNearest)
	if err != nil {
		log.Fatal(err)
	}
	if err := ebiten.Run(update, screenWidth, screenHeight, 2, "Air Ship (Ebiten Demo)"); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/configservice"
)

func resourceAwsConfigConfigRule() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsConfigConfigRulePut,
		Read:   resourceAwsConfigConfigRuleRead,
		Update: resourceAwsConfigConfigRulePut,
		Delete: resourceAwsConfigConfigRuleDelete,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:         schema.TypeString,
				Required:     true,
				ValidateFunc: validateMaxLength(64),
			},
			"rule_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"description": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateMaxLength(256),
			},
			"input_parameters": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateJsonString,
			},
			"maximum_execution_frequency": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateConfigExecutionFrequency,
			},
			"scope": {
				Type:     schema.TypeList,
				MaxItems: 1,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"compliance_resource_id": {
							Type:         schema.TypeString,
							Optional:     true,
							ValidateFunc: validateMaxLength(256),
						},
						"compliance_resource_types": {
							Type:     schema.TypeSet,
							Optional: true,
							MaxItems: 100,
							Elem: &schema.Schema{
								Type:         schema.TypeString,
								ValidateFunc: validateMaxLength(256),
							},
							Set: schema.HashString,
						},
						"tag_key": {
							Type:         schema.TypeString,
							Optional:     true,
							ValidateFunc: validateMaxLength(128),
						},
						"tag_value": {
							Type:         schema.TypeString,
							Optional:     true,
							ValidateFunc: validateMaxLength(256),
						},
					},
				},
			},
			"source": {
				Type:     schema.TypeList,
				MaxItems: 1,
				Required: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"owner": {
							Type:         schema.TypeString,
							Required:     true,
							ValidateFunc: validateConfigRuleSourceOwner,
						},
						"source_detail": {
							Type:     schema.TypeSet,
							Set:      configRuleSourceDetailsHash,
							Optional: true,
							MaxItems: 25,
							Elem: &schema.Resource{
								Schema: map[string]*schema.Schema{
									"event_source": {
										Type:     schema.TypeString,
										Optional: true,
										Default:  "aws.config",
									},
									"maximum_execution_frequency": {
										Type:         schema.TypeString,
										Optional:     true,
										ValidateFunc: validateConfigExecutionFrequency,
									},
									"message_type": {
										Type:     schema.TypeString,
										Optional: true,
									},
								},
							},
						},
						"source_identifier": {
							Type:         schema.TypeString,
							Required:     true,
							ValidateFunc: validateMaxLength(256),
						},
					},
				},
			},
		},
	}
}

func resourceAwsConfigConfigRulePut(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).configconn

	name := d.Get("name").(string)
	ruleInput := configservice.ConfigRule{
		ConfigRuleName: aws.String(name),
		Source:         expandConfigRuleSource(d.Get("source").([]interface{})),
	}

	scopes := d.Get("scope").([]interface{})
	if len(scopes) > 0 {
		ruleInput.Scope = expandConfigRuleScope(scopes[0].(map[string]interface{}))
	}

	if v, ok := d.GetOk("description"); ok {
		ruleInput.Description = aws.String(v.(string))
	}
	if v, ok := d.GetOk("input_parameters"); ok {
		ruleInput.InputParameters = aws.String(v.(string))
	}
	if v, ok := d.GetOk("maximum_execution_frequency"); ok {
		ruleInput.MaximumExecutionFrequency = aws.String(v.(string))
	}

	input := configservice.PutConfigRuleInput{
		ConfigRule: &ruleInput,
	}
	log.Printf("[DEBUG] Creating AWSConfig config rule: %s", input)
	err := resource.Retry(2*time.Minute, func() *resource.RetryError {
		_, err := conn.PutConfigRule(&input)
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				if awsErr.Code() == "InsufficientPermissionsException" {
					// IAM is eventually consistent
					return resource.RetryableError(err)
				}
			}

			return resource.NonRetryableError(fmt.Errorf("Failed to create AWSConfig rule: %s", err))
		}

		return nil
	})
	if err != nil {
		return err
	}

	d.SetId(name)

	log.Printf("[DEBUG] AWSConfig config rule %q created", name)

	return resourceAwsConfigConfigRuleRead(d, meta)
}

func resourceAwsConfigConfigRuleRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).configconn

	out, err := conn.DescribeConfigRules(&configservice.DescribeConfigRulesInput{
		ConfigRuleNames: []*string{aws.String(d.Id())},
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoSuchConfigRuleException" {
			log.Printf("[WARN] Config Rule %q is gone (NoSuchConfigRuleException)", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	numberOfRules := len(out.ConfigRules)
	if numberOfRules < 1 {
		log.Printf("[WARN] Config Rule %q is gone (no rules found)", d.Id())
		d.SetId("")
		return nil
	}

	if numberOfRules > 1 {
		return fmt.Errorf("Expected exactly 1 Config Rule, received %d: %#v",
			numberOfRules, out.ConfigRules)
	}

	log.Printf("[DEBUG] AWS Config config rule received: %s", out)

	rule := out.ConfigRules[0]
	d.Set("arn", rule.ConfigRuleArn)
	d.Set("rule_id", rule.ConfigRuleId)
	d.Set("name", rule.ConfigRuleName)
	d.Set("description", rule.Description)
	d.Set("input_parameters", rule.InputParameters)
	d.Set("maximum_execution_frequency", rule.MaximumExecutionFrequency)

	if rule.Scope != nil {
		d.Set("scope", flattenConfigRuleScope(rule.Scope))
	}

	d.Set("source", flattenConfigRuleSource(rule.Source))

	return nil
}

func resourceAwsConfigConfigRuleDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).configconn

	name := d.Get("name").(string)

	log.Printf("[DEBUG] Deleting AWS Config config rule %q", name)
	_, err := conn.DeleteConfigRule(&configservice.DeleteConfigRuleInput{
		ConfigRuleName: aws.String(name),
	})
	if err != nil {
		return fmt.Errorf("Deleting Config Rule failed: %s", err)
	}

	conf := resource.StateChangeConf{
		Pending: []string{
			configservice.ConfigRuleStateActive,
			configservice.ConfigRuleStateDeleting,
			configservice.ConfigRuleStateDeletingResults,
			configservice.ConfigRuleStateEvaluating,
		},
		Target:  []string{""},
		Timeout: 5 * time.Minute,
		Refresh: func() (interface{}, string, error) {
			out, err := conn.DescribeConfigRules(&configservice.DescribeConfigRulesInput{
				ConfigRuleNames: []*string{aws.String(d.Id())},
			})
			if err != nil {
				if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoSuchConfigRuleException" {
					return 42, "", nil
				}
				return 42, "", fmt.Errorf("Failed to describe config rule %q: %s", d.Id(), err)
			}
			if len(out.ConfigRules) < 1 {
				return 42, "", nil
			}
			rule := out.ConfigRules[0]
			return out, *rule.ConfigRuleState, nil
		},
	}
	_, err = conf.WaitForState()
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] AWS Config config rule %q deleted", name)

	d.SetId("")
	return nil
}

func configRuleSourceDetailsHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	if v, ok := m["message_type"]; ok {
		buf.WriteString(fmt.Sprintf("%s-", v.(string)))
	}
	if v, ok := m["event_source"]; ok {
		buf.WriteString(fmt.Sprintf("%s-", v.(string)))
	}
	if v, ok := m["maximum_execution_frequency"]; ok {
		buf.WriteString(fmt.Sprintf("%s-", v.(string)))
	}
	return hashcode.String(buf.String())
}
<commit_msg>provider/aws: Retry deletion of AWSConfig Rule on ResourceInUseException (#14269)<commit_after>package aws

import (
	"bytes"
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/configservice"
)

func resourceAwsConfigConfigRule() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsConfigConfigRulePut,
		Read:   resourceAwsConfigConfigRuleRead,
		Update: resourceAwsConfigConfigRulePut,
		Delete: resourceAwsConfigConfigRuleDelete,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:         schema.TypeString,
				Required:     true,
				ValidateFunc: validateMaxLength(64),
			},
			"rule_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"description": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateMaxLength(256),
			},
			"input_parameters": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateJsonString,
			},
			"maximum_execution_frequency": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateConfigExecutionFrequency,
			},
			"scope": {
				Type:     schema.TypeList,
				MaxItems: 1,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"compliance_resource_id": {
							Type:         schema.TypeString,
							Optional:     true,
							ValidateFunc: validateMaxLength(256),
						},
						"compliance_resource_types": {
							Type:     schema.TypeSet,
							Optional: true,
							MaxItems: 100,
							Elem: &schema.Schema{
								Type:         schema.TypeString,
								ValidateFunc: validateMaxLength(256),
							},
							Set: schema.HashString,
						},
						"tag_key": {
							Type:         schema.TypeString,
							Optional:     true,
							ValidateFunc: validateMaxLength(128),
						},
						"tag_value": {
							Type:         schema.TypeString,
							Optional:     true,
							ValidateFunc: validateMaxLength(256),
						},
					},
				},
			},
			"source": {
				Type:     schema.TypeList,
				MaxItems: 1,
				Required: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"owner": {
							Type:         schema.TypeString,
							Required:     true,
							ValidateFunc: validateConfigRuleSourceOwner,
						},
						"source_detail": {
							Type:     schema.TypeSet,
							Set:      configRuleSourceDetailsHash,
							Optional: true,
							MaxItems: 25,
							Elem: &schema.Resource{
								Schema: map[string]*schema.Schema{
									"event_source": {
										Type:     schema.TypeString,
										Optional: true,
										Default:  "aws.config",
									},
									"maximum_execution_frequency": {
										Type:         schema.TypeString,
										Optional:     true,
										ValidateFunc: validateConfigExecutionFrequency,
									},
									"message_type": {
										Type:     schema.TypeString,
										Optional: true,
									},
								},
							},
						},
						"source_identifier": {
							Type:         schema.TypeString,
							Required:     true,
							ValidateFunc: validateMaxLength(256),
						},
					},
				},
			},
		},
	}
}

func resourceAwsConfigConfigRulePut(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).configconn

	name := d.Get("name").(string)
	ruleInput := configservice.ConfigRule{
		ConfigRuleName: aws.String(name),
		Source:         expandConfigRuleSource(d.Get("source").([]interface{})),
	}

	scopes := d.Get("scope").([]interface{})
	if len(scopes) > 0 {
		ruleInput.Scope = expandConfigRuleScope(scopes[0].(map[string]interface{}))
	}

	if v, ok := d.GetOk("description"); ok {
		ruleInput.Description = aws.String(v.(string))
	}
	if v, ok := d.GetOk("input_parameters"); ok {
		ruleInput.InputParameters = aws.String(v.(string))
	}
	if v, ok := d.GetOk("maximum_execution_frequency"); ok {
		ruleInput.MaximumExecutionFrequency = aws.String(v.(string))
	}

	input := configservice.PutConfigRuleInput{
		ConfigRule: &ruleInput,
	}
	log.Printf("[DEBUG] Creating AWSConfig config rule: %s", input)
	err := resource.Retry(2*time.Minute, func() *resource.RetryError {
		_, err := conn.PutConfigRule(&input)
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				if awsErr.Code() == "InsufficientPermissionsException" {
					// IAM is eventually consistent
					return resource.RetryableError(err)
				}
			}

			return resource.NonRetryableError(fmt.Errorf("Failed to create AWSConfig rule: %s", err))
		}

		return nil
	})
	if err != nil {
		return err
	}

	d.SetId(name)

	log.Printf("[DEBUG] AWSConfig config rule %q created", name)

	return resourceAwsConfigConfigRuleRead(d, meta)
}

func resourceAwsConfigConfigRuleRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).configconn

	out, err := conn.DescribeConfigRules(&configservice.DescribeConfigRulesInput{
		ConfigRuleNames: []*string{aws.String(d.Id())},
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoSuchConfigRuleException" {
			log.Printf("[WARN] Config Rule %q is gone (NoSuchConfigRuleException)", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	numberOfRules := len(out.ConfigRules)
	if numberOfRules < 1 {
		log.Printf("[WARN] Config Rule %q is gone (no rules found)", d.Id())
		d.SetId("")
		return nil
	}

	if numberOfRules > 1 {
		return fmt.Errorf("Expected exactly 1 Config Rule, received %d: %#v",
			numberOfRules, out.ConfigRules)
	}

	log.Printf("[DEBUG] AWS Config config rule received: %s", out)

	rule := out.ConfigRules[0]
	d.Set("arn", rule.ConfigRuleArn)
	d.Set("rule_id", rule.ConfigRuleId)
	d.Set("name", rule.ConfigRuleName)
	d.Set("description", rule.Description)
	d.Set("input_parameters", rule.InputParameters)
	d.Set("maximum_execution_frequency", rule.MaximumExecutionFrequency)

	if rule.Scope != nil {
		d.Set("scope", flattenConfigRuleScope(rule.Scope))
	}

	d.Set("source", flattenConfigRuleSource(rule.Source))

	return nil
}

func resourceAwsConfigConfigRuleDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).configconn

	name := d.Get("name").(string)

	log.Printf("[DEBUG] Deleting AWS Config config rule %q", name)
	err := resource.Retry(2*time.Minute, func() *resource.RetryError {
		_, err := conn.DeleteConfigRule(&configservice.DeleteConfigRuleInput{
			ConfigRuleName: aws.String(name),
		})
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "ResourceInUseException" {
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("Deleting Config Rule failed: %s", err)
	}

	conf := resource.StateChangeConf{
		Pending: []string{
			configservice.ConfigRuleStateActive,
			configservice.ConfigRuleStateDeleting,
			configservice.ConfigRuleStateDeletingResults,
			configservice.ConfigRuleStateEvaluating,
		},
		Target:  []string{""},
		Timeout: 5 * time.Minute,
		Refresh: func() (interface{}, string, error) {
			out, err := conn.DescribeConfigRules(&configservice.DescribeConfigRulesInput{
				ConfigRuleNames: []*string{aws.String(d.Id())},
			})
			if err != nil {
				if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoSuchConfigRuleException" {
					return 42, "", nil
				}
				return 42, "", fmt.Errorf("Failed to describe config rule %q: %s", d.Id(), err)
			}
			if len(out.ConfigRules) < 1 {
				return 42, "", nil
			}
			rule := out.ConfigRules[0]
			return out, *rule.ConfigRuleState, nil
		},
	}
	_, err = conf.WaitForState()
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] AWS Config config rule %q deleted", name)

	d.SetId("")
	return nil
}

func configRuleSourceDetailsHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	if v, ok := m["message_type"]; ok {
		buf.WriteString(fmt.Sprintf("%s-", v.(string)))
	}
	if v, ok := m["event_source"]; ok {
		buf.WriteString(fmt.Sprintf("%s-", v.(string)))
	}
	if v, ok := m["maximum_execution_frequency"]; ok {
		buf.WriteString(fmt.Sprintf("%s-", v.(string)))
	}
	return hashcode.String(buf.String())
}
<|endoftext|>


<commit_before>package harvest

import (
	"os"
	"testing"
	"time"
)

func TestCreateUpdateDeleteProject(t *testing.T) {

	realworld := os.Getenv("HARVEST_REALWORLD")

	account := os.Getenv("HARVEST_ID")
	username := os.Getenv("HARVEST_USER")
	password := os.Getenv("HARVEST_PASS")

	if realworld == "true" && account != "" && username != "" && password != "" {
		api := NewBasicAuthAPI(account, username, password)

		clients, err := api.GetClients(Defaults())
		if err != nil {
			t.Fatal(err)
		}

		p := Project{
			Name:     "Test Harvest API Project from github.com/adlio/harvest",
			ClientID: clients[0].ID,
			Active:   true,
			BillBy:   "Project",
		}

		err = api.CreateProject(&p, Defaults())
		if err != nil {
			t.Fatal(err)
		}

		if p.ID == 0 {
			t.Errorf("Project %s should not have Zero ID", p.Name)
		}

		p.StartsOn = Date{time.Now()}
		p.EndsOn = Date{time.Now().AddDate(0, 0, 1)}
		p.Notes = "These are the automatically-generated notes from the library."

		err = api.UpdateProject(&p, Defaults())
		if err != nil {
			t.Fatal(err)
		}

		p2, err := api.GetProject(p.ID, Defaults())
		if err != nil {
			t.Fatal(err)
		}
		if p2.Notes != "These are the automatically-generated notes from the library." {
			t.Errorf("UpdateProject() seemed to have failed to change the notes on Project %d", p2.ID)
		}

		// Delete all the UserAssignments.. this avoids a mass-email about the delete
		// to all attached staff.
		assignments, err := api.GetUserAssignments(p.ID, Defaults())
		for i, ua := range assignments {
			if i == 0 {
				ua.IsProjectManager = true
				api.UpdateUserAssignment(ua, Defaults())
			}
			api.DeleteUserAssignment(ua, Defaults())
		}

		err = api.DeleteProject(&p, Defaults())
		if err != nil {
			t.Fatal(err)
		}

	} else {
		t.Skipf("Skipping realworld tests because HARVEST_REALWORLD != true or HARVEST_ID, HARVEST_USER, HARVEST_PASS not supplied as environment variables.")
	}
}
<commit_msg>Cleanup of the HARVEST_REALWORLD=true testing option.<commit_after>package harvest

import (
	"os"
	"testing"
	"time"
)

func TestCreateUpdateDeleteProject(t *testing.T) {
	api := realWorldTestAPI(t)

	clients, err := api.GetClients(Defaults())
	if err != nil {
		t.Fatal(err)
	}

	p := Project{
		Name:     "Test Harvest API Project from github.com/adlio/harvest",
		ClientID: clients[0].ID,
		Active:   true,
		BillBy:   "Project",
	}

	// Create Project
	err = api.CreateProject(&p, Defaults())
	if err != nil {
		t.Fatal(err)
	}

	if p.ID == 0 {
		t.Errorf("Project %s should not have Zero ID", p.Name)
	}

	// Now make a change
	p.StartsOn = Date{time.Now()}
	p.EndsOn = Date{time.Now().AddDate(0, 0, 1)}
	p.Notes = "These are the automatically-generated notes from the library."

	err = api.UpdateProject(&p, Defaults())
	if err != nil {
		t.Fatal(err)
	}

	// Re-fetch to verify the update
	p2, err := api.GetProject(p.ID, Defaults())
	if err != nil {
		t.Fatal(err)
	}
	if p2.Notes != "These are the automatically-generated notes from the library." {
		t.Errorf("UpdateProject() seemed to have failed to change the notes on Project %d", p2.ID)
	}

	// Delete all the UserAssignments.. this avoids a mass-email about the delete
	// to all attached staff.
	assignments, err := api.GetUserAssignments(p.ID, Defaults())
	for i, ua := range assignments {
		if i == 0 {
			ua.IsProjectManager = true
			api.UpdateUserAssignment(ua, Defaults())
		}
		api.DeleteUserAssignment(ua, Defaults())
	}

	// Clean Up Our Mess
	err = api.DeleteProject(&p, Defaults())
	if err != nil {
		t.Fatal(err)
	}
}

func realWorldTestAPI(t *testing.T) *API {
	realworld := os.Getenv("HARVEST_REALWORLD")

	account := os.Getenv("HARVEST_ID")
	username := os.Getenv("HARVEST_USER")
	password := os.Getenv("HARVEST_PASS")

	if realworld == "true" && account != "" && username != "" && password != "" {
		return NewBasicAuthAPI(account, username, password)
	} else {
		t.Skipf("Skipping realworld tests because HARVEST_REALWORLD != true or HARVEST_ID, HARVEST_USER, HARVEST_PASS not supplied as environment variables.")
		return nil
	}
}
<|endoftext|>


<commit_before>package jsonfilelog

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"time"

	"gopkg.in/fsnotify.v1"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/daemon/logger"
	"github.com/docker/docker/pkg/filenotify"
	"github.com/docker/docker/pkg/ioutils"
	"github.com/docker/docker/pkg/jsonlog"
	"github.com/docker/docker/pkg/tailfile"
)

const maxJSONDecodeRetry = 20000

func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, error) {
	l.Reset()
	if err := dec.Decode(l); err != nil {
		return nil, err
	}
	msg := &logger.Message{
		Source:    l.Stream,
		Timestamp: l.Created,
		Line:      []byte(l.Log),
		Attrs:     l.Attrs,
	}
	return msg, nil
}

// ReadLogs implements the logger's LogReader interface for the logs
// created by this driver.
func (l *JSONFileLogger) ReadLogs(config logger.ReadConfig) *logger.LogWatcher {
	logWatcher := logger.NewLogWatcher()

	go l.readLogs(logWatcher, config)
	return logWatcher
}

func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.ReadConfig) {
	defer close(logWatcher.Msg)

	// lock so the read stream doesn't get corrupted due to rotations or other log data written while we read
	// This will block writes!!!
	l.mu.Lock()

	pth := l.writer.LogPath()
	var files []io.ReadSeeker
	for i := l.writer.MaxFiles(); i > 1; i-- {
		f, err := os.Open(fmt.Sprintf("%s.%d", pth, i-1))
		if err != nil {
			if !os.IsNotExist(err) {
				logWatcher.Err <- err
				break
			}
			continue
		}
		defer f.Close()

		files = append(files, f)
	}

	latestFile, err := os.Open(pth)
	if err != nil {
		logWatcher.Err <- err
		l.mu.Unlock()
		return
	}
	defer latestFile.Close()

	if config.Tail != 0 {
		tailer := ioutils.MultiReadSeeker(append(files, latestFile)...)
		tailFile(tailer, logWatcher, config.Tail, config.Since)
	}

	// close all the rotated files
	for _, f := range files {
		if err := f.(io.Closer).Close(); err != nil {
			logrus.WithField("logger", "json-file").Warnf("error closing tailed log file: %v", err)
		}
	}

	if !config.Follow {
		if err := latestFile.Close(); err != nil {
			logrus.Errorf("Error closing file: %v", err)
		}
		l.mu.Unlock()
		return
	}

	if config.Tail >= 0 {
		latestFile.Seek(0, os.SEEK_END)
	}

	l.readers[logWatcher] = struct{}{}
	l.mu.Unlock()

	notifyRotate := l.writer.NotifyRotate()
	followLogs(latestFile, logWatcher, notifyRotate, config.Since)

	l.mu.Lock()
	delete(l.readers, logWatcher)
	l.mu.Unlock()

	l.writer.NotifyRotateEvict(notifyRotate)
}

func tailFile(f io.ReadSeeker, logWatcher *logger.LogWatcher, tail int, since time.Time) {
	var rdr io.Reader
	rdr = f
	if tail > 0 {
		ls, err := tailfile.TailFile(f, tail)
		if err != nil {
			logWatcher.Err <- err
			return
		}
		rdr = bytes.NewBuffer(bytes.Join(ls, []byte("\n")))
	}
	dec := json.NewDecoder(rdr)
	l := &jsonlog.JSONLog{}
	for {
		msg, err := decodeLogLine(dec, l)
		if err != nil {
			if err != io.EOF {
				logWatcher.Err <- err
			}
			return
		}
		if !since.IsZero() && msg.Timestamp.Before(since) {
			continue
		}
		logWatcher.Msg <- msg
	}
}

func watchFile(name string) (filenotify.FileWatcher, error) {
	fileWatcher, err := filenotify.New()
	if err != nil {
		return nil, err
	}

	if err := fileWatcher.Add(name); err != nil {
		logrus.WithField("logger", "json-file").Warnf("falling back to file poller due to error: %v", err)
		fileWatcher.Close()
		fileWatcher = filenotify.NewPollingWatcher()

		if err := fileWatcher.Add(name); err != nil {
			fileWatcher.Close()
			logrus.Debugf("error watching log file for modifications: %v", err)
			return nil, err
		}
	}
	return fileWatcher, nil
}

func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate chan interface{}, since time.Time) {
	dec := json.NewDecoder(f)
	l := &jsonlog.JSONLog{}

	name := f.Name()
	fileWatcher, err := watchFile(name)
	if err != nil {
		logWatcher.Err <- err
		return
	}
	defer func() {
		f.Close()
		fileWatcher.Close()
	}()

	var retries int
	handleRotate := func() error {
		f.Close()
		fileWatcher.Remove(name)

		// retry when the file doesn't exist
		for retries := 0; retries <= 5; retries++ {
			f, err = os.Open(name)
			if err == nil || !os.IsNotExist(err) {
				break
			}
		}
		if err != nil {
			return err
		}
		if err := fileWatcher.Add(name); err != nil {
			return err
		}
		dec = json.NewDecoder(f)
		return nil
	}

	errRetry := errors.New("retry")
	errDone := errors.New("done")
	waitRead := func() error {
		select {
		case e := <-fileWatcher.Events():
			switch e.Op {
			case fsnotify.Write:
				dec = json.NewDecoder(f)
				return nil
			case fsnotify.Rename, fsnotify.Remove:
				select {
				case <-notifyRotate:
				case <-logWatcher.WatchClose():
					fileWatcher.Remove(name)
					return errDone
				}
				if err := handleRotate(); err != nil {
					return err
				}
				return nil
			}
			return errRetry
		case err := <-fileWatcher.Errors():
			logrus.Debug("logger got error watching file: %v", err)
			// Something happened, let's try and stay alive and create a new watcher
			if retries <= 5 {
				fileWatcher.Close()
				fileWatcher, err = watchFile(name)
				if err != nil {
					return err
				}
				retries++
				return errRetry
			}
			return err
		case <-logWatcher.WatchClose():
			fileWatcher.Remove(name)
			return errDone
		}
	}

	handleDecodeErr := func(err error) error {
		if err == io.EOF {
			for err := waitRead(); err != nil; {
				if err == errRetry {
					// retry the waitRead
					continue
				}
				return err
			}
			return nil
		}
		// try again because this shouldn't happen
		if _, ok := err.(*json.SyntaxError); ok && retries <= maxJSONDecodeRetry {
			dec = json.NewDecoder(f)
			retries++
			return nil
		}
		// io.ErrUnexpectedEOF is returned from json.Decoder when there is
		// remaining data in the parser's buffer while an io.EOF occurs.
		// If the json logger writes a partial json log entry to the disk
		// while at the same time the decoder tries to decode it, the race condition happens.
		if err == io.ErrUnexpectedEOF && retries <= maxJSONDecodeRetry {
			reader := io.MultiReader(dec.Buffered(), f)
			dec = json.NewDecoder(reader)
			retries++
			return nil
		}
		return err
	}

	// main loop
	for {
		msg, err := decodeLogLine(dec, l)
		if err != nil {
			if err := handleDecodeErr(err); err != nil {
				if err == errDone {
					return
				}
				// we got an unrecoverable error, so return
				logWatcher.Err <- err
				return
			}
			// ready to try again
			continue
		}

		retries = 0 // reset retries since we've succeeded
		if !since.IsZero() && msg.Timestamp.Before(since) {
			continue
		}
		select {
		case logWatcher.Msg <- msg:
		case <-logWatcher.WatchClose():
			logWatcher.Msg <- msg
			for {
				msg, err := decodeLogLine(dec, l)
				if err != nil {
					return
				}
				if !since.IsZero() && msg.Timestamp.Before(since) {
					continue
				}
				logWatcher.Msg <- msg
			}
		}
	}
}
<commit_msg>Decouple removing the fileWatcher from reading<commit_after>package jsonfilelog

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"time"

	"golang.org/x/net/context"
	"gopkg.in/fsnotify.v1"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/daemon/logger"
	"github.com/docker/docker/pkg/filenotify"
	"github.com/docker/docker/pkg/ioutils"
	"github.com/docker/docker/pkg/jsonlog"
	"github.com/docker/docker/pkg/tailfile"
)

const maxJSONDecodeRetry = 20000

func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, error) {
	l.Reset()
	if err := dec.Decode(l); err != nil {
		return nil, err
	}
	msg := &logger.Message{
		Source:    l.Stream,
		Timestamp: l.Created,
		Line:      []byte(l.Log),
		Attrs:     l.Attrs,
	}
	return msg, nil
}

// ReadLogs implements the logger's LogReader interface for the logs
// created by this driver.
func (l *JSONFileLogger) ReadLogs(config logger.ReadConfig) *logger.LogWatcher {
	logWatcher := logger.NewLogWatcher()

	go l.readLogs(logWatcher, config)
	return logWatcher
}

func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.ReadConfig) {
	defer close(logWatcher.Msg)

	// lock so the read stream doesn't get corrupted due to rotations or other log data written while we read
	// This will block writes!!!
	l.mu.Lock()

	pth := l.writer.LogPath()
	var files []io.ReadSeeker
	for i := l.writer.MaxFiles(); i > 1; i-- {
		f, err := os.Open(fmt.Sprintf("%s.%d", pth, i-1))
		if err != nil {
			if !os.IsNotExist(err) {
				logWatcher.Err <- err
				break
			}
			continue
		}
		defer f.Close()

		files = append(files, f)
	}

	latestFile, err := os.Open(pth)
	if err != nil {
		logWatcher.Err <- err
		l.mu.Unlock()
		return
	}
	defer latestFile.Close()

	if config.Tail != 0 {
		tailer := ioutils.MultiReadSeeker(append(files, latestFile)...)
		tailFile(tailer, logWatcher, config.Tail, config.Since)
	}

	// close all the rotated files
	for _, f := range files {
		if err := f.(io.Closer).Close(); err != nil {
			logrus.WithField("logger", "json-file").Warnf("error closing tailed log file: %v", err)
		}
	}

	if !config.Follow {
		if err := latestFile.Close(); err != nil {
			logrus.Errorf("Error closing file: %v", err)
		}
		l.mu.Unlock()
		return
	}

	if config.Tail >= 0 {
		latestFile.Seek(0, os.SEEK_END)
	}

	l.readers[logWatcher] = struct{}{}
	l.mu.Unlock()

	notifyRotate := l.writer.NotifyRotate()
	followLogs(latestFile, logWatcher, notifyRotate, config.Since)

	l.mu.Lock()
	delete(l.readers, logWatcher)
	l.mu.Unlock()

	l.writer.NotifyRotateEvict(notifyRotate)
}

func tailFile(f io.ReadSeeker, logWatcher *logger.LogWatcher, tail int, since time.Time) {
	var rdr io.Reader
	rdr = f
	if tail > 0 {
		ls, err := tailfile.TailFile(f, tail)
		if err != nil {
			logWatcher.Err <- err
			return
		}
		rdr = bytes.NewBuffer(bytes.Join(ls, []byte("\n")))
	}
	dec := json.NewDecoder(rdr)
	l := &jsonlog.JSONLog{}
	for {
		msg, err := decodeLogLine(dec, l)
		if err != nil {
			if err != io.EOF {
				logWatcher.Err <- err
			}
			return
		}
		if !since.IsZero() && msg.Timestamp.Before(since) {
			continue
		}
		logWatcher.Msg <- msg
	}
}

func watchFile(name string) (filenotify.FileWatcher, error) {
	fileWatcher, err := filenotify.New()
	if err != nil {
		return nil, err
	}

	if err := fileWatcher.Add(name); err != nil {
		logrus.WithField("logger", "json-file").Warnf("falling back to file poller due to error: %v", err)
		fileWatcher.Close()
		fileWatcher = filenotify.NewPollingWatcher()

		if err := fileWatcher.Add(name); err != nil {
			fileWatcher.Close()
			logrus.Debugf("error watching log file for modifications: %v", err)
			return nil, err
		}
	}
	return fileWatcher, nil
}

func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate chan interface{}, since time.Time) {
	dec := json.NewDecoder(f)
	l := &jsonlog.JSONLog{}

	name := f.Name()
	fileWatcher, err := watchFile(name)
	if err != nil {
		logWatcher.Err <- err
		return
	}
	defer func() {
		f.Close()
		fileWatcher.Remove(name)
		fileWatcher.Close()
	}()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		select {
		case <-logWatcher.WatchClose():
			fileWatcher.Remove(name)
			cancel()
		case <-ctx.Done():
			return
		}
	}()

	var retries int
	handleRotate := func() error {
		f.Close()
		fileWatcher.Remove(name)

		// retry when the file doesn't exist
		for retries := 0; retries <= 5; retries++ {
			f, err = os.Open(name)
			if err == nil || !os.IsNotExist(err) {
				break
			}
		}
		if err != nil {
			return err
		}
		if err := fileWatcher.Add(name); err != nil {
			return err
		}
		dec = json.NewDecoder(f)
		return nil
	}

	errRetry := errors.New("retry")
	errDone := errors.New("done")
	waitRead := func() error {
		select {
		case e := <-fileWatcher.Events():
			switch e.Op {
			case fsnotify.Write:
				dec = json.NewDecoder(f)
				return nil
			case fsnotify.Rename, fsnotify.Remove:
				select {
				case <-notifyRotate:
				case <-ctx.Done():
					return errDone
				}
				if err := handleRotate(); err != nil {
					return err
				}
				return nil
			}
			return errRetry
		case err := <-fileWatcher.Errors():
			logrus.Debug("logger got error watching file: %v", err)
			// Something happened, let's try and stay alive and create a new watcher
			if retries <= 5 {
				fileWatcher.Close()
				fileWatcher, err = watchFile(name)
				if err != nil {
					return err
				}
				retries++
				return errRetry
			}
			return err
		case <-ctx.Done():
			return errDone
		}
	}

	handleDecodeErr := func(err error) error {
		if err == io.EOF {
			for err := waitRead(); err != nil; {
				if err == errRetry {
					// retry the waitRead
					continue
				}
				return err
			}
			return nil
		}
		// try again because this shouldn't happen
		if _, ok := err.(*json.SyntaxError); ok && retries <= maxJSONDecodeRetry {
			dec = json.NewDecoder(f)
			retries++
			return nil
		}
		// io.ErrUnexpectedEOF is returned from json.Decoder when there is
		// remaining data in the parser's buffer while an io.EOF occurs.
		// If the json logger writes a partial json log entry to the disk
		// while at the same time the decoder tries to decode it, the race condition happens.
		if err == io.ErrUnexpectedEOF && retries <= maxJSONDecodeRetry {
			reader := io.MultiReader(dec.Buffered(), f)
			dec = json.NewDecoder(reader)
			retries++
			return nil
		}
		return err
	}

	// main loop
	for {
		msg, err := decodeLogLine(dec, l)
		if err != nil {
			if err := handleDecodeErr(err); err != nil {
				if err == errDone {
					return
				}
				// we got an unrecoverable error, so return
				logWatcher.Err <- err
				return
			}
			// ready to try again
			continue
		}

		retries = 0 // reset retries since we've succeeded
		if !since.IsZero() && msg.Timestamp.Before(since) {
			continue
		}
		select {
		case logWatcher.Msg <- msg:
		case <-ctx.Done():
			logWatcher.Msg <- msg
			for {
				msg, err := decodeLogLine(dec, l)
				if err != nil {
					return
				}
				if !since.IsZero() && msg.Timestamp.Before(since) {
					continue
				}
				logWatcher.Msg <- msg
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Ole Krüger.

package cemi

import (
	"fmt"
)

// IndividualAddr is an address for a KNX device.
type IndividualAddr uint16

// String generates a string representation.
func (addr IndividualAddr) String() string {
	return fmt.Sprintf("%d.%d.%d", uint8(addr>>12)&15, uint8(addr>>8)&15, uint8(addr))
}

// GroupAddr is an address for a KNX group object.
type GroupAddr uint16

// String generates a string representation.
func (addr GroupAddr) String() string {
	return fmt.Sprintf("%d/%d/%d", uint8(addr>>11)&31, uint8(addr>>8)&7, uint8(addr))
}
<commit_msg>Add functions to generate GroupAddr and IndividualAddr<commit_after>// Copyright 2017 Ole Krüger.

package cemi

import (
	"fmt"
)

// IndividualAddr is an address for a KNX device.
type IndividualAddr uint16

// NewIndividualAddr3 generates a group address with format a/b/c.
func NewIndividualAddr3(a, b, c uint8) IndividualAddr {
	return IndividualAddr(a&15)<<12 | IndividualAddr(b&15)<<8 | IndividualAddr(c)
}

// NewIndividualAddr2 generates a group address with format a/b.
func NewIndividualAddr2(a, b uint8) IndividualAddr {
	return IndividualAddr(a)<<8 | IndividualAddr(b)
}

// String generates a string representation.
func (addr IndividualAddr) String() string {
	return fmt.Sprintf("%d.%d.%d", uint8(addr>>12)&15, uint8(addr>>8)&15, uint8(addr))
}

// GroupAddr is an address for a KNX group object.
type GroupAddr uint16

// NewGroupAddr3 generates a group address with format a/b/c.
func NewGroupAddr3(a, b, c uint8) GroupAddr {
	return GroupAddr(a&31)<<11 | GroupAddr(b&7)<<8 | GroupAddr(c)
}

// NewGroupAddr2 generates a group address with format a/b.
func NewGroupAddr2(a, b uint8) GroupAddr {
	return GroupAddr(a)<<8 | GroupAddr(b)
}

// String generates a string representation.
func (addr GroupAddr) String() string {
	return fmt.Sprintf("%d/%d/%d", uint8(addr>>11)&31, uint8(addr>>8)&7, uint8(addr))
}
<|endoftext|>