<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bytes

// Simple byte buffer for marshaling data.

import (
	"errors"
	"io"
	"unicode/utf8"
)

// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
// The zero value for Buffer is an empty buffer ready to use.
type Buffer struct {
	buf       []byte            // contents are the bytes buf[off : len(buf)]
	off       int               // read at &buf[off], write at &buf[len(buf)]
	runeBytes [utf8.UTFMax]byte // avoid allocation of slice on each WriteByte or Rune
	bootstrap [64]byte          // memory to hold first slice; helps small buffers (Printf) avoid allocation.
	lastRead  readOp            // last read operation, so that Unread* can work correctly.
}

// The readOp constants describe the last action performed on
// the buffer, so that UnreadRune and UnreadByte can
// check for invalid usage.
type readOp int

const (
	opInvalid  readOp = iota // Non-read operation.
	opReadRune               // Read rune.
	opRead                   // Any other read operation.
)

// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("bytes.Buffer: too large")

// Bytes returns a slice of the contents of the unread portion of the buffer;
// len(b.Bytes()) == b.Len().  If the caller changes the contents of the
// returned slice, the contents of the buffer will change provided there
// are no intervening method calls on the Buffer.
func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }

// String returns the contents of the unread portion of the buffer
// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
func (b *Buffer) String() string {
	if b == nil {
		// Special case, useful in debugging.
		return "<nil>"
	}
	return string(b.buf[b.off:])
}

// Len returns the number of bytes of the unread portion of the buffer;
// b.Len() == len(b.Bytes()).
func (b *Buffer) Len() int { return len(b.buf) - b.off }

// Truncate discards all but the first n unread bytes from the buffer.
// It is an error to call b.Truncate(n) with n > b.Len().
func (b *Buffer) Truncate(n int) {
	b.lastRead = opInvalid
	if n == 0 {
		// Reuse buffer space.
		b.off = 0
	}
	b.buf = b.buf[0 : b.off+n]
}

// Reset resets the buffer so it has no content.
// b.Reset() is the same as b.Truncate(0).
func (b *Buffer) Reset() { b.Truncate(0) }

// grow grows the buffer to guarantee space for n more bytes.
// It returns the index where bytes should be written.
// If the buffer can't grow it will panic with ErrTooLarge.
func (b *Buffer) grow(n int) int {
	m := b.Len()
	// If buffer is empty, reset to recover space.
	if m == 0 && b.off != 0 {
		b.Truncate(0)
	}
	if len(b.buf)+n > cap(b.buf) {
		var buf []byte
		if b.buf == nil && n <= len(b.bootstrap) {
			buf = b.bootstrap[0:]
		} else {
			// not enough space anywhere
			buf = makeSlice(2*cap(b.buf) + n)
			copy(buf, b.buf[b.off:])
		}
		b.buf = buf
		b.off = 0
	}
	b.buf = b.buf[0 : b.off+m+n]
	return b.off + m
}

// Write appends the contents of p to the buffer.  The return
// value n is the length of p; err is always nil.
// If the buffer becomes too large, Write will panic with
// ErrTooLarge.
func (b *Buffer) Write(p []byte) (n int, err error) {
	b.lastRead = opInvalid
	m := b.grow(len(p))
	return copy(b.buf[m:], p), nil
}

// WriteString appends the contents of s to the buffer.  The return
// value n is the length of s; err is always nil.
// If the buffer becomes too large, WriteString will panic with
// ErrTooLarge.
func (b *Buffer) WriteString(s string) (n int, err error) {
	b.lastRead = opInvalid
	m := b.grow(len(s))
	return copy(b.buf[m:], s), nil
}

// MinRead is the minimum slice size passed to a Read call by
// Buffer.ReadFrom.  As long as the Buffer has at least MinRead bytes beyond
// what is required to hold the contents of r, ReadFrom will not grow the
// underlying buffer.
const MinRead = 512

// ReadFrom reads data from r until EOF and appends it to the buffer.
// The return value n is the number of bytes read.
// Any error except io.EOF encountered during the read
// is also returned.
// If the buffer becomes too large, ReadFrom will panic with
// ErrTooLarge.
func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
	b.lastRead = opInvalid
	// If buffer is empty, reset to recover space.
	if b.off >= len(b.buf) {
		b.Truncate(0)
	}
	for {
		if free := cap(b.buf) - len(b.buf); free < MinRead {
			// not enough space at end
			newBuf := b.buf
			if b.off+free < MinRead {
				// not enough space using beginning of buffer;
				// double buffer capacity
				newBuf = makeSlice(2*cap(b.buf) + MinRead)
			}
			copy(newBuf, b.buf[b.off:])
			b.buf = newBuf[:len(b.buf)-b.off]
			b.off = 0
		}
		m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
		b.buf = b.buf[0 : len(b.buf)+m]
		n += int64(m)
		if e == io.EOF {
			break
		}
		if e != nil {
			return n, e
		}
	}
	return n, nil // err is EOF, so return nil explicitly
}

// makeSlice allocates a slice of size n. If the allocation fails, it panics
// with ErrTooLarge.
func makeSlice(n int) []byte {
	// If the make fails, give a known error.
	defer func() {
		if recover() != nil {
			panic(ErrTooLarge)
		}
	}()
	return make([]byte, n)
}

// WriteTo writes data to w until the buffer is drained or an error
// occurs. The return value n is the number of bytes written; it always
// fits into an int, but it is int64 to match the io.WriterTo interface.
// Any error encountered during the write is also returned.
func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
	b.lastRead = opInvalid
	if b.off < len(b.buf) {
		m, e := w.Write(b.buf[b.off:])
		b.off += m
		n = int64(m)
		if e != nil {
			return n, e
		}
		// otherwise all bytes were written, by definition of
		// Write method in io.Writer
	}
	// Buffer is now empty; reset.
	b.Truncate(0)
	return
}

// WriteByte appends the byte c to the buffer.
// The returned error is always nil, but is included
// to match bufio.Writer's WriteByte.
// If the buffer becomes too large, WriteByte will panic with
// ErrTooLarge.
func (b *Buffer) WriteByte(c byte) error {
	b.lastRead = opInvalid
	m := b.grow(1)
	b.buf[m] = c
	return nil
}

// WriteRune appends the UTF-8 encoding of Unicode
// code point r to the buffer, returning its length and
// an error, which is always nil but is included
// to match bufio.Writer's WriteRune.
// If the buffer becomes too large, WriteRune will panic with
// ErrTooLarge.
func (b *Buffer) WriteRune(r rune) (n int, err error) {
	if r < utf8.RuneSelf {
		b.WriteByte(byte(r))
		return 1, nil
	}
	n = utf8.EncodeRune(b.runeBytes[0:], r)
	b.Write(b.runeBytes[0:n])
	return n, nil
}

// Read reads the next len(p) bytes from the buffer or until the buffer
// is drained.  The return value n is the number of bytes read.  If the
// buffer has no data to return, err is io.EOF (unless len(p) is zero);
// otherwise it is nil.
func (b *Buffer) Read(p []byte) (n int, err error) {
	b.lastRead = opInvalid
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		if len(p) == 0 {
			return
		}
		return 0, io.EOF
	}
	n = copy(p, b.buf[b.off:])
	b.off += n
	if n > 0 {
		b.lastRead = opRead
	}
	return
}

// Next returns a slice containing the next n bytes from the buffer,
// advancing the buffer as if the bytes had been returned by Read.
// If there are fewer than n bytes in the buffer, Next returns the entire buffer.
// The slice is only valid until the next call to a read or write method.
func (b *Buffer) Next(n int) []byte {
	b.lastRead = opInvalid
	m := b.Len()
	if n > m {
		n = m
	}
	data := b.buf[b.off : b.off+n]
	b.off += n
	if n > 0 {
		b.lastRead = opRead
	}
	return data
}

// ReadByte reads and returns the next byte from the buffer.
// If no byte is available, it returns error io.EOF.
func (b *Buffer) ReadByte() (c byte, err error) {
	b.lastRead = opInvalid
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, io.EOF
	}
	c = b.buf[b.off]
	b.off++
	b.lastRead = opRead
	return c, nil
}

// ReadRune reads and returns the next UTF-8-encoded
// Unicode code point from the buffer.
// If no bytes are available, the error returned is io.EOF.
// If the bytes are an erroneous UTF-8 encoding, it
// consumes one byte and returns U+FFFD, 1.
func (b *Buffer) ReadRune() (r rune, size int, err error) {
	b.lastRead = opInvalid
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, 0, io.EOF
	}
	b.lastRead = opReadRune
	c := b.buf[b.off]
	if c < utf8.RuneSelf {
		b.off++
		return rune(c), 1, nil
	}
	r, n := utf8.DecodeRune(b.buf[b.off:])
	b.off += n
	return r, n, nil
}

// UnreadRune unreads the last rune returned by ReadRune.
// If the most recent read or write operation on the buffer was
// not a ReadRune, UnreadRune returns an error.  (In this regard
// it is stricter than UnreadByte, which will unread the last byte
// from any read operation.)
func (b *Buffer) UnreadRune() error {
	if b.lastRead != opReadRune {
		return errors.New("bytes.Buffer: UnreadRune: previous operation was not ReadRune")
	}
	b.lastRead = opInvalid
	if b.off > 0 {
		_, n := utf8.DecodeLastRune(b.buf[0:b.off])
		b.off -= n
	}
	return nil
}

// UnreadByte unreads the last byte returned by the most recent
// read operation.  If write has happened since the last read, UnreadByte
// returns an error.
func (b *Buffer) UnreadByte() error {
	if b.lastRead != opReadRune && b.lastRead != opRead {
		return errors.New("bytes.Buffer: UnreadByte: previous operation was not a read")
	}
	b.lastRead = opInvalid
	if b.off > 0 {
		b.off--
	}
	return nil
}

// ReadBytes reads until the first occurrence of delim in the input,
// returning a slice containing the data up to and including the delimiter.
// If ReadBytes encounters an error before finding a delimiter,
// it returns the data read before the error and the error itself (often io.EOF).
// ReadBytes returns err != nil if and only if the returned data does not end in
// delim.
func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) {
	i := IndexByte(b.buf[b.off:], delim)
	size := i + 1
	if i < 0 {
		size = len(b.buf) - b.off
		err = io.EOF
	}
	line = make([]byte, size)
	copy(line, b.buf[b.off:])
	b.off += size
	return
}

// ReadString reads until the first occurrence of delim in the input,
// returning a string containing the data up to and including the delimiter.
// If ReadString encounters an error before finding a delimiter,
// it returns the data read before the error and the error itself (often io.EOF).
// ReadString returns err != nil if and only if the returned data does not end
// in delim.
func (b *Buffer) ReadString(delim byte) (line string, err error) {
	bytes, err := b.ReadBytes(delim)
	return string(bytes), err
}

// NewBuffer creates and initializes a new Buffer using buf as its initial
// contents.  It is intended to prepare a Buffer to read existing data.  It
// can also be used to size the internal buffer for writing. To do that,
// buf should have the desired capacity but a length of zero.
//
// In most cases, new(Buffer) (or just declaring a Buffer variable) is
// preferable to NewBuffer.  In particular, passing a non-empty buf to
// NewBuffer and then writing to the Buffer will overwrite buf, not append to
// it.
func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} }

// NewBufferString creates and initializes a new Buffer using string s as its
// initial contents.  It is intended to prepare a buffer to read an existing
// string.  See the warnings about NewBuffer; similar issues apply here.
func NewBufferString(s string) *Buffer {
	return &Buffer{buf: []byte(s)}
}
<commit_msg>bytes: API tweaks - fix documentation for NewBuffer and NewBufferString - document and implement behavior of Truncate on invalid lengths<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bytes

// Simple byte buffer for marshaling data.

import (
	"errors"
	"io"
	"unicode/utf8"
)

// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
// The zero value for Buffer is an empty buffer ready to use.
type Buffer struct {
	buf       []byte            // contents are the bytes buf[off : len(buf)]
	off       int               // read at &buf[off], write at &buf[len(buf)]
	runeBytes [utf8.UTFMax]byte // avoid allocation of slice on each WriteByte or Rune
	bootstrap [64]byte          // memory to hold first slice; helps small buffers (Printf) avoid allocation.
	lastRead  readOp            // last read operation, so that Unread* can work correctly.
}

// The readOp constants describe the last action performed on
// the buffer, so that UnreadRune and UnreadByte can
// check for invalid usage.
type readOp int

const (
	opInvalid  readOp = iota // Non-read operation.
	opReadRune               // Read rune.
	opRead                   // Any other read operation.
)

// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("bytes.Buffer: too large")

// Bytes returns a slice of the contents of the unread portion of the buffer;
// len(b.Bytes()) == b.Len().  If the caller changes the contents of the
// returned slice, the contents of the buffer will change provided there
// are no intervening method calls on the Buffer.
func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }

// String returns the contents of the unread portion of the buffer
// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
func (b *Buffer) String() string {
	if b == nil {
		// Special case, useful in debugging.
		return "<nil>"
	}
	return string(b.buf[b.off:])
}

// Len returns the number of bytes of the unread portion of the buffer;
// b.Len() == len(b.Bytes()).
func (b *Buffer) Len() int { return len(b.buf) - b.off }

// Truncate discards all but the first n unread bytes from the buffer.
// It panics if n is negative or greater than the length of the buffer.
func (b *Buffer) Truncate(n int) {
	b.lastRead = opInvalid
	switch {
	case n < 0 || n > b.Len():
		panic("bytes.Buffer: truncation out of range")
	case n == 0:
		// Reuse buffer space.
		b.off = 0
	}
	b.buf = b.buf[0 : b.off+n]
}

// Reset resets the buffer so it has no content.
// b.Reset() is the same as b.Truncate(0).
func (b *Buffer) Reset() { b.Truncate(0) }

// grow grows the buffer to guarantee space for n more bytes.
// It returns the index where bytes should be written.
// If the buffer can't grow it will panic with ErrTooLarge.
func (b *Buffer) grow(n int) int {
	m := b.Len()
	// If buffer is empty, reset to recover space.
	if m == 0 && b.off != 0 {
		b.Truncate(0)
	}
	if len(b.buf)+n > cap(b.buf) {
		var buf []byte
		if b.buf == nil && n <= len(b.bootstrap) {
			buf = b.bootstrap[0:]
		} else {
			// not enough space anywhere
			buf = makeSlice(2*cap(b.buf) + n)
			copy(buf, b.buf[b.off:])
		}
		b.buf = buf
		b.off = 0
	}
	b.buf = b.buf[0 : b.off+m+n]
	return b.off + m
}

// Write appends the contents of p to the buffer.  The return
// value n is the length of p; err is always nil.
// If the buffer becomes too large, Write will panic with
// ErrTooLarge.
func (b *Buffer) Write(p []byte) (n int, err error) {
	b.lastRead = opInvalid
	m := b.grow(len(p))
	return copy(b.buf[m:], p), nil
}

// WriteString appends the contents of s to the buffer.  The return
// value n is the length of s; err is always nil.
// If the buffer becomes too large, WriteString will panic with
// ErrTooLarge.
func (b *Buffer) WriteString(s string) (n int, err error) {
	b.lastRead = opInvalid
	m := b.grow(len(s))
	return copy(b.buf[m:], s), nil
}

// MinRead is the minimum slice size passed to a Read call by
// Buffer.ReadFrom.  As long as the Buffer has at least MinRead bytes beyond
// what is required to hold the contents of r, ReadFrom will not grow the
// underlying buffer.
const MinRead = 512

// ReadFrom reads data from r until EOF and appends it to the buffer.
// The return value n is the number of bytes read.
// Any error except io.EOF encountered during the read
// is also returned.
// If the buffer becomes too large, ReadFrom will panic with
// ErrTooLarge.
func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
	b.lastRead = opInvalid
	// If buffer is empty, reset to recover space.
	if b.off >= len(b.buf) {
		b.Truncate(0)
	}
	for {
		if free := cap(b.buf) - len(b.buf); free < MinRead {
			// not enough space at end
			newBuf := b.buf
			if b.off+free < MinRead {
				// not enough space using beginning of buffer;
				// double buffer capacity
				newBuf = makeSlice(2*cap(b.buf) + MinRead)
			}
			copy(newBuf, b.buf[b.off:])
			b.buf = newBuf[:len(b.buf)-b.off]
			b.off = 0
		}
		m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
		b.buf = b.buf[0 : len(b.buf)+m]
		n += int64(m)
		if e == io.EOF {
			break
		}
		if e != nil {
			return n, e
		}
	}
	return n, nil // err is EOF, so return nil explicitly
}

// makeSlice allocates a slice of size n. If the allocation fails, it panics
// with ErrTooLarge.
func makeSlice(n int) []byte {
	// If the make fails, give a known error.
	defer func() {
		if recover() != nil {
			panic(ErrTooLarge)
		}
	}()
	return make([]byte, n)
}

// WriteTo writes data to w until the buffer is drained or an error
// occurs. The return value n is the number of bytes written; it always
// fits into an int, but it is int64 to match the io.WriterTo interface.
// Any error encountered during the write is also returned.
func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
	b.lastRead = opInvalid
	if b.off < len(b.buf) {
		m, e := w.Write(b.buf[b.off:])
		b.off += m
		n = int64(m)
		if e != nil {
			return n, e
		}
		// otherwise all bytes were written, by definition of
		// Write method in io.Writer
	}
	// Buffer is now empty; reset.
	b.Truncate(0)
	return
}

// WriteByte appends the byte c to the buffer.
// The returned error is always nil, but is included
// to match bufio.Writer's WriteByte.
// If the buffer becomes too large, WriteByte will panic with
// ErrTooLarge.
func (b *Buffer) WriteByte(c byte) error {
	b.lastRead = opInvalid
	m := b.grow(1)
	b.buf[m] = c
	return nil
}

// WriteRune appends the UTF-8 encoding of Unicode
// code point r to the buffer, returning its length and
// an error, which is always nil but is included
// to match bufio.Writer's WriteRune.
// If the buffer becomes too large, WriteRune will panic with
// ErrTooLarge.
func (b *Buffer) WriteRune(r rune) (n int, err error) {
	if r < utf8.RuneSelf {
		b.WriteByte(byte(r))
		return 1, nil
	}
	n = utf8.EncodeRune(b.runeBytes[0:], r)
	b.Write(b.runeBytes[0:n])
	return n, nil
}

// Read reads the next len(p) bytes from the buffer or until the buffer
// is drained.  The return value n is the number of bytes read.  If the
// buffer has no data to return, err is io.EOF (unless len(p) is zero);
// otherwise it is nil.
func (b *Buffer) Read(p []byte) (n int, err error) {
	b.lastRead = opInvalid
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		if len(p) == 0 {
			return
		}
		return 0, io.EOF
	}
	n = copy(p, b.buf[b.off:])
	b.off += n
	if n > 0 {
		b.lastRead = opRead
	}
	return
}

// Next returns a slice containing the next n bytes from the buffer,
// advancing the buffer as if the bytes had been returned by Read.
// If there are fewer than n bytes in the buffer, Next returns the entire buffer.
// The slice is only valid until the next call to a read or write method.
func (b *Buffer) Next(n int) []byte {
	b.lastRead = opInvalid
	m := b.Len()
	if n > m {
		n = m
	}
	data := b.buf[b.off : b.off+n]
	b.off += n
	if n > 0 {
		b.lastRead = opRead
	}
	return data
}

// ReadByte reads and returns the next byte from the buffer.
// If no byte is available, it returns error io.EOF.
func (b *Buffer) ReadByte() (c byte, err error) {
	b.lastRead = opInvalid
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, io.EOF
	}
	c = b.buf[b.off]
	b.off++
	b.lastRead = opRead
	return c, nil
}

// ReadRune reads and returns the next UTF-8-encoded
// Unicode code point from the buffer.
// If no bytes are available, the error returned is io.EOF.
// If the bytes are an erroneous UTF-8 encoding, it
// consumes one byte and returns U+FFFD, 1.
func (b *Buffer) ReadRune() (r rune, size int, err error) {
	b.lastRead = opInvalid
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, 0, io.EOF
	}
	b.lastRead = opReadRune
	c := b.buf[b.off]
	if c < utf8.RuneSelf {
		b.off++
		return rune(c), 1, nil
	}
	r, n := utf8.DecodeRune(b.buf[b.off:])
	b.off += n
	return r, n, nil
}

// UnreadRune unreads the last rune returned by ReadRune.
// If the most recent read or write operation on the buffer was
// not a ReadRune, UnreadRune returns an error.  (In this regard
// it is stricter than UnreadByte, which will unread the last byte
// from any read operation.)
func (b *Buffer) UnreadRune() error {
	if b.lastRead != opReadRune {
		return errors.New("bytes.Buffer: UnreadRune: previous operation was not ReadRune")
	}
	b.lastRead = opInvalid
	if b.off > 0 {
		_, n := utf8.DecodeLastRune(b.buf[0:b.off])
		b.off -= n
	}
	return nil
}

// UnreadByte unreads the last byte returned by the most recent
// read operation.  If write has happened since the last read, UnreadByte
// returns an error.
func (b *Buffer) UnreadByte() error {
	if b.lastRead != opReadRune && b.lastRead != opRead {
		return errors.New("bytes.Buffer: UnreadByte: previous operation was not a read")
	}
	b.lastRead = opInvalid
	if b.off > 0 {
		b.off--
	}
	return nil
}

// ReadBytes reads until the first occurrence of delim in the input,
// returning a slice containing the data up to and including the delimiter.
// If ReadBytes encounters an error before finding a delimiter,
// it returns the data read before the error and the error itself (often io.EOF).
// ReadBytes returns err != nil if and only if the returned data does not end in
// delim.
func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) {
	i := IndexByte(b.buf[b.off:], delim)
	size := i + 1
	if i < 0 {
		size = len(b.buf) - b.off
		err = io.EOF
	}
	line = make([]byte, size)
	copy(line, b.buf[b.off:])
	b.off += size
	return
}

// ReadString reads until the first occurrence of delim in the input,
// returning a string containing the data up to and including the delimiter.
// If ReadString encounters an error before finding a delimiter,
// it returns the data read before the error and the error itself (often io.EOF).
// ReadString returns err != nil if and only if the returned data does not end
// in delim.
func (b *Buffer) ReadString(delim byte) (line string, err error) {
	bytes, err := b.ReadBytes(delim)
	return string(bytes), err
}

// NewBuffer creates and initializes a new Buffer using buf as its initial
// contents.  It is intended to prepare a Buffer to read existing data.  It
// can also be used to size the internal buffer for writing. To do that,
// buf should have the desired capacity but a length of zero.
//
// In most cases, new(Buffer) (or just declaring a Buffer variable) is
// sufficient to initialize a Buffer.
func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} }

// NewBufferString creates and initializes a new Buffer using string s as its
// initial contents. It is intended to prepare a buffer to read an existing
// string.
//
// In most cases, new(Buffer) (or just declaring a Buffer variable) is
// sufficient to initialize a Buffer.
func NewBufferString(s string) *Buffer {
	return &Buffer{buf: []byte(s)}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements scopes and the objects they contain.

package ast

import (
	"bytes"
	"fmt"
	"go/token"
)

// A Scope maintains the set of named language entities declared
// in the scope and a link to the immediately surrounding (outer)
// scope.
//
type Scope struct {
	Outer   *Scope
	Objects map[string]*Object
}

// NewScope creates a new scope nested in the outer scope.
func NewScope(outer *Scope) *Scope {
	const n = 4 // initial scope capacity
	return &Scope{outer, make(map[string]*Object, n)}
}

// Lookup returns the object with the given name if it is
// found in scope s, otherwise it returns nil. Outer scopes
// are ignored.
//
func (s *Scope) Lookup(name string) *Object {
	return s.Objects[name]
}

// Insert attempts to insert a named object obj into the scope s.
// If the scope already contains an object alt with the same name,
// Insert leaves the scope unchanged and returns alt. Otherwise
// it inserts obj and returns nil."
//
func (s *Scope) Insert(obj *Object) (alt *Object) {
	if alt = s.Objects[obj.Name]; alt == nil {
		s.Objects[obj.Name] = obj
	}
	return
}

// Debugging support
func (s *Scope) String() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "scope %p {", s)
	if s != nil && len(s.Objects) > 0 {
		fmt.Fprintln(&buf)
		for _, obj := range s.Objects {
			fmt.Fprintf(&buf, "\t%s %s\n", obj.Kind, obj.Name)
		}
	}
	fmt.Fprintf(&buf, "}\n")
	return buf.String()
}

// ----------------------------------------------------------------------------
// Objects

// TODO(gri) Consider replacing the Object struct with an interface
//           and a corresponding set of object implementations.

// An Object describes a named language entity such as a package,
// constant, type, variable, function (incl. methods), or label.
//
// The Data fields contains object-specific data:
//
//	Kind    Data type    Data value
//	Pkg	*Scope       package scope
//	Con     int          iota for the respective declaration
//	Con     != nil       constant value
//
type Object struct {
	Kind ObjKind
	Name string      // declared name
	Decl interface{} // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil
	Data interface{} // object-specific data; or nil
	Type interface{} // place holder for type information; may be nil
}

// NewObj creates a new object of a given kind and name.
func NewObj(kind ObjKind, name string) *Object {
	return &Object{Kind: kind, Name: name}
}

// Pos computes the source position of the declaration of an object name.
// The result may be an invalid position if it cannot be computed
// (obj.Decl may be nil or not correct).
func (obj *Object) Pos() token.Pos {
	name := obj.Name
	switch d := obj.Decl.(type) {
	case *Field:
		for _, n := range d.Names {
			if n.Name == name {
				return n.Pos()
			}
		}
	case *ImportSpec:
		if d.Name != nil && d.Name.Name == name {
			return d.Name.Pos()
		}
		return d.Path.Pos()
	case *ValueSpec:
		for _, n := range d.Names {
			if n.Name == name {
				return n.Pos()
			}
		}
	case *TypeSpec:
		if d.Name.Name == name {
			return d.Name.Pos()
		}
	case *FuncDecl:
		if d.Name.Name == name {
			return d.Name.Pos()
		}
	case *LabeledStmt:
		if d.Label.Name == name {
			return d.Label.Pos()
		}
	case *AssignStmt:
		for _, x := range d.Lhs {
			if ident, isIdent := x.(*Ident); isIdent && ident.Name == name {
				return ident.Pos()
			}
		}
	case *Scope:
		// predeclared object - nothing to do for now
	}
	return token.NoPos
}

// ObKind describes what an object represents.
type ObjKind int

// The list of possible Object kinds.
const (
	Bad ObjKind = iota // for error handling
	Pkg                // package
	Con                // constant
	Typ                // type
	Var                // variable
	Fun                // function or method
	Lbl                // label
)

var objKindStrings = [...]string{
	Bad: "bad",
	Pkg: "package",
	Con: "const",
	Typ: "type",
	Var: "var",
	Fun: "func",
	Lbl: "label",
}

func (kind ObjKind) String() string { return objKindStrings[kind] }
<commit_msg>go/ast: document use of Data field for method objects<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements scopes and the objects they contain.

package ast

import (
	"bytes"
	"fmt"
	"go/token"
)

// A Scope maintains the set of named language entities declared
// in the scope and a link to the immediately surrounding (outer)
// scope.
//
type Scope struct {
	Outer   *Scope
	Objects map[string]*Object
}

// NewScope creates a new scope nested in the outer scope.
func NewScope(outer *Scope) *Scope {
	const n = 4 // initial scope capacity
	return &Scope{outer, make(map[string]*Object, n)}
}

// Lookup returns the object with the given name if it is
// found in scope s, otherwise it returns nil. Outer scopes
// are ignored.
//
func (s *Scope) Lookup(name string) *Object {
	return s.Objects[name]
}

// Insert attempts to insert a named object obj into the scope s.
// If the scope already contains an object alt with the same name,
// Insert leaves the scope unchanged and returns alt. Otherwise
// it inserts obj and returns nil."
//
func (s *Scope) Insert(obj *Object) (alt *Object) {
	if alt = s.Objects[obj.Name]; alt == nil {
		s.Objects[obj.Name] = obj
	}
	return
}

// Debugging support
func (s *Scope) String() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "scope %p {", s)
	if s != nil && len(s.Objects) > 0 {
		fmt.Fprintln(&buf)
		for _, obj := range s.Objects {
			fmt.Fprintf(&buf, "\t%s %s\n", obj.Kind, obj.Name)
		}
	}
	fmt.Fprintf(&buf, "}\n")
	return buf.String()
}

// ----------------------------------------------------------------------------
// Objects

// TODO(gri) Consider replacing the Object struct with an interface
//           and a corresponding set of object implementations.

// An Object describes a named language entity such as a package,
// constant, type, variable, function (incl. methods), or label.
//
// The Data fields contains object-specific data:
//
//	Kind    Data type    Data value
//	Pkg	*Scope       package scope
//	Con     int          iota for the respective declaration
//	Con     != nil       constant value
//	Typ     *Scope       method scope; nil if no methods
//
type Object struct {
	Kind ObjKind
	Name string      // declared name
	Decl interface{} // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil
	Data interface{} // object-specific data; or nil
	Type interface{} // place holder for type information; may be nil
}

// NewObj creates a new object of a given kind and name.
func NewObj(kind ObjKind, name string) *Object {
	return &Object{Kind: kind, Name: name}
}

// Pos computes the source position of the declaration of an object name.
// The result may be an invalid position if it cannot be computed
// (obj.Decl may be nil or not correct).
func (obj *Object) Pos() token.Pos {
	name := obj.Name
	switch d := obj.Decl.(type) {
	case *Field:
		for _, n := range d.Names {
			if n.Name == name {
				return n.Pos()
			}
		}
	case *ImportSpec:
		if d.Name != nil && d.Name.Name == name {
			return d.Name.Pos()
		}
		return d.Path.Pos()
	case *ValueSpec:
		for _, n := range d.Names {
			if n.Name == name {
				return n.Pos()
			}
		}
	case *TypeSpec:
		if d.Name.Name == name {
			return d.Name.Pos()
		}
	case *FuncDecl:
		if d.Name.Name == name {
			return d.Name.Pos()
		}
	case *LabeledStmt:
		if d.Label.Name == name {
			return d.Label.Pos()
		}
	case *AssignStmt:
		for _, x := range d.Lhs {
			if ident, isIdent := x.(*Ident); isIdent && ident.Name == name {
				return ident.Pos()
			}
		}
	case *Scope:
		// predeclared object - nothing to do for now
	}
	return token.NoPos
}

// ObKind describes what an object represents.
type ObjKind int

// The list of possible Object kinds.
const (
	Bad ObjKind = iota // for error handling
	Pkg                // package
	Con                // constant
	Typ                // type
	Var                // variable
	Fun                // function or method
	Lbl                // label
)

var objKindStrings = [...]string{
	Bad: "bad",
	Pkg: "package",
	Con: "const",
	Typ: "type",
	Var: "var",
	Fun: "func",
	Lbl: "label",
}

func (kind ObjKind) String() string { return objKindStrings[kind] }
<|endoftext|>


<commit_before>package useful

import (
	. "github.com/SimonRichardson/wishful/wishful"
)

type Option interface {
	Of(v AnyVal) Point
	Empty() Monoid
	Ap(v Applicative) Applicative
	Chain(f func(v AnyVal) Monad) Monad
	Concat(y Semigroup) Semigroup
	Map(f func(v AnyVal) AnyVal) Functor
	GetOrElse(y AnyVal) AnyVal
	OrElse(y Option) Option
}

type Some struct {
	x AnyVal
}

type None struct {
}

func NewSome(x AnyVal) Some {
	return Some{
		x: x,
	}
}

func NewNone() None {
	return None{}
}

func (x Some) Of(v AnyVal) Point {
	return NewSome(v)
}

func (x None) Of(v AnyVal) Point {
	return NewSome(v)
}

func (x Some) Empty() Monoid {
	return NewNone()
}

func (x None) Empty() Monoid {
	return NewNone()
}

func (x Some) Ap(v Applicative) Applicative {
	return fromMonadToApplicativeAp(x, v)
}

func (x None) Ap(v Applicative) Applicative {
	return x
}

func (x Some) Chain(f func(v AnyVal) Monad) Monad {
	return f(x.x)
}

func (x None) Chain(f func(v AnyVal) Monad) Monad {
	return x
}

func (x Some) Map(f func(v AnyVal) AnyVal) Functor {
	res := x.Chain(func(v AnyVal) Monad {
		return NewSome(f(v))
	})
	return res.(Functor)
}

func (x None) Map(f func(v AnyVal) AnyVal) Functor {
	return x
}

func (x Some) Concat(y Semigroup) Semigroup {
	return concat(x, y)
}

func (x None) Concat(y Semigroup) Semigroup {
	return x
}

// Derived

func (x Some) GetOrElse(y AnyVal) AnyVal {
	return x.x
}

func (x None) GetOrElse(y AnyVal) AnyVal {
	return y
}

func (x Some) OrElse(y Option) Option {
	return Some{}.Of(x.x).(Option)
}

func (x None) OrElse(y Option) Option {
	return y
}
<commit_msg>Adding option#fold<commit_after>package useful

import (
	. "github.com/SimonRichardson/wishful/wishful"
)

type Option interface {
	Of(v AnyVal) Point
	Empty() Monoid
	Ap(v Applicative) Applicative
	Chain(f func(v AnyVal) Monad) Monad
	Concat(y Semigroup) Semigroup
	Fold(f func(v AnyVal) AnyVal, g func() AnyVal) AnyVal
	Map(f func(v AnyVal) AnyVal) Functor
	GetOrElse(y AnyVal) AnyVal
	OrElse(y Option) Option
}

type Some struct {
	x AnyVal
}

type None struct {
}

func NewSome(x AnyVal) Some {
	return Some{
		x: x,
	}
}

func NewNone() None {
	return None{}
}

func (x Some) Of(v AnyVal) Point {
	return NewSome(v)
}

func (x None) Of(v AnyVal) Point {
	return NewSome(v)
}

func (x Some) Empty() Monoid {
	return NewNone()
}

func (x None) Empty() Monoid {
	return NewNone()
}

func (x Some) Ap(v Applicative) Applicative {
	return fromMonadToApplicativeAp(x, v)
}

func (x None) Ap(v Applicative) Applicative {
	return x
}

func (x Some) Chain(f func(v AnyVal) Monad) Monad {
	return f(x.x)
}

func (x None) Chain(f func(v AnyVal) Monad) Monad {
	return x
}

func (x Some) Fold(f func(v AnyVal) AnyVal, g func() AnyVal) AnyVal {
	return f(x.x)
}

func (x None) Fold(f func(v AnyVal) AnyVal, g func() AnyVal) AnyVal {
	return g()
}

func (x Some) Map(f func(v AnyVal) AnyVal) Functor {
	res := x.Chain(func(v AnyVal) Monad {
		return NewSome(f(v))
	})
	return res.(Functor)
}

func (x None) Map(f func(v AnyVal) AnyVal) Functor {
	return x
}

func (x Some) Concat(y Semigroup) Semigroup {
	return concat(x, y)
}

func (x None) Concat(y Semigroup) Semigroup {
	return x
}

// Derived

func (x Some) GetOrElse(y AnyVal) AnyVal {
	return x.x
}

func (x None) GetOrElse(y AnyVal) AnyVal {
	return y
}

func (x Some) OrElse(y Option) Option {
	return Some{}.Of(x.x).(Option)
}

func (x None) OrElse(y Option) Option {
	return y
}
<|endoftext|>


<commit_before>package mark

import (
	"github.com/a8m/expect"
	"testing"
)

func TestRender(t *testing.T) {
	expect := expect.New(t)
	cases := map[string]string{
		"foobar":                "
foobar
",
		"foo|bar":               "
foo|bar
",
		"foo  \nbar":            "
foo
bar
",
		"__bar__ foo":           "
bar foo
",
		"**bar** foo __bar__":   "
bar foo bar
",
		"**bar**__baz__":        "
barbaz
",
		"**bar**foo__bar__":     "
barfoobar
",
		"_bar_baz":              "
barbaz
",
		"_foo_~~bar~~ baz":      "
foobar baz
",
		"~~baz~~ _baz_":         "
baz baz
",
		"`bool` and that's it.": "
bool and that's it.
",
		// Emphasis mixim
		"___foo___":       "
foo
",
		"__*foo*__":       "
foo
",
		"_**mixim**_":     "
mixim
",
		"~~__*mixim*__~~": "
mixim
",
		"~~*mixim*~~":     "
mixim
",
		// Paragraph
		"1  \n2  \n3":        "
1
2
3
",
		"1\n\n2":             "
1
\n
2
",
		"1\n\n\n2":           "
1
\n
2
",
		"1\n\n\n\n\n\n\n\n2": "
1
\n
2
",
		// Heading
		"#1\n##2":                  "
1
\n
2
",
		"#1\np\n##2\n###3\n4\n===": "
1
\n
p
\n
2
\n
3
\n
4
",
		"Hello\n===":               "
Hello
",
		// Links
		"[text](link \"title\")": "
text
",
		"[text](link)":           "
text
",
		"[](link)":               "

",
		"Link: [example](#)":     "
Link: example
",
		"Link: [not really":      "
Link: [not really
",
		"http://localhost:3000":  "
http://localhost:3000
",
		"Link: http://yeah.com":  "
Link: http://yeah.com
",
		"<http://foo.com>":       "
http://foo.com
",
		"Link: <http://l.co>":    "
Link: http://l.co
",
		"Link: <not really":      "
Link: <not really
",
		// CodeBlock
		"\tfoo\n\tbar": "
foo\nbar
",
		"\tfoo\nbar":   "
foo\n

bar
",
		// GfmCodeBlock
		"```js\nvar a;\n```":  "
var a;
",
		"~~~\nvar b;~~~":      "
var b;
",
		"~~~js\nlet d = 1~~~": "
let d = 1
",
		// Hr
		"foo\n****\nbar": "
foo
\n

bar
",
		"foo\n___":       "
foo
\n
",
		// Images
		"![name](url)":           "

",
		"![name](url \"title\")": "

",
		"img: ![name]()":         "
img:
",
		// Lists
		"- foo\n- bar": "

foo

bar

",
		"* foo\n* bar": "

foo

bar

",
		"+ foo\n+ bar": "

foo

bar

",
		// Ordered Lists
		"1. one\n2. two\n3. three": "

one

two

three

",
		"1. one\n 1. one of one":   "

one

one of one

",
		"2. two\n 3. three":        "

two

three

",
	}
	for actual, expected := range cases {
		expect(Render(actual)).To.Equal(expected)
	}
}
<commit_msg>test(mark): tables<commit_after>package mark

import (
	"github.com/a8m/expect"
	"testing"
)

func TestRender(t *testing.T) {
	expect := expect.New(t)
	cases := map[string]string{
		"foobar":                "
foobar
",
		"foo|bar":               "
foo|bar
",
		"foo  \nbar":            "
foo
bar
",
		"__bar__ foo":           "
bar foo
",
		"**bar** foo __bar__":   "
bar foo bar
",
		"**bar**__baz__":        "
barbaz
",
		"**bar**foo__bar__":     "
barfoobar
",
		"_bar_baz":              "
barbaz
",
		"_foo_~~bar~~ baz":      "
foobar baz
",
		"~~baz~~ _baz_":         "
baz baz
",
		"`bool` and that's it.": "
bool and that's it.
",
		// Emphasis mixim
		"___foo___":       "
foo
",
		"__*foo*__":       "
foo
",
		"_**mixim**_":     "
mixim
",
		"~~__*mixim*__~~": "
mixim
",
		"~~*mixim*~~":     "
mixim
",
		// Paragraph
		"1  \n2  \n3":        "
1
2
3
",
		"1\n\n2":             "
1
\n
2
",
		"1\n\n\n2":           "
1
\n
2
",
		"1\n\n\n\n\n\n\n\n2": "
1
\n
2
",
		// Heading
		"#1\n##2":                  "
1
\n
2
",
		"#1\np\n##2\n###3\n4\n===": "
1
\n
p
\n
2
\n
3
\n
4
",
		"Hello\n===":               "
Hello
",
		// Links
		"[text](link \"title\")": "
text
",
		"[text](link)":           "
text
",
		"[](link)":               "

",
		"Link: [example](#)":     "
Link: example
",
		"Link: [not really":      "
Link: [not really
",
		"http://localhost:3000":  "
http://localhost:3000
",
		"Link: http://yeah.com":  "
Link: http://yeah.com
",
		"<http://foo.com>":       "
http://foo.com
",
		"Link: <http://l.co>":    "
Link: http://l.co
",
		"Link: <not really":      "
Link: <not really
",
		// CodeBlock
		"\tfoo\n\tbar": "
foo\nbar
",
		"\tfoo\nbar":   "
foo\n

bar
",
		// GfmCodeBlock
		"```js\nvar a;\n```":  "
var a;
",
		"~~~\nvar b;~~~":      "
var b;
",
		"~~~js\nlet d = 1~~~": "
let d = 1
",
		// Hr
		"foo\n****\nbar": "
foo
\n

bar
",
		"foo\n___":       "
foo
\n
",
		// Images
		"![name](url)":           "

",
		"![name](url \"title\")": "

",
		"img: ![name]()":         "
img:
",
		// Lists
		"- foo\n- bar": "

foo

bar

",
		"* foo\n* bar": "

foo

bar

",
		"+ foo\n+ bar": "

foo

bar

",
		// Ordered Lists
		"1. one\n2. two\n3. three": "

one

two

three

",
		"1. one\n 1. one of one":   "

one

one of one

",
		"2. two\n 3. three":        "

two

three

",
		// Tables
	}
	for actual, expected := range cases {
		expect(Render(actual)).To.Equal(expected)
	}
}
<|endoftext|>


<commit_before>package benchmarkbbs_test

import (
	"fmt"
	"math/rand"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/benchmarkbbs/reporter"
	"code.cloudfoundry.org/operationq"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

const (
	RepBulkFetching                   = "RepBulkFetching"
	RepBulkLoop                       = "RepBulkLoop"
	RepClaimActualLRP                 = "RepClaimActualLRP"
	RepStartActualLRP                 = "RepStartActualLRP"
	NsyncBulkerFetching               = "NsyncBulkerFetching"
	ConvergenceGathering              = "ConvergenceGathering"
	FetchActualLRPsAndSchedulingInfos = "FetchActualLRPsAndSchedulingInfos"
)

var bulkCycle = 30 * time.Second
var eventCount int32 = 0
var claimCount int32 = 0

var BenchmarkTests = func(numReps, numTrials int) {
	Describe("main benchmark test", func() {

		eventCountRunner := func(signals <-chan os.Signal, ready chan<- struct{}) error {
			eventSource, err := bbsClient.SubscribeToEvents(logger)
			Expect(err).NotTo(HaveOccurred())
			close(ready)

			eventChan := make(chan models.Event)
			go func() {
				for {
					event, err := eventSource.Next()
					if err != nil {
						logger.Error("error-getting-next-event", err)
						return
					}
					if event != nil {
						eventChan <- event
					}
				}
			}()

			for {
				select {
				case <-eventChan:
					eventCount += 1

				case <-signals:
					if eventSource != nil {
						err := eventSource.Close()
						if err != nil {
							logger.Error("failed-closing-event-source", err)
						}
					}
					return nil
				}
			}
		}

		var process ifrit.Process
		BeforeEach(func() {
			process = ifrit.Invoke(ifrit.RunFunc(eventCountRunner))
		})

		AfterEach(func() {
			ginkgomon.Kill(process)
		})

		Measure("data for benchmarks", func(b Benchmarker) {
			wg := sync.WaitGroup{}

			// start nsync
			go func() {
				defer GinkgoRecover()
				logger.Info("start-nsync-bulker-loop")
				defer logger.Info("finish-nsync-bulker-loop")
				wg.Add(1)
				defer wg.Done()
				for i := 0; i < numTrials; i++ {
					sleepDuration := getSleepDuration(i, bulkCycle)
					time.Sleep(sleepDuration)
					b.Time("fetch all desired LRP scheduling info", func() {
						desireds, err := bbsClient.DesiredLRPSchedulingInfos(logger, models.DesiredLRPFilter{})
						Expect(err).NotTo(HaveOccurred())
						Expect(len(desireds)).To(BeNumerically("~", expectedLRPCount, expectedLRPVariation), "Number of DesiredLRPs retrieved in Nsync Bulk Loop")
					}, reporter.ReporterInfo{
						MetricName: NsyncBulkerFetching,
					})
				}
			}()

			// start convergence
			go func() {
				defer GinkgoRecover()
				logger.Info("start-lrp-convergence-loop")
				defer logger.Info("finish-lrp-convergence-loop")
				wg.Add(1)
				defer wg.Done()
				for i := 0; i < numTrials; i++ {
					sleepDuration := getSleepDuration(i, bulkCycle)
					time.Sleep(sleepDuration)
					cellSet := models.NewCellSet()
					for i := 0; i < numReps; i++ {
						cellID := fmt.Sprintf("cell-%d", i)
						presence := models.NewCellPresence(cellID, "earth", "north", models.CellCapacity{}, nil, nil)
						cellSet.Add(&presence)
					}

					b.Time("BBS' internal gathering of LRPs", func() {
						activeDB.ConvergeLRPs(logger, cellSet)
					}, reporter.ReporterInfo{
						MetricName: ConvergenceGathering,
					})
				}
			}()

			// start route-emitter
			go func() {
				defer GinkgoRecover()
				logger.Info("start-route-emitter-loop")
				defer logger.Info("finish-route-emitter-loop")
				wg.Add(1)
				defer wg.Done()
				for i := 0; i < numTrials; i++ {
					sleepDuration := getSleepDuration(i, bulkCycle)
					time.Sleep(sleepDuration)
					b.Time("fetch all actualLRPs", func() {
						actuals, err := bbsClient.ActualLRPGroups(logger, models.ActualLRPFilter{})
						Expect(err).NotTo(HaveOccurred())
						Expect(len(actuals)).To(BeNumerically("~", expectedLRPCount, expectedLRPVariation), "Number of ActualLRPs retrieved in router-emitter")

						desireds, err := bbsClient.DesiredLRPSchedulingInfos(logger, models.DesiredLRPFilter{})
						Expect(err).NotTo(HaveOccurred())
						Expect(len(desireds)).To(BeNumerically("~", expectedLRPCount, expectedLRPVariation), "Number of DesiredLRPs retrieved in route-emitter")
					}, reporter.ReporterInfo{
						MetricName: FetchActualLRPsAndSchedulingInfos,
					})
				}
			}()

			totalRan := int32(0)
			totalQueued := int32(0)
			var err error
			queue := operationq.NewSlidingQueue(numTrials)

			// we need to make sure we don't run out of ports so limit amount of
			// active http requests to 25000
			semaphore := make(chan struct{}, 25000)

			for i := 0; i < numReps; i++ {
				cellID := fmt.Sprintf("cell-%d", i)
				wg.Add(1)

				go func(cellID string) {
					defer GinkgoRecover()
					defer wg.Done()

					for j := 0; j < numTrials; j++ {
						sleepDuration := getSleepDuration(j, bulkCycle)
						time.Sleep(sleepDuration)

						b.Time("rep bulk loop", func() {
							defer GinkgoRecover()
							var actuals []*models.ActualLRPGroup
							b.Time("rep bulk fetch", func() {
								semaphore <- struct{}{}
								actuals, err = bbsClient.ActualLRPGroups(logger, models.ActualLRPFilter{CellID: cellID})
								<-semaphore
								Expect(err).NotTo(HaveOccurred())
							}, reporter.ReporterInfo{
								MetricName: RepBulkFetching,
							})

							expectedActualLRPCount, ok := expectedActualLRPCounts[cellID]
							Expect(ok).To(BeTrue())

							expectedActualLRPVariation, ok := expectedActualLRPVariations[cellID]
							Expect(ok).To(BeTrue())

							Expect(len(actuals)).To(BeNumerically("~", expectedActualLRPCount, expectedActualLRPVariation), "Number of ActualLRPs retrieved by cell %s in rep bulk loop", cellID)

							numActuals := len(actuals)
							for k := 0; k < numActuals; k++ {
								actualLRP, _ := actuals[k].Resolve()
								atomic.AddInt32(&totalQueued, 1)
								queue.Push(&lrpOperation{actualLRP, percentWrites, b, &totalRan, &claimCount, semaphore})
							}
						}, reporter.ReporterInfo{
							MetricName: RepBulkLoop,
						})
					}
				}(cellID)
			}

			wg.Wait()

			eventTolerance := float64(claimCount) * errorTolerance
			Eventually(func() int32 { return eventCount }, 2*time.Minute).Should(BeNumerically("~", claimCount, eventTolerance), "events received")
			Eventually(func() int32 { return totalRan }, 2*time.Minute).Should(Equal(totalQueued), "should have run the same number of queued LRP operations")
		}, 1)
	})
}

type lrpOperation struct {
	actualLRP        *models.ActualLRP
	percentWrites    float64
	b                Benchmarker
	globalCount      *int32
	globalClaimCount *int32
	semaphore        chan struct{}
}

func (lo *lrpOperation) Key() string {
	return lo.actualLRP.ProcessGuid
}

func (lo *lrpOperation) Execute() {
	defer GinkgoRecover()
	defer atomic.AddInt32(lo.globalCount, 1)
	var err error
	randomNum := rand.Float64() * 100.0

	// divided by 2 because the start following the claim cause two writes.
	isClaiming := randomNum < (lo.percentWrites / 2)
	actualLRP := lo.actualLRP

	lo.b.Time("start actual LRP", func() {
		netInfo := models.NewActualLRPNetInfo("1.2.3.4", models.NewPortMapping(61999, 8080))
		lo.semaphore <- struct{}{}
		err = bbsClient.StartActualLRP(logger, &actualLRP.ActualLRPKey, &actualLRP.ActualLRPInstanceKey, &netInfo)
		<-lo.semaphore
		Expect(err).NotTo(HaveOccurred())
		if actualLRP.State == models.ActualLRPStateClaimed {
			defer atomic.AddInt32(lo.globalClaimCount, 1)
		}
	}, reporter.ReporterInfo{
		MetricName: RepStartActualLRP,
	})

	if isClaiming {
		lo.b.Time("claim actual LRP", func() {
			index := int(actualLRP.ActualLRPKey.Index)
			lo.semaphore <- struct{}{}
			err = bbsClient.ClaimActualLRP(logger, actualLRP.ActualLRPKey.ProcessGuid, index, &actualLRP.ActualLRPInstanceKey)
			<-lo.semaphore
			Expect(err).NotTo(HaveOccurred())
			defer atomic.AddInt32(lo.globalClaimCount, 1)
		}, reporter.ReporterInfo{
			MetricName: RepClaimActualLRP,
		})
	}
}

func getSleepDuration(loopCounter int, cycleTime time.Duration) time.Duration {
	sleepDuration := cycleTime
	if loopCounter == 0 {
		numMilli := rand.Intn(int(cycleTime.Nanoseconds() / 1000000))
		sleepDuration = time.Duration(numMilli) * time.Millisecond
	}
	return sleepDuration
}
<commit_msg>Add nil placement tags to cell presence.<commit_after>package benchmarkbbs_test

import (
	"fmt"
	"math/rand"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/benchmarkbbs/reporter"
	"code.cloudfoundry.org/operationq"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

const (
	RepBulkFetching                   = "RepBulkFetching"
	RepBulkLoop                       = "RepBulkLoop"
	RepClaimActualLRP                 = "RepClaimActualLRP"
	RepStartActualLRP                 = "RepStartActualLRP"
	NsyncBulkerFetching               = "NsyncBulkerFetching"
	ConvergenceGathering              = "ConvergenceGathering"
	FetchActualLRPsAndSchedulingInfos = "FetchActualLRPsAndSchedulingInfos"
)

var bulkCycle = 30 * time.Second
var eventCount int32 = 0
var claimCount int32 = 0

var BenchmarkTests = func(numReps, numTrials int) {
	Describe("main benchmark test", func() {

		eventCountRunner := func(signals <-chan os.Signal, ready chan<- struct{}) error {
			eventSource, err := bbsClient.SubscribeToEvents(logger)
			Expect(err).NotTo(HaveOccurred())
			close(ready)

			eventChan := make(chan models.Event)
			go func() {
				for {
					event, err := eventSource.Next()
					if err != nil {
						logger.Error("error-getting-next-event", err)
						return
					}
					if event != nil {
						eventChan <- event
					}
				}
			}()

			for {
				select {
				case <-eventChan:
					eventCount += 1

				case <-signals:
					if eventSource != nil {
						err := eventSource.Close()
						if err != nil {
							logger.Error("failed-closing-event-source", err)
						}
					}
					return nil
				}
			}
		}

		var process ifrit.Process
		BeforeEach(func() {
			process = ifrit.Invoke(ifrit.RunFunc(eventCountRunner))
		})

		AfterEach(func() {
			ginkgomon.Kill(process)
		})

		Measure("data for benchmarks", func(b Benchmarker) {
			wg := sync.WaitGroup{}

			// start nsync
			go func() {
				defer GinkgoRecover()
				logger.Info("start-nsync-bulker-loop")
				defer logger.Info("finish-nsync-bulker-loop")
				wg.Add(1)
				defer wg.Done()
				for i := 0; i < numTrials; i++ {
					sleepDuration := getSleepDuration(i, bulkCycle)
					time.Sleep(sleepDuration)
					b.Time("fetch all desired LRP scheduling info", func() {
						desireds, err := bbsClient.DesiredLRPSchedulingInfos(logger, models.DesiredLRPFilter{})
						Expect(err).NotTo(HaveOccurred())
						Expect(len(desireds)).To(BeNumerically("~", expectedLRPCount, expectedLRPVariation), "Number of DesiredLRPs retrieved in Nsync Bulk Loop")
					}, reporter.ReporterInfo{
						MetricName: NsyncBulkerFetching,
					})
				}
			}()

			// start convergence
			go func() {
				defer GinkgoRecover()
				logger.Info("start-lrp-convergence-loop")
				defer logger.Info("finish-lrp-convergence-loop")
				wg.Add(1)
				defer wg.Done()
				for i := 0; i < numTrials; i++ {
					sleepDuration := getSleepDuration(i, bulkCycle)
					time.Sleep(sleepDuration)
					cellSet := models.NewCellSet()
					for i := 0; i < numReps; i++ {
						cellID := fmt.Sprintf("cell-%d", i)
						presence := models.NewCellPresence(cellID, "earth", "north", models.CellCapacity{}, nil, nil, nil)
						cellSet.Add(&presence)
					}

					b.Time("BBS' internal gathering of LRPs", func() {
						activeDB.ConvergeLRPs(logger, cellSet)
					}, reporter.ReporterInfo{
						MetricName: ConvergenceGathering,
					})
				}
			}()

			// start route-emitter
			go func() {
				defer GinkgoRecover()
				logger.Info("start-route-emitter-loop")
				defer logger.Info("finish-route-emitter-loop")
				wg.Add(1)
				defer wg.Done()
				for i := 0; i < numTrials; i++ {
					sleepDuration := getSleepDuration(i, bulkCycle)
					time.Sleep(sleepDuration)
					b.Time("fetch all actualLRPs", func() {
						actuals, err := bbsClient.ActualLRPGroups(logger, models.ActualLRPFilter{})
						Expect(err).NotTo(HaveOccurred())
						Expect(len(actuals)).To(BeNumerically("~", expectedLRPCount, expectedLRPVariation), "Number of ActualLRPs retrieved in router-emitter")

						desireds, err := bbsClient.DesiredLRPSchedulingInfos(logger, models.DesiredLRPFilter{})
						Expect(err).NotTo(HaveOccurred())
						Expect(len(desireds)).To(BeNumerically("~", expectedLRPCount, expectedLRPVariation), "Number of DesiredLRPs retrieved in route-emitter")
					}, reporter.ReporterInfo{
						MetricName: FetchActualLRPsAndSchedulingInfos,
					})
				}
			}()

			totalRan := int32(0)
			totalQueued := int32(0)
			var err error
			queue := operationq.NewSlidingQueue(numTrials)

			// we need to make sure we don't run out of ports so limit amount of
			// active http requests to 25000
			semaphore := make(chan struct{}, 25000)

			for i := 0; i < numReps; i++ {
				cellID := fmt.Sprintf("cell-%d", i)
				wg.Add(1)

				go func(cellID string) {
					defer GinkgoRecover()
					defer wg.Done()

					for j := 0; j < numTrials; j++ {
						sleepDuration := getSleepDuration(j, bulkCycle)
						time.Sleep(sleepDuration)

						b.Time("rep bulk loop", func() {
							defer GinkgoRecover()
							var actuals []*models.ActualLRPGroup
							b.Time("rep bulk fetch", func() {
								semaphore <- struct{}{}
								actuals, err = bbsClient.ActualLRPGroups(logger, models.ActualLRPFilter{CellID: cellID})
								<-semaphore
								Expect(err).NotTo(HaveOccurred())
							}, reporter.ReporterInfo{
								MetricName: RepBulkFetching,
							})

							expectedActualLRPCount, ok := expectedActualLRPCounts[cellID]
							Expect(ok).To(BeTrue())

							expectedActualLRPVariation, ok := expectedActualLRPVariations[cellID]
							Expect(ok).To(BeTrue())

							Expect(len(actuals)).To(BeNumerically("~", expectedActualLRPCount, expectedActualLRPVariation), "Number of ActualLRPs retrieved by cell %s in rep bulk loop", cellID)

							numActuals := len(actuals)
							for k := 0; k < numActuals; k++ {
								actualLRP, _ := actuals[k].Resolve()
								atomic.AddInt32(&totalQueued, 1)
								queue.Push(&lrpOperation{actualLRP, percentWrites, b, &totalRan, &claimCount, semaphore})
							}
						}, reporter.ReporterInfo{
							MetricName: RepBulkLoop,
						})
					}
				}(cellID)
			}

			wg.Wait()

			eventTolerance := float64(claimCount) * errorTolerance
			Eventually(func() int32 { return eventCount }, 2*time.Minute).Should(BeNumerically("~", claimCount, eventTolerance), "events received")
			Eventually(func() int32 { return totalRan }, 2*time.Minute).Should(Equal(totalQueued), "should have run the same number of queued LRP operations")
		}, 1)
	})
}

type lrpOperation struct {
	actualLRP        *models.ActualLRP
	percentWrites    float64
	b                Benchmarker
	globalCount      *int32
	globalClaimCount *int32
	semaphore        chan struct{}
}

func (lo *lrpOperation) Key() string {
	return lo.actualLRP.ProcessGuid
}

func (lo *lrpOperation) Execute() {
	defer GinkgoRecover()
	defer atomic.AddInt32(lo.globalCount, 1)
	var err error
	randomNum := rand.Float64() * 100.0

	// divided by 2 because the start following the claim cause two writes.
	isClaiming := randomNum < (lo.percentWrites / 2)
	actualLRP := lo.actualLRP

	lo.b.Time("start actual LRP", func() {
		netInfo := models.NewActualLRPNetInfo("1.2.3.4", models.NewPortMapping(61999, 8080))
		lo.semaphore <- struct{}{}
		err = bbsClient.StartActualLRP(logger, &actualLRP.ActualLRPKey, &actualLRP.ActualLRPInstanceKey, &netInfo)
		<-lo.semaphore
		Expect(err).NotTo(HaveOccurred())
		if actualLRP.State == models.ActualLRPStateClaimed {
			defer atomic.AddInt32(lo.globalClaimCount, 1)
		}
	}, reporter.ReporterInfo{
		MetricName: RepStartActualLRP,
	})

	if isClaiming {
		lo.b.Time("claim actual LRP", func() {
			index := int(actualLRP.ActualLRPKey.Index)
			lo.semaphore <- struct{}{}
			err = bbsClient.ClaimActualLRP(logger, actualLRP.ActualLRPKey.ProcessGuid, index, &actualLRP.ActualLRPInstanceKey)
			<-lo.semaphore
			Expect(err).NotTo(HaveOccurred())
			defer atomic.AddInt32(lo.globalClaimCount, 1)
		}, reporter.ReporterInfo{
			MetricName: RepClaimActualLRP,
		})
	}
}

func getSleepDuration(loopCounter int, cycleTime time.Duration) time.Duration {
	sleepDuration := cycleTime
	if loopCounter == 0 {
		numMilli := rand.Intn(int(cycleTime.Nanoseconds() / 1000000))
		sleepDuration = time.Duration(numMilli) * time.Millisecond
	}
	return sleepDuration
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbgt

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

func TestOpenPIndex(t *testing.T) {
	pindex, err := OpenPIndex(nil, "not-a-real-file")
	if pindex != nil || err == nil {
		t.Errorf("expected OpenPIndex to fail on a bad file")
	}
}

func TestNewPIndex(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	pindex, err := NewPIndex(nil, "fake", "uuid",
		"blackhole", "indexName", "indexUUID", "",
		"sourceType", "sourceName", "sourceUUID",
		"sourceParams", "sourcePartitions",
		PIndexPath(emptyDir, "fake"))
	if pindex == nil || err != nil {
		t.Errorf("expected NewPIndex to work")
	}
	err = pindex.Close(true)
	if err != nil {
		t.Errorf("expected Close to work")
	}
}

func TestNewPIndexImpl(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	restart := func() {
		t.Errorf("not expecting a restart")
	}

	indexParams := ""

	pindexImpl, dest, err :=
		NewPIndexImpl("AN UNKNOWN PINDEX IMPL TYPE",
			indexParams, emptyDir, restart)
	if err == nil || pindexImpl != nil || dest != nil {
		t.Errorf("expected err on unknown impl type")
	}

	pindexImpl, dest, err =
		OpenPIndexImpl("AN UNKNOWN PINDEX IMPL TYPE", emptyDir, restart)
	if err == nil || pindexImpl != nil || dest != nil {
		t.Errorf("expected err on unknown impl type")
	}
}

func TestBlackholePIndexImpl(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	restart := func() {
		t.Errorf("not expecting a restart")
	}

	pindex, dest, err :=
		OpenBlackHolePIndexImpl("blackhole", emptyDir, restart)
	if err == nil || pindex != nil || dest != nil {
		t.Errorf("expected OpenBlackHolePIndexImpl to error on emptyDir")
	}

	pindex, dest, err =
		NewBlackHolePIndexImpl("blackhole", "", emptyDir, restart)
	if err != nil || pindex == nil || dest == nil {
		t.Errorf("expected NewBlackHolePIndexImpl to work")
	}

	pindex, dest, err =
		OpenBlackHolePIndexImpl("blackhole", emptyDir, restart)
	if err != nil || pindex == nil || dest == nil {
		t.Errorf("expected OpenBlackHolePIndexImpl to work")
	}

	if dest.Close() != nil ||
		dest.DataUpdate("", nil, 0, nil,
			0, DEST_EXTRAS_TYPE_NIL, nil) != nil ||
		dest.DataDelete("", nil, 0,
			0, DEST_EXTRAS_TYPE_NIL, nil) != nil ||
		dest.SnapshotStart("", 0, 0) != nil ||
		dest.OpaqueSet("", nil) != nil ||
		dest.Rollback("", 0) != nil ||
		dest.ConsistencyWait("", "", "", 0, nil) != nil ||
		dest.Query(nil, nil, nil, nil) != nil {
		t.Errorf("expected no errors from a blackhole pindex impl")
	}

	c, err := dest.Count(nil, nil)
	if err != nil || c != 0 {
		t.Errorf("expected 0, no err")
	}

	b := &bytes.Buffer{}
	err = dest.Stats(b)
	if err != nil {
		t.Errorf("expected 0, no err")
	}
	if string(b.Bytes()) != "null" {
		t.Errorf("expected null")
	}

	v, lastSeq, err := dest.OpaqueGet("")
	if err != nil || v != nil || lastSeq != 0 {
		t.Errorf("expected nothing from blackhole.OpaqueGet()")
	}

	bt := PIndexImplTypes["blackhole"]
	if bt == nil {
		t.Errorf("expected blackhole in PIndexImplTypes")
	}
	if bt.New == nil || bt.Open == nil {
		t.Errorf("blackhole should have open and new funcs")
	}
	if bt.Count != nil {
		t.Errorf("expected blackhole count nil")
	}
	if bt.Query != nil {
		t.Errorf("expected blackhole query nil")
	}
}

func TestErrorConsistencyWait(t *testing.T) {
	e := &ErrorConsistencyWait{}
	if e.Error() == "" {
		t.Errorf("expected err")
	}
}

func TestErrorConsistencyWaitDone(t *testing.T) {
	currSeqFunc := func() uint64 {
		return 101
	}

	cancelCh := make(chan bool)
	doneCh := make(chan error)

	var cwdErr error
	endCh := make(chan struct{})

	go func() {
		cwdErr = ConsistencyWaitDone("partition",
			cancelCh,
			doneCh,
			currSeqFunc)
		close(endCh)
	}()

	close(cancelCh)

	<-endCh

	if cwdErr == nil {
		t.Errorf("expected err")
	}

	// --------------------------

	cancelCh = make(chan bool)
	doneCh = make(chan error)

	cwdErr = nil
	endCh = make(chan struct{})

	go func() {
		cwdErr = ConsistencyWaitDone("partition",
			cancelCh,
			doneCh,
			currSeqFunc)
		close(endCh)
	}()

	doneErr := fmt.Errorf("doneErr")
	doneCh <- doneErr

	<-endCh

	if cwdErr != doneErr {
		t.Errorf("expected doneErr")
	}
}
<commit_msg>TestPIndexStoreStats<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbgt

import (
	"bytes"
	"container/list"
	"fmt"
	"io/ioutil"
	"os"
	"testing"

	"github.com/rcrowley/go-metrics"
)

func TestOpenPIndex(t *testing.T) {
	pindex, err := OpenPIndex(nil, "not-a-real-file")
	if pindex != nil || err == nil {
		t.Errorf("expected OpenPIndex to fail on a bad file")
	}
}

func TestNewPIndex(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	pindex, err := NewPIndex(nil, "fake", "uuid",
		"blackhole", "indexName", "indexUUID", "",
		"sourceType", "sourceName", "sourceUUID",
		"sourceParams", "sourcePartitions",
		PIndexPath(emptyDir, "fake"))
	if pindex == nil || err != nil {
		t.Errorf("expected NewPIndex to work")
	}
	err = pindex.Close(true)
	if err != nil {
		t.Errorf("expected Close to work")
	}
}

func TestNewPIndexImpl(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	restart := func() {
		t.Errorf("not expecting a restart")
	}

	indexParams := ""

	pindexImpl, dest, err :=
		NewPIndexImpl("AN UNKNOWN PINDEX IMPL TYPE",
			indexParams, emptyDir, restart)
	if err == nil || pindexImpl != nil || dest != nil {
		t.Errorf("expected err on unknown impl type")
	}

	pindexImpl, dest, err =
		OpenPIndexImpl("AN UNKNOWN PINDEX IMPL TYPE", emptyDir, restart)
	if err == nil || pindexImpl != nil || dest != nil {
		t.Errorf("expected err on unknown impl type")
	}
}

func TestBlackholePIndexImpl(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	restart := func() {
		t.Errorf("not expecting a restart")
	}

	pindex, dest, err :=
		OpenBlackHolePIndexImpl("blackhole", emptyDir, restart)
	if err == nil || pindex != nil || dest != nil {
		t.Errorf("expected OpenBlackHolePIndexImpl to error on emptyDir")
	}

	pindex, dest, err =
		NewBlackHolePIndexImpl("blackhole", "", emptyDir, restart)
	if err != nil || pindex == nil || dest == nil {
		t.Errorf("expected NewBlackHolePIndexImpl to work")
	}

	pindex, dest, err =
		OpenBlackHolePIndexImpl("blackhole", emptyDir, restart)
	if err != nil || pindex == nil || dest == nil {
		t.Errorf("expected OpenBlackHolePIndexImpl to work")
	}

	if dest.Close() != nil ||
		dest.DataUpdate("", nil, 0, nil,
			0, DEST_EXTRAS_TYPE_NIL, nil) != nil ||
		dest.DataDelete("", nil, 0,
			0, DEST_EXTRAS_TYPE_NIL, nil) != nil ||
		dest.SnapshotStart("", 0, 0) != nil ||
		dest.OpaqueSet("", nil) != nil ||
		dest.Rollback("", 0) != nil ||
		dest.ConsistencyWait("", "", "", 0, nil) != nil ||
		dest.Query(nil, nil, nil, nil) != nil {
		t.Errorf("expected no errors from a blackhole pindex impl")
	}

	c, err := dest.Count(nil, nil)
	if err != nil || c != 0 {
		t.Errorf("expected 0, no err")
	}

	b := &bytes.Buffer{}
	err = dest.Stats(b)
	if err != nil {
		t.Errorf("expected 0, no err")
	}
	if string(b.Bytes()) != "null" {
		t.Errorf("expected null")
	}

	v, lastSeq, err := dest.OpaqueGet("")
	if err != nil || v != nil || lastSeq != 0 {
		t.Errorf("expected nothing from blackhole.OpaqueGet()")
	}

	bt := PIndexImplTypes["blackhole"]
	if bt == nil {
		t.Errorf("expected blackhole in PIndexImplTypes")
	}
	if bt.New == nil || bt.Open == nil {
		t.Errorf("blackhole should have open and new funcs")
	}
	if bt.Count != nil {
		t.Errorf("expected blackhole count nil")
	}
	if bt.Query != nil {
		t.Errorf("expected blackhole query nil")
	}
}

func TestErrorConsistencyWait(t *testing.T) {
	e := &ErrorConsistencyWait{}
	if e.Error() == "" {
		t.Errorf("expected err")
	}
}

func TestErrorConsistencyWaitDone(t *testing.T) {
	currSeqFunc := func() uint64 {
		return 101
	}

	cancelCh := make(chan bool)
	doneCh := make(chan error)

	var cwdErr error
	endCh := make(chan struct{})

	go func() {
		cwdErr = ConsistencyWaitDone("partition",
			cancelCh,
			doneCh,
			currSeqFunc)
		close(endCh)
	}()

	close(cancelCh)

	<-endCh

	if cwdErr == nil {
		t.Errorf("expected err")
	}

	// --------------------------

	cancelCh = make(chan bool)
	doneCh = make(chan error)

	cwdErr = nil
	endCh = make(chan struct{})

	go func() {
		cwdErr = ConsistencyWaitDone("partition",
			cancelCh,
			doneCh,
			currSeqFunc)
		close(endCh)
	}()

	doneErr := fmt.Errorf("doneErr")
	doneCh <- doneErr

	<-endCh

	if cwdErr != doneErr {
		t.Errorf("expected doneErr")
	}
}

func TestPIndexStoreStats(t *testing.T) {
	s := PIndexStoreStats{
		TimerBatchStore: metrics.NewTimer(),
		Errors:          list.New(),
	}

	w := bytes.NewBuffer(nil)
	s.WriteJSON(w)
	if w.String() == "" {
		t.Errorf("expected some writes")
	}

	s.Errors.PushBack("hello")
	s.Errors.PushBack("world")

	w2 := bytes.NewBuffer(nil)
	s.WriteJSON(w2)
	if w2.String() == "" {
		t.Errorf("expected some writes")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cert

import (
	"crypto/x509"
	"encoding/pem"
	"errors"
)

const (
	// CertificateBlockType is a possible value for pem.Block.Type.
	CertificateBlockType = "CERTIFICATE"
	// CertificateRequestBlockType is a possible value for pem.Block.Type.
	CertificateRequestBlockType = "CERTIFICATE REQUEST"
)

// ParseCertsPEM returns the x509.Certificates contained in the given PEM-encoded byte array
// Returns an error if a certificate could not be parsed, or if the data does not contain any certificates
func ParseCertsPEM(pemCerts []byte) ([]*x509.Certificate, error) {
	ok := false
	certs := []*x509.Certificate{}
	for len(pemCerts) > 0 {
		var block *pem.Block
		block, pemCerts = pem.Decode(pemCerts)
		if block == nil {
			break
		}
		// Only use PEM "CERTIFICATE" blocks without extra headers
		if block.Type != CertificateBlockType || len(block.Headers) != 0 {
			continue
		}

		cert, err := x509.ParseCertificate(block.Bytes)
		if err != nil {
			return certs, err
		}

		certs = append(certs, cert)
		ok = true
	}

	if !ok {
		return certs, errors.New("data does not contain any valid RSA or ECDSA certificates")
	}
	return certs, nil
}
<commit_msg>publish cluster authentication trust via controller<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cert

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"errors"
)

const (
	// CertificateBlockType is a possible value for pem.Block.Type.
	CertificateBlockType = "CERTIFICATE"
	// CertificateRequestBlockType is a possible value for pem.Block.Type.
	CertificateRequestBlockType = "CERTIFICATE REQUEST"
)

// ParseCertsPEM returns the x509.Certificates contained in the given PEM-encoded byte array
// Returns an error if a certificate could not be parsed, or if the data does not contain any certificates
func ParseCertsPEM(pemCerts []byte) ([]*x509.Certificate, error) {
	ok := false
	certs := []*x509.Certificate{}
	for len(pemCerts) > 0 {
		var block *pem.Block
		block, pemCerts = pem.Decode(pemCerts)
		if block == nil {
			break
		}
		// Only use PEM "CERTIFICATE" blocks without extra headers
		if block.Type != CertificateBlockType || len(block.Headers) != 0 {
			continue
		}

		cert, err := x509.ParseCertificate(block.Bytes)
		if err != nil {
			return certs, err
		}

		certs = append(certs, cert)
		ok = true
	}

	if !ok {
		return certs, errors.New("data does not contain any valid RSA or ECDSA certificates")
	}
	return certs, nil
}

// EncodeCertificates returns the PEM-encoded byte array that represents by the specified certs.
func EncodeCertificates(certs ...*x509.Certificate) ([]byte, error) {
	b := bytes.Buffer{}
	for _, cert := range certs {
		if err := pem.Encode(&b, &pem.Block{Type: CertificateBlockType, Bytes: cert.Raw}); err != nil {
			return []byte{}, err
		}
	}
	return b.Bytes(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
)

const (
	intentWait = 1000
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config  *pb.Config
	serving bool
}

type mainChecker struct{}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		log.Printf("Does %v = %v and %v = %v?", s.Name, servertype, s.Identifier, servername)
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	log.Printf("Assessing server %v", server)
	ip, port := getIP("gobuildslave", server)
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(context.Background(), &pbs.Empty{})
	if err != nil {
		log.Printf("Err %v", err)
		return list, conf
	}

	r2, err := slave.GetConfig(context.Background(), &pbs.Empty{})
	if err != nil {
		log.Printf("Err %v", err)
		return list, conf
	}

	return r, r2
}

func runJob(job *pbs.JobSpec, server string) {
	log.Printf("RUNNING: %v on %v", job, server)
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		slave.Run(context.Background(), job)
		log.Printf("RUN COMMAND SENT %v", job)
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	log.Printf("DISCOVERED: %v", r)
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	// Do nothing
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Masters++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Masters: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s Server) MatchIntent() {
	//for s.serving {
	time.Sleep(intentWait)

	state := getConfig(&mainChecker{})
	diff := configDiff(s.config, state)
	joblist := runJobs(diff)
	log.Printf("FOUND %v from %v and %v", joblist, state, s.config)
	for _, job := range joblist {
		runJob(job, chooseServer(job, &mainChecker{}))
	}
	//}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}
	log.Printf("READ: %v", config)

	var sync = flag.Bool("once", false, "One pass intent match")
	s := Server{&goserver.GoServer{}, config, true}

	flag.Parse()
	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.RegisterServer("gobuildmaster", false)
		s.Serve()
	}
}
<commit_msg>Updated serving process. This closes #24.<commit_after>package main

import (
	"context"
	"flag"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
)

const (
	intentWait = 1000
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config  *pb.Config
	serving bool
}

type mainChecker struct{}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		log.Printf("Does %v = %v and %v = %v?", s.Name, servertype, s.Identifier, servername)
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	log.Printf("Assessing server %v", server)
	ip, port := getIP("gobuildslave", server)
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(context.Background(), &pbs.Empty{})
	if err != nil {
		log.Printf("Err %v", err)
		return list, conf
	}

	r2, err := slave.GetConfig(context.Background(), &pbs.Empty{})
	if err != nil {
		log.Printf("Err %v", err)
		return list, conf
	}

	return r, r2
}

func runJob(job *pbs.JobSpec, server string) {
	log.Printf("RUNNING: %v on %v", job, server)
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		slave.Run(context.Background(), job)
		log.Printf("RUN COMMAND SENT %v", job)
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	log.Printf("DISCOVERED: %v", r)
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	// Do nothing
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Masters++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Masters: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s Server) MatchIntent() {
	for s.serving {
		time.Sleep(intentWait)

		state := getConfig(&mainChecker{})
		diff := configDiff(s.config, state)
		joblist := runJobs(diff)
		log.Printf("FOUND %v from %v and %v", joblist, state, s.config)
		for _, job := range joblist {
			runJob(job, chooseServer(job, &mainChecker{}))
		}
	}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}
	log.Printf("READ: %v", config)

	var sync = flag.Bool("once", false, "One pass intent match")
	s := Server{&goserver.GoServer{}, config, true}

	flag.Parse()
	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.RegisterServer("gobuildmaster", false)
		s.RegisterServingTask(s.MatchIntent)
		s.Serve()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"sync"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
	pbg "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

const (
	intentWait = time.Second
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config     *pb.Config
	serving    bool
	LastIntent time.Time
	LastMaster time.Time
	worldMutex *sync.Mutex
	world      map[string]map[string]struct{}
	getter     getter
}

type prodGetter struct{}

func (g *prodGetter) getJobs(server *pbd.RegistryEntry) (*pbs.JobList, error) {
	list := &pbs.JobList{}

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.Dial(server.GetIp()+":"+strconv.Itoa(int(server.GetPort())), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, err
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	return r, err
}

func (s *Server) checkerThread(i *pb.Intent) {
	for true {
		time.Sleep(time.Minute)

		if len(s.world[i.GetSpec().GetName()]) != int(i.Count) {
			s.Log(fmt.Sprintf("MISMATCH: %v, %v", i, s.world[i.GetSpec().GetName()]))
		}
	}
}

func (g *prodGetter) getSlaves() (*pbd.ServiceList, error) {
	ret := &pbd.ServiceList{}

	conn, err := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	if err != nil {
		return ret, err
	}
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return ret, err
	}

	for _, s := range r.Services {
		if s.GetName() == "gobuildslave" {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret, nil
}

type mainChecker struct {
	prev   []string
	logger func(string)
}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) getprev() []string {
	return t.prev
}
func (t *mainChecker) setprev(v []string) {
	t.prev = v
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	ip, port := getIP("gobuildslave", server)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, conf
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	r2, err := slave.GetConfig(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	return r, r2
}

func (t *mainChecker) master(entry *pbd.RegistryEntry, master bool) bool {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	conn, _ := grpc.Dial(entry.GetIp()+":"+strconv.Itoa(int(entry.GetPort())), grpc.WithInsecure())
	defer conn.Close()

	server := pbg.NewGoserverServiceClient(conn)
	_, err := server.Mote(ctx, &pbg.MoteRequest{Master: master}, grpc.FailFast(false))
	if err != nil {
		t.logger(fmt.Sprintf("Master REJECT(%v): %v", entry, err))
	}

	return err == nil
}

func runJob(job *pbs.JobSpec, server string) {
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		job.Server = server
		slave.Run(ctx, job, grpc.FailFast(false))
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildMasterServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//GetState gets the state of the server
func (s Server) GetState() []*pbg.State {
	return []*pbg.State{&pbg.State{Key: "last_intent", TimeValue: s.LastIntent.Unix()},
		&pbg.State{Key: "last_master", TimeValue: s.LastMaster.Unix()},
		&pbg.State{Key: "world", Text: fmt.Sprintf("%v", s.world)}}
}

//Compare compares current state to desired state
func (s Server) Compare(ctx context.Context, in *pb.Empty) (*pb.CompareResponse, error) {
	resp := &pb.CompareResponse{}
	list, _ := getFleetStatus(&mainChecker{logger: s.Log})
	cc := &pb.Config{}
	for _, jlist := range list {
		for _, job := range jlist.GetDetails() {
			cc.Intents = append(cc.Intents, &pb.Intent{Spec: job.GetSpec()})
		}
	}
	resp.Current = cc
	resp.Desired = s.config

	return resp, nil
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Count++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Count: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s *Server) MatchIntent() {
	checker := &mainChecker{logger: s.Log}
	for s.serving {
		time.Sleep(intentWait)
		s.LastIntent = time.Now()

		state := getConfig(checker)
		diff := configDiff(s.config, state)
		joblist := runJobs(diff)
		for _, job := range joblist {
			runJob(job, chooseServer(job, checker))
		}
	}
}

// SetMaster sets up the master settings
func (s *Server) SetMaster() {
	t := time.Now()
	checker := &mainChecker{logger: s.Log}
	s.LastMaster = time.Now()

	fleet := checker.discover()
	matcher := make(map[string][]*pbd.RegistryEntry)
	hasMaster := make(map[string]int)
	for _, entry := range fleet.GetServices() {
		if !entry.GetIgnoresMaster() {
			if _, ok := matcher[entry.GetName()]; !ok {
				if entry.GetMaster() {
					hasMaster[entry.GetName()]++
				}
				matcher[entry.GetName()] = []*pbd.RegistryEntry{entry}
			} else {
				if entry.GetMaster() {
					hasMaster[entry.GetName()] = 1
					matcher[entry.GetName()] = append(matcher[entry.GetName()], entry)
				}
			}
		}
	}

	for key, entries := range matcher {
		if hasMaster[key] > 1 {
			hasMaster[key] = 1
			seen := false
			for _, entry := range entries {
				if seen && entry.GetMaster() {
					checker.master(entry, false)
				} else if entry.GetMaster() {
					seen = true
				}
			}
		}

		if hasMaster[key] == 0 {
			for _, entry := range entries {
				if checker.master(entry, true) {
					entry.Master = true
					break
				}
			}
		}
	}
	s.LogFunction("SetMasterRun", t)
}

//Init builds up the server
func Init(config *pb.Config) *Server {
	s := &Server{
		&goserver.GoServer{},
		config,
		true,
		time.Now(),
		time.Now(),
		&sync.Mutex{},
		make(map[string]map[string]struct{}),
		&prodGetter{},
	}
	return s
}

func (s *Server) becomeMaster() {
	for true {
		time.Sleep(time.Minute)
		_, _, err := utils.Resolve("gobuildmaster")
		if err != nil {
			s.Registry.Master = true
		}
	}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}

	s := Init(config)

	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	s.Register = s
	s.PrepServer()
	s.GoServer.Killme = false
	s.RegisterServer("gobuildmaster", false)
	s.RegisterServingTask(s.MatchIntent)
	s.RegisterServingTask(s.becomeMaster)
	s.RegisterRepeatingTask(s.SetMaster, time.Second)
	s.RegisterRepeatingTask(s.buildWorld, time.Minute)

	s.checkerThread(s.config.GetIntents()[0])

	err = s.Serve()
	if err != nil {
		log.Fatalf("Serve error: %v", err)
	}
}
<commit_msg>Runs thread in background<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"sync"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
	pbg "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

const (
	intentWait = time.Second
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config     *pb.Config
	serving    bool
	LastIntent time.Time
	LastMaster time.Time
	worldMutex *sync.Mutex
	world      map[string]map[string]struct{}
	getter     getter
}

type prodGetter struct{}

func (g *prodGetter) getJobs(server *pbd.RegistryEntry) (*pbs.JobList, error) {
	list := &pbs.JobList{}

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.Dial(server.GetIp()+":"+strconv.Itoa(int(server.GetPort())), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, err
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	return r, err
}

func (s *Server) checkerThread(i *pb.Intent) {
	for true {
		time.Sleep(time.Minute)

		if len(s.world[i.GetSpec().GetName()]) != int(i.Count) {
			s.Log(fmt.Sprintf("MISMATCH: %v, %v", i, s.world[i.GetSpec().GetName()]))
		}
	}
}

func (g *prodGetter) getSlaves() (*pbd.ServiceList, error) {
	ret := &pbd.ServiceList{}

	conn, err := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	if err != nil {
		return ret, err
	}
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return ret, err
	}

	for _, s := range r.Services {
		if s.GetName() == "gobuildslave" {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret, nil
}

type mainChecker struct {
	prev   []string
	logger func(string)
}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) getprev() []string {
	return t.prev
}
func (t *mainChecker) setprev(v []string) {
	t.prev = v
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	ip, port := getIP("gobuildslave", server)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, conf
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	r2, err := slave.GetConfig(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	return r, r2
}

func (t *mainChecker) master(entry *pbd.RegistryEntry, master bool) bool {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	conn, _ := grpc.Dial(entry.GetIp()+":"+strconv.Itoa(int(entry.GetPort())), grpc.WithInsecure())
	defer conn.Close()

	server := pbg.NewGoserverServiceClient(conn)
	_, err := server.Mote(ctx, &pbg.MoteRequest{Master: master}, grpc.FailFast(false))
	if err != nil {
		t.logger(fmt.Sprintf("Master REJECT(%v): %v", entry, err))
	}

	return err == nil
}

func runJob(job *pbs.JobSpec, server string) {
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		job.Server = server
		slave.Run(ctx, job, grpc.FailFast(false))
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildMasterServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//GetState gets the state of the server
func (s Server) GetState() []*pbg.State {
	return []*pbg.State{&pbg.State{Key: "last_intent", TimeValue: s.LastIntent.Unix()},
		&pbg.State{Key: "last_master", TimeValue: s.LastMaster.Unix()},
		&pbg.State{Key: "world", Text: fmt.Sprintf("%v", s.world)}}
}

//Compare compares current state to desired state
func (s Server) Compare(ctx context.Context, in *pb.Empty) (*pb.CompareResponse, error) {
	resp := &pb.CompareResponse{}
	list, _ := getFleetStatus(&mainChecker{logger: s.Log})
	cc := &pb.Config{}
	for _, jlist := range list {
		for _, job := range jlist.GetDetails() {
			cc.Intents = append(cc.Intents, &pb.Intent{Spec: job.GetSpec()})
		}
	}
	resp.Current = cc
	resp.Desired = s.config

	return resp, nil
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Count++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Count: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s *Server) MatchIntent() {
	checker := &mainChecker{logger: s.Log}
	for s.serving {
		time.Sleep(intentWait)
		s.LastIntent = time.Now()

		state := getConfig(checker)
		diff := configDiff(s.config, state)
		joblist := runJobs(diff)
		for _, job := range joblist {
			runJob(job, chooseServer(job, checker))
		}
	}
}

// SetMaster sets up the master settings
func (s *Server) SetMaster() {
	t := time.Now()
	checker := &mainChecker{logger: s.Log}
	s.LastMaster = time.Now()

	fleet := checker.discover()
	matcher := make(map[string][]*pbd.RegistryEntry)
	hasMaster := make(map[string]int)
	for _, entry := range fleet.GetServices() {
		if !entry.GetIgnoresMaster() {
			if _, ok := matcher[entry.GetName()]; !ok {
				if entry.GetMaster() {
					hasMaster[entry.GetName()]++
				}
				matcher[entry.GetName()] = []*pbd.RegistryEntry{entry}
			} else {
				if entry.GetMaster() {
					hasMaster[entry.GetName()] = 1
					matcher[entry.GetName()] = append(matcher[entry.GetName()], entry)
				}
			}
		}
	}

	for key, entries := range matcher {
		if hasMaster[key] > 1 {
			hasMaster[key] = 1
			seen := false
			for _, entry := range entries {
				if seen && entry.GetMaster() {
					checker.master(entry, false)
				} else if entry.GetMaster() {
					seen = true
				}
			}
		}

		if hasMaster[key] == 0 {
			for _, entry := range entries {
				if checker.master(entry, true) {
					entry.Master = true
					break
				}
			}
		}
	}
	s.LogFunction("SetMasterRun", t)
}

//Init builds up the server
func Init(config *pb.Config) *Server {
	s := &Server{
		&goserver.GoServer{},
		config,
		true,
		time.Now(),
		time.Now(),
		&sync.Mutex{},
		make(map[string]map[string]struct{}),
		&prodGetter{},
	}
	return s
}

func (s *Server) becomeMaster() {
	for true {
		time.Sleep(time.Minute)
		_, _, err := utils.Resolve("gobuildmaster")
		if err != nil {
			s.Registry.Master = true
		}
	}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}

	s := Init(config)

	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	s.Register = s
	s.PrepServer()
	s.GoServer.Killme = false
	s.RegisterServer("gobuildmaster", false)
	s.RegisterServingTask(s.MatchIntent)
	s.RegisterServingTask(s.becomeMaster)
	s.RegisterRepeatingTask(s.SetMaster, time.Second)
	s.RegisterRepeatingTask(s.buildWorld, time.Minute)

	go s.checkerThread(s.config.GetIntents()[0])

	err = s.Serve()
	if err != nil {
		log.Fatalf("Serve error: %v", err)
	}
}
<|endoftext|>