<commit_before>package main

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"time"
)

func pingPong(conn net.Conn, m int, buf []byte) (d time.Duration, err error) {
	var n int
	var b [16]byte
	if len(buf) != 16 {
		err = fmt.Errorf("invalid buffer size")
		return
	}
	start := time.Now()
	for i := 0; i < m; i++ {
		n, err = conn.Write(buf)
		if err != nil {
			return
		}
		_, err = io.ReadFull(conn, b[:n])
		if err != nil {
			return
		}
	}
	d = time.Since(start)
	if !bytes.Equal(buf[:n], b[:n]) {
		err = fmt.Errorf("Wrong content")
		return
	}
	return
}

type result struct {
	d   time.Duration
	err error
}

func Client(addr string, buf []byte, n int, start <-chan bool, stop <-chan bool, resChan chan<- *result) {
	<-start
	res := new(result)
	var conn net.Conn
	conn, res.err = net.Dial("tcp", addr)
	if res.err != nil {
		resChan <- res
	}
	defer conn.Close()
	res.d, res.err = pingPong(conn, n, buf)
	resChan <- res
}

type BenchClient struct {
	N       int
	M       int
	Addr    string
	start   chan bool
	stop    chan bool
	resChan chan *result
	out     io.Writer
}

func (self *BenchClient) Connect() error {
	if self.start == nil {
		self.start = make(chan bool)
	}
	if self.stop == nil {
		self.stop = make(chan bool)
	}
	if self.resChan == nil {
		self.resChan = make(chan *result)
	}
	if self.M <= 0 {
		self.M = 1
	}
	var buf [16]byte
	_, err := io.ReadFull(rand.Reader, buf[:16])
	if err != nil {
		return err
	}
	for i := 0; i < self.N; i++ {
		go Client(self.Addr, buf[:], self.M, self.start, self.stop, self.resChan)
	}
	return nil
}

func (self *BenchClient) collectResults() {
	if self.out == nil {
		self.out = os.Stdout
	}
	for r := range self.resChan {
		if r.err != nil {
			fmt.Fprintf(self.out, "Failed\n")
		} else {
			fmt.Fprintf(self.out, "%v\n", r.d.Seconds())
		}
	}
}

func (self *BenchClient) Start() {
	go self.collectResults()
	close(self.start)
}

var argvNrConn = flag.Int("n", 10, "number of concurrent connections")
var argvNrMsg = flag.Int("m", 10, "number of messages per connection")
var argvServAddr = flag.String("addr", "127.0.0.1:8080", "server address")
var argvOut = flag.String("o", "", "output file name")

func main() {
	flag.Parse()
	r := bufio.NewReader(os.Stdin)
	b := new(BenchClient)
	b.Addr = *argvServAddr
	b.N = *argvNrConn
	b.M = *argvNrMsg
	if len(*argvOut) > 0 {
		f, err := os.Create(*argvOut)
		if err != nil {
			fmt.Fprint(os.Stderr, "cannot create file: %v\n", err)
			return
		}
		b.out = f
	}

	fmt.Printf("Ready to start the connections? [Enter] ")
	r.ReadLine()
	b.Connect()

	fmt.Printf("Ready to start sending? [Enter] ")
	r.ReadLine()
	b.Start()

	fmt.Printf("Hit Enter to stop")
	r.ReadLine()
}
<commit_msg>fixed some minor bug<commit_after>package main

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"time"
)

func pingPong(conn net.Conn, m int, buf []byte) (d time.Duration, err error) {
	var n int
	var b [16]byte
	start := time.Now()
	for i := 0; i < m; i++ {
		n, err = conn.Write(buf[:])
		if err != nil {
			return
		}
		_, err = io.ReadFull(conn, b[:n])
		if err != nil {
			return
		}
	}
	d = time.Since(start)
	if !bytes.Equal(buf[:n], b[:n]) {
		err = fmt.Errorf("Wrong content")
		return
	}
	return
}

type result struct {
	d   time.Duration
	err error
}

func Client(addr string, buf []byte, n int, start <-chan bool, stop <-chan bool, resChan chan<- *result) {
	<-start
	res := new(result)
	var conn net.Conn
	conn, res.err = net.Dial("tcp", addr)
	if res.err != nil {
		resChan <- res
		return
	}
	defer conn.Close()
	res.d, res.err = pingPong(conn, n, buf)
	resChan <- res
}

type BenchClient struct {
	N       int
	M       int
	Addr    string
	start   chan bool
	stop    chan bool
	resChan chan *result
	out     io.Writer
}

func (self *BenchClient) Connect() error {
	if self.start == nil {
		self.start = make(chan bool)
	}
	if self.stop == nil {
		self.stop = make(chan bool)
	}
	if self.resChan == nil {
		self.resChan = make(chan *result)
	}
	if self.M <= 0 {
		self.M = 1
	}
	var buf [16]byte
	_, err := io.ReadFull(rand.Reader, buf[:16])
	if err != nil {
		return err
	}
	for i := 0; i < self.N; i++ {
		go Client(self.Addr, buf[:], self.M, self.start, self.stop, self.resChan)
	}
	return nil
}

func (self *BenchClient) collectResults() {
	if self.out == nil {
		self.out = os.Stdout
	}
	for r := range self.resChan {
		if r.err != nil {
			fmt.Fprintf(self.out, "Failed: %v\n", r.err)
		} else {
			fmt.Fprintf(self.out, "%v\n", r.d.Seconds())
		}
	}
}

func (self *BenchClient) Start() {
	go self.collectResults()
	close(self.start)
}

var argvNrConn = flag.Int("n", 10, "number of concurrent connections")
var argvNrMsg = flag.Int("m", 10, "number of messages per connection")
var argvServAddr = flag.String("addr", "127.0.0.1:8080", "server address")
var argvOut = flag.String("o", "", "output file name")

func main() {
	flag.Parse()
	r := bufio.NewReader(os.Stdin)
	b := new(BenchClient)
	b.Addr = *argvServAddr
	b.N = *argvNrConn
	b.M = *argvNrMsg
	if len(*argvOut) > 0 {
		f, err := os.Create(*argvOut)
		if err != nil {
			fmt.Fprint(os.Stderr, "cannot create file: %v\n", err)
			return
		}
		b.out = f
	}

	fmt.Printf("Ready to start the connections? [Enter] ")
	r.ReadLine()
	b.Connect()

	fmt.Printf("Ready to start sending? [Enter] ")
	r.ReadLine()
	b.Start()

	fmt.Printf("Hit Enter to stop")
	r.ReadLine()
}
<|endoftext|>


<commit_before>package radix

import (
	"errors"
	"net"
	"sync"
	"time"

	"github.com/mediocregopher/radix.v3/resp"
)

// ErrPoolEmpty is used by Pools created using the PoolOnEmptyErrAfter option
var ErrPoolEmpty = errors.New("connection pool is empty")

// TODO do something with errors which happen asynchronously

type staticPoolConn struct {
	Conn
	sp *Pool

	// The most recent network error which occurred when either reading
	// or writing. A critical network error is basically any non-application
	// level error, e.g. a timeout, disconnect, etc... Close is automatically
	// called on the client when it encounters a critical network error
	lastIOErr error
}

func (spc *staticPoolConn) Encode(m resp.Marshaler) error {
	err := spc.Conn.Encode(m)
	if nerr, _ := err.(net.Error); nerr != nil {
		spc.lastIOErr = err
	}
	return err
}

func (spc *staticPoolConn) Decode(m resp.Unmarshaler) error {
	err := spc.Conn.Decode(m)
	if nerr, _ := err.(net.Error); nerr != nil {
		spc.lastIOErr = err
	}
	return err
}

func (spc *staticPoolConn) Do(a Action) error {
	return a.Run(spc)
}

func (spc *staticPoolConn) Close() error {
	return spc.Conn.Close()
}

////////////////////////////////////////////////////////////////////////////////

type poolOpts struct {
	cf                    ConnFunc
	pingInterval          time.Duration
	refillInterval        time.Duration
	overflowDrainInterval time.Duration
	overflowSize          int
	onEmptyErr            bool
	onEmptyWait           time.Duration
}

// PoolOpt is an optional behavior which can be applied to the NewPool function
// to effect a Pool's behavior
type PoolOpt func(*poolOpts)

// PoolConnFunc tells the Pool to use the given ConnFunc when creating new
// Conns to its redis instance. The ConnFunc can be used to set timeouts,
// perform AUTH, or even use custom Conn implementations.
func PoolConnFunc(cf ConnFunc) PoolOpt {
	return func(po *poolOpts) {
		po.cf = cf
	}
}

// PoolPingInterval specifies the interval at which a ping event happens. On
// each ping event the Pool calls the PING redis command over one of it's
// available connections.
//
// Since connections are used in LIFO order, the ping interval * pool size is
// the duration of time it takes to ping every connection once when the pool is
// idle.
//
// A shorter interval means connections are pinged more frequently, but also
// means more traffic with the server.
func PoolPingInterval(d time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.pingInterval = d
	}
}

// PoolRefillInterval specifies the interval at which a refill event happens. On
// each refill event the Pool checks to see if it is full, and if it's not a
// single connection is created and added to it.
func PoolRefillInterval(d time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.refillInterval = d
	}
}

// PoolOnEmptyWait effects the Pool's behavior when there are no available
// connections in the Pool. The effect is to cause actions to block as long as
// it takes until a connection becomes available.
func PoolOnEmptyWait() PoolOpt {
	return func(po *poolOpts) {
		po.onEmptyWait = -1
	}
}

// PoolOnEmptyCreateAfter effects the Pool's behavior when there are no
// available connections in the Pool. The effect is to cause actions to block
// until a connection becomes available or until the duration has passed. If the
// duration is passed a new connection is created and used.
//
// If wait is 0 then a new connection is created immediately upon an empty Pool.
func PoolOnEmptyCreateAfter(wait time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.onEmptyWait = wait
		po.onEmptyErr = false
	}
}

// PoolOnEmptyErrAfter effects the Pool's behavior when there are no
// available connections in the Pool. The effect is to cause actions to block
// until a connection becomes available or until the duration has passed. If the
// duration is passed then ErrEmptyPool is returned.
//
// If wait is 0 then ErrEmptyPool is returned immediately upon an empty Pool.
func PoolOnEmptyErrAfter(wait time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.onEmptyWait = wait
		po.onEmptyErr = true
	}
}

// PoolOnFullClose effects the Pool's behavior when it is full. The effect is to
// cause any connection which is being put back into a full pool to be closed
// and discarded.
func PoolOnFullClose() PoolOpt {
	return func(po *poolOpts) {
		po.overflowSize = 0
		po.overflowDrainInterval = 0
	}
}

// PoolOnFullBuffer effects the Pool's behavior when it is full. The effect is
// to cause any connection which is being put back into a full pool to be put
// instead into an overflow buffer which can hold up to the given number of
// connections. If the overflow buffer is also full then the connection is
// closed and discarded.
//
// drainInterval specifies the interval at which a drain event happens. On each
// drain event a connection is removed from the overflow buffer and put into the
// pool. If the pool is full the connection is closed and discarded.
//
// When Actions are performed with the Pool the connection used may come from
// either the main pool or the overflow buffer. Connections do _not_ have to
// wait to be drained into the main pool before they will be used.
func PoolOnFullBuffer(size int, drainInterval time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.overflowSize = size
		po.overflowDrainInterval = drainInterval
	}
}

// Pool is a semi-dynamic pool which holds a fixed number of connections open
// and which implements the Client interface. It takes in a number of options
// which can effect its specific behavior, see the NewPool method.
type Pool struct {
	po            poolOpts
	network, addr string

	l              sync.RWMutex
	pool, overflow chan *staticPoolConn
	closed         bool

	wg       sync.WaitGroup
	closeCh  chan bool
	initDone chan struct{} // used for tests
}

// NewPool creates a *Pool which will hold up to the given number of connections
// to the redis instance at the given address.
//
// NewPool takes in a number of options which can overwrite its default
// behavior. The default options NewPool uses are:
//
//	PoolConnFunc(Dial)
//	PoolOnEmptyCreateAfter(1 * time.Second)
//	PoolRefillInterval(1 * time.Second)
//	PoolOnFullClose()
//	PoolPingInterval(10 * time.Second / size)
//
func NewPool(network, addr string, size int, opts ...PoolOpt) (*Pool, error) {
	sp := &Pool{
		network:  network,
		addr:     addr,
		pool:     make(chan *staticPoolConn, size),
		closeCh:  make(chan bool),
		initDone: make(chan struct{}),
	}

	defaultPoolOpts := []PoolOpt{
		PoolConnFunc(Dial),
		PoolOnEmptyCreateAfter(1 * time.Second),
		PoolRefillInterval(1 * time.Second),
		PoolOnFullClose(),
	}
	// if pool size is 0 don't start up a pingSpin, cause there'd be no point
	if size > 0 {
		pingOpt := PoolPingInterval(10 * time.Second / time.Duration(size))
		defaultPoolOpts = append(defaultPoolOpts, pingOpt)
	}

	for _, opt := range append(defaultPoolOpts, opts...) {
		// the other args to NewPool used to be a ConnFunc, which someone might
		// have left as nil, in which case this now gives a weird panic. Just
		// handle it
		if opt != nil {
			opt(&(sp.po))
		}
	}

	// make one Conn synchronously to ensure there's actually a redis instance
	// present. The rest will be created asynchronously
	spc, err := sp.newConn()
	if err != nil {
		return nil, err
	}
	sp.put(spc)

	sp.wg.Add(1)
	go func() {
		defer sp.wg.Done()
		for i := 0; i < size-1; i++ {
			spc, err := sp.newConn()
			if err == nil {
				sp.put(spc)
			}
		}
		close(sp.initDone)
	}()

	if sp.po.pingInterval > 0 {
		sp.atIntervalDo(sp.po.pingInterval, func() { sp.Do(Cmd(nil, "PING")) })
	}
	if sp.po.refillInterval > 0 && size > 0 {
		sp.atIntervalDo(sp.po.refillInterval, sp.doRefill)
	}
	if sp.po.overflowDrainInterval > 0 && sp.po.overflowSize > 0 {
		sp.overflow = make(chan *staticPoolConn, sp.po.overflowSize)
		sp.atIntervalDo(sp.po.overflowDrainInterval, sp.doOverflowDrain)
	}
	return sp, nil
}

func (sp *Pool) newConn() (*staticPoolConn, error) {
	c, err := sp.po.cf(sp.network, sp.addr)
	if err != nil {
		return nil, err
	}

	spc := &staticPoolConn{
		Conn: c,
		sp:   sp,
	}
	return spc, nil
}

func (sp *Pool) atIntervalDo(d time.Duration, do func()) {
	sp.wg.Add(1)
	go func() {
		defer sp.wg.Done()
		t := time.NewTicker(d)
		defer t.Stop()
		for {
			select {
			case <-t.C:
				do()
			case <-sp.closeCh:
				return
			}
		}
	}()
}

func (sp *Pool) doRefill() {
	if len(sp.pool) == cap(sp.pool) {
		return
	}
	spc, err := sp.newConn()
	if err == nil {
		sp.put(spc)
	}
}

func (sp *Pool) doOverflowDrain() {
	// If the overflow has a connection we first try to move it to the main
	// pool, but if that's full then just close the connection
	select {
	case spc := <-sp.overflow:
		select {
		case sp.pool <- spc:
		default:
			spc.Close()
		}
	default:
	}
}

func (sp *Pool) get() (*staticPoolConn, error) {
	sp.l.RLock()
	defer sp.l.RUnlock()
	if sp.closed {
		return nil, errClientClosed
	}

	// if an error is written to waitCh get returns that, otherwise if it's
	// closed get will make a new connection
	waitCh := make(chan error, 1)
	effectWaitCh := func() {
		if sp.po.onEmptyErr {
			waitCh <- ErrPoolEmpty
		} else {
			close(waitCh)
		}
	}

	if sp.po.onEmptyWait == -1 {
		// block, waitCh is never effected
	} else if sp.po.onEmptyWait == 0 {
		effectWaitCh()
	} else {
		// TODO it might be worthwhile to use a sync.Pool for timers, rather
		// than creating a new one for every single get
		t := time.AfterFunc(sp.po.onEmptyWait, effectWaitCh)
		defer t.Stop()
	}

	select {
	case spc := <-sp.pool:
		return spc, nil
	case spc := <-sp.overflow:
		return spc, nil
	case err := <-waitCh:
		if err != nil {
			return nil, err
		}
		return sp.newConn()
	}
}

func (sp *Pool) put(spc *staticPoolConn) {
	sp.l.RLock()
	defer sp.l.RUnlock()
	if spc.lastIOErr != nil || sp.closed {
		spc.Close()
		return
	}

	select {
	case sp.pool <- spc:
	default:
		select {
		case sp.overflow <- spc:
		default:
			spc.Close()
		}
	}
}

// Do implements the Do method of the Client interface by retrieving a Conn out
// of the pool, calling Run on the given Action with it, and returning the Conn
// to the pool
func (sp *Pool) Do(a Action) error {
	c, err := sp.get()
	if err != nil {
		return err
	}
	defer sp.put(c)

	return c.Do(a)
}

// NumAvailConns returns the number of connections currently available in the
// pool, as well as in the overflow buffer if that option is enabled.
func (sp *Pool) NumAvailConns() int {
	return len(sp.pool) + len(sp.overflow)
}

// Close implements the Close method of the Client
func (sp *Pool) Close() error {
	sp.l.Lock()
	if sp.closed {
		return errClientClosed
	}
	sp.closed = true
	close(sp.closeCh)
	sp.l.Unlock()

	// at this point get and put won't work anymore, so it's safe to empty and
	// close the pool channel
emptyLoop:
	for {
		select {
		case spc := <-sp.pool:
			spc.Close()
		default:
			close(sp.pool)
			break emptyLoop
		}
	}

	// by now the pool's go-routines should have bailed, wait to make sure they
	// do
	sp.wg.Wait()
	return nil
}
<commit_msg>fix pool closing and a race condition between closing and the overflow drain<commit_after>package radix

import (
	"errors"
	"net"
	"sync"
	"time"

	"github.com/mediocregopher/radix.v3/resp"
)

// ErrPoolEmpty is used by Pools created using the PoolOnEmptyErrAfter option
var ErrPoolEmpty = errors.New("connection pool is empty")

// TODO do something with errors which happen asynchronously

type staticPoolConn struct {
	Conn
	sp *Pool

	// The most recent network error which occurred when either reading
	// or writing. A critical network error is basically any non-application
	// level error, e.g. a timeout, disconnect, etc... Close is automatically
	// called on the client when it encounters a critical network error
	lastIOErr error
}

func (spc *staticPoolConn) Encode(m resp.Marshaler) error {
	err := spc.Conn.Encode(m)
	if nerr, _ := err.(net.Error); nerr != nil {
		spc.lastIOErr = err
	}
	return err
}

func (spc *staticPoolConn) Decode(m resp.Unmarshaler) error {
	err := spc.Conn.Decode(m)
	if nerr, _ := err.(net.Error); nerr != nil {
		spc.lastIOErr = err
	}
	return err
}

func (spc *staticPoolConn) Do(a Action) error {
	return a.Run(spc)
}

func (spc *staticPoolConn) Close() error {
	return spc.Conn.Close()
}

////////////////////////////////////////////////////////////////////////////////

type poolOpts struct {
	cf                    ConnFunc
	pingInterval          time.Duration
	refillInterval        time.Duration
	overflowDrainInterval time.Duration
	overflowSize          int
	onEmptyErr            bool
	onEmptyWait           time.Duration
}

// PoolOpt is an optional behavior which can be applied to the NewPool function
// to effect a Pool's behavior
type PoolOpt func(*poolOpts)

// PoolConnFunc tells the Pool to use the given ConnFunc when creating new
// Conns to its redis instance. The ConnFunc can be used to set timeouts,
// perform AUTH, or even use custom Conn implementations.
func PoolConnFunc(cf ConnFunc) PoolOpt {
	return func(po *poolOpts) {
		po.cf = cf
	}
}

// PoolPingInterval specifies the interval at which a ping event happens. On
// each ping event the Pool calls the PING redis command over one of it's
// available connections.
//
// Since connections are used in LIFO order, the ping interval * pool size is
// the duration of time it takes to ping every connection once when the pool is
// idle.
//
// A shorter interval means connections are pinged more frequently, but also
// means more traffic with the server.
func PoolPingInterval(d time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.pingInterval = d
	}
}

// PoolRefillInterval specifies the interval at which a refill event happens. On
// each refill event the Pool checks to see if it is full, and if it's not a
// single connection is created and added to it.
func PoolRefillInterval(d time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.refillInterval = d
	}
}

// PoolOnEmptyWait effects the Pool's behavior when there are no available
// connections in the Pool. The effect is to cause actions to block as long as
// it takes until a connection becomes available.
func PoolOnEmptyWait() PoolOpt {
	return func(po *poolOpts) {
		po.onEmptyWait = -1
	}
}

// PoolOnEmptyCreateAfter effects the Pool's behavior when there are no
// available connections in the Pool. The effect is to cause actions to block
// until a connection becomes available or until the duration has passed. If the
// duration is passed a new connection is created and used.
//
// If wait is 0 then a new connection is created immediately upon an empty Pool.
func PoolOnEmptyCreateAfter(wait time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.onEmptyWait = wait
		po.onEmptyErr = false
	}
}

// PoolOnEmptyErrAfter effects the Pool's behavior when there are no
// available connections in the Pool. The effect is to cause actions to block
// until a connection becomes available or until the duration has passed. If the
// duration is passed then ErrEmptyPool is returned.
//
// If wait is 0 then ErrEmptyPool is returned immediately upon an empty Pool.
func PoolOnEmptyErrAfter(wait time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.onEmptyWait = wait
		po.onEmptyErr = true
	}
}

// PoolOnFullClose effects the Pool's behavior when it is full. The effect is to
// cause any connection which is being put back into a full pool to be closed
// and discarded.
func PoolOnFullClose() PoolOpt {
	return func(po *poolOpts) {
		po.overflowSize = 0
		po.overflowDrainInterval = 0
	}
}

// PoolOnFullBuffer effects the Pool's behavior when it is full. The effect is
// to cause any connection which is being put back into a full pool to be put
// instead into an overflow buffer which can hold up to the given number of
// connections. If the overflow buffer is also full then the connection is
// closed and discarded.
//
// drainInterval specifies the interval at which a drain event happens. On each
// drain event a connection is removed from the overflow buffer and put into the
// pool. If the pool is full the connection is closed and discarded.
//
// When Actions are performed with the Pool the connection used may come from
// either the main pool or the overflow buffer. Connections do _not_ have to
// wait to be drained into the main pool before they will be used.
func PoolOnFullBuffer(size int, drainInterval time.Duration) PoolOpt {
	return func(po *poolOpts) {
		po.overflowSize = size
		po.overflowDrainInterval = drainInterval
	}
}

// Pool is a semi-dynamic pool which holds a fixed number of connections open
// and which implements the Client interface. It takes in a number of options
// which can effect its specific behavior, see the NewPool method.
type Pool struct {
	po            poolOpts
	network, addr string

	l              sync.RWMutex
	pool, overflow chan *staticPoolConn
	closed         bool

	wg       sync.WaitGroup
	closeCh  chan bool
	initDone chan struct{} // used for tests
}

// NewPool creates a *Pool which will hold up to the given number of connections
// to the redis instance at the given address.
//
// NewPool takes in a number of options which can overwrite its default
// behavior. The default options NewPool uses are:
//
//	PoolConnFunc(Dial)
//	PoolOnEmptyCreateAfter(1 * time.Second)
//	PoolRefillInterval(1 * time.Second)
//	PoolOnFullClose()
//	PoolPingInterval(10 * time.Second / size)
//
func NewPool(network, addr string, size int, opts ...PoolOpt) (*Pool, error) {
	sp := &Pool{
		network:  network,
		addr:     addr,
		pool:     make(chan *staticPoolConn, size),
		closeCh:  make(chan bool),
		initDone: make(chan struct{}),
	}

	defaultPoolOpts := []PoolOpt{
		PoolConnFunc(Dial),
		PoolOnEmptyCreateAfter(1 * time.Second),
		PoolRefillInterval(1 * time.Second),
		PoolOnFullClose(),
	}
	// if pool size is 0 don't start up a pingSpin, cause there'd be no point
	if size > 0 {
		pingOpt := PoolPingInterval(10 * time.Second / time.Duration(size))
		defaultPoolOpts = append(defaultPoolOpts, pingOpt)
	}

	for _, opt := range append(defaultPoolOpts, opts...) {
		// the other args to NewPool used to be a ConnFunc, which someone might
		// have left as nil, in which case this now gives a weird panic. Just
		// handle it
		if opt != nil {
			opt(&(sp.po))
		}
	}

	// make one Conn synchronously to ensure there's actually a redis instance
	// present. The rest will be created asynchronously
	spc, err := sp.newConn()
	if err != nil {
		return nil, err
	}
	sp.put(spc)

	sp.wg.Add(1)
	go func() {
		defer sp.wg.Done()
		for i := 0; i < size-1; i++ {
			spc, err := sp.newConn()
			if err == nil {
				sp.put(spc)
			}
		}
		close(sp.initDone)
	}()

	if sp.po.pingInterval > 0 {
		sp.atIntervalDo(sp.po.pingInterval, func() { sp.Do(Cmd(nil, "PING")) })
	}
	if sp.po.refillInterval > 0 && size > 0 {
		sp.atIntervalDo(sp.po.refillInterval, sp.doRefill)
	}
	if sp.po.overflowDrainInterval > 0 && sp.po.overflowSize > 0 {
		sp.overflow = make(chan *staticPoolConn, sp.po.overflowSize)
		sp.atIntervalDo(sp.po.overflowDrainInterval, sp.doOverflowDrain)
	}
	return sp, nil
}

func (sp *Pool) newConn() (*staticPoolConn, error) {
	c, err := sp.po.cf(sp.network, sp.addr)
	if err != nil {
		return nil, err
	}

	spc := &staticPoolConn{
		Conn: c,
		sp:   sp,
	}
	return spc, nil
}

func (sp *Pool) atIntervalDo(d time.Duration, do func()) {
	sp.wg.Add(1)
	go func() {
		defer sp.wg.Done()
		t := time.NewTicker(d)
		defer t.Stop()
		for {
			select {
			case <-t.C:
				do()
			case <-sp.closeCh:
				return
			}
		}
	}()
}

func (sp *Pool) doRefill() {
	if len(sp.pool) == cap(sp.pool) {
		return
	}
	spc, err := sp.newConn()
	if err == nil {
		sp.put(spc)
	}
}

func (sp *Pool) doOverflowDrain() {
	// the other do* processes inherently handle this case, this one needs to do
	// it manually
	sp.l.RLock()
	defer sp.l.RUnlock()
	if sp.closed {
		return
	}

	// If the overflow has a connection we first try to move it to the main
	// pool, but if that's full then just close the connection
	select {
	case spc := <-sp.overflow:
		select {
		case sp.pool <- spc:
		default:
			spc.Close()
		}
	default:
	}
}

func (sp *Pool) get() (*staticPoolConn, error) {
	sp.l.RLock()
	defer sp.l.RUnlock()
	if sp.closed {
		return nil, errClientClosed
	}

	// if an error is written to waitCh get returns that, otherwise if it's
	// closed get will make a new connection
	waitCh := make(chan error, 1)
	effectWaitCh := func() {
		if sp.po.onEmptyErr {
			waitCh <- ErrPoolEmpty
		} else {
			close(waitCh)
		}
	}

	if sp.po.onEmptyWait == -1 {
		// block, waitCh is never effected
	} else if sp.po.onEmptyWait == 0 {
		effectWaitCh()
	} else {
		// TODO it might be worthwhile to use a sync.Pool for timers, rather
		// than creating a new one for every single get
		t := time.AfterFunc(sp.po.onEmptyWait, effectWaitCh)
		defer t.Stop()
	}

	select {
	case spc := <-sp.pool:
		return spc, nil
	case spc := <-sp.overflow:
		return spc, nil
	case err := <-waitCh:
		if err != nil {
			return nil, err
		}
		return sp.newConn()
	}
}

func (sp *Pool) put(spc *staticPoolConn) {
	sp.l.RLock()
	defer sp.l.RUnlock()
	if spc.lastIOErr != nil || sp.closed {
		spc.Close()
		return
	}

	select {
	case sp.pool <- spc:
	default:
		select {
		case sp.overflow <- spc:
		default:
			spc.Close()
		}
	}
}

// Do implements the Do method of the Client interface by retrieving a Conn out
// of the pool, calling Run on the given Action with it, and returning the Conn
// to the pool
func (sp *Pool) Do(a Action) error {
	c, err := sp.get()
	if err != nil {
		return err
	}
	defer sp.put(c)

	return c.Do(a)
}

// NumAvailConns returns the number of connections currently available in the
// pool, as well as in the overflow buffer if that option is enabled.
func (sp *Pool) NumAvailConns() int {
	return len(sp.pool) + len(sp.overflow)
}

// Close implements the Close method of the Client
func (sp *Pool) Close() error {
	sp.l.Lock()
	if sp.closed {
		return errClientClosed
	}
	sp.closed = true
	close(sp.closeCh)
	sp.l.Unlock()

	// at this point get and put won't work anymore, so it's safe to empty and
	// close the pool channel
emptyLoop:
	for {
		select {
		case spc := <-sp.pool:
			spc.Close()
		case spc := <-sp.overflow:
			spc.Close()
		default:
			close(sp.pool)
			if sp.overflow != nil {
				close(sp.overflow)
			}
			break emptyLoop
		}
	}

	// by now the pool's go-routines should have bailed, wait to make sure they
	// do
	sp.wg.Wait()
	return nil
}
<|endoftext|>


<commit_before>package bridgeproxy

import "fmt"
import "net"
import "os"
import "io"
import "crypto/tls"

type Configuration struct {
	Local      string
	Bridge     string
	RemoteName string
	RemotePort string
}

func forward(src io.Reader, dst io.Writer) {
	for {
		n, err := io.Copy(dst, src)
		if n == 0 {
			break
		}
		if err != nil {
			fmt.Println("Could not forward:", err)
			break
		}
		fmt.Println("Forwarded", n, "bytes from", src, "to", dst)
	}
}

func handleRequest(browser net.Conn, item Configuration) {
	fmt.Println("handleRequest")
	conn, err := net.Dial("tcp", item.Bridge)
	if err != nil {
		fmt.Println("ERROR: Could not connect", err)
		return
	}
	fmt.Fprintf(conn, "CONNECT %s:%s HTTP/1.0\r\n\r\n\r\n", item.RemoteName, item.RemotePort)

	// Read the "HTTP/1.0 200 Connection established" and the 2 \r\n
	_, err = io.ReadFull(conn, make([]byte, 39))
	if err != nil {
		fmt.Println("Could not read:", err)
		return
	}

	// We now have access to the TLS connection.
	client := tls.Client(conn, &tls.Config{ServerName: item.RemoteName})

	// Forward traffic between the client connected to us and the remote proxy
	go forward(browser, client)
	go forward(client, browser)
}

func Serve(item Configuration) {
	// Listen for incoming connections.
	l, err := net.Listen("tcp", item.Local)
	if err != nil {
		fmt.Println("Error listening:", err.Error())
		os.Exit(1)
	}
	// Close the listener when the application closes.
	defer l.Close()
	fmt.Println("Listening on", item.Local)
	fmt.Println("- Forwarding requests to", item.RemoteName, "port", item.RemotePort, "via", item.Bridge)
	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			fmt.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// Handle connections in a new goroutine.
		go handleRequest(conn, item)
	}
}
<commit_msg>Handle closing connections<commit_after>package bridgeproxy

import "fmt"
import "net"
import "os"
import "io"
import "crypto/tls"

type Configuration struct {
	Local      string
	Bridge     string
	RemoteName string
	RemotePort string
}

func forward(src net.Conn, dst net.Conn) {
	defer src.Close()
	defer dst.Close()
	for {
		n, err := io.Copy(dst, src)
		if n == 0 {
			break
		}
		if err != nil {
			fmt.Println("Could not forward:", err)
			break
		}
		fmt.Println("Forwarded", n, "bytes from", src, "to", dst)
	}
}

func handleRequest(browser net.Conn, item Configuration) {
	fmt.Println("handleRequest")
	conn, err := net.Dial("tcp", item.Bridge)
	if err != nil {
		fmt.Println("ERROR: Could not connect", err)
		return
	}
	fmt.Fprintf(conn, "CONNECT %s:%s HTTP/1.0\r\n\r\n\r\n", item.RemoteName, item.RemotePort)

	// Read the "HTTP/1.0 200 Connection established" and the 2 \r\n
	_, err = io.ReadFull(conn, make([]byte, 39))
	if err != nil {
		fmt.Println("Could not read:", err)
		return
	}

	// We now have access to the TLS connection.
	client := tls.Client(conn, &tls.Config{ServerName: item.RemoteName})

	// Forward traffic between the client connected to us and the remote proxy
	go forward(browser, client)
	go forward(client, browser)
}

func Serve(item Configuration) {
	// Listen for incoming connections.
	l, err := net.Listen("tcp", item.Local)
	if err != nil {
		fmt.Println("Error listening:", err.Error())
		os.Exit(1)
	}
	// Close the listener when the application closes.
	defer l.Close()
	fmt.Println("Listening on", item.Local)
	fmt.Println("- Forwarding requests to", item.RemoteName, "port", item.RemotePort, "via", item.Bridge)
	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			fmt.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// Handle connections in a new goroutine.
		go handleRequest(conn, item)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"flag"
	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	"github.com/eaciit/toolkit"
	"os"
	"os/exec"
	"runtime"
	"strings"
)

var (
	conn           dbox.IConnection
	tablegroup     string
	tablename      string
	source         string
	dest           string
	tablegrouplist = []string{"allpl", "alloutlet"}
	sourcedest     = []string{"devel", "prod", "ba"}
)

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func main() {

	setinitialconnection()
	defer gdrj.CloseDb()

	flag.StringVar(&tablegroup, "tablegroup", "", "group of collection")
	flag.StringVar(&tablename, "tablename", "", "collection name")
	flag.StringVar(&source, "from", "", "source location of dumped collection")
	flag.StringVar(&dest, "to", "", "destination location of restored collection")
	flag.Parse()

	copycollection()
}

func copycollection() {
	conn, _ := modules.GetDboxIConnection("db_godrej")
	defer conn.Close()
	if source == "" || dest == "" {
		errmsg := ""
		if source == "" {
			errmsg = "source location of dumped collection"
		} else {
			errmsg = "destination location of restored collection"
		}
		toolkit.Println("\nPlease fill parameter for", errmsg)
		return
	} else {
		if !toolkit.HasMember(sourcedest, strings.ToLower(source)) {
			toolkit.Println("\nsource location is not valid, choose the valid parameter below :")
			toolkit.Println("ba\tto dump collection from go.eaciit.com:27123/ecgodrej")
			toolkit.Println("devel\tto dump collection from 52.220.25.190:27123/ecgodrej")
			toolkit.Println("prod\tto dump collection from go.eaciit.com:27123/ecgodrej_prod")
			return
		} else if !toolkit.HasMember(sourcedest, strings.ToLower(dest)) {
			toolkit.Println("\ndestination location is not valid")
			toolkit.Println("ba\tto restore collection into go.eaciit.com:27123/ecgodrej")
			toolkit.Println("devel\tto restore collection into 52.220.25.190:27123/ecgodrej")
			toolkit.Println("prod\tto restore collection into go.eaciit.com:27123/ecgodrej_prod")
			return
		}
	}

	if tablegroup == "" && tablename == "" {
		toolkit.Println("\nPlease fill parameter for tablegroup or tablename")
		return
	} else if tablegroup != "" {
		if !toolkit.HasMember(tablegrouplist, strings.ToLower(tablegroup)) {
			toolkit.Println("\ntablegroup is not valid, choose the valid parameter below :")
			toolkit.Println("allpl\t\tto dump-restore all pl collection")
			toolkit.Println("alloutlet\tto dump-restore all outlet collection")
			toolkit.Println("all\t\tto dump-restore both all pl collection and all outlet collection")
			return
		}
	}

	tablelist := conn.ObjectNames(dbox.ObjTypeTable)
	errmsg := ""

	listofcol := []string{}
	if tablename != "" {
		_tablelist := []string{}
		for _, col := range tablelist {
			_tablelist = append(_tablelist, strings.ToLower(col))
		}
		if !toolkit.HasMember(_tablelist, strings.ToLower(tablename)) {
			errmsg = toolkit.Sprintf("\n%s/%s doesn't have collection %s", conn.Info().Host,
				conn.Info().Database, tablename)
		} else {
			listofcol = append(listofcol, tablename)
		}
	}

	if tablegroup != "" {
		switch tablegroup {
		case "allpl":
			for _, col := range tablelist {
				if strings.HasPrefix(col, "pl_") {
					listofcol = append(listofcol, col)
				}
			}
		case "alloutlet":
			for _, col := range tablelist {
				if strings.HasPrefix(col, "outlet_number_") {
					listofcol = append(listofcol, col)
				}
			}
		case "all":
			for _, col := range tablelist {
				if strings.HasPrefix(col, "pl_") || strings.HasPrefix(col, "outlet_number_") {
					listofcol = append(listofcol, col)
				}
			}
		}
	}

	hostsource := ""
	dbsource := ""
	hostdest := ""
	dbdest := ""

	switch source {
	case "ba":
		hostsource = "go.eaciit.com:27123"
		dbsource = "ecgodrej"
	case "devel":
		hostsource = "52.220.25.190:27123"
		dbsource = "ecgodrej"
	case "prod":
		hostsource = "go.eaciit.com:27123"
		dbsource = "ecgodrej_prod"
	}

	switch dest {
	case "ba":
		hostdest = "go.eaciit.com:27123"
		dbdest = "ecgodrej"
	case "devel":
		hostdest = "52.220.25.190:27123"
		dbdest = "ecgodrej"
	case "prod":
		hostdest = "-h go.eaciit.com:27123"
		dbdest = "ecgodrej_prod"
	}

	if errmsg != "" {
		toolkit.Println(errmsg)
	}
	location := "/data/dump/godrej"
	numcol := len(listofcol)
	toolkit.Printfn("\nPrepare to dump & restore (%d) collections", numcol)
	toolkit.Printfn("from %s/%s to %s/%s\n",
		hostsource, dbsource, hostdest, dbdest)

	for i, col := range listofcol {
		dump := toolkit.Sprintf("mongodump -h %s -d %s -c %s --out %s", hostsource, dbsource, col, location)
		restore := toolkit.Sprintf("mongorestore -h %s -d %s -c %s --noIndexRestore --drop %s/%s/%s.bson ",
			hostdest, dbdest, col, location, dbsource, col)

		if runtime.GOOS == "windows" {
			_, err := exec.Command("sh", "-c", dump).Output()

			if err != nil {
				toolkit.Println("error", err)
			} else {
				toolkit.Printfn("(%d of %d) dump : %s", i, numcol, col)
			}

			_, err = exec.Command("cmd", "-c", restore).Output()

			if err != nil {
				toolkit.Println(err)
			} else {
				toolkit.Printfn("(%d of %d) restore : %s", i, numcol, col)
			}
		} else {
			_, err := exec.Command("/bin/bash", "-c", dump).Output()

			if err != nil {
				toolkit.Println(err)
			} else {
				toolkit.Printfn("(%d of %d) dump : %s", i, numcol, col)
			}

			_, err = exec.Command("/bin/bash", "-c", restore).Output()

			if err != nil {
				toolkit.Println(err)
			} else {
				toolkit.Printfn("(%d of %d) restore : %s", i, numcol, col)
			}
		}
	}

	toolkit.Printfn("\nDone dumping & restoring (%d) collections", numcol)
	toolkit.Printfn("from %s/%s to %s/%s\n",
		hostsource, dbsource, hostdest, dbdest)

	return
}
<commit_msg>update<commit_after>package main

import (
	"flag"
	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	"github.com/eaciit/toolkit"
	"os"
	"os/exec"
	"runtime"
	"strings"
)

var (
	conn           dbox.IConnection
	tablegroup     string
	tablename      string
	source         string
	dest           string
	tablegrouplist = []string{"allpl", "alloutlet"}
	sourcedest     = []string{"devel", "prod", "ba"}
)

func setinitialconnection(hostsource, dbsource string) {
	var err error
	ci := &dbox.ConnectionInfo{
		hostsource,
		dbsource,
		"",
		"",
		toolkit.M{}.Set("timeout", 300),
	}

	conn, err = dbox.NewConnection("mongo", ci)

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	if err = conn.Connect(); err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func main() {
	flag.StringVar(&tablegroup, "tablegroup", "", "group of collection")
	flag.StringVar(&tablename, "tablename", "", "collection name")
	flag.StringVar(&source, "from", "", "source location of dumped collection")
	flag.StringVar(&dest, "to", "", "destination location of restored collection")
	flag.Parse()

	copycollection()
}

func copycollection() {
	if source == "" || dest == "" {
		errmsg := ""
		if source == "" {
			errmsg = "source location of dumped collection"
		} else {
			errmsg = "destination location of restored collection"
		}
		toolkit.Println("\nPlease fill parameter for", errmsg)
		return
	} else {
		if !toolkit.HasMember(sourcedest, strings.ToLower(source)) {
			toolkit.Println("\nsource location is not valid, choose the valid parameter below :")
			toolkit.Println("ba\tto dump collection from go.eaciit.com:27123/ecgodrej")
			toolkit.Println("devel\tto dump collection from 52.220.25.190:27123/ecgodrej")
			toolkit.Println("prod\tto dump collection from go.eaciit.com:27123/ecgodrej_prod")
			return
		} else if !toolkit.HasMember(sourcedest, strings.ToLower(dest)) {
			toolkit.Println("\ndestination location is not valid")
			toolkit.Println("ba\tto restore collection into go.eaciit.com:27123/ecgodrej")
			toolkit.Println("devel\tto restore collection into 52.220.25.190:27123/ecgodrej")
			toolkit.Println("prod\tto restore collection into go.eaciit.com:27123/ecgodrej_prod")
			return
		}
	}

	if tablegroup == "" && tablename == "" {
		toolkit.Println("\nPlease fill parameter for tablegroup or tablename")
		return
	} else if tablegroup != "" {
		if !toolkit.HasMember(tablegrouplist, strings.ToLower(tablegroup)) {
			toolkit.Println("\ntablegroup is not valid, choose the valid parameter below :")
			toolkit.Println("allpl\t\tto dump-restore all pl collection")
			toolkit.Println("alloutlet\tto dump-restore all outlet collection")
			toolkit.Println("all\t\tto dump-restore both all pl collection and all outlet collection")
			return
		}
	}

	hostsource := ""
	dbsource := ""
	hostdest := ""
	dbdest := ""

	switch source {
	case "ba":
		hostsource = "go.eaciit.com:27123"
		dbsource = "ecgodrej"
	case "devel":
		hostsource = "52.220.25.190:27123"
		dbsource = "ecgodrej"
	case "prod":
		hostsource = "go.eaciit.com:27123"
		dbsource = "ecgodrej_prod"
	}

	switch dest {
	case "ba":
		hostdest = "go.eaciit.com:27123"
		dbdest = "ecgodrej"
	case "devel":
		hostdest = "52.220.25.190:27123"
		dbdest = "ecgodrej"
	case "prod":
		hostdest = "-h go.eaciit.com:27123"
		dbdest = "ecgodrej_prod"
	}

	setinitialconnection(hostsource, dbsource)
	defer conn.Close()

	tablelist := conn.ObjectNames(dbox.ObjTypeTable)
	errmsg := ""

	listofcol := []string{}
	if tablename != "" {
		_tablelist := []string{}
		for _, col := range tablelist {
			_tablelist = append(_tablelist, strings.ToLower(col))
		}
		if !toolkit.HasMember(_tablelist, strings.ToLower(tablename)) {
			errmsg = toolkit.Sprintf("\n%s/%s doesn't have collection %s", conn.Info().Host,
				conn.Info().Database, tablename)
		} else {
			listofcol = append(listofcol, tablename)
		}
	}

	if tablegroup != "" {
		switch tablegroup {
		case "allpl":
			for _, col := range tablelist {
				if strings.HasPrefix(col, "pl_") {
					listofcol = append(listofcol, col)
				}
			}
		case "alloutlet":
			for _, col := range tablelist {
				if strings.HasPrefix(col, "outlet_number_") {
					listofcol = append(listofcol, col)
				}
			}
		case "all":
			for _, col := range tablelist {
				if strings.HasPrefix(col, "pl_") || strings.HasPrefix(col, "outlet_number_") {
					listofcol = append(listofcol, col)
				}
			}
		}
	}

	if errmsg != "" {
		toolkit.Println(errmsg)
	}
	location := "/data/dump/godrej"
	numcol := len(listofcol)
	toolkit.Printfn("\nPrepare to dump & restore (%d) collections", numcol)
	toolkit.Printfn("from %s/%s to %s/%s\n",
		hostsource, dbsource, hostdest, dbdest)

	for i, col := range listofcol {
		dump := toolkit.Sprintf("mongodump -h %s -d %s -c %s --out %s", hostsource, dbsource, col, location)
		restore := toolkit.Sprintf("mongorestore -h %s -d %s -c %s --noIndexRestore --drop %s/%s/%s.bson ",
			hostdest, dbdest, col, location, dbsource, col)

		if runtime.GOOS == "windows" {
			_, err := exec.Command("sh", "-c", dump).Output()

			if err != nil {
				toolkit.Println("dump error", err)
				toolkit.Println("syntax", dump)
			} else {
				toolkit.Printfn("(%d of %d) dump : %s", i+1, numcol, col)
			}

			_, err = exec.Command("cmd", "-c", restore).Output()

			if err != nil {
				toolkit.Println("restore error", err)
				toolkit.Println("syntax", restore)
			} else {
				toolkit.Printfn("(%d of %d) restore : %s", i+1, numcol, col)
			}
		} else {
			_, err := exec.Command("/bin/bash", "-c", dump).Output()

			if err != nil {
				toolkit.Println("dump error", err)
				toolkit.Println("syntax", dump)
			} else {
				toolkit.Printfn("(%d of %d) dump : %s", i+1, numcol, col)
			}

			_, err = exec.Command("/bin/bash", "-c", restore).Output()

			if err != nil {
				toolkit.Println("restore error", err)
				toolkit.Println("syntax", restore)
			} else {
				toolkit.Printfn("(%d of %d) restore : %s", i+1, numcol, col)
			}
		}
	}

	toolkit.Printfn("\nDone dumping & restoring (%d) collections", numcol)
	toolkit.Printfn("from %s/%s to %s/%s\n",
		hostsource, dbsource, hostdest, dbdest)

	return
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type podLatencyData struct {
	Name    string
	Latency time.Duration
}

type latencySlice []podLatencyData

func (a latencySlice) Len() int           { return len(a) }
func (a latencySlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a latencySlice) Less(i, j int) bool { return a[i].Latency < a[j].Latency }

func printLatencies(latencies []podLatencyData, header string) {
	perc50 := latencies[len(latencies)/2].Latency
	perc90 := latencies[(len(latencies)*9)/10].Latency
	perc99 := latencies[(len(latencies)*99)/100].Latency
	Logf("10%% %s: %v", header, latencies[(len(latencies)*9)/10:len(latencies)])
	Logf("perc50: %v, perc90: %v, perc99: %v", perc50, perc90, perc99)
}

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = Describe("Density", func() {
	var c *client.Client
	var minionCount int
	var RCName string
	var additionalRCName string
	var ns string
	var uuid string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		minions, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		minionCount = len(minions.Items)
		Expect(minionCount).NotTo(BeZero())
		nsForTesting, err := createTestingNS("density", c)
		ns = nsForTesting.Name
		expectNoError(err)
		uuid = string(util.NewUUID())

		// Print latency metrics before the test.
		// TODO: Remove this once we reset metrics before the test.
		_, err = HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events"))
		expectNoError(err)

		expectNoError(os.Mkdir(fmt.Sprintf(testContext.OutputDir+"/%s", uuid), 0777))
		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "before"))
	})

	AfterEach(func() {
		// Remove any remaining pods from this test if the
		// replication controller still exists and the replica count
		// isn't 0.  This means the controller wasn't cleaned up
		// during the test so clean it up here
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, RCName)
			expectNoError(err)
		}

		rc, err = c.ReplicationControllers(ns).Get(additionalRCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, additionalRCName)
			expectNoError(err)
		}

		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}

		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after"))

		// Verify latency metrics
		// TODO: We should reset metrics before the test. Currently previous tests influence latency metrics.
		highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events"))
		expectNoError(err)
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0), "There should be no high-latency requests")
	})

	// Tests with "Skipped" substring in their name will be skipped when running
	// e2e test suite without --ginkgo.focus & --ginkgo.skip flags.
	type Density struct {
		skip bool
		// Controls if e2e latency tests should be run (they are slow)
		runLatencyTest bool
		podsPerMinion  int
		// Controls how often the apiserver is polled for pods
		interval time.Duration
	}

	densityTests := []Density{
		// This test should not be run in a regular jenkins run, because it is not isolated enough
		// (metrics from other tests affects this one).
		// TODO: Reenable once we can measure latency only from a single test.
		// TODO: Expose runLatencyTest as ginkgo flag.
		{podsPerMinion: 3, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		{podsPerMinion: 30, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		// More than 30 pods per node is outside our v1.0 goals.
		// We might want to enable those tests in the future.
		{podsPerMinion: 50, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		{podsPerMinion: 100, skip: true, runLatencyTest: false, interval: 1 * time.Second},
	}

	for _, testArg := range densityTests {
		name := fmt.Sprintf("should allow starting %d pods per node", testArg.podsPerMinion)
		if testArg.podsPerMinion <= 30 {
			name = "[Performance suite] " + name
		}
		if testArg.skip {
			name = "[Skipped] " + name
		}
		itArg := testArg
		It(name, func() {
			totalPods := itArg.podsPerMinion * minionCount
			RCName = "density" + strconv.Itoa(totalPods) + "-" + uuid
			fileHndl, err := os.Create(fmt.Sprintf(testContext.OutputDir+"/%s/pod_states.csv", uuid))
			expectNoError(err)
			defer fileHndl.Close()

			config := RCConfig{Client: c,
				Image:         "gcr.io/google_containers/pause:go",
				Name:          RCName,
				Namespace:     ns,
				PollInterval:  itArg.interval,
				PodStatusFile: fileHndl,
				Replicas:      totalPods,
			}

			// Create a listener for events.
			events := make([](*api.Event), 0)
			_, controller := framework.NewInformer(
				&cache.ListWatch{
					ListFunc: func() (runtime.Object, error) {
						return c.Events(ns).List(labels.Everything(), fields.Everything())
					},
					WatchFunc: func(rv string) (watch.Interface, error) {
						return c.Events(ns).Watch(labels.Everything(), fields.Everything(), rv)
					},
				},
				&api.Event{},
				0,
				framework.ResourceEventHandlerFuncs{
					AddFunc: func(obj interface{}) {
						events = append(events, obj.(*api.Event))
					},
				},
			)
			stop := make(chan struct{})
			go controller.Run(stop)

			// Start the replication controller.
			startTime := time.Now()
			expectNoError(RunRC(config))
			e2eStartupTime := time.Now().Sub(startTime)
			Logf("E2E startup time for %d pods: %v", totalPods, e2eStartupTime)

			By("Waiting for all events to be recorded")
			last := -1
			current := len(events)
			timeout := 10 * time.Minute
			for start := time.Now(); last < current && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				last = current
				current = len(events)
			}
			close(stop)

			if current != last {
				Logf("Warning: Not all events were recorded after waiting %.2f minutes", timeout.Minutes())
			}
			Logf("Found %d events", current)

			// Tune the threshold for allowed failures.
			badEvents := BadEvents(events)
			Expect(badEvents).NotTo(BeNumerically(">", int(math.Floor(0.01*float64(totalPods)))))

			if itArg.runLatencyTest {
				Logf("Schedling additional Pods to measure startup latencies")

				createTimes := make(map[string]util.Time, 0)
				scheduleTimes := make(map[string]util.Time, 0)
				runTimes := make(map[string]util.Time, 0)
				watchTimes := make(map[string]util.Time, 0)

				var mutex sync.Mutex
				checkPod := func(p *api.Pod) {
					mutex.Lock()
					defer mutex.Unlock()
					defer GinkgoRecover()

					if p.Status.Phase == api.PodRunning {
						if _, found := watchTimes[p.Name]; !found {
							watchTimes[p.Name] = util.Now()
							createTimes[p.Name] = p.CreationTimestamp
							var startTime util.Time
							for _, cs := range p.Status.ContainerStatuses {
								if cs.State.Running != nil {
									if startTime.Before(cs.State.Running.StartedAt) {
										startTime = cs.State.Running.StartedAt
									}
								}
							}
							if startTime != util.NewTime(time.Time{}) {
								runTimes[p.Name] = startTime
							} else {
								Failf("Pod %v is reported to be running, but none of its containers is", p.Name)
							}
						}
					}
				}

				additionalNameStr := strconv.Itoa(minionCount) + "-" + string(util.NewUUID())
				additionalRCName = "my-hostname-latency" + additionalNameStr
				_, controller := framework.NewInformer(
					&cache.ListWatch{
						ListFunc: func() (runtime.Object, error) {
							return c.Pods(ns).List(labels.SelectorFromSet(labels.Set{"name": additionalRCName}), fields.Everything())
						},
						WatchFunc: func(rv string) (watch.Interface, error) {
							return c.Pods(ns).Watch(labels.SelectorFromSet(labels.Set{"name": additionalRCName}), fields.Everything(), rv)
						},
					},
					&api.Pod{},
					time.Minute*5,
					framework.ResourceEventHandlerFuncs{
						AddFunc: func(obj interface{}) {
							p, ok := obj.(*api.Pod)
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
						UpdateFunc: func(oldObj, newObj interface{}) {
							p, ok := newObj.(*api.Pod)
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
					},
				)

				stopCh := make(chan struct{})
				go controller.Run(stopCh)

				config = RCConfig{Client: c,
					Image:        "gcr.io/google_containers/pause:go",
					Name:         additionalRCName,
					Namespace:    ns,
					PollInterval: itArg.interval,
					Replicas:     minionCount,
				}
				expectNoError(RunRC(config))

				Logf("Waiting for all Pods begin observed by the watch...")
				for start := time.Now(); len(watchTimes) < minionCount && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				}
				close(stopCh)

				schedEvents, err := c.Events(ns).List(
					labels.Everything(),
					fields.Set{
						"involvedObject.kind":      "Pod",
						"involvedObject.namespace": ns,
						"source":                   "scheduler",
					}.AsSelector())
				expectNoError(err)
				for k := range createTimes {
					for _, event := range schedEvents.Items {
						if event.InvolvedObject.Name == k {
							scheduleTimes[k] = event.FirstTimestamp
							break
						}
					}
				}

				scheduleLag := make([]podLatencyData, 0)
				startupLag := make([]podLatencyData, 0)
				watchLag := make([]podLatencyData, 0)
				schedToWatchLag := make([]podLatencyData, 0)
				e2eLag := make([]podLatencyData, 0)

				for name, create := range createTimes {
					sched, ok := scheduleTimes[name]
					Expect(ok).To(Equal(true))
					run, ok := runTimes[name]
					Expect(ok).To(Equal(true))
					watch, ok := watchTimes[name]
					Expect(ok).To(Equal(true))
					scheduleLag = append(scheduleLag, podLatencyData{name, sched.Time.Sub(create.Time)})
					startupLag = append(startupLag, podLatencyData{name, run.Time.Sub(sched.Time)})
					watchLag = append(watchLag, podLatencyData{name, watch.Time.Sub(run.Time)})
					schedToWatchLag = append(schedToWatchLag, podLatencyData{name, watch.Time.Sub(sched.Time)})
					e2eLag = append(e2eLag, podLatencyData{name, watch.Time.Sub(create.Time)})
				}

				sort.Sort(latencySlice(scheduleLag))
				sort.Sort(latencySlice(startupLag))
				sort.Sort(latencySlice(watchLag))
				sort.Sort(latencySlice(schedToWatchLag))
				sort.Sort(latencySlice(e2eLag))

				printLatencies(scheduleLag, "worst schedule latencies")
				printLatencies(startupLag, "worst run-after-schedule latencies")
				printLatencies(watchLag, "worst watch latencies")
				printLatencies(schedToWatchLag, "worst scheduled-to-end total latencies")
				printLatencies(e2eLag, "worst e2e total latencies")

				Logf("Approx throughput: %v pods/min",
					float64(minionCount)/(e2eLag[len(e2eLag)-1].Latency.Minutes()))
			}
		})
	}
})
<commit_msg>Slow down creation of additional pods in density.go e2e<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type podLatencyData struct {
	Name    string
	Latency time.Duration
}

type latencySlice []podLatencyData

func (a latencySlice) Len() int           { return len(a) }
func (a latencySlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a latencySlice) Less(i, j int) bool { return a[i].Latency < a[j].Latency }

func printLatencies(latencies []podLatencyData, header string) {
	perc50 := latencies[len(latencies)/2].Latency
	perc90 := latencies[(len(latencies)*9)/10].Latency
	perc99 := latencies[(len(latencies)*99)/100].Latency
	Logf("10%% %s: %v", header, latencies[(len(latencies)*9)/10:len(latencies)])
	Logf("perc50: %v, perc90: %v, perc99: %v", perc50, perc90, perc99)
}

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = Describe("Density", func() {
	var c *client.Client
	var minionCount int
	var RCName string
	var additionalPodsPrefix string
	var ns string
	var uuid string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		minions, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		minionCount = len(minions.Items)
		Expect(minionCount).NotTo(BeZero())
		nsForTesting, err := createTestingNS("density", c)
		ns = nsForTesting.Name
		expectNoError(err)
		uuid = string(util.NewUUID())

		// Print latency metrics before the test.
		// TODO: Remove this once we reset metrics before the test.
		_, err = HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events"))
		expectNoError(err)

		expectNoError(os.Mkdir(fmt.Sprintf(testContext.OutputDir+"/%s", uuid), 0777))
		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "before"))
	})

	AfterEach(func() {
		// Remove any remaining pods from this test if the
		// replication controller still exists and the replica count
		// isn't 0.  This means the controller wasn't cleaned up
		// during the test so clean it up here
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, RCName)
			expectNoError(err)
		}

		By("Removing additional pods if any")
		for i := 1; i <= minionCount; i++ {
			name := additionalPodsPrefix + "-" + strconv.Itoa(i)
			c.Pods(ns).Delete(name, nil)
		}

		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}

		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after"))

		// Verify latency metrics
		// TODO: We should reset metrics before the test. Currently previous tests influence latency metrics.
		highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events"))
		expectNoError(err)
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0), "There should be no high-latency requests")
	})

	// Tests with "Skipped" substring in their name will be skipped when running
	// e2e test suite without --ginkgo.focus & --ginkgo.skip flags.
	type Density struct {
		skip bool
		// Controls if e2e latency tests should be run (they are slow)
		runLatencyTest bool
		podsPerMinion  int
		// Controls how often the apiserver is polled for pods
		interval time.Duration
	}

	densityTests := []Density{
		// This test should not be run in a regular jenkins run, because it is not isolated enough
		// (metrics from other tests affects this one).
		// TODO: Reenable once we can measure latency only from a single test.
		// TODO: Expose runLatencyTest as ginkgo flag.
		{podsPerMinion: 3, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		{podsPerMinion: 30, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		// More than 30 pods per node is outside our v1.0 goals.
		// We might want to enable those tests in the future.
		{podsPerMinion: 50, skip: true, runLatencyTest: false, interval: 10 * time.Second},
		{podsPerMinion: 100, skip: true, runLatencyTest: false, interval: 1 * time.Second},
	}

	for _, testArg := range densityTests {
		name := fmt.Sprintf("should allow starting %d pods per node", testArg.podsPerMinion)
		if testArg.podsPerMinion <= 30 {
			name = "[Performance suite] " + name
		}
		if testArg.skip {
			name = "[Skipped] " + name
		}
		itArg := testArg
		It(name, func() {
			totalPods := itArg.podsPerMinion * minionCount
			RCName = "density" + strconv.Itoa(totalPods) + "-" + uuid
			fileHndl, err := os.Create(fmt.Sprintf(testContext.OutputDir+"/%s/pod_states.csv", uuid))
			expectNoError(err)
			defer fileHndl.Close()

			config := RCConfig{Client: c,
				Image:         "gcr.io/google_containers/pause:go",
				Name:          RCName,
				Namespace:     ns,
				PollInterval:  itArg.interval,
				PodStatusFile: fileHndl,
				Replicas:      totalPods,
			}

			// Create a listener for events.
			events := make([](*api.Event), 0)
			_, controller := framework.NewInformer(
				&cache.ListWatch{
					ListFunc: func() (runtime.Object, error) {
						return c.Events(ns).List(labels.Everything(), fields.Everything())
					},
					WatchFunc: func(rv string) (watch.Interface, error) {
						return c.Events(ns).Watch(labels.Everything(), fields.Everything(), rv)
					},
				},
				&api.Event{},
				0,
				framework.ResourceEventHandlerFuncs{
					AddFunc: func(obj interface{}) {
						events = append(events, obj.(*api.Event))
					},
				},
			)
			stop := make(chan struct{})
			go controller.Run(stop)

			// Start the replication controller.
			startTime := time.Now()
			expectNoError(RunRC(config))
			e2eStartupTime := time.Now().Sub(startTime)
			Logf("E2E startup time for %d pods: %v", totalPods, e2eStartupTime)

			By("Waiting for all events to be recorded")
			last := -1
			current := len(events)
			timeout := 10 * time.Minute
			for start := time.Now(); last < current && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				last = current
				current = len(events)
			}
			close(stop)

			if current != last {
				Logf("Warning: Not all events were recorded after waiting %.2f minutes", timeout.Minutes())
			}
			Logf("Found %d events", current)

			// Tune the threshold for allowed failures.
			badEvents := BadEvents(events)
			Expect(badEvents).NotTo(BeNumerically(">", int(math.Floor(0.01*float64(totalPods)))))

			if itArg.runLatencyTest {
				Logf("Schedling additional Pods to measure startup latencies")

				createTimes := make(map[string]util.Time, 0)
				scheduleTimes := make(map[string]util.Time, 0)
				runTimes := make(map[string]util.Time, 0)
				watchTimes := make(map[string]util.Time, 0)

				var mutex sync.Mutex
				checkPod := func(p *api.Pod) {
					mutex.Lock()
					defer mutex.Unlock()
					defer GinkgoRecover()

					if p.Status.Phase == api.PodRunning {
						if _, found := watchTimes[p.Name]; !found {
							watchTimes[p.Name] = util.Now()
							createTimes[p.Name] = p.CreationTimestamp
							var startTime util.Time
							for _, cs := range p.Status.ContainerStatuses {
								if cs.State.Running != nil {
									if startTime.Before(cs.State.Running.StartedAt) {
										startTime = cs.State.Running.StartedAt
									}
								}
							}
							if startTime != util.NewTime(time.Time{}) {
								runTimes[p.Name] = startTime
							} else {
								Failf("Pod %v is reported to be running, but none of its containers is", p.Name)
							}
						}
					}
				}

				additionalPodsPrefix = "density-latency-pod-" + string(util.NewUUID())
				_, controller := framework.NewInformer(
					&cache.ListWatch{
						ListFunc: func() (runtime.Object, error) {
							return c.Pods(ns).List(labels.SelectorFromSet(labels.Set{"name": additionalPodsPrefix}), fields.Everything())
						},
						WatchFunc: func(rv string) (watch.Interface, error) {
							return c.Pods(ns).Watch(labels.SelectorFromSet(labels.Set{"name": additionalPodsPrefix}), fields.Everything(), rv)
						},
					},
					&api.Pod{},
					time.Minute*5,
					framework.ResourceEventHandlerFuncs{
						AddFunc: func(obj interface{}) {
							p, ok := obj.(*api.Pod)
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
						UpdateFunc: func(oldObj, newObj interface{}) {
							p, ok := newObj.(*api.Pod)
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
					},
				)

				stopCh := make(chan struct{})
				go controller.Run(stopCh)

				// Create some additional pods with throughput ~5 pods/sec.
				var wg sync.WaitGroup
				wg.Add(minionCount)
				podLabels := map[string]string{
					"name": additionalPodsPrefix,
				}
				for i := 1; i <= minionCount; i++ {
					name := additionalPodsPrefix + "-" + strconv.Itoa(i)
					go createRunningPod(&wg, c, name, ns, "gcr.io/google_containers/pause:go", podLabels)
					time.Sleep(200 * time.Millisecond)
				}
				wg.Wait()

				Logf("Waiting for all Pods begin observed by the watch...")
				for start := time.Now(); len(watchTimes) < minionCount && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				}
				close(stopCh)

				schedEvents, err := c.Events(ns).List(
					labels.Everything(),
					fields.Set{
						"involvedObject.kind":      "Pod",
						"involvedObject.namespace": ns,
						"source":                   "scheduler",
					}.AsSelector())
				expectNoError(err)
				for k := range createTimes {
					for _, event := range schedEvents.Items {
						if event.InvolvedObject.Name == k {
							scheduleTimes[k] = event.FirstTimestamp
							break
						}
					}
				}

				scheduleLag := make([]podLatencyData, 0)
				startupLag := make([]podLatencyData, 0)
				watchLag := make([]podLatencyData, 0)
				schedToWatchLag := make([]podLatencyData, 0)
				e2eLag := make([]podLatencyData, 0)

				for name, create := range createTimes {
					sched, ok := scheduleTimes[name]
					Expect(ok).To(Equal(true))
					run, ok := runTimes[name]
					Expect(ok).To(Equal(true))
					watch, ok := watchTimes[name]
					Expect(ok).To(Equal(true))
					scheduleLag = append(scheduleLag, podLatencyData{name, sched.Time.Sub(create.Time)})
					startupLag = append(startupLag, podLatencyData{name, run.Time.Sub(sched.Time)})
					watchLag = append(watchLag, podLatencyData{name, watch.Time.Sub(run.Time)})
					schedToWatchLag = append(schedToWatchLag, podLatencyData{name, watch.Time.Sub(sched.Time)})
					e2eLag = append(e2eLag, podLatencyData{name, watch.Time.Sub(create.Time)})
				}

				sort.Sort(latencySlice(scheduleLag))
				sort.Sort(latencySlice(startupLag))
				sort.Sort(latencySlice(watchLag))
				sort.Sort(latencySlice(schedToWatchLag))
				sort.Sort(latencySlice(e2eLag))

				printLatencies(scheduleLag, "worst schedule latencies")
				printLatencies(startupLag, "worst run-after-schedule latencies")
				printLatencies(watchLag, "worst watch latencies")
				printLatencies(schedToWatchLag, "worst scheduled-to-end total latencies")
				printLatencies(e2eLag, "worst e2e total latencies")

				Logf("Approx throughput: %v pods/min",
					float64(minionCount)/(e2eLag[len(e2eLag)-1].Latency.Minutes()))
			}
		})
	}
})

func createRunningPod(wg *sync.WaitGroup, c *client.Client, name, ns, image string, labels map[string]string) {
	defer GinkgoRecover()
	defer wg.Done()
	pod := &api.Pod{
		TypeMeta: api.TypeMeta{
			Kind: "Pod",
		},
		ObjectMeta: api.ObjectMeta{
			Name:   name,
			Labels: labels,
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  name,
					Image: image,
				},
			},
		},
	}
	_, err := c.Pods(ns).Create(pod)
	expectNoError(err)
	expectNoError(waitForPodRunningInNamespace(c, name, ns))
}
<|endoftext|>


<commit_before>package controller

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"ratings/handler"

	"github.com/gavv/httpexpect"
	"github.com/labstack/echo"
)

var routes = map[string]echo.HandlerFunc{
	"OptionsRoot":    OptionsRoot,
	"OptionsRatings": OptionsRatings,
	"PostRatings":    PostRatings}

func TestOptionsRatings(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:3000"

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	response := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    200,
			"message": "OK"},
		"endpoints": []map[string]interface{}{
			{
				"method": "POST",
				"path":   "/ratings",
				"headers": map[string]interface{}{
					"Content-Type": "application/json; charset=UTF-8"}}}}

	r := e.OPTIONS("/ratings").Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")
	r.Header("Allow").Equal("OPTIONS POST")
	r.JSON().Object().Equal(response)
}

func TestPostRatings(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:3000"

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	jsonRequest := map[string]interface{}{
		"rating":      uint8(3),
		"description": "Regular",
		"range":       "e10adc3949ba59abbe56e057f20f883e",
		"app": map[string]interface{}{
			"key":     "e10adc3949ba59abbe56e057f20f883e",
			"version": "2.O"},
		"platform": map[string]interface{}{
			"key":     "e10adc3949ba59abbe56e057f20f883e",
			"version": "9.O"}}

	/*
		jsonResponse := map[string]interface{}{
			"meta": map[string]interface{}{
				"code":    201,
				"message": "Created"}}}
	*/

	e.POST("/ratings").WithJSON(jsonRequest).
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		Expect().
		Status(http.StatusCreated)
}
<commit_msg>Updated tests<commit_after>package controller

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"ratings/handler"

	"github.com/gavv/httpexpect"
	"github.com/labstack/echo"
)

var routes = map[string]echo.HandlerFunc{
	"OptionsRoot":    OptionsRoot,
	"OptionsRatings": OptionsRatings,
	"PostRatings":    PostRatings}

func TestOptionsRatings(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:3000"

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	response := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    200,
			"message": "OK"},
		"endpoints": []map[string]interface{}{
			{
				"method": "POST",
				"path":   "/ratings",
				"headers": map[string]interface{}{
					"Content-Type": "application/json; charset=UTF-8"}}}}

	r := e.OPTIONS("/ratings").
		WithHeader("Accept", "application/json").
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")
	r.Header("Allow").Equal("OPTIONS POST")
	r.JSON().Object().Equal(response)
}

func TestPostRatings(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:3000"

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	request := map[string]interface{}{
		"rating":      uint8(3),
		"description": "Regular",
		"range":       "e10adc3949ba59abbe56e057f20f883e",
		"app": map[string]interface{}{
			"key":     "e10adc3949ba59abbe56e057f20f883e",
			"version": "2.O"},
		"platform": map[string]interface{}{
			"key":     "e10adc3949ba59abbe56e057f20f883e",
			"version": "9.O"}}

	response := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    201,
			"message": "Created"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithJSON(request).
		Expect()

	r.Status(http.StatusCreated)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")
	r.JSON().Object().Equal(response)
}
<|endoftext|>


<commit_before>// +build linux

package aufs

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"sync"
	"syscall"

	"github.com/golang/glog"
	"github.com/hyperhq/hyperd/utils"
)

/*
|-- layers  // Metadata of layers
|   |---- 1
|   |---- 2
|   |---- 3
|-- diff    // Content of the layer
|   |---- 1
|   |---- 2
|   |---- 3
|-- mnt     // Mount points for the rw layers to be mounted
    |---- 1
    |---- 2
    |---- 3
*/

var (
	enableDirpermLock sync.Once
	enableDirperm     bool
)

const MsRemount = syscall.MS_REMOUNT

func MountContainerToSharedDir(containerId, rootDir, sharedDir, mountLabel string) (string, error) {
	var (
		//mntPath = path.Join(rootDir, "mnt")
		//layersPath = path.Join(rootDir, "layers")
		diffPath   = path.Join(rootDir, "diff")
		mountPoint = path.Join(sharedDir, containerId, "rootfs")
	)

	diffs, err := getParentDiffPaths(containerId, rootDir)
	if err != nil {
		return "", err
	}
	if err = os.MkdirAll(mountPoint, 0755); err != nil {
		return "", err
	}

	if err := aufsMount(diffs, path.Join(diffPath, containerId), mountPoint, mountLabel); err != nil {
		return "", fmt.Errorf("Fail to mount aufs dir to %s: %v", mountPoint, err)
	}

	return mountPoint, nil
}

func getParentDiffPaths(id, rootPath string) ([]string, error) {
	parentIds, err := getParentIds(path.Join(rootPath, "layers", id))
	if err != nil {
		return nil, err
	}
	layers := make([]string, len(parentIds))

	// Get the diff paths for all the parent ids
	for i, p := range parentIds {
		layers[i] = path.Join(rootPath, "diff", p)
	}
	return layers, nil
}

func aufsMount(ro []string, rw, target, mountLabel string) (err error) {
	defer func() {
		if err != nil {
			aufsUnmount(target)
		}
	}()

	// Mount options are clipped to page size(4096 bytes). If there are more
	// layers then these are remounted individually using append.

	offset := 54
	if useDirperm() {
		offset += len("dirperm1")
	}
	b := make([]byte, syscall.Getpagesize()-len(mountLabel)-offset) // room for xino & mountLabel
	bp := copy(b, fmt.Sprintf("br:%s=rw", rw))

	firstMount := true
	i := 0

	for {
		for ; i < len(ro); i++ {
			layer := fmt.Sprintf(":%s=ro+wh", ro[i])

			if firstMount {
				if bp+len(layer) > len(b) {
					break
				}
				bp += copy(b[bp:], layer)
			} else {
				data := utils.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel)
				if err = syscall.Mount("none", target, "aufs", MsRemount, data); err != nil {
					return
				}
			}
		}

		if firstMount {
			opts := "dio,xino=/dev/shm/aufs.xino"
			if useDirperm() {
				opts += ",dirperm1"
			}
			data := utils.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel)
			if err = syscall.Mount("none", target, "aufs", 0, data); err != nil {
				return
			}
			firstMount = false
		}

		if i == len(ro) {
			break
		}
	}

	return
}

func Unmount(mountPoint string) error {
	return aufsUnmount(mountPoint)
}

// useDirperm checks dirperm1 mount option can be used with the current
// version of aufs.
func useDirperm() bool {
	enableDirpermLock.Do(func() {
		base, err := ioutil.TempDir("", "docker-aufs-base")
		if err != nil {
			glog.Errorf("error checking dirperm1: %s", err.Error())
			return
		}
		defer os.RemoveAll(base)

		union, err := ioutil.TempDir("", "docker-aufs-union")
		if err != nil {
			glog.Errorf("error checking dirperm1: %s", err.Error())
			return
		}
		defer os.RemoveAll(union)

		opts := fmt.Sprintf("br:%s,dirperm1,xino=/dev/shm/aufs.xino", base)
		if err := syscall.Mount("none", union, "aufs", 0, opts); err != nil {
			return
		}
		enableDirperm = true
		if err := aufsUnmount(union); err != nil {
			glog.Errorf("error checking dirperm1: failed to unmount %s", err.Error())
		}
	})
	return enableDirperm
}

func aufsUnmount(target string) error {
	glog.V(1).Infof("Ready to unmount the target : %s", target)
	if _, err := os.Stat(target); err != nil && os.IsNotExist(err) {
		return nil
	}
	cmdString := fmt.Sprintf("auplink %s flush", target)
	cmd := exec.Command("/bin/sh", "-c", cmdString)
	if err := cmd.Run(); err != nil {
		glog.Warningf("Couldn't run auplink command : %s\n%s\n", err.Error())
	}
	if err := syscall.Unmount(target, 0); err != nil {
		return err
	}
	return nil
}

// Return all the directories
func loadIds(root string) ([]string, error) {
	dirs, err := ioutil.ReadDir(root)
	if err != nil {
		return nil, err
	}
	out := []string{}
	for _, d := range dirs {
		if !d.IsDir() {
			out = append(out, d.Name())
		}
	}
	return out, nil
}

// Read the layers file for the current id and return all the
// layers represented by new lines in the file
//
// If there are no lines in the file then the id has no parent
// and an empty slice is returned.
func getParentIds(id string) ([]string, error) {
	f, err := os.Open(id)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	out := []string{}
	s := bufio.NewScanner(f)

	for s.Scan() {
		if t := s.Text(); t != "" {
			out = append(out, s.Text())
		}
	}
	return out, s.Err()
}
<commit_msg>fix aufs mount option length calculation<commit_after>// +build linux

package aufs

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"sync"
	"syscall"

	"github.com/golang/glog"
	"github.com/hyperhq/hyperd/utils"
)

/*
|-- layers  // Metadata of layers
|   |---- 1
|   |---- 2
|   |---- 3
|-- diff    // Content of the layer
|   |---- 1
|   |---- 2
|   |---- 3
|-- mnt     // Mount points for the rw layers to be mounted
    |---- 1
    |---- 2
    |---- 3
*/

var (
	enableDirpermLock sync.Once
	enableDirperm     bool
)

const MsRemount = syscall.MS_REMOUNT

func MountContainerToSharedDir(containerId, rootDir, sharedDir, mountLabel string) (string, error) {
	var (
		//mntPath = path.Join(rootDir, "mnt")
		//layersPath = path.Join(rootDir, "layers")
		diffPath   = path.Join(rootDir, "diff")
		mountPoint = path.Join(sharedDir, containerId, "rootfs")
	)

	diffs, err := getParentDiffPaths(containerId, rootDir)
	if err != nil {
		return "", err
	}
	if err = os.MkdirAll(mountPoint, 0755); err != nil {
		return "", err
	}

	if err := aufsMount(diffs, path.Join(diffPath, containerId), mountPoint, mountLabel); err != nil {
		return "", fmt.Errorf("Fail to mount aufs dir to %s: %v", mountPoint, err)
	}

	return mountPoint, nil
}

func getParentDiffPaths(id, rootPath string) ([]string, error) {
	parentIds, err := getParentIds(path.Join(rootPath, "layers", id))
	if err != nil {
		return nil, err
	}
	layers := make([]string, len(parentIds))

	// Get the diff paths for all the parent ids
	for i, p := range parentIds {
		layers[i] = path.Join(rootPath, "diff", p)
	}
	return layers, nil
}

func aufsMount(ro []string, rw, target, mountLabel string) (err error) {
	defer func() {
		if err != nil {
			aufsUnmount(target)
		}
	}()

	// Mount options are clipped to page size(4096 bytes). If there are more
	// layers then these are remounted individually using append.

	offset := 54
	if useDirperm() {
		offset += len(",dirperm1")
	}
	b := make([]byte, syscall.Getpagesize()-len(mountLabel)-offset) // room for xino & mountLabel
	bp := copy(b, fmt.Sprintf("br:%s=rw", rw))

	firstMount := true
	i := 0

	for {
		for ; i < len(ro); i++ {
			layer := fmt.Sprintf(":%s=ro+wh", ro[i])

			if firstMount {
				if bp+len(layer) > len(b) {
					break
				}
				bp += copy(b[bp:], layer)
			} else {
				data := utils.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel)
				if err = syscall.Mount("none", target, "aufs", MsRemount, data); err != nil {
					glog.Errorf("error mounting aufs data(%s): %s", data, err.Error())
					return
				}
			}
		}

		if firstMount {
			opts := "dio,xino=/dev/shm/aufs.xino"
			if useDirperm() {
				opts += ",dirperm1"
			}
			data := utils.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel)
			if err = syscall.Mount("none", target, "aufs", 0, data); err != nil {
				glog.Errorf("error first mounting aufs data(%d): %s", len(data), err.Error())
				return
			}
			firstMount = false
		}

		if i == len(ro) {
			break
		}
	}

	return
}

func Unmount(mountPoint string) error {
	return aufsUnmount(mountPoint)
}

// useDirperm checks dirperm1 mount option can be used with the current
// version of aufs.
func useDirperm() bool {
	enableDirpermLock.Do(func() {
		base, err := ioutil.TempDir("", "docker-aufs-base")
		if err != nil {
			glog.Errorf("error checking dirperm1: %s", err.Error())
			return
		}
		defer os.RemoveAll(base)

		union, err := ioutil.TempDir("", "docker-aufs-union")
		if err != nil {
			glog.Errorf("error checking dirperm1: %s", err.Error())
			return
		}
		defer os.RemoveAll(union)

		opts := fmt.Sprintf("br:%s,dirperm1,xino=/dev/shm/aufs.xino", base)
		if err := syscall.Mount("none", union, "aufs", 0, opts); err != nil {
			return
		}
		enableDirperm = true
		if err := aufsUnmount(union); err != nil {
			glog.Errorf("error checking dirperm1: failed to unmount %s", err.Error())
		}
	})
	return enableDirperm
}

func aufsUnmount(target string) error {
	glog.V(1).Infof("Ready to unmount the target : %s", target)
	if _, err := os.Stat(target); err != nil && os.IsNotExist(err) {
		return nil
	}
	cmdString := fmt.Sprintf("auplink %s flush", target)
	cmd := exec.Command("/bin/sh", "-c", cmdString)
	if err := cmd.Run(); err != nil {
		glog.Warningf("Couldn't run auplink command : %s\n%s\n", err.Error())
	}
	if err := syscall.Unmount(target, 0); err != nil {
		return err
	}
	return nil
}

// Return all the directories
func loadIds(root string) ([]string, error) {
	dirs, err := ioutil.ReadDir(root)
	if err != nil {
		return nil, err
	}
	out := []string{}
	for _, d := range dirs {
		if !d.IsDir() {
			out = append(out, d.Name())
		}
	}
	return out, nil
}

// Read the layers file for the current id and return all the
// layers represented by new lines in the file
//
// If there are no lines in the file then the id has no parent
// and an empty slice is returned.
func getParentIds(id string) ([]string, error) {
	f, err := os.Open(id)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	out := []string{}
	s := bufio.NewScanner(f)

	for s.Scan() {
		if t := s.Text(); t != "" {
			out = append(out, s.Text())
		}
	}
	return out, s.Err()
}
<|endoftext|>