<commit_before>package internal

import (
	"../agent"
	"google.golang.org/grpc"
	"golang.org/x/net/context"
	"sync"
	"bytes"
	"io"
	"time"
	"errors"
	"net"
	"unsafe"
	"sync/atomic"
)

const (
	PipeChannelBuffSize int = 10
	PipeAcksMaxSize int = 100
)

var bufPool *BufPool = NewBufPool(1024 * 1024 * 1024)

func intMin(x, y int) int {
	if x < y {
		return x
	} else {
		return y
	}
}

type agentStream interface {
	Send(*agent.DataPacket) error
	Recv() (*agent.DataPacket, error)
}

type unAck struct {
	no uint32
	t  time.Time
}

func newUnAck(no uint32) *unAck {
	return &unAck{
		no: no,
		t: time.Now(),
	}
}

const (
	cmdPushAck = iota
	cmdPullAck
	cmdPopUnack
	cmdPushUnack
)

type command struct {
	cmd     int
	inAcks  []uint32
	outAcks chan uint32
}

//wrap grpc stream as net.Conn
type StreamPipe struct {
	ctx         context.Context
	cancelFunc  context.CancelFunc

	raw         agentStream
	cc          *ClientConn //may is nil

	waitGroup   sync.WaitGroup
	ioComplete  chan int

	reads       chan []byte
	writes      chan []byte
	writeable   chan int
	writeFlush  chan int

	cmds        chan *command

	acks        []uint32
	unAcks      []*unAck

	wBuffer     bytes.Buffer
	rCache      bytes.Buffer

	err         unsafe.Pointer

	acksChecker *time.Ticker

	serial      uint32
}

func NewStreamPipe(stream agentStream) *StreamPipe {
	ctx, cancelFunc := context.WithCancel(context.Background())
	pipe := &StreamPipe{
		ctx: ctx,
		cancelFunc:cancelFunc,
		raw: stream,
		ioComplete: make(chan int, 2),
		reads: make(chan []byte, PipeChannelBuffSize),
		writes: make(chan []byte, PipeChannelBuffSize),
		writeable: make(chan int, PipeChannelBuffSize),
		writeFlush: make(chan int, 1),
		cmds: make(chan *command, PipeChannelBuffSize),
		acksChecker: time.NewTicker(defaultAckCheckDelay),
	}

	pipe.waitGroup.Add(3)
	pipe.ioComplete <- 1
	pipe.ioComplete <- 1
	go pipe.readLoop()
	go pipe.writeLoop()
	go pipe.loop()

	return pipe
}

func (pipe *StreamPipe) Attach(cc *ClientConn) {
	pipe.cc = cc
}

func (pipe *StreamPipe) incrSerial() uint32 {
	pipe.serial++
	pipe.serial = pipe.serial & ^uint32(0)
	return pipe.serial
}

func (pipe *StreamPipe) newPacket() (packet *agent.DataPacket) {
	packet = &agent.DataPacket{
		No: pipe.incrSerial(),
	}
	return
}

func (pipe *StreamPipe) readLoop() {
	defer func() {
		pipe.waitGroup.Done()
		close(pipe.reads)
		<-pipe.ioComplete
	}()

	for {
		//util error(contain eof)
		packet, err := pipe.raw.Recv()
		if err != nil {
			pipe.cancel(err)
			break
		}

		//pop unack
		if len(packet.Acks) > 0 {
			cmd := &command{
				cmd: cmdPopUnack,
				inAcks: packet.Acks,
			}
			pipe.cmds <- cmd
		}

		//push ack and buf
		if len(packet.Buf) > 0 {
			cmd := &command{
				cmd: cmdPushAck,
				inAcks: []uint32{packet.No},
			}
			pipe.cmds <- cmd

			pipe.reads <- packet.Buf
		}
	}
}

//allow first == nil
func (pipe *StreamPipe) handleWrite(first []byte) (err error) {
	//until no need write
	FIRST:
	for {
		storage := bufPool.Get(defaultPacketMaxBytes)
		pos := 0

		//loop a times
		//SECOND:
		for true {
			if pipe.wBuffer.Len() > 0 {
				nr, err := pipe.wBuffer.Read(storage)
				if err != nil {
					return err
				}

				pos += nr
				if pos == len(storage) {    //full
					if first != nil && len(first) == 0 {
						_, err = pipe.wBuffer.Write(first)
						if err != nil {
							return err
						}
					}
					break    //SECOND
				}
			}

			//now, pipe.wBuffer is empty

			if first != nil && len(first) > 0 {
				nc := copy(storage[pos:], first)
				if len(first) > nc {
					pipe.wBuffer = *bytes.NewBuffer(first[nc:])
				} else {
					bufPool.Put(first)
				}
				first = nil

				pos += nc
				if pos == len(storage) {    //full
					break    //SECOND
				}
			}

			THIRD:
			for {
				//non-block
				select {
				case buf, ok := <-pipe.writes:
					if !ok {
						//pipe.writes is closed
						break THIRD
					}
					<-pipe.writeable

					nc := copy(storage[pos:], buf)

					if len(buf) > nc {
						pipe.wBuffer = *bytes.NewBuffer(buf[nc:])
					} else {
						bufPool.Put(buf)
					}

					pos += nc
					if pos == len(storage) {    //full
						break THIRD
					}
				default:
					break THIRD
				}
			}

			break    //SECOND
		}

		//request ack
		cmd := &command{
			cmd: cmdPullAck,
			outAcks: make(chan uint32),
		}
		pipe.cmds <- cmd

		acks := []uint32{}
		for ack := range cmd.outAcks {
			acks = append(acks, ack)
		}

		if pos == 0 && len(acks) == 0 {
			//no need write
			bufPool.Put(storage)
			break FIRST
		}

		packet := pipe.newPacket()
		packet.Buf = storage[:pos]
		packet.Acks = acks

		if pos > 0 {
			//need ack
			cmd = &command{
				cmd: cmdPushUnack,
				inAcks: []uint32{packet.No},
			}
			pipe.cmds <- cmd
		}

		err = pipe.raw.Send(packet)
		bufPool.Put(storage)
		if err != nil {
			return err
		}
	}

	return nil
}

func (pipe *StreamPipe) writeLoop() {
	defer func() {
		pipe.waitGroup.Done()
		<-pipe.ioComplete
	}()

	var err error = nil

	//util pipe.writes is closed
	FIRST:
	for {
		select {
		case buf, ok := <-pipe.writes:
			if !ok {
				//pipe.writes is closed
				break FIRST
			}

			<-pipe.writeable
			err = pipe.handleWrite(buf)
			if err != nil {
				pipe.cancel(err)
				break FIRST
			}
		case <-pipe.writeFlush:
			err = pipe.handleWrite(nil)
			if err != nil {
				pipe.cancel(err)
				break FIRST
			}
		}
	}

	if err != nil {
		//discard packet
		for _ = range pipe.writes {}
	} else {
		//client actively close the stream
		//server wait for the peer to close the stream
		if s, ok := pipe.raw.(grpc.ClientStream); ok {
			if err := s.CloseSend(); err != nil {
				pipe.cancel(err)
			}
		}
	}
}

func (pipe *StreamPipe) ackCheck() error {
	if len(pipe.unAcks) == 0 {
		return nil
	}

	t := pipe.unAcks[0].t
	if time.Now().Sub(t) > defaultAckMaxDelay {
		return ErrAckTimeout
	}

	return nil
}

func (pipe *StreamPipe) handleCommand(cmd *command) (err error) {
	switch cmd.cmd {
	case cmdPushAck:
		pipe.acks = append(pipe.acks, cmd.inAcks...)
		if len(pipe.writes) == 0 {
			//Do not repeat flush
			select {
			case pipe.writeFlush <- 1:
			default:
			}
		}
	case cmdPullAck:
		uplimit := intMin(len(pipe.acks), PipeAcksMaxSize)
		acks := pipe.acks[:uplimit]
		pipe.acks = pipe.acks[uplimit:]

		for _, ack := range acks {
			cmd.outAcks <- ack
		}
		close(cmd.outAcks)
	case cmdPushUnack:
		for _, unack := range cmd.inAcks {
			unAck := newUnAck(unack)
			pipe.unAcks = append(pipe.unAcks, unAck)
		}
	case cmdPopUnack:
		lenght := len(cmd.inAcks)
		if lenght > len(pipe.unAcks) {
			return errors.New("ack number error")
		}
		unAcks := pipe.unAcks[:lenght]
		pipe.unAcks = pipe.unAcks[lenght:]

		for i, unack := range cmd.inAcks {
			if unack != unAcks[i].no {
				return errors.New("ack error")
			}
		}
	}
	return nil
}

//main loop
func (pipe *StreamPipe) loop() {
	defer pipe.waitGroup.Done()

	var err error = nil

	FIRST:
	for {
		select {
		case cmd := <-pipe.cmds:
			err = pipe.handleCommand(cmd)
		case <-pipe.acksChecker.C:
			err = pipe.ackCheck()
		case <-pipe.ctx.Done():
			break FIRST
		}

		if err != nil {
			pipe.cancel(err)
			break
		}
	}

	//util writerLoop and readLoop exit
	counting := cap(pipe.ioComplete)
	SECOND:
	for {
		select {
		case cmd := <-pipe.cmds:
			if cmd.outAcks != nil {
				close(cmd.outAcks)
			}
		case pipe.ioComplete <- 1:
			counting--
			if counting == 0 {
				//io completed
				break SECOND
			}
		}
	}
}

func (pipe *StreamPipe) Err() (err error) {
	p := (*error)(atomic.LoadPointer(&pipe.err))
	if p != nil {
		return *p
	}

	return pipe.ctx.Err()
}

func (pipe*StreamPipe) cancel(err error) {
	if err == nil {
		panic("context: internal error: missing cancel error")
	}

	atomic.CompareAndSwapPointer(&pipe.err, nil, unsafe.Pointer(&err))

	if pipe.ctx.Err() != nil {
		//already canceled
		return
	}
	pipe.cancelFunc()
	close(pipe.writes)
}

//unsafe
func (pipe *StreamPipe) Read(buf []byte) (n int, err error) {
	if pipe.rCache.Len() > 0 {
		n, err = pipe.rCache.Read(buf)
		if err != nil {
			return n, err
		}

		//full or error
		if n == len(buf) || err != nil {
			return n, err
		}
	}

	//non-block
	FIRST:
	for {
		select {
		case bs, ok := <-pipe.reads:
			if !ok {
				//pipe.reads is closed
				if n > 0 {
					return n, nil
				} else {
					return n, pipe.Err()
				}
			}

			var nc int
			nc = copy(buf[n:], bs)
			n += nc

			if nc < len(bs) {    //buf must is full
				pipe.rCache = *bytes.NewBuffer(bs[nc:])
			}

		//full or error
			if n == len(buf) || err != nil {
				return n, err
			}
		default:
		//non-block
			break FIRST
		}
	}

	if n > 0 {
		return n, err
	}

	//wait data coming
	for {
		select {
		case bs, ok := <-pipe.reads:
			if !ok {
				//pipe.reads is closed
				return n, pipe.Err()        //n == 0
			}

			for {
				var nc int
				nc = copy(buf[n:], bs)
				n += nc

				if nc < len(bs) {    //buf must is full
					pipe.rCache = *bytes.NewBuffer(bs[nc:])
				}

				//full or error
				if n == len(buf) || err != nil {
					return n, err
				}

				select {
				case bs, ok = <-pipe.reads:
					if !ok {
						//pipe.reads is closed
						if n > 0 {
							return n, nil
						} else {
							return n, pipe.Err()
						}
					}
				default:
				//no more data
					return n, nil
				}
			}
		case <-pipe.ctx.Done():
			return n, pipe.Err()    //n == 0
		}
	}

	return n, err
}

//unsafe
func (pipe *StreamPipe) Write(buf []byte) (n int, err error) {
	//safe write
	select {
	case <-pipe.ctx.Done():
		return 0, pipe.Err()
	case pipe.writeable <- 1:
			select {
			case <-pipe.ctx.Done():
				return 0, pipe.Err()
			default:
				mem := bufPool.Get(len(buf))
				nc := copy(mem, buf)
				pipe.writes <- mem
				return nc, nil
			}
	}

	return 0, nil
}

func (pipe *StreamPipe) CloseWithError(e error) (err error) {
	if e == nil {
		e = io.EOF
	}
	pipe.cancel(e)

	pipe.acksChecker.Stop()

	pipe.waitGroup.Wait()

	if pipe.cc != nil {
		return pipe.cc.Close()
	}

	return nil
}

func (pipe *StreamPipe) Close() (err error) {
	return pipe.CloseWithError(nil)
}

func (pipe *StreamPipe) LocalAddr() net.Addr {
	return streamPipeAddr(0)
}

func (pipe *StreamPipe) RemoteAddr() net.Addr {
	return streamPipeAddr(0)
}

func (pipe *StreamPipe) SetDeadline(t time.Time) error {
	go func() {
		ctx, _ := context.WithDeadline(pipe.ctx, t)
		<-ctx.Done()
		pipe.cancel(ctx.Err())
	}()
	return nil
}

func (pipe *StreamPipe) SetReadDeadline(t time.Time) error {
	return errors.New("deadLine not supported")
}

func (pipe *StreamPipe) SetWriteDeadline(t time.Time) error {
	return errors.New("deadLine not supported")
}

type streamPipeAddr int

func (streamPipeAddr) Network() string {
	return "StreamPipe"
}

func (streamPipeAddr) String() string {
	return "StreamPipe"
}
<commit_msg>修正internal/StreamPipe的cancel竞争错误<commit_after>package internal

import (
	"../agent"
	"google.golang.org/grpc"
	"golang.org/x/net/context"
	"sync"
	"bytes"
	"io"
	"time"
	"errors"
	"net"
	"unsafe"
	"sync/atomic"
)

const (
	PipeChannelBuffSize int = 10
	PipeAcksMaxSize int = 100
)

var bufPool *BufPool = NewBufPool(1024 * 1024 * 1024)

func intMin(x, y int) int {
	if x < y {
		return x
	} else {
		return y
	}
}

type agentStream interface {
	Send(*agent.DataPacket) error
	Recv() (*agent.DataPacket, error)
}

type unAck struct {
	no uint32
	t  time.Time
}

func newUnAck(no uint32) *unAck {
	return &unAck{
		no: no,
		t: time.Now(),
	}
}

const (
	cmdPushAck = iota
	cmdPullAck
	cmdPopUnack
	cmdPushUnack
)

type command struct {
	cmd     int
	inAcks  []uint32
	outAcks chan uint32
}

//wrap grpc stream as net.Conn
type StreamPipe struct {
	ctx         context.Context
	cancelFunc  context.CancelFunc

	raw         agentStream
	cc          *ClientConn //may is nil

	waitGroup   sync.WaitGroup
	ioComplete  chan int

	reads       chan []byte
	writes      chan []byte
	writeable   chan int
	writeFlush  chan int

	cmds        chan *command

	acks        []uint32
	unAcks      []*unAck

	wBuffer     bytes.Buffer
	rCache      bytes.Buffer

	err         unsafe.Pointer

	acksChecker *time.Ticker

	serial      uint32
}

func NewStreamPipe(stream agentStream) *StreamPipe {
	ctx, cancelFunc := context.WithCancel(context.Background())
	pipe := &StreamPipe{
		ctx: ctx,
		cancelFunc:cancelFunc,
		raw: stream,
		ioComplete: make(chan int, 2),
		reads: make(chan []byte, PipeChannelBuffSize),
		writes: make(chan []byte, PipeChannelBuffSize),
		writeable: make(chan int, PipeChannelBuffSize),
		writeFlush: make(chan int, 1),
		cmds: make(chan *command, PipeChannelBuffSize),
		acksChecker: time.NewTicker(defaultAckCheckDelay),
	}

	pipe.waitGroup.Add(3)
	pipe.ioComplete <- 1
	pipe.ioComplete <- 1
	go pipe.readLoop()
	go pipe.writeLoop()
	go pipe.loop()

	return pipe
}

func (pipe *StreamPipe) Attach(cc *ClientConn) {
	pipe.cc = cc
}

func (pipe *StreamPipe) incrSerial() uint32 {
	pipe.serial++
	pipe.serial = pipe.serial & ^uint32(0)
	return pipe.serial
}

func (pipe *StreamPipe) newPacket() (packet *agent.DataPacket) {
	packet = &agent.DataPacket{
		No: pipe.incrSerial(),
	}
	return
}

func (pipe *StreamPipe) readLoop() {
	defer func() {
		pipe.waitGroup.Done()
		close(pipe.reads)
		<-pipe.ioComplete
	}()

	for {
		//util error(contain eof)
		packet, err := pipe.raw.Recv()
		if err != nil {
			pipe.cancel(err)
			break
		}

		//pop unack
		if len(packet.Acks) > 0 {
			cmd := &command{
				cmd: cmdPopUnack,
				inAcks: packet.Acks,
			}
			pipe.cmds <- cmd
		}

		//push ack and buf
		if len(packet.Buf) > 0 {
			cmd := &command{
				cmd: cmdPushAck,
				inAcks: []uint32{packet.No},
			}
			pipe.cmds <- cmd

			pipe.reads <- packet.Buf
		}
	}
}

//allow first == nil
func (pipe *StreamPipe) handleWrite(first []byte) (err error) {
	//until no need write
	FIRST:
	for {
		storage := bufPool.Get(defaultPacketMaxBytes)
		pos := 0

		//loop a times
		//SECOND:
		for true {
			if pipe.wBuffer.Len() > 0 {
				nr, err := pipe.wBuffer.Read(storage)
				if err != nil {
					return err
				}

				pos += nr
				if pos == len(storage) {    //full
					if first != nil && len(first) == 0 {
						_, err = pipe.wBuffer.Write(first)
						if err != nil {
							return err
						}
					}
					break    //SECOND
				}
			}

			//now, pipe.wBuffer is empty

			if first != nil && len(first) > 0 {
				nc := copy(storage[pos:], first)
				if len(first) > nc {
					pipe.wBuffer = *bytes.NewBuffer(first[nc:])
				} else {
					bufPool.Put(first)
				}
				first = nil

				pos += nc
				if pos == len(storage) {    //full
					break    //SECOND
				}
			}

			THIRD:
			for {
				//non-block
				select {
				case buf, ok := <-pipe.writes:
					if !ok {
						//pipe.writes is closed
						break THIRD
					}
					<-pipe.writeable

					nc := copy(storage[pos:], buf)

					if len(buf) > nc {
						pipe.wBuffer = *bytes.NewBuffer(buf[nc:])
					} else {
						bufPool.Put(buf)
					}

					pos += nc
					if pos == len(storage) {    //full
						break THIRD
					}
				default:
					break THIRD
				}
			}

			break    //SECOND
		}

		//request ack
		cmd := &command{
			cmd: cmdPullAck,
			outAcks: make(chan uint32),
		}
		pipe.cmds <- cmd

		acks := []uint32{}
		for ack := range cmd.outAcks {
			acks = append(acks, ack)
		}

		if pos == 0 && len(acks) == 0 {
			//no need write
			bufPool.Put(storage)
			break FIRST
		}

		packet := pipe.newPacket()
		packet.Buf = storage[:pos]
		packet.Acks = acks

		if pos > 0 {
			//need ack
			cmd = &command{
				cmd: cmdPushUnack,
				inAcks: []uint32{packet.No},
			}
			pipe.cmds <- cmd
		}

		err = pipe.raw.Send(packet)
		bufPool.Put(storage)
		if err != nil {
			return err
		}
	}

	return nil
}

func (pipe *StreamPipe) writeLoop() {
	defer func() {
		pipe.waitGroup.Done()
		<-pipe.ioComplete
	}()

	var err error = nil

	//util pipe.writes is closed
	FIRST:
	for {
		select {
		case buf, ok := <-pipe.writes:
			if !ok {
				//pipe.writes is closed
				break FIRST
			}

			<-pipe.writeable
			err = pipe.handleWrite(buf)
			if err != nil {
				pipe.cancel(err)
				break FIRST
			}
		case <-pipe.writeFlush:
			err = pipe.handleWrite(nil)
			if err != nil {
				pipe.cancel(err)
				break FIRST
			}
		}
	}

	if err != nil {
		//discard packet
		for _ = range pipe.writes {}
	} else {
		//client actively close the stream
		//server wait for the peer to close the stream
		if s, ok := pipe.raw.(grpc.ClientStream); ok {
			if err := s.CloseSend(); err != nil {
				pipe.cancel(err)
			}
		}
	}
}

func (pipe *StreamPipe) ackCheck() error {
	if len(pipe.unAcks) == 0 {
		return nil
	}

	t := pipe.unAcks[0].t
	if time.Now().Sub(t) > defaultAckMaxDelay {
		return ErrAckTimeout
	}

	return nil
}

func (pipe *StreamPipe) handleCommand(cmd *command) (err error) {
	switch cmd.cmd {
	case cmdPushAck:
		pipe.acks = append(pipe.acks, cmd.inAcks...)
		if len(pipe.writes) == 0 {
			//Do not repeat flush
			select {
			case pipe.writeFlush <- 1:
			default:
			}
		}
	case cmdPullAck:
		uplimit := intMin(len(pipe.acks), PipeAcksMaxSize)
		acks := pipe.acks[:uplimit]
		pipe.acks = pipe.acks[uplimit:]

		for _, ack := range acks {
			cmd.outAcks <- ack
		}
		close(cmd.outAcks)
	case cmdPushUnack:
		for _, unack := range cmd.inAcks {
			unAck := newUnAck(unack)
			pipe.unAcks = append(pipe.unAcks, unAck)
		}
	case cmdPopUnack:
		lenght := len(cmd.inAcks)
		if lenght > len(pipe.unAcks) {
			return errors.New("ack number error")
		}
		unAcks := pipe.unAcks[:lenght]
		pipe.unAcks = pipe.unAcks[lenght:]

		for i, unack := range cmd.inAcks {
			if unack != unAcks[i].no {
				return errors.New("ack error")
			}
		}
	}
	return nil
}

//main loop
func (pipe *StreamPipe) loop() {
	defer pipe.waitGroup.Done()

	var err error = nil

	FIRST:
	for {
		select {
		case cmd := <-pipe.cmds:
			err = pipe.handleCommand(cmd)
		case <-pipe.acksChecker.C:
			err = pipe.ackCheck()
		case <-pipe.ctx.Done():
			break FIRST
		}

		if err != nil {
			pipe.cancel(err)
			break
		}
	}

	//util writerLoop and readLoop exit
	counting := cap(pipe.ioComplete)
	SECOND:
	for {
		select {
		case cmd := <-pipe.cmds:
			if cmd.outAcks != nil {
				close(cmd.outAcks)
			}
		case pipe.ioComplete <- 1:
			counting--
			if counting == 0 {
				//io completed
				break SECOND
			}
		}
	}
}

func (pipe *StreamPipe) Err() (err error) {
	p := (*error)(atomic.LoadPointer(&pipe.err))
	if p != nil {
		return *p
	}

	return pipe.ctx.Err()
}

func (pipe*StreamPipe) cancel(err error) {
	if err == nil {
		panic("context: internal error: missing cancel error")
	}

	ok := atomic.CompareAndSwapPointer(&pipe.err, nil, unsafe.Pointer(&err))
	if !ok {
		//already canceled
		return
	}

	pipe.cancelFunc()
	close(pipe.writes)
}

//unsafe
func (pipe *StreamPipe) Read(buf []byte) (n int, err error) {
	if pipe.rCache.Len() > 0 {
		n, err = pipe.rCache.Read(buf)
		if err != nil {
			return n, err
		}

		//full or error
		if n == len(buf) || err != nil {
			return n, err
		}
	}

	//non-block
	FIRST:
	for {
		select {
		case bs, ok := <-pipe.reads:
			if !ok {
				//pipe.reads is closed
				if n > 0 {
					return n, nil
				} else {
					return n, pipe.Err()
				}
			}

			var nc int
			nc = copy(buf[n:], bs)
			n += nc

			if nc < len(bs) {    //buf must is full
				pipe.rCache = *bytes.NewBuffer(bs[nc:])
			}

		//full or error
			if n == len(buf) || err != nil {
				return n, err
			}
		default:
		//non-block
			break FIRST
		}
	}

	if n > 0 {
		return n, err
	}

	//wait data coming
	for {
		select {
		case bs, ok := <-pipe.reads:
			if !ok {
				//pipe.reads is closed
				return n, pipe.Err()        //n == 0
			}

			for {
				var nc int
				nc = copy(buf[n:], bs)
				n += nc

				if nc < len(bs) {    //buf must is full
					pipe.rCache = *bytes.NewBuffer(bs[nc:])
				}

				//full or error
				if n == len(buf) || err != nil {
					return n, err
				}

				select {
				case bs, ok = <-pipe.reads:
					if !ok {
						//pipe.reads is closed
						if n > 0 {
							return n, nil
						} else {
							return n, pipe.Err()
						}
					}
				default:
				//no more data
					return n, nil
				}
			}
		case <-pipe.ctx.Done():
			return n, pipe.Err()    //n == 0
		}
	}

	return n, err
}

//unsafe
func (pipe *StreamPipe) Write(buf []byte) (n int, err error) {
	//safe write
	select {
	case <-pipe.ctx.Done():
		return 0, pipe.Err()
	case pipe.writeable <- 1:
			select {
			case <-pipe.ctx.Done():
				return 0, pipe.Err()
			default:
				mem := bufPool.Get(len(buf))
				nc := copy(mem, buf)
				pipe.writes <- mem
				return nc, nil
			}
	}

	return 0, nil
}

func (pipe *StreamPipe) CloseWithError(e error) (err error) {
	if e == nil {
		e = io.EOF
	}
	pipe.cancel(e)

	pipe.acksChecker.Stop()

	pipe.waitGroup.Wait()

	if pipe.cc != nil {
		return pipe.cc.Close()
	}

	return nil
}

func (pipe *StreamPipe) Close() (err error) {
	return pipe.CloseWithError(nil)
}

func (pipe *StreamPipe) LocalAddr() net.Addr {
	return streamPipeAddr(0)
}

func (pipe *StreamPipe) RemoteAddr() net.Addr {
	return streamPipeAddr(0)
}

func (pipe *StreamPipe) SetDeadline(t time.Time) error {
	go func() {
		ctx, _ := context.WithDeadline(pipe.ctx, t)
		<-ctx.Done()
		pipe.cancel(ctx.Err())
	}()
	return nil
}

func (pipe *StreamPipe) SetReadDeadline(t time.Time) error {
	return errors.New("deadLine not supported")
}

func (pipe *StreamPipe) SetWriteDeadline(t time.Time) error {
	return errors.New("deadLine not supported")
}

type streamPipeAddr int

func (streamPipeAddr) Network() string {
	return "StreamPipe"
}

func (streamPipeAddr) String() string {
	return "StreamPipe"
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"

	"github.com/ConradIrwin/font/sfnt"
)

// Features prints the gpos/gsub tables (contains font features).
func Features() {
	if len(os.Args) < 2 {
		panic(fmt.Errorf("Specify a font file"))
	}

	file, err := os.Open(os.Args[1])
	if err != nil {
		panic(fmt.Errorf("Failed to open font: %s", err))
	}
	defer file.Close()

	font, err := sfnt.Parse(file)
	if err != nil {
		panic(fmt.Errorf("Failed to parse font: %s", err))
	}

	layoutTable(font, sfnt.TagGsub, "Glyph Substitution Table (GSUB)")
	layoutTable(font, sfnt.TagGpos, "Glyph Positioning Table (GPOS)")
}

func layoutTable(font *sfnt.Font, tag sfnt.Tag, name string) {
	if font.HasTable(tag) {
		fmt.Printf("%s:\n", name)

		t := font.Table(tag).(*sfnt.TableLayout)
		for _, script := range t.Scripts {
			fmt.Printf("\tScript %q (%s):\n", script.Tag, script.String())

			fmt.Printf("\t\tDefault Language:\n")
			for _, feature := range script.DefaultLanguage.Features {
				fmt.Printf("\t\t\tFeature %q (%s)\n", feature.Tag, feature.String())
			}

			for _, lang := range script.Languages {
				fmt.Printf("\t\tLanguage %q (%s):\n", lang.Tag, lang.String())
				for _, feature := range lang.Features {
					fmt.Printf("\t\t\tFeature %q (%s)\n", feature.Tag, feature.String())
				}
			}
		}
	} else {
		fmt.Printf("No %s\n", name)
	}
}
<commit_msg>Change feature to only print the description of the field if it is known.<commit_after>package commands

import (
	"fmt"
	"os"

	"github.com/ConradIrwin/font/sfnt"
)

// Features prints the gpos/gsub tables (contains font features).
func Features() {
	if len(os.Args) < 2 {
		panic(fmt.Errorf("Specify a font file"))
	}

	file, err := os.Open(os.Args[1])
	if err != nil {
		panic(fmt.Errorf("Failed to open font: %s", err))
	}
	defer file.Close()

	font, err := sfnt.Parse(file)
	if err != nil {
		panic(fmt.Errorf("Failed to parse font: %s", err))
	}

	layoutTable(font, sfnt.TagGsub, "Glyph Substitution Table (GSUB)")
	layoutTable(font, sfnt.TagGpos, "Glyph Positioning Table (GPOS)")
}

func layoutTable(font *sfnt.Font, tag sfnt.Tag, name string) {
	if font.HasTable(tag) {
		fmt.Printf("%s:\n", name)

		t := font.Table(tag).(*sfnt.TableLayout)
		for _, script := range t.Scripts {
			fmt.Printf("\tScript %q%s:\n", script.Tag, bracketString(script))

			fmt.Printf("\t\tDefault Language:\n")
			for _, feature := range script.DefaultLanguage.Features {
				fmt.Printf("\t\t\tFeature %q%s\n", feature.Tag, bracketString(feature))
			}

			for _, lang := range script.Languages {
				fmt.Printf("\t\tLanguage %q%s:\n", lang.Tag, bracketString(lang))
				for _, feature := range lang.Features {
					fmt.Printf("\t\t\tFeature %q%s\n", feature.Tag, bracketString(feature))
				}
			}
		}
	} else {
		fmt.Printf("No %s\n", name)
	}
}

func bracketString(o fmt.Stringer) string {
	if s := o.String(); s != "" {
		return fmt.Sprintf(" (%s)", s)
	}
	return ""
}
<|endoftext|>


<commit_before>package commands

import (
    "crypto/rand"
    "crypto/rsa"
    "fmt"
    "os"
    "path"
    "strings"

    log "github.com/mgutz/logxi/v1"
    "github.com/spf13/cobra"
    "github.com/spf13/viper"

    "github.com/blacklabeldata/kappa/auth"
)

// NewCertCmd is the kappa root command.
var NewCertCmd = &cobra.Command{
    Use:   "new-cert",
    Short: "new-cert creates a new certificate",
    Long:  ``,
    Run: func(cmd *cobra.Command, args []string) {

        // Create logger
        writer := log.NewConcurrentWriter(os.Stdout)
        logger := log.NewLogger(writer, "new-cert")

        err := InitializeConfig(writer)
        if err != nil {
            return
        }

        // Setup directory structure
        if err := auth.CreatePkiDirectories(logger, "."); err != nil {
            return
        }

        // Create file paths
        pki := path.Join(".", "pki")
        reqFile := path.Join(pki, "reqs", viper.GetString("Name")+".req")
        privFile := path.Join(pki, "private", viper.GetString("Name")+".key")
        crtFile := path.Join(pki, "public", viper.GetString("Name")+".crt")

        // Verify it is ok to delete files if they exist
        if !viper.GetBool("ForceOverwrite") {
            var files []string
            for _, filename := range []string{reqFile, privFile, crtFile} {
                if _, err := os.Stat(filename); err == nil {
                    files = append(files, filename)
                }
            }

            if len(files) > 0 {
                var input string
                fmt.Println("This operation will overwrite these existing files:")
                for _, file := range files {
                    fmt.Println("\t", file)
                }
                fmt.Print("Are you sure you want to overwrite these files (yN)? ")
                fmt.Scanln(&input)

                if !strings.Contains(strings.ToLower(input), "y") {
                    fmt.Println("New certificate was not created.")
                    return
                }
            }
        }

        // generate private key
        privatekey, err := rsa.GenerateKey(rand.Reader, viper.GetInt("Bits"))
        if err != nil {
            logger.Warn("Error generating private key")
            return
        }

        // Create Certificate request
        csr, req, err := auth.CreateCertificateRequest(logger, privatekey,
            viper.GetString("Name"), viper.GetString("Organization"),
            viper.GetString("Country"), viper.GetString("Hosts"))
        if err != nil {
            logger.Warn("Error creating CA", "err", err.Error())
            return
        }

        // Create Certificate
        crt, err := auth.CreateCertificate(logger, csr, privatekey,
            viper.GetInt("Years"), viper.GetString("Hosts"))
        if err != nil {
            logger.Warn("Error creating certificate", "err", err.Error())
            return
        }

        // Save cert request
        auth.SaveCertificateRequest(logger, req, reqFile)

        // Save private key
        auth.SavePrivateKey(logger, privatekey, privFile)

        // Save certificate
        auth.SaveCertificate(logger, crt, crtFile)
    },
}

// Pointer to NewCertCmd used in initialization
var newCertCmd *cobra.Command

// Command line args
var (
    Name           string
    ForceOverwrite bool
)

func init() {

    NewCertCmd.PersistentFlags().IntVarP(&KeyBits, "bits", "", 4096, "Number of bits in key")
    NewCertCmd.PersistentFlags().StringVarP(&Hosts, "hosts", "", "127.0.0.1", "IP of cert")
    NewCertCmd.PersistentFlags().IntVarP(&Years, "years", "", 10, "Number of years until the certificate expires")
    NewCertCmd.PersistentFlags().StringVarP(&Organization, "organization", "", "kappa-ca", "Organization for CA")
    NewCertCmd.PersistentFlags().StringVarP(&Country, "country", "", "USA", "Country of origin for CA")
    NewCertCmd.PersistentFlags().StringVarP(&Name, "name", "", "localhost", "Name of certificate")
    NewCertCmd.PersistentFlags().BoolVarP(&ForceOverwrite, "overwrite", "", false, "Overwrite replaces existing certs")
    newCertCmd = NewCertCmd
}

// InitializeNewCertConfig sets up the command line options for creating a new certificate
func InitializeNewCertConfig(logger log.Logger) error {
    viper.SetDefault("Name", "localhost")
    viper.SetDefault("ForceOverwrite", "false")

    if newCertCmd.PersistentFlags().Lookup("name").Changed {
        logger.Info("", "Name", Name)
        viper.Set("Name", Name)
    }
    if newCertCmd.PersistentFlags().Lookup("overwrite").Changed {
        logger.Info("", "ForceOverwrite", ForceOverwrite)
        viper.Set("ForceOverwrite", ForceOverwrite)
    }

    return nil
}
<commit_msg>Fix formatting for new-cert command<commit_after>package commands

import (
	"crypto/rand"
	"crypto/rsa"
	"fmt"
	"os"
	"path"
	"strings"

	log "github.com/mgutz/logxi/v1"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/blacklabeldata/kappa/auth"
)

// NewCertCmd is the kappa root command.
var NewCertCmd = &cobra.Command{
	Use:   "new-cert",
	Short: "new-cert creates a new certificate",
	Long:  ``,
	Run: func(cmd *cobra.Command, args []string) {

		// Create logger
		writer := log.NewConcurrentWriter(os.Stdout)
		logger := log.NewLogger(writer, "new-cert")

		err := InitializeConfig(writer)
		if err != nil {
			return
		}

		// Setup directory structure
		if err := auth.CreatePkiDirectories(logger, "."); err != nil {
			return
		}

		// Create file paths
		pki := path.Join(".", "pki")
		reqFile := path.Join(pki, "reqs", viper.GetString("Name")+".req")
		privFile := path.Join(pki, "private", viper.GetString("Name")+".key")
		crtFile := path.Join(pki, "public", viper.GetString("Name")+".crt")

		// Verify it is ok to delete files if they exist
		if !viper.GetBool("ForceOverwrite") {
			var files []string
			for _, filename := range []string{reqFile, privFile, crtFile} {
				if _, err := os.Stat(filename); err == nil {
					files = append(files, filename)
				}
			}

			if len(files) > 0 {
				var input string
				fmt.Println("This operation will overwrite these existing files:")
				for _, file := range files {
					fmt.Println("\t", file)
				}
				fmt.Print("Are you sure you want to overwrite these files (yN)? ")
				fmt.Scanln(&input)

				if !strings.Contains(strings.ToLower(input), "y") {
					fmt.Println("New certificate was not created.")
					return
				}
			}
		}

		// generate private key
		privatekey, err := rsa.GenerateKey(rand.Reader, viper.GetInt("Bits"))
		if err != nil {
			logger.Warn("Error generating private key")
			return
		}

		// Create Certificate request
		csr, req, err := auth.CreateCertificateRequest(logger, privatekey,
			viper.GetString("Name"), viper.GetString("Organization"),
			viper.GetString("Country"), viper.GetString("Hosts"))
		if err != nil {
			logger.Warn("Error creating CA", "err", err.Error())
			return
		}

		// Create Certificate
		crt, err := auth.CreateCertificate(logger, csr, privatekey,
			viper.GetInt("Years"), viper.GetString("Hosts"))
		if err != nil {
			logger.Warn("Error creating certificate", "err", err.Error())
			return
		}

		// Save cert request
		auth.SaveCertificateRequest(logger, req, reqFile)

		// Save private key
		auth.SavePrivateKey(logger, privatekey, privFile)

		// Save certificate
		auth.SaveCertificate(logger, crt, crtFile)
	},
}

// Pointer to NewCertCmd used in initialization
var newCertCmd *cobra.Command

// Command line args
var (
	Name           string
	ForceOverwrite bool
)

func init() {

	NewCertCmd.PersistentFlags().IntVarP(&KeyBits, "bits", "", 4096, "Number of bits in key")
	NewCertCmd.PersistentFlags().StringVarP(&Hosts, "hosts", "", "127.0.0.1", "IP of cert")
	NewCertCmd.PersistentFlags().IntVarP(&Years, "years", "", 10, "Number of years until the certificate expires")
	NewCertCmd.PersistentFlags().StringVarP(&Organization, "organization", "", "kappa-ca", "Organization for CA")
	NewCertCmd.PersistentFlags().StringVarP(&Country, "country", "", "USA", "Country of origin for CA")
	NewCertCmd.PersistentFlags().StringVarP(&Name, "name", "", "localhost", "Name of certificate")
	NewCertCmd.PersistentFlags().BoolVarP(&ForceOverwrite, "overwrite", "", false, "Overwrite replaces existing certs")
	newCertCmd = NewCertCmd
}

// InitializeNewCertConfig sets up the command line options for creating a new certificate
func InitializeNewCertConfig(logger log.Logger) error {
	viper.SetDefault("Name", "localhost")
	viper.SetDefault("ForceOverwrite", "false")

	if newCertCmd.PersistentFlags().Lookup("name").Changed {
		logger.Info("", "Name", Name)
		viper.Set("Name", Name)
	}
	if newCertCmd.PersistentFlags().Lookup("overwrite").Changed {
		logger.Info("", "ForceOverwrite", ForceOverwrite)
		viper.Set("ForceOverwrite", ForceOverwrite)
	}

	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/rds"
)

func TestAccAWSDBSubnetGroup_basic(t *testing.T) {
	var v rds.DBSubnetGroup

	testCheck := func(*terraform.State) error {
		return nil
	}

	rName := fmt.Sprintf("tf-test-%d", acctest.RandInt())

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.foo", &v),
					resource.TestCheckResourceAttr(
						"aws_db_subnet_group.foo", "name", rName),
					resource.TestCheckResourceAttr(
						"aws_db_subnet_group.foo", "description", "Managed by Terraform"),
					testCheck,
				),
			},
		},
	})
}

func TestAccAWSDBSubnetGroup_namePrefix(t *testing.T) {
	var v rds.DBSubnetGroup

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig_namePrefix,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.test", &v),
					resource.TestMatchResourceAttr(
						"aws_db_subnet_group.test", "name", regexp.MustCompile("^tf_test-")),
				),
			},
		},
	})
}

func TestAccAWSDBSubnetGroup_generatedName(t *testing.T) {
	var v rds.DBSubnetGroup

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig_generatedName,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.test", &v),
				),
			},
		},
	})
}

// Regression test for https://github.com/hashicorp/terraform/issues/2603 and
// https://github.com/hashicorp/terraform/issues/2664
func TestAccAWSDBSubnetGroup_withUndocumentedCharacters(t *testing.T) {
	var v rds.DBSubnetGroup

	testCheck := func(*terraform.State) error {
		return nil
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig_withUnderscoresAndPeriodsAndSpaces,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.underscores", &v),
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.periods", &v),
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.spaces", &v),
					testCheck,
				),
			},
		},
	})
}

func TestAccAWSDBSubnetGroup_updateDescription(t *testing.T) {
	var v rds.DBSubnetGroup

	rName := fmt.Sprintf("tf-test-%d", acctest.RandInt())
	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.foo", &v),
					resource.TestCheckResourceAttr(
						"aws_db_subnet_group.foo", "description", "Managed by Terraform"),
				),
			},

			resource.TestStep{
				Config: testAccDBSubnetGroupConfig_updatedDescription(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.foo", &v),
					resource.TestCheckResourceAttr(
						"aws_db_subnet_group.foo", "description", "foo description updated"),
				),
			},
		},
	})
}

func testAccCheckDBSubnetGroupDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).rdsconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_db_subnet_group" {
			continue
		}

		// Try to find the resource
		resp, err := conn.DescribeDBSubnetGroups(
			&rds.DescribeDBSubnetGroupsInput{DBSubnetGroupName: aws.String(rs.Primary.ID)})
		if err == nil {
			if len(resp.DBSubnetGroups) > 0 {
				return fmt.Errorf("still exist.")
			}

			return nil
		}

		// Verify the error is what we want
		rdserr, ok := err.(awserr.Error)
		if !ok {
			return err
		}
		if rdserr.Code() != "DBSubnetGroupNotFoundFault" {
			return err
		}
	}

	return nil
}

func testAccCheckDBSubnetGroupExists(n string, v *rds.DBSubnetGroup) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).rdsconn
		resp, err := conn.DescribeDBSubnetGroups(
			&rds.DescribeDBSubnetGroupsInput{DBSubnetGroupName: aws.String(rs.Primary.ID)})
		if err != nil {
			return err
		}
		if len(resp.DBSubnetGroups) == 0 {
			return fmt.Errorf("DbSubnetGroup not found")
		}

		*v = *resp.DBSubnetGroups[0]

		return nil
	}
}

func testAccDBSubnetGroupConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
	tags {
		Name = "terraform-testacc-db-subnet-group"
	}
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	availability_zone = "us-west-2a"
	vpc_id = "${aws_vpc.foo.id}"
	tags {
		Name = "tf-acc-db-subnet-group-1"
	}
}

resource "aws_subnet" "bar" {
	cidr_block = "10.1.2.0/24"
	availability_zone = "us-west-2b"
	vpc_id = "${aws_vpc.foo.id}"
	tags {
		Name = "tf-acc-db-subnet-group-2"
	}
}

resource "aws_db_subnet_group" "foo" {
	name = "%s"
	subnet_ids = ["${aws_subnet.foo.id}", "${aws_subnet.bar.id}"]
	tags {
		Name = "tf-dbsubnet-group-test"
	}
}`, rName)
}

func testAccDBSubnetGroupConfig_updatedDescription(rName string) string {
	return fmt.Sprintf(`
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
	tags {
		Name = "terraform-testacc-db-subnet-group-updated-description"
	}
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	availability_zone = "us-west-2a"
	vpc_id = "${aws_vpc.foo.id}"
	tags {
		Name = "tf-acc-db-subnet-group-1"
	}
}

resource "aws_subnet" "bar" {
	cidr_block = "10.1.2.0/24"
	availability_zone = "us-west-2b"
	vpc_id = "${aws_vpc.foo.id}"
	tags {
		Name = "tf-acc-db-subnet-group-2"
	}
}

resource "aws_db_subnet_group" "foo" {
	name = "%s"
	description = "foo description updated"
	subnet_ids = ["${aws_subnet.foo.id}", "${aws_subnet.bar.id}"]
	tags {
		Name = "tf-dbsubnet-group-test"
	}
}`, rName)
}

const testAccDBSubnetGroupConfig_namePrefix = `
resource "aws_vpc" "test" {
	cidr_block = "10.1.0.0/16"
	tags {
		Name = "terraform-testacc-db-subnet-group-name-prefix"
	}
}

resource "aws_subnet" "a" {
	vpc_id = "${aws_vpc.test.id}"
	cidr_block = "10.1.1.0/24"
	availability_zone = "us-west-2a"
	tags {
		Name = "tf-acc-db-subnet-group-name-prefix-a"
	}
}

resource "aws_subnet" "b" {
	vpc_id = "${aws_vpc.test.id}"
	cidr_block = "10.1.2.0/24"
	availability_zone = "us-west-2b"
	tags {
		Name = "tf-acc-db-subnet-group-name-prefix-b"
	}
}

resource "aws_db_subnet_group" "test" {
	name_prefix = "tf_test-"
	subnet_ids = ["${aws_subnet.a.id}", "${aws_subnet.b.id}"]
}`

const testAccDBSubnetGroupConfig_generatedName = `
resource "aws_vpc" "test" {
	cidr_block = "10.1.0.0/16"
	tags {
		Name = "terraform-testacc-db-subnet-group-generated-name"
	}
}

resource "aws_subnet" "a" {
	vpc_id = "${aws_vpc.test.id}"
	cidr_block = "10.1.1.0/24"
	availability_zone = "us-west-2a"
	tags {
		Name = "tf-acc-db-subnet-group-generated-name-a"
	}
}

resource "aws_subnet" "b" {
	vpc_id = "${aws_vpc.test.id}"
	cidr_block = "10.1.2.0/24"
	availability_zone = "us-west-2b"
	tags {
		Name = "tf-acc-db-subnet-group-generated-name-a"
	}
}

resource "aws_db_subnet_group" "test" {
	subnet_ids = ["${aws_subnet.a.id}", "${aws_subnet.b.id}"]
}`

const testAccDBSubnetGroupConfig_withUnderscoresAndPeriodsAndSpaces = `
resource "aws_vpc" "main" {
    cidr_block = "192.168.0.0/16"
		tags {
			Name = "terraform-testacc-db-subnet-group-w-underscores-etc"
		}
}

resource "aws_subnet" "frontend" {
    vpc_id = "${aws_vpc.main.id}"
    availability_zone = "us-west-2b"
    cidr_block = "192.168.1.0/24"
    tags {
        Name = "tf-acc-db-subnet-group-w-underscores-etc-front"
    }
}

resource "aws_subnet" "backend" {
    vpc_id = "${aws_vpc.main.id}"
    availability_zone = "us-west-2c"
    cidr_block = "192.168.2.0/24"
    tags {
        Name = "tf-acc-db-subnet-group-w-underscores-etc-back"
    }
}

resource "aws_db_subnet_group" "underscores" {
    name = "with_underscores"
    description = "Our main group of subnets"
    subnet_ids = ["${aws_subnet.frontend.id}", "${aws_subnet.backend.id}"]
}

resource "aws_db_subnet_group" "periods" {
    name = "with.periods"
    description = "Our main group of subnets"
    subnet_ids = ["${aws_subnet.frontend.id}", "${aws_subnet.backend.id}"]
}

resource "aws_db_subnet_group" "spaces" {
    name = "with spaces"
    description = "Our main group of subnets"
    subnet_ids = ["${aws_subnet.frontend.id}", "${aws_subnet.backend.id}"]
}
`
<commit_msg>add acceptance test for db subnet group<commit_after>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/rds"
)

func TestAccAWSDBSubnetGroup_basic(t *testing.T) {
	var v rds.DBSubnetGroup

	testCheck := func(*terraform.State) error {
		return nil
	}

	rName := fmt.Sprintf("tf-test-%d", acctest.RandInt())

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.foo", &v),
					resource.TestCheckResourceAttr(
						"aws_db_subnet_group.foo", "name", rName),
					resource.TestCheckResourceAttr(
						"aws_db_subnet_group.foo", "description", "Managed by Terraform"),
					resource.TestMatchResourceAttr(
						"aws_db_subnet_group.foo", "arn", regexp.MustCompile(fmt.Sprintf("^arn:[^:]+:rds:[^:]+:\\d{12}:subgrp:%s", rName))),
					testCheck,
				),
			},
		},
	})
}

func TestAccAWSDBSubnetGroup_namePrefix(t *testing.T) {
	var v rds.DBSubnetGroup

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig_namePrefix,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.test", &v),
					resource.TestMatchResourceAttr(
						"aws_db_subnet_group.test", "name", regexp.MustCompile("^tf_test-")),
				),
			},
		},
	})
}

func TestAccAWSDBSubnetGroup_generatedName(t *testing.T) {
	var v rds.DBSubnetGroup

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig_generatedName,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.test", &v),
				),
			},
		},
	})
}

// Regression test for https://github.com/hashicorp/terraform/issues/2603 and
// https://github.com/hashicorp/terraform/issues/2664
func TestAccAWSDBSubnetGroup_withUndocumentedCharacters(t *testing.T) {
	var v rds.DBSubnetGroup

	testCheck := func(*terraform.State) error {
		return nil
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig_withUnderscoresAndPeriodsAndSpaces,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.underscores", &v),
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.periods", &v),
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.spaces", &v),
					testCheck,
				),
			},
		},
	})
}

func TestAccAWSDBSubnetGroup_updateDescription(t *testing.T) {
	var v rds.DBSubnetGroup

	rName := fmt.Sprintf("tf-test-%d", acctest.RandInt())
	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDBSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDBSubnetGroupConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.foo", &v),
					resource.TestCheckResourceAttr(
						"aws_db_subnet_group.foo", "description", "Managed by Terraform"),
				),
			},

			resource.TestStep{
				Config: testAccDBSubnetGroupConfig_updatedDescription(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDBSubnetGroupExists(
						"aws_db_subnet_group.foo", &v),
					resource.TestCheckResourceAttr(
						"aws_db_subnet_group.foo", "description", "foo description updated"),
				),
			},
		},
	})
}

func testAccCheckDBSubnetGroupDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).rdsconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_db_subnet_group" {
			continue
		}

		// Try to find the resource
		resp, err := conn.DescribeDBSubnetGroups(
			&rds.DescribeDBSubnetGroupsInput{DBSubnetGroupName: aws.String(rs.Primary.ID)})
		if err == nil {
			if len(resp.DBSubnetGroups) > 0 {
				return fmt.Errorf("still exist.")
			}

			return nil
		}

		// Verify the error is what we want
		rdserr, ok := err.(awserr.Error)
		if !ok {
			return err
		}
		if rdserr.Code() != "DBSubnetGroupNotFoundFault" {
			return err
		}
	}

	return nil
}

func testAccCheckDBSubnetGroupExists(n string, v *rds.DBSubnetGroup) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).rdsconn
		resp, err := conn.DescribeDBSubnetGroups(
			&rds.DescribeDBSubnetGroupsInput{DBSubnetGroupName: aws.String(rs.Primary.ID)})
		if err != nil {
			return err
		}
		if len(resp.DBSubnetGroups) == 0 {
			return fmt.Errorf("DbSubnetGroup not found")
		}

		*v = *resp.DBSubnetGroups[0]

		return nil
	}
}

func testAccDBSubnetGroupConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
	tags {
		Name = "terraform-testacc-db-subnet-group"
	}
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	availability_zone = "us-west-2a"
	vpc_id = "${aws_vpc.foo.id}"
	tags {
		Name = "tf-acc-db-subnet-group-1"
	}
}

resource "aws_subnet" "bar" {
	cidr_block = "10.1.2.0/24"
	availability_zone = "us-west-2b"
	vpc_id = "${aws_vpc.foo.id}"
	tags {
		Name = "tf-acc-db-subnet-group-2"
	}
}

resource "aws_db_subnet_group" "foo" {
	name = "%s"
	subnet_ids = ["${aws_subnet.foo.id}", "${aws_subnet.bar.id}"]
	tags {
		Name = "tf-dbsubnet-group-test"
	}
}`, rName)
}

func testAccDBSubnetGroupConfig_updatedDescription(rName string) string {
	return fmt.Sprintf(`
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
	tags {
		Name = "terraform-testacc-db-subnet-group-updated-description"
	}
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	availability_zone = "us-west-2a"
	vpc_id = "${aws_vpc.foo.id}"
	tags {
		Name = "tf-acc-db-subnet-group-1"
	}
}

resource "aws_subnet" "bar" {
	cidr_block = "10.1.2.0/24"
	availability_zone = "us-west-2b"
	vpc_id = "${aws_vpc.foo.id}"
	tags {
		Name = "tf-acc-db-subnet-group-2"
	}
}

resource "aws_db_subnet_group" "foo" {
	name = "%s"
	description = "foo description updated"
	subnet_ids = ["${aws_subnet.foo.id}", "${aws_subnet.bar.id}"]
	tags {
		Name = "tf-dbsubnet-group-test"
	}
}`, rName)
}

const testAccDBSubnetGroupConfig_namePrefix = `
resource "aws_vpc" "test" {
	cidr_block = "10.1.0.0/16"
	tags {
		Name = "terraform-testacc-db-subnet-group-name-prefix"
	}
}

resource "aws_subnet" "a" {
	vpc_id = "${aws_vpc.test.id}"
	cidr_block = "10.1.1.0/24"
	availability_zone = "us-west-2a"
	tags {
		Name = "tf-acc-db-subnet-group-name-prefix-a"
	}
}

resource "aws_subnet" "b" {
	vpc_id = "${aws_vpc.test.id}"
	cidr_block = "10.1.2.0/24"
	availability_zone = "us-west-2b"
	tags {
		Name = "tf-acc-db-subnet-group-name-prefix-b"
	}
}

resource "aws_db_subnet_group" "test" {
	name_prefix = "tf_test-"
	subnet_ids = ["${aws_subnet.a.id}", "${aws_subnet.b.id}"]
}`

const testAccDBSubnetGroupConfig_generatedName = `
resource "aws_vpc" "test" {
	cidr_block = "10.1.0.0/16"
	tags {
		Name = "terraform-testacc-db-subnet-group-generated-name"
	}
}

resource "aws_subnet" "a" {
	vpc_id = "${aws_vpc.test.id}"
	cidr_block = "10.1.1.0/24"
	availability_zone = "us-west-2a"
	tags {
		Name = "tf-acc-db-subnet-group-generated-name-a"
	}
}

resource "aws_subnet" "b" {
	vpc_id = "${aws_vpc.test.id}"
	cidr_block = "10.1.2.0/24"
	availability_zone = "us-west-2b"
	tags {
		Name = "tf-acc-db-subnet-group-generated-name-a"
	}
}

resource "aws_db_subnet_group" "test" {
	subnet_ids = ["${aws_subnet.a.id}", "${aws_subnet.b.id}"]
}`

const testAccDBSubnetGroupConfig_withUnderscoresAndPeriodsAndSpaces = `
resource "aws_vpc" "main" {
    cidr_block = "192.168.0.0/16"
		tags {
			Name = "terraform-testacc-db-subnet-group-w-underscores-etc"
		}
}

resource "aws_subnet" "frontend" {
    vpc_id = "${aws_vpc.main.id}"
    availability_zone = "us-west-2b"
    cidr_block = "192.168.1.0/24"
    tags {
        Name = "tf-acc-db-subnet-group-w-underscores-etc-front"
    }
}

resource "aws_subnet" "backend" {
    vpc_id = "${aws_vpc.main.id}"
    availability_zone = "us-west-2c"
    cidr_block = "192.168.2.0/24"
    tags {
        Name = "tf-acc-db-subnet-group-w-underscores-etc-back"
    }
}

resource "aws_db_subnet_group" "underscores" {
    name = "with_underscores"
    description = "Our main group of subnets"
    subnet_ids = ["${aws_subnet.frontend.id}", "${aws_subnet.backend.id}"]
}

resource "aws_db_subnet_group" "periods" {
    name = "with.periods"
    description = "Our main group of subnets"
    subnet_ids = ["${aws_subnet.frontend.id}", "${aws_subnet.backend.id}"]
}

resource "aws_db_subnet_group" "spaces" {
    name = "with spaces"
    description = "Our main group of subnets"
    subnet_ids = ["${aws_subnet.frontend.id}", "${aws_subnet.backend.id}"]
}
`
<|endoftext|>


<commit_before>/*
	The mongo package is a very simple wrapper around the labix.org/v2/mgo
	package. It's purpose is to allow you to do CRUD operations with very
	little code. It's not exhaustive and not meant to do everything for you.
*/
package mongo

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"errors"
	"fmt"
	"reflect"
	"time"
)

var (
	mgoSession *mgo.Session
	servers    string
	database   string
	NoPtr      = errors.New("You must pass in a pointer")
)

// Set the mongo servers and the database
func SetServers(servers, db string) error {
	var err error

	database = db

	mgoSession, err = mgo.Dial(servers)
	return err
}

// Insert one or more structs. Must pass in a pointer to a struct. The struct must
// contain an Id field of type bson.ObjectId with a tag of `mgo:"_id"`.
func Insert(records ...interface{}) error {
	for _, rec := range records {
		if !isPtr(rec) {
			return NoPtr
		}

		if err := addNewFields(rec); err != nil {
			return err
		}

		s, err := GetSession()
		if err != nil {
			return err
		}
		defer s.Close()

		coll := GetColl(s, typeName(rec))
		err = coll.Insert(rec)
		if err != nil {
			return err
		}
	}

	return nil
}

// Find one or more records. If a single struct is passed in we'll return one record.
// If a slice is passed in all records will be returned. Must pass in a pointer to a
// struct or slice of structs.
func Find(i interface{}, q bson.M) error {
	if !isPtr(i) {
		return NoPtr
	}

	s, err := GetSession()
	if err != nil {
		return err
	}
	defer s.Close()

	coll := GetColl(s, typeName(i))

	query := coll.Find(q)

	if isSlice(reflect.TypeOf(i)) {
		err = query.All(i)
	} else {
		err = query.One(i)
	}
	return err
}

// Find a single record by id. Must pass a pointer to a struct.
func FindById(i interface{}, id string) error {
	return Find(i, bson.M{"_id": id})
}

// Updates a record. Uses the Id to identify the record to update. Must pass in a pointer
// to a struct.
func Update(i interface{}) error {
	if !isPtr(i) {
		return NoPtr
	}

	err := addCurrentDateTime(i, "UpdatedAt")
	if err != nil {
		return err
	}

	s, err := GetSession()
	if err != nil {
		return err
	}
	defer s.Close()

	id, err := getObjIdFromStruct(i)
	if err != nil {
		return err
	}

	return GetColl(s, typeName(i)).Update(bson.M{"_id": id}, i)
}

// Deletes a record. Uses the Id to identify the record to delete. Must pass in a pointer
// to a struct.
func Delete(i interface{}) error {
	if !isPtr(i) {
		return NoPtr
	}

	s, err := GetSession()
	if err != nil {
		return err
	}
	defer s.Close()

	id, err := getObjIdFromStruct(i)
	if err != nil {
		return err
	}

	return GetColl(s, typeName(i)).RemoveId(id)
}

// Returns a Mongo session. You must call Session.Close() when you're done.
func GetSession() (*mgo.Session, error) {
	var err error

	if mgoSession == nil {
		mgoSession, err = mgo.Dial(servers)
		if err != nil {
			return nil, err
		}
	}

	return mgoSession.Clone(), nil
}

// We pass in the session because that is a clone of the original and the
// caller will need to close it when finished.
func GetColl(session *mgo.Session, coll string) *mgo.Collection {
	return session.DB(database).C(coll)
}

func getObjIdFromStruct(i interface{}) (bson.ObjectId, error) {
	v := reflect.ValueOf(i)

	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}

	if v.Kind() != reflect.Struct {
		return bson.ObjectId(""), errors.New("Can't delete record. Type must be a struct.")
	}

	f := v.FieldByName("Id")
	if f.Kind() == reflect.Ptr {
		f = f.Elem()
	}

	return f.Interface().(bson.ObjectId), nil
}

func isPtr(i interface{}) bool {
	return reflect.ValueOf(i).Kind() == reflect.Ptr
}

func typeName(i interface{}) string {
	t := reflect.TypeOf(i)

	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	if isSlice(t) {
		t = t.Elem()

		if t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
	}

	return t.Name()
}

// returns true if the interface is a slice
func isSlice(t reflect.Type) bool {
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	return t.Kind() == reflect.Slice
}

func addNewFields(i interface{}) error {
	err := addId(i)
	if err != nil {
		return err
	}

	if err := addCurrentDateTime(i, "CreatedAt"); err != nil {
		return err
	}

	return addCurrentDateTime(i, "UpdatedAt")
}

func addCurrentDateTime(i interface{}, name string) error {
	if !hasStructField(i, name) {
		return nil
	}

	now := time.Now()

	v := reflect.ValueOf(i)
	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}

	f := v.FieldByName(name)
	if f.Kind() == reflect.Ptr {
		f = f.Elem()
	}

	if reflect.TypeOf(now) != f.Type() {
		return fmt.Errorf("%v must be time.Time type.", name)
	}

	if !f.CanSet() {
		return fmt.Errorf("Couldn't set time for field: %v", name)
	}

	f.Set(reflect.ValueOf(now))

	return nil
}

func hasStructField(i interface{}, field string) bool {
	t := reflect.TypeOf(i)
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	if t.Kind() != reflect.Struct {
		return false
	}

	_, found := t.FieldByName(field)
	return found
}

func addId(i interface{}) error {
	v := reflect.ValueOf(i)

	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}

	if v.Kind() != reflect.Struct {
		return errors.New("Record must be a struct")
	}

	f := v.FieldByName("Id")
	if f.Kind() == reflect.Ptr {
		f = f.Elem()
	}

	if f.Kind() == reflect.String {
		if !f.Interface().(bson.ObjectId).Valid() {
			id := reflect.ValueOf(bson.NewObjectId())
			f.Set(id)
		}
	}

	return nil
}
<commit_msg>Fix error in documentation<commit_after>/*
	The mongo package is a very simple wrapper around the labix.org/v2/mgo
	package. It's purpose is to allow you to do CRUD operations with very
	little code. It's not exhaustive and not meant to do everything for you.
*/
package mongo

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"errors"
	"fmt"
	"reflect"
	"time"
)

var (
	mgoSession *mgo.Session
	servers    string
	database   string
	NoPtr      = errors.New("You must pass in a pointer")
)

// Set the mongo servers and the database
func SetServers(servers, db string) error {
	var err error

	database = db

	mgoSession, err = mgo.Dial(servers)
	return err
}

// Insert one or more structs. Must pass in a pointer to a struct. The struct must
// contain an Id field of type bson.ObjectId with a tag of `bson:"_id"`.
func Insert(records ...interface{}) error {
	for _, rec := range records {
		if !isPtr(rec) {
			return NoPtr
		}

		if err := addNewFields(rec); err != nil {
			return err
		}

		s, err := GetSession()
		if err != nil {
			return err
		}
		defer s.Close()

		coll := GetColl(s, typeName(rec))
		err = coll.Insert(rec)
		if err != nil {
			return err
		}
	}

	return nil
}

// Find one or more records. If a single struct is passed in we'll return one record.
// If a slice is passed in all records will be returned. Must pass in a pointer to a
// struct or slice of structs.
func Find(i interface{}, q bson.M) error {
	if !isPtr(i) {
		return NoPtr
	}

	s, err := GetSession()
	if err != nil {
		return err
	}
	defer s.Close()

	coll := GetColl(s, typeName(i))

	query := coll.Find(q)

	if isSlice(reflect.TypeOf(i)) {
		err = query.All(i)
	} else {
		err = query.One(i)
	}
	return err
}

// Find a single record by id. Must pass a pointer to a struct.
func FindById(i interface{}, id string) error {
	return Find(i, bson.M{"_id": id})
}

// Updates a record. Uses the Id to identify the record to update. Must pass in a pointer
// to a struct.
func Update(i interface{}) error {
	if !isPtr(i) {
		return NoPtr
	}

	err := addCurrentDateTime(i, "UpdatedAt")
	if err != nil {
		return err
	}

	s, err := GetSession()
	if err != nil {
		return err
	}
	defer s.Close()

	id, err := getObjIdFromStruct(i)
	if err != nil {
		return err
	}

	return GetColl(s, typeName(i)).Update(bson.M{"_id": id}, i)
}

// Deletes a record. Uses the Id to identify the record to delete. Must pass in a pointer
// to a struct.
func Delete(i interface{}) error {
	if !isPtr(i) {
		return NoPtr
	}

	s, err := GetSession()
	if err != nil {
		return err
	}
	defer s.Close()

	id, err := getObjIdFromStruct(i)
	if err != nil {
		return err
	}

	return GetColl(s, typeName(i)).RemoveId(id)
}

// Returns a Mongo session. You must call Session.Close() when you're done.
func GetSession() (*mgo.Session, error) {
	var err error

	if mgoSession == nil {
		mgoSession, err = mgo.Dial(servers)
		if err != nil {
			return nil, err
		}
	}

	return mgoSession.Clone(), nil
}

// We pass in the session because that is a clone of the original and the
// caller will need to close it when finished.
func GetColl(session *mgo.Session, coll string) *mgo.Collection {
	return session.DB(database).C(coll)
}

func getObjIdFromStruct(i interface{}) (bson.ObjectId, error) {
	v := reflect.ValueOf(i)

	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}

	if v.Kind() != reflect.Struct {
		return bson.ObjectId(""), errors.New("Can't delete record. Type must be a struct.")
	}

	f := v.FieldByName("Id")
	if f.Kind() == reflect.Ptr {
		f = f.Elem()
	}

	return f.Interface().(bson.ObjectId), nil
}

func isPtr(i interface{}) bool {
	return reflect.ValueOf(i).Kind() == reflect.Ptr
}

func typeName(i interface{}) string {
	t := reflect.TypeOf(i)

	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	if isSlice(t) {
		t = t.Elem()

		if t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
	}

	return t.Name()
}

// returns true if the interface is a slice
func isSlice(t reflect.Type) bool {
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	return t.Kind() == reflect.Slice
}

func addNewFields(i interface{}) error {
	err := addId(i)
	if err != nil {
		return err
	}

	if err := addCurrentDateTime(i, "CreatedAt"); err != nil {
		return err
	}

	return addCurrentDateTime(i, "UpdatedAt")
}

func addCurrentDateTime(i interface{}, name string) error {
	if !hasStructField(i, name) {
		return nil
	}

	now := time.Now()

	v := reflect.ValueOf(i)
	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}

	f := v.FieldByName(name)
	if f.Kind() == reflect.Ptr {
		f = f.Elem()
	}

	if reflect.TypeOf(now) != f.Type() {
		return fmt.Errorf("%v must be time.Time type.", name)
	}

	if !f.CanSet() {
		return fmt.Errorf("Couldn't set time for field: %v", name)
	}

	f.Set(reflect.ValueOf(now))

	return nil
}

func hasStructField(i interface{}, field string) bool {
	t := reflect.TypeOf(i)
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	if t.Kind() != reflect.Struct {
		return false
	}

	_, found := t.FieldByName(field)
	return found
}

func addId(i interface{}) error {
	v := reflect.ValueOf(i)

	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}

	if v.Kind() != reflect.Struct {
		return errors.New("Record must be a struct")
	}

	f := v.FieldByName("Id")
	if f.Kind() == reflect.Ptr {
		f = f.Elem()
	}

	if f.Kind() == reflect.String {
		if !f.Interface().(bson.ObjectId).Valid() {
			id := reflect.ValueOf(bson.NewObjectId())
			f.Set(id)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package suffixtree

import (
	"bytes"
	"fmt"
	"math"
	"strings"
)

const infinity = math.MaxInt32

// Pos denotes position in data slice.
type Pos int32

type Token interface {
	Val() int
}

// STree is a struct representing a suffix tree.
type STree struct {
	data     []Token
	root     *state
	auxState *state // auxiliary state

	// active point
	s          *state
	start, end Pos
}

// New creates new suffix tree.
func New() *STree {
	t := new(STree)
	t.data = make([]Token, 0, 50)
	t.root = newState(t)
	t.auxState = newState(t)
	t.root.linkState = t.auxState
	t.s = t.root
	return t
}

// Update refreshes the suffix tree to by new data.
func (t *STree) Update(data ...Token) {
	t.data = append(t.data, data...)
	for _ = range data {
		t.update()
		t.s, t.start = t.canonize(t.s, t.start, t.end)
		t.end++
	}
}

// update transforms suffix tree T(n) to T(n+1).
func (t *STree) update() {
	oldr := t.root

	// (s, (start, end)) is the canonical reference pair for the active point
	s := t.s
	start, end := t.start, t.end
	var r *state
	for {
		var endPoint bool
		r, endPoint = t.testAndSplit(s, start, end-1)
		if endPoint {
			break
		}
		r.fork(end)
		if oldr != t.root {
			oldr.linkState = r
		}
		oldr = r
		s, start = t.canonize(s.linkState, start, end-1)
	}
	if oldr != t.root {
		oldr.linkState = r
	}

	// update active point
	t.s = s
	t.start = start
}

// testAndSplit tests whether a state with canonical ref. pair
// (s, (start, end)) is the end point, that is, a state that have
// a c-transition. If not, then state (exs, (start, end)) is made
// explicit (if not already so).
func (t *STree) testAndSplit(s *state, start, end Pos) (exs *state, endPoint bool) {
	c := t.data[t.end]
	if start <= end {
		tr := s.findTran(t.data[start])
		splitPoint := tr.start + end - start + 1
		if t.data[splitPoint].Val() == c.Val() {
			return s, true
		}
		// make the (s, (start, end)) state explicit
		newSt := newState(s.t)
		newSt.addTran(splitPoint, tr.end, tr.state)
		tr.end = splitPoint - 1
		tr.state = newSt
		return newSt, false
	}
	if s == t.auxState || s.findTran(c) != nil {
		return s, true
	}
	return s, false
}

// canonize returns updated state and start position for ref. pair
// (s, (start, end)) of state r so the new ref. pair is canonical,
// that is, referenced from the closest explicit ancestor of r.
func (t *STree) canonize(s *state, start, end Pos) (*state, Pos) {
	if s == t.auxState {
		s, start = t.root, start+1
	}
	if start > end {
		return s, start
	}

	var tr *tran
	for {
		if start <= end {
			tr = s.findTran(t.data[start])
			if tr == nil {
				panic(fmt.Sprintf("there should be some transition for '%d' at %d", t.data[start].Val(), start))
			}
		}
		if tr.end-tr.start > end-start {
			break
		}
		start += tr.end - tr.start + 1
		s = tr.state
	}
	if s == nil {
		panic("there should always be some suffix link resolution")
	}
	return s, start
}

func (t *STree) At(p Pos) Token {
	if p < 0 || p >= Pos(len(t.data)) {
		panic("position out of bounds")
	}
	return t.data[p]
}

func (t *STree) String() string {
	buf := new(bytes.Buffer)
	printState(buf, t.root, 0)
	return buf.String()
}

func printState(buf *bytes.Buffer, s *state, ident int) {
	for _, tr := range s.trans {
		fmt.Fprint(buf, strings.Repeat("  ", ident))
		fmt.Fprintf(buf, "* (%d, %d)\n", tr.start, tr.ActEnd())
		printState(buf, tr.state, ident+1)
	}
}

// state is an explicit state of the suffix tree.
type state struct {
	t         *STree
	trans     []*tran
	linkState *state
}

func newState(t *STree) *state {
	return &state{
		t:         t,
		trans:     make([]*tran, 0),
		linkState: nil,
	}
}

func (s *state) addTran(start, end Pos, r *state) {
	s.trans = append(s.trans, newTran(start, end, r))
}

// fork creates a new branch from the state s.
func (s *state) fork(i Pos) *state {
	r := newState(s.t)
	s.addTran(i, infinity, r)
	return r
}

// findTran finds c-transition.
func (s *state) findTran(c Token) *tran {
	for _, tran := range s.trans {
		if s.t.data[tran.start].Val() == c.Val() {
			return tran
		}
	}
	return nil
}

// tran represents a state's transition.
type tran struct {
	start, end Pos
	state      *state
}

func newTran(start, end Pos, s *state) *tran {
	return &tran{start, end, s}
}

func (t *tran) len() int {
	return int(t.end - t.start + 1)
}

// ActEnd returns actual end position as consistent with
// the actual length of the data in the STree.
func (t *tran) ActEnd() Pos {
	if t.end == infinity {
		return Pos(len(t.state.t.data)) - 1
	}
	return t.end
}
<commit_msg>typos<commit_after>package suffixtree

import (
	"bytes"
	"fmt"
	"math"
	"strings"
)

const infinity = math.MaxInt32

// Pos denotes position in data slice.
type Pos int32

type Token interface {
	Val() int
}

// STree is a struct representing a suffix tree.
type STree struct {
	data     []Token
	root     *state
	auxState *state // auxiliary state

	// active point
	s          *state
	start, end Pos
}

// New creates new suffix tree.
func New() *STree {
	t := new(STree)
	t.data = make([]Token, 0, 50)
	t.root = newState(t)
	t.auxState = newState(t)
	t.root.linkState = t.auxState
	t.s = t.root
	return t
}

// Update refreshes the suffix tree to by new data.
func (t *STree) Update(data ...Token) {
	t.data = append(t.data, data...)
	for range data {
		t.update()
		t.s, t.start = t.canonize(t.s, t.start, t.end)
		t.end++
	}
}

// update transforms suffix tree T(n) to T(n+1).
func (t *STree) update() {
	oldr := t.root

	// (s, (start, end)) is the canonical reference pair for the active point
	s := t.s
	start, end := t.start, t.end
	var r *state
	for {
		var endPoint bool
		r, endPoint = t.testAndSplit(s, start, end-1)
		if endPoint {
			break
		}
		r.fork(end)
		if oldr != t.root {
			oldr.linkState = r
		}
		oldr = r
		s, start = t.canonize(s.linkState, start, end-1)
	}
	if oldr != t.root {
		oldr.linkState = r
	}

	// update active point
	t.s = s
	t.start = start
}

// testAndSplit tests whether a state with canonical ref. pair
// (s, (start, end)) is the end point, that is, a state that have
// a c-transition. If not, then state (exs, (start, end)) is made
// explicit (if not already so).
func (t *STree) testAndSplit(s *state, start, end Pos) (exs *state, endPoint bool) {
	c := t.data[t.end]
	if start <= end {
		tr := s.findTran(t.data[start])
		splitPoint := tr.start + end - start + 1
		if t.data[splitPoint].Val() == c.Val() {
			return s, true
		}
		// make the (s, (start, end)) state explicit
		newSt := newState(s.t)
		newSt.addTran(splitPoint, tr.end, tr.state)
		tr.end = splitPoint - 1
		tr.state = newSt
		return newSt, false
	}
	if s == t.auxState || s.findTran(c) != nil {
		return s, true
	}
	return s, false
}

// canonize returns updated state and start position for ref. pair
// (s, (start, end)) of state r so the new ref. pair is canonical,
// that is, referenced from the closest explicit ancestor of r.
func (t *STree) canonize(s *state, start, end Pos) (*state, Pos) {
	if s == t.auxState {
		s, start = t.root, start+1
	}
	if start > end {
		return s, start
	}

	var tr *tran
	for {
		if start <= end {
			tr = s.findTran(t.data[start])
			if tr == nil {
				panic(fmt.Sprintf("there should be some transition for '%d' at %d",
					t.data[start].Val(), start))
			}
		}
		if tr.end-tr.start > end-start {
			break
		}
		start += tr.end - tr.start + 1
		s = tr.state
	}
	if s == nil {
		panic("there should always be some suffix link resolution")
	}
	return s, start
}

func (t *STree) At(p Pos) Token {
	if p < 0 || p >= Pos(len(t.data)) {
		panic("position out of bounds")
	}
	return t.data[p]
}

func (t *STree) String() string {
	buf := new(bytes.Buffer)
	printState(buf, t.root, 0)
	return buf.String()
}

func printState(buf *bytes.Buffer, s *state, ident int) {
	for _, tr := range s.trans {
		fmt.Fprint(buf, strings.Repeat("  ", ident))
		fmt.Fprintf(buf, "* (%d, %d)\n", tr.start, tr.ActEnd())
		printState(buf, tr.state, ident+1)
	}
}

// state is an explicit state of the suffix tree.
type state struct {
	t         *STree
	trans     []*tran
	linkState *state
}

func newState(t *STree) *state {
	return &state{
		t:         t,
		trans:     make([]*tran, 0),
		linkState: nil,
	}
}

func (s *state) addTran(start, end Pos, r *state) {
	s.trans = append(s.trans, newTran(start, end, r))
}

// fork creates a new branch from the state s.
func (s *state) fork(i Pos) *state {
	r := newState(s.t)
	s.addTran(i, infinity, r)
	return r
}

// findTran finds c-transition.
func (s *state) findTran(c Token) *tran {
	for _, tran := range s.trans {
		if s.t.data[tran.start].Val() == c.Val() {
			return tran
		}
	}
	return nil
}

// tran represents a state's transition.
type tran struct {
	start, end Pos
	state      *state
}

func newTran(start, end Pos, s *state) *tran {
	return &tran{start, end, s}
}

func (t *tran) len() int {
	return int(t.end - t.start + 1)
}

// ActEnd returns actual end position as consistent with
// the actual length of the data in the STree.
func (t *tran) ActEnd() Pos {
	if t.end == infinity {
		return Pos(len(t.state.t.data)) - 1
	}
	return t.end
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	mp "github.com/mackerelio/go-mackerel-plugin"
)

var graphDef = map[string](mp.Graphs){
	"multicore.cpu.#": mp.Graphs{
		Label: "MultiCore CPU",
		Unit:  "percentage",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "user", Label: "user", Diff: false, Stacked: true},
			mp.Metrics{Name: "nice", Label: "nice", Diff: false, Stacked: true},
			mp.Metrics{Name: "system", Label: "system", Diff: false, Stacked: true},
			mp.Metrics{Name: "idle", Label: "idle", Diff: false, Stacked: true},
			mp.Metrics{Name: "iowait", Label: "ioWait", Diff: false, Stacked: true},
			mp.Metrics{Name: "irq", Label: "irq", Diff: false, Stacked: true},
			mp.Metrics{Name: "softirq", Label: "softirq", Diff: false, Stacked: true},
			mp.Metrics{Name: "steal", Label: "steal", Diff: false, Stacked: true},
			mp.Metrics{Name: "guest", Label: "guest", Diff: false, Stacked: true},
		},
	},
	"multicore.loadavg_per_core": mp.Graphs{
		Label: "MultiCore loadavg5 per core",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "loadavg5", Label: "loadavg5", Diff: false, Stacked: false},
		},
	},
}

type saveItem struct {
	LastTime       time.Time
	ProcStatsByCPU map[string]*procStats
}

type procStats struct {
	User    float64 `json:"user"`
	Nice    float64 `json:"nice"`
	System  float64 `json:"system"`
	Idle    float64 `json:"idle"`
	IoWait  float64 `json:"iowait"`
	Irq     float64 `json:"irq"`
	SoftIrq float64 `json:"softirq"`
	Steal   float64 `json:"steal"`
	Guest   float64 `json:"guest"`
	Total   float64 `json:"total"`
}

type cpuPercentages struct {
	GroupName string
	User      float64
	Nice      float64
	System    float64
	Idle      float64
	IoWait    float64
	Irq       float64
	SoftIrq   float64
	Steal     float64
	Guest     float64
}

func getProcStat() (string, error) {
	contentbytes, err := ioutil.ReadFile("/proc/stat")
	if err != nil {
		return "", err
	}
	return string(contentbytes), nil
}

func parseFloats(values []string) ([]float64, error) {
	var result []float64
	for _, v := range values {
		f, err := strconv.ParseFloat(v, 64)
		if err != nil {
			return nil, err
		}
		result = append(result, f)
	}
	return result, nil
}

func fill(arr []float64, elementCount int) []float64 {
	if len(arr) < elementCount {
		zeroArr := make([]float64, elementCount-len(arr))
		filled := append(arr, zeroArr...)
		return filled
	}
	return arr
}

func parseProcStat(str string) (map[string]*procStats, error) {
	var result = make(map[string]*procStats)
	for _, line := range strings.Split(str, "\n") {
		if strings.HasPrefix(line, "cpu") {
			fields := strings.Fields(line)
			key := fields[0]
			values := fields[1:]

			floatValues, err := parseFloats(values)
			if err != nil {
				return nil, err
			}
			filledValues := fill(floatValues, 9)

			total := 0.0
			for _, v := range floatValues {
				total += v
			}

			ps := &procStats{
				User:    filledValues[0],
				Nice:    filledValues[1],
				System:  filledValues[2],
				Idle:    filledValues[3],
				IoWait:  filledValues[4],
				Irq:     filledValues[5],
				SoftIrq: filledValues[6],
				Steal:   filledValues[7],
				Guest:   filledValues[8],
				Total:   total,
			}
			result[key] = ps

		} else {
			break
		}
	}
	return result, nil
}

func collectProcStatValues() (map[string]*procStats, error) {
	procStats, err := getProcStat()
	if err != nil {
		return nil, err
	}
	return parseProcStat(procStats)
}

func saveValues(tempFileName string, values map[string]*procStats, now time.Time) error {
	f, err := os.Create(tempFileName)
	if err != nil {
		return err
	}
	defer f.Close()

	s := saveItem{
		LastTime:       time.Now(),
		ProcStatsByCPU: values,
	}

	encoder := json.NewEncoder(f)
	err = encoder.Encode(s)
	if err != nil {
		return err
	}

	return nil
}

func fetchLastValues(tempFileName string) (map[string]*procStats, time.Time, error) {
	f, err := os.Open(tempFileName)
	if err != nil {
		return nil, time.Now(), err
	}
	defer f.Close()

	var stat saveItem
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	if err != nil {
		return stat.ProcStatsByCPU, stat.LastTime, err
	}
	return stat.ProcStatsByCPU, stat.LastTime, nil
}

func calcCPUUsage(currentValues map[string]*procStats, now time.Time, lastValues map[string]*procStats, lastTime time.Time) ([]*cpuPercentages, error) {

	var result []*cpuPercentages
	for key, current := range currentValues {
		last, ok := lastValues[key]
		if ok {
			user, err := calcPercentage(current.User, last.User, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			nice, err := calcPercentage(current.Nice, last.Nice, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			system, err := calcPercentage(current.System, last.System, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			idle, err := calcPercentage(current.Idle, last.Idle, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			iowait, err := calcPercentage(current.IoWait, last.IoWait, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			irq, err := calcPercentage(current.Irq, last.Irq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			softirq, err := calcPercentage(current.SoftIrq, last.SoftIrq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			steal, err := calcPercentage(current.Steal, last.Steal, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			guest, err := calcPercentage(current.Guest, last.Guest, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}

			p := &cpuPercentages{
				GroupName: key,
				User:      user,
				Nice:      nice,
				System:    system,
				Idle:      idle,
				IoWait:    iowait,
				Irq:       irq,
				SoftIrq:   softirq,
				Steal:     steal,
				Guest:     guest,
			}
			result = append(result, p)
		}

	}

	return result, nil
}

func calcPercentage(currentValue float64, lastValue float64, currentTotal float64, lastTotal float64, now time.Time, lastTime time.Time) (float64, error) {
	value, err := calcDiff(currentValue, now, lastValue, lastTime)
	if err != nil {
		return 0.0, err
	}

	total, err := calcDiff(currentTotal, now, lastTotal, lastTime)
	if err != nil {
		return 0.0, err
	}

	return (value / total * 100.0), nil
}

func calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0.0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	if lastValue <= value {
		return diff, nil
	}
	return 0.0, nil
}

func fetchLoadavg5() (float64, error) {
	contentbytes, err := ioutil.ReadFile("/proc/loadavg")
	if err != nil {
		return 0.0, err
	}
	content := string(contentbytes)
	cols := strings.Fields(content)

	if len(cols) > 2 {
		f, err := strconv.ParseFloat(cols[1], 64)
		if err != nil {
			return 0.0, err
		}
		return f, nil
	}
	return 0.0, errors.New("cannot fetch loadavg5.")
}

func printValue(key string, value float64, time time.Time) {
	fmt.Printf("%s\t%f\t%d\n", key, value, time.Unix())
}

func outputCPUUsage(cpuUsage []*cpuPercentages, now time.Time) {
	if cpuUsage != nil {
		for _, u := range cpuUsage {
			if u.GroupName != "cpu" {
				printValue(fmt.Sprintf("multicore.cpu.%s.user", u.GroupName), u.User, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.nice", u.GroupName), u.Nice, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.system", u.GroupName), u.System, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.idle", u.GroupName), u.Idle, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.iowait", u.GroupName), u.IoWait, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.irq", u.GroupName), u.Irq, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.softirq", u.GroupName), u.SoftIrq, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.steal", u.GroupName), u.Steal, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.guest", u.GroupName), u.Guest, now)
			}
		}
	}
}

func outputLoadavgPerCore(loadavgPerCore float64, now time.Time) {
	printValue("multicore.loadavg_per_core.loadavg5", loadavgPerCore, now)
}

func outputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs mp.GraphDef
	graphs.Graphs = graphDef

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}

// main function
func main() {
	optTempfile := flag.String("tempfile", "", "Temp file name")
	var tempFileName string
	if *optTempfile != "" {
		tempFileName = *optTempfile
	} else {
		tempFileName = "/tmp/mackerel-plugin-multicore"
	}
	now := time.Now()

	currentValues, _ := collectProcStatValues()
	lastValues, lastTime, err := fetchLastValues(tempFileName)
	if currentValues != nil {
		saveValues(tempFileName, currentValues, now)
	}
	if err != nil {
		log.Fatalln("fetchLastValues: ", err)
	}

	var cpuUsage []*cpuPercentages
	if lastValues != nil {
		var err error
		cpuUsage, err = calcCPUUsage(currentValues, now, lastValues, lastTime)
		if err != nil {
			log.Fatalln("calcCPUUsage: ", err)
		}
	}

	loadavg5, err := fetchLoadavg5()
	if err != nil {
		log.Fatalln("fetchLoadavg5: ", err)
	}
	loadPerCPUCount := loadavg5 / (float64(len(cpuUsage) - 1))

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		outputDefinitions()
	} else {
		outputCPUUsage(cpuUsage, now)
		outputLoadavgPerCore(loadPerCPUCount, now)
	}
}
<commit_msg>golint<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	mp "github.com/mackerelio/go-mackerel-plugin"
)

var graphDef = map[string](mp.Graphs){
	"multicore.cpu.#": mp.Graphs{
		Label: "MultiCore CPU",
		Unit:  "percentage",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "user", Label: "user", Diff: false, Stacked: true},
			mp.Metrics{Name: "nice", Label: "nice", Diff: false, Stacked: true},
			mp.Metrics{Name: "system", Label: "system", Diff: false, Stacked: true},
			mp.Metrics{Name: "idle", Label: "idle", Diff: false, Stacked: true},
			mp.Metrics{Name: "iowait", Label: "ioWait", Diff: false, Stacked: true},
			mp.Metrics{Name: "irq", Label: "irq", Diff: false, Stacked: true},
			mp.Metrics{Name: "softirq", Label: "softirq", Diff: false, Stacked: true},
			mp.Metrics{Name: "steal", Label: "steal", Diff: false, Stacked: true},
			mp.Metrics{Name: "guest", Label: "guest", Diff: false, Stacked: true},
		},
	},
	"multicore.loadavg_per_core": mp.Graphs{
		Label: "MultiCore loadavg5 per core",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "loadavg5", Label: "loadavg5", Diff: false, Stacked: false},
		},
	},
}

type saveItem struct {
	LastTime       time.Time
	ProcStatsByCPU map[string]*procStats
}

type procStats struct {
	User    float64 `json:"user"`
	Nice    float64 `json:"nice"`
	System  float64 `json:"system"`
	Idle    float64 `json:"idle"`
	IoWait  float64 `json:"iowait"`
	Irq     float64 `json:"irq"`
	SoftIrq float64 `json:"softirq"`
	Steal   float64 `json:"steal"`
	Guest   float64 `json:"guest"`
	Total   float64 `json:"total"`
}

type cpuPercentages struct {
	GroupName string
	User      float64
	Nice      float64
	System    float64
	Idle      float64
	IoWait    float64
	Irq       float64
	SoftIrq   float64
	Steal     float64
	Guest     float64
}

func getProcStat() (string, error) {
	contentbytes, err := ioutil.ReadFile("/proc/stat")
	if err != nil {
		return "", err
	}
	return string(contentbytes), nil
}

func parseFloats(values []string) ([]float64, error) {
	var result []float64
	for _, v := range values {
		f, err := strconv.ParseFloat(v, 64)
		if err != nil {
			return nil, err
		}
		result = append(result, f)
	}
	return result, nil
}

func fill(arr []float64, elementCount int) []float64 {
	if len(arr) < elementCount {
		zeroArr := make([]float64, elementCount-len(arr))
		filled := append(arr, zeroArr...)
		return filled
	}
	return arr
}

func parseProcStat(str string) (map[string]*procStats, error) {
	var result = make(map[string]*procStats)
	for _, line := range strings.Split(str, "\n") {
		if strings.HasPrefix(line, "cpu") {
			fields := strings.Fields(line)
			key := fields[0]
			values := fields[1:]

			floatValues, err := parseFloats(values)
			if err != nil {
				return nil, err
			}
			filledValues := fill(floatValues, 9)

			total := 0.0
			for _, v := range floatValues {
				total += v
			}

			ps := &procStats{
				User:    filledValues[0],
				Nice:    filledValues[1],
				System:  filledValues[2],
				Idle:    filledValues[3],
				IoWait:  filledValues[4],
				Irq:     filledValues[5],
				SoftIrq: filledValues[6],
				Steal:   filledValues[7],
				Guest:   filledValues[8],
				Total:   total,
			}
			result[key] = ps

		} else {
			break
		}
	}
	return result, nil
}

func collectProcStatValues() (map[string]*procStats, error) {
	procStats, err := getProcStat()
	if err != nil {
		return nil, err
	}
	return parseProcStat(procStats)
}

func saveValues(tempFileName string, values map[string]*procStats, now time.Time) error {
	f, err := os.Create(tempFileName)
	if err != nil {
		return err
	}
	defer f.Close()

	s := saveItem{
		LastTime:       time.Now(),
		ProcStatsByCPU: values,
	}

	encoder := json.NewEncoder(f)
	err = encoder.Encode(s)
	if err != nil {
		return err
	}

	return nil
}

func fetchLastValues(tempFileName string) (map[string]*procStats, time.Time, error) {
	f, err := os.Open(tempFileName)
	if err != nil {
		return nil, time.Now(), err
	}
	defer f.Close()

	var stat saveItem
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	if err != nil {
		return stat.ProcStatsByCPU, stat.LastTime, err
	}
	return stat.ProcStatsByCPU, stat.LastTime, nil
}

func calcCPUUsage(currentValues map[string]*procStats, now time.Time, lastValues map[string]*procStats, lastTime time.Time) ([]*cpuPercentages, error) {

	var result []*cpuPercentages
	for key, current := range currentValues {
		last, ok := lastValues[key]
		if ok {
			user, err := calcPercentage(current.User, last.User, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			nice, err := calcPercentage(current.Nice, last.Nice, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			system, err := calcPercentage(current.System, last.System, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			idle, err := calcPercentage(current.Idle, last.Idle, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			iowait, err := calcPercentage(current.IoWait, last.IoWait, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			irq, err := calcPercentage(current.Irq, last.Irq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			softirq, err := calcPercentage(current.SoftIrq, last.SoftIrq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			steal, err := calcPercentage(current.Steal, last.Steal, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			guest, err := calcPercentage(current.Guest, last.Guest, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}

			p := &cpuPercentages{
				GroupName: key,
				User:      user,
				Nice:      nice,
				System:    system,
				Idle:      idle,
				IoWait:    iowait,
				Irq:       irq,
				SoftIrq:   softirq,
				Steal:     steal,
				Guest:     guest,
			}
			result = append(result, p)
		}

	}

	return result, nil
}

func calcPercentage(currentValue float64, lastValue float64, currentTotal float64, lastTotal float64, now time.Time, lastTime time.Time) (float64, error) {
	value, err := calcDiff(currentValue, now, lastValue, lastTime)
	if err != nil {
		return 0.0, err
	}

	total, err := calcDiff(currentTotal, now, lastTotal, lastTime)
	if err != nil {
		return 0.0, err
	}

	return (value / total * 100.0), nil
}

func calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0.0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	if lastValue <= value {
		return diff, nil
	}
	return 0.0, nil
}

func fetchLoadavg5() (float64, error) {
	contentbytes, err := ioutil.ReadFile("/proc/loadavg")
	if err != nil {
		return 0.0, err
	}
	content := string(contentbytes)
	cols := strings.Fields(content)

	if len(cols) > 2 {
		f, err := strconv.ParseFloat(cols[1], 64)
		if err != nil {
			return 0.0, err
		}
		return f, nil
	}
	return 0.0, errors.New("cannot fetch loadavg5")
}

func printValue(key string, value float64, time time.Time) {
	fmt.Printf("%s\t%f\t%d\n", key, value, time.Unix())
}

func outputCPUUsage(cpuUsage []*cpuPercentages, now time.Time) {
	if cpuUsage != nil {
		for _, u := range cpuUsage {
			if u.GroupName != "cpu" {
				printValue(fmt.Sprintf("multicore.cpu.%s.user", u.GroupName), u.User, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.nice", u.GroupName), u.Nice, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.system", u.GroupName), u.System, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.idle", u.GroupName), u.Idle, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.iowait", u.GroupName), u.IoWait, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.irq", u.GroupName), u.Irq, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.softirq", u.GroupName), u.SoftIrq, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.steal", u.GroupName), u.Steal, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.guest", u.GroupName), u.Guest, now)
			}
		}
	}
}

func outputLoadavgPerCore(loadavgPerCore float64, now time.Time) {
	printValue("multicore.loadavg_per_core.loadavg5", loadavgPerCore, now)
}

func outputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs mp.GraphDef
	graphs.Graphs = graphDef

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}

// main function
func main() {
	optTempfile := flag.String("tempfile", "", "Temp file name")
	var tempFileName string
	if *optTempfile != "" {
		tempFileName = *optTempfile
	} else {
		tempFileName = "/tmp/mackerel-plugin-multicore"
	}
	now := time.Now()

	currentValues, _ := collectProcStatValues()
	lastValues, lastTime, err := fetchLastValues(tempFileName)
	if currentValues != nil {
		saveValues(tempFileName, currentValues, now)
	}
	if err != nil {
		log.Fatalln("fetchLastValues: ", err)
	}

	var cpuUsage []*cpuPercentages
	if lastValues != nil {
		var err error
		cpuUsage, err = calcCPUUsage(currentValues, now, lastValues, lastTime)
		if err != nil {
			log.Fatalln("calcCPUUsage: ", err)
		}
	}

	loadavg5, err := fetchLoadavg5()
	if err != nil {
		log.Fatalln("fetchLoadavg5: ", err)
	}
	loadPerCPUCount := loadavg5 / (float64(len(cpuUsage) - 1))

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		outputDefinitions()
	} else {
		outputCPUUsage(cpuUsage, now)
		outputLoadavgPerCore(loadPerCPUCount, now)
	}
}
<|endoftext|>


<commit_before>package repository

import (
	"github.com/mgierok/monujo/db"
	"github.com/mgierok/monujo/repository/entity"
)

func OwnedStocks() (entity.OwnedStocks, error) {
	stocks := entity.OwnedStocks{}
	err := db.Connection().Select(&stocks,
		`SELECT
			portfolio_id,
			portfolio_name,
			ticker,
			short_name,
			shares,
			last_price,
			currency,
			exchange_rate,
			last_price_base_currency,
			average_price,
			average_price_base_currency,
			average_price_base_currency,
			investment_base_currency,
			market_value_base_currency,
			gain,
			percentage_gain,
			gain_base_currency,
			percentage_gain_base_currency
			FROM owned_stocks
			`)
	return stocks, err
}
<commit_msg>order owned stock summary by portfolio id<commit_after>package repository

import (
	"github.com/mgierok/monujo/db"
	"github.com/mgierok/monujo/repository/entity"
)

func OwnedStocks() (entity.OwnedStocks, error) {
	stocks := entity.OwnedStocks{}
	err := db.Connection().Select(&stocks,
		`SELECT
			portfolio_id,
			portfolio_name,
			ticker,
			short_name,
			shares,
			last_price,
			currency,
			exchange_rate,
			last_price_base_currency,
			average_price,
			average_price_base_currency,
			average_price_base_currency,
			investment_base_currency,
			market_value_base_currency,
			gain,
			percentage_gain,
			gain_base_currency,
			percentage_gain_base_currency
			FROM owned_stocks
			ORDER BY portfolio_id
			`)
	return stocks, err
}
<|endoftext|>