<commit_before>package commands

import (
	"fmt"
	"github.com/0xfoo/punchcard/schedule"
	"github.com/spf13/cobra"
)

var minCommits, maxCommits int

var randomCmd = &cobra.Command{
	Use:   "random",
	Short: "Random will add commits throughout the past 365 days.",
	Long: `Random will create a git repo at the given location and create
random commits, random meaning the number of commits per day.
This will be done for the past 365 days and the commits are in the range of
--min and --max commits.`,
	Run: randomRun,
}

func randomRun(cmd *cobra.Command, args []string) {
	schedulers.RandomSchedule()
}

func init() {
	randomCmd.Flags().IntVar(&minCommits, "min", 1,
		"minimal #commits on a given day.")
	randomCmd.Flags().IntVar(&maxCommits, "max", 10,
		"maximal #commits on a given day.")
	PunchCardCmd.AddCommand(randomCmd)
}
<commit_msg>Add min and max to random schedule call.<commit_after>package commands

import (
	"fmt"
	"github.com/0xfoo/punchcard/schedule"
	"github.com/spf13/cobra"
)

var minCommits, maxCommits int

var randomCmd = &cobra.Command{
	Use:   "random",
	Short: "Random will add commits throughout the past 365 days.",
	Long: `Random will create a git repo at the given location and create
random commits, random meaning the number of commits per day.
This will be done for the past 365 days and the commits are in the range of
--min and --max commits.`,
	Run: randomRun,
}

func randomRun(cmd *cobra.Command, args []string) {
	schedule.RandomSchedule(minCommits, maxCommits)
}

func init() {
	randomCmd.Flags().IntVar(&minCommits, "min", 1,
		"minimal #commits on a given day.")
	randomCmd.Flags().IntVar(&maxCommits, "max", 10,
		"maximal #commits on a given day.")
	PunchCardCmd.AddCommand(randomCmd)
}
<|endoftext|>


<commit_before>package commands

import (
	"flag"
	"fmt"
	"github.com/jingweno/gh/cmd"
	"github.com/jingweno/gh/git"
	shellquote "github.com/kballard/go-shellquote"
	"os/exec"
	"syscall"
)

type ExecError struct {
	Err      error
	ExitCode int
}

func (execError *ExecError) Error() string {
	return execError.Err.Error()
}

func newExecError(err error) ExecError {
	exitCode := 0
	if err != nil {
		exitCode = 1
		if exitError, ok := err.(*exec.ExitError); ok {
			if status, ok := exitError.Sys().(syscall.WaitStatus); ok {
				exitCode = status.ExitStatus()
			}
		}
	}

	return ExecError{Err: err, ExitCode: exitCode}
}

type Runner struct {
	Args []string
}

func (r *Runner) Execute() ExecError {
	args := NewArgs(r.Args)
	if args.Command == "" {
		usage()
	}

	expandAlias(args)
	slurpGlobalFlags(args)

	for _, cmd := range All() {
		if cmd.Name() == args.Command && cmd.Runnable() {
			if !cmd.GitExtension {
				cmd.Flag.Usage = func() {
					cmd.PrintUsage()
				}
				if err := cmd.Flag.Parse(args.Params); err != nil {
					if err == flag.ErrHelp {
						return newExecError(nil)
					} else {
						return newExecError(err)
					}
				}

				args.Params = cmd.Flag.Args()
			}

			cmd.Run(cmd, args)

			cmds := args.Commands()
			var err error
			if args.Noop {
				printCommands(cmds)
			} else {
				err = executeCommands(cmds)
			}

			return newExecError(err)
		}
	}

	err := git.Spawn(args.Command, args.Params...)
	return newExecError(err)
}

func slurpGlobalFlags(args *Args) {
	for i, p := range args.Params {
		if p == "--no-op" {
			args.Noop = true
			args.RemoveParam(i)
		}
	}
}

func printCommands(cmds []*cmd.Cmd) {
	for _, c := range cmds {
		fmt.Println(c)
	}
}

func executeCommands(cmds []*cmd.Cmd) error {
	for _, c := range cmds {
		err := c.Exec()
		if err != nil {
			return err
		}
	}

	return nil
}

func expandAlias(args *Args) {
	cmd := args.Command
	expandedCmd, err := git.Config(fmt.Sprintf("alias.%s", cmd))
	if err == nil && expandedCmd != "" {
		words, err := shellquote.Split(expandedCmd)
		if err != nil {
			args.Command = words[0]
			args.PrependParams(words[1:]...)
		}
	}
}
<commit_msg>Use noop flag instead of no-op<commit_after>package commands

import (
	"flag"
	"fmt"
	"github.com/jingweno/gh/cmd"
	"github.com/jingweno/gh/git"
	shellquote "github.com/kballard/go-shellquote"
	"os/exec"
	"syscall"
)

type ExecError struct {
	Err      error
	ExitCode int
}

func (execError *ExecError) Error() string {
	return execError.Err.Error()
}

func newExecError(err error) ExecError {
	exitCode := 0
	if err != nil {
		exitCode = 1
		if exitError, ok := err.(*exec.ExitError); ok {
			if status, ok := exitError.Sys().(syscall.WaitStatus); ok {
				exitCode = status.ExitStatus()
			}
		}
	}

	return ExecError{Err: err, ExitCode: exitCode}
}

type Runner struct {
	Args []string
}

func (r *Runner) Execute() ExecError {
	args := NewArgs(r.Args)
	if args.Command == "" {
		usage()
	}

	expandAlias(args)
	slurpGlobalFlags(args)

	for _, cmd := range All() {
		if cmd.Name() == args.Command && cmd.Runnable() {
			if !cmd.GitExtension {
				cmd.Flag.Usage = func() {
					cmd.PrintUsage()
				}
				if err := cmd.Flag.Parse(args.Params); err != nil {
					if err == flag.ErrHelp {
						return newExecError(nil)
					} else {
						return newExecError(err)
					}
				}

				args.Params = cmd.Flag.Args()
			}

			cmd.Run(cmd, args)

			cmds := args.Commands()
			var err error
			if args.Noop {
				printCommands(cmds)
			} else {
				err = executeCommands(cmds)
			}

			return newExecError(err)
		}
	}

	err := git.Spawn(args.Command, args.Params...)
	return newExecError(err)
}

func slurpGlobalFlags(args *Args) {
	for i, p := range args.Params {
		if p == "--noop" {
			args.Noop = true
			args.RemoveParam(i)
		}
	}
}

func printCommands(cmds []*cmd.Cmd) {
	for _, c := range cmds {
		fmt.Println(c)
	}
}

func executeCommands(cmds []*cmd.Cmd) error {
	for _, c := range cmds {
		err := c.Exec()
		if err != nil {
			return err
		}
	}

	return nil
}

func expandAlias(args *Args) {
	cmd := args.Command
	expandedCmd, err := git.Config(fmt.Sprintf("alias.%s", cmd))
	if err == nil && expandedCmd != "" {
		words, err := shellquote.Split(expandedCmd)
		if err != nil {
			args.Command = words[0]
			args.PrependParams(words[1:]...)
		}
	}
}
<|endoftext|>


<commit_before>package commands

import "bufio"
import "path/filepath"
import "fmt"
import "os"
import "os/exec"
import "strings"

import "../interpreter"

func cmd_source(cmd *exec.Cmd) (interpreter.NextT, error) {
	if len(cmd.Args) < 2 {
		return interpreter.CONTINUE, nil
	}
	envTxtPath := filepath.Join(
		os.TempDir(),
		fmt.Sprintf("nyagos-%d.tmp", os.Getpid()))

	args := make([]string, 2)
	args[0] = os.Getenv("COMSPEC")
	args[1] = "/C"
	for _, v := range cmd.Args[1:] {
		args = append(args, strings.Replace(v, " ", "^ ", -1))
	}
	args = append(args, "&")
	args = append(args, "set")
	args = append(args, ">")
	args = append(args, envTxtPath)

	var cmd2 exec.Cmd
	cmd2.Path = args[0]
	cmd2.Args = args
	cmd2.Env = nil
	cmd2.Dir = ""
	if err := cmd2.Run(); err != nil {
		return interpreter.CONTINUE, err
	}
	fp, err := os.Open(envTxtPath)
	if err != nil {
		return interpreter.CONTINUE, err
	}
	defer os.Remove(envTxtPath)
	defer fp.Close()

	scr := bufio.NewScanner(fp)
	for scr.Scan() {
		line := scr.Text()
		eqlPos := strings.Index(line, "=")
		if eqlPos > 0 {
			os.Setenv(line[:eqlPos], line[eqlPos+1:])
		}
	}
	return interpreter.CONTINUE, nil
}
<commit_msg>Fix #13: sourceのコマンド名の ( ) を ^( ^) へ置換<commit_after>package commands

import "bufio"
import "path/filepath"
import "fmt"
import "os"
import "os/exec"
import "strings"

import "../interpreter"

func cmd_source(cmd *exec.Cmd) (interpreter.NextT, error) {
	if len(cmd.Args) < 2 {
		return interpreter.CONTINUE, nil
	}
	envTxtPath := filepath.Join(
		os.TempDir(),
		fmt.Sprintf("nyagos-%d.tmp", os.Getpid()))

	args := make([]string, 2)
	args[0] = os.Getenv("COMSPEC")
	args[1] = "/C"
	for _, v := range cmd.Args[1:] {
		args = append(args,
			strings.Replace(
				strings.Replace(
					strings.Replace(v, " ", "^ ", -1), "(", "^(", -1),
				")", "^)", -1))
	}
	args = append(args, "&")
	args = append(args, "set")
	args = append(args, ">")
	args = append(args, envTxtPath)

	var cmd2 exec.Cmd
	cmd2.Path = args[0]
	cmd2.Args = args
	cmd2.Env = nil
	cmd2.Dir = ""
	if err := cmd2.Run(); err != nil {
		return interpreter.CONTINUE, err
	}
	fp, err := os.Open(envTxtPath)
	if err != nil {
		return interpreter.CONTINUE, err
	}
	defer os.Remove(envTxtPath)
	defer fp.Close()

	scr := bufio.NewScanner(fp)
	for scr.Scan() {
		line := scr.Text()
		eqlPos := strings.Index(line, "=")
		if eqlPos > 0 {
			os.Setenv(line[:eqlPos], line[eqlPos+1:])
		}
	}
	return interpreter.CONTINUE, nil
}
<|endoftext|>


<commit_before>// @author Couchbase <info@couchbase.com>
// @copyright 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/couchbase/gometa/log"
	"net"
	"sync"
)

/////////////////////////////////////////////////
// Type Declaration
/////////////////////////////////////////////////

//
// PeerPipe is to maintain the messaging channel
// between two peers. This is not used for leader
// election.
//
type PeerPipe struct {
	conn      net.Conn
	sendch    chan Packet
	receivech chan Packet
	mutex     sync.Mutex
	isClosed  bool
}

/////////////////////////////////////////////////
// Public Function
/////////////////////////////////////////////////

//
// Create a new PeerPipe. The consumer can call
// ReceiveChannel() to get the channel for receving
// message packets.
//
func NewPeerPipe(pconn net.Conn) *PeerPipe {

	pipe := &PeerPipe{conn: pconn,
		sendch:    make(chan Packet, MAX_PROPOSALS*2),
		receivech: make(chan Packet, MAX_PROPOSALS*2),
		isClosed:  false}

	go pipe.doSend()
	go pipe.doReceive()
	return pipe
}

//
// Get the net address of the remote peer.
//
func (p *PeerPipe) GetAddr() string {
	return p.conn.RemoteAddr().String()
}

//
// Return the receive channel.
//
func (p *PeerPipe) ReceiveChannel() <-chan Packet {

	// Just return receivech even if it is closed.  The caller
	// can tell if the channel is closed by using multi-value
	// recieve operator.  Returning a nil channel can cause
	// the caller being block forever.
	//
	return (<-chan Packet)(p.receivech)
}

//
// Close the PeerPipe.  It is safe to call this
// method multiple times without causing panic.
//
func (p *PeerPipe) Close() bool {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if !p.isClosed {

		log.Current.Debugf("PeerPipe.Close(): Remote Address %s", p.GetAddr())
		log.Current.Tracef("%s", "PeerPipe.Close() : Diagnostic Stack ...")
		log.Current.Tracef(log.Current.StackTrace())

		p.isClosed = true

		SafeRun("PeerPipe.Close()",
			func() {
				p.conn.Close()
			})
		SafeRun("PeerPipe.Close()",
			func() {
				close(p.sendch)
			})
		SafeRun("PeerPipe.Close()",
			func() {
				close(p.receivech)
			})
		return true
	}

	return false
}

//
// Send a packet to the peer. This method will return
// false if the pipe is already closed.
//
func (p *PeerPipe) Send(packet Packet) bool {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if !p.isClosed {
		p.sendch <- packet
		return true
	}
	return false
}

/////////////////////////////////////////////////
// Private Function
/////////////////////////////////////////////////

//
// Goroutine.  Go through the send channel and
// send out each packet to the peer as bytes.
//
func (p *PeerPipe) doSend() {
	defer func() {
		if r := recover(); r != nil {
			log.Current.Errorf("caught panic in PeerPipe.doSend() : %s.  Terminate connection upon panic.\n", r)
			log.Current.Errorf("%s", log.Current.StackTrace())
		}

		// This will close the Send and Receive channel
		p.Close()
	}()

	for {
		packet, ok := <-p.sendch
		if !ok {
			// channel close.  Terminate the loop.
			log.Current.Debugf("%s", "PeerPipe.doSend() : Send channel closed.  Terminate.")
			return
		}

		log.Current.Tracef("PeerPipe.doSend() : Prepare to send message %s to Peer %s", packet.Name(), p.GetAddr())

		msg, err := Marshall(packet)
		if err != nil {
			log.Current.Errorf("PeerPipe.doSend() : Fail to marshall message %s to Peer %s. Terminate.", packet.Name(), p.GetAddr())
			return
		}
		size := len(msg)

		// write the packet
		log.Current.Debugf("PeerPipe.doSend() : Sending message %s (len %d) to Peer %s", packet.Name(), size, p.GetAddr())
		log.Current.LazyDebug(packet.String)

		n, err := p.conn.Write(msg)
		if n < size || err != nil {
			// Network error. Close the loop.  The pipe will
			// close and cause subsequent Send() to fail.
			log.Current.Errorf("PeerPipe.doSend() : ecounter error when sending mesasage to Peer %s.  Error = %s.  Terminate.",
				p.GetAddr(), err.Error())
			return
		}
	}
}

//
// Goroutine.  Listen to the connection and
// unmarshall each packet.  Forward the packet to
// receive channel.
func (p *PeerPipe) doReceive() {
	defer func() {
		if r := recover(); r != nil {
			log.Current.Errorf("panic in PeerPipe.doReceive() : %s\n", r)
			log.Current.Errorf("%s", log.Current.StackTrace())
		}

		// This will close the Send and Receive channel
		p.Close()
	}()

	for {
		// read packet len
		lenBuf, err := p.readBytes(8, nil)
		if err != nil {
			// if encountering an error, kill the pipe.
			log.Current.Errorf("PeerPipe.doRecieve() : ecounter error when received mesasage from Peer %s.  Error = %s. Kill Pipe.",
				p.GetAddr(), err.Error())
			return
		}
		size := binary.BigEndian.Uint64(lenBuf)

		// read the content
		readahead, err := p.validateHeader(size)
		if err != nil {
			log.Current.Errorf("PeerPipe.doRecieve() : ecounter error when received mesasage from Peer %s.  Error = %s. Kill Pipe.",
				p.GetAddr(), err.Error())
			return
		}

		buf, err := p.readBytes(size, readahead)
		if err != nil {
			// if encountering an error, kill the pipe.
			log.Current.Errorf("PeerPipe.doRecieve() : ecounter error when received mesasage from Peer %s.  Error = %s. Kill Pipe.",
				p.GetAddr(), err.Error())
			return
		}
		// unmarshall the content and put it in the channel
		packet, err := UnMarshall(buf)
		if err != nil {
			log.Current.Errorf("PeerPipe.doRecieve() : ecounter error when unmarshalling mesasage from Peer %s.  Error = %s. Terminate.",
				p.GetAddr(), err.Error())
			return
		}
		log.Current.Debugf("PeerPipe.doRecieve() : Message decoded.  Packet = %s", packet.Name())
		log.Current.LazyDebug(packet.String)

		// This can block if the reciever of the channel is slow or terminated premauturely (which cause channel to fill up).
		// In this case, this can cause the TCP connection to fail.  The other end of the pipe will close as a result
		// of this.  This end of the pipe will eventually close since it can no longer send message to the other end.
		p.queue(packet)
	}
}

//
// Queue the packe to the recieve channel if
// the channel has not been closed.
//
func (p *PeerPipe) queue(packet Packet) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if !p.isClosed {
		p.receivech <- packet
	}
}

func (p *PeerPipe) readBytes(size uint64, readahead []byte) ([]byte, error) {

	result := new(bytes.Buffer)
	remaining := size

	if readahead != nil {
		result.Write(readahead)
		remaining -= uint64(len(readahead))
	}

	for {
		// read the size of the packet (uint64)
		buf := make([]byte, remaining)
		n, err := p.conn.Read(buf)
		log.Current.Tracef("PeerPipe.readBytes() : Receiving message from Peer %s, bytes read %d", p.GetAddr(), n)

		if n != 0 {
			result.Write(buf[0:n])
			remaining = remaining - uint64(n)

			if remaining == 0 {
				return result.Bytes(), nil
			}
		}

		if err != nil {
			return nil, err
		}
	}

	return result.Bytes(), nil
}

func (p *PeerPipe) validateHeader(size uint64) ([]byte, error) {

	// validate the size of packet
	if size > 20*1024*1024 {
		return nil, errors.New(fmt.Sprintf("Validate packet header: Invalid size %v", size))
	}

	result := new(bytes.Buffer)

	// read the type length
	buf, err := p.readBytes(8, nil)
	if err != nil {
		return nil, err
	}
	result.Write(buf)

	// validate the length of type string
	tyLen := binary.BigEndian.Uint64(buf)
	if tyLen > 2048 {
		return nil, errors.New(fmt.Sprintf("Validate packet header: Invalid type length %v", tyLen))
	}

	// read the type string
	buf, err = p.readBytes(tyLen, nil)
	if err != nil {
		return nil, err
	}
	result.Write(buf)

	// validate type string
	if !IsValidType(string(buf)) {
		return nil, errors.New(fmt.Sprint("Validate packet header: Unrecognize packet type"))
	}

	return result.Bytes(), nil
}
<commit_msg>MB-30774: Do not hold lock while putting packets onto channel<commit_after>// @author Couchbase <info@couchbase.com>
// @copyright 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/couchbase/gometa/log"
	"net"
	"sync"
)

/////////////////////////////////////////////////
// Type Declaration
/////////////////////////////////////////////////

//
// PeerPipe is to maintain the messaging channel
// between two peers. This is not used for leader
// election.
//
type PeerPipe struct {
	conn      net.Conn
	sendch    chan Packet
	receivech chan Packet
	mutex     sync.Mutex
	isClosed  bool
	closech   chan bool
}

/////////////////////////////////////////////////
// Public Function
/////////////////////////////////////////////////

//
// Create a new PeerPipe. The consumer can call
// ReceiveChannel() to get the channel for receving
// message packets.
//
func NewPeerPipe(pconn net.Conn) *PeerPipe {

	pipe := &PeerPipe{conn: pconn,
		sendch:    make(chan Packet, MAX_PROPOSALS*5),
		receivech: make(chan Packet, MAX_PROPOSALS*5),
		isClosed:  false,
		closech:   make(chan bool)}

	go pipe.doSend()
	go pipe.doReceive()
	return pipe
}

//
// Get the net address of the remote peer.
//
func (p *PeerPipe) GetAddr() string {
	return p.conn.RemoteAddr().String()
}

//
// Return the receive channel.
//
func (p *PeerPipe) ReceiveChannel() <-chan Packet {

	// Just return receivech even if it is closed.  The caller
	// can tell if the channel is closed by using multi-value
	// recieve operator.  Returning a nil channel can cause
	// the caller being block forever.
	//
	return (<-chan Packet)(p.receivech)
}

//
// Close the PeerPipe.  It is safe to call this
// method multiple times without causing panic.
//
func (p *PeerPipe) Close() bool {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if !p.isClosed {

		log.Current.Debugf("PeerPipe.Close(): Remote Address %s", p.GetAddr())
		log.Current.Tracef("%s", "PeerPipe.Close() : Diagnostic Stack ...")
		log.Current.Tracef(log.Current.StackTrace())

		p.isClosed = true

		SafeRun("PeerPipe.Close()",
			func() {
				p.conn.Close()
			})
		SafeRun("PeerPipe.Close()",
			func() {
				close(p.sendch)
			})
		SafeRun("PeerPipe.Close()",
			func() {
				close(p.receivech)
			})
		SafeRun("PeerPipe.Close()",
			func() {
				close(p.closech)
			})
		return true
	}

	return false
}

//
// Send a packet to the peer. This method will return
// false if the pipe is already closed.
//
func (p *PeerPipe) Send(packet Packet) bool {
	defer func() {
		recover()
	}()

	select {
	case p.sendch <- packet:
		return true
	case <-p.closech:
		return false
	}

	return false
}

/////////////////////////////////////////////////
// Private Function
/////////////////////////////////////////////////

//
// Goroutine.  Go through the send channel and
// send out each packet to the peer as bytes.
//
func (p *PeerPipe) doSend() {
	defer func() {
		if r := recover(); r != nil {
			log.Current.Errorf("caught panic in PeerPipe.doSend() : %s.  Terminate connection upon panic.\n", r)
			log.Current.Errorf("%s", log.Current.StackTrace())
		}

		// This will close the Send and Receive channel
		p.Close()
	}()

	for {
		packet, ok := <-p.sendch
		if !ok {
			// channel close.  Terminate the loop.
			log.Current.Debugf("%s", "PeerPipe.doSend() : Send channel closed.  Terminate.")
			return
		}

		log.Current.Tracef("PeerPipe.doSend() : Prepare to send message %s to Peer %s", packet.Name(), p.GetAddr())

		msg, err := Marshall(packet)
		if err != nil {
			log.Current.Errorf("PeerPipe.doSend() : Fail to marshall message %s to Peer %s. Terminate.", packet.Name(), p.GetAddr())
			return
		}
		size := len(msg)

		// write the packet
		log.Current.Debugf("PeerPipe.doSend() : Sending message %s (len %d) to Peer %s", packet.Name(), size, p.GetAddr())
		log.Current.LazyDebug(packet.String)

		n, err := p.conn.Write(msg)
		if n < size || err != nil {
			// Network error. Close the loop.  The pipe will
			// close and cause subsequent Send() to fail.
			log.Current.Errorf("PeerPipe.doSend() : ecounter error when sending mesasage to Peer %s.  Error = %s.  Terminate.",
				p.GetAddr(), err.Error())
			return
		}
	}
}

//
// Goroutine.  Listen to the connection and
// unmarshall each packet.  Forward the packet to
// receive channel.
func (p *PeerPipe) doReceive() {
	defer func() {
		if r := recover(); r != nil {
			log.Current.Errorf("panic in PeerPipe.doReceive() : %s\n", r)
			log.Current.Errorf("%s", log.Current.StackTrace())
		}

		// This will close the Send and Receive channel
		p.Close()
	}()

	for {
		// read packet len
		lenBuf, err := p.readBytes(8, nil)
		if err != nil {
			// if encountering an error, kill the pipe.
			log.Current.Errorf("PeerPipe.doRecieve() : ecounter error when received mesasage from Peer %s.  Error = %s. Kill Pipe.",
				p.GetAddr(), err.Error())
			return
		}
		size := binary.BigEndian.Uint64(lenBuf)

		// read the content
		readahead, err := p.validateHeader(size)
		if err != nil {
			log.Current.Errorf("PeerPipe.doRecieve() : ecounter error when received mesasage from Peer %s.  Error = %s. Kill Pipe.",
				p.GetAddr(), err.Error())
			return
		}

		buf, err := p.readBytes(size, readahead)
		if err != nil {
			// if encountering an error, kill the pipe.
			log.Current.Errorf("PeerPipe.doRecieve() : ecounter error when received mesasage from Peer %s.  Error = %s. Kill Pipe.",
				p.GetAddr(), err.Error())
			return
		}
		// unmarshall the content and put it in the channel
		packet, err := UnMarshall(buf)
		if err != nil {
			log.Current.Errorf("PeerPipe.doRecieve() : ecounter error when unmarshalling mesasage from Peer %s.  Error = %s. Terminate.",
				p.GetAddr(), err.Error())
			return
		}
		log.Current.Debugf("PeerPipe.doRecieve() : Message decoded.  Packet = %s", packet.Name())
		log.Current.LazyDebug(packet.String)

		// This can block if the reciever of the channel is slow or terminated premauturely (which cause channel to fill up).
		// In this case, this can cause the TCP connection to fail.  The other end of the pipe will close as a result
		// of this.  This end of the pipe will eventually close since it can no longer send message to the other end.
		p.queue(packet)
	}
}

//
// Queue the packe to the recieve channel if
// the channel has not been closed.
//
func (p *PeerPipe) queue(packet Packet) {
	defer func() {
		recover()
	}()

	select {
	case p.receivech <- packet:
		return
	case <-p.closech:
		return
	}
}

func (p *PeerPipe) readBytes(size uint64, readahead []byte) ([]byte, error) {

	result := new(bytes.Buffer)
	remaining := size

	if readahead != nil {
		result.Write(readahead)
		remaining -= uint64(len(readahead))
	}

	for {
		// read the size of the packet (uint64)
		buf := make([]byte, remaining)
		n, err := p.conn.Read(buf)
		log.Current.Tracef("PeerPipe.readBytes() : Receiving message from Peer %s, bytes read %d", p.GetAddr(), n)

		if n != 0 {
			result.Write(buf[0:n])
			remaining = remaining - uint64(n)

			if remaining == 0 {
				return result.Bytes(), nil
			}
		}

		if err != nil {
			return nil, err
		}
	}

	return result.Bytes(), nil
}

func (p *PeerPipe) validateHeader(size uint64) ([]byte, error) {

	// validate the size of packet
	if size > 20*1024*1024 {
		return nil, errors.New(fmt.Sprintf("Validate packet header: Invalid size %v", size))
	}

	result := new(bytes.Buffer)

	// read the type length
	buf, err := p.readBytes(8, nil)
	if err != nil {
		return nil, err
	}
	result.Write(buf)

	// validate the length of type string
	tyLen := binary.BigEndian.Uint64(buf)
	if tyLen > 2048 {
		return nil, errors.New(fmt.Sprintf("Validate packet header: Invalid type length %v", tyLen))
	}

	// read the type string
	buf, err = p.readBytes(tyLen, nil)
	if err != nil {
		return nil, err
	}
	result.Write(buf)

	// validate type string
	if !IsValidType(string(buf)) {
		return nil, errors.New(fmt.Sprint("Validate packet header: Unrecognize packet type"))
	}

	return result.Bytes(), nil
}
<|endoftext|>


<commit_before>package compare

import (
	"fmt"
	"reflect"

	"github.com/cloudfoundry-incubator/spiff/yaml"
)

type Diff struct {
	A yaml.Node
	B yaml.Node

	Path []string
}

func Compare(a, b yaml.Node) []Diff {
	return compare(a, b, []string{})
}

func compare(a, b yaml.Node, path []string) []Diff {
	mismatch := Diff{A: a, B: b, Path: path}

	switch a.(type) {
	case map[string]yaml.Node:
		switch b.(type) {
		case map[string]yaml.Node:
			return compareMap(a.(map[string]yaml.Node), b.(map[string]yaml.Node), path)

		case []yaml.Node:
			toMap := listToMap(b.([]yaml.Node))

			if toMap != nil {
				return compareMap(a.(map[string]yaml.Node), toMap, path)
			} else {
				return []Diff{mismatch}
			}

		default:
			return []Diff{mismatch}
		}

	case []yaml.Node:
		switch b.(type) {
		case []yaml.Node:
			return compareList(a.([]yaml.Node), b.([]yaml.Node), path)
		default:
			return []Diff{mismatch}
		}

	default:
		atype := reflect.TypeOf(a)
		btype := reflect.TypeOf(b)

		if atype != btype {
			return []Diff{mismatch}
		}

		if a != b {
			return []Diff{Diff{A: a, B: b, Path: path}}
		}
	}

	return []Diff{}
}

func listToMap(list []yaml.Node) map[string]yaml.Node {
	toMap := make(map[string]yaml.Node)

	for _, val := range list {
		name, ok := yaml.FindString(val, "name")
		if !ok {
			return nil
		}

		asMap, ok := val.(map[string]yaml.Node)
		if !ok {
			return nil
		}

		newMap := make(map[string]yaml.Node)
		for key, val := range asMap {
			if key != "name" {
				newMap[key] = val
			}
		}

		toMap[name] = newMap
	}

	return toMap
}

func compareMap(a, b map[string]yaml.Node, path []string) []Diff {
	diff := []Diff{}

	for key, aval := range a {
		bval, present := b[key]
		if present {
			diff = append(diff, compare(aval, bval, addPath(path, key))...)
		} else {
			diff = append(diff, Diff{A: aval, B: nil, Path: addPath(path, key)})
		}
	}

	for key, bval := range b {
		_, present := a[key]
		if !present {
			diff = append(diff, Diff{A: nil, B: bval, Path: addPath(path, key)})
			continue
		}
	}

	return diff
}

func compareList(a, b []yaml.Node, path []string) []Diff {
	diff := []Diff{}

	if len(path) == 1 && path[0] == "jobs" {
		return compareJobs(a, b, path)
	}

	for index, aval := range a {
		key, bval, found := findByNameOrIndex(aval, b, index)

		if !found {
			diff = append(diff, Diff{A: aval, B: nil, Path: addPath(path, key)})
			continue
		}

		diff = append(diff, compare(aval, bval, addPath(path, key))...)
	}

	for index, bval := range b {
		key := fmt.Sprintf("[%d]", index)

		if len(a) <= index {
			diff = append(diff, Diff{A: nil, B: bval, Path: addPath(path, key)})
			continue
		}
	}

	return diff
}

func compareJobs(ajobs, bjobs []yaml.Node, path []string) []Diff {
	return compareMap(jobMap(ajobs), jobMap(bjobs), path)
}

func jobMap(jobs []yaml.Node) map[string]yaml.Node {
	byName := make(map[string]yaml.Node)

	for index, job := range jobs {
		attrs, ok := job.(map[string]yaml.Node)
		attrs["index"] = index

		name, ok := yaml.FindString(job, "name")
		if !ok {
			panic("job without string name")
		}

		byName[name] = attrs
	}

	return byName
}

func findByNameOrIndex(node yaml.Node, others []yaml.Node, index int) (string, yaml.Node, bool) {
	name, ok := yaml.FindString(node, "name")
	if !ok {
		return findByIndex(others, index)
	}

	key, node, found := findByName(name, others)
	if !found {
		return findByIndex(others, index)
	}

	return key, node, true
}

func findByName(name string, nodes []yaml.Node) (string, yaml.Node, bool) {
	for _, node := range nodes {
		otherName, ok := yaml.FindString(node, "name")
		if !ok {
			continue
		}

		if otherName == name {
			return name, node, true
		}
	}

	return "", nil, false
}

func findByIndex(nodes []yaml.Node, index int) (string, yaml.Node, bool) {
	key := fmt.Sprintf("[%d]", index)

	if len(nodes) <= index {
		return key, nil, false
	}

	return key, nodes[index], true
}

// cannot use straight append for this, as it will overwrite
// previous steps, since it reuses the slice
//
// e.g. with inital path A:
//    append(A, "a")
//    append(A, "b")
//
// will result in all previous A/a paths becoming A/b
func addPath(path []string, steps ...string) []string {
	newPath := make([]string, len(path))
	copy(newPath, path)
	return append(newPath, steps...)
}
<commit_msg>use assignment + type match<commit_after>package compare

import (
	"fmt"
	"reflect"

	"github.com/cloudfoundry-incubator/spiff/yaml"
)

type Diff struct {
	A yaml.Node
	B yaml.Node

	Path []string
}

func Compare(a, b yaml.Node) []Diff {
	return compare(a, b, []string{})
}

func compare(a, b yaml.Node, path []string) []Diff {
	mismatch := Diff{A: a, B: b, Path: path}

	switch av := a.(type) {
	case map[string]yaml.Node:
		switch bv := b.(type) {
		case map[string]yaml.Node:
			return compareMap(av, bv, path)

		case []yaml.Node:
			toMap := listToMap(bv)

			if toMap != nil {
				return compareMap(av, toMap, path)
			} else {
				return []Diff{mismatch}
			}

		default:
			return []Diff{mismatch}
		}

	case []yaml.Node:
		switch bv := b.(type) {
		case []yaml.Node:
			return compareList(av, bv, path)
		default:
			return []Diff{mismatch}
		}

	default:
		atype := reflect.TypeOf(a)
		btype := reflect.TypeOf(b)

		if atype != btype {
			return []Diff{mismatch}
		}

		if a != b {
			return []Diff{Diff{A: a, B: b, Path: path}}
		}
	}

	return []Diff{}
}

func listToMap(list []yaml.Node) map[string]yaml.Node {
	toMap := make(map[string]yaml.Node)

	for _, val := range list {
		name, ok := yaml.FindString(val, "name")
		if !ok {
			return nil
		}

		asMap, ok := val.(map[string]yaml.Node)
		if !ok {
			return nil
		}

		newMap := make(map[string]yaml.Node)
		for key, val := range asMap {
			if key != "name" {
				newMap[key] = val
			}
		}

		toMap[name] = newMap
	}

	return toMap
}

func compareMap(a, b map[string]yaml.Node, path []string) []Diff {
	diff := []Diff{}

	for key, aval := range a {
		bval, present := b[key]
		if present {
			diff = append(diff, compare(aval, bval, addPath(path, key))...)
		} else {
			diff = append(diff, Diff{A: aval, B: nil, Path: addPath(path, key)})
		}
	}

	for key, bval := range b {
		_, present := a[key]
		if !present {
			diff = append(diff, Diff{A: nil, B: bval, Path: addPath(path, key)})
			continue
		}
	}

	return diff
}

func compareList(a, b []yaml.Node, path []string) []Diff {
	diff := []Diff{}

	if len(path) == 1 && path[0] == "jobs" {
		return compareJobs(a, b, path)
	}

	for index, aval := range a {
		key, bval, found := findByNameOrIndex(aval, b, index)

		if !found {
			diff = append(diff, Diff{A: aval, B: nil, Path: addPath(path, key)})
			continue
		}

		diff = append(diff, compare(aval, bval, addPath(path, key))...)
	}

	for index, bval := range b {
		key := fmt.Sprintf("[%d]", index)

		if len(a) <= index {
			diff = append(diff, Diff{A: nil, B: bval, Path: addPath(path, key)})
			continue
		}
	}

	return diff
}

func compareJobs(ajobs, bjobs []yaml.Node, path []string) []Diff {
	return compareMap(jobMap(ajobs), jobMap(bjobs), path)
}

func jobMap(jobs []yaml.Node) map[string]yaml.Node {
	byName := make(map[string]yaml.Node)

	for index, job := range jobs {
		attrs, ok := job.(map[string]yaml.Node)
		attrs["index"] = index

		name, ok := yaml.FindString(job, "name")
		if !ok {
			panic("job without string name")
		}

		byName[name] = attrs
	}

	return byName
}

func findByNameOrIndex(node yaml.Node, others []yaml.Node, index int) (string, yaml.Node, bool) {
	name, ok := yaml.FindString(node, "name")
	if !ok {
		return findByIndex(others, index)
	}

	key, node, found := findByName(name, others)
	if !found {
		return findByIndex(others, index)
	}

	return key, node, true
}

func findByName(name string, nodes []yaml.Node) (string, yaml.Node, bool) {
	for _, node := range nodes {
		otherName, ok := yaml.FindString(node, "name")
		if !ok {
			continue
		}

		if otherName == name {
			return name, node, true
		}
	}

	return "", nil, false
}

func findByIndex(nodes []yaml.Node, index int) (string, yaml.Node, bool) {
	key := fmt.Sprintf("[%d]", index)

	if len(nodes) <= index {
		return key, nil, false
	}

	return key, nodes[index], true
}

// cannot use straight append for this, as it will overwrite
// previous steps, since it reuses the slice
//
// e.g. with inital path A:
//    append(A, "a")
//    append(A, "b")
//
// will result in all previous A/a paths becoming A/b
func addPath(path []string, steps ...string) []string {
	newPath := make([]string, len(path))
	copy(newPath, path)
	return append(newPath, steps...)
}
<|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/schema"
)

const (
	defaultListUnitFilesFields = "unit,hash,dstate,state,target"
)

func mapTargetField(u schema.Unit, full bool) string {
	if suToGlobal(u) {
		return "global"
	}
	if u.MachineID == "" {
		return "-"
	}
	ms := cachedMachineState(u.MachineID)
	if ms == nil {
		ms = &machine.MachineState{ID: u.MachineID}
	}

	return machineFullLegend(*ms, full)
}

var (
	listUnitFilesFieldsFlag string
	cmdListUnitFiles        = &Command{
		Name:        "list-unit-files",
		Summary:     "List the units that exist in the cluster.",
		Usage:       "[--fields]",
		Description: `Lists all unit files that exist in the cluster (whether or not they are loaded onto a machine).`,
		Run:         runListUnitFiles,
	}
	listUnitFilesFields = map[string]unitToField{
		"unit": func(u schema.Unit, full bool) string {
			return u.Name
		},
		"global": func(u schema.Unit, full bool) string {
			return strconv.FormatBool(suToGlobal(u))
		},
		"dstate": func(u schema.Unit, full bool) string {
			if u.DesiredState == "" {
				return "-"
			}
			return u.DesiredState
		},
		"target":   mapTargetField,
		"tmachine": mapTargetField,
		"state": func(u schema.Unit, full bool) string {
			if suToGlobal(u) || u.CurrentState == "" {
				return "-"
			}
			return u.CurrentState
		},
		"hash": func(u schema.Unit, full bool) string {
			uf := schema.MapSchemaUnitOptionsToUnitFile(u.Options)
			if !full {
				return uf.Hash().Short()
			}
			return uf.Hash().String()
		},
		"desc": func(u schema.Unit, full bool) string {
			uf := schema.MapSchemaUnitOptionsToUnitFile(u.Options)
			d := uf.Description()
			if d == "" {
				return "-"
			}
			return d
		},
	}
)

type unitToField func(u schema.Unit, full bool) string

func init() {
	cmdListUnitFiles.Flags.BoolVar(&sharedFlags.Full, "full", false, "Do not ellipsize fields on output")
	cmdListUnitFiles.Flags.BoolVar(&sharedFlags.NoLegend, "no-legend", false, "Do not print a legend (column headers)")
	cmdListUnitFiles.Flags.StringVar(&listUnitFilesFieldsFlag, "fields", defaultListUnitFilesFields, fmt.Sprintf("Columns to print for each Unit file. Valid fields are %q", strings.Join(unitToFieldKeys(listUnitFilesFields), ",")))
}

func runListUnitFiles(args []string) (exit int) {
	if listUnitFilesFieldsFlag == "" {
		stderr("Must define output format")
		return 1
	}

	cols := strings.Split(listUnitFilesFieldsFlag, ",")
	for _, s := range cols {
		if _, ok := listUnitFilesFields[s]; !ok {
			stderr("Invalid key in output format: %q", s)
			return 1
		}
		if s == "tmachine" {
			stderr("WARNING: The \"tmachine\" field is deprecated. Use \"target\" instead")
		}
	}

	units, err := cAPI.Units()
	if err != nil {
		stderr("Error retrieving list of units from repository: %v", err)
		return 1
	}

	if !sharedFlags.NoLegend {
		fmt.Fprintln(out, strings.ToUpper(strings.Join(cols, "\t")))
	}

	for _, u := range units {
		var f []string
		for _, c := range cols {
			f = append(f, listUnitFilesFields[c](*u, sharedFlags.Full))
		}
		fmt.Fprintln(out, strings.Join(f, "\t"))
	}

	out.Flush()
	return
}

func unitToFieldKeys(m map[string]unitToField) (keys []string) {
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return
}
<commit_msg>fleetctl: list-unit-files fully populated for global units<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/schema"
)

const (
	defaultListUnitFilesFields = "unit,hash,dstate,state,target"
)

func mapTargetField(u schema.Unit, full bool) string {
	if suToGlobal(u) {
		return "global"
	}
	if u.MachineID == "" {
		return "-"
	}
	ms := cachedMachineState(u.MachineID)
	if ms == nil {
		ms = &machine.MachineState{ID: u.MachineID}
	}

	return machineFullLegend(*ms, full)
}

var (
	listUnitFilesFieldsFlag string
	cmdListUnitFiles        = &Command{
		Name:        "list-unit-files",
		Summary:     "List the units that exist in the cluster.",
		Usage:       "[--fields]",
		Description: `Lists all unit files that exist in the cluster (whether or not they are loaded onto a machine).`,
		Run:         runListUnitFiles,
	}
	listUnitFilesFields = map[string]unitToField{
		"unit": func(u schema.Unit, full bool) string {
			return u.Name
		},
		"global": func(u schema.Unit, full bool) string {
			return strconv.FormatBool(suToGlobal(u))
		},
		"dstate": func(u schema.Unit, full bool) string {
			if u.DesiredState == "" {
				return "-"
			}
			return u.DesiredState
		},
		"target":   mapTargetField,
		"tmachine": mapTargetField,
		"state": func(u schema.Unit, full bool) string {
			if suToGlobal(u) {
				states, err := cAPI.UnitStates()
				if err != nil {
					return "-"
				}
				var (
					inactive      int
					loaded        int
					launched      int
					currentStates []string
				)
				for _, state := range states {
					if state.Name == u.Name {
						if (state.SystemdActiveState == "active") || (state.SystemdSubState == "running") {
							launched++
							continue
						}
						if state.SystemdLoadState == "loaded" {
							loaded++
							continue
						}
						inactive++
					}
				}
				if launched != 0 {
					s := fmt.Sprintf("launched(%d)", launched)
					currentStates = append(currentStates, s)
				}
				if loaded != 0 {
					s := fmt.Sprintf("loaded(%d)", loaded)
					currentStates = append(currentStates, s)
				}
				if inactive != 0 {
					s := fmt.Sprintf("inactive(%d)", inactive)
					currentStates = append(currentStates, s)
				}
				return strings.Join(currentStates, " ")
			}
			if u.CurrentState == "" {
				return "-"
			}
			return u.CurrentState
		},
		"hash": func(u schema.Unit, full bool) string {
			uf := schema.MapSchemaUnitOptionsToUnitFile(u.Options)
			if !full {
				return uf.Hash().Short()
			}
			return uf.Hash().String()
		},
		"desc": func(u schema.Unit, full bool) string {
			uf := schema.MapSchemaUnitOptionsToUnitFile(u.Options)
			d := uf.Description()
			if d == "" {
				return "-"
			}
			return d
		},
	}
)

type unitToField func(u schema.Unit, full bool) string

func init() {
	cmdListUnitFiles.Flags.BoolVar(&sharedFlags.Full, "full", false, "Do not ellipsize fields on output")
	cmdListUnitFiles.Flags.BoolVar(&sharedFlags.NoLegend, "no-legend", false, "Do not print a legend (column headers)")
	cmdListUnitFiles.Flags.StringVar(&listUnitFilesFieldsFlag, "fields", defaultListUnitFilesFields, fmt.Sprintf("Columns to print for each Unit file. Valid fields are %q", strings.Join(unitToFieldKeys(listUnitFilesFields), ",")))
}

func runListUnitFiles(args []string) (exit int) {
	if listUnitFilesFieldsFlag == "" {
		stderr("Must define output format")
		return 1
	}

	cols := strings.Split(listUnitFilesFieldsFlag, ",")
	for _, s := range cols {
		if _, ok := listUnitFilesFields[s]; !ok {
			stderr("Invalid key in output format: %q", s)
			return 1
		}
		if s == "tmachine" {
			stderr("WARNING: The \"tmachine\" field is deprecated. Use \"target\" instead")
		}
	}

	units, err := cAPI.Units()
	if err != nil {
		stderr("Error retrieving list of units from repository: %v", err)
		return 1
	}

	if !sharedFlags.NoLegend {
		fmt.Fprintln(out, strings.ToUpper(strings.Join(cols, "\t")))
	}

	for _, u := range units {
		var f []string
		for _, c := range cols {
			f = append(f, listUnitFilesFields[c](*u, sharedFlags.Full))
		}
		fmt.Fprintln(out, strings.Join(f, "\t"))
	}

	out.Flush()
	return
}

func unitToFieldKeys(m map[string]unitToField) (keys []string) {
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return
}
<|endoftext|>


<commit_before>package sarama

import (
	"fmt"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	toxiproxy "github.com/Shopify/toxiproxy/client"
	"github.com/rcrowley/go-metrics"
)

const TestBatchSize = 1000

func TestFuncProducing(t *testing.T) {
	config := NewConfig()
	testProducingMessages(t, config)
}

func TestFuncProducingGzip(t *testing.T) {
	config := NewConfig()
	config.Producer.Compression = CompressionGZIP
	testProducingMessages(t, config)
}

func TestFuncProducingSnappy(t *testing.T) {
	config := NewConfig()
	config.Producer.Compression = CompressionSnappy
	testProducingMessages(t, config)
}

func TestFuncProducingZstd(t *testing.T) {
	config := NewConfig()
	config.Producer.Compression = CompressionZSTD
	testProducingMessages(t, config)
}

func TestFuncProducingNoResponse(t *testing.T) {
	config := NewConfig()
	config.Producer.RequiredAcks = NoResponse
	testProducingMessages(t, config)
}

func TestFuncProducingFlushing(t *testing.T) {
	config := NewConfig()
	config.Producer.Flush.Messages = TestBatchSize / 8
	config.Producer.Flush.Frequency = 250 * time.Millisecond
	testProducingMessages(t, config)
}

func TestFuncMultiPartitionProduce(t *testing.T) {
	setupFunctionalTest(t)
	defer teardownFunctionalTest(t)

	config := NewConfig()
	config.ChannelBufferSize = 20
	config.Producer.Flush.Frequency = 50 * time.Millisecond
	config.Producer.Flush.Messages = 200
	config.Producer.Return.Successes = true
	producer, err := NewSyncProducer(kafkaBrokers, config)
	if err != nil {
		t.Fatal(err)
	}

	var wg sync.WaitGroup
	wg.Add(TestBatchSize)

	for i := 1; i <= TestBatchSize; i++ {
		go func(i int) {
			defer wg.Done()
			msg := &ProducerMessage{Topic: "test.64", Key: nil, Value: StringEncoder(fmt.Sprintf("hur %d", i))}
			if _, _, err := producer.SendMessage(msg); err != nil {
				t.Error(i, err)
			}
		}(i)
	}

	wg.Wait()
	if err := producer.Close(); err != nil {
		t.Error(err)
	}
}

func TestFuncProducingToInvalidTopic(t *testing.T) {
	setupFunctionalTest(t)
	defer teardownFunctionalTest(t)

	producer, err := NewSyncProducer(kafkaBrokers, nil)
	if err != nil {
		t.Fatal(err)
	}

	if _, _, err := producer.SendMessage(&ProducerMessage{Topic: "in/valid"}); err != ErrUnknownTopicOrPartition {
		t.Error("Expected ErrUnknownTopicOrPartition, found", err)
	}

	if _, _, err := producer.SendMessage(&ProducerMessage{Topic: "in/valid"}); err != ErrUnknownTopicOrPartition {
		t.Error("Expected ErrUnknownTopicOrPartition, found", err)
	}

	safeClose(t, producer)
}

func TestFuncProducingIdempotentWithBrokerFailure(t *testing.T) {
	setupFunctionalTest(t)
	defer teardownFunctionalTest(t)

	config := NewConfig()
	config.Producer.Flush.Frequency = 250 * time.Millisecond
	config.Producer.Idempotent = true
	config.Producer.Timeout = 500 * time.Millisecond
	config.Producer.Retry.Max = 1
	config.Producer.Retry.Backoff = 500 * time.Millisecond
	config.Producer.Return.Successes = true
	config.Producer.Return.Errors = true
	config.Producer.RequiredAcks = WaitForAll
	config.Net.MaxOpenRequests = 1
	config.Version = V0_11_0_0

	producer, err := NewSyncProducer(kafkaBrokers, config)
	if err != nil {
		t.Fatal(err)
	}
	defer safeClose(t, producer)

	// Successfully publish a few messages
	for i := 0; i < 10; i++ {
		_, _, err = producer.SendMessage(&ProducerMessage{
			Topic: "test.1",
			Value: StringEncoder(fmt.Sprintf("%d message", i)),
		})
		if err != nil {
			t.Fatal(err)
		}
	}

	// break the brokers.
	for proxyName, proxy := range Proxies {
		if !strings.Contains(proxyName, "kafka") {
			continue
		}
		if err := proxy.Disable(); err != nil {
			t.Fatal(err)
		}
	}

	// This should fail hard now
	for i := 10; i < 20; i++ {
		_, _, err = producer.SendMessage(&ProducerMessage{
			Topic: "test.1",
			Value: StringEncoder(fmt.Sprintf("%d message", i)),
		})
		if err == nil {
			t.Fatal(err)
		}
	}

	// Now bring the proxy back up
	for proxyName, proxy := range Proxies {
		if !strings.Contains(proxyName, "kafka") {
			continue
		}
		if err := proxy.Enable(); err != nil {
			t.Fatal(err)
		}
	}

	// We should be able to publish again (once everything calms down)
	// (otherwise it times out)
	for {
		_, _, err = producer.SendMessage(&ProducerMessage{
			Topic: "test.1",
			Value: StringEncoder("comeback message"),
		})
		if err == nil {
			break
		}
	}
}

func testProducingMessages(t *testing.T, config *Config) {
	setupFunctionalTest(t)
	defer teardownFunctionalTest(t)

	// Configure some latency in order to properly validate the request latency metric
	for _, proxy := range Proxies {
		if _, err := proxy.AddToxic("", "latency", "", 1, toxiproxy.Attributes{"latency": 10}); err != nil {
			t.Fatal("Unable to configure latency toxicity", err)
		}
	}

	config.Producer.Return.Successes = true
	config.Consumer.Return.Errors = true

	client, err := NewClient(kafkaBrokers, config)
	if err != nil {
		t.Fatal(err)
	}

	// Keep in mind the current offset
	initialOffset, err := client.GetOffset("test.1", 0, OffsetNewest)
	if err != nil {
		t.Fatal(err)
	}

	producer, err := NewAsyncProducerFromClient(client)
	if err != nil {
		t.Fatal(err)
	}

	expectedResponses := TestBatchSize
	for i := 1; i <= TestBatchSize; {
		msg := &ProducerMessage{Topic: "test.1", Key: nil, Value: StringEncoder(fmt.Sprintf("testing %d", i))}
		select {
		case producer.Input() <- msg:
			i++
		case ret := <-producer.Errors():
			t.Fatal(ret.Err)
		case <-producer.Successes():
			expectedResponses--
		}
	}
	for expectedResponses > 0 {
		select {
		case ret := <-producer.Errors():
			t.Fatal(ret.Err)
		case <-producer.Successes():
			expectedResponses--
		}
	}
	safeClose(t, producer)

	// Validate producer metrics before using the consumer minus the offset request
	validateMetrics(t, client)

	master, err := NewConsumerFromClient(client)
	if err != nil {
		t.Fatal(err)
	}
	consumer, err := master.ConsumePartition("test.1", 0, initialOffset)
	if err != nil {
		t.Fatal(err)
	}

	for i := 1; i <= TestBatchSize; i++ {
		select {
		case <-time.After(10 * time.Second):
			t.Fatal("Not received any more events in the last 10 seconds.")

		case err := <-consumer.Errors():
			t.Error(err)

		case message := <-consumer.Messages():
			if string(message.Value) != fmt.Sprintf("testing %d", i) {
				t.Fatalf("Unexpected message with index %d: %s", i, message.Value)
			}
		}
	}
	safeClose(t, consumer)
	safeClose(t, client)
}

func validateMetrics(t *testing.T, client Client) {
	// Get the broker used by test1 topic
	var broker *Broker
	if partitions, err := client.Partitions("test.1"); err != nil {
		t.Error(err)
	} else {
		for _, partition := range partitions {
			if b, err := client.Leader("test.1", partition); err != nil {
				t.Error(err)
			} else {
				if broker != nil && b != broker {
					t.Fatal("Expected only one broker, got at least 2")
				}
				broker = b
			}
		}
	}

	metricValidators := newMetricValidators()
	noResponse := client.Config().Producer.RequiredAcks == NoResponse
	compressionEnabled := client.Config().Producer.Compression != CompressionNone

	// We are adding 10ms of latency to all requests with toxiproxy
	minRequestLatencyInMs := 10
	if noResponse {
		// but when we do not wait for a response it can be less than 1ms
		minRequestLatencyInMs = 0
	}

	// We read at least 1 byte from the broker
	metricValidators.registerForAllBrokers(broker, minCountMeterValidator("incoming-byte-rate", 1))
	// in at least 3 global requests (1 for metadata request, 1 for offset request and N for produce request)
	metricValidators.register(minCountMeterValidator("request-rate", 3))
	metricValidators.register(minCountHistogramValidator("request-size", 3))
	metricValidators.register(minValHistogramValidator("request-size", 1))
	// and at least 2 requests to the registered broker (offset + produces)
	metricValidators.registerForBroker(broker, minCountMeterValidator("request-rate", 2))
	metricValidators.registerForBroker(broker, minCountHistogramValidator("request-size", 2))
	metricValidators.registerForBroker(broker, minValHistogramValidator("request-size", 1))
	metricValidators.registerForBroker(broker, minValHistogramValidator("request-latency-in-ms", minRequestLatencyInMs))

	// We send at least 1 batch
	metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("batch-size", 1))
	metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("batch-size", 1))
	if compressionEnabled {
		// We record compression ratios between [0.50,-10.00] (50-1000 with a histogram) for at least one "fake" record
		metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("compression-ratio", 1))
		metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("compression-ratio", 50))
		metricValidators.registerForGlobalAndTopic("test_1", maxValHistogramValidator("compression-ratio", 1000))
	} else {
		// We record compression ratios of 1.00 (100 with a histogram) for every TestBatchSize record
		metricValidators.registerForGlobalAndTopic("test_1", countHistogramValidator("compression-ratio", TestBatchSize))
		metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("compression-ratio", 100))
		metricValidators.registerForGlobalAndTopic("test_1", maxValHistogramValidator("compression-ratio", 100))
	}

	// We send exactly TestBatchSize messages
	metricValidators.registerForGlobalAndTopic("test_1", countMeterValidator("record-send-rate", TestBatchSize))
	// We send at least one record per request
	metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("records-per-request", 1))
	metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("records-per-request", 1))

	// We receive at least 1 byte from the broker
	metricValidators.registerForAllBrokers(broker, minCountMeterValidator("outgoing-byte-rate", 1))
	if noResponse {
		// in exactly 2 global responses (metadata + offset)
		metricValidators.register(countMeterValidator("response-rate", 2))
		metricValidators.register(minCountHistogramValidator("response-size", 2))
		// and exactly 1 offset response for the registered broker
		metricValidators.registerForBroker(broker, countMeterValidator("response-rate", 1))
		metricValidators.registerForBroker(broker, minCountHistogramValidator("response-size", 1))
		metricValidators.registerForBroker(broker, minValHistogramValidator("response-size", 1))
	} else {
		// in at least 3 global responses (metadata + offset + produces)
		metricValidators.register(minCountMeterValidator("response-rate", 3))
		metricValidators.register(minCountHistogramValidator("response-size", 3))
		// and at least 2 for the registered broker
		metricValidators.registerForBroker(broker, minCountMeterValidator("response-rate", 2))
		metricValidators.registerForBroker(broker, minCountHistogramValidator("response-size", 2))
		metricValidators.registerForBroker(broker, minValHistogramValidator("response-size", 1))
	}

	// There should be no requests in flight anymore
	metricValidators.registerForAllBrokers(broker, counterValidator("requests-in-flight", 0))

	// Run the validators
	metricValidators.run(t, client.Config().MetricRegistry)
}

// Benchmarks

func BenchmarkProducerSmall(b *testing.B) {
	benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 128)))
}
func BenchmarkProducerMedium(b *testing.B) {
	benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 1024)))
}
func BenchmarkProducerLarge(b *testing.B) {
	benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 8192)))
}
func BenchmarkProducerSmallSinglePartition(b *testing.B) {
	benchmarkProducer(b, nil, "test.1", ByteEncoder(make([]byte, 128)))
}
func BenchmarkProducerMediumSnappy(b *testing.B) {
	conf := NewConfig()
	conf.Producer.Compression = CompressionSnappy
	benchmarkProducer(b, conf, "test.1", ByteEncoder(make([]byte, 1024)))
}

func benchmarkProducer(b *testing.B, conf *Config, topic string, value Encoder) {
	setupFunctionalTest(b)
	defer teardownFunctionalTest(b)

	metricsDisable := os.Getenv("METRICS_DISABLE")
	if metricsDisable != "" {
		previousUseNilMetrics := metrics.UseNilMetrics
		Logger.Println("Disabling metrics using no-op implementation")
		metrics.UseNilMetrics = true
		// Restore previous setting
		defer func() {
			metrics.UseNilMetrics = previousUseNilMetrics
		}()
	}

	producer, err := NewAsyncProducer(kafkaBrokers, conf)
	if err != nil {
		b.Fatal(err)
	}

	b.ResetTimer()

	for i := 1; i <= b.N; {
		msg := &ProducerMessage{Topic: topic, Key: StringEncoder(fmt.Sprintf("%d", i)), Value: value}
		select {
		case producer.Input() <- msg:
			i++
		case ret := <-producer.Errors():
			b.Fatal(ret.Err)
		}
	}
	safeClose(b, producer)
}
<commit_msg>zstd needs client >= 2.1.0.0<commit_after>package sarama

import (
	"fmt"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	toxiproxy "github.com/Shopify/toxiproxy/client"
	"github.com/rcrowley/go-metrics"
)

const TestBatchSize = 1000

func TestFuncProducing(t *testing.T) {
	config := NewConfig()
	testProducingMessages(t, config)
}

func TestFuncProducingGzip(t *testing.T) {
	config := NewConfig()
	config.Producer.Compression = CompressionGZIP
	testProducingMessages(t, config)
}

func TestFuncProducingSnappy(t *testing.T) {
	config := NewConfig()
	config.Producer.Compression = CompressionSnappy
	testProducingMessages(t, config)
}

func TestFuncProducingZstd(t *testing.T) {
	config := NewConfig()
	config.Version = V2_1_0_0
	config.Producer.Compression = CompressionZSTD
	testProducingMessages(t, config)
}

func TestFuncProducingNoResponse(t *testing.T) {
	config := NewConfig()
	config.Producer.RequiredAcks = NoResponse
	testProducingMessages(t, config)
}

func TestFuncProducingFlushing(t *testing.T) {
	config := NewConfig()
	config.Producer.Flush.Messages = TestBatchSize / 8
	config.Producer.Flush.Frequency = 250 * time.Millisecond
	testProducingMessages(t, config)
}

func TestFuncMultiPartitionProduce(t *testing.T) {
	setupFunctionalTest(t)
	defer teardownFunctionalTest(t)

	config := NewConfig()
	config.ChannelBufferSize = 20
	config.Producer.Flush.Frequency = 50 * time.Millisecond
	config.Producer.Flush.Messages = 200
	config.Producer.Return.Successes = true
	producer, err := NewSyncProducer(kafkaBrokers, config)
	if err != nil {
		t.Fatal(err)
	}

	var wg sync.WaitGroup
	wg.Add(TestBatchSize)

	for i := 1; i <= TestBatchSize; i++ {
		go func(i int) {
			defer wg.Done()
			msg := &ProducerMessage{Topic: "test.64", Key: nil, Value: StringEncoder(fmt.Sprintf("hur %d", i))}
			if _, _, err := producer.SendMessage(msg); err != nil {
				t.Error(i, err)
			}
		}(i)
	}

	wg.Wait()
	if err := producer.Close(); err != nil {
		t.Error(err)
	}
}

func TestFuncProducingToInvalidTopic(t *testing.T) {
	setupFunctionalTest(t)
	defer teardownFunctionalTest(t)

	producer, err := NewSyncProducer(kafkaBrokers, nil)
	if err != nil {
		t.Fatal(err)
	}

	if _, _, err := producer.SendMessage(&ProducerMessage{Topic: "in/valid"}); err != ErrUnknownTopicOrPartition {
		t.Error("Expected ErrUnknownTopicOrPartition, found", err)
	}

	if _, _, err := producer.SendMessage(&ProducerMessage{Topic: "in/valid"}); err != ErrUnknownTopicOrPartition {
		t.Error("Expected ErrUnknownTopicOrPartition, found", err)
	}

	safeClose(t, producer)
}

func TestFuncProducingIdempotentWithBrokerFailure(t *testing.T) {
	setupFunctionalTest(t)
	defer teardownFunctionalTest(t)

	config := NewConfig()
	config.Producer.Flush.Frequency = 250 * time.Millisecond
	config.Producer.Idempotent = true
	config.Producer.Timeout = 500 * time.Millisecond
	config.Producer.Retry.Max = 1
	config.Producer.Retry.Backoff = 500 * time.Millisecond
	config.Producer.Return.Successes = true
	config.Producer.Return.Errors = true
	config.Producer.RequiredAcks = WaitForAll
	config.Net.MaxOpenRequests = 1
	config.Version = V0_11_0_0

	producer, err := NewSyncProducer(kafkaBrokers, config)
	if err != nil {
		t.Fatal(err)
	}
	defer safeClose(t, producer)

	// Successfully publish a few messages
	for i := 0; i < 10; i++ {
		_, _, err = producer.SendMessage(&ProducerMessage{
			Topic: "test.1",
			Value: StringEncoder(fmt.Sprintf("%d message", i)),
		})
		if err != nil {
			t.Fatal(err)
		}
	}

	// break the brokers.
	for proxyName, proxy := range Proxies {
		if !strings.Contains(proxyName, "kafka") {
			continue
		}
		if err := proxy.Disable(); err != nil {
			t.Fatal(err)
		}
	}

	// This should fail hard now
	for i := 10; i < 20; i++ {
		_, _, err = producer.SendMessage(&ProducerMessage{
			Topic: "test.1",
			Value: StringEncoder(fmt.Sprintf("%d message", i)),
		})
		if err == nil {
			t.Fatal(err)
		}
	}

	// Now bring the proxy back up
	for proxyName, proxy := range Proxies {
		if !strings.Contains(proxyName, "kafka") {
			continue
		}
		if err := proxy.Enable(); err != nil {
			t.Fatal(err)
		}
	}

	// We should be able to publish again (once everything calms down)
	// (otherwise it times out)
	for {
		_, _, err = producer.SendMessage(&ProducerMessage{
			Topic: "test.1",
			Value: StringEncoder("comeback message"),
		})
		if err == nil {
			break
		}
	}
}

func testProducingMessages(t *testing.T, config *Config) {
	setupFunctionalTest(t)
	defer teardownFunctionalTest(t)

	// Configure some latency in order to properly validate the request latency metric
	for _, proxy := range Proxies {
		if _, err := proxy.AddToxic("", "latency", "", 1, toxiproxy.Attributes{"latency": 10}); err != nil {
			t.Fatal("Unable to configure latency toxicity", err)
		}
	}

	config.Producer.Return.Successes = true
	config.Consumer.Return.Errors = true

	client, err := NewClient(kafkaBrokers, config)
	if err != nil {
		t.Fatal(err)
	}

	// Keep in mind the current offset
	initialOffset, err := client.GetOffset("test.1", 0, OffsetNewest)
	if err != nil {
		t.Fatal(err)
	}

	producer, err := NewAsyncProducerFromClient(client)
	if err != nil {
		t.Fatal(err)
	}

	expectedResponses := TestBatchSize
	for i := 1; i <= TestBatchSize; {
		msg := &ProducerMessage{Topic: "test.1", Key: nil, Value: StringEncoder(fmt.Sprintf("testing %d", i))}
		select {
		case producer.Input() <- msg:
			i++
		case ret := <-producer.Errors():
			t.Fatal(ret.Err)
		case <-producer.Successes():
			expectedResponses--
		}
	}
	for expectedResponses > 0 {
		select {
		case ret := <-producer.Errors():
			t.Fatal(ret.Err)
		case <-producer.Successes():
			expectedResponses--
		}
	}
	safeClose(t, producer)

	// Validate producer metrics before using the consumer minus the offset request
	validateMetrics(t, client)

	master, err := NewConsumerFromClient(client)
	if err != nil {
		t.Fatal(err)
	}
	consumer, err := master.ConsumePartition("test.1", 0, initialOffset)
	if err != nil {
		t.Fatal(err)
	}

	for i := 1; i <= TestBatchSize; i++ {
		select {
		case <-time.After(10 * time.Second):
			t.Fatal("Not received any more events in the last 10 seconds.")

		case err := <-consumer.Errors():
			t.Error(err)

		case message := <-consumer.Messages():
			if string(message.Value) != fmt.Sprintf("testing %d", i) {
				t.Fatalf("Unexpected message with index %d: %s", i, message.Value)
			}
		}
	}
	safeClose(t, consumer)
	safeClose(t, client)
}

func validateMetrics(t *testing.T, client Client) {
	// Get the broker used by test1 topic
	var broker *Broker
	if partitions, err := client.Partitions("test.1"); err != nil {
		t.Error(err)
	} else {
		for _, partition := range partitions {
			if b, err := client.Leader("test.1", partition); err != nil {
				t.Error(err)
			} else {
				if broker != nil && b != broker {
					t.Fatal("Expected only one broker, got at least 2")
				}
				broker = b
			}
		}
	}

	metricValidators := newMetricValidators()
	noResponse := client.Config().Producer.RequiredAcks == NoResponse
	compressionEnabled := client.Config().Producer.Compression != CompressionNone

	// We are adding 10ms of latency to all requests with toxiproxy
	minRequestLatencyInMs := 10
	if noResponse {
		// but when we do not wait for a response it can be less than 1ms
		minRequestLatencyInMs = 0
	}

	// We read at least 1 byte from the broker
	metricValidators.registerForAllBrokers(broker, minCountMeterValidator("incoming-byte-rate", 1))
	// in at least 3 global requests (1 for metadata request, 1 for offset request and N for produce request)
	metricValidators.register(minCountMeterValidator("request-rate", 3))
	metricValidators.register(minCountHistogramValidator("request-size", 3))
	metricValidators.register(minValHistogramValidator("request-size", 1))
	// and at least 2 requests to the registered broker (offset + produces)
	metricValidators.registerForBroker(broker, minCountMeterValidator("request-rate", 2))
	metricValidators.registerForBroker(broker, minCountHistogramValidator("request-size", 2))
	metricValidators.registerForBroker(broker, minValHistogramValidator("request-size", 1))
	metricValidators.registerForBroker(broker, minValHistogramValidator("request-latency-in-ms", minRequestLatencyInMs))

	// We send at least 1 batch
	metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("batch-size", 1))
	metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("batch-size", 1))
	if compressionEnabled {
		// We record compression ratios between [0.50,-10.00] (50-1000 with a histogram) for at least one "fake" record
		metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("compression-ratio", 1))
		metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("compression-ratio", 50))
		metricValidators.registerForGlobalAndTopic("test_1", maxValHistogramValidator("compression-ratio", 1000))
	} else {
		// We record compression ratios of 1.00 (100 with a histogram) for every TestBatchSize record
		metricValidators.registerForGlobalAndTopic("test_1", countHistogramValidator("compression-ratio", TestBatchSize))
		metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("compression-ratio", 100))
		metricValidators.registerForGlobalAndTopic("test_1", maxValHistogramValidator("compression-ratio", 100))
	}

	// We send exactly TestBatchSize messages
	metricValidators.registerForGlobalAndTopic("test_1", countMeterValidator("record-send-rate", TestBatchSize))
	// We send at least one record per request
	metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("records-per-request", 1))
	metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("records-per-request", 1))

	// We receive at least 1 byte from the broker
	metricValidators.registerForAllBrokers(broker, minCountMeterValidator("outgoing-byte-rate", 1))
	if noResponse {
		// in exactly 2 global responses (metadata + offset)
		metricValidators.register(countMeterValidator("response-rate", 2))
		metricValidators.register(minCountHistogramValidator("response-size", 2))
		// and exactly 1 offset response for the registered broker
		metricValidators.registerForBroker(broker, countMeterValidator("response-rate", 1))
		metricValidators.registerForBroker(broker, minCountHistogramValidator("response-size", 1))
		metricValidators.registerForBroker(broker, minValHistogramValidator("response-size", 1))
	} else {
		// in at least 3 global responses (metadata + offset + produces)
		metricValidators.register(minCountMeterValidator("response-rate", 3))
		metricValidators.register(minCountHistogramValidator("response-size", 3))
		// and at least 2 for the registered broker
		metricValidators.registerForBroker(broker, minCountMeterValidator("response-rate", 2))
		metricValidators.registerForBroker(broker, minCountHistogramValidator("response-size", 2))
		metricValidators.registerForBroker(broker, minValHistogramValidator("response-size", 1))
	}

	// There should be no requests in flight anymore
	metricValidators.registerForAllBrokers(broker, counterValidator("requests-in-flight", 0))

	// Run the validators
	metricValidators.run(t, client.Config().MetricRegistry)
}

// Benchmarks

func BenchmarkProducerSmall(b *testing.B) {
	benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 128)))
}
func BenchmarkProducerMedium(b *testing.B) {
	benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 1024)))
}
func BenchmarkProducerLarge(b *testing.B) {
	benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 8192)))
}
func BenchmarkProducerSmallSinglePartition(b *testing.B) {
	benchmarkProducer(b, nil, "test.1", ByteEncoder(make([]byte, 128)))
}
func BenchmarkProducerMediumSnappy(b *testing.B) {
	conf := NewConfig()
	conf.Producer.Compression = CompressionSnappy
	benchmarkProducer(b, conf, "test.1", ByteEncoder(make([]byte, 1024)))
}

func benchmarkProducer(b *testing.B, conf *Config, topic string, value Encoder) {
	setupFunctionalTest(b)
	defer teardownFunctionalTest(b)

	metricsDisable := os.Getenv("METRICS_DISABLE")
	if metricsDisable != "" {
		previousUseNilMetrics := metrics.UseNilMetrics
		Logger.Println("Disabling metrics using no-op implementation")
		metrics.UseNilMetrics = true
		// Restore previous setting
		defer func() {
			metrics.UseNilMetrics = previousUseNilMetrics
		}()
	}

	producer, err := NewAsyncProducer(kafkaBrokers, conf)
	if err != nil {
		b.Fatal(err)
	}

	b.ResetTimer()

	for i := 1; i <= b.N; {
		msg := &ProducerMessage{Topic: topic, Key: StringEncoder(fmt.Sprintf("%d", i)), Value: value}
		select {
		case producer.Input() <- msg:
			i++
		case ret := <-producer.Errors():
			b.Fatal(ret.Err)
		}
	}
	safeClose(b, producer)
}
<|endoftext|>


<commit_before>package rccsv

/*
	CSV reader for RingCentral Call Log CSV files

	SYNOPSIS

 	csv := rccsv.NewCallLogRecordsCsvReader()
	err := csv.ReadFile("/path/to/my.csv")
	if err != nil {
		fmt.Printf("ERR %v", err)
	}
	stats := csv.CallLogRecordsCsv.GetStatsForVoiceRecordings()

	// NOTE on stripping UTF-8 BOM
	https://github.com/golang/go/issues/9588
	https://groups.google.com/forum/#!topic/golang-nuts/OToNIPdfkks
	http://cautery.blogspot.com/2013/04/stripping-utf-8-byte-order-mark-with-go.html
*/

import (
	"errors"
	"io"
	"reflect"
	"regexp"
	"strconv"
	"time"

	"github.com/grokify/gotilla/encoding/csvutil"
	"github.com/grokify/gotilla/time/timeutil"
	"github.com/ttacon/libphonenumber"
)

type CallLogStats struct {
	Days           float64
	NumCalls       int64
	TotalSeconds   int64
	TotalMinutes   float32
	TotalHours     float32
	AverageMinutes float32
	DailyCalls     float32
	DailyMinutes   float32
	DailyHours     float32
	Dt14Start      int64
	Dt14End        int64
}

func NewCallLogStats(days int64) CallLogStats {
	s := CallLogStats{}
	s.Days = -1
	s.NumCalls = 0
	s.TotalSeconds = 0
	s.TotalMinutes = 0.0
	s.TotalHours = 0.0
	s.AverageMinutes = 0.0
	s.DailyCalls = 0.0
	s.DailyMinutes = 0.0
	s.DailyHours = 0.0
	s.Dt14Start = -1
	s.Dt14End = -1
	return s
}

func (s *CallLogStats) Inflate() {
	if s.NumCalls < 1 {
		return
	}
	if s.Dt14Start > 0 && s.Dt14End > 0 && s.Dt14Start < s.Dt14End {
		dtStart, err1 := timeutil.TimeForDt14(s.Dt14Start)
		if err1 == nil {
			dtEnd, err2 := timeutil.TimeForDt14(s.Dt14End)
			if err2 == nil {
				dur := dtEnd.Sub(dtStart)
				durHrs := dur.Hours()
				days := durHrs / 24
				s.Days = days
			}
		}
	}
	s.TotalMinutes = float32(s.TotalSeconds / 60)
	s.TotalHours = float32(s.TotalSeconds / 60 / 60)
	s.AverageMinutes = s.TotalMinutes / float32(s.NumCalls)
	if s.Days > 0 {
		s.DailyCalls = float32(s.NumCalls) / float32(s.Days)
		s.DailyMinutes = s.DailyCalls * s.AverageMinutes
		s.DailyHours = s.DailyMinutes / 60
	}
}

type CallLogRecordsCsv struct {
	CallLogRecords []CallLogRecordCsv
}

func (rs *CallLogRecordsCsv) GetStatsForVoiceRecordings() CallLogStats {
	stats := NewCallLogStats(int64(30))
	stats.NumCalls = 0
	for _, rec := range rs.CallLogRecords {
		rec.Inflate()
		if rec.Action != "Phone Call" && rec.Action != "VoIP Call" && rec.Action != "FindMe" {
			//continue
		}
		if rec.ActionResult != "Accepted" && rec.ActionResult != "Call connected" {
			continue
		}
		stats.NumCalls += 1
		stats.TotalSeconds += rec.DurationSeconds

		if len(rec.TimeRfc3339) > 0 {
			dt, err := time.Parse(time.RFC3339, rec.TimeRfc3339)
			if err == nil {
				dt14 := timeutil.Dt14ForTime(dt)
				if stats.Dt14Start < 0 || dt14 < stats.Dt14Start {
					stats.Dt14Start = dt14
				}
				if stats.Dt14End < 0 || dt14 > stats.Dt14Start {
					stats.Dt14End = dt14
				}
			}
		}
	}
	stats.Inflate()
	return stats
}

type CallLogRecordCsv struct {
	Type                  string
	PhoneNumber           string
	PhoneNumberE164       string
	PhoneNumberIso31661a2 string
	Name                  string
	Date                  string
	Time                  string
	TimeRfc3339           string
	Action                string
	ActionResult          string
	ResultDescription     string
	Duration              string
	DurationSeconds       int64
}

func (r *CallLogRecordCsv) SetCountry() {}

func (r *CallLogRecordCsv) Inflate() {
	r.PhoneNumberIso31661a2 = ""
	if 1 == 0 && len(r.PhoneNumber) > 0 {
		country := "US"
		num, err := libphonenumber.Parse(r.PhoneNumber, country)
		if err == nil {
			r.PhoneNumberE164 = libphonenumber.Format(num, libphonenumber.E164)
			r.PhoneNumberIso31661a2 = country
		}
	}
	rx1 := regexp.MustCompile(`^([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$`)
	rs1 := rx1.FindStringSubmatch(r.Duration)
	if len(rs1) > 0 {
		hr, err := strconv.ParseInt(rs1[1], 10, 64)
		if err != nil {
			hr = 0
		}
		mn, err := strconv.ParseInt(rs1[2], 10, 64)
		if err != nil {
			mn = 0
		}
		sc, err := strconv.ParseInt(rs1[3], 10, 64)
		if err != nil {
			sc = 0
		}
		sc = sc + (mn * 60) + (hr * 60 * 60)
		r.DurationSeconds = sc
	}
	if len(r.Date) > 0 && len(r.Time) > 0 {
		dateTimeRaw := r.Date + " " + r.Time
		layout := "Mon 01/02/2006 3:04 PM"
		myTime, err := time.Parse(layout, dateTimeRaw)
		if err == nil {
			r.TimeRfc3339 = myTime.Format(time.RFC3339)
		}
	}
}

func (r *CallLogRecordCsv) LoadRow(cols []string, vals []string) error {
	lc := len(cols)
	lv := len(vals)
	if lc != lv {
		return errors.New("RC CSV COL ROW LENGTH MISMATCH")
	}
	rx1 := regexp.MustCompile(`\s+`)
	for i := 0; i < lc; i++ {
		col := cols[i]
		val := vals[i]
		key := rx1.ReplaceAllString(col, "")
		reflect.ValueOf(r).Elem().FieldByName(key).Set(reflect.ValueOf(val))
	}
	return nil
}

type CallLogRecordsCsvReader struct {
	CallLogRecordsCsv CallLogRecordsCsv
}

func NewCallLogRecordsCsvReader() CallLogRecordsCsvReader {
	rd := CallLogRecordsCsvReader{}
	return rd
}

func (rd *CallLogRecordsCsvReader) ReadFile(path string) error {
	return rd.ReadFileBom(path, false)
}

func (rd *CallLogRecordsCsvReader) ReadFileBom(path string, stripBom bool) error {
	reader, file, err := csvutil.NewReader(path, ',', stripBom)
	if err != nil {
		return err
	}
	i := 0
	cols := []string{}
	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			file.Close()
			if stripBom == false && err.Error() == "line 1, column 1: bare \" in non-quoted-field" {
				return rd.ReadFileBom(path, true)
			}
			return err
		}
		if i == 0 {
			cols = record
			i += 1
			continue
		}
		obj := CallLogRecordCsv{}
		_ = obj.LoadRow(cols, record)
		obj.Inflate()
		rd.CallLogRecordsCsv.CallLogRecords = append(rd.CallLogRecordsCsv.CallLogRecords, obj)
		i += 1
	}
	file.Close()
	return nil
}
<commit_msg>minor formatting<commit_after>package rccsv

/*
	CSV reader for RingCentral Call Log CSV files

	SYNOPSIS

 	csv := rccsv.NewCallLogRecordsCsvReader()
	err := csv.ReadFile("/path/to/my.csv")
	if err != nil {
		fmt.Printf("ERR %v", err)
	}
	stats := csv.CallLogRecordsCsv.GetStatsForVoiceRecordings()

	// NOTE on stripping UTF-8 BOM
	https://github.com/golang/go/issues/9588
	https://groups.google.com/forum/#!topic/golang-nuts/OToNIPdfkks
	http://cautery.blogspot.com/2013/04/stripping-utf-8-byte-order-mark-with-go.html
*/

import (
	"errors"
	"io"
	"reflect"
	"regexp"
	"strconv"
	"time"

	"github.com/grokify/gotilla/encoding/csvutil"
	"github.com/grokify/gotilla/time/timeutil"
	"github.com/ttacon/libphonenumber"
)

type CallLogStats struct {
	Days           float64
	NumCalls       int64
	TotalSeconds   int64
	TotalMinutes   float32
	TotalHours     float32
	AverageMinutes float32
	DailyCalls     float32
	DailyMinutes   float32
	DailyHours     float32
	Dt14Start      int64
	Dt14End        int64
}

func NewCallLogStats(days int64) CallLogStats {
	s := CallLogStats{
		Days:           -1,
		NumCalls:       0,
		TotalSeconds:   0,
		TotalMinutes:   0.0,
		TotalHours:     0.0,
		AverageMinutes: 0.0,
		DailyCalls:     0.0,
		DailyMinutes:   0.0,
		DailyHours:     0.0,
		Dt14Start:      -1,
		Dt14End:        -1}
	return s
}

func (s *CallLogStats) Inflate() {
	if s.NumCalls < 1 {
		return
	}
	if s.Dt14Start > 0 && s.Dt14End > 0 && s.Dt14Start < s.Dt14End {
		dtStart, err1 := timeutil.TimeForDt14(s.Dt14Start)
		if err1 == nil {
			dtEnd, err2 := timeutil.TimeForDt14(s.Dt14End)
			if err2 == nil {
				dur := dtEnd.Sub(dtStart)
				durHrs := dur.Hours()
				days := durHrs / 24
				s.Days = days
			}
		}
	}
	s.TotalMinutes = float32(s.TotalSeconds / 60)
	s.TotalHours = float32(s.TotalSeconds / 60 / 60)
	s.AverageMinutes = s.TotalMinutes / float32(s.NumCalls)
	if s.Days > 0 {
		s.DailyCalls = float32(s.NumCalls) / float32(s.Days)
		s.DailyMinutes = s.DailyCalls * s.AverageMinutes
		s.DailyHours = s.DailyMinutes / 60
	}
}

type CallLogRecordsCsv struct {
	CallLogRecords []CallLogRecordCsv
}

func (rs *CallLogRecordsCsv) GetStatsForVoiceRecordings() CallLogStats {
	stats := NewCallLogStats(int64(30))
	stats.NumCalls = 0
	for _, rec := range rs.CallLogRecords {
		rec.Inflate()
		if rec.Action != "Phone Call" && rec.Action != "VoIP Call" && rec.Action != "FindMe" {
			//continue
		}
		if rec.ActionResult != "Accepted" && rec.ActionResult != "Call connected" {
			continue
		}
		stats.NumCalls += 1
		stats.TotalSeconds += rec.DurationSeconds

		if len(rec.TimeRfc3339) > 0 {
			dt, err := time.Parse(time.RFC3339, rec.TimeRfc3339)
			if err == nil {
				dt14 := timeutil.Dt14ForTime(dt)
				if stats.Dt14Start < 0 || dt14 < stats.Dt14Start {
					stats.Dt14Start = dt14
				}
				if stats.Dt14End < 0 || dt14 > stats.Dt14Start {
					stats.Dt14End = dt14
				}
			}
		}
	}
	stats.Inflate()
	return stats
}

type CallLogRecordCsv struct {
	Type                  string
	PhoneNumber           string
	PhoneNumberE164       string
	PhoneNumberIso31661a2 string
	Name                  string
	Date                  string
	Time                  string
	TimeRfc3339           string
	Action                string
	ActionResult          string
	ResultDescription     string
	Duration              string
	DurationSeconds       int64
}

func (r *CallLogRecordCsv) SetCountry() {}

func (r *CallLogRecordCsv) Inflate() {
	r.PhoneNumberIso31661a2 = ""
	if 1 == 0 && len(r.PhoneNumber) > 0 {
		country := "US"
		num, err := libphonenumber.Parse(r.PhoneNumber, country)
		if err == nil {
			r.PhoneNumberE164 = libphonenumber.Format(num, libphonenumber.E164)
			r.PhoneNumberIso31661a2 = country
		}
	}
	rx1 := regexp.MustCompile(`^([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$`)
	rs1 := rx1.FindStringSubmatch(r.Duration)
	if len(rs1) > 0 {
		hr, err := strconv.ParseInt(rs1[1], 10, 64)
		if err != nil {
			hr = 0
		}
		mn, err := strconv.ParseInt(rs1[2], 10, 64)
		if err != nil {
			mn = 0
		}
		sc, err := strconv.ParseInt(rs1[3], 10, 64)
		if err != nil {
			sc = 0
		}
		sc = sc + (mn * 60) + (hr * 60 * 60)
		r.DurationSeconds = sc
	}
	if len(r.Date) > 0 && len(r.Time) > 0 {
		dateTimeRaw := r.Date + " " + r.Time
		layout := "Mon 01/02/2006 3:04 PM"
		myTime, err := time.Parse(layout, dateTimeRaw)
		if err == nil {
			r.TimeRfc3339 = myTime.Format(time.RFC3339)
		}
	}
}

func (r *CallLogRecordCsv) LoadRow(cols []string, vals []string) error {
	lc := len(cols)
	lv := len(vals)
	if lc != lv {
		return errors.New("RC CSV COL ROW LENGTH MISMATCH")
	}
	rx1 := regexp.MustCompile(`\s+`)
	for i := 0; i < lc; i++ {
		col := cols[i]
		val := vals[i]
		key := rx1.ReplaceAllString(col, "")
		reflect.ValueOf(r).Elem().FieldByName(key).Set(reflect.ValueOf(val))
	}
	return nil
}

type CallLogRecordsCsvReader struct {
	CallLogRecordsCsv CallLogRecordsCsv
}

func NewCallLogRecordsCsvReader() CallLogRecordsCsvReader {
	rd := CallLogRecordsCsvReader{}
	return rd
}

func (rd *CallLogRecordsCsvReader) ReadFile(path string) error {
	return rd.ReadFileBom(path, false)
}

func (rd *CallLogRecordsCsvReader) ReadFileBom(path string, stripBom bool) error {
	reader, file, err := csvutil.NewReader(path, ',', stripBom)
	if err != nil {
		return err
	}
	i := 0
	cols := []string{}
	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			file.Close()
			if stripBom == false && err.Error() == "line 1, column 1: bare \" in non-quoted-field" {
				return rd.ReadFileBom(path, true)
			}
			return err
		}
		if i == 0 {
			cols = record
			i += 1
			continue
		}
		obj := CallLogRecordCsv{}
		_ = obj.LoadRow(cols, record)
		obj.Inflate()
		rd.CallLogRecordsCsv.CallLogRecords = append(rd.CallLogRecordsCsv.CallLogRecords, obj)
		i += 1
	}
	file.Close()
	return nil
}
<|endoftext|>


<commit_before>package procexp

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"sort"
	"testing"
	"time"

	"github.com/HouzuoGuo/laitos/platform"
)

func TestGetProcIDs(t *testing.T) {
	if platform.HostIsWindows() {
		t.Skip("this test will not run on windows")
		return
	}
	allProcIDs := GetProcIDs()
	var mustFindPIDs = []struct {
		pid int
	}{{1}, {os.Getpid()}}
	for _, findPID := range mustFindPIDs {
		t.Run(fmt.Sprintf("find PID %d", findPID.pid), func(t *testing.T) {
			if foundAt := sort.SearchInts(allProcIDs, findPID.pid); allProcIDs[foundAt] != findPID.pid {
				t.Fatal(findPID, allProcIDs)
			}
		})
	}
}

func TestGetProcTaskIDs(t *testing.T) {
	if platform.HostIsWindows() {
		t.Skip("this test will not run on windows")
		return
	}
	// The program's main thread (task ID == program PID) must show up
	allTaskIDs := GetProcTaskIDs(os.Getpid())
	if foundAt := sort.SearchInts(allTaskIDs, os.Getpid()); allTaskIDs[foundAt] != os.Getpid() {
		t.Fatal(allTaskIDs)
	}
	// Get this process' own task IDs
	allTaskIDs = GetProcTaskIDs(0)
	if foundAt := sort.SearchInts(allTaskIDs, os.Getpid()); allTaskIDs[foundAt] != os.Getpid() {
		t.Fatal(allTaskIDs)
	}
}

func TestGetTaskStatus_WaitChannel(t *testing.T) {
	if platform.HostIsWindows() {
		t.Skip("this test will not run on windows")
		return
	}
	cmd := exec.Command("/bin/cat")
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard
	if err := cmd.Start(); err != nil {
		t.Fatal(err)
	}
	defer func() {
		_ = cmd.Process.Kill()
		_, _ = cmd.Process.Wait()
	}()
	time.Sleep(1 * time.Second)
	// By now kernel should see that sleep command is sleeping nicely
	status := GetTaskStatus(cmd.Process.Pid, cmd.Process.Pid)
	// On the stack there shall be at very least syscall entry, architecture-dependent sleep function, and a common sleep function.
	if status.ID != cmd.Process.Pid || status.WaitChannelName == "" {
		t.Fatalf("%+v", status)
	}
	// The stack file can only be read by root, hence it's not part of this test yet.
}

func TestGetTaskStatus_SchedulerStats(t *testing.T) {
	if platform.HostIsWindows() {
		t.Skip("this test will not run on windows")
		return
	}
	cmd := exec.Command("/usr/bin/yes")
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard
	if err := cmd.Start(); err != nil {
		t.Fatal(err)
	}
	defer func() {
		_ = cmd.Process.Kill()
		_, _ = cmd.Process.Wait()
	}()
	// It takes couple of seconds for the process to use up some CPU time
	time.Sleep(8 * time.Second)
	status := GetTaskStatus(cmd.Process.Pid, cmd.Process.Pid)
	if status.SchedulerStats.NumVoluntarySwitches == 0 || status.SchedulerStats.NumInvoluntarySwitches == 0 || status.SchedulerStats.NumRunSec == 0 {
		t.Fatalf("%+v", status)
	}
}

func TestGetProcAndTaskStatus(t *testing.T) {
	if platform.HostIsWindows() {
		t.Skip("this test will not run on windows")
		return
	}
	cmd := exec.Command("/usr/bin/yes")
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard
	if err := cmd.Start(); err != nil {
		t.Fatal(err)
	}
	defer func() {
		_ = cmd.Process.Kill()
		_, _ = cmd.Process.Wait()
	}()
	// It takes couple of seconds for the process to use up some CPU time
	time.Sleep(8 * time.Second)
	status, err := GetProcAndTaskStatus(cmd.Process.Pid)
	if err != nil {
		t.Fatal(err)
	}
	if status.Stats.NumUserModeSecInclChildren+status.Stats.NumKernelModeSecInclChildren < 1 ||
		status.Stats.ResidentSetMemSizeBytes < 100 || status.Stats.VirtualMemSizeBytes < 100 ||
		status.Stats.State == "" || status.Status.Name != "yes" ||
		len(status.Tasks) < 1 {
		t.Fatalf("%+v", status)
	}
	mainTask := status.Tasks[cmd.Process.Pid]
	if mainTask.ID != cmd.Process.Pid || mainTask.SchedulerStats.NumRunSec < 1 || mainTask.SchedulerStats.NumWaitSec < 0.001 ||
		mainTask.SchedulerStats.NumVoluntarySwitches < 10 || mainTask.SchedulerStats.NumInvoluntarySwitches < 10 {
		t.Fatalf("%+v", status)
	}
	if status.SchedulerStatsSum.NumRunSec < 1 || status.SchedulerStatsSum.NumWaitSec < 0.001 ||
		status.SchedulerStatsSum.NumVoluntarySwitches < 10 || status.SchedulerStatsSum.NumInvoluntarySwitches < 10 {
		t.Fatalf("%+v", status)
	}
}
<commit_msg>procexp: skip tests that will not run on wsl<commit_after>package procexp

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"sort"
	"testing"
	"time"

	"github.com/HouzuoGuo/laitos/platform"
)

func TestGetProcIDs(t *testing.T) {
	if platform.HostIsWindows() {
		t.Skip("this test will not run on windows")
		return
	}
	allProcIDs := GetProcIDs()
	var mustFindPIDs = []struct {
		pid int
	}{{1}, {os.Getpid()}}
	for _, findPID := range mustFindPIDs {
		t.Run(fmt.Sprintf("find PID %d", findPID.pid), func(t *testing.T) {
			if foundAt := sort.SearchInts(allProcIDs, findPID.pid); allProcIDs[foundAt] != findPID.pid {
				t.Fatal(findPID, allProcIDs)
			}
		})
	}
}

func TestGetProcTaskIDs(t *testing.T) {
	if platform.HostIsWindows() {
		t.Skip("this test will not run on windows")
		return
	}
	// The program's main thread (task ID == program PID) must show up
	allTaskIDs := GetProcTaskIDs(os.Getpid())
	if foundAt := sort.SearchInts(allTaskIDs, os.Getpid()); allTaskIDs[foundAt] != os.Getpid() {
		t.Fatal(allTaskIDs)
	}
	// Get this process' own task IDs
	allTaskIDs = GetProcTaskIDs(0)
	if foundAt := sort.SearchInts(allTaskIDs, os.Getpid()); allTaskIDs[foundAt] != os.Getpid() {
		t.Fatal(allTaskIDs)
	}
}

func TestGetTaskStatus_WaitChannel(t *testing.T) {
	if platform.HostIsWindows() || platform.HostIsWSL() {
		t.Skip("this test will not run on windows")
		return
	}
	cmd := exec.Command("/bin/cat")
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard
	if err := cmd.Start(); err != nil {
		t.Fatal(err)
	}
	defer func() {
		_ = cmd.Process.Kill()
		_, _ = cmd.Process.Wait()
	}()
	time.Sleep(1 * time.Second)
	// By now kernel should see that sleep command is sleeping nicely
	status := GetTaskStatus(cmd.Process.Pid, cmd.Process.Pid)
	// On the stack there shall be at very least syscall entry, architecture-dependent sleep function, and a common sleep function.
	if status.ID != cmd.Process.Pid || status.WaitChannelName == "" {
		t.Fatalf("%+v", status)
	}
	// The stack file can only be read by root, hence it's not part of this test yet.
}

func TestGetTaskStatus_SchedulerStats(t *testing.T) {
	if platform.HostIsWindows() || platform.HostIsWSL() {
		t.Skip("this test will not run on windows")
		return
	}
	cmd := exec.Command("/usr/bin/yes")
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard
	if err := cmd.Start(); err != nil {
		t.Fatal(err)
	}
	defer func() {
		_ = cmd.Process.Kill()
		_, _ = cmd.Process.Wait()
	}()
	// It takes couple of seconds for the process to use up some CPU time
	time.Sleep(8 * time.Second)
	status := GetTaskStatus(cmd.Process.Pid, cmd.Process.Pid)
	if status.SchedulerStats.NumVoluntarySwitches == 0 || status.SchedulerStats.NumInvoluntarySwitches == 0 || status.SchedulerStats.NumRunSec == 0 {
		t.Fatalf("%+v", status)
	}
}

func TestGetProcAndTaskStatus(t *testing.T) {
	if platform.HostIsWindows() || platform.HostIsWSL() {
		t.Skip("this test will not run on windows")
		return
	}
	cmd := exec.Command("/usr/bin/yes")
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard
	if err := cmd.Start(); err != nil {
		t.Fatal(err)
	}
	defer func() {
		_ = cmd.Process.Kill()
		_, _ = cmd.Process.Wait()
	}()
	// It takes couple of seconds for the process to use up some CPU time
	time.Sleep(8 * time.Second)
	status, err := GetProcAndTaskStatus(cmd.Process.Pid)
	if err != nil {
		t.Fatal(err)
	}
	if status.Stats.NumUserModeSecInclChildren+status.Stats.NumKernelModeSecInclChildren < 1 ||
		status.Stats.ResidentSetMemSizeBytes < 100 || status.Stats.VirtualMemSizeBytes < 100 ||
		status.Stats.State == "" || status.Status.Name != "yes" ||
		len(status.Tasks) < 1 {
		t.Fatalf("%+v", status)
	}
	mainTask := status.Tasks[cmd.Process.Pid]
	if mainTask.ID != cmd.Process.Pid || mainTask.SchedulerStats.NumRunSec < 1 || mainTask.SchedulerStats.NumWaitSec < 0.001 ||
		mainTask.SchedulerStats.NumVoluntarySwitches < 10 || mainTask.SchedulerStats.NumInvoluntarySwitches < 10 {
		t.Fatalf("%+v", status)
	}
	if status.SchedulerStatsSum.NumRunSec < 1 || status.SchedulerStatsSum.NumWaitSec < 0.001 ||
		status.SchedulerStatsSum.NumVoluntarySwitches < 10 || status.SchedulerStatsSum.NumInvoluntarySwitches < 10 {
		t.Fatalf("%+v", status)
	}
}
<|endoftext|>


<commit_before>package saml2

import (
	"encoding/xml"
	"fmt"

	"github.com/beevik/etree"
)

//ErrMissingElement is the error type that indicates an element and/or attribute is
//missing. It provides a structured error that can be more appropriately acted
//upon.
type ErrMissingElement struct {
	Tag, Attribute string
}

//ErrMissingAssertion indicates that an appropriate assertion element could not
//be found in the SAML Response
var (
	ErrMissingAssertion = ErrMissingElement{Tag: AssertionTag}
)

func (e ErrMissingElement) Error() string {
	if e.Attribute != "" {
		return fmt.Sprintf("missing %s attribute on %s element", e.Attribute, e.Tag)
	}
	return fmt.Sprintf("missing %s element", e.Tag)
}

//RetrieveAssertionInfo takes an encoded response and returns the AssertionInfo
//contained, or an error message if an error has been encountered.
func (sp *SAMLServiceProvider) RetrieveAssertionInfo(encodedResponse string) (*AssertionInfo, error) {
	assertionInfo := &AssertionInfo{}

	el, err := sp.ValidateEncodedResponse(encodedResponse)
	if err != nil {
		return nil, fmt.Errorf("error validating response: %v", err)
	}

	assertionElement := el.FindElement(AssertionTag)
	if assertionElement == nil {
		return nil, ErrMissingAssertion
	}

	//Verify all conditions for the assertion
	conditionsStatement := assertionElement.FindElement(childPath(assertionElement.Space, ConditionsTag))
	if conditionsStatement == nil {
		return nil, ErrMissingElement{Tag: ConditionsTag}
	}

	warningInfo, err := sp.VerifyAssertionConditions(assertionElement, conditionsStatement)
	if err != nil {
		return nil, err
	}

	//Get the NameID
	subjectStatement := assertionElement.FindElement(childPath(assertionElement.Space, SubjectTag))
	if subjectStatement == nil {
		return nil, ErrMissingElement{Tag: SubjectTag}
	}

	nameIDStatement := subjectStatement.FindElement(childPath(assertionElement.Space, NameIdTag))
	if nameIDStatement == nil {
		return nil, ErrMissingElement{Tag: NameIdTag}
	}
	assertionInfo.NameID = nameIDStatement.Text()

	//Get the actual assertion attributes
	attributeStatement := assertionElement.FindElement(childPath(assertionElement.Space, AttributeStatementTag))
	if attributeStatement == nil && !sp.AllowMissingAttributes {
		return nil, ErrMissingElement{Tag: AttributeStatementTag}
	}

	if attributeStatement != nil {
		doc := etree.NewDocument()
		doc.SetRoot(attributeStatement)
		bs, err := doc.WriteToBytes()

		if err != nil {
			return nil, err
		}

		err = xml.Unmarshal(bs, &assertionInfo.Values)
		if err != nil {
			return nil, err
		}
	}
	assertionInfo.WarningInfo = warningInfo
	return assertionInfo, nil
}
<commit_msg>Expose typed signature verification errors<commit_after>package saml2

import (
	"encoding/xml"
	"fmt"

	"github.com/beevik/etree"
)

//ErrMissingElement is the error type that indicates an element and/or attribute is
//missing. It provides a structured error that can be more appropriately acted
//upon.
type ErrMissingElement struct {
	Tag, Attribute string
}

type ErrVerification struct {
	Cause error
}

func (e ErrVerification) Error() string {
	return fmt.Sprintf("error validating response: %s", e.Cause.Error())
}

//ErrMissingAssertion indicates that an appropriate assertion element could not
//be found in the SAML Response
var (
	ErrMissingAssertion = ErrMissingElement{Tag: AssertionTag}
)

func (e ErrMissingElement) Error() string {
	if e.Attribute != "" {
		return fmt.Sprintf("missing %s attribute on %s element", e.Attribute, e.Tag)
	}
	return fmt.Sprintf("missing %s element", e.Tag)
}

//RetrieveAssertionInfo takes an encoded response and returns the AssertionInfo
//contained, or an error message if an error has been encountered.
func (sp *SAMLServiceProvider) RetrieveAssertionInfo(encodedResponse string) (*AssertionInfo, error) {
	assertionInfo := &AssertionInfo{}

	el, err := sp.ValidateEncodedResponse(encodedResponse)
	if err != nil {
		return nil, ErrVerification{Cause: err}
	}

	assertionElement := el.FindElement(AssertionTag)
	if assertionElement == nil {
		return nil, ErrMissingAssertion
	}

	//Verify all conditions for the assertion
	conditionsStatement := assertionElement.FindElement(childPath(assertionElement.Space, ConditionsTag))
	if conditionsStatement == nil {
		return nil, ErrMissingElement{Tag: ConditionsTag}
	}

	warningInfo, err := sp.VerifyAssertionConditions(assertionElement, conditionsStatement)
	if err != nil {
		return nil, err
	}

	//Get the NameID
	subjectStatement := assertionElement.FindElement(childPath(assertionElement.Space, SubjectTag))
	if subjectStatement == nil {
		return nil, ErrMissingElement{Tag: SubjectTag}
	}

	nameIDStatement := subjectStatement.FindElement(childPath(assertionElement.Space, NameIdTag))
	if nameIDStatement == nil {
		return nil, ErrMissingElement{Tag: NameIdTag}
	}
	assertionInfo.NameID = nameIDStatement.Text()

	//Get the actual assertion attributes
	attributeStatement := assertionElement.FindElement(childPath(assertionElement.Space, AttributeStatementTag))
	if attributeStatement == nil && !sp.AllowMissingAttributes {
		return nil, ErrMissingElement{Tag: AttributeStatementTag}
	}

	if attributeStatement != nil {
		doc := etree.NewDocument()
		doc.SetRoot(attributeStatement)
		bs, err := doc.WriteToBytes()

		if err != nil {
			return nil, err
		}

		err = xml.Unmarshal(bs, &assertionInfo.Values)
		if err != nil {
			return nil, err
		}
	}
	assertionInfo.WarningInfo = warningInfo
	return assertionInfo, nil
}
<|endoftext|>


<commit_before>package robots

import (
	"fmt"

	"github.com/trinchan/slackbot/robots"
)

type bot struct{}

func init() {
	p := &bot{}
	robots.RegisterRobot("raffl", p)
}

func (pb bot) Run(p *robots.Payload) (slashCommandImmediateReturn string) {
	go pb.DeferredAction(p)
	return "raffl this!"
}

func (pb bot) DeferredAction(p *robots.Payload) {
	response := &robots.IncomingWebhook{
		Domain:      p.TeamDomain,
		Channel:     p.ChannelID,
		Username:    "r	affl",
		Text:        fmt.Sprintf("@%s Raffl!", p.UserName),
		IconEmoji:   ":gift:",
		UnfurlLinks: true,
		Parse:       robots.ParseStyleFull,
	}
	response.Send()
}

func (pb bot) Description() (description string) {
	return "Raffl bot!\n\tUsage: /raffl\n\tExpected Response: @user: Raffl this!"
}
<commit_msg>Change immediate response<commit_after>package robots

import (
	"fmt"

	"github.com/trinchan/slackbot/robots"
)

type bot struct{}

func init() {
	p := &bot{}
	robots.RegisterRobot("raffl", p)
}

func (pb bot) Run(p *robots.Payload) (slashCommandImmediateReturn string) {
	go pb.DeferredAction(p)
	//return "raffl this!"
	return "raffl"
}

func (pb bot) DeferredAction(p *robots.Payload) {
	response := &robots.IncomingWebhook{
		Domain:      p.TeamDomain,
		Channel:     p.ChannelID,
		Username:    "raffl",
		Text:        fmt.Sprintf("@%s Raffl!", p.UserName),
		IconEmoji:   ":gift:",
		UnfurlLinks: true,
		Parse:       robots.ParseStyleFull,
	}
	response.Send()
}

func (pb bot) Description() (description string) {
	return "Raffl bot!\n\tUsage: /raffl\n\tExpected Response: @user: Raffl this!"
}
<|endoftext|>