<commit_before>package main

import (
	"github.com/go-sql-driver/mysql"
	"database/sql/driver"
	"fmt"
	"time"
	"sync"
	"runtime"
	"flag"
//	"database/sql"
//  _ "github.com/go-sql-driver/mysql"
//  _ "net/http/pprof"
//  "net/http"
	"math/big"
	"math"
)

type TestResult struct {
	connOk bool
	queryOk bool
	connTime time.Duration
	queryTime time.Duration
}

type SummeryResult struct {
	count                 int64
	connFailCount         int64
	queryFailCount        int64

	totalConnTime         time.Duration
	totalQueryTime        time.Duration
	totalSquareConnTime   big.Int
	totalSquareQueryTime  big.Int

	avgConnTime           time.Duration
	avgQueryTime          time.Duration
	stddevConnTime        time.Duration
	stddevQueryTime       time.Duration
}

func testOnce(dsn, query string) TestResult {
	result := TestResult{}
	beforeConn := time.Now()
	db, err := (mysql.MySQLDriver{}).Open(dsn)
	if err != nil {
		//fmt.Println(err.Error())
		result.connOk = false
		return result
	}
	result.connOk = true
	afterConn := time.Now()
	result.connTime = afterConn.Sub(beforeConn)
	defer db.Close()

	stmt, err := db.Prepare(query)
	if err != nil {
		result.queryOk = false
		return result
	}
	defer stmt.Close()

	rows, err := stmt.Query([]driver.Value{})
	if err != nil {
		//fmt.Println(err.Error())
		result.queryOk = false
		return result
	}
	result.queryOk = true
	defer rows.Close()
	afterQuery := time.Now()
	result.queryTime = afterQuery.Sub(afterConn)
	return result
}

func testRoutine(dsn, query string, n int, outChan chan<- TestResult) {
	for i := 0; i < n; i++ {
		outChan <- testOnce(dsn, query)
	}
}

func summeryRoutine(inChan <-chan TestResult) SummeryResult {
	var ret     SummeryResult
	var bigA, big2 big.Int
	var bigR1, bigR2, big1N, big1N1 big.Rat
	big2.SetInt64(2)
	for result := range inChan {
		ret.count++
		if result.connOk {
			ret.totalConnTime+= result.connTime
			bigA.SetInt64((int64)(result.connTime)).Mul(&bigA, &bigA)
			ret.totalSquareConnTime.Add(&ret.totalSquareConnTime, &bigA)
		} else {
			ret.connFailCount++
		}
		if result.queryOk {
			ret.totalQueryTime+= result.queryTime
			bigA.SetInt64((int64)(result.queryTime)).Mul(&bigA, &bigA)
			ret.totalSquareQueryTime.Add(&ret.totalSquareQueryTime, &bigA)
		} else {
			ret.queryFailCount++
		}
	}
	//  ∑(i-miu)2 = ∑(i2)-(∑i)2/n
	n := ret.count - ret.connFailCount
	if n > 1 {
		ret.avgConnTime = (time.Duration)((int64)(ret.totalConnTime) / n)

		big1N.SetInt64(n).Inv(&big1N) // 1/n
		big1N1.SetInt64(n-1).Inv(&big1N1) // 1/(n-1)
		bigA.SetInt64((int64)(ret.totalConnTime)).Mul(&bigA, &bigA) // (∑i)2
		bigR1.SetInt(&bigA).Mul(&bigR1, &big1N) // (∑i)2/n
		bigR2.SetInt(&ret.totalSquareConnTime).Sub(&bigR2, &bigR1)
		s2, _ := bigR2.Mul(&bigR2, &big1N1).Float64()
		ret.stddevConnTime = (time.Duration)((int64)(math.Sqrt(s2)))
	}
	n = ret.count - ret.queryFailCount
	if n > 1 {
		ret.avgQueryTime = (time.Duration)((int64)(ret.totalQueryTime) / n)

		big1N.SetInt64(n).Inv(&big1N) // 1/n
		big1N1.SetInt64(n-1).Inv(&big1N1) // 1/(n-1)
		bigA.SetInt64((int64)(ret.totalQueryTime)).Mul(&bigA, &bigA) // (∑i)2
		bigR1.SetInt(&bigA).Mul(&bigR1, &big1N) // (∑i)2/n
		bigR2.SetInt(&ret.totalSquareQueryTime).Sub(&bigR2, &bigR1)
		s2, _ := bigR2.Mul(&bigR2, &big1N1).Float64()
		ret.stddevQueryTime = (time.Duration)((int64)(math.Sqrt(s2)))
	}
	return ret
}

type NullLogger struct{}
func (*NullLogger) Print(v ...interface{}) {
}

// mysqlburst -c 2000 -r 30 -d 'mha:M616VoUJBnYFi0L02Y24@tcp(10.200.180.54:3342)/x?timeout=5s&readTimeout=3s&writeTimeout=3s'
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	//go func() {
	//      http.ListenAndServe("localhost:6060", nil)
	//}()

	procs := 0
	rounds := 0
	dsn := ""
	query := ""

	flag.IntVar(&procs, "c", 1000, "concurrency")
	flag.IntVar(&rounds, "r", 100, "rounds")
	flag.StringVar(&dsn, "d", "mysql:@tcp(127.0.0.1:3306)/mysql?timeout=5s&readTimeout=5s&writeTimeout=5s", "dsn")
	flag.StringVar(&query, "q", "select 1", "sql")
	flag.Parse()

	mysql.SetLogger(&NullLogger{})

	wg := sync.WaitGroup{}
	wg.Add(procs)
	resultChan := make(chan TestResult, 5000)
	testBegin := time.Now()
	go func() {
		for i := 0; i < procs; i++ {
			go func() {
				testRoutine(dsn, query, rounds, resultChan)
				wg.Done()
			}()
		}
		wg.Wait()
		close(resultChan)
	}()
	summery := summeryRoutine(resultChan)
	testEnd := time.Now()
	fmt.Printf("total tests: %d\n", summery.count);
	fmt.Printf("failed connections: %d\n", summery.connFailCount);
	fmt.Printf("failed queries: %d\n", summery.queryFailCount);
	fmt.Printf("test time: %s\n", testEnd.Sub(testBegin).String())

	fmt.Printf("avg conn time: %s\n", summery.avgConnTime.String())
	fmt.Printf("stddev conn time: %s\n", summery.stddevConnTime.String())

	fmt.Printf("avg query time: %s\n", summery.avgQueryTime.String())
	fmt.Printf("stddev query time: %s\n", summery.stddevQueryTime.String())
}

<commit_msg>add min/max<commit_after>package main

import (
	"github.com/go-sql-driver/mysql"
	"database/sql/driver"
	"fmt"
	"time"
	"sync"
	"runtime"
	"flag"
//	"database/sql"
//  _ "github.com/go-sql-driver/mysql"
//  _ "net/http/pprof"
//  "net/http"
	"math/big"
	"math"
)

type TestResult struct {
	connOk bool
	queryOk bool
	connTime time.Duration
	queryTime time.Duration
}

type SummeryResult struct {
	count                 int64
	connFailCount         int64
	queryFailCount        int64

	totalConnTime         time.Duration
	totalQueryTime        time.Duration
	totalSquareConnTime   big.Int
	totalSquareQueryTime  big.Int

	maxConnTime           time.Duration
	minConnTime           time.Duration
	avgConnTime           time.Duration
	stddevConnTime        time.Duration

	maxQueryTime          time.Duration
	minQueryTime          time.Duration
	avgQueryTime          time.Duration
	stddevQueryTime       time.Duration
}

func testOnce(dsn, query string) TestResult {
	result := TestResult{}
	beforeConn := time.Now()
	db, err := (mysql.MySQLDriver{}).Open(dsn)
	if err != nil {
		//fmt.Println(err.Error())
		result.connOk = false
		return result
	}
	result.connOk = true
	afterConn := time.Now()
	result.connTime = afterConn.Sub(beforeConn)
	defer db.Close()

	stmt, err := db.Prepare(query)
	if err != nil {
		result.queryOk = false
		return result
	}
	defer stmt.Close()

	rows, err := stmt.Query([]driver.Value{})
	if err != nil {
		//fmt.Println(err.Error())
		result.queryOk = false
		return result
	}
	result.queryOk = true
	defer rows.Close()
	afterQuery := time.Now()
	result.queryTime = afterQuery.Sub(afterConn)
	return result
}

func testRoutine(dsn, query string, n int, outChan chan<- TestResult) {
	for i := 0; i < n; i++ {
		outChan <- testOnce(dsn, query)
	}
}

func summeryRoutine(inChan <-chan TestResult) SummeryResult {
	var ret   SummeryResult
	var bigA  big.Int
	ret.minConnTime = math.MaxInt64
	ret.minQueryTime = math.MaxInt64
	for result := range inChan {
		ret.count++
		if result.connOk {
			if result.connTime > ret.maxConnTime {
				ret.maxConnTime = result.connTime
			}
			if result.connTime < ret.minConnTime {
				ret.minConnTime = result.connTime
			}
			ret.totalConnTime+= result.connTime
			bigA.SetInt64((int64)(result.connTime)).Mul(&bigA, &bigA)
			ret.totalSquareConnTime.Add(&ret.totalSquareConnTime, &bigA)
		} else {
			ret.connFailCount++
		}
		if result.queryOk {
			if result.queryTime > ret.maxQueryTime {
				ret.maxQueryTime = result.queryTime
			}
			if result.queryTime < ret.minQueryTime {
				ret.minQueryTime = result.queryTime
			}
			ret.totalQueryTime+= result.queryTime
			bigA.SetInt64((int64)(result.queryTime)).Mul(&bigA, &bigA)
			ret.totalSquareQueryTime.Add(&ret.totalSquareQueryTime, &bigA)
		} else {
			ret.queryFailCount++
		}
	}
	ret.Summery()
	return ret
}

func (self *SummeryResult) Summery() {
	var bigA, big2 big.Int
	var bigR1, bigR2, big1N, big1N1 big.Rat
	big2.SetInt64(2)
	//  ∑(i-miu)2 = ∑(i2)-(∑i)2/n
	n := self.count - self.connFailCount
	if n > 1 {
		self.avgConnTime = (time.Duration)((int64)(self.totalConnTime) / n)

		big1N.SetInt64(n).Inv(&big1N) // 1/n
		big1N1.SetInt64(n-1).Inv(&big1N1) // 1/(n-1)
		bigA.SetInt64((int64)(self.totalConnTime)).Mul(&bigA, &bigA) // (∑i)2
		bigR1.SetInt(&bigA).Mul(&bigR1, &big1N) // (∑i)2/n
		bigR2.SetInt(&self.totalSquareConnTime).Sub(&bigR2, &bigR1)
		s2, _ := bigR2.Mul(&bigR2, &big1N1).Float64()
		self.stddevConnTime = (time.Duration)((int64)(math.Sqrt(s2)))
	}

	n = self.count - self.queryFailCount
	if n > 1 {
		self.avgQueryTime = (time.Duration)((int64)(self.totalQueryTime) / n)

		big1N.SetInt64(n).Inv(&big1N) // 1/n
		big1N1.SetInt64(n-1).Inv(&big1N1) // 1/(n-1)
		bigA.SetInt64((int64)(self.totalQueryTime)).Mul(&bigA, &bigA) // (∑i)2
		bigR1.SetInt(&bigA).Mul(&bigR1, &big1N) // (∑i)2/n
		bigR2.SetInt(&self.totalSquareQueryTime).Sub(&bigR2, &bigR1)
		s2, _ := bigR2.Mul(&bigR2, &big1N1).Float64()
		self.stddevQueryTime = (time.Duration)((int64)(math.Sqrt(s2)))
	}

	if self.minConnTime == math.MaxInt64 {
		self.minConnTime = 0
	}
	if self.minQueryTime == math.MaxInt64 {
		self.minQueryTime = 0
	}
}

type NullLogger struct{}
func (*NullLogger) Print(v ...interface{}) {
}

// mysqlburst -c 2000 -r 30 -d 'mha:M616VoUJBnYFi0L02Y24@tcp(10.200.180.54:3342)/x?timeout=5s&readTimeout=3s&writeTimeout=3s'
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	//go func() {
	//      http.ListenAndServe("localhost:6060", nil)
	//}()

	procs := 0
	rounds := 0
	dsn := ""
	query := ""

	flag.IntVar(&procs, "c", 1000, "concurrency")
	flag.IntVar(&rounds, "r", 100, "rounds")
	flag.StringVar(&dsn, "d", "mysql:@tcp(127.0.0.1:3306)/mysql?timeout=5s&readTimeout=5s&writeTimeout=5s", "dsn")
	flag.StringVar(&query, "q", "select 1", "sql")
	flag.Parse()

	mysql.SetLogger(&NullLogger{})

	wg := sync.WaitGroup{}
	wg.Add(procs)
	resultChan := make(chan TestResult, 5000)
	testBegin := time.Now()
	go func() {
		for i := 0; i < procs; i++ {
			go func() {
				testRoutine(dsn, query, rounds, resultChan)
				wg.Done()
			}()
		}
		wg.Wait()
		close(resultChan)
	}()
	summery := summeryRoutine(resultChan)
	testEnd := time.Now()
	fmt.Printf("total tests: %d\n", summery.count);
	fmt.Printf("failed connections: %d\n", summery.connFailCount);
	fmt.Printf("failed queries: %d\n", summery.queryFailCount);
	fmt.Printf("test time: %s\n", testEnd.Sub(testBegin).String())

	fmt.Println("connect time")
	fmt.Printf("avg: %s\n", summery.avgConnTime.String())
	fmt.Printf("min: %s\n", summery.minConnTime.String())
	fmt.Printf("max: %s\n", summery.maxConnTime.String())
	fmt.Printf("stddev: %s\n", summery.stddevConnTime.String())

	fmt.Println("query time")
	fmt.Printf("avg: %s\n", summery.avgQueryTime.String())
	fmt.Printf("min: %s\n", summery.minQueryTime.String())
	fmt.Printf("max: %s\n", summery.minQueryTime.String())
	fmt.Printf("stddev: %s\n", summery.stddevQueryTime.String())
}

<|endoftext|>


<commit_before>package cli

import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

//SubCommand defines the options for a sub-command to be executed by a SubCommander.
type SubCommand interface {
	//Name returns the name of the SubCommand.
	//Names must not overlap with other SubCommand Name()s or Aliases() in the
	//same SubCommander.
	Name() string

	//Aliases returns the aliases of the SubCommand.
	//These are listed in command help output and will cause a SubCommand to execute
	//just as if the SubCommand's name used from the command line.
	//Aliases must not overlap with other SubCommand Name()s or Aliases() in the
	//same SubCommander.
	Aliases() []string

	//Synopsis returns a synopsis of the SubCommand.
	//This should be less than one line and is used for command help output.
	Synopsis() string

	//Description returns a longer description of the SubCommand.
	//This can be however long you like and is used for help specific to this
	//SubCommand.
	Description() string

	//Parameters returns the parameters of the SubCommand.
	//The order returned is important and will be enforced in parsing.
	Parameters() []*Parameter

	//ParameterUsage returns the usage text for the SubCommand.
	//This should not include the SubCommand's name or aliases.
	//An empty return value means that help output will be formatted from the
	//result of Parameters(). A non-empty value will be used directly for help output.
	ParameterUsage(params []*Parameter, format func(p *Parameter) string) string

	//SetParameter allows the SubCommand to receive parameter values when parsing.
	SetParameter(index int, value string) error

	//ParameterFlagMode returns the ParameterFlagMode the SubCommand will use
	//when parsing.
	ParameterFlagMode() ParameterFlagMode

	//ValidateParameters should ensure that the set parameters are valid.
	//This will be called after successful sub-command argument parsing to
	//allow SubCommands to ensure parameters were set properly.
	//I.e. to ensure required parameters are actually set.
	ValidateParameters() error

	//FlagSetter for SubCommand flags.
	FlagSetter

	//Execute is where the SubCommand should do its work.
	//A non-nil return value indicates the execution failed and that error will
	//be process by a SubCommander.
	Execute(ctx context.Context, out, outErr io.Writer) error
}

//SubCommander provides registering multiple SubCommands and executing them from
//command line arguments.
//
//Note that SubCommander is NOT safe for use with multiple goroutines.
type SubCommander struct {
	GlobalFlags FlagSetter

	AllowGlobalFlagsWithSubCommand bool

	names   map[string]SubCommand
	aliases map[string]SubCommand
}

//Register registers subCommand to be possibly executed later via its Name() or
//Aliases().
//This will overwrite any previously registered SubCommands with the same Name()s
//or Aliases().
func (sc *SubCommander) Register(subCommand SubCommand) {
	if sc.names == nil {
		sc.names = map[string]SubCommand{}
	}
	if sc.aliases == nil {
		sc.aliases = map[string]SubCommand{}
	}

	sc.names[subCommand.Name()] = subCommand
	for _, alias := range subCommand.Aliases() {
		sc.aliases[alias] = subCommand
	}
}

func (sc *SubCommander) Execute(args []string) error {
	return sc.ExecuteContext(context.Background(), args)
}

func (sc *SubCommander) ExecuteContext(ctx context.Context, args []string) error {
	return sc.ExecuteContextOut(ctx, args, os.Stdout, os.Stderr)
}

func (sc *SubCommander) ExecuteContextOut(ctx context.Context, args []string, out, outErr io.Writer) error {
	err := sc.executeContextOut(ctx, args, out, outErr)

	//do processing on err here.

	return err
}

func (sc *SubCommander) executeContextOut(ctx context.Context, args []string, out, outErr io.Writer) error {
	f := newFlagSet("")

	if sc.GlobalFlags != nil {
		sc.GlobalFlags.SetFlags(f)
	}
	if err := f.Parse(args); err != nil {
		return ErrParsingGlobalFlags(err)
	}

	args = f.Args()
	if len(args) == 0 {
		return ErrUnsuppliedSubCommand
	}
	name := args[0]
	args = args[1:]

	subCommand := sc.getSubCommand(name)
	if subCommand == nil {
		return ErrUnknownSubCommand(name)
	}

	return sc.executeSubCommand(ctx, f, subCommand, args, out, outErr)
}

func (sc *SubCommander) getSubCommand(name string) SubCommand {
	if subCommand, ok := sc.names[name]; ok {
		return subCommand
	}
	if subCommand, ok := sc.aliases[name]; ok {
		return subCommand
	}
	return nil
}

func (sc *SubCommander) executeSubCommand(
	ctx context.Context,
	gfs *flag.FlagSet,
	subCommand SubCommand,
	args []string,
	out, outErr io.Writer,
) (err error) {
	defer func() {
		if err != nil {
			return
		}
		if r := recover(); r != nil {
			err = ErrParsingSubCommand(fmt.Errorf("%v", r))
		}
	}()

	scf := newFlagSet(subCommand.Name())
	subCommand.SetFlags(scf)
	if sc.AllowGlobalFlagsWithSubCommand {
		subCommand.SetFlags(gfs)
	}

	err = parseSubCommandFlags(scf, subCommand, args)
	if err != nil {
		err = ErrParsingSubCommand(err)
		return
	}

	err = subCommand.Execute(ctx, out, outErr)
	if err != nil {
		err = ErrExecutingSubCommand(err)
	}

	return
}

func parseSubCommandFlags(f *flag.FlagSet, subCommand SubCommand, args []string) error {
	var err error = nil

	index := 0
	for err == nil && len(args) > 0 {
		err = f.Parse(args)
		args = f.Args()
		if err == nil && len(args) > 0 {
			err = subCommand.SetParameter(index, args[0])
			index++
			args = args[1:]
		}
	}
	if err == nil {
		err = subCommand.ValidateParameters()
	}

	return err
}

func newFlagSet(name string) *flag.FlagSet {
	f := flag.NewFlagSet(name, flag.ContinueOnError)
	f.Usage = func() {}
	f.SetOutput(ioutil.Discard)
	return f
}
<commit_msg>Update global and sub-command flag.FlagSet usage<commit_after>package cli

import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

//SubCommand defines the options for a sub-command to be executed by a SubCommander.
type SubCommand interface {
	//Name returns the name of the SubCommand.
	//Names must not overlap with other SubCommand Name()s or Aliases() in the
	//same SubCommander.
	Name() string

	//Aliases returns the aliases of the SubCommand.
	//These are listed in command help output and will cause a SubCommand to execute
	//just as if the SubCommand's name used from the command line.
	//Aliases must not overlap with other SubCommand Name()s or Aliases() in the
	//same SubCommander.
	Aliases() []string

	//Synopsis returns a synopsis of the SubCommand.
	//This should be less than one line and is used for command help output.
	Synopsis() string

	//Description returns a longer description of the SubCommand.
	//This can be however long you like and is used for help specific to this
	//SubCommand.
	Description() string

	//Parameters returns the parameters of the SubCommand.
	//The order returned is important and will be enforced in parsing.
	Parameters() []*Parameter

	//ParameterUsage returns the usage text for the SubCommand.
	//This should not include the SubCommand's name or aliases.
	//An empty return value means that help output will be formatted from the
	//result of Parameters(). A non-empty value will be used directly for help output.
	ParameterUsage(params []*Parameter, format func(p *Parameter) string) string

	//SetParameter allows the SubCommand to receive parameter values when parsing.
	SetParameter(index int, value string) error

	//ParameterFlagMode returns the ParameterFlagMode the SubCommand will use
	//when parsing.
	ParameterFlagMode() ParameterFlagMode

	//ValidateParameters should ensure that the set parameters are valid.
	//This will be called after successful sub-command argument parsing to
	//allow SubCommands to ensure parameters were set properly.
	//I.e. to ensure required parameters are actually set.
	ValidateParameters() error

	//FlagSetter for SubCommand flags.
	FlagSetter

	//Execute is where the SubCommand should do its work.
	//A non-nil return value indicates the execution failed and that error will
	//be process by a SubCommander.
	Execute(ctx context.Context, out, outErr io.Writer) error
}

//SubCommander provides registering multiple SubCommands and executing them from
//command line arguments.
//
//Note that SubCommander is NOT safe for use with multiple goroutines.
type SubCommander struct {
	GlobalFlags FlagSetter

	AllowGlobalFlagsWithSubCommand bool

	names   map[string]SubCommand
	aliases map[string]SubCommand
}

//Register registers subCommand to be possibly executed later via its Name() or
//Aliases().
//This will overwrite any previously registered SubCommands with the same Name()s
//or Aliases().
func (sc *SubCommander) Register(subCommand SubCommand) {
	if sc.names == nil {
		sc.names = map[string]SubCommand{}
	}
	if sc.aliases == nil {
		sc.aliases = map[string]SubCommand{}
	}

	sc.names[subCommand.Name()] = subCommand
	for _, alias := range subCommand.Aliases() {
		sc.aliases[alias] = subCommand
	}
}

//Execute is syntactic sugar for ExecuteContext() with context.Background().
func (sc *SubCommander) Execute(args []string) error {
	return sc.ExecuteContext(context.Background(), args)
}

//ExecuteContext is syntactic sugar for ExecuteContextOut() with os.Stdout and os.Stderr.
func (sc *SubCommander) ExecuteContext(ctx context.Context, args []string) error {
	return sc.ExecuteContextOut(ctx, args, os.Stdout, os.Stderr)
}

func (sc *SubCommander) ExecuteContextOut(ctx context.Context, args []string, out, outErr io.Writer) error {
	err := sc.executeContextOut(ctx, args, out, outErr)

	//do processing on err here.

	return err
}

func (sc *SubCommander) executeContextOut(ctx context.Context, args []string, out, outErr io.Writer) error {
	f := newFlagSet("")

	if sc.GlobalFlags != nil {
		sc.GlobalFlags.SetFlags(f)
	}
	if err := f.Parse(args); err != nil {
		return ErrParsingGlobalFlags(err)
	}

	args = f.Args()
	if len(args) == 0 {
		return ErrUnsuppliedSubCommand
	}
	name := args[0]
	args = args[1:]

	subCommand := sc.getSubCommand(name)
	if subCommand == nil {
		return ErrUnknownSubCommand(name)
	}

	return sc.executeSubCommand(ctx, f, subCommand, args, out, outErr)
}

func (sc *SubCommander) getSubCommand(name string) SubCommand {
	if subCommand, ok := sc.names[name]; ok {
		return subCommand
	}
	if subCommand, ok := sc.aliases[name]; ok {
		return subCommand
	}
	return nil
}

func (sc *SubCommander) executeSubCommand(
	ctx context.Context,
	gfs *flag.FlagSet,
	subCommand SubCommand,
	args []string,
	out, outErr io.Writer,
) (err error) {
	defer func() {
		if err != nil {
			return
		}
		if r := recover(); r != nil {
			err = ErrParsingSubCommand(fmt.Errorf("%v", r))
		}
	}()

	scf := newFlagSet(subCommand.Name())
	subCommand.SetFlags(scf)
	if sc.AllowGlobalFlagsWithSubCommand && sc.GlobalFlags != nil {
		sc.GlobalFlags.SetFlags(scf)
	}

	err = parseSubCommandFlags(scf, subCommand, args)
	if err != nil {
		err = ErrParsingSubCommand(err)
		return
	}

	err = subCommand.Execute(ctx, out, outErr)
	if err != nil {
		err = ErrExecutingSubCommand(err)
	}

	return
}

func parseSubCommandFlags(f *flag.FlagSet, subCommand SubCommand, args []string) error {
	var err error = nil

	index := 0
	for err == nil && len(args) > 0 {
		err = f.Parse(args)
		args = f.Args()
		if err == nil && len(args) > 0 {
			err = subCommand.SetParameter(index, args[0])
			index++
			args = args[1:]
		}
	}
	if err == nil {
		err = subCommand.ValidateParameters()
	}

	return err
}

func newFlagSet(name string) *flag.FlagSet {
	f := flag.NewFlagSet(name, flag.ContinueOnError)
	f.Usage = func() {}
	f.SetOutput(ioutil.Discard)
	return f
}
<|endoftext|>


<commit_before>package system

import (
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
)

// parseCPUInfo returns a newline-delimited string slice of '/proc/cpuinfo'.
func parseCPUInfo() []string {
	cached, _ := ioutil.ReadFile("/proc/cpuinfo")
	return strings.Split(string(cached), "\n")
}

// parseCPUCount returns the number of CPU cores in the system.
func parseCPUCount(cpuinfo []string) int {
	cores, _ := strconv.Atoi(strings.Fields(cpuinfo[len(cpuinfo) - 18])[3])
	return cores + 1
}

// Model returns the CPU Model
func CPUModel() string {
	modelinfo := strings.Fields(parseCPUInfo()[4])[3:]
	return modelinfo[0] + " " + modelinfo[1]
}

// CPUTemp sets the temperature of the CPU.
func CPUTemp(cputemp *int, done chan bool) {
	*cputemp = getTemperature()
	done <- true
}

// Frequencies sets '*cpufreq' with a string containing all core frequencies.
func CPUFrequencies(cpufreqs *string, done chan bool) {
	*cpufreqs = "Cores:" + getFrequencyString()
	done <- true
}
<commit_msg>Remove fmt since it's not used<commit_after>package system

import (
	"io/ioutil"
	"strconv"
	"strings"
)

// parseCPUInfo returns a newline-delimited string slice of '/proc/cpuinfo'.
func parseCPUInfo() []string {
	cached, _ := ioutil.ReadFile("/proc/cpuinfo")
	return strings.Split(string(cached), "\n")
}

// parseCPUCount returns the number of CPU cores in the system.
func parseCPUCount(cpuinfo []string) int {
	cores, _ := strconv.Atoi(strings.Fields(cpuinfo[len(cpuinfo) - 18])[3])
	return cores + 1
}

// Model returns the CPU Model
func CPUModel() string {
	modelinfo := strings.Fields(parseCPUInfo()[4])[3:]
	return modelinfo[0] + " " + modelinfo[1]
}

// CPUTemp sets the temperature of the CPU.
func CPUTemp(cputemp *int, done chan bool) {
	*cputemp = getTemperature()
	done <- true
}

// Frequencies sets '*cpufreq' with a string containing all core frequencies.
func CPUFrequencies(cpufreqs *string, done chan bool) {
	*cpufreqs = "Cores:" + getFrequencyString()
	done <- true
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import "time"

// FindProcess looks for a running process by its pid.
// The Process it returns can be used to obtain information
// about the underlying operating system process.
func FindProcess(pid int) (p *Process, err error) {
	return findProcess(pid)
}

// StartProcess starts a new process with the program, arguments and attributes
// specified by name, argv and attr.
//
// StartProcess is a low-level interface. The os/exec package provides
// higher-level interfaces.
//
// If there is an error, it will be of type *PathError.
func StartProcess(name string, argv []string, attr *ProcAttr) (*Process, error) {
	return startProcess(name, argv, attr)
}

// Release releases any resources associated with the Process p,
// rendering it unusable in the future.
// Release only needs to be called if Wait is not.
func (p *Process) Release() error {
	return p.release()
}

// Kill causes the Process to exit immediately.
func (p *Process) Kill() error {
	return p.kill()
}

// Wait waits for the Process to exit, and then returns a
// ProcessState describing its status and an error, if any.
// Wait releases any resources associated with the Process.
func (p *Process) Wait() (*ProcessState, error) {
	return p.wait()
}

// Signal sends a signal to the Process.
func (p *Process) Signal(sig Signal) error {
	return p.signal(sig)
}

// UserTime returns the user CPU time of the exited process and its children.
func (p *ProcessState) UserTime() time.Duration {
	return p.userTime()
}

// SystemTime returns the system CPU time of the exited process and its children.
func (p *ProcessState) SystemTime() time.Duration {
	return p.systemTime()
}

// Exited reports whether the program has exited.
func (p *ProcessState) Exited() bool {
	return p.exited()
}

// Success reports whether the program exited successfully,
// such as with exit status 0 on Unix.
func (p *ProcessState) Success() bool {
	return p.success()
}

// Sys returns system-dependent exit information about
// the process.  Convert it to the appropriate underlying
// type, such as syscall.WaitStatus on Unix, to access its contents.
func (p *ProcessState) Sys() interface{} {
	return p.sys()
}

// SysUsage returns system-dependent resource usage information about
// the exited process.  Convert it to the appropriate underlying
// type, such as *syscall.Rusage on Unix, to access its contents.
// (On Unix, *syscall.Rusage matches struct rusage as defined in the
// getrusage(2) manual page.)
func (p *ProcessState) SysUsage() interface{} {
	return p.sysUsage()
}

// Hostname returns the host name reported by the kernel.
func Hostname() (name string, err error) {
	return hostname()
}

// Readdir reads the contents of the directory associated with file and
// returns a slice of up to n FileInfo values, as would be returned
// by Lstat, in directory order. Subsequent calls on the same file will yield
// further FileInfos.
//
// If n > 0, Readdir returns at most n FileInfo structures. In this case, if
// Readdir returns an empty slice, it will return a non-nil error
// explaining why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdir returns all the FileInfo from the directory in
// a single slice. In this case, if Readdir succeeds (reads all
// the way to the end of the directory), it returns the slice and a
// nil error. If it encounters an error before the end of the
// directory, Readdir returns the FileInfo read until that point
// and a non-nil error.
func (f *File) Readdir(n int) (fi []FileInfo, err error) {
	if f == nil {
		return nil, ErrInvalid
	}
	return f.readdir(n)
}

// Readdirnames reads and returns a slice of names from the directory f.
//
// If n > 0, Readdirnames returns at most n names. In this case, if
// Readdirnames returns an empty slice, it will return a non-nil error
// explaining why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdirnames returns all the names from the directory in
// a single slice. In this case, if Readdirnames succeeds (reads all
// the way to the end of the directory), it returns the slice and a
// nil error. If it encounters an error before the end of the
// directory, Readdirnames returns the names read until that point and
// a non-nil error.
func (f *File) Readdirnames(n int) (names []string, err error) {
	if f == nil {
		return nil, ErrInvalid
	}
	return f.readdirnames(n)
}
<commit_msg>os: document that Process.Wait only works on child processes<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import "time"

// FindProcess looks for a running process by its pid.
// The Process it returns can be used to obtain information
// about the underlying operating system process.
func FindProcess(pid int) (p *Process, err error) {
	return findProcess(pid)
}

// StartProcess starts a new process with the program, arguments and attributes
// specified by name, argv and attr.
//
// StartProcess is a low-level interface. The os/exec package provides
// higher-level interfaces.
//
// If there is an error, it will be of type *PathError.
func StartProcess(name string, argv []string, attr *ProcAttr) (*Process, error) {
	return startProcess(name, argv, attr)
}

// Release releases any resources associated with the Process p,
// rendering it unusable in the future.
// Release only needs to be called if Wait is not.
func (p *Process) Release() error {
	return p.release()
}

// Kill causes the Process to exit immediately.
func (p *Process) Kill() error {
	return p.kill()
}

// Wait waits for the Process to exit, and then returns a
// ProcessState describing its status and an error, if any.
// Wait releases any resources associated with the Process.
// On most operating systems, the Process must be a child
// of the current process or an error will be returned.
func (p *Process) Wait() (*ProcessState, error) {
	return p.wait()
}

// Signal sends a signal to the Process.
func (p *Process) Signal(sig Signal) error {
	return p.signal(sig)
}

// UserTime returns the user CPU time of the exited process and its children.
func (p *ProcessState) UserTime() time.Duration {
	return p.userTime()
}

// SystemTime returns the system CPU time of the exited process and its children.
func (p *ProcessState) SystemTime() time.Duration {
	return p.systemTime()
}

// Exited reports whether the program has exited.
func (p *ProcessState) Exited() bool {
	return p.exited()
}

// Success reports whether the program exited successfully,
// such as with exit status 0 on Unix.
func (p *ProcessState) Success() bool {
	return p.success()
}

// Sys returns system-dependent exit information about
// the process.  Convert it to the appropriate underlying
// type, such as syscall.WaitStatus on Unix, to access its contents.
func (p *ProcessState) Sys() interface{} {
	return p.sys()
}

// SysUsage returns system-dependent resource usage information about
// the exited process.  Convert it to the appropriate underlying
// type, such as *syscall.Rusage on Unix, to access its contents.
// (On Unix, *syscall.Rusage matches struct rusage as defined in the
// getrusage(2) manual page.)
func (p *ProcessState) SysUsage() interface{} {
	return p.sysUsage()
}

// Hostname returns the host name reported by the kernel.
func Hostname() (name string, err error) {
	return hostname()
}

// Readdir reads the contents of the directory associated with file and
// returns a slice of up to n FileInfo values, as would be returned
// by Lstat, in directory order. Subsequent calls on the same file will yield
// further FileInfos.
//
// If n > 0, Readdir returns at most n FileInfo structures. In this case, if
// Readdir returns an empty slice, it will return a non-nil error
// explaining why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdir returns all the FileInfo from the directory in
// a single slice. In this case, if Readdir succeeds (reads all
// the way to the end of the directory), it returns the slice and a
// nil error. If it encounters an error before the end of the
// directory, Readdir returns the FileInfo read until that point
// and a non-nil error.
func (f *File) Readdir(n int) (fi []FileInfo, err error) {
	if f == nil {
		return nil, ErrInvalid
	}
	return f.readdir(n)
}

// Readdirnames reads and returns a slice of names from the directory f.
//
// If n > 0, Readdirnames returns at most n names. In this case, if
// Readdirnames returns an empty slice, it will return a non-nil error
// explaining why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdirnames returns all the names from the directory in
// a single slice. In this case, if Readdirnames succeeds (reads all
// the way to the end of the directory), it returns the slice and a
// nil error. If it encounters an error before the end of the
// directory, Readdirnames returns the names read until that point and
// a non-nil error.
func (f *File) Readdirnames(n int) (names []string, err error) {
	if f == nil {
		return nil, ErrInvalid
	}
	return f.readdirnames(n)
}
<|endoftext|>


<commit_before>package main

import (
	"net"
	"fmt"
	"golang.org/x/net/ipv4"
	"errors"
	//"time"
)

const tcp_buff_sz = 10

type TCP_Client_Manager struct {
	tcp_reader *IP_Reader
	readBuffer map[uint16](map[uint16](map[string](chan []byte))) // dst, src port, ip
}

func New_TCP_Client_Manager() (*TCP_Client_Manager, error) {
	nr, err := NewNetwork_Reader() // TODO: create a global var for the network reader
	if err != nil {
		return nil, err
	}

	ipr, err := nr.NewIP_Reader("*", TCP_PROTO)
	if err != nil {
		return nil, err
	}

	cm := &TCP_Client_Manager{
		tcp_reader: ipr,
		readBuffer: make(map[uint16](map[uint16](map[string](chan []byte)))),
	}

	go cm.readAll()

	return cm, nil
}

func (cm *TCP_Client_Manager) bind(srcport, dstport uint16, ip string) (chan []byte, error) {
	if _, ok := cm.readBuffer[dstport]; !ok {
		cm.readBuffer[dstport] = make(map[uint16](map[string](chan []byte)))
	}

	if _, ok := cm.readBuffer[dstport][srcport]; !ok {
		cm.readBuffer[dstport][srcport] = make(map[string](chan []byte))
	}

	if _, ok := cm.readBuffer[dstport][srcport][ip]; ok {
		return nil, errors.New("Ports and IP already binded to")
	}

	cm.readBuffer[dstport][srcport][ip] = make(chan []byte, tcp_buff_sz)
	return cm.readBuffer[dstport][srcport][ip], nil
}

func (cm *TCP_Client_Manager) readAll() {
	for {
		ip, _, payload, err := cm.tcp_reader.ReadFrom()
		if err != nil {
			fmt.Println("TCP readAll error", err)
			continue
		}
		dstport := uint16(payload[0]) << 8 | uint16(payload[1]) // reversed to account
		srcport := uint16(payload[2]) << 8 | uint16(payload[3]) // for server sending

		if _, ok := cm.readBuffer[dstport]; ok {
			if _, ok := cm.readBuffer[dstport][srcport]; ok {
				if _, ok := cm.readBuffer[dstport][srcport][ip]; ok {
					cm.readBuffer[dstport][srcport][ip] <- payload
					continue
				} else if _, ok := cm.readBuffer[dstport][srcport]["*"]; ok {
					cm.readBuffer[dstport][srcport]["*"] <- payload
					continue
				}
			}
		}
		//fmt.Println(errors.New("Dst/Src port + ip not binded to"))
	}
}

type TCP_Client struct {
	manager   *TCP_Client_Manager
	read      chan []byte
	writer    *ipv4.RawConn
	ipAddress string // destination ip address
	srcIP     string // src ip address
	src, dst  uint16 // ports
	seqNum, ackNum    uint32 // sequence number
	status    uint
}

const (
	LISTEN = 1
	SYN_SENT
	SYN_RCVD
	ESTABLISHED
	FIN_WAIT_1
	FIN_WAIT_2
	CLOSE_WAIT
	CLOSING
	LAST_ACK
	TIME_WAIT
	CLOSED
)

func (x *TCP_Client_Manager) New_TCP_Client(src, dst uint16, dstIP string) (*TCP_Client, error) {
	/*write, err := NewIP_Writer(dstIP, TCP_PROTO)
	if err != nil {
		return nil, err
	}*/

	read, err := x.bind(src, dst, dstIP)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	p, err := net.ListenPacket(fmt.Sprintf("ip4:%d", TCP_PROTO), dstIP)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	r, err := ipv4.NewRawConn(p)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	return &TCP_Client{
		src: src,
		dst: dst,
		ipAddress: dstIP,
		srcIP: "127.0.0.1", // TODO: don't hardcode the srcIP
		manager: x,
		read: read,
		writer: r,
		seqNum: uint32(3425), // Can be any number between 0 and 2^32 inclusive, TODO this needs to be changed later, it can't be predictable.
		ackNum: uint32(0),    // Always 0 at start
	}, nil
}

const (
	TCP_FIN = 0x01
	TCP_SYN = 0x02
	TCP_RSH = 0x04
	TCP_PSH = 0x08
	TCP_ACK = 0x10
	TCP_URG = 0x20
	TCP_ECE = 0x40
	TCP_CWR = 0x80
)

func (c *TCP_Client) Connect() error {
	// SYN
	window := uint16(43690)       // TODO calc using http://ithitman.blogspot.com/2013/02/understanding-tcp-window-window-scaling.html
	headerLenSYN := 40
	SYN := []byte{
		(byte)(c.src >> 8), (byte)(c.src), // Source port in byte
		(byte)(c.dst >> 8), (byte)(c.dst), // Destination port in byte slice
		(byte)(c.seqNum >> 24), (byte)(c.seqNum >> 16), (byte)(c.seqNum >> 8), (byte)(c.seqNum),
		(byte)(c.ackNum >> 24), (byte)(c.ackNum >> 16), (byte)(c.ackNum >> 8), (byte)(c.ackNum),
		(byte)(
			(headerLenSYN/4) << 4, // Size of header in 32 bit chunks. It is always 5 unless options are used. This is also the data offset.
			// bits 5-7 inclusive are reserved, always 0
			// bit 8 is flag 0(NS flag), set to 0 here because only SYN
		),
		(byte)(// Flags 1-8 inclusive
			// CWR
			// ECE
			// URG
			// ACK
			// PSH
			// RST
			0 | TCP_SYN, //SYN
			// FIN
		),
		(byte)(window >> 8), (byte)(window),
		0, 0, // checksum
		0, 0, // URG pointer, only matters where URG flag is set.
		0x02, 0x04, 0xff, 0xd7, 0x04, 0x02, 0x08, 0x0a, 0x02, 0x64, 0x80, 0x8b, 0x0, 0x0, 0x0, 0x0, 0x01, 0x03, 0x03, 0x07,
	}

	checksumSYN := checksum(append(append(append(SYN, net.ParseIP(/*c.writer.src*/c.srcIP)...), net.ParseIP(/*c.writer.dst*/c.ipAddress)...), []byte{byte(TCP_PROTO >> 8), byte(TCP_PROTO), byte(headerLenSYN >> 8), byte(headerLenSYN)}...))
	SYN[16] = byte(checksumSYN >> 8)
	SYN[17] = byte(checksumSYN)

	//c.writer.WriteTo(SYN)
	err := c.writer.WriteTo(&ipv4.Header{
		Version:  ipv4.Version, // protocol version
		Len:      20, // header length
		TOS:      0, // type-of-service (0 is everything normal)
		TotalLen: len(SYN) + 20, // packet total length (octets)
		ID:       0, // identification
		Flags:    ipv4.DontFragment, // flags
		FragOff:  0, // fragment offset
		TTL:      64, // time-to-live (maximum lifespan in seconds)
		Protocol: TCP_PROTO, // next protocol
		Checksum: 0, // checksum (apparently autocomputed)
		Dst: net.ParseIP(c.ipAddress), // destination address
		//Options                         // options, extension headers
	}, SYN, nil)
	fmt.Println("Sent SYN")
	if err != nil {
		fmt.Println("Raw conn send", err)
	}

	// TODO: resend syn if needed

	// SYN-ACK
	fmt.Println("Waiting for syn-ack")
	synack := <- c.read
	fmt.Println("Syn-Ack Rcvd:", synack)

	// ACK
	headerLenACK := 20
	// TODO: verify the syn-ack flags (seq, ack, options-window scale, etc), and checksum
	c.seqNum++ // A+1
	B := uint32(synack[4]) << 24 | uint32(synack[5]) << 16 | uint32(synack[6]) << 8 | uint32(synack[7])
	c.ackNum = B+1

	ACK := []byte{
		(byte)(c.src >> 8), (byte)(c.src), // Source port in byte
		(byte)(c.dst >> 8), (byte)(c.dst), // Destination port in byte slice
		(byte)(c.seqNum >> 24), (byte)(c.seqNum >> 16), (byte)(c.seqNum >> 8), (byte)(c.seqNum),
		(byte)(c.ackNum >> 24), (byte)(c.ackNum >> 16), (byte)(c.ackNum >> 8), (byte)(c.ackNum),
		byte((headerLenACK/4) << 4), // data offset
		byte(TCP_ACK), // flags
		(byte)(window >> 8), (byte)(window), // window size
		0, 0, // checksum
		0, 0, // urg
	}

	checksumACK := checksum(append(append(append(ACK, net.ParseIP(c.srcIP)...), net.ParseIP(c.ipAddress)...), []byte{byte(TCP_PROTO >> 8), byte(TCP_PROTO), byte(headerLenACK >> 8), byte(headerLenACK)}...))
	ACK[16] = byte(checksumACK >> 8)
	ACK[17] = byte(checksumACK)

	err = c.writer.WriteTo(&ipv4.Header{
		Version:  ipv4.Version, // protocol version
		Len:      20, // header length
		TOS:      0, // type-of-service (0 is everything normal)
		TotalLen: len(ACK) + 20, // packet total length (octets)
		ID:       0, // identification
		Flags:    ipv4.DontFragment, // flags
		FragOff:  0, // fragment offset
		TTL:      64, // time-to-live (maximum lifespan in seconds)
		Protocol: TCP_PROTO, // next protocol
		Checksum: 0, // checksum (apparently autocomputed)
		Dst: net.ParseIP(c.ipAddress), // destination address
		//Options                         // options, extension headers
	}, ACK, nil)
	fmt.Println("Sent ACK data")
	if err != nil {
		fmt.Println("Raw conn send", err)
	}

	return nil
}

func (c *TCP_Client) Send(data []byte) error {
	return nil // TODO: implement TCP_Client send
}

func (c *TCP_Client) Close() error {
	return nil // TODO: free manager read buffer
}
<commit_msg>Name changes to the structs. Uses the standard names<commit_after>package main

import (
	"net"
	"fmt"
	"golang.org/x/net/ipv4"
	"errors"
	//"time"
)

const tcp_buff_sz = 10

type TCP_Main struct {
	tcp_reader *IP_Reader
	readBuffer map[uint16](map[uint16](map[string](chan []byte))) // dst, src port, ip
}

func New_TCP_Main() (*TCP_Main, error) {
	nr, err := NewNetwork_Reader() // TODO: create a global var for the network reader
	if err != nil {
		return nil, err
	}

	ipr, err := nr.NewIP_Reader("*", TCP_PROTO)
	if err != nil {
		return nil, err
	}

	cm := &TCP_Main{
		tcp_reader: ipr,
		readBuffer: make(map[uint16](map[uint16](map[string](chan []byte)))),
	}

	go cm.readAll()

	return cm, nil
}

func (cm *TCP_Main) bind(srcport, dstport uint16, ip string) (chan []byte, error) {
	if _, ok := cm.readBuffer[dstport]; !ok {
		cm.readBuffer[dstport] = make(map[uint16](map[string](chan []byte)))
	}

	if _, ok := cm.readBuffer[dstport][srcport]; !ok {
		cm.readBuffer[dstport][srcport] = make(map[string](chan []byte))
	}

	if _, ok := cm.readBuffer[dstport][srcport][ip]; ok {
		return nil, errors.New("Ports and IP already binded to")
	}

	cm.readBuffer[dstport][srcport][ip] = make(chan []byte, tcp_buff_sz)
	return cm.readBuffer[dstport][srcport][ip], nil
}

func (cm *TCP_Main) readAll() {
	for {
		ip, _, payload, err := cm.tcp_reader.ReadFrom()
		if err != nil {
			fmt.Println("TCP readAll error", err)
			continue
		}
		dstport := uint16(payload[0]) << 8 | uint16(payload[1]) // reversed to account
		srcport := uint16(payload[2]) << 8 | uint16(payload[3]) // for server sending

		if _, ok := cm.readBuffer[dstport]; ok {
			if _, ok := cm.readBuffer[dstport][srcport]; ok {
				if _, ok := cm.readBuffer[dstport][srcport][ip]; ok {
					cm.readBuffer[dstport][srcport][ip] <- payload
					continue
				} else if _, ok := cm.readBuffer[dstport][srcport]["*"]; ok {
					cm.readBuffer[dstport][srcport]["*"] <- payload
					continue
				}
			}
		}
		//fmt.Println(errors.New("Dst/Src port + ip not binded to"))
	}
}

type TCB struct {
	manager   *TCP_Main
	read      chan []byte
	writer    *ipv4.RawConn
	ipAddress string // destination ip address
	srcIP     string // src ip address
	src, dst  uint16 // ports
	seqNum, ackNum    uint32 // sequence number
	status    uint
}

const (
	LISTEN = 1
	SYN_SENT
	SYN_RCVD
	ESTABLISHED
	FIN_WAIT_1
	FIN_WAIT_2
	CLOSE_WAIT
	CLOSING
	LAST_ACK
	TIME_WAIT
	CLOSED
)

func (x *TCP_Main) New_TCB(src, dst uint16, dstIP string) (*TCB, error) {
	/*write, err := NewIP_Writer(dstIP, TCP_PROTO)
	if err != nil {
		return nil, err
	}*/

	read, err := x.bind(src, dst, dstIP)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	p, err := net.ListenPacket(fmt.Sprintf("ip4:%d", TCP_PROTO), dstIP)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	r, err := ipv4.NewRawConn(p)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	return &TCB{
		src: src,
		dst: dst,
		ipAddress: dstIP,
		srcIP: "127.0.0.1", // TODO: don't hardcode the srcIP
		manager: x,
		read: read,
		writer: r,
		seqNum: uint32(3425), // Can be any number between 0 and 2^32 inclusive, TODO this needs to be changed later, it can't be predictable.
		ackNum: uint32(0),    // Always 0 at start
	}, nil
}

const (
	TCP_FIN = 0x01
	TCP_SYN = 0x02
	TCP_RSH = 0x04
	TCP_PSH = 0x08
	TCP_ACK = 0x10
	TCP_URG = 0x20
	TCP_ECE = 0x40
	TCP_CWR = 0x80
)

func (c *TCB) Connect() error {
	// SYN
	window := uint16(43690)       // TODO calc using http://ithitman.blogspot.com/2013/02/understanding-tcp-window-window-scaling.html
	headerLenSYN := 40
	SYN := []byte{
		(byte)(c.src >> 8), (byte)(c.src), // Source port in byte
		(byte)(c.dst >> 8), (byte)(c.dst), // Destination port in byte slice
		(byte)(c.seqNum >> 24), (byte)(c.seqNum >> 16), (byte)(c.seqNum >> 8), (byte)(c.seqNum),
		(byte)(c.ackNum >> 24), (byte)(c.ackNum >> 16), (byte)(c.ackNum >> 8), (byte)(c.ackNum),
		(byte)(
			(headerLenSYN/4) << 4, // Size of header in 32 bit chunks. It is always 5 unless options are used. This is also the data offset.
			// bits 5-7 inclusive are reserved, always 0
			// bit 8 is flag 0(NS flag), set to 0 here because only SYN
		),
		(byte)(// Flags 1-8 inclusive
			// CWR
			// ECE
			// URG
			// ACK
			// PSH
			// RST
			0 | TCP_SYN, //SYN
			// FIN
		),
		(byte)(window >> 8), (byte)(window),
		0, 0, // checksum
		0, 0, // URG pointer, only matters where URG flag is set.
		0x02, 0x04, 0xff, 0xd7, 0x04, 0x02, 0x08, 0x0a, 0x02, 0x64, 0x80, 0x8b, 0x0, 0x0, 0x0, 0x0, 0x01, 0x03, 0x03, 0x07,
	}

	checksumSYN := checksum(append(append(append(SYN, net.ParseIP(/*c.writer.src*/c.srcIP)...), net.ParseIP(/*c.writer.dst*/c.ipAddress)...), []byte{byte(TCP_PROTO >> 8), byte(TCP_PROTO), byte(headerLenSYN >> 8), byte(headerLenSYN)}...))
	SYN[16] = byte(checksumSYN >> 8)
	SYN[17] = byte(checksumSYN)

	//c.writer.WriteTo(SYN)
	err := c.writer.WriteTo(&ipv4.Header{
		Version:  ipv4.Version, // protocol version
		Len:      20, // header length
		TOS:      0, // type-of-service (0 is everything normal)
		TotalLen: len(SYN) + 20, // packet total length (octets)
		ID:       0, // identification
		Flags:    ipv4.DontFragment, // flags
		FragOff:  0, // fragment offset
		TTL:      64, // time-to-live (maximum lifespan in seconds)
		Protocol: TCP_PROTO, // next protocol
		Checksum: 0, // checksum (apparently autocomputed)
		Dst: net.ParseIP(c.ipAddress), // destination address
		//Options                         // options, extension headers
	}, SYN, nil)
	fmt.Println("Sent SYN")
	if err != nil {
		fmt.Println("Raw conn send", err)
	}

	// TODO: resend syn if needed

	// SYN-ACK
	fmt.Println("Waiting for syn-ack")
	synack := <- c.read
	fmt.Println("Syn-Ack Rcvd:", synack)

	// ACK
	headerLenACK := 20
	// TODO: verify the syn-ack flags (seq, ack, options-window scale, etc), and checksum
	c.seqNum++ // A+1
	B := uint32(synack[4]) << 24 | uint32(synack[5]) << 16 | uint32(synack[6]) << 8 | uint32(synack[7])
	c.ackNum = B+1

	ACK := []byte{
		(byte)(c.src >> 8), (byte)(c.src), // Source port in byte
		(byte)(c.dst >> 8), (byte)(c.dst), // Destination port in byte slice
		(byte)(c.seqNum >> 24), (byte)(c.seqNum >> 16), (byte)(c.seqNum >> 8), (byte)(c.seqNum),
		(byte)(c.ackNum >> 24), (byte)(c.ackNum >> 16), (byte)(c.ackNum >> 8), (byte)(c.ackNum),
		byte((headerLenACK/4) << 4), // data offset
		byte(TCP_ACK), // flags
		(byte)(window >> 8), (byte)(window), // window size
		0, 0, // checksum
		0, 0, // urg
	}

	checksumACK := checksum(append(append(append(ACK, net.ParseIP(c.srcIP)...), net.ParseIP(c.ipAddress)...), []byte{byte(TCP_PROTO >> 8), byte(TCP_PROTO), byte(headerLenACK >> 8), byte(headerLenACK)}...))
	ACK[16] = byte(checksumACK >> 8)
	ACK[17] = byte(checksumACK)

	err = c.writer.WriteTo(&ipv4.Header{
		Version:  ipv4.Version, // protocol version
		Len:      20, // header length
		TOS:      0, // type-of-service (0 is everything normal)
		TotalLen: len(ACK) + 20, // packet total length (octets)
		ID:       0, // identification
		Flags:    ipv4.DontFragment, // flags
		FragOff:  0, // fragment offset
		TTL:      64, // time-to-live (maximum lifespan in seconds)
		Protocol: TCP_PROTO, // next protocol
		Checksum: 0, // checksum (apparently autocomputed)
		Dst: net.ParseIP(c.ipAddress), // destination address
		//Options                         // options, extension headers
	}, ACK, nil)
	fmt.Println("Sent ACK data")
	if err != nil {
		fmt.Println("Raw conn send", err)
	}

	return nil
}

func (c *TCB) Send(data []byte) error {
	return nil // TODO: implement TCP_Client send
}

func (c *TCB) Close() error {
	return nil // TODO: free manager read buffer
}
<|endoftext|>


<commit_before>package engine

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/drone/drone/model"
	"github.com/drone/drone/shared/docker"
	"github.com/drone/drone/shared/envconfig"
	"github.com/drone/drone/store"
	"github.com/samalba/dockerclient"
	"golang.org/x/net/context"
)

type Engine interface {
	Schedule(context.Context, *Task)
	Cancel(int64, int64, *model.Node) error
	Stream(int64, int64, *model.Node) (io.ReadCloser, error)
	Deallocate(*model.Node)
	Allocate(*model.Node) error
	Subscribe(chan *Event)
	Unsubscribe(chan *Event)
}

var (
	// options to fetch the stdout and stderr logs
	logOpts = &dockerclient.LogOptions{
		Stdout: true,
		Stderr: true,
	}

	// options to fetch the stdout and stderr logs
	// by tailing the output.
	logOptsTail = &dockerclient.LogOptions{
		Follow: true,
		Stdout: true,
		Stderr: true,
	}

	// error when the system cannot find logs
	errLogging = errors.New("Logs not available")
)

type engine struct {
	bus     *eventbus
	updater *updater
	pool    *pool
	envs    []string
}

// Load creates a new build engine, loaded with registered nodes from the
// database. The registered nodes are added to the pool of nodes to immediately
// start accepting workloads.
func Load(env envconfig.Env, s store.Store) Engine {
	engine := &engine{}
	engine.bus = newEventbus()
	engine.pool = newPool()
	engine.updater = &updater{engine.bus}

	// quick fix to propogate HTTP_PROXY variables
	// throughout the build environment.
	var proxyVars = []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy"}
	for _, proxyVar := range proxyVars {
		proxyVal := env.Get(proxyVar)
		if len(proxyVal) != 0 {
			engine.envs = append(engine.envs, proxyVar+"="+proxyVal)
		}
	}

	nodes, err := s.Nodes().GetList()
	if err != nil {
		log.Fatalf("failed to get nodes from database. %s", err)
	}
	for _, node := range nodes {
		engine.pool.allocate(node)
		log.Infof("registered docker daemon %s", node.Addr)
	}

	return engine
}

// Cancel cancels the job running on the specified Node.
func (e *engine) Cancel(build, job int64, node *model.Node) error {
	client, err := newDockerClient(node.Addr, node.Cert, node.Key, node.CA)
	if err != nil {
		return err
	}

	id := fmt.Sprintf("drone_build_%d_job_%d", build, job)
	return client.StopContainer(id, 30)
}

// Stream streams the job output from the specified Node.
func (e *engine) Stream(build, job int64, node *model.Node) (io.ReadCloser, error) {
	client, err := newDockerClient(node.Addr, node.Cert, node.Key, node.CA)
	if err != nil {
		log.Errorf("cannot create Docker client for node %s", node.Addr)
		return nil, err
	}

	id := fmt.Sprintf("drone_build_%d_job_%d", build, job)
	log.Debugf("streaming container logs %s", id)
	return client.ContainerLogs(id, logOptsTail)
}

// Subscribe subscribes the channel to all build events.
func (e *engine) Subscribe(c chan *Event) {
	e.bus.subscribe(c)
}

// Unsubscribe unsubscribes the channel from all build events.
func (e *engine) Unsubscribe(c chan *Event) {
	e.bus.unsubscribe(c)
}

func (e *engine) Allocate(node *model.Node) error {

	// run the full build!
	client, err := newDockerClient(node.Addr, node.Cert, node.Key, node.CA)
	if err != nil {
		log.Errorf("error creating docker client %s. %s.", node.Addr, err)
		return err
	}
	version, err := client.Version()
	if err != nil {
		log.Errorf("error connecting to docker daemon %s. %s.", node.Addr, err)
		return err
	}

	log.Infof("registered docker daemon %s running version %s", node.Addr, version.Version)
	e.pool.allocate(node)
	return nil
}

func (e *engine) Deallocate(n *model.Node) {
	nodes := e.pool.list()
	for _, node := range nodes {
		if node.ID == n.ID {
			log.Infof("un-registered docker daemon %s", node.Addr)
			e.pool.deallocate(node)
			break
		}
	}
}

func (e *engine) Schedule(c context.Context, req *Task) {
	node := <-e.pool.reserve()

	// since we are probably running in a go-routine
	// make sure we recover from any panics so that
	// a bug doesn't crash the whole system.
	defer func() {
		if err := recover(); err != nil {

			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			log.Errorf("panic running build: %v\n%s", err, string(buf))
		}
		e.pool.release(node)
	}()

	// update the node that was allocated to each job
	func(id int64) {
		for _, job := range req.Jobs {
			job.NodeID = id
			store.UpdateJob(c, job)
		}
	}(node.ID)

	// run the full build!
	client, err := newDockerClient(node.Addr, node.Cert, node.Key, node.CA)
	if err != nil {
		log.Errorln("error creating docker client", err)
	}

	// update the build state if any of the sub-tasks
	// had a non-success status
	req.Build.Started = time.Now().UTC().Unix()
	req.Build.Status = model.StatusRunning
	e.updater.SetBuild(c, req)

	// run all bulid jobs
	for _, job := range req.Jobs {
		req.Job = job
		e.runJob(c, req, e.updater, client)
	}

	// update overall status based on each job
	req.Build.Status = model.StatusSuccess
	for _, job := range req.Jobs {
		if job.Status != model.StatusSuccess {
			req.Build.Status = job.Status
			break
		}
	}
	req.Build.Finished = time.Now().UTC().Unix()
	err = e.updater.SetBuild(c, req)
	if err != nil {
		log.Errorf("error updating build completion status. %s", err)
	}

	// run notifications
	err = e.runJobNotify(req, client)
	if err != nil {
		log.Errorf("error executing notification step. %s", err)
	}
}

func newDockerClient(addr, cert, key, ca string) (dockerclient.Client, error) {
	var tlc *tls.Config

	// create the Docket client TLS config
	if len(cert) != 0 {
		pem, err := tls.X509KeyPair([]byte(cert), []byte(key))
		if err != nil {
			log.Errorf("error loading X509 key pair. %s.", err)
			return dockerclient.NewDockerClient(addr, nil)
		}

		// create the TLS configuration for secure
		// docker communications.
		tlc = &tls.Config{}
		tlc.Certificates = []tls.Certificate{pem}

		// use the certificate authority if provided.
		// else don't use a certificate authority and set
		// skip verify to true
		if len(ca) != 0 {
			log.Infof("creating docker client %s with CA", addr)
			pool := x509.NewCertPool()
			pool.AppendCertsFromPEM([]byte(ca))
			tlc.RootCAs = pool

		} else {
			log.Infof("creating docker client %s WITHOUT CA", addr)
			tlc.InsecureSkipVerify = true
		}
	}

	// create the Docker client. In this version of Drone (alpha)
	// we do not spread builds across clients, but this can and
	// (probably) will change in the future.
	return dockerclient.NewDockerClient(addr, tlc)
}

func (e *engine) runJob(c context.Context, r *Task, updater *updater, client dockerclient.Client) error {

	name := fmt.Sprintf("drone_build_%d_job_%d", r.Build.ID, r.Job.ID)

	defer func() {
		if r.Job.Status == model.StatusRunning {
			r.Job.Status = model.StatusError
			r.Job.Finished = time.Now().UTC().Unix()
			r.Job.ExitCode = 255
		}
		if r.Job.Status == model.StatusPending {
			r.Job.Status = model.StatusError
			r.Job.Started = time.Now().UTC().Unix()
			r.Job.Finished = time.Now().UTC().Unix()
			r.Job.ExitCode = 255
		}
		updater.SetJob(c, r)

		client.KillContainer(name, "9")
		client.RemoveContainer(name, true, true)
	}()

	// marks the task as running
	r.Job.Status = model.StatusRunning
	r.Job.Started = time.Now().UTC().Unix()

	// encode the build payload to write to stdin
	// when launching the build container
	in, err := encodeToLegacyFormat(r)
	if err != nil {
		log.Errorf("failure to marshal work. %s", err)
		return err
	}

	// CREATE AND START BUILD
	args := DefaultBuildArgs
	if r.Build.Event == model.EventPull {
		args = DefaultPullRequestArgs
	}
	args = append(args, "--")
	args = append(args, string(in))

	conf := &dockerclient.ContainerConfig{
		Image:      DefaultAgent,
		Entrypoint: DefaultEntrypoint,
		Cmd:        args,
		Env:        e.envs,
		HostConfig: dockerclient.HostConfig{
			Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
		},
		Volumes: map[string]struct{}{
			"/var/run/docker.sock": struct{}{},
		},
	}

	log.Infof("preparing container %s", name)
	client.PullImage(conf.Image, nil)

	_, err = docker.RunDaemon(client, conf, name)
	if err != nil {
		log.Errorf("error starting build container. %s", err)
		return err
	}

	// UPDATE STATUS

	err = updater.SetJob(c, r)
	if err != nil {
		log.Errorf("error updating job status as running. %s", err)
		return err
	}

	// WAIT FOR OUTPUT
	info, builderr := docker.Wait(client, name)

	switch {
	case info.State.Running:
		// A build unblocked before actually being completed.
		log.Errorf("incomplete build: %s", name)
		r.Job.ExitCode = 1
		r.Job.Status = model.StatusError
	case info.State.ExitCode == 128:
		r.Job.ExitCode = info.State.ExitCode
		r.Job.Status = model.StatusKilled
	case info.State.ExitCode == 130:
		r.Job.ExitCode = info.State.ExitCode
		r.Job.Status = model.StatusKilled
	case builderr != nil:
		r.Job.Status = model.StatusError
	case info.State.ExitCode != 0:
		r.Job.ExitCode = info.State.ExitCode
		r.Job.Status = model.StatusFailure
	default:
		r.Job.Status = model.StatusSuccess
	}

	// send the logs to the datastore
	var buf bytes.Buffer
	rc, err := client.ContainerLogs(name, docker.LogOpts)
	if err != nil && builderr != nil {
		buf.WriteString("Error launching build")
		buf.WriteString(builderr.Error())
	} else if err != nil {
		buf.WriteString("Error launching build")
		buf.WriteString(err.Error())
		log.Errorf("error opening connection to logs. %s", err)
		return err
	} else {
		defer rc.Close()
		stdcopy.StdCopy(&buf, &buf, io.LimitReader(rc, 5000000))
	}

	// update the task in the datastore
	r.Job.Finished = time.Now().UTC().Unix()
	err = updater.SetJob(c, r)
	if err != nil {
		log.Errorf("error updating job after completion. %s", err)
		return err
	}

	err = updater.SetLogs(c, r, ioutil.NopCloser(&buf))
	if err != nil {
		log.Errorf("error updating logs. %s", err)
		return err
	}

	log.Debugf("completed job %d with status %s.", r.Job.ID, r.Job.Status)
	return nil
}

func (e *engine) runJobNotify(r *Task, client dockerclient.Client) error {

	name := fmt.Sprintf("drone_build_%d_notify", r.Build.ID)

	defer func() {
		client.KillContainer(name, "9")
		client.RemoveContainer(name, true, true)
	}()

	// encode the build payload to write to stdin
	// when launching the build container
	in, err := encodeToLegacyFormat(r)
	if err != nil {
		log.Errorf("failure to marshal work. %s", err)
		return err
	}

	args := DefaultNotifyArgs
	args = append(args, "--")
	args = append(args, string(in))

	conf := &dockerclient.ContainerConfig{
		Image:      DefaultAgent,
		Entrypoint: DefaultEntrypoint,
		Cmd:        args,
		Env:        e.envs,
		HostConfig: dockerclient.HostConfig{
			Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
		},
		Volumes: map[string]struct{}{
			"/var/run/docker.sock": struct{}{},
		},
	}

	log.Infof("preparing container %s", name)
	info, err := docker.Run(client, conf, name)
	if err != nil {
		log.Errorf("Error starting notification container %s. %s", name, err)
	}

	// for debugging purposes we print a failed notification executions
	// output to the logs. Otherwise we have no way to troubleshoot failed
	// notifications. This is temporary code until I've come up with
	// a better solution.
	if info != nil && info.State.ExitCode != 0 && log.GetLevel() >= log.InfoLevel {
		var buf bytes.Buffer
		rc, err := client.ContainerLogs(name, docker.LogOpts)
		if err == nil {
			defer rc.Close()
			stdcopy.StdCopy(&buf, &buf, io.LimitReader(rc, 50000))
		}
		log.Infof("Notification container %s exited with %d", name, info.State.ExitCode)
		log.Infoln(buf.String())
	}

	return err
}
<commit_msg>Do not attempt to set memory.swappiness to zero<commit_after>package engine

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/drone/drone/model"
	"github.com/drone/drone/shared/docker"
	"github.com/drone/drone/shared/envconfig"
	"github.com/drone/drone/store"
	"github.com/samalba/dockerclient"
	"golang.org/x/net/context"
)

type Engine interface {
	Schedule(context.Context, *Task)
	Cancel(int64, int64, *model.Node) error
	Stream(int64, int64, *model.Node) (io.ReadCloser, error)
	Deallocate(*model.Node)
	Allocate(*model.Node) error
	Subscribe(chan *Event)
	Unsubscribe(chan *Event)
}

var (
	// options to fetch the stdout and stderr logs
	logOpts = &dockerclient.LogOptions{
		Stdout: true,
		Stderr: true,
	}

	// options to fetch the stdout and stderr logs
	// by tailing the output.
	logOptsTail = &dockerclient.LogOptions{
		Follow: true,
		Stdout: true,
		Stderr: true,
	}

	// error when the system cannot find logs
	errLogging = errors.New("Logs not available")
)

type engine struct {
	bus     *eventbus
	updater *updater
	pool    *pool
	envs    []string
}

// Load creates a new build engine, loaded with registered nodes from the
// database. The registered nodes are added to the pool of nodes to immediately
// start accepting workloads.
func Load(env envconfig.Env, s store.Store) Engine {
	engine := &engine{}
	engine.bus = newEventbus()
	engine.pool = newPool()
	engine.updater = &updater{engine.bus}

	// quick fix to propogate HTTP_PROXY variables
	// throughout the build environment.
	var proxyVars = []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy"}
	for _, proxyVar := range proxyVars {
		proxyVal := env.Get(proxyVar)
		if len(proxyVal) != 0 {
			engine.envs = append(engine.envs, proxyVar+"="+proxyVal)
		}
	}

	nodes, err := s.Nodes().GetList()
	if err != nil {
		log.Fatalf("failed to get nodes from database. %s", err)
	}
	for _, node := range nodes {
		engine.pool.allocate(node)
		log.Infof("registered docker daemon %s", node.Addr)
	}

	return engine
}

// Cancel cancels the job running on the specified Node.
func (e *engine) Cancel(build, job int64, node *model.Node) error {
	client, err := newDockerClient(node.Addr, node.Cert, node.Key, node.CA)
	if err != nil {
		return err
	}

	id := fmt.Sprintf("drone_build_%d_job_%d", build, job)
	return client.StopContainer(id, 30)
}

// Stream streams the job output from the specified Node.
func (e *engine) Stream(build, job int64, node *model.Node) (io.ReadCloser, error) {
	client, err := newDockerClient(node.Addr, node.Cert, node.Key, node.CA)
	if err != nil {
		log.Errorf("cannot create Docker client for node %s", node.Addr)
		return nil, err
	}

	id := fmt.Sprintf("drone_build_%d_job_%d", build, job)
	log.Debugf("streaming container logs %s", id)
	return client.ContainerLogs(id, logOptsTail)
}

// Subscribe subscribes the channel to all build events.
func (e *engine) Subscribe(c chan *Event) {
	e.bus.subscribe(c)
}

// Unsubscribe unsubscribes the channel from all build events.
func (e *engine) Unsubscribe(c chan *Event) {
	e.bus.unsubscribe(c)
}

func (e *engine) Allocate(node *model.Node) error {

	// run the full build!
	client, err := newDockerClient(node.Addr, node.Cert, node.Key, node.CA)
	if err != nil {
		log.Errorf("error creating docker client %s. %s.", node.Addr, err)
		return err
	}
	version, err := client.Version()
	if err != nil {
		log.Errorf("error connecting to docker daemon %s. %s.", node.Addr, err)
		return err
	}

	log.Infof("registered docker daemon %s running version %s", node.Addr, version.Version)
	e.pool.allocate(node)
	return nil
}

func (e *engine) Deallocate(n *model.Node) {
	nodes := e.pool.list()
	for _, node := range nodes {
		if node.ID == n.ID {
			log.Infof("un-registered docker daemon %s", node.Addr)
			e.pool.deallocate(node)
			break
		}
	}
}

func (e *engine) Schedule(c context.Context, req *Task) {
	node := <-e.pool.reserve()

	// since we are probably running in a go-routine
	// make sure we recover from any panics so that
	// a bug doesn't crash the whole system.
	defer func() {
		if err := recover(); err != nil {

			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			log.Errorf("panic running build: %v\n%s", err, string(buf))
		}
		e.pool.release(node)
	}()

	// update the node that was allocated to each job
	func(id int64) {
		for _, job := range req.Jobs {
			job.NodeID = id
			store.UpdateJob(c, job)
		}
	}(node.ID)

	// run the full build!
	client, err := newDockerClient(node.Addr, node.Cert, node.Key, node.CA)
	if err != nil {
		log.Errorln("error creating docker client", err)
	}

	// update the build state if any of the sub-tasks
	// had a non-success status
	req.Build.Started = time.Now().UTC().Unix()
	req.Build.Status = model.StatusRunning
	e.updater.SetBuild(c, req)

	// run all bulid jobs
	for _, job := range req.Jobs {
		req.Job = job
		e.runJob(c, req, e.updater, client)
	}

	// update overall status based on each job
	req.Build.Status = model.StatusSuccess
	for _, job := range req.Jobs {
		if job.Status != model.StatusSuccess {
			req.Build.Status = job.Status
			break
		}
	}
	req.Build.Finished = time.Now().UTC().Unix()
	err = e.updater.SetBuild(c, req)
	if err != nil {
		log.Errorf("error updating build completion status. %s", err)
	}

	// run notifications
	err = e.runJobNotify(req, client)
	if err != nil {
		log.Errorf("error executing notification step. %s", err)
	}
}

func newDockerClient(addr, cert, key, ca string) (dockerclient.Client, error) {
	var tlc *tls.Config

	// create the Docket client TLS config
	if len(cert) != 0 {
		pem, err := tls.X509KeyPair([]byte(cert), []byte(key))
		if err != nil {
			log.Errorf("error loading X509 key pair. %s.", err)
			return dockerclient.NewDockerClient(addr, nil)
		}

		// create the TLS configuration for secure
		// docker communications.
		tlc = &tls.Config{}
		tlc.Certificates = []tls.Certificate{pem}

		// use the certificate authority if provided.
		// else don't use a certificate authority and set
		// skip verify to true
		if len(ca) != 0 {
			log.Infof("creating docker client %s with CA", addr)
			pool := x509.NewCertPool()
			pool.AppendCertsFromPEM([]byte(ca))
			tlc.RootCAs = pool

		} else {
			log.Infof("creating docker client %s WITHOUT CA", addr)
			tlc.InsecureSkipVerify = true
		}
	}

	// create the Docker client. In this version of Drone (alpha)
	// we do not spread builds across clients, but this can and
	// (probably) will change in the future.
	return dockerclient.NewDockerClient(addr, tlc)
}

func (e *engine) runJob(c context.Context, r *Task, updater *updater, client dockerclient.Client) error {

	name := fmt.Sprintf("drone_build_%d_job_%d", r.Build.ID, r.Job.ID)

	defer func() {
		if r.Job.Status == model.StatusRunning {
			r.Job.Status = model.StatusError
			r.Job.Finished = time.Now().UTC().Unix()
			r.Job.ExitCode = 255
		}
		if r.Job.Status == model.StatusPending {
			r.Job.Status = model.StatusError
			r.Job.Started = time.Now().UTC().Unix()
			r.Job.Finished = time.Now().UTC().Unix()
			r.Job.ExitCode = 255
		}
		updater.SetJob(c, r)

		client.KillContainer(name, "9")
		client.RemoveContainer(name, true, true)
	}()

	// marks the task as running
	r.Job.Status = model.StatusRunning
	r.Job.Started = time.Now().UTC().Unix()

	// encode the build payload to write to stdin
	// when launching the build container
	in, err := encodeToLegacyFormat(r)
	if err != nil {
		log.Errorf("failure to marshal work. %s", err)
		return err
	}

	// CREATE AND START BUILD
	args := DefaultBuildArgs
	if r.Build.Event == model.EventPull {
		args = DefaultPullRequestArgs
	}
	args = append(args, "--")
	args = append(args, string(in))

	conf := &dockerclient.ContainerConfig{
		Image:      DefaultAgent,
		Entrypoint: DefaultEntrypoint,
		Cmd:        args,
		Env:        e.envs,
		HostConfig: dockerclient.HostConfig{
			Binds:            []string{"/var/run/docker.sock:/var/run/docker.sock"},
			MemorySwappiness: -1,
		},
		Volumes: map[string]struct{}{
			"/var/run/docker.sock": struct{}{},
		},
	}

	log.Infof("preparing container %s", name)
	client.PullImage(conf.Image, nil)

	_, err = docker.RunDaemon(client, conf, name)
	if err != nil {
		log.Errorf("error starting build container. %s", err)
		return err
	}

	// UPDATE STATUS

	err = updater.SetJob(c, r)
	if err != nil {
		log.Errorf("error updating job status as running. %s", err)
		return err
	}

	// WAIT FOR OUTPUT
	info, builderr := docker.Wait(client, name)

	switch {
	case info.State.Running:
		// A build unblocked before actually being completed.
		log.Errorf("incomplete build: %s", name)
		r.Job.ExitCode = 1
		r.Job.Status = model.StatusError
	case info.State.ExitCode == 128:
		r.Job.ExitCode = info.State.ExitCode
		r.Job.Status = model.StatusKilled
	case info.State.ExitCode == 130:
		r.Job.ExitCode = info.State.ExitCode
		r.Job.Status = model.StatusKilled
	case builderr != nil:
		r.Job.Status = model.StatusError
	case info.State.ExitCode != 0:
		r.Job.ExitCode = info.State.ExitCode
		r.Job.Status = model.StatusFailure
	default:
		r.Job.Status = model.StatusSuccess
	}

	// send the logs to the datastore
	var buf bytes.Buffer
	rc, err := client.ContainerLogs(name, docker.LogOpts)
	if err != nil && builderr != nil {
		buf.WriteString("Error launching build")
		buf.WriteString(builderr.Error())
	} else if err != nil {
		buf.WriteString("Error launching build")
		buf.WriteString(err.Error())
		log.Errorf("error opening connection to logs. %s", err)
		return err
	} else {
		defer rc.Close()
		stdcopy.StdCopy(&buf, &buf, io.LimitReader(rc, 5000000))
	}

	// update the task in the datastore
	r.Job.Finished = time.Now().UTC().Unix()
	err = updater.SetJob(c, r)
	if err != nil {
		log.Errorf("error updating job after completion. %s", err)
		return err
	}

	err = updater.SetLogs(c, r, ioutil.NopCloser(&buf))
	if err != nil {
		log.Errorf("error updating logs. %s", err)
		return err
	}

	log.Debugf("completed job %d with status %s.", r.Job.ID, r.Job.Status)
	return nil
}

func (e *engine) runJobNotify(r *Task, client dockerclient.Client) error {

	name := fmt.Sprintf("drone_build_%d_notify", r.Build.ID)

	defer func() {
		client.KillContainer(name, "9")
		client.RemoveContainer(name, true, true)
	}()

	// encode the build payload to write to stdin
	// when launching the build container
	in, err := encodeToLegacyFormat(r)
	if err != nil {
		log.Errorf("failure to marshal work. %s", err)
		return err
	}

	args := DefaultNotifyArgs
	args = append(args, "--")
	args = append(args, string(in))

	conf := &dockerclient.ContainerConfig{
		Image:      DefaultAgent,
		Entrypoint: DefaultEntrypoint,
		Cmd:        args,
		Env:        e.envs,
		HostConfig: dockerclient.HostConfig{
			Binds:            []string{"/var/run/docker.sock:/var/run/docker.sock"},
			MemorySwappiness: -1,
		},
		Volumes: map[string]struct{}{
			"/var/run/docker.sock": struct{}{},
		},
	}

	log.Infof("preparing container %s", name)
	info, err := docker.Run(client, conf, name)
	if err != nil {
		log.Errorf("Error starting notification container %s. %s", name, err)
	}

	// for debugging purposes we print a failed notification executions
	// output to the logs. Otherwise we have no way to troubleshoot failed
	// notifications. This is temporary code until I've come up with
	// a better solution.
	if info != nil && info.State.ExitCode != 0 && log.GetLevel() >= log.InfoLevel {
		var buf bytes.Buffer
		rc, err := client.ContainerLogs(name, docker.LogOpts)
		if err == nil {
			defer rc.Close()
			stdcopy.StdCopy(&buf, &buf, io.LimitReader(rc, 50000))
		}
		log.Infof("Notification container %s exited with %d", name, info.State.ExitCode)
		log.Infoln(buf.String())
	}

	return err
}
<|endoftext|>


<commit_before>package engine

import (
	"crypto/tls"
	"io/ioutil"
	"log"
	"net"

	"github.com/imdario/mergo"
	"github.com/influx6/relay"
	"gopkg.in/yaml.v2"
)

// DefaultConfig provides a default configuration for the app
var DefaultConfig = Config{
	Addr:    ":8080",
	UseTLS:  false,
	Folders: Folders{},
}

//TLSConfig provides a base config for tls configuration
type TLSConfig struct {
	Certs *tls.Config
	Key   string `yaml:"key"`
	Cert  string `yaml:"cert"`
}

type tlsconf struct {
	Key  string `yaml:"key"`
	Cert string `yaml:"cert"`
}

// UnmarshalYaml unmarshalls the incoming data for use
func (t *TLSConfig) UnmarshalYaml(unmarshal func(interface{}) error) error {
	to := tlsconf{}

	if err := unmarshal(&to); err != nil {
		return err
	}

	co, err := relay.LoadTLS(to.Cert, to.Key)

	if err != nil {
		return err
	}

	t.Certs = co
	return nil
}

// Folders provide a configuration for app-used folders
type Folders struct {
	Assets string `yaml:"assets"`
	Models string `yaml:"models"`
	Views  string `yaml:"views"`
}

// UnmarshalYaml unmarshalls the incoming data for use
func (t *Folders) UnmarshalYaml(unmarshal func(interface{}) error) error {
	if err := unmarshal(t); err != nil {
		return err
	}

	return nil
}

// Config provides configuration for Afro
type Config struct {
	Addr    string    `yaml:"addr"`
	UseTLS  bool      `yaml:"usetls"`
	C       TLSConfig `yaml:"tls"`
	Folders Folders   `yaml:"folders"`
}

// NewConfig returns a new configuration file
func NewConfig() *Config {
	c := DefaultConfig
	return &c
}

// Load loads the configuration from a yaml file
func (c *Config) Load(file string) error {
	data, err := ioutil.ReadFile(file)

	if err != nil {
		return err
	}

	conf := Config{}
	err = yaml.Unmarshal(data, &conf)

	log.Printf("load: %+s", conf)
	if err != nil {
		return err
	}

	return mergo.MergeWithOverwrite(c, conf)
}

// UnmarshalYAML unmarshals and sets the configuration options
func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {

	return nil
}

//Engine provides a base luncher for a service
type Engine struct {
	*relay.Routes
	li     net.Listener
	config *Config
}

//NewEngine returns a new app configuration
func NewEngine(c *Config) *Engine {
	return &Engine{
		Routes: relay.NewRoutes(),
		config: c,
	}
}

// Serve serves the app and configuration and loads the routes and serivices settings
func (a *Engine) Serve() error {
	var err error
	var li net.Listener

	if a.config.UseTLS && a.config.C.Certs != nil {
		_, li, err = relay.CreateTLS(a.config.Addr, a.config.C.Certs, a)
	} else {
		_, li, err = relay.CreateHTTP(a.config.Addr, a)
	}

	if err != nil {
		log.Fatalf("Server failed to start: %+s", err.Error())
		return err
	}

	a.li = li

	return nil
}

// Addr returns the address of the app
func (a *Engine) Addr() net.Addr {
	return a.li.Addr()
}

// Close closes and returns an error of the internal listener
func (a *Engine) Close() error {
	return a.li.Close()
}
<commit_msg>adding certification fixes<commit_after>package engine

import (
	"crypto/tls"
	"io/ioutil"
	"log"
	"net"

	"github.com/imdario/mergo"
	"github.com/influx6/relay"
	"gopkg.in/yaml.v2"
)

// DefaultConfig provides a default configuration for the app
var DefaultConfig = Config{
	Addr:    ":8080",
	UseTLS:  false,
	Folders: Folders{},
}

//TLSConfig provides a base config for tls configuration
type TLSConfig struct {
	Certs *tls.Config
	Key   string `yaml:"key"`
	Cert  string `yaml:"cert"`
}

type tlsconf struct {
	Key  string `yaml:"key"`
	Cert string `yaml:"cert"`
}

// UnmarshalYaml unmarshalls the incoming data for use
func (t *TLSConfig) UnmarshalYaml(unmarshal func(interface{}) error) error {
	to := tlsconf{}

	if err := unmarshal(&to); err != nil {
		return err
	}

	co, err := relay.LoadTLS(to.Cert, to.Key)

	if err != nil {
		return err
	}

	t.Certs = co
	return nil
}

// Folders provide a configuration for app-used folders
type Folders struct {
	Assets string `yaml:"assets"`
	Models string `yaml:"models"`
	Views  string `yaml:"views"`
}

// UnmarshalYaml unmarshalls the incoming data for use
func (t *Folders) UnmarshalYaml(unmarshal func(interface{}) error) error {
	if err := unmarshal(t); err != nil {
		return err
	}

	return nil
}

// Config provides configuration for Afro
type Config struct {
	Addr    string    `yaml:"addr"`
	UseTLS  bool      `yaml:"usetls"`
	C       TLSConfig `yaml:"tls"`
	Folders Folders   `yaml:"folders"`
}

// NewConfig returns a new configuration file
func NewConfig() *Config {
	c := DefaultConfig
	return &c
}

// Load loads the configuration from a yaml file
func (c *Config) Load(file string) error {
	data, err := ioutil.ReadFile(file)

	if err != nil {
		return err
	}

	conf := Config{}
	err = yaml.Unmarshal(data, &conf)

	log.Printf("load: %+s %+s", conf, data)
	if err != nil {
		return err
	}

	return mergo.MergeWithOverwrite(c, conf)
}

// UnmarshalYAML unmarshals and sets the configuration options
func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {

	return nil
}

//Engine provides a base luncher for a service
type Engine struct {
	*relay.Routes
	li     net.Listener
	config *Config
}

//NewEngine returns a new app configuration
func NewEngine(c *Config) *Engine {
	return &Engine{
		Routes: relay.NewRoutes(),
		config: c,
	}
}

// Serve serves the app and configuration and loads the routes and serivices settings
func (a *Engine) Serve() error {
	var err error
	var li net.Listener

	if a.config.UseTLS && a.config.C.Certs != nil {
		_, li, err = relay.CreateTLS(a.config.Addr, a.config.C.Certs, a)
	} else {
		_, li, err = relay.CreateHTTP(a.config.Addr, a)
	}

	if err != nil {
		log.Fatalf("Server failed to start: %+s", err.Error())
		return err
	}

	a.li = li

	return nil
}

// Addr returns the address of the app
func (a *Engine) Addr() net.Addr {
	return a.li.Addr()
}

// Close closes and returns an error of the internal listener
func (a *Engine) Close() error {
	return a.li.Close()
}
<|endoftext|>


<commit_before>package engine

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/funkygao/golib/gofmt"
	log "github.com/funkygao/log4go"
)

type messageRouter struct {
	hub chan *PipelinePack

	stats routerStats

	removeFilterMatcher chan *matcher
	removeOutputMatcher chan *matcher

	filterMatchers []*matcher
	outputMatchers []*matcher
}

func newMessageRouter() (this *messageRouter) {
	this = new(messageRouter)
	this.hub = make(chan *PipelinePack, Globals().PluginChanSize)
	this.stats = routerStats{}
	this.removeFilterMatcher = make(chan *matcher)
	this.removeOutputMatcher = make(chan *matcher)
	this.filterMatchers = make([]*matcher, 0, 10)
	this.outputMatchers = make([]*matcher, 0, 10)

	return
}

func (this *messageRouter) addFilterMatcher(matcher *matcher) {
	this.filterMatchers = append(this.filterMatchers, matcher)
}

func (this *messageRouter) addOutputMatcher(matcher *matcher) {
	this.outputMatchers = append(this.outputMatchers, matcher)
}

func (this *messageRouter) reportMatcherQueues() {
	globals := Globals()
	s := fmt.Sprintf("Queued hub=%d", len(this.hub))
	if len(this.hub) == globals.PluginChanSize {
		s = fmt.Sprintf("%s(F)", s)
	}

	for _, m := range this.filterMatchers {
		s = fmt.Sprintf("%s %s:%d", s, m.runner.Name(), len(m.InChan()))
		if len(m.InChan()) == globals.PluginChanSize {
			s = fmt.Sprintf("%s(F)", s)
		}
	}
	for _, m := range this.outputMatchers {
		s = fmt.Sprintf("%s %s:%d", s, m.runner.Name(), len(m.InChan()))
		if len(m.InChan()) == globals.PluginChanSize {
			s = fmt.Sprintf("%s(F)", s)
		}
	}

	log.Trace(s)
}

// Dispatch pack from Input to MatchRunners
func (this *messageRouter) Start(wg *sync.WaitGroup) {
	defer wg.Done()

	var (
		globals    = Globals()
		ok         = true
		pack       *PipelinePack
		ticker     *time.Ticker
		matcher    *matcher
		foundMatch bool
	)

	ticker = time.NewTicker(globals.WatchdogTick)
	defer ticker.Stop()

	go func() {
		t := time.NewTicker(globals.WatchdogTick)
		defer t.Stop()

		for range t.C {
			this.reportMatcherQueues()
		}
	}()

	log.Trace("Router started with ticker=%s", globals.WatchdogTick)

LOOP:
	for ok {
		select {
		case matcher = <-this.removeOutputMatcher:
			this.removeMatcher(matcher, this.outputMatchers)

		case matcher = <-this.removeFilterMatcher:
			this.removeMatcher(matcher, this.filterMatchers)

		case <-ticker.C:
			this.stats.render(int(globals.WatchdogTick.Seconds()))
			this.stats.resetPeriodCounters()

		case pack, ok = <-this.hub:
			if !ok {
				globals.Stopping = true
				break LOOP
			}

			this.stats.update(pack)

			pack.diagnostics.Reset()
			foundMatch = false

			// If we send pack to filterMatchers and then outputMatchers
			// because filter may change pack Ident, and this pack because
			// of shared mem, may match both filterMatcher and outputMatcher
			// then dup dispatching happens!!!
			//
			// We have to dispatch to Output then Filter to avoid that case
			for _, matcher = range this.outputMatchers {
				// a pack can match several Output
				if matcher != nil && matcher.Match(pack) {
					foundMatch = true

					pack.incRef()
					pack.diagnostics.AddStamp(matcher.runner)
					matcher.InChan() <- pack
				}
			}

			// got pack from Input, now dispatch
			// for each target, pack will inc ref count
			// and the router will dec ref count only once
			for _, matcher = range this.filterMatchers {
				// a pack can match several Filter
				if matcher != nil && matcher.Match(pack) {
					foundMatch = true

					pack.incRef()
					pack.diagnostics.AddStamp(matcher.runner)
					matcher.InChan() <- pack
				}
			}

			if !foundMatch {
				// Maybe we closed all filter/output inChan, but there
				// still exits some remnant packs in router.hub
				log.Warn("no match: %+v", pack)
			}

			// never forget this!
			pack.Recycle()
		}
	}

}

func (this *messageRouter) removeMatcher(matcher *matcher, matchers []*matcher) {
	for idx, m := range matchers {
		if m == matcher {
			log.Trace("closing matcher for %s", m.runner.Name())

			// in golang, close means we can no longer send to that chan
			// but consumers can still recv from the chan
			close(m.InChan())
			matchers[idx] = nil
			return
		}
	}
}

type routerStats struct {
	TotalInputMsgN       int64
	PeriodInputMsgN      int32
	TotalInputBytes      int64
	PeriodInputBytes     int64
	TotalProcessedBytes  int64
	TotalProcessedMsgN   int64 // 16 BilionBillion
	PeriodProcessedMsgN  int32
	PeriodProcessedBytes int64
	TotalMaxMsgBytes     int64
	PeriodMaxMsgBytes    int64
}

func (this *routerStats) update(pack *PipelinePack) {
	atomic.AddInt64(&this.TotalProcessedMsgN, 1)
	atomic.AddInt32(&this.PeriodProcessedMsgN, 1)

	if pack.input {
		atomic.AddInt64(&this.TotalInputMsgN, 1)
		atomic.AddInt32(&this.PeriodInputMsgN, 1)
		atomic.AddInt64(&this.TotalInputBytes, int64(pack.Payload.Length()))
		atomic.AddInt64(&this.PeriodInputBytes, int64(pack.Payload.Length()))
	}
}

func (this *routerStats) resetPeriodCounters() {
	this.PeriodProcessedBytes = int64(0)
	this.PeriodInputBytes = int64(0)
	this.PeriodInputMsgN = int32(0)
	this.PeriodProcessedMsgN = int32(0)
	this.PeriodMaxMsgBytes = int64(0)
}

func (this *routerStats) render(elapsed int) {
	log.Trace("Total:%10s %10s speed:%6s/s %10s/s max: %s/%s",
		gofmt.Comma(this.TotalProcessedMsgN),
		gofmt.ByteSize(this.TotalProcessedBytes),
		gofmt.Comma(int64(this.PeriodProcessedMsgN/int32(elapsed))),
		gofmt.ByteSize(this.PeriodProcessedBytes/int64(elapsed)),
		gofmt.ByteSize(this.PeriodMaxMsgBytes),
		gofmt.ByteSize(this.TotalMaxMsgBytes))
	log.Trace("Input:%10s %10s speed:%6s/s %10s/s",
		gofmt.Comma(int64(this.PeriodInputMsgN)),
		gofmt.ByteSize(this.PeriodInputBytes),
		gofmt.Comma(int64(this.PeriodInputMsgN/int32(elapsed))),
		gofmt.ByteSize(this.PeriodInputBytes/int64(elapsed)))
}
<commit_msg>some notes<commit_after>package engine

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/funkygao/golib/gofmt"
	log "github.com/funkygao/log4go"
)

type messageRouter struct {
	hub chan *PipelinePack

	stats routerStats

	removeFilterMatcher chan *matcher
	removeOutputMatcher chan *matcher

	filterMatchers []*matcher
	outputMatchers []*matcher
}

func newMessageRouter() (this *messageRouter) {
	this = new(messageRouter)
	this.hub = make(chan *PipelinePack, Globals().PluginChanSize)
	this.stats = routerStats{}
	this.removeFilterMatcher = make(chan *matcher)
	this.removeOutputMatcher = make(chan *matcher)
	this.filterMatchers = make([]*matcher, 0, 10)
	this.outputMatchers = make([]*matcher, 0, 10)

	return
}

func (this *messageRouter) addFilterMatcher(matcher *matcher) {
	this.filterMatchers = append(this.filterMatchers, matcher)
}

func (this *messageRouter) addOutputMatcher(matcher *matcher) {
	this.outputMatchers = append(this.outputMatchers, matcher)
}

func (this *messageRouter) reportMatcherQueues() {
	globals := Globals()
	s := fmt.Sprintf("Queued hub=%d", len(this.hub))
	if len(this.hub) == globals.PluginChanSize {
		s = fmt.Sprintf("%s(F)", s)
	}

	for _, m := range this.filterMatchers {
		s = fmt.Sprintf("%s %s:%d", s, m.runner.Name(), len(m.InChan()))
		if len(m.InChan()) == globals.PluginChanSize {
			s = fmt.Sprintf("%s(F)", s)
		}
	}
	for _, m := range this.outputMatchers {
		s = fmt.Sprintf("%s %s:%d", s, m.runner.Name(), len(m.InChan()))
		if len(m.InChan()) == globals.PluginChanSize {
			s = fmt.Sprintf("%s(F)", s)
		}
	}

	log.Trace(s)
}

// Dispatch pack from Input to MatchRunners
func (this *messageRouter) Start(wg *sync.WaitGroup) {
	defer wg.Done()

	var (
		globals    = Globals()
		ok         = true
		pack       *PipelinePack
		ticker     *time.Ticker
		matcher    *matcher
		foundMatch bool
	)

	ticker = time.NewTicker(globals.WatchdogTick)
	defer ticker.Stop()

	go func() {
		t := time.NewTicker(globals.WatchdogTick)
		defer t.Stop()

		for range t.C {
			this.reportMatcherQueues()
		}
	}()

	log.Trace("Router started with ticker=%s", globals.WatchdogTick)

LOOP:
	for ok {
		select {
		case matcher = <-this.removeOutputMatcher:
			this.removeMatcher(matcher, this.outputMatchers)

		case matcher = <-this.removeFilterMatcher:
			this.removeMatcher(matcher, this.filterMatchers)

		case <-ticker.C:
			this.stats.render(int(globals.WatchdogTick.Seconds()))
			this.stats.resetPeriodCounters()

		case pack, ok = <-this.hub:
			if !ok {
				globals.Stopping = true
				break LOOP
			}

			this.stats.update(pack)

			pack.diagnostics.Reset()
			foundMatch = false

			// If we send pack to filterMatchers and then outputMatchers
			// because filter may change pack Ident, and this pack because
			// of shared mem, may match both filterMatcher and outputMatcher
			// then dup dispatching happens!!!
			//
			// We have to dispatch to Output then Filter to avoid that case
			for _, matcher = range this.outputMatchers {
				// a pack can match several Output
				if matcher != nil && matcher.Match(pack) {
					foundMatch = true

					pack.incRef()
					pack.diagnostics.AddStamp(matcher.runner)
					matcher.InChan() <- pack
				}
			}

			// got pack from Input, now dispatch
			// for each target, pack will inc ref count
			// and the router will dec ref count only once
			for _, matcher = range this.filterMatchers {
				// a pack can match several Filter
				if matcher != nil && matcher.Match(pack) {
					foundMatch = true

					pack.incRef()
					pack.diagnostics.AddStamp(matcher.runner)
					matcher.InChan() <- pack
				}
			}

			if !foundMatch {
				// Maybe we closed all filter/output inChan, but there
				// still exits some remnant packs in router.hub.
				// To handle this issue, Input/Output should be stateful.
				log.Debug("no match: %+v", pack)
			}

			// never forget this!
			pack.Recycle()
		}
	}

}

func (this *messageRouter) removeMatcher(matcher *matcher, matchers []*matcher) {
	for idx, m := range matchers {
		if m == matcher {
			log.Trace("closing matcher for %s", m.runner.Name())

			// in golang, close means we can no longer send to that chan
			// but consumers can still recv from the chan
			close(m.InChan())
			matchers[idx] = nil
			return
		}
	}
}

type routerStats struct {
	TotalInputMsgN       int64
	PeriodInputMsgN      int32
	TotalInputBytes      int64
	PeriodInputBytes     int64
	TotalProcessedBytes  int64
	TotalProcessedMsgN   int64 // 16 BilionBillion
	PeriodProcessedMsgN  int32
	PeriodProcessedBytes int64
	TotalMaxMsgBytes     int64
	PeriodMaxMsgBytes    int64
}

func (this *routerStats) update(pack *PipelinePack) {
	atomic.AddInt64(&this.TotalProcessedMsgN, 1)
	atomic.AddInt32(&this.PeriodProcessedMsgN, 1)

	if pack.input {
		atomic.AddInt64(&this.TotalInputMsgN, 1)
		atomic.AddInt32(&this.PeriodInputMsgN, 1)
		atomic.AddInt64(&this.TotalInputBytes, int64(pack.Payload.Length()))
		atomic.AddInt64(&this.PeriodInputBytes, int64(pack.Payload.Length()))
	}
}

func (this *routerStats) resetPeriodCounters() {
	this.PeriodProcessedBytes = int64(0)
	this.PeriodInputBytes = int64(0)
	this.PeriodInputMsgN = int32(0)
	this.PeriodProcessedMsgN = int32(0)
	this.PeriodMaxMsgBytes = int64(0)
}

func (this *routerStats) render(elapsed int) {
	log.Trace("Total:%10s %10s speed:%6s/s %10s/s max: %s/%s",
		gofmt.Comma(this.TotalProcessedMsgN),
		gofmt.ByteSize(this.TotalProcessedBytes),
		gofmt.Comma(int64(this.PeriodProcessedMsgN/int32(elapsed))),
		gofmt.ByteSize(this.PeriodProcessedBytes/int64(elapsed)),
		gofmt.ByteSize(this.PeriodMaxMsgBytes),
		gofmt.ByteSize(this.TotalMaxMsgBytes))
	log.Trace("Input:%10s %10s speed:%6s/s %10s/s",
		gofmt.Comma(int64(this.PeriodInputMsgN)),
		gofmt.ByteSize(this.PeriodInputBytes),
		gofmt.Comma(int64(this.PeriodInputMsgN/int32(elapsed))),
		gofmt.ByteSize(this.PeriodInputBytes/int64(elapsed)))
}
<|endoftext|>


<commit_before>package isolated

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Unshare Private Domain", func() {
	var (
		domainName      string
		owningOrgName   string
		sharedToOrgName string
	)

	BeforeEach(func() {
		domainName = helpers.NewDomainName()
		sharedToOrgName = helpers.NewOrgName()
	})

	Describe("Help Text", func() {
		It("Displays the help text", func() {
			session := helpers.CF("unshare-private-domain", "--help")
			Eventually(session).Should(Say("NAME:"))
			Eventually(session).Should(Say("unshare-private-domain - Unshare a private domain with an org"))
			Eventually(session).Should(Say("USAGE:"))
			Eventually(session).Should(Say("cf unshare-private-domain ORG DOMAIN"))
			Eventually(session).Should(Say("SEE ALSO:"))
			Eventually(session).Should(Say("delete-domain, domains"))
		})
	})

	Describe("When the environment is not set up correctly", func() {
		When("The user is not logged in", func() {
			It("lets the user know", func() {
				session := helpers.CF("unshare-private-domain", sharedToOrgName, domainName)
				Eventually(session).Should(Say("FAILED"))
				Eventually(session).Should(Say("Not logged in. Use 'cf login' to log in."))
			})
		})
	})

	Describe("When the environment is set up correctly", func() {
		BeforeEach(func() {
			helpers.LoginCF()
			owningOrgName = helpers.CreateAndTargetOrg()
			helpers.CreateOrg(sharedToOrgName)
			domain := helpers.NewDomain(owningOrgName, domainName)
			domain.CreatePrivate()
			domain.V7Share(sharedToOrgName)
		})

		It("unshares the domain from the org", func() {
			session := helpers.CF("unshare-private-domain", sharedToOrgName, domainName)
			Eventually(session).Should(Say("Unsharing domain %s from org %s as admin...", domainName, sharedToOrgName))
			Eventually(session).Should(Say("OK"))
			Eventually(session).Should(Exit(0))

			helpers.TargetOrg(sharedToOrgName)
			session = helpers.CF("domains")
			Consistently(session).Should(Not(Say("%s", domainName)))
		})
	})
})
<commit_msg>Remove incorrectly placed integration test<commit_after><|endoftext|>