<commit_before>package mysql

import (
	"context"
	"database/sql"
	"fmt"
	"math"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"gorm.io/gorm"
	"gorm.io/gorm/callbacks"
	"gorm.io/gorm/clause"
	"gorm.io/gorm/logger"
	"gorm.io/gorm/migrator"
	"gorm.io/gorm/schema"
)

type Config struct {
	DriverName                string
	ServerVersion             string
	DSN                       string
	Conn                      gorm.ConnPool
	SkipInitializeWithVersion bool
	DefaultStringSize         uint
	DefaultDatetimePrecision  *int
	DisableDatetimePrecision  bool
	DontSupportRenameIndex    bool
	DontSupportRenameColumn   bool
	DontSupportForShareClause bool
}

type Dialector struct {
	*Config
}

var (
	// CreateClauses create clauses
	CreateClauses = []string{"INSERT", "VALUES", "ON CONFLICT"}
	// UpdateClauses update clauses
	UpdateClauses = []string{"UPDATE", "SET", "WHERE", "ORDER BY", "LIMIT"}
	// DeleteClauses delete clauses
	DeleteClauses = []string{"DELETE", "FROM", "WHERE", "ORDER BY", "LIMIT"}

	defaultDatetimePrecision = 3
)

func Open(dsn string) gorm.Dialector {
	return &Dialector{Config: &Config{DSN: dsn}}
}

func New(config Config) gorm.Dialector {
	return &Dialector{Config: &config}
}

func (dialector Dialector) Name() string {
	return "mysql"
}

// NowFunc return now func
func (dialector Dialector) NowFunc(n int) func() time.Time {
	return func() time.Time {
		round := time.Second / time.Duration(math.Pow10(n))
		return time.Now().Local().Round(round)
	}
}

func (dialector Dialector) Apply(config *gorm.Config) error {
	if config.NowFunc == nil {
		if dialector.DefaultDatetimePrecision == nil {
			dialector.DefaultDatetimePrecision = &defaultDatetimePrecision
		}

		// while maintaining the readability of the code, separate the business logic from
		// the general part and leave it to the function to do it here.
		config.NowFunc = dialector.NowFunc(*dialector.DefaultDatetimePrecision)
	}

	return nil
}

func (dialector Dialector) Initialize(db *gorm.DB) (err error) {
	ctx := context.Background()

	// register callbacks
	callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{
		CreateClauses: CreateClauses,
		UpdateClauses: UpdateClauses,
		DeleteClauses: DeleteClauses,
	})

	if dialector.DriverName == "" {
		dialector.DriverName = "mysql"
	}

	if dialector.DefaultDatetimePrecision == nil {
		dialector.DefaultDatetimePrecision = &defaultDatetimePrecision
	}

	if dialector.Conn != nil {
		db.ConnPool = dialector.Conn
	} else {
		db.ConnPool, err = sql.Open(dialector.DriverName, dialector.DSN)
		if err != nil {
			return err
		}
	}

	if !dialector.Config.SkipInitializeWithVersion {
		err = db.ConnPool.QueryRowContext(ctx, "SELECT VERSION()").Scan(&dialector.ServerVersion)
		if err != nil {
			return err
		}

		if strings.Contains(dialector.ServerVersion, "MariaDB") {
			dialector.Config.DontSupportRenameIndex = true
			dialector.Config.DontSupportRenameColumn = true
			dialector.Config.DontSupportForShareClause = true
		} else if strings.HasPrefix(dialector.ServerVersion, "5.6.") {
			dialector.Config.DontSupportRenameIndex = true
			dialector.Config.DontSupportRenameColumn = true
			dialector.Config.DontSupportForShareClause = true
		} else if strings.HasPrefix(dialector.ServerVersion, "5.7.") {
			dialector.Config.DontSupportRenameColumn = true
			dialector.Config.DontSupportForShareClause = true
		} else if strings.HasPrefix(dialector.ServerVersion, "5.") {
			dialector.Config.DisableDatetimePrecision = true
			dialector.Config.DontSupportRenameIndex = true
			dialector.Config.DontSupportRenameColumn = true
			dialector.Config.DontSupportForShareClause = true
		}
	}

	for k, v := range dialector.ClauseBuilders() {
		db.ClauseBuilders[k] = v
	}
	return
}

const (
	// ClauseOnConflict for clause.ClauseBuilder ON CONFLICT key
	ClauseOnConflict = "ON CONFLICT"
	// ClauseValues for clause.ClauseBuilder VALUES key
	ClauseValues = "VALUES"
	// ClauseValues for clause.ClauseBuilder FOR key
	ClauseFor = "FOR"
)

func (dialector Dialector) ClauseBuilders() map[string]clause.ClauseBuilder {
	clauseBuilders := map[string]clause.ClauseBuilder{
		ClauseOnConflict: func(c clause.Clause, builder clause.Builder) {
			onConflict, ok := c.Expression.(clause.OnConflict)
			if !ok {
				c.Build(builder)
				return
			}

			builder.WriteString("ON DUPLICATE KEY UPDATE ")
			if len(onConflict.DoUpdates) == 0 {
				if s := builder.(*gorm.Statement).Schema; s != nil {
					var column clause.Column
					onConflict.DoNothing = false

					if s.PrioritizedPrimaryField != nil {
						column = clause.Column{Name: s.PrioritizedPrimaryField.DBName}
					} else if len(s.DBNames) > 0 {
						column = clause.Column{Name: s.DBNames[0]}
					}

					if column.Name != "" {
						onConflict.DoUpdates = []clause.Assignment{{Column: column, Value: column}}
					}
				}
			}

			for idx, assignment := range onConflict.DoUpdates {
				if idx > 0 {
					builder.WriteByte(',')
				}

				builder.WriteQuoted(assignment.Column)
				builder.WriteByte('=')
				if column, ok := assignment.Value.(clause.Column); ok && column.Table == "excluded" {
					column.Table = ""
					builder.WriteString("VALUES(")
					builder.WriteQuoted(column)
					builder.WriteByte(')')
				} else {
					builder.AddVar(builder, assignment.Value)
				}
			}
		},
		ClauseValues: func(c clause.Clause, builder clause.Builder) {
			if values, ok := c.Expression.(clause.Values); ok && len(values.Columns) == 0 {
				builder.WriteString("VALUES()")
				return
			}
			c.Build(builder)
		},
	}

	if dialector.Config.DontSupportForShareClause {
		clauseBuilders[ClauseFor] = func(c clause.Clause, builder clause.Builder) {
			if values, ok := c.Expression.(clause.Locking); ok && strings.EqualFold(values.Strength, "SHARE") {
				builder.WriteString("LOCK IN SHARE MODE")
				return
			}
			c.Build(builder)
		}
	}

	return clauseBuilders
}

func (dialector Dialector) DefaultValueOf(field *schema.Field) clause.Expression {
	return clause.Expr{SQL: "DEFAULT"}
}

func (dialector Dialector) Migrator(db *gorm.DB) gorm.Migrator {
	return Migrator{
		Migrator: migrator.Migrator{
			Config: migrator.Config{
				DB:        db,
				Dialector: dialector,
			},
		},
		Dialector: dialector,
	}
}

func (dialector Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{}) {
	writer.WriteByte('?')
}

func (dialector Dialector) QuoteTo(writer clause.Writer, str string) {
	var (
		underQuoted, selfQuoted bool
		continuousBacktick      int8
		shiftDelimiter          int8
	)

	for _, v := range []byte(str) {
		switch v {
		case '`':
			continuousBacktick++
			if continuousBacktick == 2 {
				writer.WriteString("``")
				continuousBacktick = 0
			}
		case '.':
			if continuousBacktick > 0 || !selfQuoted {
				shiftDelimiter = 0
				underQuoted = false
				continuousBacktick = 0
				writer.WriteString("`")
			}
			writer.WriteByte(v)
			continue
		default:
			if shiftDelimiter-continuousBacktick <= 0 && !underQuoted {
				writer.WriteByte('`')
				underQuoted = true
				if selfQuoted = continuousBacktick > 0; selfQuoted {
					continuousBacktick -= 1
				}
			}

			for ; continuousBacktick > 0; continuousBacktick -= 1 {
				writer.WriteString("``")
			}

			writer.WriteByte(v)
		}
		shiftDelimiter++
	}

	if continuousBacktick > 0 && !selfQuoted {
		writer.WriteString("``")
	}
	writer.WriteString("`")
}

func (dialector Dialector) Explain(sql string, vars ...interface{}) string {
	return logger.ExplainSQL(sql, nil, `'`, vars...)
}

func (dialector Dialector) DataTypeOf(field *schema.Field) string {
	switch field.DataType {
	case schema.Bool:
		return "boolean"
	case schema.Int, schema.Uint:
		return dialector.getSchemaIntAndUnitType(field)
	case schema.Float:
		return dialector.getSchemaFloatType(field)
	case schema.String:
		return dialector.getSchemaStringType(field)
	case schema.Time:
		return dialector.getSchemaTimeType(field)
	case schema.Bytes:
		return dialector.getSchemaBytesType(field)
	}

	return string(field.DataType)
}

func (dialector Dialector) getSchemaFloatType(field *schema.Field) string {
	if field.Precision > 0 {
		return fmt.Sprintf("decimal(%d, %d)", field.Precision, field.Scale)
	}

	if field.Size <= 32 {
		return "float"
	}

	return "double"
}

func (dialector Dialector) getSchemaStringType(field *schema.Field) string {
	size := field.Size
	if size == 0 {
		if dialector.DefaultStringSize > 0 {
			size = int(dialector.DefaultStringSize)
		} else {
			hasIndex := field.TagSettings["INDEX"] != "" || field.TagSettings["UNIQUE"] != ""
			// TEXT, GEOMETRY or JSON column can't have a default value
			if field.PrimaryKey || field.HasDefaultValue || hasIndex {
				size = 191 // utf8mb4
			}
		}
	}

	if size >= 65536 && size <= int(math.Pow(2, 24)) {
		return "mediumtext"
	}

	if size > int(math.Pow(2, 24)) || size <= 0 {
		return "longtext"
	}

	return fmt.Sprintf("varchar(%d)", size)
}

func (dialector Dialector) getSchemaTimeType(field *schema.Field) string {
	precision := ""
	if !dialector.DisableDatetimePrecision && field.Precision == 0 {
		field.Precision = *dialector.DefaultDatetimePrecision
	}

	if field.Precision > 0 {
		precision = fmt.Sprintf("(%d)", field.Precision)
	}

	if field.NotNull || field.PrimaryKey {
		return "datetime" + precision
	}
	return "datetime" + precision + " NULL"
}

func (dialector Dialector) getSchemaBytesType(field *schema.Field) string {
	if field.Size > 0 && field.Size < 65536 {
		return fmt.Sprintf("varbinary(%d)", field.Size)
	}

	if field.Size >= 65536 && field.Size <= int(math.Pow(2, 24)) {
		return "mediumblob"
	}

	return "longblob"
}

func (dialector Dialector) getSchemaIntAndUnitType(field *schema.Field) string {
	sqlType := "bigint"
	switch {
	case field.Size <= 8:
		sqlType = "tinyint"
	case field.Size <= 16:
		sqlType = "smallint"
	case field.Size <= 24:
		sqlType = "mediumint"
	case field.Size <= 32:
		sqlType = "int"
	}

	if field.DataType == schema.Uint {
		sqlType += " unsigned"
	}

	if field.AutoIncrement {
		sqlType += " AUTO_INCREMENT"
	}

	return sqlType
}

func (dialector Dialector) SavePoint(tx *gorm.DB, name string) error {
	return tx.Exec("SAVEPOINT " + name).Error
}

func (dialector Dialector) RollbackTo(tx *gorm.DB, name string) error {
	return tx.Exec("ROLLBACK TO SAVEPOINT " + name).Error
}
<commit_msg>add global variable QueryClauses (#65)<commit_after>package mysql

import (
	"context"
	"database/sql"
	"fmt"
	"math"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"gorm.io/gorm"
	"gorm.io/gorm/callbacks"
	"gorm.io/gorm/clause"
	"gorm.io/gorm/logger"
	"gorm.io/gorm/migrator"
	"gorm.io/gorm/schema"
)

type Config struct {
	DriverName                string
	ServerVersion             string
	DSN                       string
	Conn                      gorm.ConnPool
	SkipInitializeWithVersion bool
	DefaultStringSize         uint
	DefaultDatetimePrecision  *int
	DisableDatetimePrecision  bool
	DontSupportRenameIndex    bool
	DontSupportRenameColumn   bool
	DontSupportForShareClause bool
}

type Dialector struct {
	*Config
}

var (
	// CreateClauses create clauses
	CreateClauses = []string{"INSERT", "VALUES", "ON CONFLICT"}
	// QueryClauses query clauses
	QueryClauses = []string{}
	// UpdateClauses update clauses
	UpdateClauses = []string{"UPDATE", "SET", "WHERE", "ORDER BY", "LIMIT"}
	// DeleteClauses delete clauses
	DeleteClauses = []string{"DELETE", "FROM", "WHERE", "ORDER BY", "LIMIT"}

	defaultDatetimePrecision = 3
)

func Open(dsn string) gorm.Dialector {
	return &Dialector{Config: &Config{DSN: dsn}}
}

func New(config Config) gorm.Dialector {
	return &Dialector{Config: &config}
}

func (dialector Dialector) Name() string {
	return "mysql"
}

// NowFunc return now func
func (dialector Dialector) NowFunc(n int) func() time.Time {
	return func() time.Time {
		round := time.Second / time.Duration(math.Pow10(n))
		return time.Now().Local().Round(round)
	}
}

func (dialector Dialector) Apply(config *gorm.Config) error {
	if config.NowFunc == nil {
		if dialector.DefaultDatetimePrecision == nil {
			dialector.DefaultDatetimePrecision = &defaultDatetimePrecision
		}

		// while maintaining the readability of the code, separate the business logic from
		// the general part and leave it to the function to do it here.
		config.NowFunc = dialector.NowFunc(*dialector.DefaultDatetimePrecision)
	}

	return nil
}

func (dialector Dialector) Initialize(db *gorm.DB) (err error) {
	ctx := context.Background()

	// register callbacks
	callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{
		CreateClauses: CreateClauses,
		QueryClauses:  QueryClauses,
		UpdateClauses: UpdateClauses,
		DeleteClauses: DeleteClauses,
	})

	if dialector.DriverName == "" {
		dialector.DriverName = "mysql"
	}

	if dialector.DefaultDatetimePrecision == nil {
		dialector.DefaultDatetimePrecision = &defaultDatetimePrecision
	}

	if dialector.Conn != nil {
		db.ConnPool = dialector.Conn
	} else {
		db.ConnPool, err = sql.Open(dialector.DriverName, dialector.DSN)
		if err != nil {
			return err
		}
	}

	if !dialector.Config.SkipInitializeWithVersion {
		err = db.ConnPool.QueryRowContext(ctx, "SELECT VERSION()").Scan(&dialector.ServerVersion)
		if err != nil {
			return err
		}

		if strings.Contains(dialector.ServerVersion, "MariaDB") {
			dialector.Config.DontSupportRenameIndex = true
			dialector.Config.DontSupportRenameColumn = true
			dialector.Config.DontSupportForShareClause = true
		} else if strings.HasPrefix(dialector.ServerVersion, "5.6.") {
			dialector.Config.DontSupportRenameIndex = true
			dialector.Config.DontSupportRenameColumn = true
			dialector.Config.DontSupportForShareClause = true
		} else if strings.HasPrefix(dialector.ServerVersion, "5.7.") {
			dialector.Config.DontSupportRenameColumn = true
			dialector.Config.DontSupportForShareClause = true
		} else if strings.HasPrefix(dialector.ServerVersion, "5.") {
			dialector.Config.DisableDatetimePrecision = true
			dialector.Config.DontSupportRenameIndex = true
			dialector.Config.DontSupportRenameColumn = true
			dialector.Config.DontSupportForShareClause = true
		}
	}

	for k, v := range dialector.ClauseBuilders() {
		db.ClauseBuilders[k] = v
	}
	return
}

const (
	// ClauseOnConflict for clause.ClauseBuilder ON CONFLICT key
	ClauseOnConflict = "ON CONFLICT"
	// ClauseValues for clause.ClauseBuilder VALUES key
	ClauseValues = "VALUES"
	// ClauseValues for clause.ClauseBuilder FOR key
	ClauseFor = "FOR"
)

func (dialector Dialector) ClauseBuilders() map[string]clause.ClauseBuilder {
	clauseBuilders := map[string]clause.ClauseBuilder{
		ClauseOnConflict: func(c clause.Clause, builder clause.Builder) {
			onConflict, ok := c.Expression.(clause.OnConflict)
			if !ok {
				c.Build(builder)
				return
			}

			builder.WriteString("ON DUPLICATE KEY UPDATE ")
			if len(onConflict.DoUpdates) == 0 {
				if s := builder.(*gorm.Statement).Schema; s != nil {
					var column clause.Column
					onConflict.DoNothing = false

					if s.PrioritizedPrimaryField != nil {
						column = clause.Column{Name: s.PrioritizedPrimaryField.DBName}
					} else if len(s.DBNames) > 0 {
						column = clause.Column{Name: s.DBNames[0]}
					}

					if column.Name != "" {
						onConflict.DoUpdates = []clause.Assignment{{Column: column, Value: column}}
					}
				}
			}

			for idx, assignment := range onConflict.DoUpdates {
				if idx > 0 {
					builder.WriteByte(',')
				}

				builder.WriteQuoted(assignment.Column)
				builder.WriteByte('=')
				if column, ok := assignment.Value.(clause.Column); ok && column.Table == "excluded" {
					column.Table = ""
					builder.WriteString("VALUES(")
					builder.WriteQuoted(column)
					builder.WriteByte(')')
				} else {
					builder.AddVar(builder, assignment.Value)
				}
			}
		},
		ClauseValues: func(c clause.Clause, builder clause.Builder) {
			if values, ok := c.Expression.(clause.Values); ok && len(values.Columns) == 0 {
				builder.WriteString("VALUES()")
				return
			}
			c.Build(builder)
		},
	}

	if dialector.Config.DontSupportForShareClause {
		clauseBuilders[ClauseFor] = func(c clause.Clause, builder clause.Builder) {
			if values, ok := c.Expression.(clause.Locking); ok && strings.EqualFold(values.Strength, "SHARE") {
				builder.WriteString("LOCK IN SHARE MODE")
				return
			}
			c.Build(builder)
		}
	}

	return clauseBuilders
}

func (dialector Dialector) DefaultValueOf(field *schema.Field) clause.Expression {
	return clause.Expr{SQL: "DEFAULT"}
}

func (dialector Dialector) Migrator(db *gorm.DB) gorm.Migrator {
	return Migrator{
		Migrator: migrator.Migrator{
			Config: migrator.Config{
				DB:        db,
				Dialector: dialector,
			},
		},
		Dialector: dialector,
	}
}

func (dialector Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{}) {
	writer.WriteByte('?')
}

func (dialector Dialector) QuoteTo(writer clause.Writer, str string) {
	var (
		underQuoted, selfQuoted bool
		continuousBacktick      int8
		shiftDelimiter          int8
	)

	for _, v := range []byte(str) {
		switch v {
		case '`':
			continuousBacktick++
			if continuousBacktick == 2 {
				writer.WriteString("``")
				continuousBacktick = 0
			}
		case '.':
			if continuousBacktick > 0 || !selfQuoted {
				shiftDelimiter = 0
				underQuoted = false
				continuousBacktick = 0
				writer.WriteString("`")
			}
			writer.WriteByte(v)
			continue
		default:
			if shiftDelimiter-continuousBacktick <= 0 && !underQuoted {
				writer.WriteByte('`')
				underQuoted = true
				if selfQuoted = continuousBacktick > 0; selfQuoted {
					continuousBacktick -= 1
				}
			}

			for ; continuousBacktick > 0; continuousBacktick -= 1 {
				writer.WriteString("``")
			}

			writer.WriteByte(v)
		}
		shiftDelimiter++
	}

	if continuousBacktick > 0 && !selfQuoted {
		writer.WriteString("``")
	}
	writer.WriteString("`")
}

func (dialector Dialector) Explain(sql string, vars ...interface{}) string {
	return logger.ExplainSQL(sql, nil, `'`, vars...)
}

func (dialector Dialector) DataTypeOf(field *schema.Field) string {
	switch field.DataType {
	case schema.Bool:
		return "boolean"
	case schema.Int, schema.Uint:
		return dialector.getSchemaIntAndUnitType(field)
	case schema.Float:
		return dialector.getSchemaFloatType(field)
	case schema.String:
		return dialector.getSchemaStringType(field)
	case schema.Time:
		return dialector.getSchemaTimeType(field)
	case schema.Bytes:
		return dialector.getSchemaBytesType(field)
	}

	return string(field.DataType)
}

func (dialector Dialector) getSchemaFloatType(field *schema.Field) string {
	if field.Precision > 0 {
		return fmt.Sprintf("decimal(%d, %d)", field.Precision, field.Scale)
	}

	if field.Size <= 32 {
		return "float"
	}

	return "double"
}

func (dialector Dialector) getSchemaStringType(field *schema.Field) string {
	size := field.Size
	if size == 0 {
		if dialector.DefaultStringSize > 0 {
			size = int(dialector.DefaultStringSize)
		} else {
			hasIndex := field.TagSettings["INDEX"] != "" || field.TagSettings["UNIQUE"] != ""
			// TEXT, GEOMETRY or JSON column can't have a default value
			if field.PrimaryKey || field.HasDefaultValue || hasIndex {
				size = 191 // utf8mb4
			}
		}
	}

	if size >= 65536 && size <= int(math.Pow(2, 24)) {
		return "mediumtext"
	}

	if size > int(math.Pow(2, 24)) || size <= 0 {
		return "longtext"
	}

	return fmt.Sprintf("varchar(%d)", size)
}

func (dialector Dialector) getSchemaTimeType(field *schema.Field) string {
	precision := ""
	if !dialector.DisableDatetimePrecision && field.Precision == 0 {
		field.Precision = *dialector.DefaultDatetimePrecision
	}

	if field.Precision > 0 {
		precision = fmt.Sprintf("(%d)", field.Precision)
	}

	if field.NotNull || field.PrimaryKey {
		return "datetime" + precision
	}
	return "datetime" + precision + " NULL"
}

func (dialector Dialector) getSchemaBytesType(field *schema.Field) string {
	if field.Size > 0 && field.Size < 65536 {
		return fmt.Sprintf("varbinary(%d)", field.Size)
	}

	if field.Size >= 65536 && field.Size <= int(math.Pow(2, 24)) {
		return "mediumblob"
	}

	return "longblob"
}

func (dialector Dialector) getSchemaIntAndUnitType(field *schema.Field) string {
	sqlType := "bigint"
	switch {
	case field.Size <= 8:
		sqlType = "tinyint"
	case field.Size <= 16:
		sqlType = "smallint"
	case field.Size <= 24:
		sqlType = "mediumint"
	case field.Size <= 32:
		sqlType = "int"
	}

	if field.DataType == schema.Uint {
		sqlType += " unsigned"
	}

	if field.AutoIncrement {
		sqlType += " AUTO_INCREMENT"
	}

	return sqlType
}

func (dialector Dialector) SavePoint(tx *gorm.DB, name string) error {
	return tx.Exec("SAVEPOINT " + name).Error
}

func (dialector Dialector) RollbackTo(tx *gorm.DB, name string) error {
	return tx.Exec("ROLLBACK TO SAVEPOINT " + name).Error
}
<|endoftext|>


<commit_before>package app

import (
	"errors"
	"io"
	"log"
	"strconv"
	"time"

	"github.com/mesos/mesos-go/api/v1/lib"
	"github.com/mesos/mesos-go/api/v1/lib/backoff"
	xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules"
	"github.com/mesos/mesos-go/api/v1/lib/extras/store"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler/calls"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler/events"
)

var (
	RegistrationMinBackoff = 1 * time.Second
	RegistrationMaxBackoff = 15 * time.Second
)

// StateError is returned when the system encounters an unresolvable state transition error and
// should likely exit.
type StateError string

func (err StateError) Error() string { return string(err) }

func Run(cfg Config) error {
	log.Printf("scheduler running with configuration: %+v", cfg)
	shutdown := make(chan struct{})
	defer close(shutdown)

	state, err := newInternalState(cfg)
	if err != nil {
		return err
	}

	// TODO(jdef) how to track/handle timeout errors that occur for SUBSCRIBE calls? we should
	// probably tolerate X number of subsequent subscribe failures before bailing. we'll need
	// to track the lastCallAttempted along with subsequentSubscribeTimeouts.

	frameworkIDStore := store.DecorateSingleton(
		store.NewInMemorySingleton(),
		store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error {
			log.Println("FrameworkID", v)
			return nil
		}))

	state.cli = calls.Decorators{
		callMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics),
		logCalls(map[scheduler.Call_Type]string{scheduler.Call_SUBSCRIBE: "connecting..."}),
		calls.SubscribedCaller(store.GetIgnoreErrors(frameworkIDStore)), // automatically set the frameworkID for all outgoing calls
	}.Apply(state.cli)

	err = controller.Run(
		buildFrameworkInfo(state.config),
		state.cli,
		controller.WithDone(state.done.Closed),
		controller.WithEventHandler(
			buildEventHandler(state, frameworkIDStore),
			eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics),
			events.Decorator(logAllEvents).If(state.config.verbose),
		),
		controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)),
		controller.WithRegistrationTokens(
			backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, shutdown),
		),
		controller.WithSubscriptionTerminated(func(err error) {
			if err != nil {
				if err != io.EOF {
					log.Println(err)
				}
				if _, ok := err.(StateError); ok {
					state.done.Close()
				}
				return
			}
			log.Println("disconnected")
		}),
	)
	if state.err != nil {
		err = state.err
	}
	return err
}

// buildEventHandler generates and returns a handler to process events received from the subscription.
func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) events.Handler {
	logger := controller.LogEvents()
	return controller.LiftErrors().Handle(events.HandlerSet{
		scheduler.Event_FAILURE: logger.HandleF(failure),
		scheduler.Event_OFFERS: trackOffersReceived(state).AndThen().HandleF(
			func(e *scheduler.Event) error {
				if state.config.verbose {
					log.Println("received an OFFERS event")
				}
				return resourceOffers(state, e.GetOffers().GetOffers())
			}),
		scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)),
		scheduler.Event_SUBSCRIBED: eventrules.Rules{
			logger,
			controller.TrackSubscription(frameworkIDStore, state.config.failoverTimeout),
		},
	})
}

func trackOffersReceived(state *internalState) eventrules.Rule {
	return func(e *scheduler.Event, err error, chain eventrules.Chain) (*scheduler.Event, error) {
		if err == nil {
			state.metricsAPI.offersReceived.Int(len(e.GetOffers().GetOffers()))
		}
		return chain(e, nil)

	}
}

func failure(e *scheduler.Event) error {
	var (
		f              = e.GetFailure()
		eid, aid, stat = f.ExecutorID, f.AgentID, f.Status
	)
	if eid != nil {
		// executor failed..
		msg := "executor '" + eid.Value + "' terminated"
		if aid != nil {
			msg += " on agent '" + aid.Value + "'"
		}
		if stat != nil {
			msg += " with status=" + strconv.Itoa(int(*stat))
		}
		log.Println(msg)
	} else if aid != nil {
		// agent failed..
		log.Println("agent '" + aid.Value + "' terminated")
	}
	return nil
}

func resourceOffers(state *internalState, offers []mesos.Offer) error {
	callOption := calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds)
	tasksLaunchedThisCycle := 0
	offersDeclined := 0
	for i := range offers {
		var (
			remaining = mesos.Resources(offers[i].Resources)
			tasks     = []mesos.TaskInfo{}
		)

		if state.config.verbose {
			log.Println("received offer id '" + offers[i].ID.Value + "' with resources " + remaining.String())
		}

		var wantsExecutorResources mesos.Resources
		if len(offers[i].ExecutorIDs) == 0 {
			wantsExecutorResources = mesos.Resources(state.executor.Resources)
		}

		flattened := remaining.Flatten()

		// avoid the expense of computing these if we can...
		if state.config.summaryMetrics && state.config.resourceTypeMetrics {
			for name, restype := range flattened.Types() {
				if restype == mesos.SCALAR {
					sum := flattened.SumScalars(mesos.NamedResources(name))
					state.metricsAPI.offeredResources(sum.GetValue(), name)
				}
			}
		}

		taskWantsResources := state.wantsTaskResources.Plus(wantsExecutorResources...)
		for state.tasksLaunched < state.totalTasks && flattened.ContainsAll(taskWantsResources) {
			state.tasksLaunched++
			taskID := state.tasksLaunched

			if state.config.verbose {
				log.Println("launching task " + strconv.Itoa(taskID) + " using offer " + offers[i].ID.Value)
			}

			task := mesos.TaskInfo{
				TaskID:    mesos.TaskID{Value: strconv.Itoa(taskID)},
				AgentID:   offers[i].AgentID,
				Executor:  state.executor,
				Resources: remaining.Find(state.wantsTaskResources.Flatten(mesos.RoleName(state.role).Assign())),
			}
			task.Name = "Task " + task.TaskID.Value

			remaining.Subtract(task.Resources...)
			tasks = append(tasks, task)

			flattened = remaining.Flatten()
		}

		// build Accept call to launch all of the tasks we've assembled
		accept := calls.Accept(
			calls.OfferOperations{calls.OpLaunch(tasks...)}.WithOffers(offers[i].ID),
		).With(callOption)

		// send Accept call to mesos
		err := calls.CallNoData(state.cli, accept)
		if err != nil {
			log.Printf("failed to launch tasks: %+v", err)
		} else {
			if n := len(tasks); n > 0 {
				tasksLaunchedThisCycle += n
			} else {
				offersDeclined++
			}
		}
	}
	state.metricsAPI.offersDeclined.Int(offersDeclined)
	state.metricsAPI.tasksLaunched.Int(tasksLaunchedThisCycle)
	if state.config.summaryMetrics {
		state.metricsAPI.launchesPerOfferCycle(float64(tasksLaunchedThisCycle))
	}
	return nil
}

func statusUpdate(state *internalState) events.HandlerFunc {
	return func(e *scheduler.Event) error {
		s := e.GetUpdate().GetStatus()
		if state.config.verbose {
			msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String()
			if m := s.GetMessage(); m != "" {
				msg += " with message '" + m + "'"
			}
			log.Println(msg)
		}

		switch st := s.GetState(); st {
		case mesos.TASK_FINISHED:
			state.tasksFinished++
			state.metricsAPI.tasksFinished()

			if state.tasksFinished == state.totalTasks {
				log.Println("mission accomplished, terminating")
				state.done.Close()
			} else {
				tryReviveOffers(state)
			}

		case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR:
			state.err = errors.New("Exiting because task " + s.GetTaskID().Value +
				" is in an unexpected state " + st.String() +
				" with reason " + s.GetReason().String() +
				" from source " + s.GetSource().String() +
				" with message '" + s.GetMessage() + "'")
			state.done.Close()
		}
		return nil
	}
}

func tryReviveOffers(state *internalState) {
	// limit the rate at which we request offer revival
	select {
	case <-state.reviveTokens:
		// not done yet, revive offers!
		err := calls.CallNoData(state.cli, calls.Revive())
		if err != nil {
			log.Printf("failed to revive offers: %+v", err)
			return
		}
	default:
		// noop
	}
}

// logAllEvents logs every observed event; this is somewhat expensive to do
func logAllEvents(h events.Handler) events.Handler {
	return events.HandlerFunc(func(e *scheduler.Event) error {
		log.Printf("%+v\n", *e)
		return h.HandleEvent(e)
	})
}

// eventMetrics logs metrics for every processed API event
func eventMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) events.Decorator {
	timed := metricsAPI.eventReceivedLatency
	if !timingMetrics {
		timed = nil
	}
	harness := xmetrics.NewHarness(metricsAPI.eventReceivedCount, metricsAPI.eventErrorCount, timed, clock)
	return events.Metrics(harness)
}

// callMetrics logs metrics for every outgoing Mesos call
func callMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) calls.Decorator {
	timed := metricsAPI.callLatency
	if !timingMetrics {
		timed = nil
	}
	harness := xmetrics.NewHarness(metricsAPI.callCount, metricsAPI.callErrorCount, timed, clock)
	return calls.CallerMetrics(harness)
}

// logCalls logs a specific message string when a particular call-type is observed
func logCalls(messages map[scheduler.Call_Type]string) calls.Decorator {
	return func(caller calls.Caller) calls.Caller {
		return calls.CallerFunc(func(c *scheduler.Call) (mesos.Response, error) {
			if message, ok := messages[c.GetType()]; ok {
				log.Println(message)
			}
			return caller.Call(c)
		})
	}
}
<commit_msg>example-scheduler: tidy up event handler builder<commit_after>package app

import (
	"errors"
	"io"
	"log"
	"strconv"
	"time"

	"github.com/mesos/mesos-go/api/v1/lib"
	"github.com/mesos/mesos-go/api/v1/lib/backoff"
	xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules"
	"github.com/mesos/mesos-go/api/v1/lib/extras/store"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler/calls"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler/events"
)

var (
	RegistrationMinBackoff = 1 * time.Second
	RegistrationMaxBackoff = 15 * time.Second
)

// StateError is returned when the system encounters an unresolvable state transition error and
// should likely exit.
type StateError string

func (err StateError) Error() string { return string(err) }

func Run(cfg Config) error {
	log.Printf("scheduler running with configuration: %+v", cfg)
	shutdown := make(chan struct{})
	defer close(shutdown)

	state, err := newInternalState(cfg)
	if err != nil {
		return err
	}

	// TODO(jdef) how to track/handle timeout errors that occur for SUBSCRIBE calls? we should
	// probably tolerate X number of subsequent subscribe failures before bailing. we'll need
	// to track the lastCallAttempted along with subsequentSubscribeTimeouts.

	frameworkIDStore := store.DecorateSingleton(
		store.NewInMemorySingleton(),
		store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error {
			log.Println("FrameworkID", v)
			return nil
		}))

	state.cli = calls.Decorators{
		callMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics),
		logCalls(map[scheduler.Call_Type]string{scheduler.Call_SUBSCRIBE: "connecting..."}),
		calls.SubscribedCaller(store.GetIgnoreErrors(frameworkIDStore)), // automatically set the frameworkID for all outgoing calls
	}.Apply(state.cli)

	err = controller.Run(
		buildFrameworkInfo(state.config),
		state.cli,
		controller.WithDone(state.done.Closed),
		controller.WithEventHandler(
			buildEventHandler(state, frameworkIDStore),
			eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics),
			events.Decorator(logAllEvents).If(state.config.verbose),
		),
		controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)),
		controller.WithRegistrationTokens(
			backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, shutdown),
		),
		controller.WithSubscriptionTerminated(func(err error) {
			if err != nil {
				if err != io.EOF {
					log.Println(err)
				}
				if _, ok := err.(StateError); ok {
					state.done.Close()
				}
				return
			}
			log.Println("disconnected")
		}),
	)
	if state.err != nil {
		err = state.err
	}
	return err
}

// buildEventHandler generates and returns a handler to process events received from the subscription.
func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) events.Handler {
	logger := controller.LogEvents()
	return controller.LiftErrors().DropOnError().Handle(events.HandlerSet{
		scheduler.Event_FAILURE: logger.HandleF(failure),
		scheduler.Event_OFFERS: eventrules.Concat(
			trackOffersReceived(state),
			logger.If(state.config.verbose),
		).HandleF(resourceOffers(state)),
		scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)),
		scheduler.Event_SUBSCRIBED: eventrules.Rules{
			logger,
			controller.TrackSubscription(frameworkIDStore, state.config.failoverTimeout),
		},
	})
}

func trackOffersReceived(state *internalState) eventrules.Rule {
	return func(e *scheduler.Event, err error, chain eventrules.Chain) (*scheduler.Event, error) {
		if err == nil {
			state.metricsAPI.offersReceived.Int(len(e.GetOffers().GetOffers()))
		}
		return chain(e, nil)

	}
}

func failure(e *scheduler.Event) error {
	var (
		f              = e.GetFailure()
		eid, aid, stat = f.ExecutorID, f.AgentID, f.Status
	)
	if eid != nil {
		// executor failed..
		msg := "executor '" + eid.Value + "' terminated"
		if aid != nil {
			msg += " on agent '" + aid.Value + "'"
		}
		if stat != nil {
			msg += " with status=" + strconv.Itoa(int(*stat))
		}
		log.Println(msg)
	} else if aid != nil {
		// agent failed..
		log.Println("agent '" + aid.Value + "' terminated")
	}
	return nil
}

func resourceOffers(state *internalState) events.HandlerFunc {
	return func(e *scheduler.Event) error {
		var (
			offers                 = e.GetOffers().GetOffers()
			callOption             = calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds)
			tasksLaunchedThisCycle = 0
			offersDeclined         = 0
		)
		for i := range offers {
			var (
				remaining = mesos.Resources(offers[i].Resources)
				tasks     = []mesos.TaskInfo{}
			)

			if state.config.verbose {
				log.Println("received offer id '" + offers[i].ID.Value +
					"' with resources " + remaining.String())
			}

			var wantsExecutorResources mesos.Resources
			if len(offers[i].ExecutorIDs) == 0 {
				wantsExecutorResources = mesos.Resources(state.executor.Resources)
			}

			flattened := remaining.Flatten()

			// avoid the expense of computing these if we can...
			if state.config.summaryMetrics && state.config.resourceTypeMetrics {
				for name, restype := range flattened.Types() {
					if restype == mesos.SCALAR {
						sum := flattened.SumScalars(mesos.NamedResources(name))
						state.metricsAPI.offeredResources(sum.GetValue(), name)
					}
				}
			}

			taskWantsResources := state.wantsTaskResources.Plus(wantsExecutorResources...)
			for state.tasksLaunched < state.totalTasks && flattened.ContainsAll(taskWantsResources) {
				state.tasksLaunched++
				taskID := state.tasksLaunched

				if state.config.verbose {
					log.Println("launching task " + strconv.Itoa(taskID) + " using offer " + offers[i].ID.Value)
				}

				task := mesos.TaskInfo{
					TaskID:    mesos.TaskID{Value: strconv.Itoa(taskID)},
					AgentID:   offers[i].AgentID,
					Executor:  state.executor,
					Resources: remaining.Find(state.wantsTaskResources.Flatten(mesos.RoleName(state.role).Assign())),
				}
				task.Name = "Task " + task.TaskID.Value

				remaining.Subtract(task.Resources...)
				tasks = append(tasks, task)

				flattened = remaining.Flatten()
			}

			// build Accept call to launch all of the tasks we've assembled
			accept := calls.Accept(
				calls.OfferOperations{calls.OpLaunch(tasks...)}.WithOffers(offers[i].ID),
			).With(callOption)

			// send Accept call to mesos
			err := calls.CallNoData(state.cli, accept)
			if err != nil {
				log.Printf("failed to launch tasks: %+v", err)
			} else {
				if n := len(tasks); n > 0 {
					tasksLaunchedThisCycle += n
				} else {
					offersDeclined++
				}
			}
		}
		state.metricsAPI.offersDeclined.Int(offersDeclined)
		state.metricsAPI.tasksLaunched.Int(tasksLaunchedThisCycle)
		if state.config.summaryMetrics {
			state.metricsAPI.launchesPerOfferCycle(float64(tasksLaunchedThisCycle))
		}
		return nil
	}
}

func statusUpdate(state *internalState) events.HandlerFunc {
	return func(e *scheduler.Event) error {
		s := e.GetUpdate().GetStatus()
		if state.config.verbose {
			msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String()
			if m := s.GetMessage(); m != "" {
				msg += " with message '" + m + "'"
			}
			log.Println(msg)
		}

		switch st := s.GetState(); st {
		case mesos.TASK_FINISHED:
			state.tasksFinished++
			state.metricsAPI.tasksFinished()

			if state.tasksFinished == state.totalTasks {
				log.Println("mission accomplished, terminating")
				state.done.Close()
			} else {
				tryReviveOffers(state)
			}

		case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR:
			state.err = errors.New("Exiting because task " + s.GetTaskID().Value +
				" is in an unexpected state " + st.String() +
				" with reason " + s.GetReason().String() +
				" from source " + s.GetSource().String() +
				" with message '" + s.GetMessage() + "'")
			state.done.Close()
		}
		return nil
	}
}

func tryReviveOffers(state *internalState) {
	// limit the rate at which we request offer revival
	select {
	case <-state.reviveTokens:
		// not done yet, revive offers!
		err := calls.CallNoData(state.cli, calls.Revive())
		if err != nil {
			log.Printf("failed to revive offers: %+v", err)
			return
		}
	default:
		// noop
	}
}

// logAllEvents logs every observed event; this is somewhat expensive to do
func logAllEvents(h events.Handler) events.Handler {
	return events.HandlerFunc(func(e *scheduler.Event) error {
		log.Printf("%+v\n", *e)
		return h.HandleEvent(e)
	})
}

// eventMetrics logs metrics for every processed API event
func eventMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) events.Decorator {
	timed := metricsAPI.eventReceivedLatency
	if !timingMetrics {
		timed = nil
	}
	harness := xmetrics.NewHarness(metricsAPI.eventReceivedCount, metricsAPI.eventErrorCount, timed, clock)
	return events.Metrics(harness)
}

// callMetrics logs metrics for every outgoing Mesos call
func callMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) calls.Decorator {
	timed := metricsAPI.callLatency
	if !timingMetrics {
		timed = nil
	}
	harness := xmetrics.NewHarness(metricsAPI.callCount, metricsAPI.callErrorCount, timed, clock)
	return calls.CallerMetrics(harness)
}

// logCalls logs a specific message string when a particular call-type is observed
func logCalls(messages map[scheduler.Call_Type]string) calls.Decorator {
	return func(caller calls.Caller) calls.Caller {
		return calls.CallerFunc(func(c *scheduler.Call) (mesos.Response, error) {
			if message, ok := messages[c.GetType()]; ok {
				log.Println(message)
			}
			return caller.Call(c)
		})
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2017, 2018 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"go/doc"
	"strings"
)

type target struct {
	Target       string
	Phony        bool
	Dependencies []string
	MakeScript   string
}

type targetType interface {
	name() string
	help() string
	targets() []target
}

type helpTarget struct {
	getTargetTypes func() []targetType
}

func createHelpTarget(getTargetTypes func() []targetType) targetType {
	return &helpTarget{getTargetTypes}
}

func (*helpTarget) name() string {
	return "help"
}

func (*helpTarget) help() string {
	return "Display this help message. Unless overridden by the '--" +
		maketargetOption + "' option, this is the default target."
}

func (ht *helpTarget) targets() []target {
	script := `	@echo "Usage:"
	@echo "    make [target...]"
	@echo
	@echo "Global targets:"
`

	for _, t := range ht.getTargetTypes() {
		script += "\t@echo \"    " + t.name() + "\"\n"

		var buffer bytes.Buffer

		doc.ToText(&buffer, t.help(), "", "    ", 52)

		help := buffer.String()

		for _, l := range strings.Split(help, "\n") {
			if l != "" {
				script += "\t@echo \"        " + l + "\"\n"
			} else {
				script += "\t@echo\n"
			}
		}
	}

	return []target{{
		Target:     "help",
		Phony:      true,
		MakeScript: script}}
}

type bootstrapTarget struct {
	selection packageDefinitionList
}

func createBootstrapTarget(selection packageDefinitionList) targetType {
	return &bootstrapTarget{selection}
}

func (*bootstrapTarget) name() string {
	return "bootstrap"
}

func (*bootstrapTarget) help() string {
	return "Unconditionally regenerate the 'configure' " +
		"scripts for the selected packages."
}

func (bt *bootstrapTarget) targets() []target {
	prefix := "bootstrap_"

	var dependencies []string

	for _, pd := range bt.selection {
		dependencies = append(dependencies, prefix+pd.PackageName)
	}

	bootstrapTargets := []target{{
		Target:       "bootstrap",
		Phony:        true,
		Dependencies: dependencies,
	}}

	scriptTemplate := `	@echo "[bootstrap] %[1]s"
	@cd ` + privateDirName + "/" + pkgDirName + `/%[1]s && ./autogen.sh
`

	for i, pd := range bt.selection {
		bootstrapTargets = append(bootstrapTargets,
			target{
				Target: dependencies[i],
				Phony:  true,
				MakeScript: fmt.Sprintf(scriptTemplate,
					pd.PackageName),
			},
			target{
				Target: privateDirName + "/" + pkgDirName +
					"/" + pd.PackageName + "/configure",
				MakeScript: "	@$(MAKE) -s " +
					dependencies[i] + "\n",
			})
	}

	return bootstrapTargets
}

type configureTarget struct {
}

func createConfigureTarget() targetType {
	return &configureTarget{}
}

func (*configureTarget) name() string {
	return "configure"
}

func (*configureTarget) help() string {
	return "Configure the selected packages using the " +
		"current options specified in the 'conftab' file."
}

func (*configureTarget) targets() []target {
	return nil
}

type buildTarget struct {
}

func createBuildTarget() targetType {
	return &buildTarget{}
}

func (*buildTarget) name() string {
	return "build"
}

func (*buildTarget) help() string {
	return "Build (compile and link) the selected packages. " +
		"For the packages that have not been configured, the " +
		"configuration step will be performed automatically."
}

func (*buildTarget) targets() []target {
	return nil
}

type checkTarget struct {
}

func createCheckTarget() targetType {
	return &checkTarget{}
}

func (*checkTarget) name() string {
	return "check"
}

func (*checkTarget) help() string {
	return "Build and run unit tests for the selected packages."
}

func (*checkTarget) targets() []target {
	return nil
}
<commit_msg>Extract makeTargetData from bootstrapTarget<commit_after>// Copyright (C) 2017, 2018 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"go/doc"
	"strings"
)

type target struct {
	Target       string
	Phony        bool
	Dependencies []string
	MakeScript   string
}

type targetType interface {
	name() string
	help() string
	targets() []target
}

type helpTarget struct {
	getTargetTypes func() []targetType
}

func createHelpTarget(getTargetTypes func() []targetType) targetType {
	return &helpTarget{getTargetTypes}
}

func (*helpTarget) name() string {
	return "help"
}

func (*helpTarget) help() string {
	return "Display this help message. Unless overridden by the '--" +
		maketargetOption + "' option, this is the default target."
}

func (ht *helpTarget) targets() []target {
	script := `	@echo "Usage:"
	@echo "    make [target...]"
	@echo
	@echo "Global targets:"
`

	for _, t := range ht.getTargetTypes() {
		script += "\t@echo \"    " + t.name() + "\"\n"

		var buffer bytes.Buffer

		doc.ToText(&buffer, t.help(), "", "    ", 52)

		help := buffer.String()

		for _, l := range strings.Split(help, "\n") {
			if l != "" {
				script += "\t@echo \"        " + l + "\"\n"
			} else {
				script += "\t@echo\n"
			}
		}
	}

	return []target{{
		Target:     "help",
		Phony:      true,
		MakeScript: script}}
}

type makeTargetData struct {
	targetName string
	selection  packageDefinitionList
}

func (mtd *makeTargetData) name() string {
	return mtd.targetName
}

func (mtd *makeTargetData) globalTarget() target {
	prefix := "bootstrap_"

	var dependencies []string

	for _, pd := range mtd.selection {
		dependencies = append(dependencies, prefix+pd.PackageName)
	}

	return target{
		Target:       "bootstrap",
		Phony:        true,
		Dependencies: dependencies}
}

type bootstrapTarget struct {
	makeTargetData
}

func createBootstrapTarget(selection packageDefinitionList) targetType {
	return &bootstrapTarget{makeTargetData{"bootstrap", selection}}
}

func (*bootstrapTarget) help() string {
	return "Unconditionally regenerate the 'configure' " +
		"scripts for the selected packages."
}

func (bt *bootstrapTarget) targets() []target {
	globalTarget := bt.globalTarget()

	bootstrapTargets := []target{globalTarget}

	scriptTemplate := `	@echo "[bootstrap] %[1]s"
	@cd ` + privateDirName + "/" + pkgDirName + `/%[1]s && ./autogen.sh
`

	for i, pd := range bt.selection {
		bootstrapTargets = append(bootstrapTargets,
			target{
				Target: globalTarget.Dependencies[i],
				Phony:  true,
				MakeScript: fmt.Sprintf(scriptTemplate,
					pd.PackageName),
			},
			target{
				Target: privateDirName + "/" + pkgDirName +
					"/" + pd.PackageName + "/configure",
				MakeScript: "	@$(MAKE) -s " +
					globalTarget.Dependencies[i] + "\n",
			})
	}

	return bootstrapTargets
}

type configureTarget struct {
}

func createConfigureTarget() targetType {
	return &configureTarget{}
}

func (*configureTarget) name() string {
	return "configure"
}

func (*configureTarget) help() string {
	return "Configure the selected packages using the " +
		"current options specified in the 'conftab' file."
}

func (*configureTarget) targets() []target {
	return nil
}

type buildTarget struct {
}

func createBuildTarget() targetType {
	return &buildTarget{}
}

func (*buildTarget) name() string {
	return "build"
}

func (*buildTarget) help() string {
	return "Build (compile and link) the selected packages. " +
		"For the packages that have not been configured, the " +
		"configuration step will be performed automatically."
}

func (*buildTarget) targets() []target {
	return nil
}

type checkTarget struct {
}

func createCheckTarget() targetType {
	return &checkTarget{}
}

func (*checkTarget) name() string {
	return "check"
}

func (*checkTarget) help() string {
	return "Build and run unit tests for the selected packages."
}

func (*checkTarget) targets() []target {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"runtime"
	"strings"
	"time"

	"github.com/couchbaselabs/clog"
)

const vbucketCount=2

type NodeStatus struct {
	status  bool
	retries int
}

type vbucketMap struct {
	id int
	nodes []string
}

var (
	address string
	port    int
	logPath string
	hosts   string
	nodes   = make(map[string]NodeStatus)
	bucketMap = make(map[string]string)
)

func init() {

	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.StringVar(&address, "address", "", "Address to listen on, Default is to all")
	flag.IntVar(&port, "port", 8091, "Port to listen on. Default is 8091")
	flag.StringVar(&logPath, "path", "manager", "cluster manager logging dir")
	flag.StringVar(&hosts, "host", "localhost:11212", "nodes to manage")
	flag.Parse()

}

func Nodes(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(200)

	//onlineNodes := make([]string, 0)
	//onlineNodes := make(map[string]string)
	//for node, _ := range bucketMap {
	//	onlineNodes = append(onlineNodes, node)
	//}

	//oNodes, _ := json.Marshal(onlineNodes)
	oNodes, _ := json.Marshal(bucketMap)

	fmt.Fprintf(w, fmt.Sprintf("{\"nodes\":%s}", oNodes))
}

func main() {

	log.Printf("listening on %s:%d\n", address, port)
	log.Printf("cluster manager Path: %s\n", logPath)

	for _, host := range strings.Split(hosts, ",") {

		conn, err := net.Dial("tcp", host)
		defer conn.Close()

		nodes[host] = NodeStatus{status: true, retries: 0}
		if err != nil {
			clog.Error(err)
			nodes[host] = NodeStatus{status: false, retries: 0}
			break
		}
	}

	nodeCount := len(nodes)

	servers := make([]string, 0, nodeCount)
	for n := range nodes {
		servers = append(servers, n)
	}

	fmt.Printf("%#v\n", nodes)

	vbmap := make(map[int][]string)

	//TODO - fix it
	for i := 0; i < nodeCount; i++ {
		if i % 2 == 0 {
			vbmap[0] = append(vbmap[0], servers[i])
		} else {
			vbmap[1] = append(vbmap[1], servers[i])
		}
	}

	bucketMap["serverList"] = hosts
	bucketMap["luxmap"] = strings.Join(vbmap[0], ";") + "," + strings.Join(vbmap[1], ";")

	fmt.Println("vbmap:", bucketMap)

	//Polling nodes, needs cleanup
	go func() {
		for {
			for node, _ := range nodes {
				conn, err := net.Dial(node)
				defer conn.Close()
				nodes[node] = NodeStatus{status: true, retries: 0}

				if err != nil {
					clog.Error(err)
					retryCount := nodes[node].retries + 1

					if retryCount <= 3 {
						nodes[node] = NodeStatus{status: false, retries: retryCount}
					} else {
						delete(nodes, node)
					}
					break
				}
			}

			nodeCount := len(nodes)
			servers := make([]string, 0, nodeCount)
			for n := range nodes {
				servers = append(servers, n)
			}

			fmt.Printf("%#v\n", nodes)

			vbmap = make(map[int][]string)

			//TODO - fix it
			for i := 0; i < nodeCount; i++ {
				if i % 2 == 0 {
					vbmap[0] = append(vbmap[0], servers[i])
				} else {
					vbmap[1] = append(vbmap[1], servers[i])
				}
			}

			bucketMap["serverList"] = strings.Join(servers, ",")
			bucketMap["luxmap"] = strings.Join(vbmap[0], ";") + "," + strings.Join(vbmap[1], ";")
			//bucketMap["luxmap"] = "0:" + strings.Join(vbmap[0], ";" + ", 1:" + strings.Join(vbmap[1], ";")
			fmt.Printf("%#v\n", nodes)
			time.Sleep(time.Second)
		}
	}()

	http.HandleFunc("/nodes", Nodes)

	err := http.ListenAndServe(fmt.Sprintf("%s:%d", address, port), nil)
	if err != nil {
		log.Fatalf("Failed to start cluster manager: %v", err)
	}
}
<commit_msg>make sure vbmap is consistent on node failure<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"hash/fnv"
	"log"
	"net"
	"net/http"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/couchbaselabs/clog"
)

const vbucketCount=2

type NodeStatus struct {
	status  string
	retries int
}

type vbucketMap struct {
	id int
	nodes []string
}

var (
	address string
	port    int
	logPath string
	hosts   string
	nodes   = make(map[string]NodeStatus)
	bucketMap = make(map[string]string)
)

func init() {

	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.StringVar(&address, "address", "", "Address to listen on, Default is to all")
	flag.IntVar(&port, "port", 8091, "Port to listen on. Default is 8091")
	flag.StringVar(&logPath, "path", "manager", "cluster manager logging dir")
	flag.StringVar(&hosts, "host", "localhost:11212", "nodes to manage")
	flag.Parse()

}

func hash(s string) uint32 {
	h := fnv.New32a()
	h.Write([]byte(s))
	return h.Sum32()
}

func Nodes(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(200)

	//onlineNodes := make([]string, 0)
	//onlineNodes := make(map[string]string)
	//for node, _ := range bucketMap {
	//	onlineNodes = append(onlineNodes, node)
	//}

	//oNodes, _ := json.Marshal(onlineNodes)
	oNodes, _ := json.Marshal(bucketMap)

	fmt.Fprintf(w, fmt.Sprintf("{\"nodes\":%s}", oNodes))
}

func main() {

	log.Printf("listening on %s:%d\n", address, port)
	log.Printf("cluster manager Path: %s\n", logPath)

	for _, host := range strings.Split(hosts, ",") {

		//conn, err := net.Dial("tcp", host)
		_, err := net.Dial("tcp", host)
		//defer conn.Close()

		nodes[host] = NodeStatus{status: "up", retries: 0}
		if err != nil {
			clog.Error(err)
			nodes[host] = NodeStatus{status: "down", retries: 0}
			break
		}
	}

	nodeCount := len(nodes)

	servers := make([]string, 0, nodeCount)
	for n := range nodes {
		servers = append(servers, n)
	}
	sort.Strings(servers)

	fmt.Printf("%#v\n", nodes)

	vbmap := make(map[int][]string)

	//TODO - fix it
	for i := 0; i < nodeCount; i++ {
		if hash(servers[i])  % 2 == 0 {
			vbmap[0] = append(vbmap[0], servers[i])
		} else {
			vbmap[1] = append(vbmap[1], servers[i])
		}
	}

	bucketMap["serverList"] = hosts
	//bucketMap["luxmap"] = strings.Join(vbmap[0], ";") + "," + strings.Join(vbmap[1], ";")
	bucketMap["luxmap"] = "0:" + strings.Join(vbmap[0], ";") + ", 1:" + strings.Join(vbmap[1], ";")

	fmt.Println("vbmap:", bucketMap)

	//Polling nodes, needs cleanup
	go func() {
		for {
			for node, _ := range nodes {
				_, err := net.Dial("tcp", node)
				//conn, err := net.Dial("tcp", node)
				//defer conn.Close()

				if err != nil {
					clog.Error(err)
					fmt.Println("retry count:", nodes[node].retries, " node:", node)
					retryCount := nodes[node].retries + 1

					if retryCount <= 3 {
						nodes[node] = NodeStatus{status: "down", retries: retryCount}
					} else {
						delete(nodes, node)
					}
					break
				} else {
					nodes[node] = NodeStatus{status: "up", retries: 0}
				}
			}

			nodeCount := len(nodes)
			servers := make([]string, 0, nodeCount)
			for n := range nodes {
				servers = append(servers, n)
			}
			sort.Strings(servers)
			fmt.Printf("%#v\n", nodes)

			vbmap = make(map[int][]string)

			//TODO - fix it
			for i := 0; i < nodeCount; i++ {
				if hash(servers[i]) % 2 == 0 {
					vbmap[0] = append(vbmap[0], servers[i])
				} else {
					vbmap[1] = append(vbmap[1], servers[i])
				}
			}

			bucketMap["serverList"] = strings.Join(servers, ",")
			//bucketMap["luxmap"] = strings.Join(vbmap[0], ";") + "," + strings.Join(vbmap[1], ";")
			bucketMap["luxmap"] = "0:" + strings.Join(vbmap[0], ";") + ", 1:" + strings.Join(vbmap[1], ";")
			fmt.Printf("%#v\n", nodes)
			time.Sleep(time.Second)
		}
	}()

	http.HandleFunc("/nodes", Nodes)

	err := http.ListenAndServe(fmt.Sprintf("%s:%d", address, port), nil)
	if err != nil {
		log.Fatalf("Failed to start cluster manager: %v", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package progressutil

import (
	"fmt"
	"io"
	"sync"
	"time"
)

type copyReader struct {
	reader   io.Reader
	current  int64
	total    int64
	done     bool
	doneLock sync.Mutex
	pb       *ProgressBar
}

func (cr *copyReader) getDone() bool {
	cr.doneLock.Lock()
	val := cr.done
	cr.doneLock.Unlock()
	return val
}

func (cr *copyReader) setDone(val bool) {
	cr.doneLock.Lock()
	cr.done = val
	cr.doneLock.Unlock()
}

func (cr *copyReader) Read(p []byte) (int, error) {
	n, err := cr.reader.Read(p)
	cr.current += int64(n)
	err1 := cr.updateProgressBar()
	if err == nil {
		err = err1
	}
	return n, err
}

func (cr *copyReader) updateProgressBar() error {
	cr.pb.SetPrintAfter(cr.formattedProgress())

	progress := float64(cr.current) / float64(cr.total)
	if progress > 1 {
		progress = 1
	}
	return cr.pb.SetCurrentProgress(progress)
}

// CopyProgressPrinter will perform an arbitrary number of io.Copy calls, while
// continually printing the progress of each copy.
type CopyProgressPrinter struct {
	readers []*copyReader
	errors  []error
	lock    sync.Mutex
	pbp     *ProgressBarPrinter
}

// AddCopy adds a copy for this CopyProgressPrinter to perform. An io.Copy call
// will be made to copy bytes from reader to dest, and name and size will be
// used to label the progress bar and display how much progress has been made.
// If size is 0, the total size of the reader is assumed to be unknown.
func (cpp *CopyProgressPrinter) AddCopy(reader io.Reader, name string, size int64, dest io.Writer) {
	cpp.lock.Lock()
	if cpp.pbp == nil {
		cpp.pbp = &ProgressBarPrinter{}
		cpp.pbp.PadToBeEven = true
	}

	cr := &copyReader{
		reader:  reader,
		current: 0,
		total:   size,
		pb:      cpp.pbp.AddProgressBar(),
	}
	cr.pb.SetPrintBefore(name)
	cr.pb.SetPrintAfter(cr.formattedProgress())

	cpp.readers = append(cpp.readers, cr)
	cpp.lock.Unlock()

	go func() {
		_, err := io.Copy(dest, cr)
		if err != nil {
			cpp.lock.Lock()
			cpp.errors = append(cpp.errors, err)
			cpp.lock.Unlock()
		}
		cr.setDone(true)
	}()
}

// PrintAndWait will print the progress for each copy operation added with
// AddCopy to printTo every printInterval. This will continue until every added
// copy is finished, or until cancel is written to.
func (cpp *CopyProgressPrinter) PrintAndWait(printTo io.Writer, printInterval time.Duration, cancel chan struct{}) error {
	for {
		// If cancel is not nil, see if anything has been written to it. If
		// something has, return, otherwise keep drawing.
		if cancel != nil {
			select {
			case <-cancel:
				return nil
			default:
			}
		}

		cpp.lock.Lock()
		readers := cpp.readers
		errors := cpp.errors
		cpp.lock.Unlock()

		if len(errors) > 0 {
			return errors[0]
		}

		if len(readers) > 0 {
			_, err := cpp.pbp.Print(printTo)
			if err != nil {
				return err
			}
		} else {
		}

		allDone := true
		for _, r := range readers {
			allDone = allDone && r.getDone()
		}
		if allDone && len(readers) > 0 {
			return nil
		}

		time.Sleep(printInterval)
	}
}

func (cr *copyReader) formattedProgress() string {
	var totalStr string
	if cr.total == 0 {
		totalStr = "?"
	} else {
		totalStr = ByteUnitStr(cr.total)
	}
	return fmt.Sprintf("%s / %s", ByteUnitStr(cr.current), totalStr)
}

var byteUnits = []string{"B", "KB", "MB", "GB", "TB", "PB"}

// ByteUnitStr pretty prints a number of bytes.
func ByteUnitStr(n int64) string {
	var unit string
	size := float64(n)
	for i := 1; i < len(byteUnits); i++ {
		if size < 1000 {
			unit = byteUnits[i-1]
			break
		}

		size = size / 1000
	}

	return fmt.Sprintf("%.3g %s", size, unit)
}
<commit_msg>progressutil: remove extraneous else<commit_after>// Copyright 2016 CoreOS Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package progressutil

import (
	"fmt"
	"io"
	"sync"
	"time"
)

type copyReader struct {
	reader   io.Reader
	current  int64
	total    int64
	done     bool
	doneLock sync.Mutex
	pb       *ProgressBar
}

func (cr *copyReader) getDone() bool {
	cr.doneLock.Lock()
	val := cr.done
	cr.doneLock.Unlock()
	return val
}

func (cr *copyReader) setDone(val bool) {
	cr.doneLock.Lock()
	cr.done = val
	cr.doneLock.Unlock()
}

func (cr *copyReader) Read(p []byte) (int, error) {
	n, err := cr.reader.Read(p)
	cr.current += int64(n)
	err1 := cr.updateProgressBar()
	if err == nil {
		err = err1
	}
	return n, err
}

func (cr *copyReader) updateProgressBar() error {
	cr.pb.SetPrintAfter(cr.formattedProgress())

	progress := float64(cr.current) / float64(cr.total)
	if progress > 1 {
		progress = 1
	}
	return cr.pb.SetCurrentProgress(progress)
}

// CopyProgressPrinter will perform an arbitrary number of io.Copy calls, while
// continually printing the progress of each copy.
type CopyProgressPrinter struct {
	readers []*copyReader
	errors  []error
	lock    sync.Mutex
	pbp     *ProgressBarPrinter
}

// AddCopy adds a copy for this CopyProgressPrinter to perform. An io.Copy call
// will be made to copy bytes from reader to dest, and name and size will be
// used to label the progress bar and display how much progress has been made.
// If size is 0, the total size of the reader is assumed to be unknown.
func (cpp *CopyProgressPrinter) AddCopy(reader io.Reader, name string, size int64, dest io.Writer) {
	cpp.lock.Lock()
	if cpp.pbp == nil {
		cpp.pbp = &ProgressBarPrinter{}
		cpp.pbp.PadToBeEven = true
	}

	cr := &copyReader{
		reader:  reader,
		current: 0,
		total:   size,
		pb:      cpp.pbp.AddProgressBar(),
	}
	cr.pb.SetPrintBefore(name)
	cr.pb.SetPrintAfter(cr.formattedProgress())

	cpp.readers = append(cpp.readers, cr)
	cpp.lock.Unlock()

	go func() {
		_, err := io.Copy(dest, cr)
		if err != nil {
			cpp.lock.Lock()
			cpp.errors = append(cpp.errors, err)
			cpp.lock.Unlock()
		}
		cr.setDone(true)
	}()
}

// PrintAndWait will print the progress for each copy operation added with
// AddCopy to printTo every printInterval. This will continue until every added
// copy is finished, or until cancel is written to.
func (cpp *CopyProgressPrinter) PrintAndWait(printTo io.Writer, printInterval time.Duration, cancel chan struct{}) error {
	for {
		// If cancel is not nil, see if anything has been written to it. If
		// something has, return, otherwise keep drawing.
		if cancel != nil {
			select {
			case <-cancel:
				return nil
			default:
			}
		}

		cpp.lock.Lock()
		readers := cpp.readers
		errors := cpp.errors
		cpp.lock.Unlock()

		if len(errors) > 0 {
			return errors[0]
		}

		if len(readers) > 0 {
			_, err := cpp.pbp.Print(printTo)
			if err != nil {
				return err
			}
		}

		allDone := true
		for _, r := range readers {
			allDone = allDone && r.getDone()
		}
		if allDone && len(readers) > 0 {
			return nil
		}

		time.Sleep(printInterval)
	}
}

func (cr *copyReader) formattedProgress() string {
	var totalStr string
	if cr.total == 0 {
		totalStr = "?"
	} else {
		totalStr = ByteUnitStr(cr.total)
	}
	return fmt.Sprintf("%s / %s", ByteUnitStr(cr.current), totalStr)
}

var byteUnits = []string{"B", "KB", "MB", "GB", "TB", "PB"}

// ByteUnitStr pretty prints a number of bytes.
func ByteUnitStr(n int64) string {
	var unit string
	size := float64(n)
	for i := 1; i < len(byteUnits); i++ {
		if size < 1000 {
			unit = byteUnits[i-1]
			break
		}

		size = size / 1000
	}

	return fmt.Sprintf("%.3g %s", size, unit)
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "math/rand"
    "time"
    "strconv"
    "runtime"
    "sync"
)
var wg sync.WaitGroup

func main() {
  runtime.GOMAXPROCS(2)
  wg.Add(2)
  var mode, numTables, total, tries, avg int = 0, 0, 0, 0, 0
  fmt.Println("\n\tGame modes:")
  fmt.Println("1: Pick a random table")
  fmt.Println("2: Find the avg tries it takes to solve a puzzle\n")
  fmt.Println("Please choose 1 or 2: ")
  fmt.Scanf("%d", &mode)
  if mode == 1 {
    picker()
  } else if mode == 2 {
    fmt.Print("How many tables would you like to calculate for: ")
    fmt.Scanf("%d", &numTables)
    fmt.Print("How many times would you like to run the experiment: ")
    fmt.Scanf("%d", &tries)
    done := make(chan bool)
    go calc(done)
    go printAvg(numTables, total, avg, tries, done)
    wg.Wait()
  } else {
    fmt.Println("Goodbye!")
  }
}

func printAvg(numTables int, total int, avg int, tries int, done chan<- bool) {
  defer wg.Done()
  for i := 1; i <= tries; i++ {
    total += avgFinder(numTables)
  }
  close(done)
  avg = total/tries
  fmt.Println("\nThe avg tries it took to solve", numTables, "tables was", avg)
}

func calc(done <-chan bool) {
  defer wg.Done()
  for {
    for i := "Calculating";; i += "." {
      fmt.Printf("\r%s", i)
      time.Sleep(500 * time.Millisecond)
    }
    select {
      case _ = <- done:
        return
    }
  }
}

func printWelcome() {
  welcome := []string{"\nWelcome to Colin's Random Table Picker", "Tables are eliminated when number is drawn,", "And put back in the game when their number is drawn again.", "Last number left wins!", "Good Luck.\n"}
  for _, item := range welcome {
    fmt.Println(item)
    time.Sleep(1000 * time.Millisecond)
  }
}

func picker() {
  var tables, comp, wait int
  var cont bool = false
  var end bool = false
  var compString string
  printWelcome()
  for cont == false {
    fmt.Print("How many tables are playing: ")
    fmt.Scanf("%d", &tables)
    fmt.Print("How many ms would you like to wait between each iteration: ")
    fmt.Scanf("%d", &wait)
    if tables > 0 {
      cont = true
    }
  }
  tableArray := []string{}
  for i := 1; i <= tables; i++ {
    tableArray = append(tableArray, strconv.Itoa(i))
  }
  var total int = 0
  for end == false {
    r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
    comp = r.Intn(tables) + 1;
    compString = strconv.Itoa(comp)
    var numX int = 0
    total++
    fmt.Println("Random number is:", compString)
    if tableArray[comp - 1] == compString {
      tableArray[comp - 1] = "x"
    } else {
      tableArray[comp - 1] = compString
    }
    fmt.Println(tableArray)
    for _, element := range tableArray {
      if element == "x" {
        numX++
      }
    }
    if numX >= tables - 1 {
      end = true
      for _, element := range tableArray {
        if element != "x" {
          fmt.Println("The lucky winner is", element, "chosen after", total, "rounds.")
          break
        }
      }
    }
    time.Sleep(time.Duration(wait) * time.Millisecond)
  }
}

func avgFinder(tables int) int{
  var comp, total int = 0, 0
  var end bool = false
  var compString string
  tableArray := []string{}
  for i := 1; i <= int(tables); i++ {
    tableArray = append(tableArray, strconv.Itoa(i))
  }
  for end == false {
    r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
    comp = r.Intn(tables) + 1;
    compString = strconv.Itoa(comp)
    var numX int = 0
    total++
    if tableArray[comp - 1] == compString {
      tableArray[comp - 1] = "x"
    } else {
      tableArray[comp - 1] = compString
    }
    for _, element := range tableArray {
      if element == "x" {
        numX++
      }
    }
    if numX >= tables - 1 {
      break
    }
  }
  return total
}

/*
AVERAGES:
1:  0
2:  1
3:  2
4:  6
5:  11
6:  19
7:  34
8:  57
9:  95
10: 164
11: 283
12: 494
*/
<commit_msg>Removed done channel, fixed termination of calc()<commit_after>package main

import (
    "fmt"
    "math/rand"
    "time"
    "strconv"
    "runtime"
    "sync"
)
var wg sync.WaitGroup

func main() {
  runtime.GOMAXPROCS(2)
  wg.Add(1)
  var mode, numTables, total, tries, avg int = 0, 0, 0, 0, 0
  fmt.Println("\n\tGame modes:")
  fmt.Println("1: Pick a random table")
  fmt.Println("2: Find the avg tries it takes to solve a puzzle\n")
  fmt.Println("Please choose 1 or 2: ")
  fmt.Scanf("%d", &mode)
  if mode == 1 {
    picker()
  } else if mode == 2 {
    fmt.Print("How many tables would you like to calculate for: ")
    fmt.Scanf("%d", &numTables)
    fmt.Print("How many times would you like to run the experiment: ")
    fmt.Scanf("%d", &tries)
    fmt.Println("\n")
    go calc()
    go printAvg(numTables, total, avg, tries, )
    wg.Wait()
  } else {
    fmt.Println("Goodbye!")
  }
}

func printAvg(numTables int, total int, avg int, tries int) {
  defer wg.Done()
  for i := 1; i <= tries; i++ {
    total += avgFinder(numTables)
  }
  avg = total/tries
  fmt.Printf("\rThe avg tries it took to solve %v tables was %v\n\n", numTables, avg)
}

func calc() {
  for {
    for i := "Calculating";; i += "." {
      fmt.Printf("\r%s", i)
      time.Sleep(500 * time.Millisecond)
    }
  }
}

func printWelcome() {
  welcome := []string{"\nWelcome to Colin's Random Table Picker", "Tables are eliminated when number is drawn,", "And put back in the game when their number is drawn again.", "Last number left wins!", "Good Luck.\n"}
  for _, item := range welcome {
    fmt.Println(item)
    time.Sleep(900 * time.Millisecond)
  }
}

func picker() {
  var tables, comp, wait int
  var cont bool = false
  var end bool = false
  var compString string
  printWelcome()
  for cont == false {
    fmt.Print("How many tables are playing: ")
    fmt.Scanf("%d", &tables)
    fmt.Print("How many ms would you like to wait between each iteration: ")
    fmt.Scanf("%d", &wait)
    if tables > 0 {
      cont = true
    }
  }
  tableArray := []string{}
  for i := 1; i <= tables; i++ {
    tableArray = append(tableArray, strconv.Itoa(i))
  }
  var total int = 0
  for end == false {
    r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
    comp = r.Intn(tables) + 1;
    compString = strconv.Itoa(comp)
    var numX int = 0
    total++
    fmt.Println("Random number is:", compString)
    if tableArray[comp - 1] == compString {
      tableArray[comp - 1] = "x"
    } else {
      tableArray[comp - 1] = compString
    }
    fmt.Println(tableArray)
    for _, element := range tableArray {
      if element == "x" {
        numX++
      }
    }
    if numX >= tables - 1 {
      end = true
      for _, element := range tableArray {
        if element != "x" {
          fmt.Println("The lucky winner is", element, "chosen after", total, "rounds.")
          break
        }
      }
    }
    time.Sleep(time.Duration(wait) * time.Millisecond)
  }
}

func avgFinder(tables int) int{
  var comp, total int = 0, 0
  var end bool = false
  var compString string
  tableArray := []string{}
  for i := 1; i <= int(tables); i++ {
    tableArray = append(tableArray, strconv.Itoa(i))
  }
  for end == false {
    r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
    comp = r.Intn(tables) + 1;
    compString = strconv.Itoa(comp)
    var numX int = 0
    total++
    if tableArray[comp - 1] == compString {
      tableArray[comp - 1] = "x"
    } else {
      tableArray[comp - 1] = compString
    }
    for _, element := range tableArray {
      if element == "x" {
        numX++
      }
    }
    if numX >= tables - 1 {
      break
    }
  }
  return total
}

/*
AVERAGES:
1:  0
2:  1
3:  2
4:  6
5:  11
6:  19
7:  34
8:  57
9:  95
10: 164
11: 283
12: 494
*/
<|endoftext|>


<commit_before>package util

import (
	"io/ioutil"
	"testing"
)

func ReadFixture(t *testing.T, path string) []byte {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatalf("can't read fixture %s: %v", path, err)
	}
	return b
}
<commit_msg>[util][test] Add ReadJson<commit_after>package util

import (
	"io/ioutil"
	"testing"
	"encoding/json"
)

func ReadFixture(t *testing.T, path string) []byte {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatalf("can't read fixture %s: %v", path, err)
	}
	return b
}

func ReadJson(t *testing.T, path string, v interface{}) {
	b := ReadFixture(t, path)
	if err := json.Unmarshal(b, v); err != nil {
		t.Fatalf("can't unmarshal fixture %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/mocks"
	"github.com/urfave/cli"
)

var defVM lib.VirtualMachineName
var defGroup lib.GroupName

func baseTestSetup() (config *mocks.Config, client *mocks.Client) {
	config = new(mocks.Config)
	client = new(mocks.Client)
	global.Client = client
	global.Config = config

	baseAppSetup()
	return
}

func traverseAllCommands(cmds []cli.Command, fn func(cli.Command)) {
	if cmds == nil {
		return
	}
	for _, c := range cmds {
		fn(c)
		traverseAllCommands(c.Subcommands, fn)
	}
}

func getFixtureVM() lib.VirtualMachine {
	return lib.VirtualMachine{
		Name:     "test-server",
		Hostname: "test-server.test-group",
		GroupID:  1,
		ZoneName: "test-zone",
	}
}

func getFixtureGroup() lib.Group {
	vms := make([]*lib.VirtualMachine, 1, 1)
	vm := getFixtureVM()
	vms[0] = &vm

	return lib.Group{
		Name:            "test-group",
		VirtualMachines: vms,
	}
}
<commit_msg>Revert "Add ZoneName to FixtureVM"<commit_after>package main

import (
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/mocks"
	"github.com/urfave/cli"
)

var defVM lib.VirtualMachineName
var defGroup lib.GroupName

func baseTestSetup() (config *mocks.Config, client *mocks.Client) {
	config = new(mocks.Config)
	client = new(mocks.Client)
	global.Client = client
	global.Config = config

	baseAppSetup()
	return
}

func traverseAllCommands(cmds []cli.Command, fn func(cli.Command)) {
	if cmds == nil {
		return
	}
	for _, c := range cmds {
		fn(c)
		traverseAllCommands(c.Subcommands, fn)
	}
}

func getFixtureVM() lib.VirtualMachine {
	return lib.VirtualMachine{
		Name:     "test-server",
		Hostname: "test-server.test-group",
		GroupID:  1,
	}
}

func getFixtureGroup() lib.Group {
	vms := make([]*lib.VirtualMachine, 1, 1)
	vm := getFixtureVM()
	vms[0] = &vm

	return lib.Group{
		Name:            "test-group",
		VirtualMachines: vms,
	}
}
<|endoftext|>


<commit_before>// +build !go1.6

package shellwords

import (
	"os"
	"os/exec"
	"runtime"
	"strings"
)

func shellRun(line, dir string) (string, error) {
	var b []byte
	var err error
	var cmd *exec.Cmd
	if runtime.GOOS == "windows" {
		cmd = exec.Command(os.Getenv("COMSPEC"), "/c", line)
	} else {
		cmd = exec.Command(os.Getenv("SHELL"), "-c", line)
	}
	if dir != "" {
		cmd.Dir = dir
	}
	b, err = cmd.Output()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(b)), nil
}
<commit_msg>Drop go1.5<commit_after><|endoftext|>


<commit_before>package graphql_test

import (
	"encoding/json"
	"log"
	"reflect"
	"testing"

	"github.com/graphql-go/graphql"
	"github.com/graphql-go/graphql/testutil"
)

type Person struct {
	Human
	Name    string   `json:"name"`
	Home    Address  `json:"home"`
	Friends []Friend `json:"friends"`
}

type Human struct {
	Alive  bool    `json:"alive"`
	Age    int     `json:"age"`
	Weight float64 `json:"weight"`
}

type Friend struct {
	Name    string `json:"name"`
	Address string `json:"address"`
}

type Address struct {
	Street string `json:"street"`
	City   string `json:"city"`
}

var personSource = Person{
	Human: Human{
		Age:    24,
		Weight: 70.1,
		Alive:  true,
	},
	Name: "John Doe",
	Home: Address{
		Street: "Jl. G1",
		City:   "Jakarta",
	},
	Friends: friendSource,
}

var friendSource = []Friend{
	{Name: "Arief", Address: "palembang"},
	{Name: "Al", Address: "semarang"},
}

func TestBindFields(t *testing.T) {
	// create person type based on Person struct
	personType := graphql.NewObject(graphql.ObjectConfig{
		Name: "Person",
		// pass empty Person struct to bind all of it's fields
		Fields: graphql.BindFields(Person{}),
	})
	fields := graphql.Fields{
		"person": &graphql.Field{
			Type: personType,
			Resolve: func(p graphql.ResolveParams) (interface{}, error) {
				return personSource, nil
			},
		},
	}
	rootQuery := graphql.ObjectConfig{Name: "RootQuery", Fields: fields}
	schemaConfig := graphql.SchemaConfig{Query: graphql.NewObject(rootQuery)}
	schema, err := graphql.NewSchema(schemaConfig)
	if err != nil {
		log.Fatalf("failed to create new schema, error: %v", err)
	}

	// Query
	query := `
		{
			person{
				name,
				home{street,city},
				friends{name,address},
				age,
				weight,
				alive
			}
		}
	`
	params := graphql.Params{Schema: schema, RequestString: query}
	r := graphql.Do(params)
	if len(r.Errors) > 0 {
		log.Fatalf("failed to execute graphql operation, errors: %+v", r.Errors)
	}

	rJSON, _ := json.Marshal(r)
	data := struct {
		Data struct {
			Person Person `json:"person"`
		} `json:"data"`
	}{}
	err = json.Unmarshal(rJSON, &data)
	if err != nil {
		log.Fatalf("failed to unmarshal. error: %v", err)
	}

	newPerson := data.Data.Person
	if !reflect.DeepEqual(newPerson, personSource) {
		t.Fatalf("Unexpected result, Diff: %v", testutil.Diff(personSource, newPerson))
	}
}

func TestBindArg(t *testing.T) {
	var friendObj = graphql.NewObject(graphql.ObjectConfig{
		Name:   "friend",
		Fields: graphql.BindFields(Friend{}),
	})

	fields := graphql.Fields{
		"friend": &graphql.Field{
			Type: friendObj,
			//it can be added more than one since it's a slice
			Args: graphql.BindArg(Friend{}, "name"),
			Resolve: func(p graphql.ResolveParams) (interface{}, error) {
				if name, ok := p.Args["name"].(string); ok {
					for _, friend := range friendSource {
						if friend.Name == name {
							return friend, nil
						}
					}
				}
				return nil, nil
			},
		},
	}
	rootQuery := graphql.ObjectConfig{Name: "RootQuery", Fields: fields}
	schemaConfig := graphql.SchemaConfig{Query: graphql.NewObject(rootQuery)}
	schema, err := graphql.NewSchema(schemaConfig)
	if err != nil {
		log.Fatalf("failed to create new schema, error: %v", err)
	}

	// Query
	query := `
		{
			friend(name:"Arief"){
				address
			}
		}
	`
	params := graphql.Params{Schema: schema, RequestString: query}
	r := graphql.Do(params)
	if len(r.Errors) > 0 {
		log.Fatalf("failed to execute graphql operation, errors: %+v", r.Errors)
	}

	rJSON, _ := json.Marshal(r)

	data := struct {
		Data struct {
			Friend Friend `json:"friend"`
		} `json:"data"`
	}{}
	err = json.Unmarshal(rJSON, &data)
	if err != nil {
		log.Fatalf("failed to unmarshal. error: %v", err)
	}

	expectedAddress := "palembang"
	newFriend := data.Data.Friend
	if newFriend.Address != expectedAddress {
		t.Fatalf("Unexpected result, expected address to be %s but got %s", expectedAddress, newFriend.Address)
	}
}
<commit_msg>add scalar slice<commit_after>package graphql_test

import (
	"encoding/json"
	"log"
	"reflect"
	"testing"

	"github.com/graphql-go/graphql"
	"github.com/graphql-go/graphql/testutil"
)

type Person struct {
	Human
	Name    string   `json:"name"`
	Home    Address  `json:"home"`
	Hobbies []string `json:"hobbies"`
	Friends []Friend `json:"friends"`
}

type Human struct {
	Alive  bool    `json:"alive"`
	Age    int     `json:"age"`
	Weight float64 `json:"weight"`
}

type Friend struct {
	Name    string `json:"name"`
	Address string `json:"address"`
}

type Address struct {
	Street string `json:"street"`
	City   string `json:"city"`
}

var personSource = Person{
	Human: Human{
		Age:    24,
		Weight: 70.1,
		Alive:  true,
	},
	Name: "John Doe",
	Home: Address{
		Street: "Jl. G1",
		City:   "Jakarta",
	},
	Friends: friendSource,
	Hobbies:[]string{"eat","sleep","code"},
}

var friendSource = []Friend{
	{Name: "Arief", Address: "palembang"},
	{Name: "Al", Address: "semarang"},
}

func TestBindFields(t *testing.T) {
	// create person type based on Person struct
	personType := graphql.NewObject(graphql.ObjectConfig{
		Name: "Person",
		// pass empty Person struct to bind all of it's fields
		Fields: graphql.BindFields(Person{}),
	})
	fields := graphql.Fields{
		"person": &graphql.Field{
			Type: personType,
			Resolve: func(p graphql.ResolveParams) (interface{}, error) {
				return personSource, nil
			},
		},
	}
	rootQuery := graphql.ObjectConfig{Name: "RootQuery", Fields: fields}
	schemaConfig := graphql.SchemaConfig{Query: graphql.NewObject(rootQuery)}
	schema, err := graphql.NewSchema(schemaConfig)
	if err != nil {
		log.Fatalf("failed to create new schema, error: %v", err)
	}

	// Query
	query := `
		{
			person{
				name,
				home{street,city},
				friends{name,address},
				age,
				weight,
				alive,
				hobbies
			}
		}
	`
	params := graphql.Params{Schema: schema, RequestString: query}
	r := graphql.Do(params)
	if len(r.Errors) > 0 {
		log.Fatalf("failed to execute graphql operation, errors: %+v", r.Errors)
	}

	rJSON, _ := json.Marshal(r)
	data := struct {
		Data struct {
			Person Person `json:"person"`
		} `json:"data"`
	}{}
	err = json.Unmarshal(rJSON, &data)
	if err != nil {
		log.Fatalf("failed to unmarshal. error: %v", err)
	}

	newPerson := data.Data.Person
	if !reflect.DeepEqual(newPerson, personSource) {
		t.Fatalf("Unexpected result, Diff: %v", testutil.Diff(personSource, newPerson))
	}
}

func TestBindArg(t *testing.T) {
	var friendObj = graphql.NewObject(graphql.ObjectConfig{
		Name:   "friend",
		Fields: graphql.BindFields(Friend{}),
	})

	fields := graphql.Fields{
		"friend": &graphql.Field{
			Type: friendObj,
			//it can be added more than one since it's a slice
			Args: graphql.BindArg(Friend{}, "name"),
			Resolve: func(p graphql.ResolveParams) (interface{}, error) {
				if name, ok := p.Args["name"].(string); ok {
					for _, friend := range friendSource {
						if friend.Name == name {
							return friend, nil
						}
					}
				}
				return nil, nil
			},
		},
	}
	rootQuery := graphql.ObjectConfig{Name: "RootQuery", Fields: fields}
	schemaConfig := graphql.SchemaConfig{Query: graphql.NewObject(rootQuery)}
	schema, err := graphql.NewSchema(schemaConfig)
	if err != nil {
		log.Fatalf("failed to create new schema, error: %v", err)
	}

	// Query
	query := `
		{
			friend(name:"Arief"){
				address
			}
		}
	`
	params := graphql.Params{Schema: schema, RequestString: query}
	r := graphql.Do(params)
	if len(r.Errors) > 0 {
		log.Fatalf("failed to execute graphql operation, errors: %+v", r.Errors)
	}

	rJSON, _ := json.Marshal(r)

	data := struct {
		Data struct {
			Friend Friend `json:"friend"`
		} `json:"data"`
	}{}
	err = json.Unmarshal(rJSON, &data)
	if err != nil {
		log.Fatalf("failed to unmarshal. error: %v", err)
	}

	expectedAddress := "palembang"
	newFriend := data.Data.Friend
	if newFriend.Address != expectedAddress {
		t.Fatalf("Unexpected result, expected address to be %s but got %s", expectedAddress, newFriend.Address)
	}
}
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	"github.com/ryanuber/columnize"
)

// consul members will include:
// - zk cluster as server
// - agents
//   - brokers
//   - kateway
type Members struct {
	Ui  cli.Ui
	Cmd string

	brokerHosts, zkHosts, katewayHosts map[string]struct{}
	nodeHostMap                        map[string]string // consul members node->ip
}

func (this *Members) Run(args []string) (exitCode int) {
	var (
		zone        string
		showLoadAvg bool
	)
	cmdFlags := flag.NewFlagSet("members", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&showLoadAvg, "l", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.fillTheHosts(zkzone)

	consulLiveNode, consulDeadNodes := this.consulMembers()
	for _, node := range consulDeadNodes {
		this.Ui.Error(fmt.Sprintf("%s consul dead", node))
	}

	consulLiveMap := make(map[string]struct{})
	brokerN, zkN, katewayN, unknownN := 0, 0, 0, 0
	for _, node := range consulLiveNode {
		_, presentInBroker := this.brokerHosts[node]
		_, presentInZk := this.zkHosts[node]
		_, presentInKateway := this.katewayHosts[node]
		if presentInBroker {
			brokerN++
		}
		if presentInZk {
			zkN++
		}
		if presentInKateway {
			katewayN++
		}

		if !presentInBroker && !presentInZk && !presentInKateway {
			unknownN++
		}

		consulLiveMap[node] = struct{}{}
	}

	// all brokers should run consul
	for broker, _ := range this.brokerHosts {
		if _, present := consulLiveMap[broker]; !present {
			this.Ui.Warn(fmt.Sprintf("- %s", broker))
		}
	}

	if showLoadAvg {
		this.displayLoadAvg()
	}

	this.Ui.Output(fmt.Sprintf("zk:%s broker:%s kateway:%s ?:%s",
		color.Magenta("%d", zkN),
		color.Magenta("%d", brokerN),
		color.Magenta("%d", katewayN),
		color.Green("%d", unknownN)))

	return
}

func (this *Members) fillTheHosts(zkzone *zk.ZkZone) {
	this.brokerHosts = make(map[string]struct{})
	zkzone.ForSortedBrokers(func(cluster string, brokers map[string]*zk.BrokerZnode) {
		for _, brokerInfo := range brokers {
			this.brokerHosts[brokerInfo.Host] = struct{}{}
		}
	})

	this.zkHosts = make(map[string]struct{})
	for _, addr := range zkzone.ZkAddrList() {
		zkNode, _, err := net.SplitHostPort(addr)
		swallow(err)
		this.zkHosts[zkNode] = struct{}{}
	}

	this.katewayHosts = make(map[string]struct{})
	kws, err := zkzone.KatewayInfos()
	swallow(err)
	for _, kw := range kws {
		host, _, err := net.SplitHostPort(kw.PubAddr)
		swallow(err)
		this.katewayHosts[host] = struct{}{}
	}
}

func (this *Members) displayLoadAvg() {
	cmd := pipestream.New("consul", "exec",
		"uptime", "|", "grep", "load")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	lines := make([]string, 0)
	header := "Node|Host|Role|Load Avg"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		host := this.nodeHostMap[node]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, this.roleOfHost(host), parts[1]))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) roleOfHost(host string) string {
	if _, present := this.brokerHosts[host]; present {
		return "B"
	}
	if _, present := this.zkHosts[host]; present {
		return "Z"
	}
	if _, present := this.katewayHosts[host]; present {
		return "K"
	}
	return "?"
}

func (this *Members) consulMembers() ([]string, []string) {
	cmd := pipestream.New("consul", "members")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	liveHosts, deadHosts := []string{}, []string{}
	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	this.nodeHostMap = make(map[string]string)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "Protocol") {
			// the header
			continue
		}

		fields := strings.Fields(scanner.Text())
		node, addr, alive := fields[0], fields[1], fields[2]
		host, _, err := net.SplitHostPort(addr)
		swallow(err)

		this.nodeHostMap[node] = host

		if alive == "alive" {
			liveHosts = append(liveHosts, host)
		} else {
			deadHosts = append(deadHosts, host)
		}
	}

	return liveHosts, deadHosts
}

func (*Members) Synopsis() string {
	return "Verify consul members match kafka zone"
}

func (this *Members) Help() string {
	help := fmt.Sprintf(`
Usage: %s members [options]

    Verify consul members match kafka zone

    -z zone

    -l
      Display each member load average

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>exec cmd on all hosts<commit_after>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	"github.com/ryanuber/columnize"
)

// consul members will include:
// - zk cluster as server
// - agents
//   - brokers
//   - kateway
type Members struct {
	Ui  cli.Ui
	Cmd string

	brokerHosts, zkHosts, katewayHosts map[string]struct{}
	nodeHostMap                        map[string]string // consul members node->ip
}

func (this *Members) Run(args []string) (exitCode int) {
	var (
		zone        string
		showLoadAvg bool
		exec        string
	)
	cmdFlags := flag.NewFlagSet("members", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&showLoadAvg, "l", false, "")
	cmdFlags.StringVar(&exec, "exec", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.fillTheHosts(zkzone)

	consulLiveNode, consulDeadNodes := this.consulMembers()
	for _, node := range consulDeadNodes {
		this.Ui.Error(fmt.Sprintf("%s consul dead", node))
	}

	consulLiveMap := make(map[string]struct{})
	brokerN, zkN, katewayN, unknownN := 0, 0, 0, 0
	for _, node := range consulLiveNode {
		_, presentInBroker := this.brokerHosts[node]
		_, presentInZk := this.zkHosts[node]
		_, presentInKateway := this.katewayHosts[node]
		if presentInBroker {
			brokerN++
		}
		if presentInZk {
			zkN++
		}
		if presentInKateway {
			katewayN++
		}

		if !presentInBroker && !presentInZk && !presentInKateway {
			unknownN++
		}

		consulLiveMap[node] = struct{}{}
	}

	// all brokers should run consul
	for broker, _ := range this.brokerHosts {
		if _, present := consulLiveMap[broker]; !present {
			this.Ui.Warn(fmt.Sprintf("- %s", broker))
		}
	}

	switch {
	case showLoadAvg:
		this.displayLoadAvg()

	case exec != "":
		this.executeOnAll(exec)
	}

	// summary
	this.Ui.Output(fmt.Sprintf("zk:%s broker:%s kateway:%s ?:%s",
		color.Magenta("%d", zkN),
		color.Magenta("%d", brokerN),
		color.Magenta("%d", katewayN),
		color.Green("%d", unknownN)))

	return
}

func (this *Members) fillTheHosts(zkzone *zk.ZkZone) {
	this.brokerHosts = make(map[string]struct{})
	zkzone.ForSortedBrokers(func(cluster string, brokers map[string]*zk.BrokerZnode) {
		for _, brokerInfo := range brokers {
			this.brokerHosts[brokerInfo.Host] = struct{}{}
		}
	})

	this.zkHosts = make(map[string]struct{})
	for _, addr := range zkzone.ZkAddrList() {
		zkNode, _, err := net.SplitHostPort(addr)
		swallow(err)
		this.zkHosts[zkNode] = struct{}{}
	}

	this.katewayHosts = make(map[string]struct{})
	kws, err := zkzone.KatewayInfos()
	swallow(err)
	for _, kw := range kws {
		host, _, err := net.SplitHostPort(kw.PubAddr)
		swallow(err)
		this.katewayHosts[host] = struct{}{}
	}
}

func (this *Members) executeOnAll(execCmd string) {
	args := []string{"exec"}
	args = append(args, strings.Split(execCmd, " ")...)
	cmd := pipestream.New("consul", args...)
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	this.Ui.Info(fmt.Sprintf("%s ...", execCmd))

	lines := make([]string, 0)
	header := "Node|Host|Role|Result"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "finished with exit code 0") ||
			strings.Contains(line, "completed / acknowledged") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) == 1 {
			continue
		}

		node := fields[0]
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		host := this.nodeHostMap[node]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, this.roleOfHost(host),
			strings.Join(fields[1:], " ")))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) displayLoadAvg() {
	cmd := pipestream.New("consul", "exec",
		"uptime", "|", "grep", "load")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	lines := make([]string, 0)
	header := "Node|Host|Role|Load Avg"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		host := this.nodeHostMap[node]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, this.roleOfHost(host), parts[1]))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) roleOfHost(host string) string {
	if _, present := this.brokerHosts[host]; present {
		return "B"
	}
	if _, present := this.zkHosts[host]; present {
		return "Z"
	}
	if _, present := this.katewayHosts[host]; present {
		return "K"
	}
	return "?"
}

func (this *Members) consulMembers() ([]string, []string) {
	cmd := pipestream.New("consul", "members")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	liveHosts, deadHosts := []string{}, []string{}
	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	this.nodeHostMap = make(map[string]string)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "Protocol") {
			// the header
			continue
		}

		fields := strings.Fields(scanner.Text())
		node, addr, alive := fields[0], fields[1], fields[2]
		host, _, err := net.SplitHostPort(addr)
		swallow(err)

		this.nodeHostMap[node] = host

		if alive == "alive" {
			liveHosts = append(liveHosts, host)
		} else {
			deadHosts = append(deadHosts, host)
		}
	}

	return liveHosts, deadHosts
}

func (*Members) Synopsis() string {
	return "Verify consul members match kafka zone"
}

func (this *Members) Help() string {
	help := fmt.Sprintf(`
Usage: %s members [options]

    Verify consul members match kafka zone

    -z zone

    -l
      Display each member load average

    -exec <cmd>
      Execute cmd on all members and print the result by host
      e,g. gk members -exec "ifconfig bond0 | grep 'TX bytes'"

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"os"
)

var showHelp bool

func main() {
	validateCommandLine()
	anyBagFailed := false
	for i := 1; i < len(os.Args); i++ {
		filePath := os.Args[i]
		validator, err := bagman.NewValidator(filePath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error creating validator for %s: %s\n", filePath, err)
			os.Exit(1)
		}
		if validator.IsValid() {
			fmt.Printf("[PASS] %s is a valid APTrust bag\n", filePath)
		} else {
			fmt.Printf("[FAIL] %s is not valid for the following reasons:\n", filePath)
			fmt.Println(" ", validator.ErrorMessage)
			anyBagFailed = true
		}
	}
	if anyBagFailed {
		fmt.Println("")
		printSpecUrl()
		os.Exit(1)
	}
}

func validateCommandLine() {
	flag.BoolVar(&showHelp, "h", false, "Show help")
	flag.Parse()
	if showHelp {
		printUsage()
		os.Exit(0)
	}
	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "Please specify a bag to validate. ")
		fmt.Fprintf(os.Stderr, "Or use apt_validator -help for help.\n")
		os.Exit(1)
	}
}

func printUsage() {
	fmt.Println("apt_validator <path1> <path2> ... <pathN>")
	fmt.Println("Validates bags for APTrust.")
	fmt.Printf("Each path param should be the path to a tar file,")
	fmt.Println("or the path to a directory that ")
	fmt.Println("you want to tar up and send to APTrust.\n")
	fmt.Println("Examples:")
	fmt.Println("    apt_validator /home/josie/university.edu.my_archive.tar")
	fmt.Println("    apt_validator university.edu.archive_one.tar university.edu.archive_two.tar")
	fmt.Println("    apt_validator /home/josie/university.edu.my_archive/\n")
	printSpecUrl()
}

func printSpecUrl() {
	fmt.Println("The full APTrust bagit specification is available at")
	fmt.Println("https://sites.google.com/a/aptrust.org/aptrust-wiki/technical-documentation/processing-ingest/aptrust-bagit-profile")
}
<commit_msg>Cleaned up usage messages<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"os"
)

var showHelp bool

func main() {
	validateCommandLine()
	anyBagFailed := false
	for i := 1; i < len(os.Args); i++ {
		filePath := os.Args[i]
		validator, err := bagman.NewValidator(filePath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error creating validator for %s: %s\n", filePath, err)
			os.Exit(1)
		}
		if validator.IsValid() {
			fmt.Printf("[PASS] %s is a valid APTrust bag\n", filePath)
		} else {
			fmt.Printf("[FAIL] %s is not valid for the following reasons:\n", filePath)
			fmt.Println(" ", validator.ErrorMessage)
			anyBagFailed = true
		}
	}
	if anyBagFailed {
		fmt.Println("")
		printSpecUrl()
		os.Exit(1)
	}
}

func validateCommandLine() {
	flag.BoolVar(&showHelp, "h", false, "Show help")
	flag.Parse()
	if showHelp {
		printUsage()
		os.Exit(0)
	}
	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "Please specify a bag to validate. ")
		fmt.Fprintf(os.Stderr, "Or use apt_validator -h for help.\n")
		os.Exit(1)
	}
}

func printUsage() {
	usage := `
apt_validator <path1> <path2> ... <pathN>
Validates bags for APTrust.
Each path param should be the path to a tar file, or the path to a directory
that you want to tar up and send to APTrust.
	Examples:
	    apt_validator /home/josie/university.edu.my_archive.tar
	    apt_validator university.edu.archive_one.tar university.edu.archive_two.tar
	    apt_validator /home/josie/university.edu.my_archive/
`
	fmt.Println(usage)
	printSpecUrl()
}

func printSpecUrl() {
	fmt.Println("The full APTrust bagit specification is available at")
	fmt.Println("https://sites.google.com/a/aptrust.org/aptrust-wiki/technical-documentation/processing-ingest/aptrust-bagit-profile")
}
<|endoftext|>