<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tabletmanager

import (
	"flag"
	"fmt"
	"html/template"
	"time"

	"vitess.io/vitess/go/vt/vterrors"

	"golang.org/x/net/context"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/vt/health"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/mysqlctl"
)

var (
	enableReplicationReporter = flag.Bool("enable_replication_reporter", false, "Register the health check module that monitors MySQL replication")
)

// replicationReporter implements health.Reporter
type replicationReporter struct {
	// set at construction time
	agent *ActionAgent
	now   func() time.Time

	// store the last time we successfully got the lag, so if we
	// can't get the lag any more, we can extrapolate.
	lastKnownValue time.Duration
	lastKnownTime  time.Time
}

// Report is part of the health.Reporter interface
func (r *replicationReporter) Report(isSlaveType, shouldQueryServiceBeRunning bool) (time.Duration, error) {
	if !isSlaveType {
		return 0, nil
	}

	status, statusErr := r.agent.MysqlDaemon.SlaveStatus()
	if statusErr == mysql.ErrNotSlave ||
		(statusErr == nil && !status.SlaveSQLRunning && !status.SlaveIORunning) {
		// MySQL is up, but slave is either not configured or not running.
		// Both SQL and IO threads are stopped, so it's probably either
		// stopped on purpose, or stopped because of a mysqld restart.
		if !r.agent.slaveStopped() {
			// As far as we've been told, it isn't stopped on purpose,
			// so let's try to start it.
			if *mysqlctl.DisableActiveReparents {
				log.Infof("Slave is stopped. Running with --disable_active_reparents so will not try to reconnect to master...")
			} else {
				log.Infof("Slave is stopped. Trying to reconnect to master...")
				ctx, cancel := context.WithTimeout(r.agent.batchCtx, 5*time.Second)
				if err := repairReplication(ctx, r.agent); err != nil {
					log.Infof("Failed to reconnect to master: %v", err)
				}
				cancel()
				// Check status again.
				status, statusErr = r.agent.MysqlDaemon.SlaveStatus()
			}
		}
	}
	if statusErr != nil {
		// mysqld is not running or slave is not configured.
		// We can't report healthy.
		return 0, statusErr
	}
	if !status.SlaveRunning() {
		// mysqld is running, but slave is not replicating (most likely,
		// replication has been stopped). See if we can extrapolate.
		if r.lastKnownTime.IsZero() {
			// we can't.
			return 0, health.ErrSlaveNotRunning
		}

		// we can extrapolate with the worst possible
		// value (that is we made no replication
		// progress since last time, and just fell more behind).
		elapsed := r.now().Sub(r.lastKnownTime)
		return elapsed + r.lastKnownValue, nil
	}

	// we got a real value, save it.
	r.lastKnownValue = time.Duration(status.SecondsBehindMaster) * time.Second
	r.lastKnownTime = r.now()
	return r.lastKnownValue, nil
}

// HTMLName is part of the health.Reporter interface
func (r *replicationReporter) HTMLName() template.HTML {
	return template.HTML("MySQLReplicationLag")
}

// repairReplication tries to connect this slave to whoever is
// the current master of the shard, and start replicating.
func repairReplication(ctx context.Context, agent *ActionAgent) error {
	if *mysqlctl.DisableActiveReparents {
		return fmt.Errorf("can't repair replication with --disable_active_reparents")
	}

	ts := agent.TopoServer
	tablet := agent.Tablet()

	si, err := ts.GetShard(ctx, tablet.Keyspace, tablet.Shard)
	if err != nil {
		return err
	}
	if !si.HasMaster() {
		return fmt.Errorf("no master tablet for shard %v/%v", tablet.Keyspace, tablet.Shard)
	}

	// If Orchestrator is configured and if Orchestrator is actively reparenting, we should not repairReplication
	if agent.orc != nil {
		re, err := agent.orc.InActiveShardRecovery(tablet)
		if err != nil {
			return err
		}
		if re {
			return fmt.Errorf("orchestrator actively reparenting shard %v, skipping repairReplication", si)
		}

		// Before repairing replication, tell Orchestrator to enter maintenance mode for this tablet and to
		// lock any other actions on this tablet by Orchestrator.
		if err := agent.orc.BeginMaintenance(agent.Tablet(), "vttablet has been told to StopSlave"); err != nil {
			log.Warningf("Orchestrator BeginMaintenance failed: %v", err)
			return vterrors.Wrap(err, "orchestrator BeginMaintenance failed, skipping repairReplication")
		}
	}

	return agent.setMasterRepairReplication(ctx, si.MasterAlias, 0, true)
}

func registerReplicationReporter(agent *ActionAgent) {
	if *enableReplicationReporter {
		health.DefaultAggregator.Register("replication_reporter",
			&replicationReporter{
				agent: agent,
				now:   time.Now,
			})
	}
}
<commit_msg>replication_reporter: Don't try to reparent to yourself.<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tabletmanager

import (
	"flag"
	"fmt"
	"html/template"
	"time"

	"vitess.io/vitess/go/vt/vterrors"

	"golang.org/x/net/context"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/vt/health"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/mysqlctl"
	"vitess.io/vitess/go/vt/topo/topoproto"
)

var (
	enableReplicationReporter = flag.Bool("enable_replication_reporter", false, "Register the health check module that monitors MySQL replication")
)

// replicationReporter implements health.Reporter
type replicationReporter struct {
	// set at construction time
	agent *ActionAgent
	now   func() time.Time

	// store the last time we successfully got the lag, so if we
	// can't get the lag any more, we can extrapolate.
	lastKnownValue time.Duration
	lastKnownTime  time.Time
}

// Report is part of the health.Reporter interface
func (r *replicationReporter) Report(isSlaveType, shouldQueryServiceBeRunning bool) (time.Duration, error) {
	if !isSlaveType {
		return 0, nil
	}

	status, statusErr := r.agent.MysqlDaemon.SlaveStatus()
	if statusErr == mysql.ErrNotSlave ||
		(statusErr == nil && !status.SlaveSQLRunning && !status.SlaveIORunning) {
		// MySQL is up, but slave is either not configured or not running.
		// Both SQL and IO threads are stopped, so it's probably either
		// stopped on purpose, or stopped because of a mysqld restart.
		if !r.agent.slaveStopped() {
			// As far as we've been told, it isn't stopped on purpose,
			// so let's try to start it.
			if *mysqlctl.DisableActiveReparents {
				log.Infof("Slave is stopped. Running with --disable_active_reparents so will not try to reconnect to master...")
			} else {
				log.Infof("Slave is stopped. Trying to reconnect to master...")
				ctx, cancel := context.WithTimeout(r.agent.batchCtx, 5*time.Second)
				if err := repairReplication(ctx, r.agent); err != nil {
					log.Infof("Failed to reconnect to master: %v", err)
				}
				cancel()
				// Check status again.
				status, statusErr = r.agent.MysqlDaemon.SlaveStatus()
			}
		}
	}
	if statusErr != nil {
		// mysqld is not running or slave is not configured.
		// We can't report healthy.
		return 0, statusErr
	}
	if !status.SlaveRunning() {
		// mysqld is running, but slave is not replicating (most likely,
		// replication has been stopped). See if we can extrapolate.
		if r.lastKnownTime.IsZero() {
			// we can't.
			return 0, health.ErrSlaveNotRunning
		}

		// we can extrapolate with the worst possible
		// value (that is we made no replication
		// progress since last time, and just fell more behind).
		elapsed := r.now().Sub(r.lastKnownTime)
		return elapsed + r.lastKnownValue, nil
	}

	// we got a real value, save it.
	r.lastKnownValue = time.Duration(status.SecondsBehindMaster) * time.Second
	r.lastKnownTime = r.now()
	return r.lastKnownValue, nil
}

// HTMLName is part of the health.Reporter interface
func (r *replicationReporter) HTMLName() template.HTML {
	return template.HTML("MySQLReplicationLag")
}

// repairReplication tries to connect this slave to whoever is
// the current master of the shard, and start replicating.
func repairReplication(ctx context.Context, agent *ActionAgent) error {
	if *mysqlctl.DisableActiveReparents {
		return fmt.Errorf("can't repair replication with --disable_active_reparents")
	}

	ts := agent.TopoServer
	tablet := agent.Tablet()

	si, err := ts.GetShard(ctx, tablet.Keyspace, tablet.Shard)
	if err != nil {
		return err
	}
	if !si.HasMaster() {
		return fmt.Errorf("no master tablet for shard %v/%v", tablet.Keyspace, tablet.Shard)
	}

	if topoproto.TabletAliasEqual(si.MasterAlias, tablet.Alias) {
		// The shard record says we are master, but we disagree; we wouldn't
		// reach this point unless we were told to check replication as a slave
		// type. Hopefully someone is working on fixing that, but in any case,
		// we should not try to reparent to ourselves.
		return fmt.Errorf("shard %v/%v record claims tablet %v is master, but its type is %v", tablet.Keyspace, tablet.Shard, topoproto.TabletAliasString(tablet.Alias), tablet.Type)
	}

	// If Orchestrator is configured and if Orchestrator is actively reparenting, we should not repairReplication
	if agent.orc != nil {
		re, err := agent.orc.InActiveShardRecovery(tablet)
		if err != nil {
			return err
		}
		if re {
			return fmt.Errorf("orchestrator actively reparenting shard %v, skipping repairReplication", si)
		}

		// Before repairing replication, tell Orchestrator to enter maintenance mode for this tablet and to
		// lock any other actions on this tablet by Orchestrator.
		if err := agent.orc.BeginMaintenance(agent.Tablet(), "vttablet has been told to StopSlave"); err != nil {
			log.Warningf("Orchestrator BeginMaintenance failed: %v", err)
			return vterrors.Wrap(err, "orchestrator BeginMaintenance failed, skipping repairReplication")
		}
	}

	return agent.setMasterRepairReplication(ctx, si.MasterAlias, 0, true)
}

func registerReplicationReporter(agent *ActionAgent) {
	if *enableReplicationReporter {
		health.DefaultAggregator.Register("replication_reporter",
			&replicationReporter{
				agent: agent,
				now:   time.Now,
			})
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vstreamer

import (
	"context"
	"fmt"
	"math"
	"time"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/dbconfigs"
	"vitess.io/vitess/go/vt/log"
	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
	querypb "vitess.io/vitess/go/vt/proto/query"
	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/sqlparser"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vtgate/vindexes"
	"vitess.io/vitess/go/vt/vttablet/tabletserver/schema"
)

// RowStreamer exposes an externally usable interface to rowStreamer.
type RowStreamer interface {
	Stream() error
	Cancel()
}

// NewRowStreamer returns a RowStreamer
func NewRowStreamer(ctx context.Context, cp dbconfigs.Connector, se *schema.Engine, query string, lastpk []sqltypes.Value, send func(*binlogdatapb.VStreamRowsResponse) error, vse *Engine) RowStreamer {
	return newRowStreamer(ctx, cp, se, query, lastpk, &localVSchema{vschema: &vindexes.VSchema{}}, send, vse)
}

// rowStreamer is used for copying the existing rows of a table
// before vreplication begins streaming binlogs. The rowStreamer
// responds to a request with the GTID position as of which it
// streams the rows of a table. This allows vreplication to synchronize
// its events as of the returned GTID before adding the new rows.
// For every set of rows sent, the last pk value is also sent.
// This allows for the streaming to be resumed based on the last
// pk value processed.
type rowStreamer struct {
	ctx    context.Context
	cancel func()

	cp      dbconfigs.Connector
	se      *schema.Engine
	query   string
	lastpk  []sqltypes.Value
	send    func(*binlogdatapb.VStreamRowsResponse) error
	vschema *localVSchema

	plan      *Plan
	pkColumns []int
	ukName    string
	sendQuery string
	vse       *Engine
	pktsize   PacketSizer
}

func newRowStreamer(ctx context.Context, cp dbconfigs.Connector, se *schema.Engine, query string, lastpk []sqltypes.Value, vschema *localVSchema, send func(*binlogdatapb.VStreamRowsResponse) error, vse *Engine) *rowStreamer {
	ctx, cancel := context.WithCancel(ctx)
	return &rowStreamer{
		ctx:     ctx,
		cancel:  cancel,
		cp:      cp,
		se:      se,
		query:   query,
		lastpk:  lastpk,
		send:    send,
		vschema: vschema,
		vse:     vse,
		pktsize: DefaultPacketSizer(),
	}
}

func (rs *rowStreamer) Cancel() {
	log.Info("Rowstreamer Cancel() called")
	rs.cancel()
}

func (rs *rowStreamer) Stream() error {
	// Ensure sh is Open. If vttablet came up in a non_serving role,
	// the schema engine may not have been initialized.
	if err := rs.se.Open(); err != nil {
		return err
	}
	if err := rs.buildPlan(); err != nil {
		return err
	}
	conn, err := snapshotConnect(rs.ctx, rs.cp)
	if err != nil {
		return err
	}
	defer conn.Close()
	if _, err := conn.ExecuteFetch("set names binary", 1, false); err != nil {
		return err
	}
	return rs.streamQuery(conn, rs.send)
}

func (rs *rowStreamer) buildPlan() error {
	// This pre-parsing is required to extract the table name
	// and create its metadata.
	sel, fromTable, err := analyzeSelect(rs.query)
	if err != nil {
		return err
	}
	st, err := rs.se.GetTableForPos(fromTable, "")
	if err != nil {
		return err
	}
	ti := &Table{
		Name:   st.Name,
		Fields: st.Fields,
	}
	// The plan we build is identical to the one for vstreamer.
	// This is because the row format of a read is identical
	// to the row format of a binlog event. So, the same
	// filtering will work.
	rs.plan, err = buildTablePlan(ti, rs.vschema, rs.query)
	if err != nil {
		log.Errorf("%s", err.Error())
		return err
	}

	directives := sqlparser.ExtractCommentDirectives(sel.Comments)
	rs.ukName = directives.GetString("ukName", "")

	rs.pkColumns, err = rs.buildPKColumns(st)
	if err != nil {
		return err
	}
	rs.sendQuery, err = rs.buildSelect()
	if err != nil {
		return err
	}
	return err
}

// buildPKColumnsFromUniqueKey assumes a unique key is indicated,
func (rs *rowStreamer) buildPKColumnsFromUniqueKey() ([]int, error) {
	var pkColumns = make([]int, 0)
	fmt.Printf("========== rs.ukName=%s\n", rs.ukName)
	// We wish to utilize a UNIQUE KEY which is not the PRIMARY KEY/
	// For PRIMARY KEY, we already know what columns partiipate in the key,
	// but for any other key, we need to go ahead and inspect the table/key definition
	conn, err := snapshotConnect(rs.ctx, rs.cp)
	if err != nil {
		return pkColumns, err
	}
	defer conn.Close()

	query, err := sqlparser.ParseAndBind(mysql.BaseShowTableUniqueKey,
		sqltypes.StringBindVariable(rs.plan.Table.Name),
		sqltypes.StringBindVariable(rs.ukName),
	)
	if err != nil {
		return pkColumns, err
	}
	fmt.Printf("========== query=%s\n", query)
	r, err := conn.ExecuteFetch(query, math.MaxInt64, true)
	if err != nil {
		return pkColumns, err
	}

	for _, row := range r.Named().Rows {
		colName := row["column_name"].ToString()
		index := rs.plan.Table.FindColumn(sqlparser.NewColIdent(colName))
		if index < 0 {
			return pkColumns, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "column %v is listed as unique key, but not present in table %v", colName, rs.plan.Table.Name)
		}
		pkColumns = append(pkColumns, index)
	}
	fmt.Printf("========== pkColumns=%v\n", pkColumns)
	return pkColumns, nil
}

func (rs *rowStreamer) buildPKColumns(st *binlogdatapb.MinimalTable) ([]int, error) {
	if rs.ukName != "" && rs.ukName != "PRIMARY" {
		return rs.buildPKColumnsFromUniqueKey()
	}
	var pkColumns = make([]int, 0)
	if len(st.PKColumns) == 0 {
		pkColumns = make([]int, len(st.Fields))
		for i := range st.Fields {
			pkColumns[i] = i
		}
		return pkColumns, nil
	}
	for _, pk := range st.PKColumns {
		if pk >= int64(len(st.Fields)) {
			return nil, fmt.Errorf("primary key %d refers to non-existent column", pk)
		}
		pkColumns = append(pkColumns, int(pk))
	}
	return pkColumns, nil
}

func (rs *rowStreamer) buildSelect() (string, error) {
	buf := sqlparser.NewTrackedBuffer(nil)
	// We could have used select *, but being explicit is more predictable.
	buf.Myprintf("select ")
	prefix := ""
	for _, col := range rs.plan.Table.Fields {
		if rs.plan.isConvertColumnUsingUTF8(col.Name) {
			buf.Myprintf("%sconvert(%v using utf8mb4)", prefix, sqlparser.NewColIdent(col.Name))
		} else {
			buf.Myprintf("%s%v", prefix, sqlparser.NewColIdent(col.Name))
		}
		prefix = ", "
	}
	buf.Myprintf(" from %v", sqlparser.NewTableIdent(rs.plan.Table.Name))
	if len(rs.lastpk) != 0 {
		if len(rs.lastpk) != len(rs.pkColumns) {
			return "", fmt.Errorf("primary key values don't match length: %v vs %v", rs.lastpk, rs.pkColumns)
		}
		buf.WriteString(" where ")
		prefix := ""
		// This loop handles the case for composite pks. For example,
		// if lastpk was (1,2), the where clause would be:
		// (col1 = 1 and col2 > 2) or (col1 > 1).
		// A tuple inequality like (col1,col2) > (1,2) ends up
		// being a full table scan for mysql.
		for lastcol := len(rs.pkColumns) - 1; lastcol >= 0; lastcol-- {
			buf.Myprintf("%s(", prefix)
			prefix = " or "
			for i, pk := range rs.pkColumns[:lastcol] {
				buf.Myprintf("%v = ", sqlparser.NewColIdent(rs.plan.Table.Fields[pk].Name))
				rs.lastpk[i].EncodeSQL(buf)
				buf.Myprintf(" and ")
			}
			buf.Myprintf("%v > ", sqlparser.NewColIdent(rs.plan.Table.Fields[rs.pkColumns[lastcol]].Name))
			rs.lastpk[lastcol].EncodeSQL(buf)
			buf.Myprintf(")")
		}
	}
	buf.Myprintf(" order by ", sqlparser.NewTableIdent(rs.plan.Table.Name))
	prefix = ""
	for _, pk := range rs.pkColumns {
		buf.Myprintf("%s%v", prefix, sqlparser.NewColIdent(rs.plan.Table.Fields[pk].Name))
		prefix = ", "
	}
	return buf.String(), nil
}

func (rs *rowStreamer) streamQuery(conn *snapshotConn, send func(*binlogdatapb.VStreamRowsResponse) error) error {
	log.Infof("Streaming query: %v\n", rs.sendQuery)
	gtid, err := conn.streamWithSnapshot(rs.ctx, rs.plan.Table.Name, rs.sendQuery)
	if err != nil {
		return err
	}

	// first call the callback with the fields
	flds, err := conn.Fields()
	if err != nil {
		return err
	}
	pkfields := make([]*querypb.Field, len(rs.pkColumns))
	for i, pk := range rs.pkColumns {
		pkfields[i] = &querypb.Field{
			Name: flds[pk].Name,
			Type: flds[pk].Type,
		}
	}

	err = send(&binlogdatapb.VStreamRowsResponse{
		Fields:   rs.plan.fields(),
		Pkfields: pkfields,
		Gtid:     gtid,
	})
	if err != nil {
		return fmt.Errorf("stream send error: %v", err)
	}

	var response binlogdatapb.VStreamRowsResponse
	var rows []*querypb.Row
	var rowCount int
	var mysqlrow []sqltypes.Value

	filtered := make([]sqltypes.Value, len(rs.plan.ColExprs))
	lastpk := make([]sqltypes.Value, len(rs.pkColumns))
	byteCount := 0
	for {
		//log.Infof("StreamResponse for loop iteration starts")
		if rs.ctx.Err() != nil {
			log.Infof("Stream ended because of ctx.Done")
			return fmt.Errorf("stream ended: %v", rs.ctx.Err())
		}

		// check throttler.
		if !rs.vse.throttlerClient.ThrottleCheckOKOrWait(rs.ctx) {
			continue
		}

		if mysqlrow != nil {
			mysqlrow = mysqlrow[:0]
		}
		mysqlrow, err = conn.FetchNext(mysqlrow)
		if err != nil {
			return err
		}
		if mysqlrow == nil {
			break
		}
		// Compute lastpk here, because we'll need it
		// at the end after the loop exits.
		for i, pk := range rs.pkColumns {
			lastpk[i] = mysqlrow[pk]
		}
		// Reuse the vstreamer's filter.
		ok, err := rs.plan.filter(mysqlrow, filtered)
		if err != nil {
			return err
		}
		if ok {
			if rowCount >= len(rows) {
				rows = append(rows, &querypb.Row{})
			}
			byteCount += sqltypes.RowToProto3Inplace(filtered, rows[rowCount])
			rowCount++
		}

		if rs.pktsize.ShouldSend(byteCount) {
			response.Rows = rows[:rowCount]
			response.Lastpk = sqltypes.RowToProto3(lastpk)

			rs.vse.rowStreamerNumRows.Add(int64(len(response.Rows)))
			rs.vse.rowStreamerNumPackets.Add(int64(1))

			startSend := time.Now()
			err = send(&response)
			if err != nil {
				log.Infof("Rowstreamer send returned error %v", err)
				return err
			}
			rs.pktsize.Record(byteCount, time.Since(startSend))
			rowCount = 0
			byteCount = 0
		}
	}

	if rowCount > 0 {
		response.Rows = rows[:rowCount]
		response.Lastpk = sqltypes.RowToProto3(lastpk)

		rs.vse.rowStreamerNumRows.Add(int64(len(response.Rows)))
		err = send(&response)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>provide alias for textual columns with convert() function, so that field name still reflects column name<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vstreamer

import (
	"context"
	"fmt"
	"math"
	"time"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/dbconfigs"
	"vitess.io/vitess/go/vt/log"
	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
	querypb "vitess.io/vitess/go/vt/proto/query"
	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/sqlparser"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vtgate/vindexes"
	"vitess.io/vitess/go/vt/vttablet/tabletserver/schema"
)

// RowStreamer exposes an externally usable interface to rowStreamer.
type RowStreamer interface {
	Stream() error
	Cancel()
}

// NewRowStreamer returns a RowStreamer
func NewRowStreamer(ctx context.Context, cp dbconfigs.Connector, se *schema.Engine, query string, lastpk []sqltypes.Value, send func(*binlogdatapb.VStreamRowsResponse) error, vse *Engine) RowStreamer {
	return newRowStreamer(ctx, cp, se, query, lastpk, &localVSchema{vschema: &vindexes.VSchema{}}, send, vse)
}

// rowStreamer is used for copying the existing rows of a table
// before vreplication begins streaming binlogs. The rowStreamer
// responds to a request with the GTID position as of which it
// streams the rows of a table. This allows vreplication to synchronize
// its events as of the returned GTID before adding the new rows.
// For every set of rows sent, the last pk value is also sent.
// This allows for the streaming to be resumed based on the last
// pk value processed.
type rowStreamer struct {
	ctx    context.Context
	cancel func()

	cp      dbconfigs.Connector
	se      *schema.Engine
	query   string
	lastpk  []sqltypes.Value
	send    func(*binlogdatapb.VStreamRowsResponse) error
	vschema *localVSchema

	plan      *Plan
	pkColumns []int
	ukName    string
	sendQuery string
	vse       *Engine
	pktsize   PacketSizer
}

func newRowStreamer(ctx context.Context, cp dbconfigs.Connector, se *schema.Engine, query string, lastpk []sqltypes.Value, vschema *localVSchema, send func(*binlogdatapb.VStreamRowsResponse) error, vse *Engine) *rowStreamer {
	ctx, cancel := context.WithCancel(ctx)
	return &rowStreamer{
		ctx:     ctx,
		cancel:  cancel,
		cp:      cp,
		se:      se,
		query:   query,
		lastpk:  lastpk,
		send:    send,
		vschema: vschema,
		vse:     vse,
		pktsize: DefaultPacketSizer(),
	}
}

func (rs *rowStreamer) Cancel() {
	log.Info("Rowstreamer Cancel() called")
	rs.cancel()
}

func (rs *rowStreamer) Stream() error {
	// Ensure sh is Open. If vttablet came up in a non_serving role,
	// the schema engine may not have been initialized.
	if err := rs.se.Open(); err != nil {
		return err
	}
	if err := rs.buildPlan(); err != nil {
		return err
	}
	conn, err := snapshotConnect(rs.ctx, rs.cp)
	if err != nil {
		return err
	}
	defer conn.Close()
	if _, err := conn.ExecuteFetch("set names binary", 1, false); err != nil {
		return err
	}
	return rs.streamQuery(conn, rs.send)
}

func (rs *rowStreamer) buildPlan() error {
	// This pre-parsing is required to extract the table name
	// and create its metadata.
	sel, fromTable, err := analyzeSelect(rs.query)
	if err != nil {
		return err
	}
	st, err := rs.se.GetTableForPos(fromTable, "")
	if err != nil {
		return err
	}
	ti := &Table{
		Name:   st.Name,
		Fields: st.Fields,
	}
	// The plan we build is identical to the one for vstreamer.
	// This is because the row format of a read is identical
	// to the row format of a binlog event. So, the same
	// filtering will work.
	rs.plan, err = buildTablePlan(ti, rs.vschema, rs.query)
	if err != nil {
		log.Errorf("%s", err.Error())
		return err
	}

	directives := sqlparser.ExtractCommentDirectives(sel.Comments)
	rs.ukName = directives.GetString("ukName", "")

	rs.pkColumns, err = rs.buildPKColumns(st)
	if err != nil {
		return err
	}
	rs.sendQuery, err = rs.buildSelect()
	if err != nil {
		return err
	}
	return err
}

// buildPKColumnsFromUniqueKey assumes a unique key is indicated,
func (rs *rowStreamer) buildPKColumnsFromUniqueKey() ([]int, error) {
	var pkColumns = make([]int, 0)
	// We wish to utilize a UNIQUE KEY which is not the PRIMARY KEY/
	// For PRIMARY KEY, we already know what columns partiipate in the key,
	// but for any other key, we need to go ahead and inspect the table/key definition
	conn, err := snapshotConnect(rs.ctx, rs.cp)
	if err != nil {
		return pkColumns, err
	}
	defer conn.Close()

	query, err := sqlparser.ParseAndBind(mysql.BaseShowTableUniqueKey,
		sqltypes.StringBindVariable(rs.plan.Table.Name),
		sqltypes.StringBindVariable(rs.ukName),
	)
	if err != nil {
		return pkColumns, err
	}
	r, err := conn.ExecuteFetch(query, math.MaxInt64, true)
	if err != nil {
		return pkColumns, err
	}

	for _, row := range r.Named().Rows {
		colName := row["column_name"].ToString()
		index := rs.plan.Table.FindColumn(sqlparser.NewColIdent(colName))
		if index < 0 {
			return pkColumns, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "column %v is listed as unique key, but not present in table %v", colName, rs.plan.Table.Name)
		}
		pkColumns = append(pkColumns, index)
	}
	return pkColumns, nil
}

func (rs *rowStreamer) buildPKColumns(st *binlogdatapb.MinimalTable) ([]int, error) {
	if rs.ukName != "" && rs.ukName != "PRIMARY" {
		return rs.buildPKColumnsFromUniqueKey()
	}
	var pkColumns = make([]int, 0)
	if len(st.PKColumns) == 0 {
		pkColumns = make([]int, len(st.Fields))
		for i := range st.Fields {
			pkColumns[i] = i
		}
		return pkColumns, nil
	}
	for _, pk := range st.PKColumns {
		if pk >= int64(len(st.Fields)) {
			return nil, fmt.Errorf("primary key %d refers to non-existent column", pk)
		}
		pkColumns = append(pkColumns, int(pk))
	}
	return pkColumns, nil
}

func (rs *rowStreamer) buildSelect() (string, error) {
	buf := sqlparser.NewTrackedBuffer(nil)
	// We could have used select *, but being explicit is more predictable.
	buf.Myprintf("select ")
	prefix := ""
	for _, col := range rs.plan.Table.Fields {
		if rs.plan.isConvertColumnUsingUTF8(col.Name) {
			buf.Myprintf("%sconvert(%v using utf8mb4) as %v", prefix, sqlparser.NewColIdent(col.Name), sqlparser.NewColIdent(col.Name))
		} else {
			buf.Myprintf("%s%v", prefix, sqlparser.NewColIdent(col.Name))
		}
		prefix = ", "
	}
	buf.Myprintf(" from %v", sqlparser.NewTableIdent(rs.plan.Table.Name))
	if len(rs.lastpk) != 0 {
		if len(rs.lastpk) != len(rs.pkColumns) {
			return "", fmt.Errorf("primary key values don't match length: %v vs %v", rs.lastpk, rs.pkColumns)
		}
		buf.WriteString(" where ")
		prefix := ""
		// This loop handles the case for composite pks. For example,
		// if lastpk was (1,2), the where clause would be:
		// (col1 = 1 and col2 > 2) or (col1 > 1).
		// A tuple inequality like (col1,col2) > (1,2) ends up
		// being a full table scan for mysql.
		for lastcol := len(rs.pkColumns) - 1; lastcol >= 0; lastcol-- {
			buf.Myprintf("%s(", prefix)
			prefix = " or "
			for i, pk := range rs.pkColumns[:lastcol] {
				buf.Myprintf("%v = ", sqlparser.NewColIdent(rs.plan.Table.Fields[pk].Name))
				rs.lastpk[i].EncodeSQL(buf)
				buf.Myprintf(" and ")
			}
			buf.Myprintf("%v > ", sqlparser.NewColIdent(rs.plan.Table.Fields[rs.pkColumns[lastcol]].Name))
			rs.lastpk[lastcol].EncodeSQL(buf)
			buf.Myprintf(")")
		}
	}
	buf.Myprintf(" order by ", sqlparser.NewTableIdent(rs.plan.Table.Name))
	prefix = ""
	for _, pk := range rs.pkColumns {
		buf.Myprintf("%s%v", prefix, sqlparser.NewColIdent(rs.plan.Table.Fields[pk].Name))
		prefix = ", "
	}
	return buf.String(), nil
}

func (rs *rowStreamer) streamQuery(conn *snapshotConn, send func(*binlogdatapb.VStreamRowsResponse) error) error {
	log.Infof("Streaming query: %v\n", rs.sendQuery)
	gtid, err := conn.streamWithSnapshot(rs.ctx, rs.plan.Table.Name, rs.sendQuery)
	if err != nil {
		return err
	}

	// first call the callback with the fields
	flds, err := conn.Fields()
	if err != nil {
		return err
	}
	pkfields := make([]*querypb.Field, len(rs.pkColumns))
	for i, pk := range rs.pkColumns {
		pkfields[i] = &querypb.Field{
			Name: flds[pk].Name,
			Type: flds[pk].Type,
		}
	}

	err = send(&binlogdatapb.VStreamRowsResponse{
		Fields:   rs.plan.fields(),
		Pkfields: pkfields,
		Gtid:     gtid,
	})
	if err != nil {
		return fmt.Errorf("stream send error: %v", err)
	}

	var response binlogdatapb.VStreamRowsResponse
	var rows []*querypb.Row
	var rowCount int
	var mysqlrow []sqltypes.Value

	filtered := make([]sqltypes.Value, len(rs.plan.ColExprs))
	lastpk := make([]sqltypes.Value, len(rs.pkColumns))
	byteCount := 0
	for {
		//log.Infof("StreamResponse for loop iteration starts")
		if rs.ctx.Err() != nil {
			log.Infof("Stream ended because of ctx.Done")
			return fmt.Errorf("stream ended: %v", rs.ctx.Err())
		}

		// check throttler.
		if !rs.vse.throttlerClient.ThrottleCheckOKOrWait(rs.ctx) {
			continue
		}

		if mysqlrow != nil {
			mysqlrow = mysqlrow[:0]
		}
		mysqlrow, err = conn.FetchNext(mysqlrow)
		if err != nil {
			return err
		}
		if mysqlrow == nil {
			break
		}
		// Compute lastpk here, because we'll need it
		// at the end after the loop exits.
		for i, pk := range rs.pkColumns {
			lastpk[i] = mysqlrow[pk]
		}
		// Reuse the vstreamer's filter.
		ok, err := rs.plan.filter(mysqlrow, filtered)
		if err != nil {
			return err
		}
		if ok {
			if rowCount >= len(rows) {
				rows = append(rows, &querypb.Row{})
			}
			byteCount += sqltypes.RowToProto3Inplace(filtered, rows[rowCount])
			rowCount++
		}

		if rs.pktsize.ShouldSend(byteCount) {
			response.Rows = rows[:rowCount]
			response.Lastpk = sqltypes.RowToProto3(lastpk)

			rs.vse.rowStreamerNumRows.Add(int64(len(response.Rows)))
			rs.vse.rowStreamerNumPackets.Add(int64(1))

			startSend := time.Now()
			err = send(&response)
			if err != nil {
				log.Infof("Rowstreamer send returned error %v", err)
				return err
			}
			rs.pktsize.Record(byteCount, time.Since(startSend))
			rowCount = 0
			byteCount = 0
		}
	}

	if rowCount > 0 {
		response.Rows = rows[:rowCount]
		response.Lastpk = sqltypes.RowToProto3(lastpk)

		rs.vse.rowStreamerNumRows.Add(int64(len(response.Rows)))
		err = send(&response)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

// Stackdriver tests cannot be run in parallel otherwise they will error out with:
// Error 503: Too many concurrent edits to the project configuration. Please try again.

func TestAccMonitoringAlertPolicy_basic(t *testing.T) {

	alertName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	conditionName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	filter := `metric.type=\"compute.googleapis.com/instance/disk/write_bytes_count\" AND resource.type=\"gce_instance\"`

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAlertPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccMonitoringAlertPolicy_basic(alertName, conditionName, "ALIGN_RATE", filter),
			},
			{
				ResourceName:      "google_monitoring_alert_policy.basic",
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccMonitoringAlertPolicy_update(t *testing.T) {

	alertName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	conditionName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	filter1 := `metric.type=\"compute.googleapis.com/instance/disk/write_bytes_count\" AND resource.type=\"gce_instance\"`
	aligner1 := "ALIGN_RATE"
	filter2 := `metric.type=\"compute.googleapis.com/instance/cpu/utilization\" AND resource.type=\"gce_instance\"`
	aligner2 := "ALIGN_MAX"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAlertPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccMonitoringAlertPolicy_basic(alertName, conditionName, aligner1, filter1),
			},
			{
				ResourceName:      "google_monitoring_alert_policy.basic",
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccMonitoringAlertPolicy_basic(alertName, conditionName, aligner2, filter2),
			},
			{
				ResourceName:      "google_monitoring_alert_policy.basic",
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccMonitoringAlertPolicy_full(t *testing.T) {

	alertName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	conditionName1 := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	conditionName2 := fmt.Sprintf("tf-test-%s", acctest.RandString(10))

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAlertPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccMonitoringAlertPolicy_full(alertName, conditionName1, conditionName2),
			},
			{
				ResourceName:      "google_monitoring_alert_policy.full",
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccCheckAlertPolicyDestroy(s *terraform.State) error {
	config := testAccProvider.Meta().(*Config)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "google_monitoring_alert_policy" {
			continue
		}

		name := rs.Primary.Attributes["name"]

		url := fmt.Sprintf("https://monitoring.googleapis.com/v3/%s", name)
		_, err := sendRequest(config, "GET", url, nil)

		if err == nil {
			return fmt.Errorf("Error, alert policy %s still exists", name)
		}
	}

	return nil
}

func testAccMonitoringAlertPolicy_basic(alertName, conditionName, aligner, filter string) string {
	return fmt.Sprintf(`
resource "google_monitoring_alert_policy" "basic" {
  display_name = "%s"
  enabled      = true
  combiner     = "OR"

  conditions {
    display_name = "%s"

    condition_threshold {
      aggregations {
        alignment_period   = "60s"
        per_series_aligner = "%s"
      }

      duration        = "60s"
      comparison      = "COMPARISON_GT"
      filter          = "%s"
      threshold_value = "0.5"
    }
  }
}
`, alertName, conditionName, aligner, filter)
}

func testAccMonitoringAlertPolicy_full(alertName, conditionName1, conditionName2 string) string {
	return fmt.Sprintf(`
resource "google_monitoring_alert_policy" "full" {
  display_name = "%s"
  combiner     = "OR"
  enabled      = true

  conditions {
    display_name = "%s"

    condition_threshold {
      threshold_value = 50
      filter          = "metric.type=\"compute.googleapis.com/instance/disk/write_bytes_count\" AND resource.type=\"gce_instance\""
      duration        = "60s"
      comparison      = "COMPARISON_GT"

      aggregations {
        alignment_period     = "60s"
        per_series_aligner   = "ALIGN_RATE"
        cross_series_reducer = "REDUCE_MEAN"

        group_by_fields = [
          "metric.label.device_name",
          "project",
          "resource.label.instance_id",
          "resource.label.zone",
        ]
      }

      trigger {
        percent = 10
      }
    }
  }

  conditions {
    display_name = "%s"

    condition_absent {
      duration = "3600s"
      filter   = "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" AND resource.type=\"gce_instance\""

      aggregations {
        alignment_period     = "60s"
        cross_series_reducer = "REDUCE_MEAN"
        per_series_aligner   = "ALIGN_MEAN"

        group_by_fields = [
          "project",
          "resource.label.instance_id",
          "resource.label.zone",
        ]
      }

      trigger {
        count = 1
      }
    }
  }

  documentation {
    content   = "test content"
    mime_type = "text/markdown"
  }
}
`, alertName, conditionName1, conditionName2)
}
<commit_msg>Move AlertPolicy tests to a matrix (#813)<commit_after>package google

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

// Stackdriver tests cannot be run in parallel otherwise they will error out with:
// Error 503: Too many concurrent edits to the project configuration. Please try again.

func TestAccMonitoringAlertPolicy(t *testing.T) {
	testCases := map[string]func(t *testing.T){
		"basic":  testAccMonitoringAlertPolicy_basic,
		"full":   testAccMonitoringAlertPolicy_full,
		"update": testAccMonitoringAlertPolicy_update,
	}

	for name, tc := range testCases {
		// shadow the tc variable into scope so that when
		// the loop continues, if t.Run hasn't executed tc(t)
		// yet, we don't have a race condition
		// see https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
		tc := tc
		t.Run(name, func(t *testing.T) {
			tc(t)
		})
	}
}

func testAccMonitoringAlertPolicy_basic(t *testing.T) {

	alertName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	conditionName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	filter := `metric.type=\"compute.googleapis.com/instance/disk/write_bytes_count\" AND resource.type=\"gce_instance\"`

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAlertPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccMonitoringAlertPolicy_basicCfg(alertName, conditionName, "ALIGN_RATE", filter),
			},
			{
				ResourceName:      "google_monitoring_alert_policy.basic",
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccMonitoringAlertPolicy_update(t *testing.T) {

	alertName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	conditionName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	filter1 := `metric.type=\"compute.googleapis.com/instance/disk/write_bytes_count\" AND resource.type=\"gce_instance\"`
	aligner1 := "ALIGN_RATE"
	filter2 := `metric.type=\"compute.googleapis.com/instance/cpu/utilization\" AND resource.type=\"gce_instance\"`
	aligner2 := "ALIGN_MAX"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAlertPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccMonitoringAlertPolicy_basicCfg(alertName, conditionName, aligner1, filter1),
			},
			{
				ResourceName:      "google_monitoring_alert_policy.basic",
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccMonitoringAlertPolicy_basicCfg(alertName, conditionName, aligner2, filter2),
			},
			{
				ResourceName:      "google_monitoring_alert_policy.basic",
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccMonitoringAlertPolicy_full(t *testing.T) {

	alertName := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	conditionName1 := fmt.Sprintf("tf-test-%s", acctest.RandString(10))
	conditionName2 := fmt.Sprintf("tf-test-%s", acctest.RandString(10))

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAlertPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccMonitoringAlertPolicy_fullCfg(alertName, conditionName1, conditionName2),
			},
			{
				ResourceName:      "google_monitoring_alert_policy.full",
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccCheckAlertPolicyDestroy(s *terraform.State) error {
	config := testAccProvider.Meta().(*Config)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "google_monitoring_alert_policy" {
			continue
		}

		name := rs.Primary.Attributes["name"]

		url := fmt.Sprintf("https://monitoring.googleapis.com/v3/%s", name)
		_, err := sendRequest(config, "GET", url, nil)

		if err == nil {
			return fmt.Errorf("Error, alert policy %s still exists", name)
		}
	}

	return nil
}

func testAccMonitoringAlertPolicy_basicCfg(alertName, conditionName, aligner, filter string) string {
	return fmt.Sprintf(`
resource "google_monitoring_alert_policy" "basic" {
  display_name = "%s"
  enabled      = true
  combiner     = "OR"

  conditions {
    display_name = "%s"

    condition_threshold {
      aggregations {
        alignment_period   = "60s"
        per_series_aligner = "%s"
      }

      duration        = "60s"
      comparison      = "COMPARISON_GT"
      filter          = "%s"
      threshold_value = "0.5"
    }
  }
}
`, alertName, conditionName, aligner, filter)
}

func testAccMonitoringAlertPolicy_fullCfg(alertName, conditionName1, conditionName2 string) string {
	return fmt.Sprintf(`
resource "google_monitoring_alert_policy" "full" {
  display_name = "%s"
  combiner     = "OR"
  enabled      = true

  conditions {
    display_name = "%s"

    condition_threshold {
      threshold_value = 50
      filter          = "metric.type=\"compute.googleapis.com/instance/disk/write_bytes_count\" AND resource.type=\"gce_instance\""
      duration        = "60s"
      comparison      = "COMPARISON_GT"

      aggregations {
        alignment_period     = "60s"
        per_series_aligner   = "ALIGN_RATE"
        cross_series_reducer = "REDUCE_MEAN"

        group_by_fields = [
          "metric.label.device_name",
          "project",
          "resource.label.instance_id",
          "resource.label.zone",
        ]
      }

      trigger {
        percent = 10
      }
    }
  }

  conditions {
    display_name = "%s"

    condition_absent {
      duration = "3600s"
      filter   = "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" AND resource.type=\"gce_instance\""

      aggregations {
        alignment_period     = "60s"
        cross_series_reducer = "REDUCE_MEAN"
        per_series_aligner   = "ALIGN_MEAN"

        group_by_fields = [
          "project",
          "resource.label.instance_id",
          "resource.label.zone",
        ]
      }

      trigger {
        count = 1
      }
    }
  }

  documentation {
    content   = "test content"
    mime_type = "text/markdown"
  }
}
`, alertName, conditionName1, conditionName2)
}
<|endoftext|>


<commit_before>package game

import (
	"time"

	"sdl"
)

const (
	// TODO: replace with bigger sheet
	tileSheetFile                   = "assets/default.png"
	tileWidth, tileHeight           = 32, 32
	tileSheetWidth, tileSheetHeight = 1, 1
)

var (
	terrainTileSheet *sdl.Texture
)

type tile struct {
	// x and y position in pixels - woo arbitrary!
	// id index in tile sheet
	x, y, id int
}

// Tile needs basically no work.
func (t *tile) AddChild(Object)      {}
func (t *tile) Children() []Object   { return nil }
func (t *tile) Update(time.Duration) {}
func (t *tile) Destroy()             {}

func (t *tile) Draw(r *sdl.Renderer) {
	if terrainTileSheet == nil {
		panic("Tilesheet not initialized")
	}
	r.Copy(terrainTileSheet, sdl.Rect((t.id%tileSheetWidth)*32, (t.id/tileSheetWidth)*32, tileWidth, tileHeight),
		sdl.Rect(t.x, t.y, tileWidth, tileHeight))
}

type layer struct {
	Base
}

func newLayer() *layer {
	l := &layer{}
	for i := 0; i < 32; i++ {
		for j := 0; j < 24; j++ {
			l.AddChild(&tile{
				x:  i * tileWidth,
				y:  j * tileHeight,
				id: 1, // TODO: load from somewhere
			})
		}
	}
	return l
}

type Terrain struct {
	Base
}

func NewTerrain() *Terrain {
	t := &Terrain{}
	t.AddChild(newLayer())
	return t
}

func InitTerrainTextures(r *sdl.Renderer) error {
	if terrainTileSheet == nil {
		t, err := r.LoadImage("assets/default.png")
		if err != nil {
			return err
		}
		terrainTileSheet = t
	}
	return nil
}
<commit_msg>Made tile terrain work, ish<commit_after>package game

import (
	"time"

	"sdl"
)

const (
	// TODO: replace with bigger sheet
	tileSheetFile                   = "assets/default.png"
	tileWidth, tileHeight           = 32, 32
	tileSheetWidth, tileSheetHeight = 1, 1
)

var (
	terrainTileSheet *sdl.Texture
)

type tile struct {
	// x and y position in pixels - woo arbitrary!
	// id index in tile sheet
	x, y, id int
}

// Tile needs basically no work.
func (t *tile) AddChild(Object)      {}
func (t *tile) Children() []Object   { return nil }
func (t *tile) Update(time.Duration) {}
func (t *tile) Destroy()             {}

func (t *tile) Draw(r *sdl.Renderer) {
	if terrainTileSheet == nil {
		panic("Tilesheet not initialized")
	}
	r.Copy(terrainTileSheet, sdl.Rect((t.id%tileSheetWidth)*32, (t.id/tileSheetWidth)*32, tileWidth, tileHeight),
		sdl.Rect(t.x, t.y, tileWidth, tileHeight))
}

type layer struct {
	Base
}

func newLayer() *layer {
	l := &layer{}
	for i := 0; i < 32; i++ {
		for j := 0; j < 24; j++ {
			l.AddChild(&tile{
				x:  i * tileWidth,
				y:  j * tileHeight,
				id: 0, // TODO: load from somewhere
			})
		}
	}
	return l
}

type Terrain struct {
	Base
}

func NewTerrain() *Terrain {
	t := &Terrain{}
	t.AddChild(newLayer())
	return t
}

func InitTerrainTextures(r *sdl.Renderer) error {
	if terrainTileSheet == nil {
		t, err := r.LoadImage("assets/default.png")
		if err != nil {
			return err
		}
		terrainTileSheet = t
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package benchmark

import (
	"fmt"
	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/kubernetes/pkg/scheduler"
	testutils "k8s.io/kubernetes/test/utils"
	"math"
	"strconv"
	"testing"
	"time"
)

const (
	warning3K    = 100
	threshold3K  = 30
	threshold30K = 30
	threshold60K = 30
)

var (
	basePodTemplate = &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "sched-perf-pod-",
		},
		// TODO: this needs to be configurable.
		Spec: testutils.MakePodSpec(),
	}
	baseNodeTemplate = &v1.Node{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "sample-node-",
		},
		Status: v1.NodeStatus{
			Capacity: v1.ResourceList{
				v1.ResourcePods:   *resource.NewQuantity(110, resource.DecimalSI),
				v1.ResourceCPU:    resource.MustParse("4"),
				v1.ResourceMemory: resource.MustParse("32Gi"),
			},
			Phase: v1.NodeRunning,
			Conditions: []v1.NodeCondition{
				{Type: v1.NodeReady, Status: v1.ConditionTrue},
			},
		},
	}
)

// TestSchedule100Node3KPods schedules 3k pods on 100 nodes.
func TestSchedule100Node3KPods(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping because we want to run short tests")
	}

	config := getBaseConfig(100, 3000)
	err := writePodAndNodeTopologyToConfig(config)
	if err != nil {
		t.Errorf("Misconfiguration happened for nodes/pods chosen to have predicates and priorities")
	}
	min := schedulePods(config)
	if min < threshold3K {
		t.Errorf("Failing: Scheduling rate was too low for an interval, we saw rate of %v, which is the allowed minimum of %v ! ", min, threshold3K)
	} else if min < warning3K {
		fmt.Printf("Warning: pod scheduling throughput for 3k pods was slow for an interval... Saw a interval with very low (%v) scheduling rate!", min)
	} else {
		fmt.Printf("Minimal observed throughput for 3k pod test: %v\n", min)
	}
}

// TestSchedule2000Node60KPods schedules 60k pods on 2000 nodes.
// This test won't fit in normal 10 minutes time window.
// func TestSchedule2000Node60KPods(t *testing.T) {
// 	if testing.Short() {
// 		t.Skip("Skipping because we want to run short tests")
// 	}
// 	config := defaultSchedulerBenchmarkConfig(2000, 60000)
// 	if min := schedulePods(config); min < threshold60K {
// 		t.Errorf("To small pod scheduling throughput for 60k pods. Expected %v got %v", threshold60K, min)
// 	} else {
// 		fmt.Printf("Minimal observed throughput for 60k pod test: %v\n", min)
// 	}
// }

// testConfig contains the some input parameters needed for running test-suite
type testConfig struct {
	numPods                   int
	numNodes                  int
	mutatedNodeTemplate       *v1.Node
	mutatedPodTemplate        *v1.Pod
	schedulerSupportFunctions scheduler.Configurator
	destroyFunc               func()
}

// getBaseConfig returns baseConfig after initializing number of nodes and pods.
func getBaseConfig(nodes int, pods int) *testConfig {
	schedulerConfigFactory, destroyFunc := mustSetupScheduler()
	return &testConfig{
		schedulerSupportFunctions: schedulerConfigFactory,
		destroyFunc:               destroyFunc,
		numNodes:                  nodes,
		numPods:                   pods,
	}
}

// schedulePods schedules specific number of pods on specific number of nodes.
// This is used to learn the scheduling throughput on various
// sizes of cluster and changes as more and more pods are scheduled.
// It won't stop until all pods are scheduled.
// It returns the minimum of throughput over whole run.
func schedulePods(config *testConfig) int32 {
	defer config.destroyFunc()
	prev := 0
	// On startup there may be a latent period where NO scheduling occurs (qps = 0).
	// We are interested in low scheduling rates (i.e. qps=2),
	minQps := int32(math.MaxInt32)
	start := time.Now()
	// Bake in time for the first pod scheduling event.
	for {
		time.Sleep(50 * time.Millisecond)
		scheduled, err := config.schedulerSupportFunctions.GetScheduledPodLister().List(labels.Everything())
		if err != nil {
			glog.Fatalf("%v", err)
		}
		// 30,000 pods -> wait till @ least 300 are scheduled to start measuring.
		// TODO Find out why sometimes there may be scheduling blips in the beginning.
		if len(scheduled) > config.numPods/100 {
			break
		}
	}
	// map minimum QPS entries in a counter, useful for debugging tests.
	qpsStats := map[int]int{}

	// Now that scheduling has started, lets start taking the pulse on how many pods are happening per second.
	for {
		// This can potentially affect performance of scheduler, since List() is done under mutex.
		// Listing 10000 pods is an expensive operation, so running it frequently may impact scheduler.
		// TODO: Setup watch on apiserver and wait until all pods scheduled.
		scheduled, err := config.schedulerSupportFunctions.GetScheduledPodLister().List(labels.Everything())
		if err != nil {
			glog.Fatalf("%v", err)
		}

		// We will be completed when all pods are done being scheduled.
		// return the worst-case-scenario interval that was seen during this time.
		// Note this should never be low due to cold-start, so allow bake in sched time if necessary.
		if len(scheduled) >= config.numPods {
			fmt.Printf("Scheduled %v Pods in %v seconds (%v per second on average). min QPS was %v\n",
				config.numPods, int(time.Since(start)/time.Second), config.numPods/int(time.Since(start)/time.Second), minQps)
			return minQps
		}

		// There's no point in printing it for the last iteration, as the value is random
		qps := len(scheduled) - prev
		qpsStats[qps] += 1
		if int32(qps) < minQps {
			minQps = int32(qps)
		}
		fmt.Printf("%ds\trate: %d\ttotal: %d (qps frequency: %v)\n", time.Since(start)/time.Second, qps, len(scheduled), qpsStats)
		prev = len(scheduled)
		time.Sleep(1 * time.Second)
	}
}

// mutateNodeTemplate returns the modified node needed for creation of nodes.
func (na nodeAffinity) mutateNodeTemplate(node *v1.Node) {
	labels := make(map[string]string)
	for i := 0; i < na.LabelCount; i++ {
		value := strconv.Itoa(i)
		key := na.nodeAffinityKey + value
		labels[key] = value
	}
	node.ObjectMeta.Labels = labels
	return
}

// mutatePodTemplate returns the modified pod template after applying mutations.
func (na nodeAffinity) mutatePodTemplate(pod *v1.Pod) {
	var nodeSelectorRequirements []v1.NodeSelectorRequirement
	for i := 0; i < na.LabelCount; i++ {
		value := strconv.Itoa(i)
		key := na.nodeAffinityKey + value
		nodeSelector := v1.NodeSelectorRequirement{Key: key, Values: []string{value}, Operator: v1.NodeSelectorOpIn}
		nodeSelectorRequirements = append(nodeSelectorRequirements, nodeSelector)
	}
	pod.Spec.Affinity = &v1.Affinity{
		NodeAffinity: &v1.NodeAffinity{
			RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
				NodeSelectorTerms: []v1.NodeSelectorTerm{
					{
						MatchExpressions: nodeSelectorRequirements,
					},
				},
			},
		},
	}
}

// generateNodes generates nodes to be used for scheduling.
func (inputConfig *schedulerPerfConfig) generateNodes(config *testConfig) {
	for i := 0; i < inputConfig.NodeCount; i++ {
		config.schedulerSupportFunctions.GetClient().CoreV1().Nodes().Create(config.mutatedNodeTemplate)

	}
	for i := 0; i < config.numNodes-inputConfig.NodeCount; i++ {
		config.schedulerSupportFunctions.GetClient().CoreV1().Nodes().Create(baseNodeTemplate)

	}
}

// generatePods generates pods to be used for scheduling.
func (inputConfig *schedulerPerfConfig) generatePods(config *testConfig) {
	testutils.CreatePod(config.schedulerSupportFunctions.GetClient(), "sample", inputConfig.PodCount, config.mutatedPodTemplate)
	testutils.CreatePod(config.schedulerSupportFunctions.GetClient(), "sample", config.numPods-inputConfig.PodCount, basePodTemplate)
}

// generatePodAndNodeTopology is the wrapper function for modifying both pods and node objects.
func (inputConfig *schedulerPerfConfig) generatePodAndNodeTopology(config *testConfig) error {
	if config.numNodes < inputConfig.NodeCount || config.numPods < inputConfig.PodCount {
		return fmt.Errorf("NodeCount cannot be greater than numNodes")
	}
	nodeAffinity := inputConfig.NodeAffinity
	// Node template that needs to be mutated.
	mutatedNodeTemplate := baseNodeTemplate
	// Pod template that needs to be mutated.
	mutatedPodTemplate := basePodTemplate
	if nodeAffinity != nil {
		nodeAffinity.mutateNodeTemplate(mutatedNodeTemplate)
		nodeAffinity.mutatePodTemplate(mutatedPodTemplate)

	} // TODO: other predicates/priorities will be processed in subsequent if statements or a switch:).
	config.mutatedPodTemplate = mutatedPodTemplate
	config.mutatedNodeTemplate = mutatedNodeTemplate
	inputConfig.generateNodes(config)
	inputConfig.generatePods(config)
	return nil
}

// writePodAndNodeTopologyToConfig reads a configuration and then applies it to a test configuration.
//TODO: As of now, this function is not doing anything except for reading input values to priority structs.
func writePodAndNodeTopologyToConfig(config *testConfig) error {
	// High Level structure that should be filled for every predicate or priority.
	inputConfig := &schedulerPerfConfig{
		NodeCount: 100,
		PodCount:  3000,
		NodeAffinity: &nodeAffinity{
			nodeAffinityKey: "kubernetes.io/sched-perf-node-affinity-",
			LabelCount:      10,
		},
	}
	err := inputConfig.generatePodAndNodeTopology(config)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>fix integer divide by zero panic<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package benchmark

import (
	"fmt"
	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/kubernetes/pkg/scheduler"
	testutils "k8s.io/kubernetes/test/utils"
	"math"
	"strconv"
	"testing"
	"time"
)

const (
	warning3K    = 100
	threshold3K  = 30
	threshold30K = 30
	threshold60K = 30
)

var (
	basePodTemplate = &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "sched-perf-pod-",
		},
		// TODO: this needs to be configurable.
		Spec: testutils.MakePodSpec(),
	}
	baseNodeTemplate = &v1.Node{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "sample-node-",
		},
		Status: v1.NodeStatus{
			Capacity: v1.ResourceList{
				v1.ResourcePods:   *resource.NewQuantity(110, resource.DecimalSI),
				v1.ResourceCPU:    resource.MustParse("4"),
				v1.ResourceMemory: resource.MustParse("32Gi"),
			},
			Phase: v1.NodeRunning,
			Conditions: []v1.NodeCondition{
				{Type: v1.NodeReady, Status: v1.ConditionTrue},
			},
		},
	}
)

// TestSchedule100Node3KPods schedules 3k pods on 100 nodes.
func TestSchedule100Node3KPods(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping because we want to run short tests")
	}

	config := getBaseConfig(100, 3000)
	err := writePodAndNodeTopologyToConfig(config)
	if err != nil {
		t.Errorf("Misconfiguration happened for nodes/pods chosen to have predicates and priorities")
	}
	min := schedulePods(config)
	if min < threshold3K {
		t.Errorf("Failing: Scheduling rate was too low for an interval, we saw rate of %v, which is the allowed minimum of %v ! ", min, threshold3K)
	} else if min < warning3K {
		fmt.Printf("Warning: pod scheduling throughput for 3k pods was slow for an interval... Saw a interval with very low (%v) scheduling rate!", min)
	} else {
		fmt.Printf("Minimal observed throughput for 3k pod test: %v\n", min)
	}
}

// TestSchedule2000Node60KPods schedules 60k pods on 2000 nodes.
// This test won't fit in normal 10 minutes time window.
// func TestSchedule2000Node60KPods(t *testing.T) {
// 	if testing.Short() {
// 		t.Skip("Skipping because we want to run short tests")
// 	}
// 	config := defaultSchedulerBenchmarkConfig(2000, 60000)
// 	if min := schedulePods(config); min < threshold60K {
// 		t.Errorf("To small pod scheduling throughput for 60k pods. Expected %v got %v", threshold60K, min)
// 	} else {
// 		fmt.Printf("Minimal observed throughput for 60k pod test: %v\n", min)
// 	}
// }

// testConfig contains the some input parameters needed for running test-suite
type testConfig struct {
	numPods                   int
	numNodes                  int
	mutatedNodeTemplate       *v1.Node
	mutatedPodTemplate        *v1.Pod
	schedulerSupportFunctions scheduler.Configurator
	destroyFunc               func()
}

// getBaseConfig returns baseConfig after initializing number of nodes and pods.
func getBaseConfig(nodes int, pods int) *testConfig {
	schedulerConfigFactory, destroyFunc := mustSetupScheduler()
	return &testConfig{
		schedulerSupportFunctions: schedulerConfigFactory,
		destroyFunc:               destroyFunc,
		numNodes:                  nodes,
		numPods:                   pods,
	}
}

// schedulePods schedules specific number of pods on specific number of nodes.
// This is used to learn the scheduling throughput on various
// sizes of cluster and changes as more and more pods are scheduled.
// It won't stop until all pods are scheduled.
// It returns the minimum of throughput over whole run.
func schedulePods(config *testConfig) int32 {
	defer config.destroyFunc()
	prev := 0
	// On startup there may be a latent period where NO scheduling occurs (qps = 0).
	// We are interested in low scheduling rates (i.e. qps=2),
	minQps := int32(math.MaxInt32)
	start := time.Now()
	// Bake in time for the first pod scheduling event.
	for {
		time.Sleep(50 * time.Millisecond)
		scheduled, err := config.schedulerSupportFunctions.GetScheduledPodLister().List(labels.Everything())
		if err != nil {
			glog.Fatalf("%v", err)
		}
		// 30,000 pods -> wait till @ least 300 are scheduled to start measuring.
		// TODO Find out why sometimes there may be scheduling blips in the beginning.
		if len(scheduled) > config.numPods/100 {
			break
		}
	}
	// map minimum QPS entries in a counter, useful for debugging tests.
	qpsStats := map[int]int{}

	// Now that scheduling has started, lets start taking the pulse on how many pods are happening per second.
	for {
		// This can potentially affect performance of scheduler, since List() is done under mutex.
		// Listing 10000 pods is an expensive operation, so running it frequently may impact scheduler.
		// TODO: Setup watch on apiserver and wait until all pods scheduled.
		scheduled, err := config.schedulerSupportFunctions.GetScheduledPodLister().List(labels.Everything())
		if err != nil {
			glog.Fatalf("%v", err)
		}

		// We will be completed when all pods are done being scheduled.
		// return the worst-case-scenario interval that was seen during this time.
		// Note this should never be low due to cold-start, so allow bake in sched time if necessary.
		if len(scheduled) >= config.numPods {
			consumed := int(time.Since(start) / time.Second)
			if consumed <= 0 {
				consumed = 1
			}
			fmt.Printf("Scheduled %v Pods in %v seconds (%v per second on average). min QPS was %v\n",
				config.numPods, consumed, config.numPods/consumed, minQps)
			return minQps
		}

		// There's no point in printing it for the last iteration, as the value is random
		qps := len(scheduled) - prev
		qpsStats[qps] += 1
		if int32(qps) < minQps {
			minQps = int32(qps)
		}
		fmt.Printf("%ds\trate: %d\ttotal: %d (qps frequency: %v)\n", time.Since(start)/time.Second, qps, len(scheduled), qpsStats)
		prev = len(scheduled)
		time.Sleep(1 * time.Second)
	}
}

// mutateNodeTemplate returns the modified node needed for creation of nodes.
func (na nodeAffinity) mutateNodeTemplate(node *v1.Node) {
	labels := make(map[string]string)
	for i := 0; i < na.LabelCount; i++ {
		value := strconv.Itoa(i)
		key := na.nodeAffinityKey + value
		labels[key] = value
	}
	node.ObjectMeta.Labels = labels
	return
}

// mutatePodTemplate returns the modified pod template after applying mutations.
func (na nodeAffinity) mutatePodTemplate(pod *v1.Pod) {
	var nodeSelectorRequirements []v1.NodeSelectorRequirement
	for i := 0; i < na.LabelCount; i++ {
		value := strconv.Itoa(i)
		key := na.nodeAffinityKey + value
		nodeSelector := v1.NodeSelectorRequirement{Key: key, Values: []string{value}, Operator: v1.NodeSelectorOpIn}
		nodeSelectorRequirements = append(nodeSelectorRequirements, nodeSelector)
	}
	pod.Spec.Affinity = &v1.Affinity{
		NodeAffinity: &v1.NodeAffinity{
			RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
				NodeSelectorTerms: []v1.NodeSelectorTerm{
					{
						MatchExpressions: nodeSelectorRequirements,
					},
				},
			},
		},
	}
}

// generateNodes generates nodes to be used for scheduling.
func (inputConfig *schedulerPerfConfig) generateNodes(config *testConfig) {
	for i := 0; i < inputConfig.NodeCount; i++ {
		config.schedulerSupportFunctions.GetClient().CoreV1().Nodes().Create(config.mutatedNodeTemplate)

	}
	for i := 0; i < config.numNodes-inputConfig.NodeCount; i++ {
		config.schedulerSupportFunctions.GetClient().CoreV1().Nodes().Create(baseNodeTemplate)

	}
}

// generatePods generates pods to be used for scheduling.
func (inputConfig *schedulerPerfConfig) generatePods(config *testConfig) {
	testutils.CreatePod(config.schedulerSupportFunctions.GetClient(), "sample", inputConfig.PodCount, config.mutatedPodTemplate)
	testutils.CreatePod(config.schedulerSupportFunctions.GetClient(), "sample", config.numPods-inputConfig.PodCount, basePodTemplate)
}

// generatePodAndNodeTopology is the wrapper function for modifying both pods and node objects.
func (inputConfig *schedulerPerfConfig) generatePodAndNodeTopology(config *testConfig) error {
	if config.numNodes < inputConfig.NodeCount || config.numPods < inputConfig.PodCount {
		return fmt.Errorf("NodeCount cannot be greater than numNodes")
	}
	nodeAffinity := inputConfig.NodeAffinity
	// Node template that needs to be mutated.
	mutatedNodeTemplate := baseNodeTemplate
	// Pod template that needs to be mutated.
	mutatedPodTemplate := basePodTemplate
	if nodeAffinity != nil {
		nodeAffinity.mutateNodeTemplate(mutatedNodeTemplate)
		nodeAffinity.mutatePodTemplate(mutatedPodTemplate)

	} // TODO: other predicates/priorities will be processed in subsequent if statements or a switch:).
	config.mutatedPodTemplate = mutatedPodTemplate
	config.mutatedNodeTemplate = mutatedNodeTemplate
	inputConfig.generateNodes(config)
	inputConfig.generatePods(config)
	return nil
}

// writePodAndNodeTopologyToConfig reads a configuration and then applies it to a test configuration.
//TODO: As of now, this function is not doing anything except for reading input values to priority structs.
func writePodAndNodeTopologyToConfig(config *testConfig) error {
	// High Level structure that should be filled for every predicate or priority.
	inputConfig := &schedulerPerfConfig{
		NodeCount: 100,
		PodCount:  3000,
		NodeAffinity: &nodeAffinity{
			nodeAffinityKey: "kubernetes.io/sched-perf-node-affinity-",
			LabelCount:      10,
		},
	}
	err := inputConfig.generatePodAndNodeTopology(config)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>/**
 * Copyright 2014 @ s1n1 Team.
 * name :
 * author : jarryliu
 * date : 2014-02-05 21:53
 * description :
 * history :
 */
package core

import (
	"fmt"
	"github.com/atnet/gof"
	"log"
	"github.com/garyburd/redigo/redis"
	"time"
)

func createRedisPool(c *gof.Config) *redis.Pool {
	redisHost := c.GetString("redis_host")
	redisDb := c.GetString("redis_db")
	redisPort := c.GetInt("redis_port")
	redisAuth := c.GetString("redis_auth")

	if redisPort <= 0 {
		redisPort = 6379
	}
	redisMaxIdle := c.GetInt("redis_maxIdle")
	if redisMaxIdle <= 0 {
		redisMaxIdle = 10000
	}
	redisIdleTimeout := c.GetInt("redis_idleTimeout")
	if redisIdleTimeout <= 0 {
		redisIdleTimeout = 20000
	}

	return &redis.Pool{
		MaxIdle:     redisMaxIdle,
		IdleTimeout: time.Duration(redisIdleTimeout) * time.Second,
		Dial: func() (redis.Conn, error) {
			dial:
			c, err := redis.Dial("tcp", fmt.Sprintf("%s:%d", redisHost, redisPort))
			if err != nil {
				for {
					log.Printf("[ Redis] - redis(%s:%d) dial failed - %s , Redial after 5 seconds\n",
						redisHost, redisPort, err.Error())
					time.Sleep(time.Second * 5)
					goto dial
				}
			}

			if len(redisAuth) != 0 {
				if _,err := c.Do("AUTH",redisAuth);err != nil{
					c.Close()
					log.Fatalf("[ Redis][ AUTH] - %s\n",err.Error())
				}
			}
			if _, err = c.Do("SELECT", redisDb); err != nil {
				c.Close()
				log.Fatalf("FATAL: redis(%s:%d) initialize failed - %s",
					redisHost, redisPort, err.Error())
			}

			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
<commit_msg>redis auth<commit_after>/**
 * Copyright 2014 @ s1n1 Team.
 * name :
 * author : jarryliu
 * date : 2014-02-05 21:53
 * description :
 * history :
 */
package core

import (
	"fmt"
	"github.com/atnet/gof"
	"log"
	"github.com/garyburd/redigo/redis"
	"time"
)

func createRedisPool(c *gof.Config) *redis.Pool {
	redisHost := c.GetString("redis_host")
	redisDb := c.GetString("redis_db")
	redisPort := c.GetInt("redis_port")
	redisAuth := c.GetString("redis_auth")

	if redisPort <= 0 {
		redisPort = 6379
	}
	redisMaxIdle := c.GetInt("redis_maxIdle")
	if redisMaxIdle <= 0 {
		redisMaxIdle = 10000
	}
	redisIdleTimeout := c.GetInt("redis_idleTimeout")
	if redisIdleTimeout <= 0 {
		redisIdleTimeout = 20000
	}

	return &redis.Pool{
		MaxIdle:     redisMaxIdle,
		IdleTimeout: time.Duration(redisIdleTimeout) * time.Second,
		Dial: func() (redis.Conn, error) {
			dial:
			c, err := redis.Dial("tcp", fmt.Sprintf("%s:%d", redisHost, redisPort))
			if err != nil {
				for {
					log.Printf("[ Redis] - redis(%s:%d) dial failed - %s , Redial after 5 seconds\n",
						redisHost, redisPort, err.Error())
					time.Sleep(time.Second * 5)
					goto dial
				}
			}

			if len(redisAuth) != 0 {
				if _,err := c.Do("AUTH",redisAuth);err != nil{
					c.Close()
					log.Fatalf("[ Redis][ AUTH] - %s\n",err.Error())
				}
			}
			if _, err = c.Do("SELECT", redisDb); err != nil {
				c.Close()
				log.Fatalf("[ Redis][ SELECT] - redis(%s:%d) select db failed - %s",
					redisHost, redisPort, err.Error())
			}

			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
<|endoftext|>


<commit_before>// +build linux

package unshare

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"syscall"

	"github.com/containers/buildah/util"
	"github.com/containers/storage/pkg/reexec"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
)

// Cmd wraps an exec.Cmd created by the reexec package in unshare(), and
// handles setting ID maps and other related settings by triggering
// initialization code in the child.
type Cmd struct {
	*exec.Cmd
	UnshareFlags               int
	UseNewuidmap               bool
	UidMappings                []specs.LinuxIDMapping
	UseNewgidmap               bool
	GidMappings                []specs.LinuxIDMapping
	GidMappingsEnableSetgroups bool
	Setsid                     bool
	Setpgrp                    bool
	Ctty                       *os.File
	OOMScoreAdj                *int
	Hook                       func(pid int) error
}

// Command creates a new Cmd which can be customized.
func Command(args ...string) *Cmd {
	cmd := reexec.Command(args...)
	return &Cmd{
		Cmd: cmd,
	}
}

func (c *Cmd) Start() error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// Set an environment variable to tell the child to synchronize its startup.
	if c.Env == nil {
		c.Env = os.Environ()
	}
	c.Env = append(c.Env, fmt.Sprintf("_Buildah-unshare=%d", c.UnshareFlags))

	// Create the pipe for reading the child's PID.
	pidRead, pidWrite, err := os.Pipe()
	if err != nil {
		return errors.Wrapf(err, "error creating pid pipe")
	}
	c.Env = append(c.Env, fmt.Sprintf("_Buildah-pid-pipe=%d", len(c.ExtraFiles)+3))
	c.ExtraFiles = append(c.ExtraFiles, pidWrite)

	// Create the pipe for letting the child know to proceed.
	continueRead, continueWrite, err := os.Pipe()
	if err != nil {
		pidRead.Close()
		pidWrite.Close()
		return errors.Wrapf(err, "error creating pid pipe")
	}
	c.Env = append(c.Env, fmt.Sprintf("_Buildah-continue-pipe=%d", len(c.ExtraFiles)+3))
	c.ExtraFiles = append(c.ExtraFiles, continueRead)

	// Pass along other instructions.
	if c.Setsid {
		c.Env = append(c.Env, "_Buildah-setsid=1")
	}
	if c.Setpgrp {
		c.Env = append(c.Env, "_Buildah-setpgrp=1")
	}
	if c.Ctty != nil {
		c.Env = append(c.Env, fmt.Sprintf("_Buildah-ctty=%d", len(c.ExtraFiles)+3))
		c.ExtraFiles = append(c.ExtraFiles, c.Ctty)
	}

	// Make sure we clean up our pipes.
	defer func() {
		if pidRead != nil {
			pidRead.Close()
		}
		if pidWrite != nil {
			pidWrite.Close()
		}
		if continueRead != nil {
			continueRead.Close()
		}
		if continueWrite != nil {
			continueWrite.Close()
		}
	}()

	// Start the new process.
	err = c.Cmd.Start()
	if err != nil {
		return err
	}

	// Close the ends of the pipes that the parent doesn't need.
	continueRead.Close()
	continueRead = nil
	pidWrite.Close()
	pidWrite = nil

	// Read the child's PID from the pipe.
	pidString := ""
	b := new(bytes.Buffer)
	io.Copy(b, pidRead)
	pidString = b.String()
	pid, err := strconv.Atoi(pidString)
	if err != nil {
		fmt.Fprintf(continueWrite, "error parsing PID %q: %v", pidString, err)
		return errors.Wrapf(err, "error parsing PID %q", pidString)
	}
	pidString = fmt.Sprintf("%d", pid)

	// If we created a new user namespace, set any specified mappings.
	if c.UnshareFlags&syscall.CLONE_NEWUSER != 0 {
		// Always set "setgroups".
		setgroups, err := os.OpenFile(fmt.Sprintf("/proc/%s/setgroups", pidString), os.O_TRUNC|os.O_WRONLY, 0)
		if err != nil {
			fmt.Fprintf(continueWrite, "error opening setgroups: %v", err)
			return errors.Wrapf(err, "error opening /proc/%s/setgroups", pidString)
		}
		defer setgroups.Close()
		if c.GidMappingsEnableSetgroups {
			if _, err := fmt.Fprintf(setgroups, "allow"); err != nil {
				fmt.Fprintf(continueWrite, "error writing \"allow\" to setgroups: %v", err)
				return errors.Wrapf(err, "error opening \"allow\" to /proc/%s/setgroups", pidString)
			}
		} else {
			if _, err := fmt.Fprintf(setgroups, "deny"); err != nil {
				fmt.Fprintf(continueWrite, "error writing \"deny\" to setgroups: %v", err)
				return errors.Wrapf(err, "error writing \"deny\" to /proc/%s/setgroups", pidString)
			}
		}

		if len(c.UidMappings) == 0 || len(c.GidMappings) == 0 {
			uidmap, gidmap, err := util.GetHostIDMappings("")
			if err != nil {
				fmt.Fprintf(continueWrite, "error reading ID mappings in parent: %v", err)
				return errors.Wrapf(err, "error reading ID mappings in parent")
			}
			if len(c.UidMappings) == 0 {
				c.UidMappings = uidmap
				for i := range c.UidMappings {
					c.UidMappings[i].HostID = c.UidMappings[i].ContainerID
				}
			}
			if len(c.GidMappings) == 0 {
				c.GidMappings = gidmap
				for i := range c.GidMappings {
					c.GidMappings[i].HostID = c.GidMappings[i].ContainerID
				}
			}
		}

		if len(c.GidMappings) > 0 {
			// Build the GID map, since writing to the proc file has to be done all at once.
			g := new(bytes.Buffer)
			for _, m := range c.GidMappings {
				fmt.Fprintf(g, "%d %d %d\n", m.ContainerID, m.HostID, m.Size)
			}
			// Set the GID map.
			if c.UseNewgidmap {
				cmd := exec.Command("newgidmap", append([]string{pidString}, strings.Fields(strings.Replace(g.String(), "\n", " ", -1))...)...)
				g.Reset()
				cmd.Stdout = g
				cmd.Stderr = g
				err := cmd.Run()
				if err != nil {
					fmt.Fprintf(continueWrite, "error running newgidmap: %v: %s", err, g.String())
					return errors.Wrapf(err, "error running newgidmap: %s", g.String())
				}
			} else {
				gidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/gid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0)
				if err != nil {
					fmt.Fprintf(continueWrite, "error opening /proc/%s/gid_map: %v", pidString, err)
					return errors.Wrapf(err, "error opening /proc/%s/gid_map", pidString)
				}
				defer gidmap.Close()
				if _, err := fmt.Fprintf(gidmap, "%s", g.String()); err != nil {
					fmt.Fprintf(continueWrite, "error writing %q to /proc/%s/gid_map: %v", g.String(), pidString, err)
					return errors.Wrapf(err, "error writing %q to /proc/%s/gid_map", g.String(), pidString)
				}
			}
		}

		if len(c.UidMappings) > 0 {
			// Build the UID map, since writing to the proc file has to be done all at once.
			u := new(bytes.Buffer)
			for _, m := range c.UidMappings {
				fmt.Fprintf(u, "%d %d %d\n", m.ContainerID, m.HostID, m.Size)
			}
			// Set the GID map.
			if c.UseNewuidmap {
				cmd := exec.Command("newuidmap", append([]string{pidString}, strings.Fields(strings.Replace(u.String(), "\n", " ", -1))...)...)
				u.Reset()
				cmd.Stdout = u
				cmd.Stderr = u
				err := cmd.Run()
				if err != nil {
					fmt.Fprintf(continueWrite, "error running newuidmap: %v: %s", err, u.String())
					return errors.Wrapf(err, "error running newuidmap: %s", u.String())
				}
			} else {
				uidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/uid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0)
				if err != nil {
					fmt.Fprintf(continueWrite, "error opening /proc/%s/uid_map: %v", pidString, err)
					return errors.Wrapf(err, "error opening /proc/%s/uid_map", pidString)
				}
				defer uidmap.Close()
				if _, err := fmt.Fprintf(uidmap, "%s", u.String()); err != nil {
					fmt.Fprintf(continueWrite, "error writing %q to /proc/%s/uid_map: %v", u.String(), pidString, err)
					return errors.Wrapf(err, "error writing %q to /proc/%s/uid_map", u.String(), pidString)
				}
			}
		}
	}

	if c.OOMScoreAdj != nil {
		oomScoreAdj, err := os.OpenFile(fmt.Sprintf("/proc/%s/oom_score_adj", pidString), os.O_TRUNC|os.O_WRONLY, 0)
		if err != nil {
			fmt.Fprintf(continueWrite, "error opening oom_score_adj: %v", err)
			return errors.Wrapf(err, "error opening /proc/%s/oom_score_adj", pidString)
		}
		defer oomScoreAdj.Close()
		if _, err := fmt.Fprintf(oomScoreAdj, "%d\n", *c.OOMScoreAdj); err != nil {
			fmt.Fprintf(continueWrite, "error writing \"%d\" to oom_score_adj: %v", c.OOMScoreAdj, err)
			return errors.Wrapf(err, "error writing \"%d\" to /proc/%s/oom_score_adj", c.OOMScoreAdj, pidString)
		}
	}
	// Run any additional setup that we want to do before the child starts running proper.
	if c.Hook != nil {
		if err = c.Hook(pid); err != nil {
			fmt.Fprintf(continueWrite, "hook error: %v", err)
			return err
		}
	}

	return nil
}

func (c *Cmd) Run() error {
	if err := c.Start(); err != nil {
		return err
	}
	return c.Wait()
}

func (c *Cmd) CombinedOutput() ([]byte, error) {
	return nil, errors.New("unshare: CombinedOutput() not implemented")
}

func (c *Cmd) Output() ([]byte, error) {
	return nil, errors.New("unshare: Output() not implemented")
}
<commit_msg>unshare: support libpod rootless pkg<commit_after>// +build linux

package unshare

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"syscall"

	"github.com/containers/buildah/util"
	"github.com/containers/storage/pkg/reexec"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
)

// Cmd wraps an exec.Cmd created by the reexec package in unshare(), and
// handles setting ID maps and other related settings by triggering
// initialization code in the child.
type Cmd struct {
	*exec.Cmd
	UnshareFlags               int
	UseNewuidmap               bool
	UidMappings                []specs.LinuxIDMapping
	UseNewgidmap               bool
	GidMappings                []specs.LinuxIDMapping
	GidMappingsEnableSetgroups bool
	Setsid                     bool
	Setpgrp                    bool
	Ctty                       *os.File
	OOMScoreAdj                *int
	Hook                       func(pid int) error
}

// Command creates a new Cmd which can be customized.
func Command(args ...string) *Cmd {
	cmd := reexec.Command(args...)
	return &Cmd{
		Cmd: cmd,
	}
}

func (c *Cmd) Start() error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// Set an environment variable to tell the child to synchronize its startup.
	if c.Env == nil {
		c.Env = os.Environ()
	}
	c.Env = append(c.Env, fmt.Sprintf("_Buildah-unshare=%d", c.UnshareFlags))

	// Please the libpod "rootless" package to find the expected env variables.
	c.Env = append(c.Env, "_LIBPOD_USERNS_CONFIGURED=done")
	c.Env = append(c.Env, fmt.Sprintf("_LIBPOD_ROOTLESS_UID=%d", os.Geteuid()))

	// Create the pipe for reading the child's PID.
	pidRead, pidWrite, err := os.Pipe()
	if err != nil {
		return errors.Wrapf(err, "error creating pid pipe")
	}
	c.Env = append(c.Env, fmt.Sprintf("_Buildah-pid-pipe=%d", len(c.ExtraFiles)+3))
	c.ExtraFiles = append(c.ExtraFiles, pidWrite)

	// Create the pipe for letting the child know to proceed.
	continueRead, continueWrite, err := os.Pipe()
	if err != nil {
		pidRead.Close()
		pidWrite.Close()
		return errors.Wrapf(err, "error creating pid pipe")
	}
	c.Env = append(c.Env, fmt.Sprintf("_Buildah-continue-pipe=%d", len(c.ExtraFiles)+3))
	c.ExtraFiles = append(c.ExtraFiles, continueRead)

	// Pass along other instructions.
	if c.Setsid {
		c.Env = append(c.Env, "_Buildah-setsid=1")
	}
	if c.Setpgrp {
		c.Env = append(c.Env, "_Buildah-setpgrp=1")
	}
	if c.Ctty != nil {
		c.Env = append(c.Env, fmt.Sprintf("_Buildah-ctty=%d", len(c.ExtraFiles)+3))
		c.ExtraFiles = append(c.ExtraFiles, c.Ctty)
	}

	// Make sure we clean up our pipes.
	defer func() {
		if pidRead != nil {
			pidRead.Close()
		}
		if pidWrite != nil {
			pidWrite.Close()
		}
		if continueRead != nil {
			continueRead.Close()
		}
		if continueWrite != nil {
			continueWrite.Close()
		}
	}()

	// Start the new process.
	err = c.Cmd.Start()
	if err != nil {
		return err
	}

	// Close the ends of the pipes that the parent doesn't need.
	continueRead.Close()
	continueRead = nil
	pidWrite.Close()
	pidWrite = nil

	// Read the child's PID from the pipe.
	pidString := ""
	b := new(bytes.Buffer)
	io.Copy(b, pidRead)
	pidString = b.String()
	pid, err := strconv.Atoi(pidString)
	if err != nil {
		fmt.Fprintf(continueWrite, "error parsing PID %q: %v", pidString, err)
		return errors.Wrapf(err, "error parsing PID %q", pidString)
	}
	pidString = fmt.Sprintf("%d", pid)

	// If we created a new user namespace, set any specified mappings.
	if c.UnshareFlags&syscall.CLONE_NEWUSER != 0 {
		// Always set "setgroups".
		setgroups, err := os.OpenFile(fmt.Sprintf("/proc/%s/setgroups", pidString), os.O_TRUNC|os.O_WRONLY, 0)
		if err != nil {
			fmt.Fprintf(continueWrite, "error opening setgroups: %v", err)
			return errors.Wrapf(err, "error opening /proc/%s/setgroups", pidString)
		}
		defer setgroups.Close()
		if c.GidMappingsEnableSetgroups {
			if _, err := fmt.Fprintf(setgroups, "allow"); err != nil {
				fmt.Fprintf(continueWrite, "error writing \"allow\" to setgroups: %v", err)
				return errors.Wrapf(err, "error opening \"allow\" to /proc/%s/setgroups", pidString)
			}
		} else {
			if _, err := fmt.Fprintf(setgroups, "deny"); err != nil {
				fmt.Fprintf(continueWrite, "error writing \"deny\" to setgroups: %v", err)
				return errors.Wrapf(err, "error writing \"deny\" to /proc/%s/setgroups", pidString)
			}
		}

		if len(c.UidMappings) == 0 || len(c.GidMappings) == 0 {
			uidmap, gidmap, err := util.GetHostIDMappings("")
			if err != nil {
				fmt.Fprintf(continueWrite, "error reading ID mappings in parent: %v", err)
				return errors.Wrapf(err, "error reading ID mappings in parent")
			}
			if len(c.UidMappings) == 0 {
				c.UidMappings = uidmap
				for i := range c.UidMappings {
					c.UidMappings[i].HostID = c.UidMappings[i].ContainerID
				}
			}
			if len(c.GidMappings) == 0 {
				c.GidMappings = gidmap
				for i := range c.GidMappings {
					c.GidMappings[i].HostID = c.GidMappings[i].ContainerID
				}
			}
		}

		if len(c.GidMappings) > 0 {
			// Build the GID map, since writing to the proc file has to be done all at once.
			g := new(bytes.Buffer)
			for _, m := range c.GidMappings {
				fmt.Fprintf(g, "%d %d %d\n", m.ContainerID, m.HostID, m.Size)
			}
			// Set the GID map.
			if c.UseNewgidmap {
				cmd := exec.Command("newgidmap", append([]string{pidString}, strings.Fields(strings.Replace(g.String(), "\n", " ", -1))...)...)
				g.Reset()
				cmd.Stdout = g
				cmd.Stderr = g
				err := cmd.Run()
				if err != nil {
					fmt.Fprintf(continueWrite, "error running newgidmap: %v: %s", err, g.String())
					return errors.Wrapf(err, "error running newgidmap: %s", g.String())
				}
			} else {
				gidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/gid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0)
				if err != nil {
					fmt.Fprintf(continueWrite, "error opening /proc/%s/gid_map: %v", pidString, err)
					return errors.Wrapf(err, "error opening /proc/%s/gid_map", pidString)
				}
				defer gidmap.Close()
				if _, err := fmt.Fprintf(gidmap, "%s", g.String()); err != nil {
					fmt.Fprintf(continueWrite, "error writing %q to /proc/%s/gid_map: %v", g.String(), pidString, err)
					return errors.Wrapf(err, "error writing %q to /proc/%s/gid_map", g.String(), pidString)
				}
			}
		}

		if len(c.UidMappings) > 0 {
			// Build the UID map, since writing to the proc file has to be done all at once.
			u := new(bytes.Buffer)
			for _, m := range c.UidMappings {
				fmt.Fprintf(u, "%d %d %d\n", m.ContainerID, m.HostID, m.Size)
			}
			// Set the GID map.
			if c.UseNewuidmap {
				cmd := exec.Command("newuidmap", append([]string{pidString}, strings.Fields(strings.Replace(u.String(), "\n", " ", -1))...)...)
				u.Reset()
				cmd.Stdout = u
				cmd.Stderr = u
				err := cmd.Run()
				if err != nil {
					fmt.Fprintf(continueWrite, "error running newuidmap: %v: %s", err, u.String())
					return errors.Wrapf(err, "error running newuidmap: %s", u.String())
				}
			} else {
				uidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/uid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0)
				if err != nil {
					fmt.Fprintf(continueWrite, "error opening /proc/%s/uid_map: %v", pidString, err)
					return errors.Wrapf(err, "error opening /proc/%s/uid_map", pidString)
				}
				defer uidmap.Close()
				if _, err := fmt.Fprintf(uidmap, "%s", u.String()); err != nil {
					fmt.Fprintf(continueWrite, "error writing %q to /proc/%s/uid_map: %v", u.String(), pidString, err)
					return errors.Wrapf(err, "error writing %q to /proc/%s/uid_map", u.String(), pidString)
				}
			}
		}
	}

	if c.OOMScoreAdj != nil {
		oomScoreAdj, err := os.OpenFile(fmt.Sprintf("/proc/%s/oom_score_adj", pidString), os.O_TRUNC|os.O_WRONLY, 0)
		if err != nil {
			fmt.Fprintf(continueWrite, "error opening oom_score_adj: %v", err)
			return errors.Wrapf(err, "error opening /proc/%s/oom_score_adj", pidString)
		}
		defer oomScoreAdj.Close()
		if _, err := fmt.Fprintf(oomScoreAdj, "%d\n", *c.OOMScoreAdj); err != nil {
			fmt.Fprintf(continueWrite, "error writing \"%d\" to oom_score_adj: %v", c.OOMScoreAdj, err)
			return errors.Wrapf(err, "error writing \"%d\" to /proc/%s/oom_score_adj", c.OOMScoreAdj, pidString)
		}
	}
	// Run any additional setup that we want to do before the child starts running proper.
	if c.Hook != nil {
		if err = c.Hook(pid); err != nil {
			fmt.Fprintf(continueWrite, "hook error: %v", err)
			return err
		}
	}

	return nil
}

func (c *Cmd) Run() error {
	if err := c.Start(); err != nil {
		return err
	}
	return c.Wait()
}

func (c *Cmd) CombinedOutput() ([]byte, error) {
	return nil, errors.New("unshare: CombinedOutput() not implemented")
}

func (c *Cmd) Output() ([]byte, error) {
	return nil, errors.New("unshare: Output() not implemented")
}
<|endoftext|>


<commit_before>package jsonpath

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
)

type Slice struct{ start, stop int }

type pathTypeMismatch struct{}
type notFound struct{}
type unknownPathType struct{}

func recursiveGet(data interface{}, path []interface{}) interface{} {

	if len(path) == 0 {
		switch data.(type) {
		case string:
			return data.(string)
		case float64:
			return data.(float64)
		case bool:
			return data.(bool)
		case nil:
			return nil
		case []interface{}:
			return data
		case map[string]interface{}:
			return data
		}
	}

	switch path[0].(type) {

	case string:
		switch data.(type) {
		case map[string]interface{}:
			for k, v := range data.(map[string]interface{}) {
				if k == path[0].(string) {
					return recursiveGet(v, path[1:])
				}
			}
			return notFound{}
		default:
			return pathTypeMismatch{}
		}

	case int:
		switch data.(type) {
		case []interface{}:
			for i, v := range data.([]interface{}) {
				if i == path[0].(int) {
					return recursiveGet(v, path[1:])
				}
			}
			return notFound{}
		default:
			return pathTypeMismatch{}
		}

	case func(int, interface{}) bool:
		switch data.(type) {
		case []interface{}:
			ret := make([]interface{}, 0)
			for i, v := range data.([]interface{}) {
				if path[0].(func(int, interface{}) bool)(i, v) {
					val := recursiveGet(v, path[1:])
					switch val.(type) {
					case pathTypeMismatch, notFound:
						continue
					case unknownPathType:
						return val
					}
					ret = append(ret, val)
				}
			}
			return ret
		}

	case func(string, interface{}) bool:
		switch data.(type) {
		case map[string]interface{}:
			ret := make([]interface{}, 0)
			for k, v := range data.(map[string]interface{}) {
				if path[0].(func(string, interface{}) bool)(k, v) {
					val := recursiveGet(v, path[1:])
					switch val.(type) {
					case pathTypeMismatch, notFound:
						continue
					case unknownPathType:
						return val
					}
					ret = append(ret, val)
				}
			}
			return ret
		}

	case Slice:
		switch data.(type) {
		case []interface{}:
			ret := make([]interface{}, 0)
			start := path[0].(Slice).start
			stop := path[0].(Slice).stop
			for i, v := range data.([]interface{}) {
				if start <= i && i < stop {
					ret = append(ret, recursiveGet(v, path[1:]))
				}
			}
			return ret
		}
	}

	return unknownPathType{}

}

func Get(decoded interface{}, path []interface{}, defaultValue interface{}) (interface{}, error) {
	val := recursiveGet(decoded, path)
	switch val.(type) {
	case notFound:
		return defaultValue, fmt.Errorf("not found")
	case unknownPathType:
		return defaultValue, fmt.Errorf("unknown path type")
	case pathTypeMismatch:
		return defaultValue, fmt.Errorf("mismatched path type")
	}
	return val, nil
}

func GetString(decoded interface{}, path []interface{}, defaultValue string) (string, error) {
	value, err := Get(decoded, path, nil)
	if err != nil {
		return defaultValue, err
	}
	switch value.(type) {
	case string:
		return value.(string), nil
	}
	return defaultValue, fmt.Errorf("unexpected type")
}

func GetNumber(decoded interface{}, path []interface{}, defaultValue float64) (float64, error) {
	value, err := Get(decoded, path, nil)
	if err != nil {
		return defaultValue, err
	}
	switch value.(type) {
	case float64:
		return value.(float64), nil
	}
	return defaultValue, fmt.Errorf("unexpected type")
}

func GetBool(decoded interface{}, path []interface{}, defaultValue bool) (bool, error) {
	value, err := Get(decoded, path, nil)
	if err != nil {
		return defaultValue, err
	}
	switch value.(type) {
	case bool:
		return value.(bool), nil
	}
	return defaultValue, fmt.Errorf("unexpected type")
}

func DecodeJsonString(s string) (interface{}, error) {
	r := bytes.NewBufferString(s)
	return DecodeJsonReader(r)
}

func DecodeJsonReader(r io.Reader) (interface{}, error) {
	var data interface{}
	dec := json.NewDecoder(r)
	err := dec.Decode(&data)
	return data, err
}

func Read(s string, path []interface{}, defaultValue interface{}) (interface{}, error) {
	data, err := DecodeJsonString(s)
	if err != nil {
		return nil, err
	}
	return Get(data, path, defaultValue)
}

func ReadString(s string, path []interface{}, defaultValue string) (string, error) {
	data, err := DecodeJsonString(s)
	if err != nil {
		return defaultValue, err
	}
	return GetString(data, path, defaultValue)
}

func ReadNumber(s string, path []interface{}, defaultValue float64) (float64, error) {
	data, err := DecodeJsonString(s)
	if err != nil {
		return defaultValue, err
	}
	return GetNumber(data, path, defaultValue)
}

func ReadBool(s string, path []interface{}, defaultValue bool) (bool, error) {
	data, err := DecodeJsonString(s)
	if err != nil {
		return defaultValue, err
	}
	return GetBool(data, path, defaultValue)
}
<commit_msg>Refactoring interface for string/io.Reader<commit_after>package jsonpath

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
)

type Slice struct{ start, stop int }

type pathTypeMismatch struct{}
type notFound struct{}
type unknownPathType struct{}

func recursiveGet(data interface{}, path []interface{}) interface{} {

	if len(path) == 0 {
		switch data.(type) {
		case string:
			return data.(string)
		case float64:
			return data.(float64)
		case bool:
			return data.(bool)
		case nil:
			return nil
		case []interface{}:
			return data
		case map[string]interface{}:
			return data
		}
	}

	switch path[0].(type) {

	case string:
		switch data.(type) {
		case map[string]interface{}:
			for k, v := range data.(map[string]interface{}) {
				if k == path[0].(string) {
					return recursiveGet(v, path[1:])
				}
			}
			return notFound{}
		default:
			return pathTypeMismatch{}
		}

	case int:
		switch data.(type) {
		case []interface{}:
			for i, v := range data.([]interface{}) {
				if i == path[0].(int) {
					return recursiveGet(v, path[1:])
				}
			}
			return notFound{}
		default:
			return pathTypeMismatch{}
		}

	case func(int, interface{}) bool:
		switch data.(type) {
		case []interface{}:
			ret := make([]interface{}, 0)
			for i, v := range data.([]interface{}) {
				if path[0].(func(int, interface{}) bool)(i, v) {
					val := recursiveGet(v, path[1:])
					switch val.(type) {
					case pathTypeMismatch, notFound:
						continue
					case unknownPathType:
						return val
					}
					ret = append(ret, val)
				}
			}
			return ret
		}

	case func(string, interface{}) bool:
		switch data.(type) {
		case map[string]interface{}:
			ret := make([]interface{}, 0)
			for k, v := range data.(map[string]interface{}) {
				if path[0].(func(string, interface{}) bool)(k, v) {
					val := recursiveGet(v, path[1:])
					switch val.(type) {
					case pathTypeMismatch, notFound:
						continue
					case unknownPathType:
						return val
					}
					ret = append(ret, val)
				}
			}
			return ret
		}

	case Slice:
		switch data.(type) {
		case []interface{}:
			ret := make([]interface{}, 0)
			start := path[0].(Slice).start
			stop := path[0].(Slice).stop
			for i, v := range data.([]interface{}) {
				if start <= i && i < stop {
					ret = append(ret, recursiveGet(v, path[1:]))
				}
			}
			return ret
		}
	}

	return unknownPathType{}

}

func Get(decoded interface{}, path []interface{}, defaultValue interface{}) (interface{}, error) {
	val := recursiveGet(decoded, path)
	switch val.(type) {
	case notFound:
		return defaultValue, fmt.Errorf("not found")
	case unknownPathType:
		return defaultValue, fmt.Errorf("unknown path type")
	case pathTypeMismatch:
		return defaultValue, fmt.Errorf("mismatched path type")
	}
	return val, nil
}

func GetString(decoded interface{}, path []interface{}, defaultValue string) (string, error) {
	value, err := Get(decoded, path, nil)
	if err != nil {
		return defaultValue, err
	}
	switch value.(type) {
	case string:
		return value.(string), nil
	}
	return defaultValue, fmt.Errorf("unexpected type")
}

func GetNumber(decoded interface{}, path []interface{}, defaultValue float64) (float64, error) {
	value, err := Get(decoded, path, nil)
	if err != nil {
		return defaultValue, err
	}
	switch value.(type) {
	case float64:
		return value.(float64), nil
	}
	return defaultValue, fmt.Errorf("unexpected type")
}

func GetBool(decoded interface{}, path []interface{}, defaultValue bool) (bool, error) {
	value, err := Get(decoded, path, nil)
	if err != nil {
		return defaultValue, err
	}
	switch value.(type) {
	case bool:
		return value.(bool), nil
	}
	return defaultValue, fmt.Errorf("unexpected type")
}

func DecodeString(s string) (interface{}, error) {
	r := bytes.NewBufferString(s)
	return DecodeReader(r)
}

func DecodeReader(r io.Reader) (interface{}, error) {
	var data interface{}
	dec := json.NewDecoder(r)
	err := dec.Decode(&data)
	return data, err
}

func Read(s string, path []interface{}, defaultValue interface{}) (interface{}, error) {
	data, err := DecodeString(s)
	if err != nil {
		return nil, err
	}
	return Get(data, path, defaultValue)
}

func ReadString(s string, path []interface{}, defaultValue string) (string, error) {
	data, err := DecodeString(s)
	if err != nil {
		return defaultValue, err
	}
	return GetString(data, path, defaultValue)
}

func ReadNumber(s string, path []interface{}, defaultValue float64) (float64, error) {
	data, err := DecodeString(s)
	if err != nil {
		return defaultValue, err
	}
	return GetNumber(data, path, defaultValue)
}

func ReadBool(s string, path []interface{}, defaultValue bool) (bool, error) {
	data, err := DecodeString(s)
	if err != nil {
		return defaultValue, err
	}
	return GetBool(data, path, defaultValue)
}
<|endoftext|>