<commit_before>package myslave

import (
	"context"
	"fmt"
	"time"

	"github.com/funkygao/dbus/pkg/checkpoint"
	"github.com/funkygao/dbus/pkg/model"
	log "github.com/funkygao/log4go"
	uuid "github.com/satori/go.uuid"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

// StopReplication stops the slave and do necessary cleanups.
func (m *MySlave) StopReplication() {
	if !m.started.Get() {
		return
	}

	log.Debug("[%s] stopping replication from %s", m.name, m.masterAddr)

	m.r.Close()
	m.m.Close()

	if err := m.p.Shutdown(); err != nil {
		log.Error("[%s] %s", m.name, err)
	}

	m.started.Set(false)
}

// StartReplication start the mysql binlog replication.
// TODO graceful shutdown
// TODO GTID
func (m *MySlave) StartReplication(ready chan struct{}) {
	m.started.Set(true)

	m.rowsEvent = make(chan *model.RowsEvent, m.c.Int("event_buffer_len", 100))
	m.errors = make(chan error, 1)

	m.r = replication.NewBinlogSyncer(&replication.BinlogSyncerConfig{
		ServerID:        uint32(m.c.Int("server_id", 137)), // 137 unique enough? TODO
		Flavor:          m.c.String("flavor", mysql.MySQLFlavor),
		Host:            m.host,
		Port:            m.port,
		User:            m.user,
		Password:        m.passwd,
		SemiSyncEnabled: m.c.Bool("semi_sync", false),
	})

	// resume replication position from the checkpoint
	err := m.p.LastPersistedState(m.state)
	if err != nil && err != checkpoint.ErrStateNotFound {
		close(ready)
		m.emitFatalError(err)
		return
	}

	file, offset := m.state.File, m.state.Offset

	var syncer *replication.BinlogStreamer
	if m.GTID {
		// SELECT @@gtid_mode
		// SHOW GLOBAL VARIABLES LIKE 'SERVER_UUID'
		var masterUUID uuid.UUID
		// 07c93cd7-a7d3-12a5-94e1-a0369a7c3790:1-313225133
		set, _ := mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d-%d", masterUUID.String(), 1, 2))
		syncer, err = m.r.StartSyncGTID(set)
	} else {
		syncer, err = m.r.StartSync(mysql.Position{
			Name: file,
			Pos:  offset,
		})
	}

	if err != nil {
		// unrecoverable err encountered
		// e,g.
		// ERROR 1045 (28000): Access denied for user 'xx'@'1.1.1.1'
		close(ready)
		m.emitFatalError(err)
		return
	}

	close(ready)
	log.Debug("[%s] ready to receive mysql binlog from %s", m.name, m.masterAddr)

	timeout := time.Second
	maxTimeout := time.Minute
	for m.started.Get() {
		// what if the conn broken?
		// 2017/02/08 08:31:39 binlogsyncer.go:529: [info] receive EOF packet, retry ReadPacket
		// 2017/02/08 08:31:39 binlogsyncer.go:484: [error] connection was bad
		// 2017/02/08 08:31:40 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:40 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:40 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		// 2017/02/08 08:31:42 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:42 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:42 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		//
		// or invalid position sent?
		// 2017/02/08 08:42:09 binlogstreamer.go:47: [error] close sync with err: ERROR 1236 (HY000): Client requested master to start replication from position > file size; the first event 'mysql.000004' at 2274, the last event read from '/usr/local/var/mysql.000004' at 4, the last byte read from '/usr/local/var/mysql.000004' at 4.
		//
		// [02/08/17 17:39:20] [EROR] ( mysqlbinlog.go:64) backoff 5s: invalid table id 2413, no correspond table map event
		// mysql> SHOW BINLOG EVENTS IN '8623306-bin.008517' FROM 117752230 limit 10;
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | Log_name           | Pos       | Event_type | Server_id | End_log_pos | Info                                                                        |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | 8623306-bin.008517 | 117732471 | Query      |  81413306 |   117732534 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117732534 | Table_map  |  81413306 |   117732590 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117732590 | Write_rows |  81413306 |   117735750 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117735750 | Table_map  |  81413306 |   117735810 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117735810 | Write_rows |  81413306 |   117744020 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117744020 | Write_rows |  81413306 |   117752230 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117752230 | Write_rows |  81413306 |   117755340 | table_id: 2968 flags: STMT_END_F                                            | <- checkpoint
		// | 8623306-bin.008517 | 117755340 | Xid        |  81413306 |   117755371 | COMMIT /* xid=2243095633 */                                                 |
		// | 8623306-bin.008517 | 117755371 | Gtid       |  81413306 |   117755419 | SET @@SESSION.GTID_NEXT= 'a0f36bb1-fdbb-11e5-8413-a0369f7c3bb4:16393135780' |
		// | 8623306-bin.008517 | 117755419 | Query      |  81413306 |   117755482 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117755482 | Table_map  |  81413306 |   117755538 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117755538 | Write_rows |  81413306 |   117758123 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117758123 | Table_map  |  81413306 |   117758183 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117758183 | Write_rows |  81413306 |   117766393 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117766393 | Write_rows |  81413306 |   117772728 | table_id: 2968 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117772728 | Xid        |  81413306 |   117772759 | COMMIT /* xid=2243095636 */                                                 |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		ev, err := syncer.GetEvent(ctx)
		//ev, err := syncer.GetEvent(context.Background())
		cancel()

		if err != nil {
			if err == context.DeadlineExceeded {
				if timeout < maxTimeout {
					// backoff
					timeout *= 2
				}

				continue
			}

			// TODO try out all the cases
			m.emitFatalError(err)
			return
		}

		m.m.Events.Mark(1)

		// insert into tbtest values(1) will trigger the following events:
		// QueryEvent    BEGIN, Log position: 4800
		// TableMapEvent Schema: mydb, Table: tbtest, TableID: 81, Log position: 4844
		// RowsEvent     Values: 0:1, TableID: 81, Log position: 4884
		// XIDEvent      XID: 356, Log position: 4915 // COMMMIT
		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			// e,g.
			// Position: 4
			// Next log name: mysql.000002
			file = string(e.NextLogName)
			// e.Position is End_log_pos(i,e. next log position)
			log.Trace("[%s] events rotate to (%s, %d)", m.name, file, e.Position)

		case *replication.RowsEvent:
			m.m.TPS.Mark(1)
			// ev.Header.ServerID, where the event originated
			// depends on ntp to sync clock, but there is sure clock drift
			lag := time.Now().Unix() - int64(ev.Header.Timestamp)
			if lag < 0 {
				lag = 0
			}
			m.m.Lag.Update(lag)
			m.handleRowsEvent(file, ev.Header, e)

		case *replication.QueryEvent:
			// DDL comes this way
			// e,g. create table y(id int)
			// e,g. BEGIN
			// e,g. flush tables
			// e,g. ALTER TABLE
			//
			// only handles alter table query
			if db, table, yes := isAlterTableQuery(e.Query); yes {
				if len(db) == 0 {
					db = string(e.Schema)
				}
				log.Trace("[%s] %s.%s table schema changed", m.name, db, table)
				m.clearTableCache(db, table)
			}

		case *replication.XIDEvent:
			// e,g. COMMIT /* xid=403013040 */

		case *replication.FormatDescriptionEvent:
			// Version: 4
			// Server version: 5.6.23-log
			// Checksum algorithm: 1

		case *replication.TableMapEvent:
			// e,g.
			// TableID: 170
			// TableID size: 6
			// Flags: 1
			// Schema: test
			// Table: y
			// Column count: 1
			// Column type:
			// 00000000  03
			// NULL bitmap:
			// 00000000  01
			//
			// https://github.com/siddontang/go-mysql/issues/48
			// If we have processed TableMapEvent and then restart, the following
			// rows event may miss the table id when we sync from last saved position.

		case *replication.GenericEvent:
			// known misc types of event

		case *replication.GTIDEvent:
			// only GTID mode will recv this event

		default:
			log.Warn("[%s] unexpected event: %+v", m.name, e)
		}
	}

}

func (m *MySlave) emitFatalError(err error) {
	m.errors <- err
}
<commit_msg>GTID explained<commit_after>package myslave

import (
	"context"
	"fmt"
	"time"

	"github.com/funkygao/dbus/pkg/checkpoint"
	"github.com/funkygao/dbus/pkg/model"
	log "github.com/funkygao/log4go"
	uuid "github.com/satori/go.uuid"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

// StopReplication stops the slave and do necessary cleanups.
func (m *MySlave) StopReplication() {
	if !m.started.Get() {
		return
	}

	log.Debug("[%s] stopping replication from %s", m.name, m.masterAddr)

	m.r.Close()
	m.m.Close()

	if err := m.p.Shutdown(); err != nil {
		log.Error("[%s] %s", m.name, err)
	}

	m.started.Set(false)
}

// StartReplication start the mysql binlog replication.
// TODO graceful shutdown
// TODO GTID
func (m *MySlave) StartReplication(ready chan struct{}) {
	m.started.Set(true)

	m.rowsEvent = make(chan *model.RowsEvent, m.c.Int("event_buffer_len", 100))
	m.errors = make(chan error, 1)

	m.r = replication.NewBinlogSyncer(&replication.BinlogSyncerConfig{
		ServerID:        uint32(m.c.Int("server_id", 137)), // 137 unique enough? TODO
		Flavor:          m.c.String("flavor", mysql.MySQLFlavor),
		Host:            m.host,
		Port:            m.port,
		User:            m.user,
		Password:        m.passwd,
		SemiSyncEnabled: m.c.Bool("semi_sync", false),
	})

	// resume replication position from the checkpoint
	err := m.p.LastPersistedState(m.state)
	if err != nil && err != checkpoint.ErrStateNotFound {
		close(ready)
		m.emitFatalError(err)
		return
	}

	file, offset := m.state.File, m.state.Offset

	var syncer *replication.BinlogStreamer
	if m.GTID {
		// SELECT @@gtid_mode
		// SHOW GLOBAL VARIABLES LIKE 'SERVER_UUID'
		var masterUUID uuid.UUID
		// 07c93cd7-a7d3-12a5-94e1-a0369a7c3790:1-313225133
		set, _ := mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d-%d", masterUUID.String(), 1, 2))
		syncer, err = m.r.StartSyncGTID(set)
	} else {
		syncer, err = m.r.StartSync(mysql.Position{
			Name: file,
			Pos:  offset,
		})
	}

	if err != nil {
		// unrecoverable err encountered
		// e,g.
		// ERROR 1045 (28000): Access denied for user 'xx'@'1.1.1.1'
		close(ready)
		m.emitFatalError(err)
		return
	}

	close(ready)
	log.Debug("[%s] ready to receive mysql binlog from %s", m.name, m.masterAddr)

	timeout := time.Second
	maxTimeout := time.Minute
	for m.started.Get() {
		// what if the conn broken?
		// 2017/02/08 08:31:39 binlogsyncer.go:529: [info] receive EOF packet, retry ReadPacket
		// 2017/02/08 08:31:39 binlogsyncer.go:484: [error] connection was bad
		// 2017/02/08 08:31:40 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:40 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:40 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		// 2017/02/08 08:31:42 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:42 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:42 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		//
		// or invalid position sent?
		// 2017/02/08 08:42:09 binlogstreamer.go:47: [error] close sync with err: ERROR 1236 (HY000): Client requested master to start replication from position > file size; the first event 'mysql.000004' at 2274, the last event read from '/usr/local/var/mysql.000004' at 4, the last byte read from '/usr/local/var/mysql.000004' at 4.
		//
		// [02/08/17 17:39:20] [EROR] ( mysqlbinlog.go:64) backoff 5s: invalid table id 2413, no correspond table map event
		// mysql> SHOW BINLOG EVENTS IN '8623306-bin.008517' FROM 117752230 limit 10;
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | Log_name           | Pos       | Event_type | Server_id | End_log_pos | Info                                                                        |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | 8623306-bin.008517 | 117732471 | Query      |  81413306 |   117732534 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117732534 | Table_map  |  81413306 |   117732590 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117732590 | Write_rows |  81413306 |   117735750 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117735750 | Table_map  |  81413306 |   117735810 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117735810 | Write_rows |  81413306 |   117744020 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117744020 | Write_rows |  81413306 |   117752230 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117752230 | Write_rows |  81413306 |   117755340 | table_id: 2968 flags: STMT_END_F                                            | <- checkpoint
		// | 8623306-bin.008517 | 117755340 | Xid        |  81413306 |   117755371 | COMMIT /* xid=2243095633 */                                                 |
		// | 8623306-bin.008517 | 117755371 | Gtid       |  81413306 |   117755419 | SET @@SESSION.GTID_NEXT= 'a0f36bb1-fdbb-11e5-8413-a0369f7c3bb4:16393135780' |
		// | 8623306-bin.008517 | 117755419 | Query      |  81413306 |   117755482 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117755482 | Table_map  |  81413306 |   117755538 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117755538 | Write_rows |  81413306 |   117758123 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117758123 | Table_map  |  81413306 |   117758183 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117758183 | Write_rows |  81413306 |   117766393 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117766393 | Write_rows |  81413306 |   117772728 | table_id: 2968 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117772728 | Xid        |  81413306 |   117772759 | COMMIT /* xid=2243095636 */                                                 |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		ev, err := syncer.GetEvent(ctx)
		//ev, err := syncer.GetEvent(context.Background())
		cancel()

		if err != nil {
			if err == context.DeadlineExceeded {
				if timeout < maxTimeout {
					// backoff
					timeout *= 2
				}

				continue
			}

			// TODO try out all the cases
			m.emitFatalError(err)
			return
		}

		m.m.Events.Mark(1)

		// insert into tbtest values(1) will trigger the following events:
		// QueryEvent    BEGIN, Log position: 4800
		// TableMapEvent Schema: mydb, Table: tbtest, TableID: 81, Log position: 4844
		// RowsEvent     Values: 0:1, TableID: 81, Log position: 4884
		// XIDEvent      XID: 356, Log position: 4915 // COMMMIT
		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			// e,g.
			// Position: 4
			// Next log name: mysql.000002
			file = string(e.NextLogName)
			// e.Position is End_log_pos(i,e. next log position)
			log.Trace("[%s] events rotate to (%s, %d)", m.name, file, e.Position)

		case *replication.RowsEvent:
			m.m.TPS.Mark(1)
			// ev.Header.ServerID, where the event originated
			// depends on ntp to sync clock, but there is sure clock drift
			lag := time.Now().Unix() - int64(ev.Header.Timestamp)
			if lag < 0 {
				lag = 0
			}
			m.m.Lag.Update(lag)
			m.handleRowsEvent(file, ev.Header, e)

		case *replication.QueryEvent:
			// DDL comes this way
			// e,g. create table y(id int)
			// e,g. BEGIN
			// e,g. flush tables
			// e,g. ALTER TABLE
			//
			// only handles alter table query
			if db, table, yes := isAlterTableQuery(e.Query); yes {
				if len(db) == 0 {
					db = string(e.Schema)
				}
				log.Trace("[%s] %s.%s table schema changed", m.name, db, table)
				m.clearTableCache(db, table)
			}

		case *replication.XIDEvent:
			// e,g. COMMIT /* xid=403013040 */

		case *replication.FormatDescriptionEvent:
			// Version: 4
			// Server version: 5.6.23-log
			// Checksum algorithm: 1

		case *replication.TableMapEvent:
			// e,g.
			// TableID: 170
			// TableID size: 6
			// Flags: 1
			// Schema: test
			// Table: y
			// Column count: 1
			// Column type:
			// 00000000  03
			// NULL bitmap:
			// 00000000  01
			//
			// https://github.com/siddontang/go-mysql/issues/48
			// If we have processed TableMapEvent and then restart, the following
			// rows event may miss the table id when we sync from last saved position.

		case *replication.GenericEvent:
			// known misc types of event

		case *replication.GTIDEvent:
			// only GTID mode will recv this event
			// GTID is for easier failover because each slave pos coordinate is different: SET MASTER_AUTO_POSITION=1
			// GTID is for strong master/slave W/R split: check if the GTID on master is replicated to slave
			//
			// This is how we can see the GTID inside the binary logs:
			// # at 300
			// #130221 13:08:58 server id 101 end_log_pos 348 CRC32 0xc18cdbda GTID [commit=yes] SET @@SESSION.GTID_NEXT= '8182213e-7c1e-11e2-a6e2-080027635ef5:2'/*!*/;
			// # at 348
			// BEGIN
			// insert into t values(1)
			// COMMIT/*!*/;
			// # at 565
			// #130221 13:09:03 server id 101 end_log_pos 613 CRC32 0x5b25189e
			// SET @@SESSION.GTID_NEXT= '8182213e-7c1e-11e2-a6e2-080027635ef5:3'/*!*/;
			// # at 697
			// BEGIN
			//
			// what is GTID like?
			// 8182213e-7c1e-11e2-a6e2-080027635ef5:2  SID:GNO
			// SID is 128 bit server uuid which identifies where the transaction was originated
			// GNO is txn id which increments with every new transaction

		default:
			log.Warn("[%s] unexpected event: %+v", m.name, e)
		}
	}

}

func (m *MySlave) emitFatalError(err error) {
	m.errors <- err
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// +build !appengine

package osutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

// UmountAll recurusively unmounts all mounts in dir.
func UmountAll(dir string) {
	files, _ := ioutil.ReadDir(dir)
	for _, f := range files {
		name := filepath.Join(dir, f.Name())
		if f.IsDir() {
			UmountAll(name)
		}
		fn := []byte(name + "\x00")
		syscall.Syscall(syscall.SYS_UMOUNT2, uintptr(unsafe.Pointer(&fn[0])), syscall.MNT_FORCE, 0)
	}
}

func Sandbox(cmd *exec.Cmd, user, net bool) error {
	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	if user {
		uid, gid, err := initSandbox()
		if err != nil {
			return err
		}
		cmd.SysProcAttr.Credential = &syscall.Credential{
			Uid: uid,
			Gid: gid,
		}
	}
	if net {
		cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET | syscall.CLONE_NEWIPC |
			syscall.CLONE_NEWNS | syscall.CLONE_NEWUTS | syscall.CLONE_NEWPID
	}
	return nil
}

func SandboxChown(file string) error {
	uid, _, err := initSandbox()
	if err != nil {
		return err
	}
	return os.Chown(file, int(uid), int(uid))
}

var (
	sandboxOnce     sync.Once
	sandboxUsername = "syzkaller"
	sandboxUID      = ^uint32(0)
	sandboxGID      = ^uint32(0)
)

func initSandbox() (uint32, uint32, error) {
	sandboxOnce.Do(func() {
		uid, err := usernameToID("-u")
		if err != nil {
			return
		}
		gid, err := usernameToID("-g")
		if err != nil {
			return
		}
		sandboxUID = uid
		sandboxGID = gid
	})
	if sandboxUID == ^uint32(0) {
		return 0, 0, fmt.Errorf("user %q is not found, can't sandbox command", sandboxUsername)
	}
	return sandboxUID, sandboxGID, nil
}

func usernameToID(what string) (uint32, error) {
	out, err := RunCmd(time.Minute, "", "id", what, sandboxUsername)
	if err != nil {
		return 0, err
	}
	str := strings.Trim(string(out), " \t\n")
	id, err := strconv.ParseUint(str, 10, 32)
	if err != nil {
		return 0, err
	}
	return uint32(id), nil
}

func setPdeathsig(cmd *exec.Cmd) {
	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL
}

func prolongPipe(r, w *os.File) {
	for sz := 128 << 10; sz <= 2<<20; sz *= 2 {
		syscall.Syscall(syscall.SYS_FCNTL, w.Fd(), syscall.F_SETPIPE_SZ, uintptr(sz))
	}
}
<commit_msg>pkg/osutil: use proper gid during sandboxing<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// +build !appengine

package osutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

// UmountAll recurusively unmounts all mounts in dir.
func UmountAll(dir string) {
	files, _ := ioutil.ReadDir(dir)
	for _, f := range files {
		name := filepath.Join(dir, f.Name())
		if f.IsDir() {
			UmountAll(name)
		}
		fn := []byte(name + "\x00")
		syscall.Syscall(syscall.SYS_UMOUNT2, uintptr(unsafe.Pointer(&fn[0])), syscall.MNT_FORCE, 0)
	}
}

func Sandbox(cmd *exec.Cmd, user, net bool) error {
	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	if user {
		uid, gid, err := initSandbox()
		if err != nil {
			return err
		}
		cmd.SysProcAttr.Credential = &syscall.Credential{
			Uid: uid,
			Gid: gid,
		}
	}
	if net {
		cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET | syscall.CLONE_NEWIPC |
			syscall.CLONE_NEWNS | syscall.CLONE_NEWUTS | syscall.CLONE_NEWPID
	}
	return nil
}

func SandboxChown(file string) error {
	uid, gid, err := initSandbox()
	if err != nil {
		return err
	}
	return os.Chown(file, int(uid), int(gid))
}

var (
	sandboxOnce     sync.Once
	sandboxUsername = "syzkaller"
	sandboxUID      = ^uint32(0)
	sandboxGID      = ^uint32(0)
)

func initSandbox() (uint32, uint32, error) {
	sandboxOnce.Do(func() {
		uid, err := usernameToID("-u")
		if err != nil {
			return
		}
		gid, err := usernameToID("-g")
		if err != nil {
			return
		}
		sandboxUID = uid
		sandboxGID = gid
	})
	if sandboxUID == ^uint32(0) {
		return 0, 0, fmt.Errorf("user %q is not found, can't sandbox command", sandboxUsername)
	}
	return sandboxUID, sandboxGID, nil
}

func usernameToID(what string) (uint32, error) {
	out, err := RunCmd(time.Minute, "", "id", what, sandboxUsername)
	if err != nil {
		return 0, err
	}
	str := strings.Trim(string(out), " \t\n")
	id, err := strconv.ParseUint(str, 10, 32)
	if err != nil {
		return 0, err
	}
	return uint32(id), nil
}

func setPdeathsig(cmd *exec.Cmd) {
	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL
}

func prolongPipe(r, w *os.File) {
	for sz := 128 << 10; sz <= 2<<20; sz *= 2 {
		syscall.Syscall(syscall.SYS_FCNTL, w.Fd(), syscall.F_SETPIPE_SZ, uintptr(sz))
	}
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"fmt"
	lang "github.com/Aptomi/aptomi/pkg/slinga/language"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/object/codec"
	"github.com/Aptomi/aptomi/pkg/slinga/object/codec/yaml"
	"github.com/Aptomi/aptomi/pkg/slinga/server/controller"
	"github.com/gosuri/uitable"
	"github.com/mattn/go-zglob"
	"github.com/spf13/viper"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"time"
)

func Apply(config *viper.Viper) error {
	policyPaths := config.GetStringSlice("apply.policyPaths")

	catalog := object.NewObjectCatalog(lang.ServiceObject, lang.ContractObject, lang.ClusterObject, lang.RuleObject, lang.DependencyObject, controller.PolicyDataObject)
	cod := yaml.NewCodec(catalog)

	allObjects, err := readFiles(policyPaths, cod)
	if err != nil {
		return err
	}

	// todo(slukjanov): here we can use some more efficient marshaller like gob, etc
	// prepare single []byte to send to API
	data, err := cod.MarshalMany(allObjects)
	if err != nil {
		return fmt.Errorf("Error while marshaling data for sending to API: %s", err)
	}

	client := &http.Client{
		Timeout: 5 * time.Second,
	}

	host := config.GetString("server.host")
	port := config.GetInt("server.port")

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://%s:%d/api/v1/policy", host, port), bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/yaml")
	req.Header.Set("User-Agent", "aptomictl")

	fmt.Println("Request:", req)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// todo(slujanov): process response - check status and print returned data
	fmt.Println("Response:", resp)

	respData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(fmt.Sprintf("Error while reading bytes from response Body: %s", err))
	}

	// todo bad logging
	fmt.Println("Response data: " + string(respData))

	objects, err := cod.UnmarshalOneOrMany(respData)
	if err != nil {
		panic(fmt.Sprintf("Error while unmarshaling response: %s", err))
	}

	// todo(slukjanov): pretty print response
	table := uitable.New()
	table.MaxColWidth = 50

	table.AddRow("#", "Namespace", "Kind", "Name", "Generation", "Object")
	for idx, obj := range objects {
		table.AddRow(idx, obj.GetNamespace(), obj.GetKind(), obj.GetName(), obj.GetGeneration(), obj)
	}
	fmt.Println(table)

	return nil
}

func readFiles(policyPaths []string, codec codec.MarshalUnmarshaler) ([]object.Base, error) {
	files, err := findPolicyFiles(policyPaths)
	if err != nil {
		return nil, fmt.Errorf("Error while searching for policy files: %s", err)
	}

	allObjects := make([]object.Base, 0)
	for _, file := range files {
		data, err := ioutil.ReadFile(file)
		if err != nil {
			return nil, fmt.Errorf("Can't read file %s error: %s", file, err)
		}

		// todo(slukjanov): here we can try multiple marshalers, toml for example
		objects, err := codec.UnmarshalOneOrMany(data)
		if err != nil {
			return nil, fmt.Errorf("Can't unmarshal file %s error: %s", file, err)
		}

		allObjects = append(allObjects, objects...)
	}

	if len(allObjects) == 0 {
		return nil, fmt.Errorf("No objects found in %s", policyPaths)
	}

	return allObjects, nil
}

func findPolicyFiles(policyPaths []string) ([]string, error) {
	allFiles := make([]string, 0, len(policyPaths))

	for _, rawPolicyPath := range policyPaths {
		policyPath, err := filepath.Abs(rawPolicyPath)
		if err != nil {
			return nil, fmt.Errorf("Error reading filepath: %s", err)
		}

		if stat, err := os.Stat(policyPath); err == nil {
			if stat.IsDir() { // if dir provided, use all yaml files from it
				files, err := zglob.Glob(filepath.Join(policyPath, "**", "*.yaml"))
				if err != nil {
					return nil, fmt.Errorf("Error while searching yaml files in directory: %s error: %s", policyPath, err)
				}
				allFiles = append(allFiles, files...)
			} else { // if specific file provided, use it
				allFiles = append(allFiles, policyPath)
			}
		} else if os.IsNotExist(err) {
			return nil, fmt.Errorf("Path doesn't exists: %s error: %s", policyPath, err)
		} else {
			return nil, fmt.Errorf("Error while processing path: %s", err)
		}
	}

	sort.Strings(allFiles) // todo(slukjanov): do we really need to sort files?

	// todo(slukjanov): log list of files from which we're applying policy
	//fmt.Println("Apply policy from following files:")
	//for idx, policyPath := range allFiles {
	//	fmt.Println(idx, "-", policyPath)
	//}

	return allFiles, nil
}
<commit_msg>fixed a typo in error message<commit_after>package client

import (
	"bytes"
	"fmt"
	lang "github.com/Aptomi/aptomi/pkg/slinga/language"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/object/codec"
	"github.com/Aptomi/aptomi/pkg/slinga/object/codec/yaml"
	"github.com/Aptomi/aptomi/pkg/slinga/server/controller"
	"github.com/gosuri/uitable"
	"github.com/mattn/go-zglob"
	"github.com/spf13/viper"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"time"
)

func Apply(config *viper.Viper) error {
	policyPaths := config.GetStringSlice("apply.policyPaths")

	catalog := object.NewObjectCatalog(lang.ServiceObject, lang.ContractObject, lang.ClusterObject, lang.RuleObject, lang.DependencyObject, controller.PolicyDataObject)
	cod := yaml.NewCodec(catalog)

	allObjects, err := readFiles(policyPaths, cod)
	if err != nil {
		return err
	}

	// todo(slukjanov): here we can use some more efficient marshaller like gob, etc
	// prepare single []byte to send to API
	data, err := cod.MarshalMany(allObjects)
	if err != nil {
		return fmt.Errorf("Error while marshaling data for sending to API: %s", err)
	}

	client := &http.Client{
		Timeout: 5 * time.Second,
	}

	host := config.GetString("server.host")
	port := config.GetInt("server.port")

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://%s:%d/api/v1/policy", host, port), bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/yaml")
	req.Header.Set("User-Agent", "aptomictl")

	fmt.Println("Request:", req)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// todo(slujanov): process response - check status and print returned data
	fmt.Println("Response:", resp)

	respData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(fmt.Sprintf("Error while reading bytes from response Body: %s", err))
	}

	// todo bad logging
	fmt.Println("Response data: " + string(respData))

	objects, err := cod.UnmarshalOneOrMany(respData)
	if err != nil {
		panic(fmt.Sprintf("Error while unmarshaling response: %s", err))
	}

	// todo(slukjanov): pretty print response
	table := uitable.New()
	table.MaxColWidth = 50

	table.AddRow("#", "Namespace", "Kind", "Name", "Generation", "Object")
	for idx, obj := range objects {
		table.AddRow(idx, obj.GetNamespace(), obj.GetKind(), obj.GetName(), obj.GetGeneration(), obj)
	}
	fmt.Println(table)

	return nil
}

func readFiles(policyPaths []string, codec codec.MarshalUnmarshaler) ([]object.Base, error) {
	files, err := findPolicyFiles(policyPaths)
	if err != nil {
		return nil, fmt.Errorf("Error while searching for policy files: %s", err)
	}

	allObjects := make([]object.Base, 0)
	for _, file := range files {
		data, err := ioutil.ReadFile(file)
		if err != nil {
			return nil, fmt.Errorf("Can't read file %s error: %s", file, err)
		}

		// todo(slukjanov): here we can try multiple marshalers, toml for example
		objects, err := codec.UnmarshalOneOrMany(data)
		if err != nil {
			return nil, fmt.Errorf("Can't unmarshal file %s error: %s", file, err)
		}

		allObjects = append(allObjects, objects...)
	}

	if len(allObjects) == 0 {
		return nil, fmt.Errorf("No objects found in %s", policyPaths)
	}

	return allObjects, nil
}

func findPolicyFiles(policyPaths []string) ([]string, error) {
	allFiles := make([]string, 0, len(policyPaths))

	for _, rawPolicyPath := range policyPaths {
		policyPath, err := filepath.Abs(rawPolicyPath)
		if err != nil {
			return nil, fmt.Errorf("Error reading filepath: %s", err)
		}

		if stat, err := os.Stat(policyPath); err == nil {
			if stat.IsDir() { // if dir provided, use all yaml files from it
				files, err := zglob.Glob(filepath.Join(policyPath, "**", "*.yaml"))
				if err != nil {
					return nil, fmt.Errorf("Error while searching yaml files in directory: %s error: %s", policyPath, err)
				}
				allFiles = append(allFiles, files...)
			} else { // if specific file provided, use it
				allFiles = append(allFiles, policyPath)
			}
		} else if os.IsNotExist(err) {
			return nil, fmt.Errorf("Path doesn't exist: %s error: %s", policyPath, err)
		} else {
			return nil, fmt.Errorf("Error while processing path: %s", err)
		}
	}

	sort.Strings(allFiles) // todo(slukjanov): do we really need to sort files?

	// todo(slukjanov): log list of files from which we're applying policy
	//fmt.Println("Apply policy from following files:")
	//for idx, policyPath := range allFiles {
	//	fmt.Println(idx, "-", policyPath)
	//}

	return allFiles, nil
}
<|endoftext|>


<commit_before>package ssh

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

func (o *SSH) runLocalCommandClient(kind, namespace, name string) error {
	args := []string{}
	args = append(args, o.buildProxyCommandOption(kind, namespace, name))
	args = append(args, o.buildSSHTarget(kind, namespace, name))

	cmd := exec.Command("ssh", args...)
	fmt.Println("running:", cmd)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	return cmd.Run()
}

func (o *SSH) buildProxyCommandOption(kind, namespace, name string) string {
	proxyCommand := strings.Builder{}
	proxyCommand.WriteString("-o ProxyCommand=")
	proxyCommand.WriteString(os.Args[0])
	proxyCommand.WriteString(" port-forward --stdio=true ")
	proxyCommand.WriteString(fmt.Sprintf("%s/%s.%s", kind, name, namespace))
	proxyCommand.WriteString(" ")

	proxyCommand.WriteString(strconv.Itoa(o.options.SshPort))

	return proxyCommand.String()
}

func (o *SSH) buildSSHTarget(kind, namespace, name string) string {
	target := strings.Builder{}
	if len(o.options.SshUsername) > 0 {
		target.WriteString(o.options.SshUsername)
		target.WriteRune('@')
	}
	target.WriteString(kind)
	target.WriteRune('/')
	target.WriteString(name)
	target.WriteRune('.')
	target.WriteString(namespace)
	return target.String()
}
<commit_msg>append identity file with local-ssh<commit_after>package ssh

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

func (o *SSH) runLocalCommandClient(kind, namespace, name string) error {
	args := []string{}
	args = append(args, o.buildProxyCommandOption(kind, namespace, name))
	args = append(args, o.buildSSHTarget(kind, namespace, name))

	cmd := exec.Command("ssh", args...)
	fmt.Println("running:", cmd)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	return cmd.Run()
}

func (o *SSH) buildProxyCommandOption(kind, namespace, name string) string {
	proxyCommand := strings.Builder{}
	proxyCommand.WriteString("-o ProxyCommand=")
	proxyCommand.WriteString(os.Args[0])
	proxyCommand.WriteString(" port-forward --stdio=true ")
	proxyCommand.WriteString(fmt.Sprintf("%s/%s.%s", kind, name, namespace))
	proxyCommand.WriteString(" ")

	proxyCommand.WriteString(strconv.Itoa(o.options.SshPort))

	return proxyCommand.String()
}

func (o *SSH) buildSSHTarget(kind, namespace, name string) string {
	target := strings.Builder{}
	if o.options.IdentityFilePathProvided {
		target.WriteString(fmt.Sprintf(" -i %s ", o.options.IdentityFilePath))
	}
	if len(o.options.SshUsername) > 0 {
		target.WriteString(o.options.SshUsername)
		target.WriteRune('@')
	}
	target.WriteString(kind)
	target.WriteRune('/')
	target.WriteString(name)
	target.WriteRune('.')
	target.WriteString(namespace)
	return target.String()
}
<|endoftext|>


<commit_before>package ssh

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var runCommand = func(cmd *exec.Cmd) error {
	return cmd.Run()
}

func (o *SSH) runLocalCommandClient(kind, namespace, name string) error {

	args := []string{}
	args = append(args, o.buildProxyCommandOption(kind, namespace, name))
	args = append(args, o.buildSSHTarget(kind, namespace, name))

	cmd := exec.Command("ssh", args...)
	fmt.Println("running:", cmd)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	return runCommand(cmd)
}

func (o *SSH) buildProxyCommandOption(kind, namespace, name string) string {
	proxyCommand := strings.Builder{}
	proxyCommand.WriteString("-o ProxyCommand=")
	proxyCommand.WriteString(os.Args[0])
	proxyCommand.WriteString(" port-forward --stdio=true ")
	proxyCommand.WriteString(fmt.Sprintf("%s/%s.%s", kind, name, namespace))
	proxyCommand.WriteString(" ")

	proxyCommand.WriteString(strconv.Itoa(o.options.SshPort))

	return proxyCommand.String()
}

func (o *SSH) buildSSHTarget(kind, namespace, name string) string {
	target := strings.Builder{}
	if o.options.IdentityFilePathProvided {
		target.WriteString(fmt.Sprintf(" -i %s ", o.options.IdentityFilePath))
	}
	if len(o.options.SshUsername) > 0 {
		target.WriteString(o.options.SshUsername)
		target.WriteRune('@')
	}
	target.WriteString(kind)
	target.WriteRune('/')
	target.WriteString(name)
	target.WriteRune('.')
	target.WriteString(namespace)
	return target.String()
}
<commit_msg>fix parsing of local-ssh options<commit_after>package ssh

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var runCommand = func(cmd *exec.Cmd) error {
	return cmd.Run()
}

func (o *SSH) runLocalCommandClient(kind, namespace, name string) error {

	args := []string{"-o"}
	args = append(args, o.buildProxyCommandOption(kind, namespace, name))
	args = append(args, o.buildSSHTarget(kind, namespace, name)...)

	cmd := exec.Command("ssh", args...)
	fmt.Println("running:", cmd)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	return runCommand(cmd)
}

func (o *SSH) buildProxyCommandOption(kind, namespace, name string) string {
	proxyCommand := strings.Builder{}
	proxyCommand.WriteString("ProxyCommand=")
	proxyCommand.WriteString(os.Args[0])
	proxyCommand.WriteString(" port-forward --stdio=true ")
	proxyCommand.WriteString(fmt.Sprintf("%s/%s.%s", kind, name, namespace))
	proxyCommand.WriteString(" ")

	proxyCommand.WriteString(strconv.Itoa(o.options.SshPort))

	return proxyCommand.String()
}

func (o *SSH) buildSSHTarget(kind, namespace, name string) (opts []string) {
	target := strings.Builder{}
	if o.options.IdentityFilePathProvided {
		opts = append(opts, "-i", o.options.IdentityFilePath)
	}
	if len(o.options.SshUsername) > 0 {
		target.WriteString(o.options.SshUsername)
		target.WriteRune('@')
	}
	target.WriteString(kind)
	target.WriteRune('/')
	target.WriteString(name)
	target.WriteRune('.')
	target.WriteString(namespace)
	opts = append(opts, target.String())
	return
}
<|endoftext|>


<commit_before>package intersect

import (
	"context"
	"errors"
	"log"
	"sort"

	"github.com/go-spatial/geom"
	"github.com/go-spatial/geom/cmp"
	"github.com/go-spatial/geom/planar"
)

// ErrStopIteration is used to stop the iteration.
var ErrStopIteration = errors.New("Stop Iteration")

type eventType uint8

const (
	LEFT eventType = iota
	RIGHT
)

func (et eventType) String() string {
	switch et {
	default:
		return "UNKNOWN"
	case LEFT:
		return "LEFT"
	case RIGHT:
		return "RIGHT"
	}
}

type event struct {
	edge     int        // the indext number of the edge in the segment list.
	edgeType eventType  // Is this the left or right edge.
	ev       geom.Point // event vertex
}

type EventQueue struct {
	events   []event
	segments []geom.Line
}

func (e *event) Point() geom.Point { return e.ev }
func (e *event) Edge() int         { return e.edge }

func (eq EventQueue) Len() int      { return len(eq.events) }
func (eq EventQueue) Swap(i, j int) { eq.events[i], eq.events[j] = eq.events[j], eq.events[i] }
func (eq EventQueue) Less(i, j int) bool {
	pt1, pt2 := eq.events[i].ev, eq.events[j].ev
	if pt1[0] != pt2[0] {
		return pt1[0] < pt2[0]
	}
	// We want to sort all the lines that are ending to the bottom of the list. So, that the lines that are
	// starting will get picked up.
	if eq.events[i].edgeType != eq.events[j].edgeType {
		return eq.events[i].edgeType < eq.events[j].edgeType
	}
	return pt1[1] < pt2[1]
}

// Ref: http://geomalgorithms.com/a09-_intersect-3.html#simple_Polygon()
func NewEventQueue(segments []geom.Line) (eq EventQueue) {

	// the event queue
	eq.events = make([]event, len(segments)*2)
	eq.segments = make([]geom.Line, len(segments))
	copy(eq.segments, segments)

	for i := range eq.segments {
		idx := 2 * i
		eq.events[idx].edge = i
		eq.events[idx+1].edge = i
		p1 := eq.segments[i][0]
		p2 := eq.segments[i][1]
		eq.events[idx].ev = p1
		eq.events[idx+1].ev = p2
		if p1[0] < p2[0] || (p1[0] != p2[0] && p1[1] < p2[1]) {
			eq.events[idx].edgeType = LEFT
			eq.events[idx+1].edgeType = RIGHT
		} else {
			eq.events[idx].edgeType = RIGHT
			eq.events[idx+1].edgeType = LEFT
		}
	}
	sort.Sort(eq)
	if debug {
		log.Println("Got the following for eq:\n\tevents:   ")
		for i, ev := range eq.events {
			log.Printf("\t\t%03v:i: %02v : %v - [%v, %v]", i, ev.edge, ev.edgeType, ev.ev[0], ev.ev[1])
		}
		log.Println("\n\tsegments: ")
		for i, seg := range eq.segments {
			log.Printf("\t\t%03v:%v", i, seg)
		}
	}
	return eq
}

func (eq *EventQueue) FindIntersects(ctx context.Context, connected bool, fn func(src, dest int, pt [2]float64) error) error {
	segmap := make(map[int]struct{})
	keys := make([]int, 0, 2)
	for _, ev := range eq.events {
		if err := ctx.Err(); err != nil {
			return err
		}
		edgeidx := ev.edge
		if debug {
			log.Printf("Checkout out edge: %v", edgeidx)
		}
		// this is the first point in the segment, so we will just add it to our segment map and move on.
		if ev.edgeType == LEFT {
			segmap[edgeidx] = struct{}{}
			continue
		}
		// We have found the end of a line. Let's see if there it intersects with the other lines.
		// first we need to remove the line from our set of lines.
		delete(segmap, edgeidx)
		seg := eq.segments[edgeidx]
		if debug {
			log.Printf("Got to the edge of a segment: %v", seg)
		}

		if len(segmap) == 0 {
			continue
		}
		keys = keys[:0]
		{
			for k := range segmap {
				keys = append(keys, k)
			}
			sort.Sort(sort.IntSlice(keys))
		}
		if debug {
			log.Printf("Keys are: %v", keys)
		}
		for _, edge := range keys {
			seg1 := eq.segments[edge]

			ipt, ok := planar.SegmentIntersect(seg, seg1)
			if debug {
				log.Printf("Looking at \n\tLine1(%v): %v \n\tLine2(%v): %v", edgeidx, seg, edge, seg1)
				if ok {
					log.Printf("Found ipt: %v", ipt)
				} else {
					log.Printf("Did not find point")
				}
			}
			if !ok {
				// Check the next edge.
				continue
			}
			// we need to see if , the ipt is the endpoint of both lines, (it's a connecting point) and
			// the polygonCheck is true, then it should not count as an intersect.
			if connected {
				matchseg := cmp.PointEqual(ipt, seg.Point1().XY()) || cmp.PointEqual(ipt, seg.Point2().XY())
				matchseg1 := cmp.PointEqual(ipt, seg1.Point1().XY()) || cmp.PointEqual(ipt, seg1.Point2().XY())
				// Check the next edge.
				if matchseg && matchseg1 {
					continue
				}
			}
			if err := fn(edgeidx, edge, ipt); err != nil {
				// We were told not to continue.
				if err == ErrStopIteration {
					return nil
				}
				return err
			}
		}
	}
	return nil
}
<commit_msg>fix comparision in EventQueue<commit_after>package intersect

import (
	"context"
	"errors"
	"log"
	"sort"

	"github.com/go-spatial/geom"
	"github.com/go-spatial/geom/cmp"
	"github.com/go-spatial/geom/planar"
)

// ErrStopIteration is used to stop the iteration.
var ErrStopIteration = errors.New("Stop Iteration")

type eventType uint8

const (
	LEFT eventType = iota
	RIGHT
)

func (et eventType) String() string {
	switch et {
	default:
		return "UNKNOWN"
	case LEFT:
		return "LEFT"
	case RIGHT:
		return "RIGHT"
	}
}

type event struct {
	edge     int        // the indext number of the edge in the segment list.
	edgeType eventType  // Is this the left or right edge.
	ev       geom.Point // event vertex
}

type EventQueue struct {
	events   []event
	segments []geom.Line
}

func (e *event) Point() geom.Point { return e.ev }
func (e *event) Edge() int         { return e.edge }

func (eq EventQueue) Len() int      { return len(eq.events) }
func (eq EventQueue) Swap(i, j int) { eq.events[i], eq.events[j] = eq.events[j], eq.events[i] }
func (eq EventQueue) Less(i, j int) bool {
	pt1, pt2 := eq.events[i].ev, eq.events[j].ev
	if pt1[0] != pt2[0] {
		return pt1[0] < pt2[0]
	}
	// We want to sort all the lines that are ending to the bottom of the list. So, that the lines that are
	// starting will get picked up.
	if eq.events[i].edgeType != eq.events[j].edgeType {
		return eq.events[i].edgeType < eq.events[j].edgeType
	}
	return pt1[1] < pt2[1]
}

// Ref: http://geomalgorithms.com/a09-_intersect-3.html#simple_Polygon()
func NewEventQueue(segments []geom.Line) (eq EventQueue) {

	// the event queue
	eq.events = make([]event, len(segments)*2)
	eq.segments = make([]geom.Line, len(segments))
	copy(eq.segments, segments)

	for i := range eq.segments {
		idx := 2 * i
		eq.events[idx].edge = i
		eq.events[idx+1].edge = i
		p1 := eq.segments[i][0]
		p2 := eq.segments[i][1]
		eq.events[idx].ev = p1
		eq.events[idx+1].ev = p2
		if p1[0] < p2[0] || (p1[0] == p2[0] && p1[1] < p2[1]) {
			eq.events[idx].edgeType = LEFT
			eq.events[idx+1].edgeType = RIGHT
		} else {
			eq.events[idx].edgeType = RIGHT
			eq.events[idx+1].edgeType = LEFT
		}
	}
	sort.Sort(eq)
	if debug {
		log.Println("Got the following for eq:\n\tevents:   ")
		for i, ev := range eq.events {
			log.Printf("\t\t%03v:i: %02v : %v - [%v, %v]", i, ev.edge, ev.edgeType, ev.ev[0], ev.ev[1])
		}
		log.Println("\n\tsegments: ")
		for i, seg := range eq.segments {
			log.Printf("\t\t%03v:%v", i, seg)
		}
	}
	return eq
}

func (eq *EventQueue) FindIntersects(ctx context.Context, connected bool, fn func(src, dest int, pt [2]float64) error) error {
	segmap := make(map[int]struct{})
	keys := make([]int, 0, 2)
	for _, ev := range eq.events {
		if err := ctx.Err(); err != nil {
			return err
		}
		edgeidx := ev.edge
		if debug {
			log.Printf("Checkout out edge: %v", edgeidx)
		}
		// this is the first point in the segment, so we will just add it to our segment map and move on.
		if ev.edgeType == LEFT {
			segmap[edgeidx] = struct{}{}
			continue
		}
		// We have found the end of a line. Let's see if there it intersects with the other lines.
		// first we need to remove the line from our set of lines.
		delete(segmap, edgeidx)
		seg := eq.segments[edgeidx]
		if debug {
			log.Printf("Got to the edge of a segment: %v", seg)
		}

		if len(segmap) == 0 {
			continue
		}
		keys = keys[:0]
		{
			for k := range segmap {
				keys = append(keys, k)
			}
			sort.Sort(sort.IntSlice(keys))
		}
		if debug {
			log.Printf("Keys are: %v", keys)
		}
		for _, edge := range keys {
			seg1 := eq.segments[edge]

			ipt, ok := planar.SegmentIntersect(seg, seg1)
			if debug {
				log.Printf("Looking at \n\tLine1(%v): %v \n\tLine2(%v): %v", edgeidx, seg, edge, seg1)
				if ok {
					log.Printf("Found ipt: %v", ipt)
				} else {
					log.Printf("Did not find point")
				}
			}
			if !ok {
				// Check the next edge.
				continue
			}
			// we need to see if , the ipt is the endpoint of both lines, (it's a connecting point) and
			// the polygonCheck is true, then it should not count as an intersect.
			if connected {
				matchseg := cmp.PointEqual(ipt, seg.Point1().XY()) || cmp.PointEqual(ipt, seg.Point2().XY())
				matchseg1 := cmp.PointEqual(ipt, seg1.Point1().XY()) || cmp.PointEqual(ipt, seg1.Point2().XY())
				// Check the next edge.
				if matchseg && matchseg1 {
					continue
				}
			}
			if err := fn(edgeidx, edge, ipt); err != nil {
				// We were told not to continue.
				if err == ErrStopIteration {
					return nil
				}
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/coredns/coredns/core/dnsserver"
	"github.com/coredns/coredns/plugin"
	"github.com/coredns/coredns/plugin/pkg/dnsutil"
	clog "github.com/coredns/coredns/plugin/pkg/log"
	"github.com/coredns/coredns/plugin/pkg/parse"
	"github.com/coredns/coredns/plugin/pkg/upstream"

	"github.com/mholt/caddy"
	"github.com/miekg/dns"
	meta "k8s.io/apimachinery/pkg/apis/meta/v1"

	// Pull this in for logtostderr flag parsing
	"k8s.io/klog"

	// Excluding azure because it is failing to compile
	// pull this in here, because we want it excluded if plugin.cfg doesn't have k8s
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	// pull this in here, because we want it excluded if plugin.cfg doesn't have k8s
	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
	// pull this in here, because we want it excluded if plugin.cfg doesn't have k8s
	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
	"k8s.io/client-go/tools/clientcmd"
)

var log = clog.NewWithPlugin("kubernetes")

func init() {
	// Kubernetes plugin uses the kubernetes library, which now uses klog, we must set and parse this flag
	// so we don't log to the filesystem, which can fill up and crash CoreDNS indirectly by calling os.Exit().
	// We also set: os.Stderr = os.Stdout in the setup function below so we output to standard out; as we do for
	// all CoreDNS logging. We can't do *that* in the init function, because we, when starting, also barf some
	// things to stderr.
	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
	klog.InitFlags(klogFlags)
	logtostderr := klogFlags.Lookup("logtostderr")
	logtostderr.Value.Set("true")

	caddy.RegisterPlugin("kubernetes", caddy.Plugin{
		ServerType: "dns",
		Action:     setup,
	})
}

func setup(c *caddy.Controller) error {
	// See comment in the init function.
	os.Stderr = os.Stdout

	k, err := kubernetesParse(c)
	if err != nil {
		return plugin.Error("kubernetes", err)
	}

	err = k.InitKubeCache()
	if err != nil {
		return plugin.Error("kubernetes", err)
	}

	k.RegisterKubeCache(c)

	dnsserver.GetConfig(c).AddPlugin(func(next plugin.Handler) plugin.Handler {
		k.Next = next
		return k
	})

	return nil
}

// RegisterKubeCache registers KubeCache start and stop functions with Caddy
func (k *Kubernetes) RegisterKubeCache(c *caddy.Controller) {
	c.OnStartup(func() error {
		go k.APIConn.Run()

		timeout := time.After(5 * time.Second)
		ticker := time.NewTicker(100 * time.Millisecond)
		for {
			select {
			case <-ticker.C:
				if k.APIConn.HasSynced() {
					return nil
				}
			case <-timeout:
				return nil
			}
		}
	})

	c.OnShutdown(func() error {
		return k.APIConn.Stop()
	})
}

func kubernetesParse(c *caddy.Controller) (*Kubernetes, error) {
	var (
		k8s *Kubernetes
		err error
	)

	i := 0
	for c.Next() {
		if i > 0 {
			return nil, plugin.ErrOnce
		}
		i++

		k8s, err = ParseStanza(c)
		if err != nil {
			return k8s, err
		}
	}
	return k8s, nil
}

// ParseStanza parses a kubernetes stanza
func ParseStanza(c *caddy.Controller) (*Kubernetes, error) {

	k8s := New([]string{""})
	k8s.interfaceAddrsFunc = localPodIP
	k8s.autoPathSearch = searchFromResolvConf()

	opts := dnsControlOpts{
		initEndpointsCache: true,
		ignoreEmptyService: false,
		resyncPeriod:       defaultResyncPeriod,
	}
	k8s.opts = opts

	zones := c.RemainingArgs()

	if len(zones) != 0 {
		k8s.Zones = zones
		for i := 0; i < len(k8s.Zones); i++ {
			k8s.Zones[i] = plugin.Host(k8s.Zones[i]).Normalize()
		}
	} else {
		k8s.Zones = make([]string, len(c.ServerBlockKeys))
		for i := 0; i < len(c.ServerBlockKeys); i++ {
			k8s.Zones[i] = plugin.Host(c.ServerBlockKeys[i]).Normalize()
		}
	}

	k8s.primaryZoneIndex = -1
	for i, z := range k8s.Zones {
		if dnsutil.IsReverse(z) > 0 {
			continue
		}
		k8s.primaryZoneIndex = i
		break
	}

	if k8s.primaryZoneIndex == -1 {
		return nil, errors.New("non-reverse zone name must be used")
	}

	for c.NextBlock() {
		switch c.Val() {
		case "endpoint_pod_names":
			args := c.RemainingArgs()
			if len(args) > 0 {
				return nil, c.ArgErr()
			}
			k8s.endpointNameMode = true
			continue
		case "pods":
			args := c.RemainingArgs()
			if len(args) == 1 {
				switch args[0] {
				case podModeDisabled, podModeInsecure, podModeVerified:
					k8s.podMode = args[0]
				default:
					return nil, fmt.Errorf("wrong value for pods: %s,  must be one of: disabled, verified, insecure", args[0])
				}
				continue
			}
			return nil, c.ArgErr()
		case "namespaces":
			args := c.RemainingArgs()
			if len(args) > 0 {
				for _, a := range args {
					k8s.Namespaces[a] = struct{}{}
				}
				continue
			}
			return nil, c.ArgErr()
		case "endpoint":
			args := c.RemainingArgs()
			if len(args) > 0 {
				// Multiple endoints are deprecated but still could be specified,
				// only the first one be used, though
				k8s.APIServerList = args
				if len(args) > 1 {
					log.Warningf("Multiple endpoints have been deprecated, only the first specified endpoint '%s' is used", args[0])
				}
				continue
			}
			return nil, c.ArgErr()
		case "tls": // cert key cacertfile
			args := c.RemainingArgs()
			if len(args) == 3 {
				k8s.APIClientCert, k8s.APIClientKey, k8s.APICertAuth = args[0], args[1], args[2]
				continue
			}
			return nil, c.ArgErr()
		case "resyncperiod":
			args := c.RemainingArgs()
			if len(args) > 0 {
				rp, err := time.ParseDuration(args[0])
				if err != nil {
					return nil, fmt.Errorf("unable to parse resync duration value: '%v': %v", args[0], err)
				}
				k8s.opts.resyncPeriod = rp
				continue
			}
			return nil, c.ArgErr()
		case "labels":
			args := c.RemainingArgs()
			if len(args) > 0 {
				labelSelectorString := strings.Join(args, " ")
				ls, err := meta.ParseToLabelSelector(labelSelectorString)
				if err != nil {
					return nil, fmt.Errorf("unable to parse label selector value: '%v': %v", labelSelectorString, err)
				}
				k8s.opts.labelSelector = ls
				continue
			}
			return nil, c.ArgErr()
		case "namespace_labels":
			args := c.RemainingArgs()
			if len(args) > 0 {
				namespaceLabelSelectorString := strings.Join(args, " ")
				nls, err := meta.ParseToLabelSelector(namespaceLabelSelectorString)
				if err != nil {
					return nil, fmt.Errorf("unable to parse namespace_label selector value: '%v': %v", namespaceLabelSelectorString, err)
				}
				k8s.opts.namespaceLabelSelector = nls
				continue
			}
			return nil, c.ArgErr()
		case "fallthrough":
			k8s.Fall.SetZonesFromArgs(c.RemainingArgs())
		case "upstream":
			c.RemainingArgs() // eat remaining args
			k8s.Upstream = upstream.New()
		case "ttl":
			args := c.RemainingArgs()
			if len(args) == 0 {
				return nil, c.ArgErr()
			}
			t, err := strconv.Atoi(args[0])
			if err != nil {
				return nil, err
			}
			if t < 0 || t > 3600 {
				return nil, c.Errf("ttl must be in range [0, 3600]: %d", t)
			}
			k8s.ttl = uint32(t)
		case "transfer":
			tos, froms, err := parse.Transfer(c, false)
			if err != nil {
				return nil, err
			}
			if len(froms) != 0 {
				return nil, c.Errf("transfer from is not supported with this plugin")
			}
			k8s.TransferTo = tos
		case "noendpoints":
			if len(c.RemainingArgs()) != 0 {
				return nil, c.ArgErr()
			}
			k8s.opts.initEndpointsCache = false
		case "ignore":
			args := c.RemainingArgs()
			if len(args) > 0 {
				ignore := args[0]
				if ignore == "empty_service" {
					k8s.opts.ignoreEmptyService = true
					continue
				} else {
					return nil, fmt.Errorf("unable to parse ignore value: '%v'", ignore)
				}
			}
		case "kubeconfig":
			args := c.RemainingArgs()
			if len(args) == 2 {
				config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
					&clientcmd.ClientConfigLoadingRules{ExplicitPath: args[0]},
					&clientcmd.ConfigOverrides{CurrentContext: args[1]},
				)
				k8s.ClientConfig = config
				continue
			}
			return nil, c.ArgErr()
		default:
			return nil, c.Errf("unknown property '%s'", c.Val())
		}
	}

	if len(k8s.Namespaces) != 0 && k8s.opts.namespaceLabelSelector != nil {
		return nil, c.Errf("namespaces and namespace_labels cannot both be set")
	}

	return k8s, nil
}

func searchFromResolvConf() []string {
	rc, err := dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil {
		return nil
	}
	plugin.Zones(rc.Search).Normalize()
	return rc.Search
}

const defaultResyncPeriod = 5 * time.Minute
<commit_msg>Bump the resync period to five hours (#2748)<commit_after>package kubernetes

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/coredns/coredns/core/dnsserver"
	"github.com/coredns/coredns/plugin"
	"github.com/coredns/coredns/plugin/pkg/dnsutil"
	clog "github.com/coredns/coredns/plugin/pkg/log"
	"github.com/coredns/coredns/plugin/pkg/parse"
	"github.com/coredns/coredns/plugin/pkg/upstream"

	"github.com/mholt/caddy"
	"github.com/miekg/dns"
	meta "k8s.io/apimachinery/pkg/apis/meta/v1"

	// Pull this in for logtostderr flag parsing
	"k8s.io/klog"

	// Excluding azure because it is failing to compile
	// pull this in here, because we want it excluded if plugin.cfg doesn't have k8s
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	// pull this in here, because we want it excluded if plugin.cfg doesn't have k8s
	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
	// pull this in here, because we want it excluded if plugin.cfg doesn't have k8s
	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
	"k8s.io/client-go/tools/clientcmd"
)

var log = clog.NewWithPlugin("kubernetes")

func init() {
	// Kubernetes plugin uses the kubernetes library, which now uses klog, we must set and parse this flag
	// so we don't log to the filesystem, which can fill up and crash CoreDNS indirectly by calling os.Exit().
	// We also set: os.Stderr = os.Stdout in the setup function below so we output to standard out; as we do for
	// all CoreDNS logging. We can't do *that* in the init function, because we, when starting, also barf some
	// things to stderr.
	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
	klog.InitFlags(klogFlags)
	logtostderr := klogFlags.Lookup("logtostderr")
	logtostderr.Value.Set("true")

	caddy.RegisterPlugin("kubernetes", caddy.Plugin{
		ServerType: "dns",
		Action:     setup,
	})
}

func setup(c *caddy.Controller) error {
	// See comment in the init function.
	os.Stderr = os.Stdout

	k, err := kubernetesParse(c)
	if err != nil {
		return plugin.Error("kubernetes", err)
	}

	err = k.InitKubeCache()
	if err != nil {
		return plugin.Error("kubernetes", err)
	}

	k.RegisterKubeCache(c)

	dnsserver.GetConfig(c).AddPlugin(func(next plugin.Handler) plugin.Handler {
		k.Next = next
		return k
	})

	return nil
}

// RegisterKubeCache registers KubeCache start and stop functions with Caddy
func (k *Kubernetes) RegisterKubeCache(c *caddy.Controller) {
	c.OnStartup(func() error {
		go k.APIConn.Run()

		timeout := time.After(5 * time.Second)
		ticker := time.NewTicker(100 * time.Millisecond)
		for {
			select {
			case <-ticker.C:
				if k.APIConn.HasSynced() {
					return nil
				}
			case <-timeout:
				return nil
			}
		}
	})

	c.OnShutdown(func() error {
		return k.APIConn.Stop()
	})
}

func kubernetesParse(c *caddy.Controller) (*Kubernetes, error) {
	var (
		k8s *Kubernetes
		err error
	)

	i := 0
	for c.Next() {
		if i > 0 {
			return nil, plugin.ErrOnce
		}
		i++

		k8s, err = ParseStanza(c)
		if err != nil {
			return k8s, err
		}
	}
	return k8s, nil
}

// ParseStanza parses a kubernetes stanza
func ParseStanza(c *caddy.Controller) (*Kubernetes, error) {

	k8s := New([]string{""})
	k8s.interfaceAddrsFunc = localPodIP
	k8s.autoPathSearch = searchFromResolvConf()

	opts := dnsControlOpts{
		initEndpointsCache: true,
		ignoreEmptyService: false,
		resyncPeriod:       defaultResyncPeriod,
	}
	k8s.opts = opts

	zones := c.RemainingArgs()

	if len(zones) != 0 {
		k8s.Zones = zones
		for i := 0; i < len(k8s.Zones); i++ {
			k8s.Zones[i] = plugin.Host(k8s.Zones[i]).Normalize()
		}
	} else {
		k8s.Zones = make([]string, len(c.ServerBlockKeys))
		for i := 0; i < len(c.ServerBlockKeys); i++ {
			k8s.Zones[i] = plugin.Host(c.ServerBlockKeys[i]).Normalize()
		}
	}

	k8s.primaryZoneIndex = -1
	for i, z := range k8s.Zones {
		if dnsutil.IsReverse(z) > 0 {
			continue
		}
		k8s.primaryZoneIndex = i
		break
	}

	if k8s.primaryZoneIndex == -1 {
		return nil, errors.New("non-reverse zone name must be used")
	}

	for c.NextBlock() {
		switch c.Val() {
		case "endpoint_pod_names":
			args := c.RemainingArgs()
			if len(args) > 0 {
				return nil, c.ArgErr()
			}
			k8s.endpointNameMode = true
			continue
		case "pods":
			args := c.RemainingArgs()
			if len(args) == 1 {
				switch args[0] {
				case podModeDisabled, podModeInsecure, podModeVerified:
					k8s.podMode = args[0]
				default:
					return nil, fmt.Errorf("wrong value for pods: %s,  must be one of: disabled, verified, insecure", args[0])
				}
				continue
			}
			return nil, c.ArgErr()
		case "namespaces":
			args := c.RemainingArgs()
			if len(args) > 0 {
				for _, a := range args {
					k8s.Namespaces[a] = struct{}{}
				}
				continue
			}
			return nil, c.ArgErr()
		case "endpoint":
			args := c.RemainingArgs()
			if len(args) > 0 {
				// Multiple endoints are deprecated but still could be specified,
				// only the first one be used, though
				k8s.APIServerList = args
				if len(args) > 1 {
					log.Warningf("Multiple endpoints have been deprecated, only the first specified endpoint '%s' is used", args[0])
				}
				continue
			}
			return nil, c.ArgErr()
		case "tls": // cert key cacertfile
			args := c.RemainingArgs()
			if len(args) == 3 {
				k8s.APIClientCert, k8s.APIClientKey, k8s.APICertAuth = args[0], args[1], args[2]
				continue
			}
			return nil, c.ArgErr()
		case "resyncperiod":
			args := c.RemainingArgs()
			if len(args) > 0 {
				rp, err := time.ParseDuration(args[0])
				if err != nil {
					return nil, fmt.Errorf("unable to parse resync duration value: '%v': %v", args[0], err)
				}
				k8s.opts.resyncPeriod = rp
				continue
			}
			return nil, c.ArgErr()
		case "labels":
			args := c.RemainingArgs()
			if len(args) > 0 {
				labelSelectorString := strings.Join(args, " ")
				ls, err := meta.ParseToLabelSelector(labelSelectorString)
				if err != nil {
					return nil, fmt.Errorf("unable to parse label selector value: '%v': %v", labelSelectorString, err)
				}
				k8s.opts.labelSelector = ls
				continue
			}
			return nil, c.ArgErr()
		case "namespace_labels":
			args := c.RemainingArgs()
			if len(args) > 0 {
				namespaceLabelSelectorString := strings.Join(args, " ")
				nls, err := meta.ParseToLabelSelector(namespaceLabelSelectorString)
				if err != nil {
					return nil, fmt.Errorf("unable to parse namespace_label selector value: '%v': %v", namespaceLabelSelectorString, err)
				}
				k8s.opts.namespaceLabelSelector = nls
				continue
			}
			return nil, c.ArgErr()
		case "fallthrough":
			k8s.Fall.SetZonesFromArgs(c.RemainingArgs())
		case "upstream":
			c.RemainingArgs() // eat remaining args
			k8s.Upstream = upstream.New()
		case "ttl":
			args := c.RemainingArgs()
			if len(args) == 0 {
				return nil, c.ArgErr()
			}
			t, err := strconv.Atoi(args[0])
			if err != nil {
				return nil, err
			}
			if t < 0 || t > 3600 {
				return nil, c.Errf("ttl must be in range [0, 3600]: %d", t)
			}
			k8s.ttl = uint32(t)
		case "transfer":
			tos, froms, err := parse.Transfer(c, false)
			if err != nil {
				return nil, err
			}
			if len(froms) != 0 {
				return nil, c.Errf("transfer from is not supported with this plugin")
			}
			k8s.TransferTo = tos
		case "noendpoints":
			if len(c.RemainingArgs()) != 0 {
				return nil, c.ArgErr()
			}
			k8s.opts.initEndpointsCache = false
		case "ignore":
			args := c.RemainingArgs()
			if len(args) > 0 {
				ignore := args[0]
				if ignore == "empty_service" {
					k8s.opts.ignoreEmptyService = true
					continue
				} else {
					return nil, fmt.Errorf("unable to parse ignore value: '%v'", ignore)
				}
			}
		case "kubeconfig":
			args := c.RemainingArgs()
			if len(args) == 2 {
				config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
					&clientcmd.ClientConfigLoadingRules{ExplicitPath: args[0]},
					&clientcmd.ConfigOverrides{CurrentContext: args[1]},
				)
				k8s.ClientConfig = config
				continue
			}
			return nil, c.ArgErr()
		default:
			return nil, c.Errf("unknown property '%s'", c.Val())
		}
	}

	if len(k8s.Namespaces) != 0 && k8s.opts.namespaceLabelSelector != nil {
		return nil, c.Errf("namespaces and namespace_labels cannot both be set")
	}

	return k8s, nil
}

func searchFromResolvConf() []string {
	rc, err := dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil {
		return nil
	}
	plugin.Zones(rc.Search).Normalize()
	return rc.Search
}

const defaultResyncPeriod = 5 * time.Hour
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/Sirupsen/logrus/hooks/sentry"
	"github.com/codegangsta/cli"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/librato"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/config"
	"github.com/travis-ci/worker/context"
	gocontext "golang.org/x/net/context"
)

var (
	bootTime = time.Now().UTC()
)

func main() {
	app := cli.NewApp()
	app.Usage = "Travis Worker daemon"
	app.Version = worker.VersionString
	app.Author = "Travis CI GmbH"
	app.Email = "contact+travis-worker@travis-ci.org"

	app.Flags = config.Flags
	app.Action = runWorker

	app.Run(os.Args)
}

func runWorker(c *cli.Context) {
	ctx, cancel := gocontext.WithCancel(gocontext.Background())
	logger := context.LoggerFromContext(ctx)

	logrus.SetFormatter(&logrus.TextFormatter{DisableColors: true})

	if c.String("pprof-port") != "" {
		// Start net/http/pprof server
		go func() {
			http.ListenAndServe(fmt.Sprintf("localhost:%s", c.String("pprof-port")), nil)
		}()
	}

	if c.Bool("debug") {
		logrus.SetLevel(logrus.DebugLevel)
	}

	cfg := config.ConfigFromCLIContext(c)

	if c.Bool("echo-config") {
		config.WriteEnvConfig(cfg, os.Stdout)
		return
	}

	logger.WithFields(logrus.Fields{
		"cfg": fmt.Sprintf("%#v", cfg),
	}).Debug("read config")

	logger.Info("worker started")
	defer logger.Info("worker finished")

	if cfg.SentryDSN != "" {
		sentryHook, err := logrus_sentry.NewSentryHook(cfg.SentryDSN, []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel})
		if err != nil {
			logger.WithField("err", err).Error("couldn't create sentry hook")
		}

		logrus.AddHook(sentryHook)
	}

	if cfg.LibratoEmail != "" && cfg.LibratoToken != "" && cfg.LibratoSource != "" {
		logger.Info("starting librato metrics reporter")
		go librato.Librato(metrics.DefaultRegistry, time.Minute, cfg.LibratoEmail, cfg.LibratoToken, cfg.LibratoSource, []float64{0.95}, time.Millisecond)
	} else if !c.Bool("silence-metrics") {
		logger.Info("starting logger metrics reporter")
		go metrics.Log(metrics.DefaultRegistry, time.Minute, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}

	amqpConn, err := amqp.Dial(cfg.AmqpURI)
	if err != nil {
		logger.WithField("err", err).Error("couldn't connect to AMQP")
		return
	}

	go func() {
		errChan := make(chan *amqp.Error)
		errChan = amqpConn.NotifyClose(errChan)

		err, ok := <-errChan
		if ok {
			logger.WithField("err", err).Error("amqp connection errored, terminating")
			cancel()
		}
	}()

	logger.Debug("connected to AMQP")

	generator := worker.NewBuildScriptGenerator(cfg)
	logger.WithFields(logrus.Fields{
		"build_script_generator": fmt.Sprintf("%#v", generator),
	}).Debug("built")

	provider, err := backend.NewProvider(cfg.ProviderName, cfg.ProviderConfig)
	if err != nil {
		logger.WithField("err", err).Error("couldn't create backend provider")
		return
	}
	logger.WithFields(logrus.Fields{
		"provider": fmt.Sprintf("%#v", provider),
	}).Debug("built")

	commandDispatcher := worker.NewCommandDispatcher(ctx, amqpConn)
	logger.WithFields(logrus.Fields{
		"command_dispatcher": fmt.Sprintf("%#v", commandDispatcher),
	}).Debug("built")

	go commandDispatcher.Run()

	pool := worker.NewProcessorPool(cfg.Hostname, ctx, cfg.HardTimeout, amqpConn,
		provider, generator, commandDispatcher)

	pool.SkipShutdownOnLogTimeout = cfg.SkipShutdownOnLogTimeout
	logger.WithFields(logrus.Fields{
		"pool": pool,
	}).Debug("built")

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1)
	go func() {
		for {
			select {
			case sig := <-signalChan:
				if sig == syscall.SIGINT {
					logger.Info("SIGINT received, starting graceful shutdown")
					pool.GracefulShutdown()
				} else if sig == syscall.SIGTERM {
					logger.Info("SIGTERM received, shutting down immediately")
					cancel()
				} else if sig == syscall.SIGUSR1 {
					logger.WithFields(logrus.Fields{
						"version":   worker.VersionString,
						"revision":  worker.RevisionString,
						"generated": worker.GeneratedString,
						"boot_time": bootTime,
						"uptime":    time.Since(bootTime),
					}).Info("SIGUSR1 received, dumping info")
					pool.Each(func(n int, proc *worker.Processor) {
						logger.WithFields(logrus.Fields{
							"n":         n,
							"id":        proc.ID,
							"job":       fmt.Sprintf("%#v", proc.CurrentJob),
							"processed": proc.ProcessedCount,
						}).Info("processor info")
					})
				} else {
					logger.WithField("signal", sig).Info("ignoring unknown signal")
				}
			default:
				time.Sleep(time.Second)
			}
		}
	}()

	logger.WithFields(logrus.Fields{
		"pool_size":  cfg.PoolSize,
		"queue_name": cfg.QueueName,
	}).Debug("running pool")

	pool.Run(cfg.PoolSize, cfg.QueueName)

	err = amqpConn.Close()
	if err != nil {
		logger.WithField("err", err).Error("couldn't close AMQP connection cleanly")
		return
	}
}
<commit_msg>main: change email address to travis-ci.com domain<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/Sirupsen/logrus/hooks/sentry"
	"github.com/codegangsta/cli"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/librato"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/config"
	"github.com/travis-ci/worker/context"
	gocontext "golang.org/x/net/context"
)

var (
	bootTime = time.Now().UTC()
)

func main() {
	app := cli.NewApp()
	app.Usage = "Travis Worker daemon"
	app.Version = worker.VersionString
	app.Author = "Travis CI GmbH"
	app.Email = "contact+travis-worker@travis-ci.com"

	app.Flags = config.Flags
	app.Action = runWorker

	app.Run(os.Args)
}

func runWorker(c *cli.Context) {
	ctx, cancel := gocontext.WithCancel(gocontext.Background())
	logger := context.LoggerFromContext(ctx)

	logrus.SetFormatter(&logrus.TextFormatter{DisableColors: true})

	if c.String("pprof-port") != "" {
		// Start net/http/pprof server
		go func() {
			http.ListenAndServe(fmt.Sprintf("localhost:%s", c.String("pprof-port")), nil)
		}()
	}

	if c.Bool("debug") {
		logrus.SetLevel(logrus.DebugLevel)
	}

	cfg := config.ConfigFromCLIContext(c)

	if c.Bool("echo-config") {
		config.WriteEnvConfig(cfg, os.Stdout)
		return
	}

	logger.WithFields(logrus.Fields{
		"cfg": fmt.Sprintf("%#v", cfg),
	}).Debug("read config")

	logger.Info("worker started")
	defer logger.Info("worker finished")

	if cfg.SentryDSN != "" {
		sentryHook, err := logrus_sentry.NewSentryHook(cfg.SentryDSN, []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel})
		if err != nil {
			logger.WithField("err", err).Error("couldn't create sentry hook")
		}

		logrus.AddHook(sentryHook)
	}

	if cfg.LibratoEmail != "" && cfg.LibratoToken != "" && cfg.LibratoSource != "" {
		logger.Info("starting librato metrics reporter")
		go librato.Librato(metrics.DefaultRegistry, time.Minute, cfg.LibratoEmail, cfg.LibratoToken, cfg.LibratoSource, []float64{0.95}, time.Millisecond)
	} else if !c.Bool("silence-metrics") {
		logger.Info("starting logger metrics reporter")
		go metrics.Log(metrics.DefaultRegistry, time.Minute, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}

	amqpConn, err := amqp.Dial(cfg.AmqpURI)
	if err != nil {
		logger.WithField("err", err).Error("couldn't connect to AMQP")
		return
	}

	go func() {
		errChan := make(chan *amqp.Error)
		errChan = amqpConn.NotifyClose(errChan)

		err, ok := <-errChan
		if ok {
			logger.WithField("err", err).Error("amqp connection errored, terminating")
			cancel()
		}
	}()

	logger.Debug("connected to AMQP")

	generator := worker.NewBuildScriptGenerator(cfg)
	logger.WithFields(logrus.Fields{
		"build_script_generator": fmt.Sprintf("%#v", generator),
	}).Debug("built")

	provider, err := backend.NewProvider(cfg.ProviderName, cfg.ProviderConfig)
	if err != nil {
		logger.WithField("err", err).Error("couldn't create backend provider")
		return
	}
	logger.WithFields(logrus.Fields{
		"provider": fmt.Sprintf("%#v", provider),
	}).Debug("built")

	commandDispatcher := worker.NewCommandDispatcher(ctx, amqpConn)
	logger.WithFields(logrus.Fields{
		"command_dispatcher": fmt.Sprintf("%#v", commandDispatcher),
	}).Debug("built")

	go commandDispatcher.Run()

	pool := worker.NewProcessorPool(cfg.Hostname, ctx, cfg.HardTimeout, amqpConn,
		provider, generator, commandDispatcher)

	pool.SkipShutdownOnLogTimeout = cfg.SkipShutdownOnLogTimeout
	logger.WithFields(logrus.Fields{
		"pool": pool,
	}).Debug("built")

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1)
	go func() {
		for {
			select {
			case sig := <-signalChan:
				if sig == syscall.SIGINT {
					logger.Info("SIGINT received, starting graceful shutdown")
					pool.GracefulShutdown()
				} else if sig == syscall.SIGTERM {
					logger.Info("SIGTERM received, shutting down immediately")
					cancel()
				} else if sig == syscall.SIGUSR1 {
					logger.WithFields(logrus.Fields{
						"version":   worker.VersionString,
						"revision":  worker.RevisionString,
						"generated": worker.GeneratedString,
						"boot_time": bootTime,
						"uptime":    time.Since(bootTime),
					}).Info("SIGUSR1 received, dumping info")
					pool.Each(func(n int, proc *worker.Processor) {
						logger.WithFields(logrus.Fields{
							"n":         n,
							"id":        proc.ID,
							"job":       fmt.Sprintf("%#v", proc.CurrentJob),
							"processed": proc.ProcessedCount,
						}).Info("processor info")
					})
				} else {
					logger.WithField("signal", sig).Info("ignoring unknown signal")
				}
			default:
				time.Sleep(time.Second)
			}
		}
	}()

	logger.WithFields(logrus.Fields{
		"pool_size":  cfg.PoolSize,
		"queue_name": cfg.QueueName,
	}).Debug("running pool")

	pool.Run(cfg.PoolSize, cfg.QueueName)

	err = amqpConn.Close()
	if err != nil {
		logger.WithField("err", err).Error("couldn't close AMQP connection cleanly")
		return
	}
}
<|endoftext|>


<commit_before>package farm

import (
	"errors"
	"fmt"
	"reflect"
	"sort"
	"sync/atomic"
	"testing"

	"github.com/soundcloud/roshi/cluster"
	"github.com/soundcloud/roshi/common"
)

func TestMockCluster(t *testing.T) {
	c := newMockCluster()

	// Insert 2
	if err := c.Insert([]common.KeyScoreMember{
		common.KeyScoreMember{Key: "foo", Score: 1, Member: "bar"},
		common.KeyScoreMember{Key: "foo", Score: 2, Member: "baz"},
	}); err != nil {
		t.Fatal(err)
	}

	// Select
	expected := map[string][]common.KeyScoreMember{
		"foo": []common.KeyScoreMember{
			common.KeyScoreMember{Key: "foo", Score: 2, Member: "baz"},
			common.KeyScoreMember{Key: "foo", Score: 1, Member: "bar"},
		},
	}
	ch := c.Select([]string{"foo"}, 0, 10)
	got := map[string][]common.KeyScoreMember{}
	for e := range ch {
		if e.Error != nil {
			t.Errorf("%s: %s", e.Key, e.Error)
		}
		got[e.Key] = e.KeyScoreMembers
	}
	if !reflect.DeepEqual(expected, got) {
		t.Errorf("expected %+v, got %+v", expected, got)
	}

	// Delete 1
	if err := c.Delete([]common.KeyScoreMember{
		common.KeyScoreMember{Key: "foo", Score: 999, Member: "bar"},
	}); err != nil {
		t.Fatal(err)
	}

	// Select
	expected = map[string][]common.KeyScoreMember{
		"foo": []common.KeyScoreMember{
			common.KeyScoreMember{Key: "foo", Score: 2, Member: "baz"},
		},
	}
	ch = c.Select([]string{"foo"}, 0, 10)
	got = map[string][]common.KeyScoreMember{}
	for e := range ch {
		if e.Error != nil {
			t.Errorf("%s: %s", e.Key, e.Error)
		}
		got[e.Key] = e.KeyScoreMembers
	}
	if !reflect.DeepEqual(expected, got) {
		t.Errorf("expected %+v, got %+v", expected, got)
	}
}

// mockCluster is *not* safe for concurrent use.
type mockCluster struct {
	id                int32
	m                 map[string]common.KeyScoreMembers
	failing           bool
	countInsert       int32
	countSelect       int32
	countDelete       int32
	countScore        int32
	countKeys         int32
	countOpenChannels int32
}

var mockClusterIDs int32

func newMockCluster() *mockCluster {
	return &mockCluster{
		id: atomic.AddInt32(&mockClusterIDs, 1),
		m:  map[string]common.KeyScoreMembers{},
	}
}

func newFailingMockCluster() *mockCluster {
	return &mockCluster{
		m:       map[string]common.KeyScoreMembers{},
		failing: true,
	}
}

func (c *mockCluster) Insert(tuples []common.KeyScoreMember) error {
	atomic.AddInt32(&c.countInsert, 1)
	if c.failing {
		return errors.New("failtown, population you")
	}
	for _, tuple := range tuples {
		a := c.m[tuple.Key]
		a = append(c.m[tuple.Key], common.KeyScoreMember{
			Key:    tuple.Key,
			Score:  tuple.Score,
			Member: tuple.Member,
		})
		sort.Sort(a)
		c.m[tuple.Key] = a
	}
	return nil
}

func (c *mockCluster) Select(keys []string, offset, limit int) <-chan cluster.Element {
	atomic.AddInt32(&c.countSelect, 1)
	ch := make(chan cluster.Element)
	if c.failing {
		close(ch)
		return ch
	}
	atomic.AddInt32(&c.countOpenChannels, 1)
	go func() {
		defer func() {
			close(ch)
			atomic.AddInt32(&c.countOpenChannels, -1)
		}()
		for _, key := range keys {
			ksms := c.m[key]
			if len(ksms) <= offset {
				ch <- cluster.Element{Key: key, KeyScoreMembers: []common.KeyScoreMember{}}
				continue
			}
			ksms = ksms[offset:]
			if len(ksms) > limit {
				ksms = ksms[:limit]
			}
			ch <- cluster.Element{Key: key, KeyScoreMembers: uniqueMembers(ksms)}
		}
	}()
	return ch
}

func uniqueMembers(ksms common.KeyScoreMembers) common.KeyScoreMembers {
	a := make(common.KeyScoreMembers, 0, len(ksms))
	members := map[string]struct{}{}
	for _, ksm := range ksms {
		if _, ok := members[ksm.Member]; ok {
			continue
		}
		a = append(a, ksm)
		members[ksm.Member] = struct{}{}
	}
	return a
}

func (c *mockCluster) Delete(tuples []common.KeyScoreMember) error {
	atomic.AddInt32(&c.countDelete, 1)
	if c.failing {
		return errors.New("failtown, population you")
	}
	for _, toDelete := range tuples {
		for key, existingTuples := range c.m {
			replacementTuples := []common.KeyScoreMember{}
			for _, existingTuple := range existingTuples {
				if existingTuple.Key == toDelete.Key && existingTuple.Member == toDelete.Member {
					continue
				}
				replacementTuples = append(replacementTuples, existingTuple)
			}
			c.m[key] = replacementTuples
		}
	}
	return nil
}

// Score in this mock implementation will never return a score for
// deleted entries.
func (c *mockCluster) Score(key, member string) (float64, bool, error) {
	atomic.AddInt32(&c.countScore, 1)
	if c.failing {
		return 0, false, errors.New("failtown, population you")
	}
	tuples, ok := c.m[key]
	if !ok {
		return 0, false, fmt.Errorf("no member '%s' found for key '%s'", member, key)
	}
	for _, tuple := range tuples {
		if tuple.Member == member {
			return tuple.Score, true, nil
		}
	}
	return 0, false, fmt.Errorf("no member '%s' found for key '%s'", member, key)
}

func (c *mockCluster) Keys() <-chan string {
	atomic.AddInt32(&c.countKeys, 1)
	ch := make(chan string)
	m := map[string]common.KeyScoreMembers{}
	// Copy c.m so that at least after this method has returned,
	// we don't run into issues with concurrent modifications.
	for k, v := range c.m {
		m[k] = v
	}
	go func() {
		defer close(ch)
		for key := range m {
			ch <- key
		}
	}()
	return ch
}

func (c *mockCluster) clear() {
	c.m = map[string]common.KeyScoreMembers{}
}

func newMockClusters(n int) []cluster.Cluster {
	a := make([]cluster.Cluster, n)
	for i := 0; i < n; i++ {
		a[i] = newMockCluster()
	}
	return a
}

func newFailingMockClusters(n int) []cluster.Cluster {
	a := make([]cluster.Cluster, n)
	for i := 0; i < n; i++ {
		a[i] = newFailingMockCluster()
	}
	return a
}
<commit_msg>farm: fix mockCluster Keys<commit_after>package farm

import (
	"errors"
	"fmt"
	"reflect"
	"sort"
	"sync/atomic"
	"testing"

	"github.com/soundcloud/roshi/cluster"
	"github.com/soundcloud/roshi/common"
)

func TestMockCluster(t *testing.T) {
	c := newMockCluster()

	// Insert 2
	if err := c.Insert([]common.KeyScoreMember{
		common.KeyScoreMember{Key: "foo", Score: 1, Member: "bar"},
		common.KeyScoreMember{Key: "foo", Score: 2, Member: "baz"},
	}); err != nil {
		t.Fatal(err)
	}

	// Select
	expected := map[string][]common.KeyScoreMember{
		"foo": []common.KeyScoreMember{
			common.KeyScoreMember{Key: "foo", Score: 2, Member: "baz"},
			common.KeyScoreMember{Key: "foo", Score: 1, Member: "bar"},
		},
	}
	ch := c.Select([]string{"foo"}, 0, 10)
	got := map[string][]common.KeyScoreMember{}
	for e := range ch {
		if e.Error != nil {
			t.Errorf("%s: %s", e.Key, e.Error)
		}
		got[e.Key] = e.KeyScoreMembers
	}
	if !reflect.DeepEqual(expected, got) {
		t.Errorf("expected %+v, got %+v", expected, got)
	}

	// Delete 1
	if err := c.Delete([]common.KeyScoreMember{
		common.KeyScoreMember{Key: "foo", Score: 999, Member: "bar"},
	}); err != nil {
		t.Fatal(err)
	}

	// Select
	expected = map[string][]common.KeyScoreMember{
		"foo": []common.KeyScoreMember{
			common.KeyScoreMember{Key: "foo", Score: 2, Member: "baz"},
		},
	}
	ch = c.Select([]string{"foo"}, 0, 10)
	got = map[string][]common.KeyScoreMember{}
	for e := range ch {
		if e.Error != nil {
			t.Errorf("%s: %s", e.Key, e.Error)
		}
		got[e.Key] = e.KeyScoreMembers
	}
	if !reflect.DeepEqual(expected, got) {
		t.Errorf("expected %+v, got %+v", expected, got)
	}
}

// mockCluster is *not* safe for concurrent use.
type mockCluster struct {
	id                int32
	m                 map[string]common.KeyScoreMembers
	failing           bool
	countInsert       int32
	countSelect       int32
	countDelete       int32
	countScore        int32
	countKeys         int32
	countOpenChannels int32
}

var mockClusterIDs int32

func newMockCluster() *mockCluster {
	return &mockCluster{
		id: atomic.AddInt32(&mockClusterIDs, 1),
		m:  map[string]common.KeyScoreMembers{},
	}
}

func newFailingMockCluster() *mockCluster {
	return &mockCluster{
		m:       map[string]common.KeyScoreMembers{},
		failing: true,
	}
}

func (c *mockCluster) Insert(tuples []common.KeyScoreMember) error {
	atomic.AddInt32(&c.countInsert, 1)
	if c.failing {
		return errors.New("failtown, population you")
	}
	for _, tuple := range tuples {
		a := c.m[tuple.Key]
		a = append(c.m[tuple.Key], common.KeyScoreMember{
			Key:    tuple.Key,
			Score:  tuple.Score,
			Member: tuple.Member,
		})
		sort.Sort(a)
		c.m[tuple.Key] = a
	}
	return nil
}

func (c *mockCluster) Select(keys []string, offset, limit int) <-chan cluster.Element {
	atomic.AddInt32(&c.countSelect, 1)
	ch := make(chan cluster.Element)
	if c.failing {
		close(ch)
		return ch
	}
	atomic.AddInt32(&c.countOpenChannels, 1)
	go func() {
		defer func() {
			close(ch)
			atomic.AddInt32(&c.countOpenChannels, -1)
		}()
		for _, key := range keys {
			ksms := c.m[key]
			if len(ksms) <= offset {
				ch <- cluster.Element{Key: key, KeyScoreMembers: []common.KeyScoreMember{}}
				continue
			}
			ksms = ksms[offset:]
			if len(ksms) > limit {
				ksms = ksms[:limit]
			}
			ch <- cluster.Element{Key: key, KeyScoreMembers: uniqueMembers(ksms)}
		}
	}()
	return ch
}

func uniqueMembers(ksms common.KeyScoreMembers) common.KeyScoreMembers {
	a := make(common.KeyScoreMembers, 0, len(ksms))
	members := map[string]struct{}{}
	for _, ksm := range ksms {
		if _, ok := members[ksm.Member]; ok {
			continue
		}
		a = append(a, ksm)
		members[ksm.Member] = struct{}{}
	}
	return a
}

func (c *mockCluster) Delete(tuples []common.KeyScoreMember) error {
	atomic.AddInt32(&c.countDelete, 1)
	if c.failing {
		return errors.New("failtown, population you")
	}
	for _, toDelete := range tuples {
		for key, existingTuples := range c.m {
			replacementTuples := []common.KeyScoreMember{}
			for _, existingTuple := range existingTuples {
				if existingTuple.Key == toDelete.Key && existingTuple.Member == toDelete.Member {
					continue
				}
				replacementTuples = append(replacementTuples, existingTuple)
			}
			c.m[key] = replacementTuples
		}
	}
	return nil
}

// Score in this mock implementation will never return a score for
// deleted entries.
func (c *mockCluster) Score(key, member string) (float64, bool, error) {
	atomic.AddInt32(&c.countScore, 1)
	if c.failing {
		return 0, false, errors.New("failtown, population you")
	}
	tuples, ok := c.m[key]
	if !ok {
		return 0, false, fmt.Errorf("no member '%s' found for key '%s'", member, key)
	}
	for _, tuple := range tuples {
		if tuple.Member == member {
			return tuple.Score, true, nil
		}
	}
	return 0, false, fmt.Errorf("no member '%s' found for key '%s'", member, key)
}

func (c *mockCluster) Keys(batchSize int) <-chan []string {
	atomic.AddInt32(&c.countKeys, 1)
	ch := make(chan []string)
	m := map[string]common.KeyScoreMembers{}
	// Copy c.m so that at least after this method has returned,
	// we don't run into issues with concurrent modifications.
	for k, v := range c.m {
		m[k] = v
	}
	go func() {
		defer close(ch)
		batch := []string{}
		for key := range m {
			batch = append(batch, key)
			if len(batch) >= batchSize {
				ch <- batch
				batch = []string{}
			}
		}
		if len(batch) > 0 {
			ch <- batch
		}
	}()
	return ch
}

func (c *mockCluster) clear() {
	c.m = map[string]common.KeyScoreMembers{}
}

func newMockClusters(n int) []cluster.Cluster {
	a := make([]cluster.Cluster, n)
	for i := 0; i < n; i++ {
		a[i] = newMockCluster()
	}
	return a
}

func newFailingMockClusters(n int) []cluster.Cluster {
	a := make([]cluster.Cluster, n)
	for i := 0; i < n; i++ {
		a[i] = newFailingMockCluster()
	}
	return a
}
<|endoftext|>