<commit_before>package river

import (
	"bytes"
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/siddontang/go-mysql-elasticsearch/elastic"
	"github.com/siddontang/go-mysql/canal"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
	"github.com/siddontang/go-mysql/schema"
)

const (
	syncInsertDoc = iota
	syncDeleteDoc
	syncUpdateDoc
)

const (
	fieldTypeList = "list"
)

type posSaver struct {
	pos   mysql.Position
	force bool
}

type eventHandler struct {
	r *River
}

func (h *eventHandler) OnRotate(e *replication.RotateEvent) error {
	pos := mysql.Position{
		string(e.NextLogName),
		uint32(e.Position),
	}

	h.r.syncCh <- posSaver{pos, true}

	return h.r.ctx.Err()
}

func (h *eventHandler) OnDDL(nextPos mysql.Position, _ *replication.QueryEvent) error {
	h.r.syncCh <- posSaver{nextPos, true}
	return h.r.ctx.Err()
}

func (h *eventHandler) OnXID(nextPos mysql.Position) error {
	h.r.syncCh <- posSaver{nextPos, false}
	return h.r.ctx.Err()
}

func (h *eventHandler) OnRow(e *canal.RowsEvent) error {
	rule, ok := h.r.rules[ruleKey(e.Table.Schema, e.Table.Name)]
	if !ok {
		return nil
	}

	var reqs []*elastic.BulkRequest
	var err error
	switch e.Action {
	case canal.InsertAction:
		reqs, err = h.r.makeInsertRequest(rule, e.Rows)
	case canal.DeleteAction:
		reqs, err = h.r.makeDeleteRequest(rule, e.Rows)
	case canal.UpdateAction:
		reqs, err = h.r.makeUpdateRequest(rule, e.Rows)
	default:
		err = errors.Errorf("invalid rows action %s", e.Action)
	}

	if err != nil {
		h.r.cancel()
		return errors.Errorf("make %s ES request err %v, close sync", e.Action, err)
	}

	h.r.syncCh <- reqs

	return h.r.ctx.Err()
}

func (h *eventHandler) String() string {
	return "ESRiverEventHandler"
}

func (r *River) syncLoop() {
	bulkSize := r.c.BulkSize
	if bulkSize == 0 {
		bulkSize = 128
	}

	interval := r.c.FlushBulkTime.Duration
	if interval == 0 {
		interval = 200 * time.Millisecond
	}

	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	defer r.wg.Done()

	lastSavedTime := time.Now()
	reqs := make([]*elastic.BulkRequest, 0, 1024)

	var pos mysql.Position

	for {
		needFlush := false
		needSavePos := false

		select {
		case v := <-r.syncCh:
			switch v := v.(type) {
			case posSaver:
				now := time.Now()
				if v.force || now.Sub(lastSavedTime) > 3*time.Second {
					lastSavedTime = now
					needFlush = true
					needSavePos = true
					pos = v.pos
				}
			case []*elastic.BulkRequest:
				reqs = append(reqs, v...)
				needFlush = len(reqs) >= bulkSize
			}
		case <-ticker.C:
			needFlush = true
		case <-r.ctx.Done():
			return
		}

		if needFlush {
			// TODO: retry some times?
			if err := r.doBulk(reqs); err != nil {
				log.Errorf("do ES bulk err %v, close sync", err)
				r.cancel()
				return
			}
			reqs = reqs[0:0]
		}

		if needSavePos {
			if err := r.master.Save(pos); err != nil {
				log.Errorf("save sync position %s err %v, close sync", pos, err)
				r.cancel()
				return
			}
		}
	}
}

// for insert and delete
func (r *River) makeRequest(rule *Rule, action string, rows [][]interface{}) ([]*elastic.BulkRequest, error) {
	reqs := make([]*elastic.BulkRequest, 0, len(rows))

	for _, values := range rows {
		id, err := r.getDocID(rule, values)
		if err != nil {
			return nil, errors.Trace(err)
		}

		parentID := ""
		if len(rule.Parent) > 0 {
			if parentID, err = r.getParentID(rule, values, rule.Parent); err != nil {
				return nil, errors.Trace(err)
			}
		}

		req := &elastic.BulkRequest{Index: rule.Index, Type: rule.Type, ID: id, Parent: parentID}

		if action == canal.DeleteAction {
			req.Action = elastic.ActionDelete
			r.st.DeleteNum.Add(1)
		} else {
			r.makeInsertReqData(req, rule, values)
			r.st.InsertNum.Add(1)
		}

		reqs = append(reqs, req)
	}

	return reqs, nil
}

func (r *River) makeInsertRequest(rule *Rule, rows [][]interface{}) ([]*elastic.BulkRequest, error) {
	return r.makeRequest(rule, canal.InsertAction, rows)
}

func (r *River) makeDeleteRequest(rule *Rule, rows [][]interface{}) ([]*elastic.BulkRequest, error) {
	return r.makeRequest(rule, canal.DeleteAction, rows)
}

func (r *River) makeUpdateRequest(rule *Rule, rows [][]interface{}) ([]*elastic.BulkRequest, error) {
	if len(rows)%2 != 0 {
		return nil, errors.Errorf("invalid update rows event, must have 2x rows, but %d", len(rows))
	}

	reqs := make([]*elastic.BulkRequest, 0, len(rows))

	for i := 0; i < len(rows); i += 2 {
		beforeID, err := r.getDocID(rule, rows[i])
		if err != nil {
			return nil, errors.Trace(err)
		}

		afterID, err := r.getDocID(rule, rows[i+1])

		if err != nil {
			return nil, errors.Trace(err)
		}

		beforeParentID, afterParentID := "", ""
		if len(rule.Parent) > 0 {
			if beforeParentID, err = r.getParentID(rule, rows[i], rule.Parent); err != nil {
				return nil, errors.Trace(err)
			}
			if afterParentID, err = r.getParentID(rule, rows[i+1], rule.Parent); err != nil {
				return nil, errors.Trace(err)
			}
		}

		req := &elastic.BulkRequest{Index: rule.Index, Type: rule.Type, ID: beforeID, Parent: beforeParentID}

		if beforeID != afterID || beforeParentID != afterParentID {
			req.Action = elastic.ActionDelete
			reqs = append(reqs, req)

			req = &elastic.BulkRequest{Index: rule.Index, Type: rule.Type, ID: afterID, Parent: afterParentID}
			r.makeInsertReqData(req, rule, rows[i+1])

			r.st.DeleteNum.Add(1)
			r.st.InsertNum.Add(1)
		} else {
			r.makeUpdateReqData(req, rule, rows[i], rows[i+1])
			r.st.UpdateNum.Add(1)
		}

		reqs = append(reqs, req)
	}

	return reqs, nil
}

func (r *River) makeReqColumnData(col *schema.TableColumn, value interface{}) interface{} {
	switch col.Type {
	case schema.TYPE_ENUM:
		switch value := value.(type) {
		case int64:
			// for binlog, ENUM may be int64, but for dump, enum is string
			eNum := value - 1
			if eNum < 0 || eNum >= int64(len(col.EnumValues)) {
				// we insert invalid enum value before, so return empty
				log.Warnf("invalid binlog enum index %d, for enum %v", eNum, col.EnumValues)
				return ""
			}

			return col.EnumValues[eNum]
		}
	case schema.TYPE_SET:
		switch value := value.(type) {
		case int64:
			// for binlog, SET may be int64, but for dump, SET is string
			bitmask := value
			sets := make([]string, 0, len(col.SetValues))
			for i, s := range col.SetValues {
				if bitmask&int64(1<<uint(i)) > 0 {
					sets = append(sets, s)
				}
			}
			return strings.Join(sets, ",")
		}
	case schema.TYPE_BIT:
		switch value := value.(type) {
		case string:
			// for binlog, BIT is int64, but for dump, BIT is string
			// for dump 0x01 is for 1, \0 is for 0
			if value == "\x01" {
				return int64(1)
			}

			return int64(0)
		}
	case schema.TYPE_STRING:
		switch value := value.(type) {
		case []byte:
			return string(value[:])
		}
	}

	return value
}

func (r *River) getFieldParts(k string, v string) (string, string, string) {
	composedField := strings.Split(v, ",")

	mysql := k
	elastic := composedField[0]
	fieldType := ""

	if 0 == len(elastic) {
		elastic = mysql
	}
	if 2 == len(composedField) {
		fieldType = composedField[1]
	}

	return mysql, elastic, fieldType
}

func (r *River) makeInsertReqData(req *elastic.BulkRequest, rule *Rule, values []interface{}) {
	req.Data = make(map[string]interface{}, len(values))
	req.Action = elastic.ActionIndex

	for i, c := range rule.TableInfo.Columns {
		if !rule.CheckFilter(c.Name) {
			continue
		}
		mapped := false
		for k, v := range rule.FieldMapping {
			mysql, elastic, fieldType := r.getFieldParts(k, v)
			if mysql == c.Name {
				mapped = true
				v := r.makeReqColumnData(&c, values[i])
				if fieldType == fieldTypeList {
					if str, ok := v.(string); ok {
						req.Data[elastic] = strings.Split(str, ",")
					} else {
						req.Data[elastic] = v
					}
				} else {
					req.Data[elastic] = v
				}
			}
		}
		if mapped == false {
			req.Data[c.Name] = r.makeReqColumnData(&c, values[i])
		}
	}
}

func (r *River) makeUpdateReqData(req *elastic.BulkRequest, rule *Rule,
	beforeValues []interface{}, afterValues []interface{}) {
	req.Data = make(map[string]interface{}, len(beforeValues))

	// maybe dangerous if something wrong delete before?
	req.Action = elastic.ActionUpdate

	for i, c := range rule.TableInfo.Columns {
		mapped := false
		if !rule.CheckFilter(c.Name) {
			continue
		}
		if reflect.DeepEqual(beforeValues[i], afterValues[i]) {
			//nothing changed
			continue
		}
		for k, v := range rule.FieldMapping {
			mysql, elastic, fieldType := r.getFieldParts(k, v)
			if mysql == c.Name {
				mapped = true
				// has custom field mapping
				v := r.makeReqColumnData(&c, afterValues[i])
				str, ok := v.(string)
				if ok == false {
					req.Data[c.Name] = v
				} else {
					if fieldType == fieldTypeList {
						req.Data[elastic] = strings.Split(str, ",")
					} else {
						req.Data[elastic] = str
					}
				}
			}
		}
		if mapped == false {
			req.Data[c.Name] = r.makeReqColumnData(&c, afterValues[i])
		}

	}
}

// If id in toml file is none, get primary keys in one row and format them into a string, and PK must not be nil
// Else get the ID's column in one row and format them into a string
func (r *River) getDocID(rule *Rule, row []interface{}) (string, error) {
	var (
  		ids []interface{}
  		err error
	)
	if rule.ID == nil {
		ids, err = canal.GetPKValues(rule.TableInfo, row)
		if err != nil {
			return "", err
		}
	} else {
		ids = make([]interface{}, 0, len(rule.ID))
		for _, column := range rule.ID {
			value, err := canal.GetColumnValue(rule.TableInfo, column, row)
			if err != nil {
				return "", err
			}
			ids = append(ids, value)
		}
	}

	var buf bytes.Buffer

	sep := ""
	for i, value := range ids {
		if value == nil {
			return "", errors.Errorf("The %ds id or PK value is nil", i)
		}

		buf.WriteString(fmt.Sprintf("%s%v", sep, value))
		sep = ":"
	}

	return buf.String(), nil
}

func (r *River) getParentID(rule *Rule, row []interface{}, columnName string) (string, error) {
	index := rule.TableInfo.FindColumn(columnName)
	if index < 0 {
		return "", errors.Errorf("parent id not found %s(%s)", rule.TableInfo.Name, columnName)
	}

	return fmt.Sprint(row[index]), nil
}

func (r *River) doBulk(reqs []*elastic.BulkRequest) error {
	if len(reqs) == 0 {
		return nil
	}

	if resp, err := r.es.Bulk(reqs); err != nil {
		log.Errorf("sync docs err %v after binlog %s", err, r.canal.SyncedPosition())
		return errors.Trace(err)
	} else if resp.Code / 100 == 2 || resp.Errors {
		for i := 0; i < len(resp.Items); i++ {
			for action, item := range resp.Items[i] {
				if len(item.Error) > 0 {
					log.Errorf("%s index: %s, type: %s, id: %s, status: %d, error: %s",
						action, item.Index, item.Type, item.ID, item.Status, item.Error)
				}
			}
		}
	}

	return nil
}
<commit_msg>add support of parse json field<commit_after>package river

import (
	"bytes"
	"fmt"
	"reflect"
	"strings"
	"time"
	"encoding/json"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/siddontang/go-mysql-elasticsearch/elastic"
	"github.com/siddontang/go-mysql/canal"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
	"github.com/siddontang/go-mysql/schema"
)

const (
	syncInsertDoc = iota
	syncDeleteDoc
	syncUpdateDoc
)

const (
	fieldTypeList = "list"
)

type posSaver struct {
	pos   mysql.Position
	force bool
}

type eventHandler struct {
	r *River
}

func (h *eventHandler) OnRotate(e *replication.RotateEvent) error {
	pos := mysql.Position{
		string(e.NextLogName),
		uint32(e.Position),
	}

	h.r.syncCh <- posSaver{pos, true}

	return h.r.ctx.Err()
}

func (h *eventHandler) OnDDL(nextPos mysql.Position, _ *replication.QueryEvent) error {
	h.r.syncCh <- posSaver{nextPos, true}
	return h.r.ctx.Err()
}

func (h *eventHandler) OnXID(nextPos mysql.Position) error {
	h.r.syncCh <- posSaver{nextPos, false}
	return h.r.ctx.Err()
}

func (h *eventHandler) OnRow(e *canal.RowsEvent) error {
	rule, ok := h.r.rules[ruleKey(e.Table.Schema, e.Table.Name)]
	if !ok {
		return nil
	}

	var reqs []*elastic.BulkRequest
	var err error
	switch e.Action {
	case canal.InsertAction:
		reqs, err = h.r.makeInsertRequest(rule, e.Rows)
	case canal.DeleteAction:
		reqs, err = h.r.makeDeleteRequest(rule, e.Rows)
	case canal.UpdateAction:
		reqs, err = h.r.makeUpdateRequest(rule, e.Rows)
	default:
		err = errors.Errorf("invalid rows action %s", e.Action)
	}

	if err != nil {
		h.r.cancel()
		return errors.Errorf("make %s ES request err %v, close sync", e.Action, err)
	}

	h.r.syncCh <- reqs

	return h.r.ctx.Err()
}

func (h *eventHandler) String() string {
	return "ESRiverEventHandler"
}

func (r *River) syncLoop() {
	bulkSize := r.c.BulkSize
	if bulkSize == 0 {
		bulkSize = 128
	}

	interval := r.c.FlushBulkTime.Duration
	if interval == 0 {
		interval = 200 * time.Millisecond
	}

	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	defer r.wg.Done()

	lastSavedTime := time.Now()
	reqs := make([]*elastic.BulkRequest, 0, 1024)

	var pos mysql.Position

	for {
		needFlush := false
		needSavePos := false

		select {
		case v := <-r.syncCh:
			switch v := v.(type) {
			case posSaver:
				now := time.Now()
				if v.force || now.Sub(lastSavedTime) > 3*time.Second {
					lastSavedTime = now
					needFlush = true
					needSavePos = true
					pos = v.pos
				}
			case []*elastic.BulkRequest:
				reqs = append(reqs, v...)
				needFlush = len(reqs) >= bulkSize
			}
		case <-ticker.C:
			needFlush = true
		case <-r.ctx.Done():
			return
		}

		if needFlush {
			// TODO: retry some times?
			if err := r.doBulk(reqs); err != nil {
				log.Errorf("do ES bulk err %v, close sync", err)
				r.cancel()
				return
			}
			reqs = reqs[0:0]
		}

		if needSavePos {
			if err := r.master.Save(pos); err != nil {
				log.Errorf("save sync position %s err %v, close sync", pos, err)
				r.cancel()
				return
			}
		}
	}
}

// for insert and delete
func (r *River) makeRequest(rule *Rule, action string, rows [][]interface{}) ([]*elastic.BulkRequest, error) {
	reqs := make([]*elastic.BulkRequest, 0, len(rows))

	for _, values := range rows {
		id, err := r.getDocID(rule, values)
		if err != nil {
			return nil, errors.Trace(err)
		}

		parentID := ""
		if len(rule.Parent) > 0 {
			if parentID, err = r.getParentID(rule, values, rule.Parent); err != nil {
				return nil, errors.Trace(err)
			}
		}

		req := &elastic.BulkRequest{Index: rule.Index, Type: rule.Type, ID: id, Parent: parentID}

		if action == canal.DeleteAction {
			req.Action = elastic.ActionDelete
			r.st.DeleteNum.Add(1)
		} else {
			r.makeInsertReqData(req, rule, values)
			r.st.InsertNum.Add(1)
		}

		reqs = append(reqs, req)
	}

	return reqs, nil
}

func (r *River) makeInsertRequest(rule *Rule, rows [][]interface{}) ([]*elastic.BulkRequest, error) {
	return r.makeRequest(rule, canal.InsertAction, rows)
}

func (r *River) makeDeleteRequest(rule *Rule, rows [][]interface{}) ([]*elastic.BulkRequest, error) {
	return r.makeRequest(rule, canal.DeleteAction, rows)
}

func (r *River) makeUpdateRequest(rule *Rule, rows [][]interface{}) ([]*elastic.BulkRequest, error) {
	if len(rows)%2 != 0 {
		return nil, errors.Errorf("invalid update rows event, must have 2x rows, but %d", len(rows))
	}

	reqs := make([]*elastic.BulkRequest, 0, len(rows))

	for i := 0; i < len(rows); i += 2 {
		beforeID, err := r.getDocID(rule, rows[i])
		if err != nil {
			return nil, errors.Trace(err)
		}

		afterID, err := r.getDocID(rule, rows[i+1])

		if err != nil {
			return nil, errors.Trace(err)
		}

		beforeParentID, afterParentID := "", ""
		if len(rule.Parent) > 0 {
			if beforeParentID, err = r.getParentID(rule, rows[i], rule.Parent); err != nil {
				return nil, errors.Trace(err)
			}
			if afterParentID, err = r.getParentID(rule, rows[i+1], rule.Parent); err != nil {
				return nil, errors.Trace(err)
			}
		}

		req := &elastic.BulkRequest{Index: rule.Index, Type: rule.Type, ID: beforeID, Parent: beforeParentID}

		if beforeID != afterID || beforeParentID != afterParentID {
			req.Action = elastic.ActionDelete
			reqs = append(reqs, req)

			req = &elastic.BulkRequest{Index: rule.Index, Type: rule.Type, ID: afterID, Parent: afterParentID}
			r.makeInsertReqData(req, rule, rows[i+1])

			r.st.DeleteNum.Add(1)
			r.st.InsertNum.Add(1)
		} else {
			r.makeUpdateReqData(req, rule, rows[i], rows[i+1])
			r.st.UpdateNum.Add(1)
		}

		reqs = append(reqs, req)
	}

	return reqs, nil
}

func (r *River) makeReqColumnData(col *schema.TableColumn, value interface{}) interface{} {
	switch col.Type {
	case schema.TYPE_ENUM:
		switch value := value.(type) {
		case int64:
			// for binlog, ENUM may be int64, but for dump, enum is string
			eNum := value - 1
			if eNum < 0 || eNum >= int64(len(col.EnumValues)) {
				// we insert invalid enum value before, so return empty
				log.Warnf("invalid binlog enum index %d, for enum %v", eNum, col.EnumValues)
				return ""
			}

			return col.EnumValues[eNum]
		}
	case schema.TYPE_SET:
		switch value := value.(type) {
		case int64:
			// for binlog, SET may be int64, but for dump, SET is string
			bitmask := value
			sets := make([]string, 0, len(col.SetValues))
			for i, s := range col.SetValues {
				if bitmask&int64(1<<uint(i)) > 0 {
					sets = append(sets, s)
				}
			}
			return strings.Join(sets, ",")
		}
	case schema.TYPE_BIT:
		switch value := value.(type) {
		case string:
			// for binlog, BIT is int64, but for dump, BIT is string
			// for dump 0x01 is for 1, \0 is for 0
			if value == "\x01" {
				return int64(1)
			}

			return int64(0)
		}
	case schema.TYPE_STRING:
		switch value := value.(type) {
		case []byte:
			return string(value[:])
		}
	case schema.TYPE_JSON:
		var f interface{}
		var err error
		switch v := value.(type) {
		case string:
			err = json.Unmarshal([]byte(v), &f)
		case []byte:
			err = json.Unmarshal(v, &f)
		}
		if err == nil && f != nil {
			return f
		}
	}

	return value
}

func (r *River) getFieldParts(k string, v string) (string, string, string) {
	composedField := strings.Split(v, ",")

	mysql := k
	elastic := composedField[0]
	fieldType := ""

	if 0 == len(elastic) {
		elastic = mysql
	}
	if 2 == len(composedField) {
		fieldType = composedField[1]
	}

	return mysql, elastic, fieldType
}

func (r *River) makeInsertReqData(req *elastic.BulkRequest, rule *Rule, values []interface{}) {
	req.Data = make(map[string]interface{}, len(values))
	req.Action = elastic.ActionIndex

	for i, c := range rule.TableInfo.Columns {
		if !rule.CheckFilter(c.Name) {
			continue
		}
		mapped := false
		for k, v := range rule.FieldMapping {
			mysql, elastic, fieldType := r.getFieldParts(k, v)
			if mysql == c.Name {
				mapped = true
				v := r.makeReqColumnData(&c, values[i])
				if fieldType == fieldTypeList {
					if str, ok := v.(string); ok {
						req.Data[elastic] = strings.Split(str, ",")
					} else {
						req.Data[elastic] = v
					}
				} else {
					req.Data[elastic] = v
				}
			}
		}
		if mapped == false {
			req.Data[c.Name] = r.makeReqColumnData(&c, values[i])
		}
	}
}

func (r *River) makeUpdateReqData(req *elastic.BulkRequest, rule *Rule,
	beforeValues []interface{}, afterValues []interface{}) {
	req.Data = make(map[string]interface{}, len(beforeValues))

	// maybe dangerous if something wrong delete before?
	req.Action = elastic.ActionUpdate

	for i, c := range rule.TableInfo.Columns {
		mapped := false
		if !rule.CheckFilter(c.Name) {
			continue
		}
		if reflect.DeepEqual(beforeValues[i], afterValues[i]) {
			//nothing changed
			continue
		}
		for k, v := range rule.FieldMapping {
			mysql, elastic, fieldType := r.getFieldParts(k, v)
			if mysql == c.Name {
				mapped = true
				// has custom field mapping
				v := r.makeReqColumnData(&c, afterValues[i])
				str, ok := v.(string)
				if ok == false {
					req.Data[c.Name] = v
				} else {
					if fieldType == fieldTypeList {
						req.Data[elastic] = strings.Split(str, ",")
					} else {
						req.Data[elastic] = str
					}
				}
			}
		}
		if mapped == false {
			req.Data[c.Name] = r.makeReqColumnData(&c, afterValues[i])
		}

	}
}

// If id in toml file is none, get primary keys in one row and format them into a string, and PK must not be nil
// Else get the ID's column in one row and format them into a string
func (r *River) getDocID(rule *Rule, row []interface{}) (string, error) {
	var (
  		ids []interface{}
  		err error
	)
	if rule.ID == nil {
		ids, err = canal.GetPKValues(rule.TableInfo, row)
		if err != nil {
			return "", err
		}
	} else {
		ids = make([]interface{}, 0, len(rule.ID))
		for _, column := range rule.ID {
			value, err := canal.GetColumnValue(rule.TableInfo, column, row)
			if err != nil {
				return "", err
			}
			ids = append(ids, value)
		}
	}

	var buf bytes.Buffer

	sep := ""
	for i, value := range ids {
		if value == nil {
			return "", errors.Errorf("The %ds id or PK value is nil", i)
		}

		buf.WriteString(fmt.Sprintf("%s%v", sep, value))
		sep = ":"
	}

	return buf.String(), nil
}

func (r *River) getParentID(rule *Rule, row []interface{}, columnName string) (string, error) {
	index := rule.TableInfo.FindColumn(columnName)
	if index < 0 {
		return "", errors.Errorf("parent id not found %s(%s)", rule.TableInfo.Name, columnName)
	}

	return fmt.Sprint(row[index]), nil
}

func (r *River) doBulk(reqs []*elastic.BulkRequest) error {
	if len(reqs) == 0 {
		return nil
	}

	if resp, err := r.es.Bulk(reqs); err != nil {
		log.Errorf("sync docs err %v after binlog %s", err, r.canal.SyncedPosition())
		return errors.Trace(err)
	} else if resp.Code / 100 == 2 || resp.Errors {
		for i := 0; i < len(resp.Items); i++ {
			for action, item := range resp.Items[i] {
				if len(item.Error) > 0 {
					log.Errorf("%s index: %s, type: %s, id: %s, status: %d, error: %s",
						action, item.Index, item.Type, item.ID, item.Status, item.Error)
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package river

import (
	"bytes"
	"fmt"
	"reflect"
	"strings"
	"time"
	"encoding/json"
	"encoding/hex"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/WangXiangUSTC/go-mysql-mongodb/mongodb"
	"github.com/siddontang/go-mysql/canal"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
	"github.com/siddontang/go-mysql/schema"
)

const (
	syncInsertDoc = iota
	syncDeleteDoc
	syncUpdateDoc
)

const (
	fieldTypeList = "list"
)

type posSaver struct {
	pos   mysql.Position
	force bool
}

type eventHandler struct {
	r *River
}

func (h *eventHandler) OnRotate(e *replication.RotateEvent) error {
	pos := mysql.Position{
		string(e.NextLogName),
		uint32(e.Position),
	}

	h.r.syncCh <- posSaver{pos, true}

	return h.r.ctx.Err()
}

func (h *eventHandler) OnDDL(nextPos mysql.Position, _ *replication.QueryEvent) error {
	h.r.syncCh <- posSaver{nextPos, true}
	return h.r.ctx.Err()
}

func (h *eventHandler) OnXID(nextPos mysql.Position) error {
	h.r.syncCh <- posSaver{nextPos, false}
	return h.r.ctx.Err()
}

func (h *eventHandler) OnRow(e *canal.RowsEvent) error {
    var err error
	rule, ok := h.r.rules[ruleKey(e.Table.Schema, e.Table.Name)]
	if !ok {
        if h.r.c.AllDB == "yes" {
            rule = newDefaultRule(e.Table.Schema, e.Table.Name)
            rule.TableInfo, err = h.r.canal.GetTable(e.Table.Schema, e.Table.Name)
            if err != nil {
                return nil
            }
            h.r.rules[ruleKey(e.Table.Schema, e.Table.Name)] = rule
        } else {
            return nil
        }
	}

	var reqs []*mongodb.BulkRequest
	switch e.Action {
	case canal.InsertAction:
		reqs, err = h.r.makeInsertRequest(rule, e.Rows)
	case canal.DeleteAction:
		reqs, err = h.r.makeDeleteRequest(rule, e.Rows)
	case canal.UpdateAction:
		reqs, err = h.r.makeUpdateRequest(rule, e.Rows)
	default:
		err = errors.Errorf("invalid rows action %s", e.Action)
	}

	if err != nil {
		h.r.cancel()
        log.Warnf("make %s MongoDB request err %v, close sync", e.Action, err)
		return errors.Errorf("make %s MongoDB request err %v, close sync", e.Action, err)
	}

	h.r.syncCh <- reqs

	return h.r.ctx.Err()
}

func (h *eventHandler) String() string {
	return "MongoRiverEventHandler"
}

func (r *River) syncLoop() {
	bulkSize := r.c.BulkSize
	if bulkSize == 0 {
		bulkSize = 512
	}

	interval := r.c.FlushBulkTime.Duration
	if interval == 0 {
		interval = 500 * time.Millisecond
	}

	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	defer r.wg.Done()

	lastSavedTime := time.Now()
	reqs := make([]*mongodb.BulkRequest, 0, 2048)

	var pos mysql.Position

	for {
		needFlush := false
		needSavePos := false

		select {
		case v := <-r.syncCh:
			switch v := v.(type) {
			case posSaver:
				now := time.Now()
				if v.force || now.Sub(lastSavedTime) > 3*time.Second {
					lastSavedTime = now
					needFlush = true
					needSavePos = true
					pos = v.pos
				}
			case []*mongodb.BulkRequest:
				reqs = append(reqs, v...)
				needFlush = len(reqs) >= bulkSize
			}
		case <-ticker.C:
			needFlush = true
		case <-r.ctx.Done():
			return
		}

		if needFlush {
			// TODO: retry some times?
			if err := r.doBulk(reqs); err != nil {
				log.Errorf("do MongoDB bulk err %v, close sync", err)
				r.cancel()
				return
			}
			reqs = reqs[0:0]
		}

		if needSavePos {
			if err := r.master.Save(pos); err != nil {
				log.Errorf("save sync position %s err %v, close sync", pos, err)
				r.cancel()
				return
			}
		}
	}
}

// for insert and delete
func (r *River) makeRequest(rule *Rule, action string, rows [][]interface{}) ([]*mongodb.BulkRequest, error) {
	reqs := make([]*mongodb.BulkRequest, 0, len(rows))

	for _, values := range rows {
		id, err := r.getDocID(rule, values)
		if err != nil {
			return nil, errors.Trace(err)
		}

		req := &mongodb.BulkRequest{Database: rule.Database, Collection: rule.Collection, ID: id}

		if action == canal.DeleteAction {
			req.Action = mongodb.ActionDelete
			r.st.DeleteNum.Add(1)
		} else {
			r.makeInsertReqData(req, rule, values)
			r.st.InsertNum.Add(1)
		}

		reqs = append(reqs, req)
	}

	return reqs, nil
}

func (r *River) makeInsertRequest(rule *Rule, rows [][]interface{}) ([]*mongodb.BulkRequest, error) {
	return r.makeRequest(rule, canal.InsertAction, rows)
}

func (r *River) makeDeleteRequest(rule *Rule, rows [][]interface{}) ([]*mongodb.BulkRequest, error) {
	return r.makeRequest(rule, canal.DeleteAction, rows)
}

func (r *River) makeUpdateRequest(rule *Rule, rows [][]interface{}) ([]*mongodb.BulkRequest, error) {
	if len(rows)%2 != 0 {
		return nil, errors.Errorf("invalid update rows event, must have 2x rows, but %d", len(rows))
	}

	reqs := make([]*mongodb.BulkRequest, 0, len(rows))

	for i := 0; i < len(rows); i += 2 {
		beforeID, err := r.getDocID(rule, rows[i])
		if err != nil {
			return nil, errors.Trace(err)
		}

		afterID, err := r.getDocID(rule, rows[i+1])

		if err != nil {
			return nil, errors.Trace(err)
		}

		req := &mongodb.BulkRequest{Database: rule.Database, Collection: rule.Collection, ID: beforeID}

		if beforeID != afterID {
			req.Action = mongodb.ActionDelete
			reqs = append(reqs, req)

			req = &mongodb.BulkRequest{Database: rule.Database, Collection: rule.Collection, ID: afterID}
			r.makeInsertReqData(req, rule, rows[i+1])

			r.st.DeleteNum.Add(1)
			r.st.InsertNum.Add(1)
		} else {
			r.makeUpdateReqData(req, rule, rows[i], rows[i+1])
			r.st.UpdateNum.Add(1)
		}

		reqs = append(reqs, req)
	}

	return reqs, nil
}

func (r *River) makeReqColumnData(col *schema.TableColumn, value interface{}) interface{} {
	switch col.Type {
	case schema.TYPE_ENUM:
		switch value := value.(type) {
		case int64:
			// for binlog, ENUM may be int64, but for dump, enum is string
			eNum := value - 1
			if eNum < 0 || eNum >= int64(len(col.EnumValues)) {
				// we insert invalid enum value before, so return empty
				log.Warnf("invalid binlog enum index %d, for enum %v", eNum, col.EnumValues)
				return ""
			}

			return col.EnumValues[eNum]
		}
	case schema.TYPE_SET:
		switch value := value.(type) {
		case int64:
			// for binlog, SET may be int64, but for dump, SET is string
			bitmask := value
			sets := make([]string, 0, len(col.SetValues))
			for i, s := range col.SetValues {
				if bitmask&int64(1<<uint(i)) > 0 {
					sets = append(sets, s)
				}
			}
			return strings.Join(sets, ",")
		}
	case schema.TYPE_BIT:
		switch value := value.(type) {
		case string:
			// for binlog, BIT is int64, but for dump, BIT is string
			// for dump 0x01 is for 1, \0 is for 0
			if value == "\x01" {
				return int64(1)
			}

			return int64(0)
		}
	case schema.TYPE_STRING:
		switch value := value.(type) {
		case []byte:
			return string(value[:])
		}
	case schema.TYPE_JSON:
			var f interface{}
			var err error
			switch v := value.(type) {
				case string:
					err = json.Unmarshal([]byte(v), &f)
				case []byte:
					err = json.Unmarshal(v, &f)
			}
			if err == nil && f != nil {
					return f
			}
	}

	return value
}

func (r *River) getFieldParts(k string, v string) (string, string, string) {
	composedField := strings.Split(v, ",")

	mysql := k
	mongodb := composedField[0]
	fieldType := ""

	if 0 == len(mongodb) {
		mongodb = mysql
	}
	if 2 == len(composedField) {
		fieldType = composedField[1]
	}

	return mysql, mongodb, fieldType
}

func (r *River) makeInsertReqData(req *mongodb.BulkRequest, rule *Rule, values []interface{}) {
	req.Data = make(map[string]interface{}, len(values))
	req.Action = mongodb.ActionInsert

	for i, c := range rule.TableInfo.Columns {
		if !rule.CheckFilter(c.Name) {
			continue
		}
		mapped := false
		for k, v := range rule.FieldMapping {
			mysql, mongodb, fieldType := r.getFieldParts(k, v)
			if mysql == c.Name {
				mapped = true
				v := r.makeReqColumnData(&c, values[i])
				if fieldType == fieldTypeList {
					if str, ok := v.(string); ok {
						req.Data[mongodb] = strings.Split(str, ",")
					} else {
						req.Data[mongodb] = v
					}
				} else {
					req.Data[mongodb] = v
				}
			}
		}
		if mapped == false {
			req.Data[c.Name] = r.makeReqColumnData(&c, values[i])
		}
	}
}

func (r *River) makeUpdateReqData(req *mongodb.BulkRequest, rule *Rule,
	beforeValues []interface{}, afterValues []interface{}) {
	req.Data = make(map[string]interface{}, len(beforeValues))

	// maybe dangerous if something wrong delete before?
	req.Action = mongodb.ActionUpdate

	for i, c := range rule.TableInfo.Columns {
		mapped := false
		if !rule.CheckFilter(c.Name) {
			continue
		}
		if reflect.DeepEqual(beforeValues[i], afterValues[i]) {
			//nothing changed
			continue
		}
		for k, v := range rule.FieldMapping {
			mysql, mongodb, fieldType := r.getFieldParts(k, v)
			if mysql == c.Name {
				mapped = true
				// has custom field mapping
				v := r.makeReqColumnData(&c, afterValues[i])
				str, ok := v.(string)
				if ok == false {
					req.Data[c.Name] = v
				} else {
					if fieldType == fieldTypeList {
						req.Data[mongodb] = strings.Split(str, ",")
					} else {
						req.Data[mongodb] = str
					}
				}
			}
		}
		if mapped == false {
			req.Data[c.Name] = r.makeReqColumnData(&c, afterValues[i])
		}

	}
}

// If id in toml file is none, get primary keys in one row and format them into a string, and PK must not be nil
// Else get the ID's column in one row and format them into a string
func (r *River) getDocID(rule *Rule, row []interface{}) (string, error) {
	var (
        flag bool
        ids []interface{}
	)
	if rule.ID == nil {
        ids = make([]interface{}, 0, len(rule.TableInfo.PKColumns))

        if len(rule.TableInfo.PKColumns) == 0 {
            flag = true
        }
        for _, num := range rule.TableInfo.PKColumns {
            ids = append(ids, r.makeReqColumnData(&(rule.TableInfo.Columns[num]), row[num]))
        }
	} else {
		ids = make([]interface{}, 0, len(rule.ID))
		for _, column := range rule.ID {
            index := rule.TableInfo.FindColumn(column)
            ids = append(ids, r.makeReqColumnData(&(rule.TableInfo.Columns[index]), row[index]))
		}
	}
    if flag {
        ids = make([]interface{}, 0, len(rule.TableInfo.Columns))
        for i, column := range rule.TableInfo.Columns {
            ids = append(ids, r.makeReqColumnData(&column, row[i]))
        }
    }

	var buf bytes.Buffer

	sep := ""
	for i, value := range ids {
		if value == nil {
            value = "<nil>"
            if !flag {
                log.Warnf("Position: %d id or PK value is nil, row: %s", i, row)
            }
		}

		buf.WriteString(fmt.Sprintf("%s%v", sep, value))
		sep = ":"
	}

    if flag {
		r.md5Ctx.Write(buf.Bytes())
		cipherStr := r.md5Ctx.Sum(nil)
        r.md5Ctx.Reset()
        return hex.EncodeToString(cipherStr), nil
	}
	return buf.String(), nil
}

func (r *River) doBulk(reqs []*mongodb.BulkRequest) error {
	if len(reqs) == 0 {
		return nil
	}

	if err := r.mongo.Bulk(reqs); err != nil {
		log.Errorf("sync docs err %v after binlog %s", err, r.canal.SyncedPosition())
		return errors.Trace(err)
	}

	return nil
}
<commit_msg>Update sync.go<commit_after>package river

import (
	"bytes"
	"fmt"
	"reflect"
	"strings"
	"time"
	"encoding/json"
	"encoding/hex"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/WangXiangUSTC/go-mysql-mongodb/mongodb"
	"github.com/siddontang/go-mysql/canal"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
	"github.com/siddontang/go-mysql/schema"
)

const (
	syncInsertDoc = iota
	syncDeleteDoc
	syncUpdateDoc
)

const (
	fieldTypeList = "list"
)

type posSaver struct {
	pos   mysql.Position
	force bool
}

type eventHandler struct {
	r *River
}

func (h *eventHandler) OnRotate(e *replication.RotateEvent) error {
	pos := mysql.Position{
		string(e.NextLogName),
		uint32(e.Position),
	}

	h.r.syncCh <- posSaver{pos, true}

	return h.r.ctx.Err()
}

func (h *eventHandler) OnDDL(nextPos mysql.Position, _ *replication.QueryEvent) error {
	h.r.syncCh <- posSaver{nextPos, true}
	return h.r.ctx.Err()
}

func (h *eventHandler) OnXID(nextPos mysql.Position) error {
	h.r.syncCh <- posSaver{nextPos, false}
	return h.r.ctx.Err()
}

func (h *eventHandler) OnRow(e *canal.RowsEvent) error {
    var err error
	rule, ok := h.r.rules[ruleKey(e.Table.Schema, e.Table.Name)]
	if !ok {
        if h.r.c.AllDB == "yes" {
            rule = newDefaultRule(e.Table.Schema, e.Table.Name)
            rule.TableInfo, err = h.r.canal.GetTable(e.Table.Schema, e.Table.Name)
            if err != nil {
                return nil
            }
            h.r.rules[ruleKey(e.Table.Schema, e.Table.Name)] = rule
        } else {
            return nil
        }
	}

	var reqs []*mongodb.BulkRequest
	switch e.Action {
	case canal.InsertAction:
		reqs, err = h.r.makeInsertRequest(rule, e.Rows)
	case canal.DeleteAction:
		reqs, err = h.r.makeDeleteRequest(rule, e.Rows)
	case canal.UpdateAction:
		reqs, err = h.r.makeUpdateRequest(rule, e.Rows)
	default:
		err = errors.Errorf("invalid rows action %s", e.Action)
	}

	if err != nil {
		h.r.cancel()
        log.Warnf("make %s MongoDB request err %v, close sync", e.Action, err)
		return errors.Errorf("make %s MongoDB request err %v, close sync", e.Action, err)
	}

	h.r.syncCh <- reqs

	return h.r.ctx.Err()
}

func (h *eventHandler) String() string {
	return "MongoRiverEventHandler"
}

func (r *River) syncLoop() {
	bulkSize := r.c.BulkSize
	if bulkSize == 0 {
		bulkSize = 512
	}

	interval := r.c.FlushBulkTime.Duration
	if interval == 0 {
		interval = 500 * time.Millisecond
	}

	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	defer r.wg.Done()

	lastSavedTime := time.Now()
	reqs := make([]*mongodb.BulkRequest, 0, 2048)

	var pos mysql.Position

	for {
		needFlush := false
		needSavePos := false

		select {
		case v := <-r.syncCh:
			switch v := v.(type) {
			case posSaver:
				now := time.Now()
				if v.force || now.Sub(lastSavedTime) > 3*time.Second {
					lastSavedTime = now
					needFlush = true
					needSavePos = true
					pos = v.pos
				}
			case []*mongodb.BulkRequest:
				reqs = append(reqs, v...)
				needFlush = len(reqs) >= bulkSize
			}
		case <-ticker.C:
			needFlush = true
		case <-r.ctx.Done():
			return
		}

		if needFlush {
			// TODO: retry some times?
			if err := r.doBulk(reqs); err != nil {
				log.Errorf("do MongoDB bulk err %v, close sync", err)
				r.cancel()
				return
			}
			reqs = reqs[0:0]
		}

		if needSavePos {
			if err := r.master.Save(pos); err != nil {
				log.Errorf("save sync position %s err %v, close sync", pos, err)
				r.cancel()
				return
			}
		}
	}
}

// for insert and delete
func (r *River) makeRequest(rule *Rule, action string, rows [][]interface{}) ([]*mongodb.BulkRequest, error) {
	reqs := make([]*mongodb.BulkRequest, 0, len(rows))

	for _, values := range rows {
		id, err := r.getDocID(rule, values)
		if err != nil {
			return nil, errors.Trace(err)
		}

		req := &mongodb.BulkRequest{Database: rule.Database, Collection: rule.Collection, ID: id}

		if action == canal.DeleteAction {
			req.Action = mongodb.ActionDelete
			r.st.DeleteNum.Add(1)
		} else {
			r.makeInsertReqData(req, rule, values)
			r.st.InsertNum.Add(1)
		}

		reqs = append(reqs, req)
	}

	return reqs, nil
}

func (r *River) makeInsertRequest(rule *Rule, rows [][]interface{}) ([]*mongodb.BulkRequest, error) {
	return r.makeRequest(rule, canal.InsertAction, rows)
}

func (r *River) makeDeleteRequest(rule *Rule, rows [][]interface{}) ([]*mongodb.BulkRequest, error) {
	return r.makeRequest(rule, canal.DeleteAction, rows)
}

func (r *River) makeUpdateRequest(rule *Rule, rows [][]interface{}) ([]*mongodb.BulkRequest, error) {
	if len(rows)%2 != 0 {
		return nil, errors.Errorf("invalid update rows event, must have 2x rows, but %d", len(rows))
	}

	reqs := make([]*mongodb.BulkRequest, 0, len(rows))

	for i := 0; i < len(rows); i += 2 {
		beforeID, err := r.getDocID(rule, rows[i])
		if err != nil {
			return nil, errors.Trace(err)
		}

		afterID, err := r.getDocID(rule, rows[i+1])

		if err != nil {
			return nil, errors.Trace(err)
		}

		req := &mongodb.BulkRequest{Database: rule.Database, Collection: rule.Collection, ID: beforeID}

		if beforeID != afterID {
			req.Action = mongodb.ActionDelete
			reqs = append(reqs, req)

			req = &mongodb.BulkRequest{Database: rule.Database, Collection: rule.Collection, ID: afterID}
			r.makeInsertReqData(req, rule, rows[i+1])

			r.st.DeleteNum.Add(1)
			r.st.InsertNum.Add(1)
		} else {
			r.makeUpdateReqData(req, rule, rows[i], rows[i+1])
			r.st.UpdateNum.Add(1)
		}

		reqs = append(reqs, req)
	}

	return reqs, nil
}

func (r *River) makeReqColumnData(col *schema.TableColumn, value interface{}) interface{} {
	switch col.Type {
	case schema.TYPE_ENUM:
		switch value := value.(type) {
		case int64:
			// for binlog, ENUM may be int64, but for dump, enum is string
			eNum := value - 1
			if eNum < 0 || eNum >= int64(len(col.EnumValues)) {
				// we insert invalid enum value before, so return empty
				log.Warnf("invalid binlog enum index %d, for enum %v", eNum, col.EnumValues)
				return ""
			}

			return col.EnumValues[eNum]
		}
	case schema.TYPE_SET:
		switch value := value.(type) {
		case int64:
			// for binlog, SET may be int64, but for dump, SET is string
			bitmask := value
			sets := make([]string, 0, len(col.SetValues))
			for i, s := range col.SetValues {
				if bitmask&int64(1<<uint(i)) > 0 {
					sets = append(sets, s)
				}
			}
			return strings.Join(sets, ",")
		}
	case schema.TYPE_BIT:
		switch value := value.(type) {
		case string:
			// for binlog, BIT is int64, but for dump, BIT is string
			// for dump 0x01 is for 1, \0 is for 0
			if value == "\x01" {
				return int64(1)
			}

			return int64(0)
		}
	case schema.TYPE_STRING:
		switch value := value.(type) {
		case []byte:
			return string(value[:])
		}
	case schema.TYPE_JSON:
			var f interface{}
			var err error
			switch v := value.(type) {
				case string:
					err = json.Unmarshal([]byte(v), &f)
				case []byte:
					err = json.Unmarshal(v, &f)
			}
			if err == nil && f != nil {
					return f
			}
	}

	return value
}

func (r *River) getFieldParts(k string, v string) (string, string, string) {
	composedField := strings.Split(v, ",")

	mysql := k
	mongodb := composedField[0]
	fieldType := ""

	if 0 == len(mongodb) {
		mongodb = mysql
	}
	if 2 == len(composedField) {
		fieldType = composedField[1]
	}

	return mysql, mongodb, fieldType
}

func (r *River) makeInsertReqData(req *mongodb.BulkRequest, rule *Rule, values []interface{}) {
	req.Data = make(map[string]interface{}, len(values))
	req.Action = mongodb.ActionInsert

	for i, c := range rule.TableInfo.Columns {
		if !rule.CheckFilter(c.Name) {
			continue
		}
		mapped := false
		for k, v := range rule.FieldMapping {
			mysql, mongodb, fieldType := r.getFieldParts(k, v)
			if mysql == c.Name {
				mapped = true
				v := r.makeReqColumnData(&c, values[i])
				if fieldType == fieldTypeList {
					if str, ok := v.(string); ok {
						req.Data[mongodb] = strings.Split(str, ",")
					} else {
						req.Data[mongodb] = v
					}
				} else {
					req.Data[mongodb] = v
				}
			}
		}
		if mapped == false {
			req.Data[c.Name] = r.makeReqColumnData(&c, values[i])
		}
	}
}

func (r *River) makeUpdateReqData(req *mongodb.BulkRequest, rule *Rule,
	beforeValues []interface{}, afterValues []interface{}) {
	req.Data = make(map[string]interface{}, len(beforeValues))

	// maybe dangerous if something wrong delete before?
	req.Action = mongodb.ActionUpdate

	for i, c := range rule.TableInfo.Columns {
		mapped := false
		if !rule.CheckFilter(c.Name) {
			continue
		}
		if reflect.DeepEqual(beforeValues[i], afterValues[i]) {
			//nothing changed
			continue
		}
		for k, v := range rule.FieldMapping {
			mysql, mongodb, fieldType := r.getFieldParts(k, v)
			if mysql == c.Name {
				mapped = true
				// has custom field mapping
				v := r.makeReqColumnData(&c, afterValues[i])
				str, ok := v.(string)
				if ok == false {
					req.Data[c.Name] = v
				} else {
					if fieldType == fieldTypeList {
						req.Data[mongodb] = strings.Split(str, ",")
					} else {
						req.Data[mongodb] = str
					}
				}
			}
		}
		if mapped == false {
			req.Data[c.Name] = r.makeReqColumnData(&c, afterValues[i])
		}

	}
}

// If id in toml file is none, get primary keys in one row and format them into a string, and PK must not be nil
// Else get the ID's column in one row and format them into a string
func (r *River) getDocID(rule *Rule, row []interface{}) (string, error) {
	var (
        flag bool
        ids []interface{}
	)
	if rule.ID == nil {
        ids = make([]interface{}, 0, len(rule.TableInfo.PKColumns))

        if len(rule.TableInfo.PKColumns) == 0 {
            flag = true
        }
        for _, num := range rule.TableInfo.PKColumns {
            ids = append(ids, r.makeReqColumnData(&(rule.TableInfo.Columns[num]), row[num]))
        }
	} else {
		ids = make([]interface{}, 0, len(rule.ID))
		for _, column := range rule.ID {
            index := rule.TableInfo.FindColumn(column)
            ids = append(ids, r.makeReqColumnData(&(rule.TableInfo.Columns[index]), row[index]))
		}
	}
    if flag {
        ids = make([]interface{}, 0, len(rule.TableInfo.Columns))
        for i, column := range rule.TableInfo.Columns {
            ids = append(ids, r.makeReqColumnData(&column, row[i]))
        }
    }

	var buf bytes.Buffer

	sep := ""
	for i, value := range ids {
		if value == nil {
            value = "<nil>"
            if !flag {
                log.Warnf("Position: %d id or PK value is nil, row: %s", i, row)
            }
		}

		buf.WriteString(fmt.Sprintf("%s%v", sep, value))
		sep = ":"
	}

    	if flag {
		r.md5Ctx.Write(buf.Bytes())
		cipherStr := r.md5Ctx.Sum(nil)
        	r.md5Ctx.Reset()
        	return hex.EncodeToString(cipherStr), nil
	}
	return buf.String(), nil
}

func (r *River) doBulk(reqs []*mongodb.BulkRequest) error {
	if len(reqs) == 0 {
		return nil
	}

	if err := r.mongo.Bulk(reqs); err != nil {
		log.Errorf("sync docs err %v after binlog %s", err, r.canal.SyncedPosition())
		return errors.Trace(err)
	}

	return nil
}
<|endoftext|>


<commit_before>package immortal

import (
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"testing"
	"time"
)

func TestHelperProcessSignals(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	select {
	case <-c:
		os.Exit(0)
	case <-time.After(30 * time.Second):
		os.Exit(1)
	}
}

func TestSignals(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	base := filepath.Base(os.Args[0]) // "exec.test"
	dir := filepath.Dir(os.Args[0])   // "/tmp/go-buildNNNN/os/exec/_test"
	if dir == "." {
		t.Skip("skipping; running test at root somehow")
	}
	parentDir := filepath.Dir(dir) // "/tmp/go-buildNNNN/os/exec"
	dirBase := filepath.Base(dir)  // "_test"
	if dirBase == "." {
		t.Skipf("skipping; unexpected shallow dir of %q", dir)
	}
	cfg := &Config{
		Env:     map[string]string{"GO_WANT_HELPER_PROCESS": "1"},
		command: []string{filepath.Join(dirBase, base), "-test.run=TestHelperProcessSignals"},
		Cwd:     parentDir,
		Pid: Pid{
			Parent: filepath.Join(parentDir, "parent.pid"),
			Child:  filepath.Join(parentDir, "child.pid"),
		},
	}
	c := make(chan os.Signal)
	wait := make(chan struct{})
	d := &Daemon{
		Config: cfg,
		Control: &Control{
			fifo:  make(chan Return),
			quit:  make(chan struct{}),
			state: make(chan error),
		},
		Forker: &myFork{},
		Logger: &LogWriter{
			logger: NewLogger(cfg),
		},
		process: &catchSignals{&os.Process{}, c, wait},
	}
	d.Run()
	sup := new(Sup)
	go Supervise(sup, d)

	// wait for process to startup
	select {
	case <-wait:
	case <-time.After(1 * time.Second):
		t.Fatal("timeout waiting for pid")
	}
	var testSignals = []struct {
		signal   string
		expected os.Signal
	}{
		{"p", syscall.SIGSTOP},
		{"pause", syscall.SIGSTOP},
		{"s", syscall.SIGSTOP},
		{"stop", syscall.SIGSTOP},
		{"c", syscall.SIGCONT},
		{"cont", syscall.SIGCONT},
		{"h", syscall.SIGHUP},
		{"hup", syscall.SIGHUP},
		{"a", syscall.SIGALRM},
		{"alrm", syscall.SIGALRM},
		{"i", syscall.SIGINT},
		{"int", syscall.SIGINT},
		{"q", syscall.SIGQUIT},
		{"quit", syscall.SIGQUIT},
		{"1", syscall.SIGUSR1},
		{"usr1", syscall.SIGUSR1},
		{"2", syscall.SIGUSR2},
		{"2", syscall.SIGUSR2},
		{"t", syscall.SIGTERM},
		{"term", syscall.SIGTERM},
		{"in", syscall.SIGTTIN},
		{"TTIN", syscall.SIGTTIN},
		{"ou", syscall.SIGTTOU},
		{"out", syscall.SIGTTOU},
		{"TTOU", syscall.SIGTTOU},
		{"w", syscall.SIGWINCH},
		{"winch", syscall.SIGWINCH},
	}
	for _, s := range testSignals {
		d.Control.fifo <- Return{err: nil, msg: s.signal}
		waitSig(t, c, s.expected)
	}

	// test kill process will restart
	old_pid := d.process.GetPid()
	d.Control.fifo <- Return{err: nil, msg: "k"}
	expect(t, d.count, uint32(1), "in 114")
	expect(t, d.count_defer, uint32(0), "in 115")

	// wait for process to came up and then send signal "once"
	d.Control.fifo <- Return{err: nil, msg: "o"}
	for d.count_defer != 1 {
	}
	expect(t, d.count, uint32(1), "in 121")
	expect(t, old_pid, d.process.GetPid())

	// kill for test once (should re-sestart)
	d.Control.fifo <- Return{err: nil, msg: "k"}
	// process shuld not start and pids remains the same
	expect(t, d.count, uint32(1), "in 127")
	expect(t, d.count_defer, uint32(1), "in 128")
	expect(t, old_pid, d.process.GetPid())

	for sup.IsRunning(d.process.GetPid()) {
	}

	var testSignalsError = []struct {
		signal   string
		expected os.Signal
	}{
		{"p", syscall.SIGILL},
		{"pause", syscall.SIGILL},
		{"s", syscall.SIGILL},
		{"stop", syscall.SIGILL},
		{"c", syscall.SIGILL},
		{"cont", syscall.SIGILL},
		{"h", syscall.SIGILL},
		{"hup", syscall.SIGILL},
		{"a", syscall.SIGILL},
		{"alrm", syscall.SIGILL},
		{"i", syscall.SIGILL},
		{"int", syscall.SIGILL},
		{"q", syscall.SIGILL},
		{"quit", syscall.SIGILL},
		{"1", syscall.SIGILL},
		{"usr1", syscall.SIGILL},
		{"2", syscall.SIGILL},
		{"2", syscall.SIGILL},
		{"t", syscall.SIGILL},
		{"term", syscall.SIGILL},
		{"in", syscall.SIGILL},
		{"TTIN", syscall.SIGILL},
		{"ou", syscall.SIGILL},
		{"out", syscall.SIGILL},
		{"TTOU", syscall.SIGILL},
		{"w", syscall.SIGILL},
		{"winch", syscall.SIGILL},
	}
	for _, s := range testSignalsError {
		d.Control.fifo <- Return{err: nil, msg: s.signal}
		waitSig(t, c, s.expected)
	}

	// test u
	// bring up the service (new pid expected)
	d.Control.fifo <- Return{err: nil, msg: "u"}
	select {
	case <-wait:
	case <-time.After(2 * time.Second):
		t.Fatal("timeout waiting for pid")
	}
	for old_pid == d.process.GetPid() {
	}

	// test down
	d.Control.fifo <- Return{err: nil, msg: "down"}
	for sup.IsRunning(d.process.GetPid()) {
		// waiting for process to exit
	}

	// test up
	// bring up the service (new pid expected)
	d.Control.fifo <- Return{err: nil, msg: "up"}
	for !sup.IsRunning(d.process.GetPid()) {
	}
	d.Control.fifo <- Return{err: nil, msg: "once"}
	for d.count_defer != 1 {
	}
	expect(t, d.count, uint32(1), "in 196")
	expect(t, d.count_defer, uint32(1), "in 197")

	// save old pid
	old_pid = d.process.GetPid()

	// send kill (should not start)
	d.Control.fifo <- Return{err: nil, msg: "k"}
	for sup.IsRunning(d.process.GetPid()) {
	}
	expect(t, old_pid, d.process.GetPid(), "in 206")
	expect(t, false, sup.IsRunning(d.process.GetPid()), "in 207")

	// test up
	// bring up the service (new pid expected)
	d.Control.fifo <- Return{err: nil, msg: "up"}
	for !sup.IsRunning(d.process.GetPid()) {
	}
	old_pid = d.process.GetPid()

	// send kill (should re-start, and get new pid)
	d.Control.fifo <- Return{err: nil, msg: "k"}
	for sup.IsRunning(d.process.GetPid()) {
	}
	select {
	case <-wait:
	case <-time.After(2 * time.Second):
		t.Fatal("timeout waiting for pid")
	}
	for old_pid == d.process.GetPid() {
	}

	// should be running
	expect(t, true, sup.IsRunning(d.process.GetPid()))

	// quit
	d.Control.fifo <- Return{err: nil, msg: "k"}
	d.Control.fifo <- Return{err: nil, msg: "exit"}
}

func waitSig(t *testing.T, c <-chan os.Signal, sig os.Signal) {
	select {
	case s := <-c:
		if s != sig {
			t.Fatalf("signal was %v, want %v", s, sig)
		}
	case <-time.After(1 * time.Second):
		t.Fatalf("timeout waiting for %v", sig)
	}
}
<commit_msg>	modified:   signals_test.go<commit_after>package immortal

import (
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"testing"
	"time"
)

func TestHelperProcessSignals(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	select {
	case <-c:
		os.Exit(0)
	case <-time.After(30 * time.Second):
		os.Exit(1)
	}
}

func TestSignals(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	base := filepath.Base(os.Args[0]) // "exec.test"
	dir := filepath.Dir(os.Args[0])   // "/tmp/go-buildNNNN/os/exec/_test"
	if dir == "." {
		t.Skip("skipping; running test at root somehow")
	}
	parentDir := filepath.Dir(dir) // "/tmp/go-buildNNNN/os/exec"
	dirBase := filepath.Base(dir)  // "_test"
	if dirBase == "." {
		t.Skipf("skipping; unexpected shallow dir of %q", dir)
	}
	cfg := &Config{
		Env:     map[string]string{"GO_WANT_HELPER_PROCESS": "1"},
		command: []string{filepath.Join(dirBase, base), "-test.run=TestHelperProcessSignals"},
		Cwd:     parentDir,
		Pid: Pid{
			Parent: filepath.Join(parentDir, "parent.pid"),
			Child:  filepath.Join(parentDir, "child.pid"),
		},
	}
	c := make(chan os.Signal)
	wait := make(chan struct{})
	d := &Daemon{
		Config: cfg,
		Control: &Control{
			fifo:  make(chan Return),
			quit:  make(chan struct{}),
			state: make(chan error),
		},
		Forker: &myFork{},
		Logger: &LogWriter{
			logger: NewLogger(cfg),
		},
		process: &catchSignals{&os.Process{}, c, wait},
	}
	d.Run()
	sup := new(Sup)
	go Supervise(sup, d)

	// wait for process to startup
	select {
	case <-wait:
	case <-time.After(1 * time.Second):
		t.Fatal("timeout waiting for pid")
	}
	var testSignals = []struct {
		signal   string
		expected os.Signal
	}{
		{"p", syscall.SIGSTOP},
		{"pause", syscall.SIGSTOP},
		{"s", syscall.SIGSTOP},
		{"stop", syscall.SIGSTOP},
		{"c", syscall.SIGCONT},
		{"cont", syscall.SIGCONT},
		{"h", syscall.SIGHUP},
		{"hup", syscall.SIGHUP},
		{"a", syscall.SIGALRM},
		{"alrm", syscall.SIGALRM},
		{"i", syscall.SIGINT},
		{"int", syscall.SIGINT},
		{"q", syscall.SIGQUIT},
		{"quit", syscall.SIGQUIT},
		{"1", syscall.SIGUSR1},
		{"usr1", syscall.SIGUSR1},
		{"2", syscall.SIGUSR2},
		{"2", syscall.SIGUSR2},
		{"t", syscall.SIGTERM},
		{"term", syscall.SIGTERM},
		{"in", syscall.SIGTTIN},
		{"TTIN", syscall.SIGTTIN},
		{"ou", syscall.SIGTTOU},
		{"out", syscall.SIGTTOU},
		{"TTOU", syscall.SIGTTOU},
		{"w", syscall.SIGWINCH},
		{"winch", syscall.SIGWINCH},
	}
	for _, s := range testSignals {
		d.Control.fifo <- Return{err: nil, msg: s.signal}
		waitSig(t, c, s.expected)
	}

	// test kill process will restart
	old_pid := d.process.GetPid()
	d.Control.fifo <- Return{err: nil, msg: "k"}
	expect(t, d.count, uint32(1), "in 114")
	expect(t, d.count_defer, uint32(0), "in 115")

	// wait for process to came up and then send signal "once"
	d.Control.fifo <- Return{err: nil, msg: "o"}
	for d.count_defer != 1 {
	}
	expect(t, d.count, uint32(1), "in 121")
	expect(t, old_pid, d.process.GetPid())

	// kill for test once (should re-sestart)
	d.Control.fifo <- Return{err: nil, msg: "k"}
	// process shuld not start and pids remains the same
	expect(t, d.count, uint32(1), "in 127")
	expect(t, d.count_defer, uint32(1), "in 128")
	expect(t, old_pid, d.process.GetPid())

	for sup.IsRunning(d.process.GetPid()) {
		// wait for process to die
	}

	var testSignalsError = []struct {
		signal   string
		expected os.Signal
	}{
		{"p", syscall.SIGILL},
		{"pause", syscall.SIGILL},
		{"s", syscall.SIGILL},
		{"stop", syscall.SIGILL},
		{"c", syscall.SIGILL},
		{"cont", syscall.SIGILL},
		{"h", syscall.SIGILL},
		{"hup", syscall.SIGILL},
		{"a", syscall.SIGILL},
		{"alrm", syscall.SIGILL},
		{"i", syscall.SIGILL},
		{"int", syscall.SIGILL},
		{"q", syscall.SIGILL},
		{"quit", syscall.SIGILL},
		{"1", syscall.SIGILL},
		{"usr1", syscall.SIGILL},
		{"2", syscall.SIGILL},
		{"2", syscall.SIGILL},
		{"t", syscall.SIGILL},
		{"term", syscall.SIGILL},
		{"in", syscall.SIGILL},
		{"TTIN", syscall.SIGILL},
		{"ou", syscall.SIGILL},
		{"out", syscall.SIGILL},
		{"TTOU", syscall.SIGILL},
		{"w", syscall.SIGILL},
		{"winch", syscall.SIGILL},
	}
	for _, s := range testSignalsError {
		d.Control.fifo <- Return{err: nil, msg: s.signal}
		waitSig(t, c, s.expected)
	}

	// test u
	// bring up the service (new pid expected)
	d.Control.fifo <- Return{err: nil, msg: "u"}
	select {
	case <-wait:
	case <-time.After(2 * time.Second):
		t.Fatal("timeout waiting for pid")
	}
	for old_pid == d.process.GetPid() {
	}

	// test down
	d.Control.fifo <- Return{err: nil, msg: "down"}
	for sup.IsRunning(d.process.GetPid()) {
		// waiting for process to exit
	}

	// test up
	// bring up the service (new pid expected)
	d.Control.fifo <- Return{err: nil, msg: "up"}
	for !sup.IsRunning(d.process.GetPid()) {
	}
	d.Control.fifo <- Return{err: nil, msg: "once"}
	for d.count_defer != 1 {
	}
	expect(t, d.count, uint32(1), "in 197")
	expect(t, d.count_defer, uint32(1), "in 198")

	// save old pid
	old_pid = d.process.GetPid()

	// send kill (should not start)
	d.Control.fifo <- Return{err: nil, msg: "k"}
	for sup.IsRunning(d.process.GetPid()) {
	}
	expect(t, old_pid, d.process.GetPid(), "in 207")
	expect(t, false, sup.IsRunning(d.process.GetPid()), "in 208")

	// test up
	// bring up the service (new pid expected)
	d.Control.fifo <- Return{err: nil, msg: "up"}
	for !sup.IsRunning(d.process.GetPid()) {
	}
	old_pid = d.process.GetPid()

	// send kill (should re-start, and get new pid)
	d.Control.fifo <- Return{err: nil, msg: "k"}
	for sup.IsRunning(d.process.GetPid()) {
	}
	select {
	case <-wait:
	case <-time.After(2 * time.Second):
		t.Fatal("timeout waiting for pid")
	}
	for old_pid == d.process.GetPid() {
	}

	// should be running
	expect(t, true, sup.IsRunning(d.process.GetPid()))

	// quit
	d.Control.fifo <- Return{err: nil, msg: "k"}
	d.Control.fifo <- Return{err: nil, msg: "exit"}
}

func waitSig(t *testing.T, c <-chan os.Signal, sig os.Signal) {
	select {
	case s := <-c:
		if s != sig {
			t.Fatalf("signal was %v, want %v", s, sig)
		}
	case <-time.After(1 * time.Second):
		t.Fatalf("timeout waiting for %v", sig)
	}
}
<|endoftext|>


<commit_before>package route

import (
	"log"
	"net/http"
	Pathp "path"
	"reflect"
	"strings"
)

func isSpecial(s string) bool {
	switch s {
	case "&", "":
		return true
	default:
		return false
	}
}

//A PathingRouter is part of the filepath, and can take part
//in the descent of the filepath trie.
type PathingRouter interface {
	Child(subpath string) (n Router, remainingSubpath string)
}

var (
	_ PathingRouter = new(Path)
	_ Router        = new(Path)
)

//NoExtPath is a PathRouter that ignores
//the extensions of folders and files when matching
//request URIs.
type NoExtPath Path

func (i NoExtPath) underlying() Path {
	return Path(i)
}

//Function Child is provided by all types implementing the PathingRouter
//interface.
func (i NoExtPath) Child(subpath string) (Router, string) {
	return i.underlying().ChildProcess(
		subpath,
		func(s string) string {
			ext := Pathp.Ext(s)
			return strings.TrimSuffix(s, ext)
		},
	)
}

func (i NoExtPath) RouteHTTP(r *http.Request) Router {
	return PathRouteHTTP(i, r)
}

//Type Path is a Router which routes by URL path. Files or directories
//with empty names are not allowed. The empty name routes to the terminal
//Router, the one used if the path stops here, and the ampersand ("&") path
//swallows the next file segment in the path, regardless of its contents.
//
//It should be noted that when RouteHTTP is called
//the PathRouter is followed to completion from the
//start to end of the URL, thus using two routers separately
//(i.e. separated by a different Router in the routing tree)
//causes two separate and independant operations on the path;
//a Path that routes a/b followed by one that routes a/b/c does
//not result in a route of a/b/a/c, instead resulting in a route of
//just a/b.
type Path map[string]Router

func (p Path) self() Path {
	if p == nil {
		p = make(Path)
	}
	return p
}

//Function AddChild adds a child PathRouter which is used
//as the next name in the path.
//
//A non PathRouter child will cause the Router to
//be returned to the caller of RouteHTTP, causing the
//path to terminate there if it is a Handler.
func (p Path) AddChild(pR Router, name string) Path {
	p[name] = pR
	return p
}

//Sets the Handler that is used when the path terminates here.
//This is the same as AddChild(r, ""); the empty string routes
//to here.
func (p Path) Handler(r Router) Path {
	return p.AddChild(r, "")
}

func isPathRouter(r Router) (b bool) {
	_, b = r.(Path)
	return
}

//Performs RouteHTTP on a trie of PathingRouters.
func PathRouteHTTP(p PathingRouter, rq *http.Request) Router {
	/*
		Now like SubdomainRouter!
	*/
	var (
		currentPathingRouter PathingRouter = p
		path                 string        = strings.Trim(rq.URL.Path, "/")
		currentRouter        Router
	)

	//fix paths
	path = Pathp.Clean(path)

	//We break spec a bit to allow direcories called "."
	if path == "." {
		path = ""
	}
	for {
		currentRouter, path = currentPathingRouter.Child(path)

		var ok bool
		if currentPathingRouter, ok = currentRouter.(PathingRouter); !ok {
			if debug {
				log.Printf("[-] Path routing terminated at %v [%v] and pathRouter %v [%v]", currentRouter, reflect.TypeOf(currentRouter), currentPathingRouter, reflect.TypeOf(currentPathingRouter))
			}
			break
		}

	}

	return currentRouter
}

//RouteHTTP traverses the tree of Paths until the end of the URL path
//is encountered, returning the terminal router or nil.
func (p Path) RouteHTTP(rq *http.Request) Router {
	return PathRouteHTTP(p, rq)
}

//Function Child is provided by all types that implement the PathingRouter interface.
func (p Path) Child(subpath string) (n Router, remainingSubpath string) {
	return p.ChildProcess(subpath, nil)
}

func swallowOne(subpath string) (subsectpath, swallowed string) {
	slashPos := strings.Index(subpath, "/")
	if slashPos == -1 {
		swallowed = subpath
		return
	}

	slashPos += 1
	if slashPos > len(subpath) {
		swallowed = subpath
		return
	}
	subsectpath, swallowed = subpath[slashPos:], subpath[:slashPos-1]
	return
}

//Function Child returns the next Router associated with the next
//'hop' in the path.
func (p Path) ChildProcess(subpath string, process func(string) string) (n Router, remainingSubpath string) {

	if process == nil {
		process = func(s string) string {
			return s
		}
	}

	if debug {
		log.Printf("[?] Currently in path %+v, with requested path %+q \n", p, subpath)
	}

	//strip leading slashes
	subpath = strings.TrimLeft(subpath, "/")

	//If strings.SplitN(subpath, "/", 3)'s length is two, then this
	//is the only item left in the path, and thus we must terminate here.
	if subpath == "" || subpath == "/" {
		if debug {
			log.Println("[?] Routing into current level (path is empty).")
		}
		return p[""], ""
	}

	remaining, popped := swallowOne(subpath)

	if debug {
		log.Printf("If we chop one off, we get %s, %s\n", remaining, popped)
	}

	if pathRouter, ok := p[process(popped)]; ok {
		return pathRouter, remaining
	} else if debug {
		log.Printf("%s was not present.\n", process(popped))
	}

	//Check if we have a route that begins with the subpath
	/*
		for path, pR := range p {
			if path != "" && strings.HasPrefix(processedSubpath, path) {
				if debug {
					log.Printf(
						"[?] Routed into %v - %+q is a prefix of %+q",
						pR,
						path,
						subpath,
					)
				}
				return pR, strings.TrimLeft(subpath, path)
			} else if debug {
				log.Printf(
					"[?] Did not reoute %+q is not a prefix of %+q",
					path,
					subpath,
				)
			}
		}
		//I actually have no idea what this does
	*/

	if p["&"] != nil {
		log.Printf("Ampersand present, swallowing one.")
		return p["&"], remaining
	} else if debug {
		log.Printf("[?] No ampersand present in Path, no swallow.")
	}

	//Not Found.
	return nil, subpath
}

//Returns s up until a char in terminators, or the whole string.
func TrimPast(s, terminators string) string {
	pos := strings.IndexAny(s, terminators)
	if pos == -1 {
		//to take the whole string
		pos = len(s)
	}

	return s[:pos]
}

//Function Ampersand is used to retrieve the URL part that was swallowed
//by an ampersand path. To achieve this, `prefix` is trimmed from the
//beginning of the url, and the resulting string is returned up
//until an instance of a char in `terminators` or the end of the string.
func Ampersand(url, prefix, terminators string) string {
	return TrimPast(strings.TrimLeft(url, prefix), terminators)
}

//Function PartN returns the Nth part of a URL, separated by '/'.
//A leading slash ('/usr') is ignored.
//If there are too few parts, empty string is returned.
func PartN(url string, n int) string {
	if url[0] == '/' {
		if len(url) < 2 {
			return ""
		}
		url = url[1:]
	}
	for i, ch := range url {
		if ch == '/' {
			n--
		}

		if n == 0 {
			return TrimPast(
				strings.TrimLeft(url[i+1:], "/"),
				"/",
			)
		}
	}
	return "fail"
}
<commit_msg>Remove non-debug print statement<commit_after>package route

import (
	"log"
	"net/http"
	Pathp "path"
	"reflect"
	"strings"
)

func isSpecial(s string) bool {
	switch s {
	case "&", "":
		return true
	default:
		return false
	}
}

//A PathingRouter is part of the filepath, and can take part
//in the descent of the filepath trie.
type PathingRouter interface {
	Child(subpath string) (n Router, remainingSubpath string)
}

var (
	_ PathingRouter = new(Path)
	_ Router        = new(Path)
)

//NoExtPath is a PathRouter that ignores
//the extensions of folders and files when matching
//request URIs.
type NoExtPath Path

func (i NoExtPath) underlying() Path {
	return Path(i)
}

//Function Child is provided by all types implementing the PathingRouter
//interface.
func (i NoExtPath) Child(subpath string) (Router, string) {
	return i.underlying().ChildProcess(
		subpath,
		func(s string) string {
			ext := Pathp.Ext(s)
			return strings.TrimSuffix(s, ext)
		},
	)
}

func (i NoExtPath) RouteHTTP(r *http.Request) Router {
	return PathRouteHTTP(i, r)
}

//Type Path is a Router which routes by URL path. Files or directories
//with empty names are not allowed. The empty name routes to the terminal
//Router, the one used if the path stops here, and the ampersand ("&") path
//swallows the next file segment in the path, regardless of its contents.
//
//It should be noted that when RouteHTTP is called
//the PathRouter is followed to completion from the
//start to end of the URL, thus using two routers separately
//(i.e. separated by a different Router in the routing tree)
//causes two separate and independant operations on the path;
//a Path that routes a/b followed by one that routes a/b/c does
//not result in a route of a/b/a/c, instead resulting in a route of
//just a/b.
type Path map[string]Router

func (p Path) self() Path {
	if p == nil {
		p = make(Path)
	}
	return p
}

//Function AddChild adds a child PathRouter which is used
//as the next name in the path.
//
//A non PathRouter child will cause the Router to
//be returned to the caller of RouteHTTP, causing the
//path to terminate there if it is a Handler.
func (p Path) AddChild(pR Router, name string) Path {
	p[name] = pR
	return p
}

//Sets the Handler that is used when the path terminates here.
//This is the same as AddChild(r, ""); the empty string routes
//to here.
func (p Path) Handler(r Router) Path {
	return p.AddChild(r, "")
}

func isPathRouter(r Router) (b bool) {
	_, b = r.(Path)
	return
}

//Performs RouteHTTP on a trie of PathingRouters.
func PathRouteHTTP(p PathingRouter, rq *http.Request) Router {
	/*
		Now like SubdomainRouter!
	*/
	var (
		currentPathingRouter PathingRouter = p
		path                 string        = strings.Trim(rq.URL.Path, "/")
		currentRouter        Router
	)

	//fix paths
	path = Pathp.Clean(path)

	//We break spec a bit to allow direcories called "."
	if path == "." {
		path = ""
	}
	for {
		currentRouter, path = currentPathingRouter.Child(path)

		var ok bool
		if currentPathingRouter, ok = currentRouter.(PathingRouter); !ok {
			if debug {
				log.Printf("[-] Path routing terminated at %v [%v] and pathRouter %v [%v]", currentRouter, reflect.TypeOf(currentRouter), currentPathingRouter, reflect.TypeOf(currentPathingRouter))
			}
			break
		}

	}

	return currentRouter
}

//RouteHTTP traverses the tree of Paths until the end of the URL path
//is encountered, returning the terminal router or nil.
func (p Path) RouteHTTP(rq *http.Request) Router {
	return PathRouteHTTP(p, rq)
}

//Function Child is provided by all types that implement the PathingRouter interface.
func (p Path) Child(subpath string) (n Router, remainingSubpath string) {
	return p.ChildProcess(subpath, nil)
}

func swallowOne(subpath string) (subsectpath, swallowed string) {
	slashPos := strings.Index(subpath, "/")
	if slashPos == -1 {
		swallowed = subpath
		return
	}

	slashPos += 1
	if slashPos > len(subpath) {
		swallowed = subpath
		return
	}
	subsectpath, swallowed = subpath[slashPos:], subpath[:slashPos-1]
	return
}

//Function Child returns the next Router associated with the next
//'hop' in the path.
func (p Path) ChildProcess(subpath string, process func(string) string) (n Router, remainingSubpath string) {

	if process == nil {
		process = func(s string) string {
			return s
		}
	}

	if debug {
		log.Printf("[?] Currently in path %+v, with requested path %+q \n", p, subpath)
	}

	//strip leading slashes
	subpath = strings.TrimLeft(subpath, "/")

	//If strings.SplitN(subpath, "/", 3)'s length is two, then this
	//is the only item left in the path, and thus we must terminate here.
	if subpath == "" || subpath == "/" {
		if debug {
			log.Println("[?] Routing into current level (path is empty).")
		}
		return p[""], ""
	}

	remaining, popped := swallowOne(subpath)

	if debug {
		log.Printf("If we chop one off, we get %s, %s\n", remaining, popped)
	}

	if pathRouter, ok := p[process(popped)]; ok {
		return pathRouter, remaining
	} else if debug {
		log.Printf("%s was not present.\n", process(popped))
	}

	//Check if we have a route that begins with the subpath
	/*
		for path, pR := range p {
			if path != "" && strings.HasPrefix(processedSubpath, path) {
				if debug {
					log.Printf(
						"[?] Routed into %v - %+q is a prefix of %+q",
						pR,
						path,
						subpath,
					)
				}
				return pR, strings.TrimLeft(subpath, path)
			} else if debug {
				log.Printf(
					"[?] Did not reoute %+q is not a prefix of %+q",
					path,
					subpath,
				)
			}
		}
		//I actually have no idea what this does
	*/

	if p["&"] != nil {
		return p["&"], remaining
	} else if debug {
		log.Printf("[?] No ampersand present in Path, no swallow.")
	}

	//Not Found.
	return nil, subpath
}

//Returns s up until a char in terminators, or the whole string.
func TrimPast(s, terminators string) string {
	pos := strings.IndexAny(s, terminators)
	if pos == -1 {
		//to take the whole string
		pos = len(s)
	}

	return s[:pos]
}

//Function Ampersand is used to retrieve the URL part that was swallowed
//by an ampersand path. To achieve this, `prefix` is trimmed from the
//beginning of the url, and the resulting string is returned up
//until an instance of a char in `terminators` or the end of the string.
func Ampersand(url, prefix, terminators string) string {
	return TrimPast(strings.TrimLeft(url, prefix), terminators)
}

//Function PartN returns the Nth part of a URL, separated by '/'.
//A leading slash ('/usr') is ignored.
//If there are too few parts, empty string is returned.
func PartN(url string, n int) string {
	if url[0] == '/' {
		if len(url) < 2 {
			return ""
		}
		url = url[1:]
	}
	for i, ch := range url {
		if ch == '/' {
			n--
		}

		if n == 0 {
			return TrimPast(
				strings.TrimLeft(url[i+1:], "/"),
				"/",
			)
		}
	}
	return "fail"
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sharedmain

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/user"
	"path/filepath"
	"time"

	"go.opencensus.io/stats/view"
	"golang.org/x/sync/errgroup"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"go.uber.org/zap"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/configmap"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/metrics"
	"knative.dev/pkg/profiling"
	"knative.dev/pkg/signals"
	"knative.dev/pkg/system"
	"knative.dev/pkg/version"
	"knative.dev/pkg/webhook"
)

// GetConfig returns a rest.Config to be used for kubernetes client creation.
// It does so in the following order:
//   1. Use the passed kubeconfig/masterURL.
//   2. Fallback to the KUBECONFIG environment variable.
//   3. Fallback to in-cluster config.
//   4. Fallback to the ~/.kube/config.
func GetConfig(masterURL, kubeconfig string) (*rest.Config, error) {
	if kubeconfig == "" {
		kubeconfig = os.Getenv("KUBECONFIG")
	}
	// If we have an explicit indication of where the kubernetes config lives, read that.
	if kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
	}
	// If not, try the in-cluster config.
	if c, err := rest.InClusterConfig(); err == nil {
		return c, nil
	}
	// If no in-cluster config, try the default location in the user's home directory.
	if usr, err := user.Current(); err == nil {
		if c, err := clientcmd.BuildConfigFromFlags("", filepath.Join(usr.HomeDir, ".kube", "config")); err == nil {
			return c, nil
		}
	}

	return nil, fmt.Errorf("could not create a valid kubeconfig")
}

// GetLoggingConfig gets the logging config from either the file system if present
// or via reading a configMap from the API.
// The context is expected to be initialized with injection.
func GetLoggingConfig(ctx context.Context) (*logging.Config, error) {
	loggingConfigMap, err := kubeclient.Get(ctx).CoreV1().ConfigMaps(system.Namespace()).Get(logging.ConfigMapName(), metav1.GetOptions{})
	if err != nil {
		if apierrors.IsNotFound(err) {
			return logging.NewConfigFromMap(nil)
		} else {
			return nil, err
		}
	}

	return logging.NewConfigFromConfigMap(loggingConfigMap)
}

func Main(component string, ctors ...injection.ControllerConstructor) {
	// Set up signals so we handle the first shutdown signal gracefully.
	MainWithContext(signals.NewContext(), component, ctors...)
}

func MainWithContext(ctx context.Context, component string, ctors ...injection.ControllerConstructor) {
	var (
		masterURL  = flag.String("master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
		kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
	)
	flag.Parse()

	cfg, err := GetConfig(*masterURL, *kubeconfig)
	if err != nil {
		log.Fatal("Error building kubeconfig", err)
	}
	MainWithConfig(ctx, component, cfg, ctors...)
}

func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, ctors ...injection.ControllerConstructor) {
	log.Printf("Registering %d clients", len(injection.Default.GetClients()))
	log.Printf("Registering %d informer factories", len(injection.Default.GetInformerFactories()))
	log.Printf("Registering %d informers", len(injection.Default.GetInformers()))
	log.Printf("Registering %d controllers", len(ctors))

	// Report stats on Go memory usage every 30 seconds.
	msp := metrics.NewMemStatsAll()
	msp.Start(ctx, 30*time.Second)

	if err := view.Register(msp.DefaultViews()...); err != nil {
		log.Fatalf("Error exporting go memstats view: %v", err)
	}

	// Adjust our client's rate limits based on the number of controller's we are running.
	cfg.QPS = float32(len(ctors)) * rest.DefaultQPS
	cfg.Burst = len(ctors) * rest.DefaultBurst

	ctx, informers := injection.Default.SetupInformers(ctx, cfg)

	// Set up our logger.
	loggingConfig, err := GetLoggingConfig(ctx)
	if err != nil {
		log.Fatal("Error reading/parsing logging configuration:", err)
	}
	logger, atomicLevel := logging.NewLoggerFromConfig(loggingConfig, component)
	defer flush(logger)
	ctx = logging.WithLogger(ctx, logger)

	kc := kubeclient.Get(ctx)
	if err := version.CheckMinimumVersion(kc.Discovery()); err != nil {
		logger.Fatalw("Version check failed", zap.Error(err))
	}

	// TODO(mattmoor): This should itself take a context and be injection-based.
	cmw := configmap.NewInformedWatcher(kc, system.Namespace())

	// Based on the reconcilers we have linked, build up the set of controllers to run.
	controllers := make([]*controller.Impl, 0, len(ctors))
	webhooks := make([]webhook.AdmissionController, 0)
	for _, cf := range ctors {
		ctrl := cf(ctx, cmw)
		controllers = append(controllers, ctrl)

		// Build a list of any reconcilers that implement webhook.AdmissionController
		if ac, ok := ctrl.Reconciler.(webhook.AdmissionController); ok {
			webhooks = append(webhooks, ac)
		}
	}

	profilingHandler := profiling.NewHandler(logger, false)

	// Watch the logging config map and dynamically update logging levels.
	if _, err := kubeclient.Get(ctx).CoreV1().ConfigMaps(system.Namespace()).Get(logging.ConfigMapName(),
		metav1.GetOptions{}); err == nil {
		cmw.Watch(logging.ConfigMapName(), logging.UpdateLevelFromConfigMap(logger, atomicLevel, component))
	} else if !apierrors.IsNotFound(err) {
		logger.Fatalw("Error reading ConfigMap: "+logging.ConfigMapName(), zap.Error(err))
	}

	// Watch the observability config map
	if _, err := kubeclient.Get(ctx).CoreV1().ConfigMaps(system.Namespace()).Get(metrics.ConfigMapName(),
		metav1.GetOptions{}); err == nil {
		cmw.Watch(metrics.ConfigMapName(),
			metrics.UpdateExporterFromConfigMap(component, logger),
			profilingHandler.UpdateFromConfigMap)
	} else if !apierrors.IsNotFound(err) {
		logger.Fatalw("Error reading ConfigMap: "+metrics.ConfigMapName(), zap.Error(err))
	}

	if err := cmw.Start(ctx.Done()); err != nil {
		logger.Fatalw("failed to start configuration manager", zap.Error(err))
	}

	// Start all of the informers and wait for them to sync.
	logger.Info("Starting informers.")
	if err := controller.StartInformers(ctx.Done(), informers...); err != nil {
		logger.Fatalw("Failed to start informers", err)
	}

	// Start all of the controllers.
	logger.Info("Starting controllers...")
	go controller.StartAll(ctx.Done(), controllers...)

	profilingServer := profiling.NewServer(profilingHandler)

	eg, egCtx := errgroup.WithContext(ctx)
	eg.Go(profilingServer.ListenAndServe)

	// If we have one or more admission controllers, then start the webhook
	// and pass them in.
	if len(webhooks) > 0 {
		// Register webhook metrics
		webhook.RegisterMetrics()

		wh, err := webhook.New(ctx, webhooks)
		if err != nil {
			logger.Fatalw("Failed to create admission controller", zap.Error(err))
		}
		eg.Go(func() error {
			return wh.Run(ctx.Done())
		})
	}

	// This will block until either a signal arrives or one of the grouped functions
	// returns an error.
	<-egCtx.Done()

	profilingServer.Shutdown(context.Background())
	// Don't forward ErrServerClosed as that indicates we're already shutting down.
	if err := eg.Wait(); err != nil && err != http.ErrServerClosed {
		logger.Errorw("Error while running server", zap.Error(err))
	}
}

func flush(logger *zap.SugaredLogger) {
	logger.Sync()
	metrics.FlushExporter()
}
<commit_msg>Fix sharedmain log messages format (#974)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sharedmain

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/user"
	"path/filepath"
	"time"

	"go.opencensus.io/stats/view"
	"golang.org/x/sync/errgroup"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"go.uber.org/zap"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/configmap"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/metrics"
	"knative.dev/pkg/profiling"
	"knative.dev/pkg/signals"
	"knative.dev/pkg/system"
	"knative.dev/pkg/version"
	"knative.dev/pkg/webhook"
)

// GetConfig returns a rest.Config to be used for kubernetes client creation.
// It does so in the following order:
//   1. Use the passed kubeconfig/masterURL.
//   2. Fallback to the KUBECONFIG environment variable.
//   3. Fallback to in-cluster config.
//   4. Fallback to the ~/.kube/config.
func GetConfig(masterURL, kubeconfig string) (*rest.Config, error) {
	if kubeconfig == "" {
		kubeconfig = os.Getenv("KUBECONFIG")
	}
	// If we have an explicit indication of where the kubernetes config lives, read that.
	if kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
	}
	// If not, try the in-cluster config.
	if c, err := rest.InClusterConfig(); err == nil {
		return c, nil
	}
	// If no in-cluster config, try the default location in the user's home directory.
	if usr, err := user.Current(); err == nil {
		if c, err := clientcmd.BuildConfigFromFlags("", filepath.Join(usr.HomeDir, ".kube", "config")); err == nil {
			return c, nil
		}
	}

	return nil, fmt.Errorf("could not create a valid kubeconfig")
}

// GetLoggingConfig gets the logging config from either the file system if present
// or via reading a configMap from the API.
// The context is expected to be initialized with injection.
func GetLoggingConfig(ctx context.Context) (*logging.Config, error) {
	loggingConfigMap, err := kubeclient.Get(ctx).CoreV1().ConfigMaps(system.Namespace()).Get(logging.ConfigMapName(), metav1.GetOptions{})
	if err != nil {
		if apierrors.IsNotFound(err) {
			return logging.NewConfigFromMap(nil)
		} else {
			return nil, err
		}
	}

	return logging.NewConfigFromConfigMap(loggingConfigMap)
}

func Main(component string, ctors ...injection.ControllerConstructor) {
	// Set up signals so we handle the first shutdown signal gracefully.
	MainWithContext(signals.NewContext(), component, ctors...)
}

func MainWithContext(ctx context.Context, component string, ctors ...injection.ControllerConstructor) {
	var (
		masterURL  = flag.String("master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
		kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
	)
	flag.Parse()

	cfg, err := GetConfig(*masterURL, *kubeconfig)
	if err != nil {
		log.Fatalf("Error building kubeconfig: %v", err)
	}
	MainWithConfig(ctx, component, cfg, ctors...)
}

func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, ctors ...injection.ControllerConstructor) {
	log.Printf("Registering %d clients", len(injection.Default.GetClients()))
	log.Printf("Registering %d informer factories", len(injection.Default.GetInformerFactories()))
	log.Printf("Registering %d informers", len(injection.Default.GetInformers()))
	log.Printf("Registering %d controllers", len(ctors))

	// Report stats on Go memory usage every 30 seconds.
	msp := metrics.NewMemStatsAll()
	msp.Start(ctx, 30*time.Second)

	if err := view.Register(msp.DefaultViews()...); err != nil {
		log.Fatalf("Error exporting go memstats view: %v", err)
	}

	// Adjust our client's rate limits based on the number of controller's we are running.
	cfg.QPS = float32(len(ctors)) * rest.DefaultQPS
	cfg.Burst = len(ctors) * rest.DefaultBurst

	ctx, informers := injection.Default.SetupInformers(ctx, cfg)

	// Set up our logger.
	loggingConfig, err := GetLoggingConfig(ctx)
	if err != nil {
		log.Fatalf("Error reading/parsing logging configuration: %v", err)
	}
	logger, atomicLevel := logging.NewLoggerFromConfig(loggingConfig, component)
	defer flush(logger)
	ctx = logging.WithLogger(ctx, logger)

	kc := kubeclient.Get(ctx)
	if err := version.CheckMinimumVersion(kc.Discovery()); err != nil {
		logger.Fatalw("Version check failed", zap.Error(err))
	}

	// TODO(mattmoor): This should itself take a context and be injection-based.
	cmw := configmap.NewInformedWatcher(kc, system.Namespace())

	// Based on the reconcilers we have linked, build up the set of controllers to run.
	controllers := make([]*controller.Impl, 0, len(ctors))
	webhooks := make([]webhook.AdmissionController, 0)
	for _, cf := range ctors {
		ctrl := cf(ctx, cmw)
		controllers = append(controllers, ctrl)

		// Build a list of any reconcilers that implement webhook.AdmissionController
		if ac, ok := ctrl.Reconciler.(webhook.AdmissionController); ok {
			webhooks = append(webhooks, ac)
		}
	}

	profilingHandler := profiling.NewHandler(logger, false)

	// Watch the logging config map and dynamically update logging levels.
	if _, err := kubeclient.Get(ctx).CoreV1().ConfigMaps(system.Namespace()).Get(logging.ConfigMapName(),
		metav1.GetOptions{}); err == nil {
		cmw.Watch(logging.ConfigMapName(), logging.UpdateLevelFromConfigMap(logger, atomicLevel, component))
	} else if !apierrors.IsNotFound(err) {
		logger.With(zap.Error(err)).Fatalf("Error reading ConfigMap %q", logging.ConfigMapName())
	}

	// Watch the observability config map
	if _, err := kubeclient.Get(ctx).CoreV1().ConfigMaps(system.Namespace()).Get(metrics.ConfigMapName(),
		metav1.GetOptions{}); err == nil {
		cmw.Watch(metrics.ConfigMapName(),
			metrics.UpdateExporterFromConfigMap(component, logger),
			profilingHandler.UpdateFromConfigMap)
	} else if !apierrors.IsNotFound(err) {
		logger.With(zap.Error(err)).Fatalf("Error reading ConfigMap %q", metrics.ConfigMapName())
	}

	if err := cmw.Start(ctx.Done()); err != nil {
		logger.Fatalw("Failed to start configuration manager", zap.Error(err))
	}

	// Start all of the informers and wait for them to sync.
	logger.Info("Starting informers.")
	if err := controller.StartInformers(ctx.Done(), informers...); err != nil {
		logger.Fatalw("Failed to start informers", zap.Error(err))
	}

	// Start all of the controllers.
	logger.Info("Starting controllers...")
	go controller.StartAll(ctx.Done(), controllers...)

	profilingServer := profiling.NewServer(profilingHandler)

	eg, egCtx := errgroup.WithContext(ctx)
	eg.Go(profilingServer.ListenAndServe)

	// If we have one or more admission controllers, then start the webhook
	// and pass them in.
	if len(webhooks) > 0 {
		// Register webhook metrics
		webhook.RegisterMetrics()

		wh, err := webhook.New(ctx, webhooks)
		if err != nil {
			logger.Fatalw("Failed to create admission controller", zap.Error(err))
		}
		eg.Go(func() error {
			return wh.Run(ctx.Done())
		})
	}

	// This will block until either a signal arrives or one of the grouped functions
	// returns an error.
	<-egCtx.Done()

	profilingServer.Shutdown(context.Background())
	// Don't forward ErrServerClosed as that indicates we're already shutting down.
	if err := eg.Wait(); err != nil && err != http.ErrServerClosed {
		logger.Errorw("Error while running server", zap.Error(err))
	}
}

func flush(logger *zap.SugaredLogger) {
	logger.Sync()
	metrics.FlushExporter()
}
<|endoftext|>


<commit_before>package contact

import (
	"math"

	"github.com/hueypark/physics/core/body"
	"github.com/hueypark/physics/core/shape"
	"github.com/hueypark/physics/core/shape/circle"
	"github.com/hueypark/physics/core/shape/convex"
	"github.com/hueypark/physics/core/vector"
)

func (c *Contact) DetectCollision() {
	lhsType := c.lhs.Shape.Type()
	rhsType := c.rhs.Shape.Type()

	switch lhsType {
	case shape.BULLET:
		switch rhsType {
		case shape.BULLET:
			break
		case shape.CIRCLE:
			c.bulletToCircle(c.rhs.Shape.(*circle.Circle))
			break
		case shape.CONVEX:
			c.normal, c.penetration, c.points = bulletToConvex(c.lhs, c.rhs)
			break
		}
		break
	case shape.CIRCLE:
		switch rhsType {
		case shape.BULLET:
			c.circleToBullet(c.lhs.Shape.(*circle.Circle))
			break
		case shape.CIRCLE:
			c.circleToCircle(c.lhs.Shape.(*circle.Circle), c.rhs.Shape.(*circle.Circle))
			break
		case shape.CONVEX:
			c.circleToConvex(c.lhs.Shape.(*circle.Circle), c.rhs.Shape.(*convex.Convex))
			break
		}
		break
	case shape.CONVEX:
		switch rhsType {
		case shape.BULLET:
			c.normal, c.penetration, c.points = convexToBullet(c.lhs, c.rhs)
		case shape.CIRCLE:
			c.convexToCircle(c.lhs.Shape.(*convex.Convex), c.rhs.Shape.(*circle.Circle))
			break
		case shape.CONVEX:
			c.convexToConvex(c.lhs.Shape.(*convex.Convex), c.rhs.Shape.(*convex.Convex))
			break
		}
		break
	}
}

func (c *Contact) bulletToCircle(rhs *circle.Circle) {
	c.normal = vector.Subtract(c.rhs.Position(), c.lhs.Position())

	distanceSquared := c.normal.SizeSquared()

	if distanceSquared >= rhs.Radius*rhs.Radius {
		return
	}

	distance := math.Sqrt(distanceSquared)

	c.penetration = rhs.Radius - distance
	c.normal.Normalize()
	c.points = append(c.points, c.lhs.Position())
}

func (c *Contact) circleToBullet(lhs *circle.Circle) {
	c.normal = vector.Subtract(c.rhs.Position(), c.lhs.Position())

	distanceSquared := c.normal.SizeSquared()

	if distanceSquared >= lhs.Radius*lhs.Radius {
		return
	}

	distance := math.Sqrt(distanceSquared)

	c.penetration = lhs.Radius - distance
	c.normal.Normalize()
	c.points = append(c.points, c.lhs.Position())
}

func bulletToConvex(lhs, rhs *body.Body) (normal vector.Vector, penetration float64, points []vector.Vector) {
	rhsConvex := rhs.Shape.(*convex.Convex)

	penetration = math.MaxFloat64

	for _, edge := range rhsConvex.Edges() {
		worldStart := vector.Add(rhs.Position(), edge.Start)
		worldEnd := vector.Add(rhs.Position(), edge.End)
		edgeVector := vector.Subtract(worldEnd, worldStart)
		pointVector := vector.Subtract(lhs.Position(), worldStart)

		if !pointVector.OnTheRight(edgeVector) {
			normal = vector.Vector{}
			penetration = 0
			return normal, penetration, points
		}

		perpendicular := vector.Vector{-edgeVector.Y, edgeVector.X}
		perpendicular.Normalize()

		lhsVector := vector.Subtract(lhs.Position(), worldStart)

		proj := vector.Multiply(perpendicular, vector.Dot(lhsVector, perpendicular))

		if proj.Size() < penetration {
			normal = perpendicular
			penetration = proj.Size()
		}
	}

	points = append(points, lhs.Position())
	return normal, penetration, points
}

func convexToBullet(lhs, rhs *body.Body) (normal vector.Vector, penetration float64, points []vector.Vector) {
	normal, penetration, points = bulletToConvex(rhs, lhs)

	normal = vector.Vector{-normal.X, -normal.Y}

	return normal, penetration, points
}

func (c *Contact) circleToCircle(lhsCircle, rhsCircle *circle.Circle) {
	c.normal = vector.Subtract(c.rhs.Position(), c.lhs.Position())

	distanceSquared := c.normal.SizeSquared()
	radius := lhsCircle.Radius + rhsCircle.Radius

	if distanceSquared >= radius*radius {
		return
	}

	distance := math.Sqrt(distanceSquared)

	c.penetration = radius - distance
	c.normal.Normalize()
	c.points = append(c.points, vector.Add(vector.Multiply(c.normal, lhsCircle.Radius), c.lhs.Position()))
}

func (c *Contact) circleToConvex(lhs *circle.Circle, rhs *convex.Convex) {

}

func (c *Contact) convexToCircle(lhs *convex.Convex, rhs *circle.Circle) {

}

func (c *Contact) convexToConvex(lhs, rhs *convex.Convex) {

}
<commit_msg>In collision check, swap lhs, rhs to remove unnecessary duplicate implementations<commit_after>package contact

import (
	"math"

	"github.com/hueypark/physics/core/body"
	"github.com/hueypark/physics/core/shape"
	"github.com/hueypark/physics/core/shape/circle"
	"github.com/hueypark/physics/core/shape/convex"
	"github.com/hueypark/physics/core/vector"
)

func (c *Contact) DetectCollision() {
	lhsType := c.lhs.Shape.Type()
	rhsType := c.rhs.Shape.Type()

	switch lhsType {
	case shape.BULLET:
		switch rhsType {
		case shape.BULLET:
			break
		case shape.CIRCLE:
			c.bulletToCircle(c.rhs.Shape.(*circle.Circle))
			break
		case shape.CONVEX:
			c.normal, c.penetration, c.points = bulletToConvex(c.lhs, c.rhs)
			break
		}
		break
	case shape.CIRCLE:
		switch rhsType {
		case shape.BULLET:
			c.circleToBullet(c.lhs.Shape.(*circle.Circle))
			break
		case shape.CIRCLE:
			c.circleToCircle(c.lhs.Shape.(*circle.Circle), c.rhs.Shape.(*circle.Circle))
			break
		case shape.CONVEX:
			c.circleToConvex(c.lhs.Shape.(*circle.Circle), c.rhs.Shape.(*convex.Convex))
			break
		}
		break
	case shape.CONVEX:
		switch rhsType {
		case shape.BULLET:
			c.swap()
			c.normal, c.penetration, c.points = bulletToConvex(c.lhs, c.rhs)
		case shape.CIRCLE:
			c.convexToCircle(c.lhs.Shape.(*convex.Convex), c.rhs.Shape.(*circle.Circle))
			break
		case shape.CONVEX:
			c.convexToConvex(c.lhs.Shape.(*convex.Convex), c.rhs.Shape.(*convex.Convex))
			break
		}
		break
	}
}

func (c *Contact) swap() {
	c.lhs, c.rhs = c.rhs, c.lhs
}

func (c *Contact) bulletToCircle(rhs *circle.Circle) {
	c.normal = vector.Subtract(c.rhs.Position(), c.lhs.Position())

	distanceSquared := c.normal.SizeSquared()

	if distanceSquared >= rhs.Radius*rhs.Radius {
		return
	}

	distance := math.Sqrt(distanceSquared)

	c.penetration = rhs.Radius - distance
	c.normal.Normalize()
	c.points = append(c.points, c.lhs.Position())
}

func (c *Contact) circleToBullet(lhs *circle.Circle) {
	c.normal = vector.Subtract(c.rhs.Position(), c.lhs.Position())

	distanceSquared := c.normal.SizeSquared()

	if distanceSquared >= lhs.Radius*lhs.Radius {
		return
	}

	distance := math.Sqrt(distanceSquared)

	c.penetration = lhs.Radius - distance
	c.normal.Normalize()
	c.points = append(c.points, c.lhs.Position())
}

func bulletToConvex(lhs, rhs *body.Body) (normal vector.Vector, penetration float64, points []vector.Vector) {
	rhsConvex := rhs.Shape.(*convex.Convex)

	penetration = math.MaxFloat64

	for _, edge := range rhsConvex.Edges() {
		worldStart := vector.Add(rhs.Position(), edge.Start)
		worldEnd := vector.Add(rhs.Position(), edge.End)
		edgeVector := vector.Subtract(worldEnd, worldStart)
		pointVector := vector.Subtract(lhs.Position(), worldStart)

		if !pointVector.OnTheRight(edgeVector) {
			normal = vector.Vector{}
			penetration = 0
			return normal, penetration, points
		}

		perpendicular := vector.Vector{-edgeVector.Y, edgeVector.X}
		perpendicular.Normalize()

		lhsVector := vector.Subtract(lhs.Position(), worldStart)

		proj := vector.Multiply(perpendicular, vector.Dot(lhsVector, perpendicular))

		if proj.Size() < penetration {
			normal = perpendicular
			penetration = proj.Size()
		}
	}

	points = append(points, lhs.Position())
	return normal, penetration, points
}

func (c *Contact) circleToCircle(lhsCircle, rhsCircle *circle.Circle) {
	c.normal = vector.Subtract(c.rhs.Position(), c.lhs.Position())

	distanceSquared := c.normal.SizeSquared()
	radius := lhsCircle.Radius + rhsCircle.Radius

	if distanceSquared >= radius*radius {
		return
	}

	distance := math.Sqrt(distanceSquared)

	c.penetration = radius - distance
	c.normal.Normalize()
	c.points = append(c.points, vector.Add(vector.Multiply(c.normal, lhsCircle.Radius), c.lhs.Position()))
}

func (c *Contact) circleToConvex(lhs *circle.Circle, rhs *convex.Convex) {

}

func (c *Contact) convexToCircle(lhs *convex.Convex, rhs *circle.Circle) {

}

func (c *Contact) convexToConvex(lhs, rhs *convex.Convex) {

}
<|endoftext|>


<commit_before>package concourse_test

import (
	"net/http"

	"github.com/concourse/atc"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("ATC Handler Containers", func() {
	Describe("ListContainers", func() {
		Context("when passed an empty specification list", func() {
			var expectedContainers []atc.Container

			BeforeEach(func() {
				expectedURL := "/api/v1/containers"

				expectedContainers = []atc.Container{
					{
						ID:               "myid-1",
						PipelineName:     "mypipeline-1",
						WorkingDirectory: "/tmp/build/some-guid",
					},
					{
						ID:               "myid-2",
						PipelineName:     "mypipeline-2",
						WorkingDirectory: "/tmp/build/some-other-guid",
					},
				}

				atcServer.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", expectedURL),
						ghttp.RespondWithJSONEncoded(http.StatusOK, expectedContainers),
					),
				)
			})

			It("returns all the containers", func() {
				containers, err := client.ListContainers(map[string]string{})
				Expect(err).NotTo(HaveOccurred())
				Expect(containers).To(Equal(expectedContainers))
			})
		})

		Context("when passed a nonempty specification list", func() {
			var (
				expectedContainers []atc.Container
				expectedQueryList  map[string]string
			)

			BeforeEach(func() {
				expectedURL := "/api/v1/containers"
				expectedQueryList = map[string]string{
					"query1": "value1",
				}

				expectedContainers = []atc.Container{
					{
						ID:               "myid-1",
						PipelineName:     "mypipeline-1",
						WorkingDirectory: "/tmp/build/some-guid",
					},
				}

				atcServer.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", expectedURL, "query1=value1"),
						ghttp.RespondWithJSONEncoded(http.StatusOK, expectedContainers),
					),
				)
			})

			It("returns the specified containers", func() {
				containers, err := client.ListContainers(expectedQueryList)
				Expect(err).NotTo(HaveOccurred())
				Expect(containers).To(Equal(expectedContainers))
			})
		})
	})
})
<commit_msg>Add env vars to container tests<commit_after>package concourse_test

import (
	"net/http"

	"github.com/concourse/atc"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("ATC Handler Containers", func() {
	Describe("ListContainers", func() {
		Context("when passed an empty specification list", func() {
			var expectedContainers []atc.Container

			BeforeEach(func() {
				expectedURL := "/api/v1/containers"

				expectedContainers = []atc.Container{
					{
						ID:               "myid-1",
						PipelineName:     "mypipeline-1",
						WorkingDirectory: "/tmp/build/some-guid",
					},
					{
						ID:               "myid-2",
						PipelineName:     "mypipeline-2",
						WorkingDirectory: "/tmp/build/some-other-guid",
					},
				}

				atcServer.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", expectedURL),
						ghttp.RespondWithJSONEncoded(http.StatusOK, expectedContainers),
					),
				)
			})

			It("returns all the containers", func() {
				containers, err := client.ListContainers(map[string]string{})
				Expect(err).NotTo(HaveOccurred())
				Expect(containers).To(Equal(expectedContainers))
			})
		})

		Context("when passed a nonempty specification list", func() {
			var (
				expectedContainers []atc.Container
				expectedQueryList  map[string]string
			)

			BeforeEach(func() {
				expectedURL := "/api/v1/containers"
				expectedQueryList = map[string]string{
					"query1": "value1",
				}

				expectedContainers = []atc.Container{
					{
						ID:                   "myid-1",
						PipelineName:         "mypipeline-1",
						WorkingDirectory:     "/tmp/build/some-guid",
						EnvironmentVariables: []string{"VAR1=val1"},
					},
				}

				atcServer.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", expectedURL, "query1=value1"),
						ghttp.RespondWithJSONEncoded(http.StatusOK, expectedContainers),
					),
				)
			})

			It("returns the specified containers", func() {
				containers, err := client.ListContainers(expectedQueryList)
				Expect(err).NotTo(HaveOccurred())
				Expect(containers).To(Equal(expectedContainers))
			})
		})
	})
})
<|endoftext|>