<commit_before>// Copyright 2019 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package parser

import (
	"errors"
	"fmt"
	"strings"

	"github.com/issue9/orm/v2/sqlbuilder"
)

var (
	// mysql 和 sqlite3 使用相同的字符
	quoteReplacer = strings.NewReplacer("`", "")
)

// Table 表信息
type Table struct {
	Columns     map[string]string                // 列信息，名称=>类型
	Constraints map[string]sqlbuilder.Constraint // 约束信息，名称=>约束类型
	Indexes     map[string]sqlbuilder.Index      // 索引信息，名称=>索引类型
}

// ParseCreateTable 分析 create table 的语法
func ParseCreateTable(driverName, table string, engine sqlbuilder.Engine) (*Table, error) {
	switch driverName {
	case "mysql":
		return getMysqlTableInfo(table, engine)
	case "sqlite3":
		return getSqlite3TableInfo(table, engine)
	}
	panic("未实现的数据库：" + driverName)
}

func getMysqlTableInfo(table string, engine sqlbuilder.Engine) (*Table, error) {
	rows, err := engine.Query("SHOW CREATE TABLE `" + table + "`")
	if err != nil {
		return nil, err
	}
	defer func() {
		if err1 := rows.Close(); err1 != nil {
			err = errors.New(err1.Error() + err.Error())
		}
	}()

	var tableName, sql string
	if err = rows.Scan(&tableName, &sql); err != nil {
		return nil, err
	}

	return parseMysqlCreateTable(filterCreateTableSQL(sql))
}

func getSqlite3TableInfo(tableName string, engine sqlbuilder.Engine) (*Table, error) {
	table := &Table{
		Columns:     make(map[string]string, 10),
		Constraints: make(map[string]sqlbuilder.Constraint, 5),
		Indexes:     make(map[string]sqlbuilder.Index, 2),
	}

	if err := parseSqlite3CreateTable(table, tableName, engine); err != nil {
		return nil, err
	}

	if err := parseSqlite3Indexes(table, tableName, engine); err != nil {
		return nil, err
	}

	return table, nil
}

// https://dev.mysql.com/doc/refman/8.0/en/create-table.html
func parseMysqlCreateTable(lines []string) (*Table, error) {
	table := &Table{
		Columns:     make(map[string]string, len(lines)),
		Constraints: make(map[string]sqlbuilder.Constraint, len(lines)),
		Indexes:     make(map[string]sqlbuilder.Index, len(lines)),
	}

	for _, line := range lines {
		line = strings.TrimSpace(line)
		index := strings.IndexByte(line, ' ')
		if index <= 0 {
			continue
		}
		first := line[:index]
		line = line[index+1:]

		switch strings.ToUpper(first) {
		case "FULLTEXT", "SPATIAL": // FULLTEXT|SPATIAL INDEX
			index = strings.IndexByte(line, ' ')
			if index <= 0 {
				continue
			}
			first = line[:index]
			line = line[index+1:]
			fallthrough
		case "INDEX", "KEY": // 索引
			index = strings.IndexByte(line, ' ')
			if index <= 0 {
				continue
			}
			table.Indexes[line[:index]] = sqlbuilder.IndexDefault
		case "CONSTRAINT":
			index = strings.IndexByte(line, ' ')
			if index <= 0 {
				continue
			}
			first = line[:index]
			line = line[index+1:]
			fallthrough
		case "UNIQUE", "PRIMARY", "FOREIGN": // 约束
		// TODO
		default: // 普通列定义，第一个字符串即为列名
			table.Columns[line[:index]] = line[index+1:]
		}
	}

	return table, nil
}

// https://www.sqlite.org/draft/lang_createtable.html
func parseSqlite3CreateTable(table *Table, tableName string, engine sqlbuilder.Engine) error {
	rows, err := engine.Query("SELECT sql FROM sqlite_master WHERE type='table' and name='" + tableName + "'")
	if err != nil {
		return err
	}
	defer func() {
		if err1 := rows.Close(); err1 != nil {
			err = errors.New(err1.Error() + err.Error())
		}
	}()

	var sql string
	if err = rows.Scan(&sql); err != nil {
		return err
	}

	lines := filterCreateTableSQL(sql)

LOOP:
	for _, line := range lines {
		line = strings.TrimSpace(line)
		index := strings.IndexByte(line, ' ')
		if index <= 0 {
			continue
		}
		first := line[:index]

		switch strings.ToUpper(first) {
		case "CONSTRAINT": // 约束
			line = strings.TrimSpace(line[index+1:])
			index = strings.IndexByte(line, ' ')
			if index <= 0 {
				continue LOOP
			}
			name := line[:index]

			line = strings.TrimSpace(line[index+1:])
			index = strings.IndexByte(line, ' ')
			if index <= 0 {
				continue LOOP
			}
			switch line[:index] {
			case "PRIMARY":
				table.Constraints[name] = sqlbuilder.ConstraintPK
			case "UNIQUE":
				table.Constraints[name] = sqlbuilder.ConstraintUnique
			case "CHECK":
				table.Constraints[name] = sqlbuilder.ConstraintCheck
			case "FOREIGN":
				table.Constraints[name] = sqlbuilder.ConstraintFK
			default:
				return fmt.Errorf("未知的约束名：%s", line[:index])
			}
		default: // 普通列定义，第一个字符串即为列名
			table.Columns[line[:index]] = line[index+1:]
		}
	}

	return nil
}
func parseSqlite3Indexes(table *Table, tableName string, engine sqlbuilder.Engine) error {
	rows, err := engine.Query("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='" + tableName + "'")
	if err != nil {
		return err
	}
	defer func() {
		if err1 := rows.Close(); err1 != nil {
			err = errors.New(err1.Error() + err.Error())
		}
	}()

	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			return err
		}
		table.Indexes[name] = sqlbuilder.IndexDefault
	}

	return nil
}

func filterCreateTableSQL(sql string) []string {
	sql = quoteReplacer.Replace(sql)
	var deep, start int
	var lines []string

	for index, c := range sql {
		switch c {
		case ',':
			if deep == 1 && index > start {
				lines = append(lines, sql[start:index])
				start = index
			}
		case '(':
			deep++
			if deep == 1 {
				start = index
			}
		case ')':
			deep--
			if deep == 0 { // 不需要 create table xx() 之后的内容
				break
			}
		} // end switch
	} // end for

	return lines
}
<commit_msg>[internal/parser] 基本功能完成<commit_after>// Copyright 2019 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package parser

import (
	"errors"
	"fmt"
	"strings"

	"github.com/issue9/orm/v2/sqlbuilder"
)

var (
	// mysql 和 sqlite3 使用相同的字符
	quoteReplacer = strings.NewReplacer("`", "")
)

// Table 表信息
type Table struct {
	Columns     map[string]string                // 列信息，名称=>类型
	Constraints map[string]sqlbuilder.Constraint // 约束信息，名称=>约束类型
	Indexes     map[string]sqlbuilder.Index      // 索引信息，名称=>索引类型
}

// ParseCreateTable 分析 create table 的语法
func ParseCreateTable(driverName, table string, engine sqlbuilder.Engine) (*Table, error) {
	switch driverName {
	case "mysql":
		return getMysqlTableInfo(table, engine)
	case "sqlite3":
		return getSqlite3TableInfo(table, engine)
	}
	panic("未实现的数据库：" + driverName)
}

func getMysqlTableInfo(table string, engine sqlbuilder.Engine) (*Table, error) {
	// show index 语句无法获取 check 约束的相关信息
	rows, err := engine.Query("SHOW CREATE TABLE `" + table + "`")
	if err != nil {
		return nil, err
	}
	defer func() {
		if err1 := rows.Close(); err1 != nil {
			err = errors.New(err1.Error() + err.Error())
		}
	}()

	var tableName, sql string
	if err = rows.Scan(&tableName, &sql); err != nil {
		return nil, err
	}

	return parseMysqlCreateTable(filterCreateTableSQL(sql))
}

func getSqlite3TableInfo(tableName string, engine sqlbuilder.Engine) (*Table, error) {
	table := &Table{
		Columns:     make(map[string]string, 10),
		Constraints: make(map[string]sqlbuilder.Constraint, 5),
		Indexes:     make(map[string]sqlbuilder.Index, 2),
	}

	if err := parseSqlite3CreateTable(table, tableName, engine); err != nil {
		return nil, err
	}

	if err := parseSqlite3Indexes(table, tableName, engine); err != nil {
		return nil, err
	}

	return table, nil
}

// show create table 产生的格式比较统一，不像 create table 那样多样化。
// https://dev.mysql.com/doc/refman/8.0/en/show-create-table.html
func parseMysqlCreateTable(lines []string) (*Table, error) {
	table := &Table{
		Columns:     make(map[string]string, len(lines)),
		Constraints: make(map[string]sqlbuilder.Constraint, len(lines)),
		Indexes:     make(map[string]sqlbuilder.Index, len(lines)),
	}

	for _, line := range lines {
		line = strings.TrimSpace(line)
		index := strings.IndexByte(line, ' ')
		if index <= 0 {
			continue
		}
		first := line[:index]
		line = line[index+1:]

		switch strings.ToUpper(first) {
		case "INDEX", "KEY": // 索引
			index = strings.IndexByte(line, ' ')
			if index <= 0 {
				continue
			}
			table.Indexes[line[:index]] = sqlbuilder.IndexDefault
		case "PRIMARY":
			words := strings.Fields(line)
			table.Constraints[words[1]] = sqlbuilder.ConstraintPK
		case "UNIQUE":
			words := strings.Fields(line)
			table.Constraints[words[1]] = sqlbuilder.ConstraintUnique
		case "CONSTRAINT": // check 或是 fk 约束
			words := strings.Fields(line)
			switch strings.ToUpper(words[1]) {
			case "FOREIGN":
				table.Constraints[words[0]] = sqlbuilder.ConstraintFK
			case "CHECK":
				table.Constraints[words[0]] = sqlbuilder.ConstraintCheck
			default:
				return nil, fmt.Errorf("未知的约束类型:%s", words[1])
			}
		default: // 普通列定义，第一个字符串即为列名
			table.Columns[line[:index]] = line
		}
	}

	return table, nil
}

// https://www.sqlite.org/draft/lang_createtable.html
func parseSqlite3CreateTable(table *Table, tableName string, engine sqlbuilder.Engine) error {
	rows, err := engine.Query("SELECT sql FROM sqlite_master WHERE type='table' and name='" + tableName + "'")
	if err != nil {
		return err
	}
	defer func() {
		if err1 := rows.Close(); err1 != nil {
			err = errors.New(err1.Error() + err.Error())
		}
	}()

	var sql string
	if err = rows.Scan(&sql); err != nil {
		return err
	}

	lines := filterCreateTableSQL(sql)

LOOP:
	for _, line := range lines {
		line = strings.TrimSpace(line)
		index := strings.IndexByte(line, ' ')
		if index <= 0 {
			continue
		}
		first := line[:index]
		line = line[index+1:]

		switch strings.ToUpper(first) {
		case "CONSTRAINT": // 约束
			index = strings.IndexByte(line, ' ')
			if index <= 0 {
				continue LOOP
			}
			name := line[:index]

			line = strings.TrimSpace(line[index+1:])
			index = strings.IndexByte(line, ' ')
			if index <= 0 {
				continue LOOP
			}
			switch line[:index] {
			case "PRIMARY":
				table.Constraints[name] = sqlbuilder.ConstraintPK
			case "UNIQUE":
				table.Constraints[name] = sqlbuilder.ConstraintUnique
			case "CHECK":
				table.Constraints[name] = sqlbuilder.ConstraintCheck
			case "FOREIGN":
				table.Constraints[name] = sqlbuilder.ConstraintFK
			default:
				return fmt.Errorf("未知的约束名：%s", line[:index])
			}
		default: // 普通列定义，第一个字符串即为列名
			table.Columns[line[:index]] = line
		}
	}

	return nil
}
func parseSqlite3Indexes(table *Table, tableName string, engine sqlbuilder.Engine) error {
	rows, err := engine.Query("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='" + tableName + "'")
	if err != nil {
		return err
	}
	defer func() {
		if err1 := rows.Close(); err1 != nil {
			err = errors.New(err1.Error() + err.Error())
		}
	}()

	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			return err
		}
		table.Indexes[name] = sqlbuilder.IndexDefault
	}

	return nil
}

func filterCreateTableSQL(sql string) []string {
	sql = quoteReplacer.Replace(sql)
	var deep, start int
	var lines []string

	for index, c := range sql {
		switch c {
		case ',':
			if deep == 1 && index > start {
				lines = append(lines, sql[start:index])
				start = index
			}
		case '(':
			deep++
			if deep == 1 {
				start = index
			}
		case ')':
			deep--
			if deep == 0 { // 不需要 create table xx() 之后的内容
				break
			}
		} // end switch
	} // end for

	return lines
}
<|endoftext|>


<commit_before>package runner

import (
	"context"
	"errors"
	"fmt"
	"sync"

	"io"
	"time"

	core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
	cds "github.com/envoyproxy/go-control-plane/envoy/service/cluster/v3"
	discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"

	"github.com/ii/xds-test-harness/internal/parser"
	pb "github.com/ii/xds-test-harness/api/adapter"
	"google.golang.org/grpc"
	"github.com/rs/zerolog/log"
)

var (
	opts []grpc.DialOption = []grpc.DialOption{
		grpc.WithInsecure(),
		grpc.WithBlock(),
		grpc.WithTimeout(time.Second * 10),
	}
)

type ClientConfig struct {
	Port string
	Conn *grpc.ClientConn
}

type Cache struct {
	StartState *pb.Snapshot
	FinalResponse *discovery.DiscoveryResponse
}

type CDS struct {
	Req   chan *discovery.DiscoveryRequest
	Res   chan *discovery.DiscoveryResponse
	Err   chan error
	Done  chan bool
	Cache struct {
		InitResource []string
		Requests  []*discovery.DiscoveryRequest
		Responses []*discovery.DiscoveryResponse
	}
}

type Runner struct {
	Adapter *ClientConfig
	Target  *ClientConfig
	NodeID  string
	Cache   *Cache
	CDS     *CDS
}

func NewRunner() *Runner {
	return &Runner{
		Adapter: &ClientConfig{},
		Target:  &ClientConfig{},
		NodeID:  "",
		Cache:   &Cache{},
		CDS:     &CDS{},
	}
}

func FreshRunner (current *Runner) *Runner {
	return &Runner{
		Adapter: current.Adapter,
		Target: current.Target,
		NodeID: current.NodeID,
		Cache: &Cache{},
		CDS: &CDS{},
	}
}

func connectViaGRPC(client *ClientConfig, server string) (conn *grpc.ClientConn, err error) {
	conn, err = grpc.Dial(client.Port, opts...)
	if err != nil {
		err = fmt.Errorf("Cannot connect at %v: %v\n", client.Port, err)
		return nil, err
	}
	log.Debug().
		Msgf("Runner connected to %v", server)
	return conn, nil
}

func (r *Runner) ConnectToTarget(address string) error {
	r.Target.Port = address
	conn, err := connectViaGRPC(r.Target, "target")
	if err != nil {
		return err
	}
	r.Target.Conn = conn
	return nil
}

func (r *Runner) ConnectToAdapter(address string) error {
	r.Adapter.Port = address
	conn, err := connectViaGRPC(r.Adapter, "adapter")
	if err != nil {
		return err
	}
	r.Adapter.Conn = conn
	return nil
}

func (r *Runner) NewCDSRequest(resourceList []string) *discovery.DiscoveryRequest {
	clusters := []string{}
	for _, cluster := range resourceList {
		clusters = append(clusters, cluster)
	}
	return &discovery.DiscoveryRequest{
		VersionInfo: "",
		Node: &core.Node{
			Id: r.NodeID,
		},
		ResourceNames: clusters,
		TypeUrl:       "type.googleapis.com/envoy.config.cluster.v3.Cluster",
	}
}

func (r *Runner) NewCDSAckFromResponse(res *discovery.DiscoveryResponse) (*discovery.DiscoveryRequest, error) {
	response, err := parser.ParseDiscoveryResponse(res)
	if err != nil {
		err := fmt.Errorf("error parsing dres for acking: %v", err)
		return nil, err
	}

	request := &discovery.DiscoveryRequest{
		VersionInfo:   response.VersionInfo,
		ResourceNames: r.CDS.Cache.InitResource,
		TypeUrl:       "type.googleapis.com/envoy.config.cluster.v3.Cluster",
		ResponseNonce: response.Nonce,
	}
	return request, nil
}

func (r *Runner) AckCDS(initReq *discovery.DiscoveryRequest) {

	log.Debug().Msgf("Sending First Discovery Request", initReq)
	r.CDS.Req <- initReq
	r.CDS.Cache.Requests = append(r.CDS.Cache.Requests, initReq)

	for {
		select {
		case res := <-r.CDS.Res:
			r.CDS.Cache.Responses = append(r.CDS.Cache.Responses, res)
			ack, err := r.NewCDSAckFromResponse(res)
			if err != nil {
				log.Err(err).Msg("Error creating Ack Request")
			}
			log.Debug().
				Msgf("Sending Ack: %v", ack)
			r.CDS.Req <- ack
	        r.CDS.Cache.Requests = append(r.CDS.Cache.Requests, ack)
		case <-r.CDS.Done:
			log.Debug().Msg("Received Done signal, shutting down request channel")
			close(r.CDS.Req)
			return
		}
	}
}

func (r *Runner) CDSStream() error {

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	defer close(r.CDS.Err)

	client := cds.NewClusterDiscoveryServiceClient(r.Target.Conn)
	stream, err := client.StreamClusters(ctx)
	if err != nil {
		err = errors.New(fmt.Sprintf("Cannot start CDS stream %v. error: %v", stream, err))
		log.Debug().
			Err(err).
			Msg("")
		r.CDS.Err <- err
	}
	var wg sync.WaitGroup
	go func() {
		for {
			wg.Add(1)
			in, err := stream.Recv()
			if err == io.EOF {
				log.Debug().
					Msg("No more Discovery Responses from CDS stream")
				close(r.CDS.Res)
				return
			}
			if err != nil {
				log.Err(err).Msg("error receiving responses on CDS stream")
				r.CDS.Err <- err
				return
			}
			log.Debug().
				Msgf("Received discovery response: %v", in)
			r.CDS.Res <- in
		}
	}()

	for req := range r.CDS.Req {
		if err := stream.Send(req); err != nil {
			log.Err(err).
				Msg("Error sending discovery request")
			r.CDS.Err <- err
		}
	}
	stream.CloseSend()
	wg.Wait()
	return err
}
<commit_msg>log error as error<commit_after>package runner

import (
	"context"
	"errors"
	"fmt"
	"sync"

	"io"
	"time"

	core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
	cds "github.com/envoyproxy/go-control-plane/envoy/service/cluster/v3"
	discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"

	"github.com/ii/xds-test-harness/internal/parser"
	pb "github.com/ii/xds-test-harness/api/adapter"
	"google.golang.org/grpc"
	"github.com/rs/zerolog/log"
)

var (
	opts []grpc.DialOption = []grpc.DialOption{
		grpc.WithInsecure(),
		grpc.WithBlock(),
		grpc.WithTimeout(time.Second * 10),
	}
)

type ClientConfig struct {
	Port string
	Conn *grpc.ClientConn
}

type Cache struct {
	StartState *pb.Snapshot
	FinalResponse *discovery.DiscoveryResponse
}

type CDS struct {
	Req   chan *discovery.DiscoveryRequest
	Res   chan *discovery.DiscoveryResponse
	Err   chan error
	Done  chan bool
	Cache struct {
		InitResource []string
		Requests  []*discovery.DiscoveryRequest
		Responses []*discovery.DiscoveryResponse
	}
}

type Runner struct {
	Adapter *ClientConfig
	Target  *ClientConfig
	NodeID  string
	Cache   *Cache
	CDS     *CDS
}

func NewRunner() *Runner {
	return &Runner{
		Adapter: &ClientConfig{},
		Target:  &ClientConfig{},
		NodeID:  "",
		Cache:   &Cache{},
		CDS:     &CDS{},
	}
}

func FreshRunner (current *Runner) *Runner {
	return &Runner{
		Adapter: current.Adapter,
		Target: current.Target,
		NodeID: current.NodeID,
		Cache: &Cache{},
		CDS: &CDS{},
	}
}

func connectViaGRPC(client *ClientConfig, server string) (conn *grpc.ClientConn, err error) {
	conn, err = grpc.Dial(client.Port, opts...)
	if err != nil {
		err = fmt.Errorf("Cannot connect at %v: %v\n", client.Port, err)
		return nil, err
	}
	log.Debug().
		Msgf("Runner connected to %v", server)
	return conn, nil
}

func (r *Runner) ConnectToTarget(address string) error {
	r.Target.Port = address
	conn, err := connectViaGRPC(r.Target, "target")
	if err != nil {
		return err
	}
	r.Target.Conn = conn
	return nil
}

func (r *Runner) ConnectToAdapter(address string) error {
	r.Adapter.Port = address
	conn, err := connectViaGRPC(r.Adapter, "adapter")
	if err != nil {
		return err
	}
	r.Adapter.Conn = conn
	return nil
}

func (r *Runner) NewCDSRequest(resourceList []string) *discovery.DiscoveryRequest {
	clusters := []string{}
	for _, cluster := range resourceList {
		clusters = append(clusters, cluster)
	}
	return &discovery.DiscoveryRequest{
		VersionInfo: "",
		Node: &core.Node{
			Id: r.NodeID,
		},
		ResourceNames: clusters,
		TypeUrl:       "type.googleapis.com/envoy.config.cluster.v3.Cluster",
	}
}

func (r *Runner) NewCDSAckFromResponse(res *discovery.DiscoveryResponse) (*discovery.DiscoveryRequest, error) {
	response, err := parser.ParseDiscoveryResponse(res)
	if err != nil {
		err := fmt.Errorf("error parsing dres for acking: %v", err)
		return nil, err
	}

	request := &discovery.DiscoveryRequest{
		VersionInfo:   response.VersionInfo,
		ResourceNames: r.CDS.Cache.InitResource,
		TypeUrl:       "type.googleapis.com/envoy.config.cluster.v3.Cluster",
		ResponseNonce: response.Nonce,
	}
	return request, nil
}

func (r *Runner) AckCDS(initReq *discovery.DiscoveryRequest) {

	log.Debug().Msgf("Sending First Discovery Request", initReq)
	r.CDS.Req <- initReq
	r.CDS.Cache.Requests = append(r.CDS.Cache.Requests, initReq)

	for {
		select {
		case res := <-r.CDS.Res:
			r.CDS.Cache.Responses = append(r.CDS.Cache.Responses, res)
			ack, err := r.NewCDSAckFromResponse(res)
			if err != nil {
				log.Err(err).Msg("Error creating Ack Request")
			}
			log.Debug().
				Msgf("Sending Ack: %v", ack)
			r.CDS.Req <- ack
	        r.CDS.Cache.Requests = append(r.CDS.Cache.Requests, ack)
		case <-r.CDS.Done:
			log.Debug().Msg("Received Done signal, shutting down request channel")
			close(r.CDS.Req)
			return
		}
	}
}

func (r *Runner) CDSStream() error {

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	defer close(r.CDS.Err)

	client := cds.NewClusterDiscoveryServiceClient(r.Target.Conn)
	stream, err := client.StreamClusters(ctx)
	if err != nil {
		err = errors.New(fmt.Sprintf("Cannot start CDS stream %v. error: %v", stream, err))
		log.Error().
			Err(err).
			Msg("")
		r.CDS.Err <- err
	}
	var wg sync.WaitGroup
	go func() {
		for {
			wg.Add(1)
			in, err := stream.Recv()
			if err == io.EOF {
				log.Debug().
					Msg("No more Discovery Responses from CDS stream")
				close(r.CDS.Res)
				return
			}
			if err != nil {
				log.Err(err).Msg("error receiving responses on CDS stream")
				r.CDS.Err <- err
				return
			}
			log.Debug().
				Msgf("Received discovery response: %v", in)
			r.CDS.Res <- in
		}
	}()

	for req := range r.CDS.Req {
		if err := stream.Send(req); err != nil {
			log.Err(err).
				Msg("Error sending discovery request")
			r.CDS.Err <- err
		}
	}
	stream.CloseSend()
	wg.Wait()
	return err
}
<|endoftext|>


<commit_before>package virt

import (
	"context"
	"errors"
	"io"

	"github.com/jimmyfrasche/etlite/internal/internal/errint"
	"github.com/jimmyfrasche/etlite/internal/internal/errusr"
	"github.com/jimmyfrasche/etlite/internal/token"
	"github.com/jimmyfrasche/etlite/internal/virt/internal/builder"
)

type ImportSpec struct {
	Pos            token.Position
	Internal, Temp bool
	Table, Frame   string
	Header         []string
	DDL            string
	Limit, Offset  int
}

func (s ImportSpec) Valid() error {
	if s.DDL != "" {
		if s.Internal {
			return errint.New("CREATE TABLE FROM marked as internal table")
		}
		if s.Temp {
			return errint.New("CREATE TABLE FROM marked as temporary table (by system, not user)")
		}
		if len(s.Header) > 0 {
			return errint.New("CREATE TABLE FROM provides user defined header")
		}
		if s.Table == "" {
			return errint.New("CREATE TABLE FROM provides no table name")
		}
	} else if s.Internal && s.Temp {
		return errint.New("internal table marked as temporary (by system, not user)")
	}
	if s.Table == "" {
		return errint.New("table did not provide name")
	}
	return nil
}

func Import(s ImportSpec) Instruction {
	return func(ctx context.Context, m *Machine) error {
		if err := s.Valid(); err != nil {
			return err
		}

		//CREATE TABLE FROM
		if s.DDL != "" {
			//build table and compute header
			if err := m.exec(s.DDL); err != nil {
				return errusr.Wrap(s.Pos, err)
			}
			p, err := m.conn.Prepare("SELECT * FROM " + s.Table)
			if err != nil {
				return err
			}
			s.Header = p.Columns()
			if len(s.Header) == 0 {
				return errint.New("could not retrieve columns in CREATE TABLE FROM")
			}
			if err := p.Close(); err != nil {
				return err
			}
		}

		//prep the decoder
		dec := m.decoder
		if dec == nil {
			return errint.New("no decoder available when importing")
		}
		inHeader, err := dec.ReadHeader(s.Frame, s.Header)
		if err != nil {
			return err
		}

		//no header specified, use the one from the decoder
		if len(s.Header) == 0 {
			if len(inHeader) == 0 {
				return errors.New("no header specified and none returned by " + m.decoder.Name() + " format")
			}
			s.Header = inHeader
		}

		//internal tables can be created en masse so have to take care of their own savepointing.
		if !s.Internal {
			if err := m.savepoint(); err != nil {
				return err
			}
		}

		//not a CREATE TABLE FROM so we need to make the table
		if s.DDL == "" {
			ddl := m.createTable(s.Temp || s.Internal, s.Table, s.Header)
			if err := m.exec(ddl); err != nil {
				return err
			}
		}

		ins := m.createInserter(s.Table, s.Header)
		if err := m.bulkInsert(s.Table, ins, s.Limit, s.Offset); err != nil {
			return err
		}

		if !s.Internal {
			if err := m.release(); err != nil {
				return err
			}
		}

		return nil
	}
}

func (m *Machine) createTable(temp bool, name string, header []string) string {
	b := builder.New("CREATE")

	if temp {
		b.Push("TEMPORARY")
	}

	b.Push("TABLE", name, "(")

	b.CSV(header, func(h string) {
		b.Push(h, "TEXT")
	})

	b.Push(")")

	return b.Join(" ")
}

func (m *Machine) createInserter(name string, header []string) string {
	b := builder.New("INSERT INTO", name, "(")

	b.CSV(header, func(h string) {
		b.Push(h)
	})

	b.Push(") VALUES (")

	b.CSV(header, func(string) {
		b.Push("?")
	})
	b.Push(")")

	return b.Join(" ")
}

func (m *Machine) bulkInsert(name, ins string, limit, offset int) error {
	//make sure we have a decoder
	dec := m.decoder
	if dec == nil {
		return errint.Newf("no decoder when attempting to import %s", name)
	}

	//build the bulk loader
	p, err := m.conn.Prepare(ins)
	if err != nil {
		return err
	}
	defer p.Close()

	bulk, err := p.Loader()
	if err != nil {
		return err
	}

	if offset > 0 {
		if err := dec.Skip(offset); err != nil {
			return err
		}
	}

	for rows := 0; limit > 0 && rows == limit || true; rows++ {
		row, err := dec.ReadRow()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if err := bulk.Load(row); err != nil {
			return err
		}
	}

	if err := bulk.Close(); err != nil {
		return err
	}

	return dec.Reset()
}
<commit_msg>createrTable/Inserter need not be methods<commit_after>package virt

import (
	"context"
	"errors"
	"io"

	"github.com/jimmyfrasche/etlite/internal/internal/errint"
	"github.com/jimmyfrasche/etlite/internal/internal/errusr"
	"github.com/jimmyfrasche/etlite/internal/token"
	"github.com/jimmyfrasche/etlite/internal/virt/internal/builder"
)

type ImportSpec struct {
	Pos            token.Position
	Internal, Temp bool
	Table, Frame   string
	Header         []string
	DDL            string
	Limit, Offset  int
}

func (s ImportSpec) Valid() error {
	if s.DDL != "" {
		if s.Internal {
			return errint.New("CREATE TABLE FROM marked as internal table")
		}
		if s.Temp {
			return errint.New("CREATE TABLE FROM marked as temporary table (by system, not user)")
		}
		if len(s.Header) > 0 {
			return errint.New("CREATE TABLE FROM provides user defined header")
		}
		if s.Table == "" {
			return errint.New("CREATE TABLE FROM provides no table name")
		}
	} else if s.Internal && s.Temp {
		return errint.New("internal table marked as temporary (by system, not user)")
	}
	if s.Table == "" {
		return errint.New("table did not provide name")
	}
	return nil
}

func Import(s ImportSpec) Instruction {
	return func(ctx context.Context, m *Machine) error {
		if err := s.Valid(); err != nil {
			return err
		}

		//CREATE TABLE FROM
		if s.DDL != "" {
			//build table and compute header
			if err := m.exec(s.DDL); err != nil {
				return errusr.Wrap(s.Pos, err)
			}
			p, err := m.conn.Prepare("SELECT * FROM " + s.Table)
			if err != nil {
				return err
			}
			s.Header = p.Columns()
			if len(s.Header) == 0 {
				return errint.New("could not retrieve columns in CREATE TABLE FROM")
			}
			if err := p.Close(); err != nil {
				return err
			}
		}

		//prep the decoder
		dec := m.decoder
		if dec == nil {
			return errint.New("no decoder available when importing")
		}
		inHeader, err := dec.ReadHeader(s.Frame, s.Header)
		if err != nil {
			return err
		}

		//no header specified, use the one from the decoder
		if len(s.Header) == 0 {
			if len(inHeader) == 0 {
				return errors.New("no header specified and none returned by " + m.decoder.Name() + " format")
			}
			s.Header = inHeader
		}

		//internal tables can be created en masse so have to take care of their own savepointing.
		if !s.Internal {
			if err := m.savepoint(); err != nil {
				return err
			}
		}

		//not a CREATE TABLE FROM so we need to make the table
		if s.DDL == "" {
			ddl := createTable(s.Temp || s.Internal, s.Table, s.Header)
			if err := m.exec(ddl); err != nil {
				return err
			}
		}

		ins := createInserter(s.Table, s.Header)
		if err := m.bulkInsert(s.Table, ins, s.Limit, s.Offset); err != nil {
			return err
		}

		if !s.Internal {
			if err := m.release(); err != nil {
				return err
			}
		}

		return nil
	}
}

func createTable(temp bool, name string, header []string) string {
	b := builder.New("CREATE")

	if temp {
		b.Push("TEMPORARY")
	}

	b.Push("TABLE", name, "(")

	b.CSV(header, func(h string) {
		b.Push(h, "TEXT")
	})

	b.Push(")")

	return b.Join(" ")
}

func createInserter(name string, header []string) string {
	b := builder.New("INSERT INTO", name, "(")

	b.CSV(header, func(h string) {
		b.Push(h)
	})

	b.Push(") VALUES (")

	b.CSV(header, func(string) {
		b.Push("?")
	})
	b.Push(")")

	return b.Join(" ")
}

func (m *Machine) bulkInsert(name, ins string, limit, offset int) error {
	//make sure we have a decoder
	dec := m.decoder
	if dec == nil {
		return errint.Newf("no decoder when attempting to import %s", name)
	}

	//build the bulk loader
	p, err := m.conn.Prepare(ins)
	if err != nil {
		return err
	}
	defer p.Close()

	bulk, err := p.Loader()
	if err != nil {
		return err
	}

	if offset > 0 {
		if err := dec.Skip(offset); err != nil {
			return err
		}
	}

	for rows := 0; limit > 0 && rows == limit || true; rows++ {
		row, err := dec.ReadRow()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if err := bulk.Load(row); err != nil {
			return err
		}
	}

	if err := bulk.Close(); err != nil {
		return err
	}

	return dec.Reset()
}
<|endoftext|>


<commit_before>/*
Copyright 2022 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrateapiversion

import (
	"context"
	"fmt"
	"io"
	"time"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"

	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
)

type Migrator struct {
	// Client used for API interactions
	Client client.Client

	// If true, skip checking the 'status.storedVersion' before running the migration.
	// By default, migration will only be run if the CRD contains storedVersions other
	// than the desired target version.
	SkipStoredVersionCheck bool

	// Writers to write informational & error messages to
	Out, ErrOut io.Writer
}

// NewMigrator creates a new migrator with the given API client.
// If either of out or errOut are nil, log messages will be discarded.
func NewMigrator(client client.Client, skipStoredVersionCheck bool, out, errOut io.Writer) *Migrator {
	if out == nil {
		out = io.Discard
	}
	if errOut == nil {
		errOut = io.Discard
	}

	return &Migrator{
		Client:                 client,
		SkipStoredVersionCheck: skipStoredVersionCheck,
		Out:                    out,
		ErrOut:                 errOut,
	}
}

// Run begins the migration of all the named CRDs.
// It will attempt to migrate all resources defined as part of these CRDs to the
// given 'targetVersion', and after completion will update the `status.storedVersions`
// field on the corresponding CRD version to only contain the given targetVersion.
// Returns 'true' if a migration was actually performed, and false if migration was not required.
func (m *Migrator) Run(ctx context.Context, targetVersion string, names []string) (bool, error) {
	fmt.Fprintf(m.Out, "Checking all CustomResourceDefinitions have storage version set to \"%s\"\n", targetVersion)
	allTargetVersion, allCRDs, err := m.ensureCRDStorageVersionEquals(ctx, targetVersion, names)
	if err != nil {
		return false, err
	}
	if !allTargetVersion {
		fmt.Fprintf(m.ErrOut, "It looks like you are running a version of cert-manager that does not set the storage version of CRDs to %q. You MUST upgrade to cert-manager v1.0-v1.6 before migrating resources for v1.7.\n", targetVersion)
		return false, fmt.Errorf("preflight checks failed")
	}
	fmt.Fprintf(m.Out, "All CustomResourceDefinitions have %q configured as the storage version.\n", targetVersion)

	crdsRequiringMigration := allCRDs
	if !m.SkipStoredVersionCheck {
		fmt.Fprintf(m.Out, "Looking for CRDs that contain resources that require migrating to %q...\n", targetVersion)
		crdsRequiringMigration, err = m.discoverCRDsRequiringMigration(ctx, targetVersion, names)
		if err != nil {
			fmt.Fprintf(m.ErrOut, "Failed to determine resource types that require migration: %v\n", err)
			return false, err
		}
		if len(crdsRequiringMigration) == 0 {
			fmt.Fprintln(m.Out, "Nothing to do. cert-manager CRDs do not have \"status.storedVersions\" containing old API versions. You may proceed to upgrade to cert-manager v1.7.")
			return false, nil
		}
	} else {
		fmt.Fprintln(m.Out, "Forcing migration of all CRD resources as --skip-stored-version-check=true")
	}

	fmt.Fprintf(m.Out, "Found %d resource types that require migration:\n", len(crdsRequiringMigration))
	for _, crd := range crdsRequiringMigration {
		fmt.Fprintf(m.Out, " - %s (%s)\n", crd.Name, crd.Spec.Names.Kind)
	}

	for _, crd := range crdsRequiringMigration {
		if err := m.migrateResourcesForCRD(ctx, crd); err != nil {
			fmt.Fprintf(m.ErrOut, "Failed to migrate resource: %v\n", err)
			return false, err
		}
	}

	fmt.Fprintf(m.Out, "Patching CRD resources to set \"status.storedVersions\" to %q...\n", targetVersion)
	if err := m.patchCRDStoredVersions(ctx, crdsRequiringMigration); err != nil {
		fmt.Fprintf(m.ErrOut, "Failed to patch \"status.storedVersions\" field: %v\n", err)
		return false, err
	}

	fmt.Fprintln(m.Out, "Successfully migrated all cert-manager resource types. It is now safe to proceed to upgrade to cert-manager v1.7.")
	return true, nil
}

func (m *Migrator) ensureCRDStorageVersionEquals(ctx context.Context, vers string, names []string) (bool, []*apiext.CustomResourceDefinition, error) {
	var crds []*apiext.CustomResourceDefinition
	for _, crdName := range names {
		crd := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
			return false, nil, err
		}

		// Discover the storage version
		storageVersion := storageVersionForCRD(crd)

		if storageVersion != vers {
			fmt.Fprintf(m.Out, "CustomResourceDefinition object %q has storage version set to %q.\n", crdName, storageVersion)
			return false, nil, nil
		}

		crds = append(crds, crd)
	}

	return true, crds, nil
}

func (m *Migrator) discoverCRDsRequiringMigration(ctx context.Context, desiredStorageVersion string, names []string) ([]*apiext.CustomResourceDefinition, error) {
	var requireMigration []*apiext.CustomResourceDefinition
	for _, name := range names {
		crd := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, crd); err != nil {
			return nil, err
		}
		// If no versions are stored, there's nothing to migrate.
		if len(crd.Status.StoredVersions) == 0 {
			continue
		}
		// If more than one entry exists in `storedVersions` OR if the only element in there is not
		// the desired version, perform a migration.
		if len(crd.Status.StoredVersions) > 1 || crd.Status.StoredVersions[0] != desiredStorageVersion {
			requireMigration = append(requireMigration, crd)
		}
	}
	return requireMigration, nil
}

func (m *Migrator) migrateResourcesForCRD(ctx context.Context, crd *apiext.CustomResourceDefinition) error {
	startTime := time.Now()
	timeFormat := "15:04:05"
	fmt.Fprintf(m.Out, "Migrating %q objects in group %q - this may take a while (started at %s)...\n", crd.Spec.Names.Kind, crd.Spec.Group, startTime.Format(timeFormat))
	list := &unstructured.UnstructuredList{}
	list.SetGroupVersionKind(schema.GroupVersionKind{
		Group:   crd.Spec.Group,
		Version: storageVersionForCRD(crd),
		Kind:    crd.Spec.Names.ListKind,
	})
	if err := m.Client.List(ctx, list); err != nil {
		return err
	}
	fmt.Fprintf(m.Out, " %d resources to migrate...\n", len(list.Items))
	for _, obj := range list.Items {
		// retry on any kind of error to handle cases where e.g. the network connection to the apiserver fails
		if err := retry.OnError(wait.Backoff{
			Duration: time.Second, // wait 1s between attempts
			Steps:    3,           // allow up to 3 attempts per object
		}, func(err error) bool {
			// Retry on any errors that are not otherwise skipped/ignored
			return handleUpdateErr(err) != nil
		}, func() error { return m.Client.Update(ctx, &obj) }); handleUpdateErr(err) != nil {
			return err
		}
	}
	// add 500ms to the duration to ensure we always round up
	duration := time.Now().Sub(startTime) + (time.Millisecond * 500)
	fmt.Fprintf(m.Out, " Successfully migrated %d %s objects in %s\n", len(list.Items), crd.Spec.Names.Kind, duration.Round(time.Second))
	return nil
}

// patchCRDStoredVersions will patch the `status.storedVersions` field of all passed in CRDs to be
// set to an array containing JUST the current storage version.
// This is only safe to run after a successful migration (i.e. a read/write of all resources of the given CRD type).
func (m *Migrator) patchCRDStoredVersions(ctx context.Context, crds []*apiext.CustomResourceDefinition) error {
	for _, crd := range crds {
		// fetch a fresh copy of the CRD to avoid any conflict errors
		freshCRD := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: crd.Name}, freshCRD); err != nil {
			return err
		}

		// Check the latest copy of the CRD to ensure that:
		//   1) the storage version is the same as it was at the start of the migration
		//   2) the status.storedVersion field has not changed, and if it has, it has only added the new/desired storage version
		// This helps to avoid cases where the storage version was changed by a third-party midway through the migration,
		// which could lead to corrupted apiservers when we patch the status.storedVersions field below.
		expectedStorageVersion := storageVersionForCRD(crd)
		if storageVersionForCRD(freshCRD) != expectedStorageVersion {
			return newUnexpectedChangeError(crd)
		}
		newlyAddedVersions := storedVersionsAdded(crd, freshCRD)
		if newlyAddedVersions.Len() != 0 && !newlyAddedVersions.Equal(sets.NewString(expectedStorageVersion)) {
			return newUnexpectedChangeError(crd)
		}

		// Set the `status.storedVersions` field to the target storage version
		freshCRD.Status.StoredVersions = []string{storageVersionForCRD(crd)}

		if err := m.Client.Status().Update(ctx, freshCRD); err != nil {
			return err
		}
	}

	return nil
}

// storageVersionForCRD discovers the storage version for a given CRD.
func storageVersionForCRD(crd *apiext.CustomResourceDefinition) string {
	storageVersion := ""
	for _, v := range crd.Spec.Versions {
		if v.Storage {
			storageVersion = v.Name
			break
		}
	}
	return storageVersion
}

// storedVersionsAdded returns a list of any versions added to the `status.storedVersions` field on
// a CRD resource.
func storedVersionsAdded(old, new *apiext.CustomResourceDefinition) sets.String {
	oldStoredVersions := sets.NewString(old.Status.StoredVersions...)
	newStoredVersions := sets.NewString(new.Status.StoredVersions...)
	return newStoredVersions.Difference(oldStoredVersions)
}

// newUnexpectedChangeError creates a new 'error' that informs users that a change to the CRDs
// was detected during the migration process and so the migration must be re-run.
func newUnexpectedChangeError(crd *apiext.CustomResourceDefinition) error {
	errorFmt := "" +
		"The CRD %q unexpectedly changed during the migration. " +
		"This means that either an object was persisted in a non-storage version during the migration, " +
		"or the storage version was changed by someone else (or some automated deployment tooling) whilst the migration " +
		"was in progress.\n\n" +
		"All automated deployment tooling should be in a stable state (i.e. no upgrades to cert-manager CRDs should be" +
		"in progress whilst the migration is running).\n\n" +
		"Please ensure no changes to the CRDs are made during the migration process and re-run the migration until you" +
		"no longer see this message."
	return fmt.Errorf(errorFmt, crd.Name)
}

// handleUpdateErr will absorb certain types of errors that we know can be skipped/passed on
// during a migration of a particular object.
func handleUpdateErr(err error) error {
	if err == nil {
		return nil
	}
	// If the resource no longer exists, don't return the error as the object no longer
	// needs updating to the new API version.
	if apierrors.IsNotFound(err) {
		return nil
	}
	// If there was a conflict, another client must have written the object already which
	// means we don't need to force an update.
	if apierrors.IsConflict(err) {
		return nil
	}
	return err
}
<commit_msg>Simplify successfully migrated message<commit_after>/*
Copyright 2022 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrateapiversion

import (
	"context"
	"fmt"
	"io"
	"time"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"

	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
)

type Migrator struct {
	// Client used for API interactions
	Client client.Client

	// If true, skip checking the 'status.storedVersion' before running the migration.
	// By default, migration will only be run if the CRD contains storedVersions other
	// than the desired target version.
	SkipStoredVersionCheck bool

	// Writers to write informational & error messages to
	Out, ErrOut io.Writer
}

// NewMigrator creates a new migrator with the given API client.
// If either of out or errOut are nil, log messages will be discarded.
func NewMigrator(client client.Client, skipStoredVersionCheck bool, out, errOut io.Writer) *Migrator {
	if out == nil {
		out = io.Discard
	}
	if errOut == nil {
		errOut = io.Discard
	}

	return &Migrator{
		Client:                 client,
		SkipStoredVersionCheck: skipStoredVersionCheck,
		Out:                    out,
		ErrOut:                 errOut,
	}
}

// Run begins the migration of all the named CRDs.
// It will attempt to migrate all resources defined as part of these CRDs to the
// given 'targetVersion', and after completion will update the `status.storedVersions`
// field on the corresponding CRD version to only contain the given targetVersion.
// Returns 'true' if a migration was actually performed, and false if migration was not required.
func (m *Migrator) Run(ctx context.Context, targetVersion string, names []string) (bool, error) {
	fmt.Fprintf(m.Out, "Checking all CustomResourceDefinitions have storage version set to \"%s\"\n", targetVersion)
	allTargetVersion, allCRDs, err := m.ensureCRDStorageVersionEquals(ctx, targetVersion, names)
	if err != nil {
		return false, err
	}
	if !allTargetVersion {
		fmt.Fprintf(m.ErrOut, "It looks like you are running a version of cert-manager that does not set the storage version of CRDs to %q. You MUST upgrade to cert-manager v1.0-v1.6 before migrating resources for v1.7.\n", targetVersion)
		return false, fmt.Errorf("preflight checks failed")
	}
	fmt.Fprintf(m.Out, "All CustomResourceDefinitions have %q configured as the storage version.\n", targetVersion)

	crdsRequiringMigration := allCRDs
	if !m.SkipStoredVersionCheck {
		fmt.Fprintf(m.Out, "Looking for CRDs that contain resources that require migrating to %q...\n", targetVersion)
		crdsRequiringMigration, err = m.discoverCRDsRequiringMigration(ctx, targetVersion, names)
		if err != nil {
			fmt.Fprintf(m.ErrOut, "Failed to determine resource types that require migration: %v\n", err)
			return false, err
		}
		if len(crdsRequiringMigration) == 0 {
			fmt.Fprintln(m.Out, "Nothing to do. cert-manager CRDs do not have \"status.storedVersions\" containing old API versions. You may proceed to upgrade to cert-manager v1.7.")
			return false, nil
		}
	} else {
		fmt.Fprintln(m.Out, "Forcing migration of all CRD resources as --skip-stored-version-check=true")
	}

	fmt.Fprintf(m.Out, "Found %d resource types that require migration:\n", len(crdsRequiringMigration))
	for _, crd := range crdsRequiringMigration {
		fmt.Fprintf(m.Out, " - %s (%s)\n", crd.Name, crd.Spec.Names.Kind)
	}

	for _, crd := range crdsRequiringMigration {
		if err := m.migrateResourcesForCRD(ctx, crd); err != nil {
			fmt.Fprintf(m.ErrOut, "Failed to migrate resource: %v\n", err)
			return false, err
		}
	}

	fmt.Fprintf(m.Out, "Patching CRD resources to set \"status.storedVersions\" to %q...\n", targetVersion)
	if err := m.patchCRDStoredVersions(ctx, crdsRequiringMigration); err != nil {
		fmt.Fprintf(m.ErrOut, "Failed to patch \"status.storedVersions\" field: %v\n", err)
		return false, err
	}

	fmt.Fprintln(m.Out, "Successfully migrated all cert-manager resource types. It is now safe to upgrade to cert-manager v1.7.")
	return true, nil
}

func (m *Migrator) ensureCRDStorageVersionEquals(ctx context.Context, vers string, names []string) (bool, []*apiext.CustomResourceDefinition, error) {
	var crds []*apiext.CustomResourceDefinition
	for _, crdName := range names {
		crd := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
			return false, nil, err
		}

		// Discover the storage version
		storageVersion := storageVersionForCRD(crd)

		if storageVersion != vers {
			fmt.Fprintf(m.Out, "CustomResourceDefinition object %q has storage version set to %q.\n", crdName, storageVersion)
			return false, nil, nil
		}

		crds = append(crds, crd)
	}

	return true, crds, nil
}

func (m *Migrator) discoverCRDsRequiringMigration(ctx context.Context, desiredStorageVersion string, names []string) ([]*apiext.CustomResourceDefinition, error) {
	var requireMigration []*apiext.CustomResourceDefinition
	for _, name := range names {
		crd := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, crd); err != nil {
			return nil, err
		}
		// If no versions are stored, there's nothing to migrate.
		if len(crd.Status.StoredVersions) == 0 {
			continue
		}
		// If more than one entry exists in `storedVersions` OR if the only element in there is not
		// the desired version, perform a migration.
		if len(crd.Status.StoredVersions) > 1 || crd.Status.StoredVersions[0] != desiredStorageVersion {
			requireMigration = append(requireMigration, crd)
		}
	}
	return requireMigration, nil
}

func (m *Migrator) migrateResourcesForCRD(ctx context.Context, crd *apiext.CustomResourceDefinition) error {
	startTime := time.Now()
	timeFormat := "15:04:05"
	fmt.Fprintf(m.Out, "Migrating %q objects in group %q - this may take a while (started at %s)...\n", crd.Spec.Names.Kind, crd.Spec.Group, startTime.Format(timeFormat))
	list := &unstructured.UnstructuredList{}
	list.SetGroupVersionKind(schema.GroupVersionKind{
		Group:   crd.Spec.Group,
		Version: storageVersionForCRD(crd),
		Kind:    crd.Spec.Names.ListKind,
	})
	if err := m.Client.List(ctx, list); err != nil {
		return err
	}
	fmt.Fprintf(m.Out, " %d resources to migrate...\n", len(list.Items))
	for _, obj := range list.Items {
		// retry on any kind of error to handle cases where e.g. the network connection to the apiserver fails
		if err := retry.OnError(wait.Backoff{
			Duration: time.Second, // wait 1s between attempts
			Steps:    3,           // allow up to 3 attempts per object
		}, func(err error) bool {
			// Retry on any errors that are not otherwise skipped/ignored
			return handleUpdateErr(err) != nil
		}, func() error { return m.Client.Update(ctx, &obj) }); handleUpdateErr(err) != nil {
			return err
		}
	}
	// add 500ms to the duration to ensure we always round up
	duration := time.Now().Sub(startTime) + (time.Millisecond * 500)
	fmt.Fprintf(m.Out, " Successfully migrated %d %s objects in %s\n", len(list.Items), crd.Spec.Names.Kind, duration.Round(time.Second))
	return nil
}

// patchCRDStoredVersions will patch the `status.storedVersions` field of all passed in CRDs to be
// set to an array containing JUST the current storage version.
// This is only safe to run after a successful migration (i.e. a read/write of all resources of the given CRD type).
func (m *Migrator) patchCRDStoredVersions(ctx context.Context, crds []*apiext.CustomResourceDefinition) error {
	for _, crd := range crds {
		// fetch a fresh copy of the CRD to avoid any conflict errors
		freshCRD := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: crd.Name}, freshCRD); err != nil {
			return err
		}

		// Check the latest copy of the CRD to ensure that:
		//   1) the storage version is the same as it was at the start of the migration
		//   2) the status.storedVersion field has not changed, and if it has, it has only added the new/desired storage version
		// This helps to avoid cases where the storage version was changed by a third-party midway through the migration,
		// which could lead to corrupted apiservers when we patch the status.storedVersions field below.
		expectedStorageVersion := storageVersionForCRD(crd)
		if storageVersionForCRD(freshCRD) != expectedStorageVersion {
			return newUnexpectedChangeError(crd)
		}
		newlyAddedVersions := storedVersionsAdded(crd, freshCRD)
		if newlyAddedVersions.Len() != 0 && !newlyAddedVersions.Equal(sets.NewString(expectedStorageVersion)) {
			return newUnexpectedChangeError(crd)
		}

		// Set the `status.storedVersions` field to the target storage version
		freshCRD.Status.StoredVersions = []string{storageVersionForCRD(crd)}

		if err := m.Client.Status().Update(ctx, freshCRD); err != nil {
			return err
		}
	}

	return nil
}

// storageVersionForCRD discovers the storage version for a given CRD.
func storageVersionForCRD(crd *apiext.CustomResourceDefinition) string {
	storageVersion := ""
	for _, v := range crd.Spec.Versions {
		if v.Storage {
			storageVersion = v.Name
			break
		}
	}
	return storageVersion
}

// storedVersionsAdded returns a list of any versions added to the `status.storedVersions` field on
// a CRD resource.
func storedVersionsAdded(old, new *apiext.CustomResourceDefinition) sets.String {
	oldStoredVersions := sets.NewString(old.Status.StoredVersions...)
	newStoredVersions := sets.NewString(new.Status.StoredVersions...)
	return newStoredVersions.Difference(oldStoredVersions)
}

// newUnexpectedChangeError creates a new 'error' that informs users that a change to the CRDs
// was detected during the migration process and so the migration must be re-run.
func newUnexpectedChangeError(crd *apiext.CustomResourceDefinition) error {
	errorFmt := "" +
		"The CRD %q unexpectedly changed during the migration. " +
		"This means that either an object was persisted in a non-storage version during the migration, " +
		"or the storage version was changed by someone else (or some automated deployment tooling) whilst the migration " +
		"was in progress.\n\n" +
		"All automated deployment tooling should be in a stable state (i.e. no upgrades to cert-manager CRDs should be" +
		"in progress whilst the migration is running).\n\n" +
		"Please ensure no changes to the CRDs are made during the migration process and re-run the migration until you" +
		"no longer see this message."
	return fmt.Errorf(errorFmt, crd.Name)
}

// handleUpdateErr will absorb certain types of errors that we know can be skipped/passed on
// during a migration of a particular object.
func handleUpdateErr(err error) error {
	if err == nil {
		return nil
	}
	// If the resource no longer exists, don't return the error as the object no longer
	// needs updating to the new API version.
	if apierrors.IsNotFound(err) {
		return nil
	}
	// If there was a conflict, another client must have written the object already which
	// means we don't need to force an update.
	if apierrors.IsConflict(err) {
		return nil
	}
	return err
}
<|endoftext|>


<commit_before>// Prober is an HTTP prober that periodically sends out HTTP requests to specified
// endpoints and reports if the returned results match the expectations. The results
// of the probe, including latency, are recored in Carbon, which is presumed to run
// on the same machine as the prober. See probers.json as an example of the config
// file format.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/rcrowley/go-metrics"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"time"
)

var (
	config = flag.String("config", "probers.json", "Name of prober config file.")
	prefix = flag.String("prefix", "prober", "Prefix to add to all prober values sent to Carbon.")
	carbon = flag.String("carbon", "localhost:2003", "Address of Carbon server and port.")
)

const (
	SAMPLE_PERIOD = time.Minute
	TIMEOUT       = time.Duration(20 * time.Second)
)

// Probe is a single endpoint we are probing.
type Probe struct {
	// URL is the HTTP URL to probe.
	URL string `json:"url"`

	// Method is the HTTP method to use when probing.
	Method string `json:"method"`

	// Expected is the list of expected HTTP status code, i.e. [200, 201]
	Expected []int `json:"expected"`

	// Body is the body of the request to send if the method is POST.
	Body string `json:"body"`

	// The mimetype of the Body.
	MimeType string `json:"mimetype"`

	Success metrics.Counter
	Failure metrics.Counter
	Latency metrics.Timer
}

// Probes is all the probes that are to be run.
type Probes map[string]*Probe

func readConfigFile(filename string) (*Probes, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("Failed to open config file: %s", err)
	}
	d := json.NewDecoder(file)
	p := &Probes{}
	if err := d.Decode(p); err != nil {
		return nil, fmt.Errorf("Failed to decode JSON in config file: %s", err)
	}
	return p, nil
}

// In returns true if n is found in list.
func In(n int, list []int) bool {
	for _, x := range list {
		if x == n {
			return true
		}
	}
	return false
}

// dialTimeout is a dialer that sets a timeout.
func dialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, TIMEOUT)
}

func main() {
	flag.Parse()
	log.Println("Looking for Carbon server.")
	addr, err := net.ResolveTCPAddr("tcp", *carbon)
	if err != nil {
		log.Fatalln("Failed to resolve the Carbon server: ", err)
	}
	log.Println("Found Carbon server.")
	go metrics.Graphite(metrics.DefaultRegistry, SAMPLE_PERIOD, *prefix, addr)

	// TODO(jcgregorio) Monitor config file and reload if it changes.
	cfg, err := readConfigFile(*config)
	if err != nil {
		log.Fatalln("Failed to read config file: ", err)
	}
	log.Println("Successfully read config file.")
	// Register counters for each probe.
	for name, probe := range *cfg {
		probe.Success = metrics.NewRegisteredCounter(name+".success", metrics.DefaultRegistry)
		probe.Failure = metrics.NewRegisteredCounter(name+".failure", metrics.DefaultRegistry)
		probe.Latency = metrics.NewRegisteredTimer(name+".latency", metrics.DefaultRegistry)
	}
	var resp *http.Response
	var begin time.Time

	// Create a client that uses our dialer with a timeout.
	c := &http.Client{
		Transport: &http.Transport{
			Dial: dialTimeout,
		},
	}
	for _ = range time.Tick(SAMPLE_PERIOD) {
		for name, probe := range *cfg {
			log.Println("Running probe: ", name)
			begin = time.Now()
			if probe.Method == "GET" {
				resp, err = c.Get(probe.URL)
			} else if probe.Method == "POST" {
				resp, err = c.Post(probe.URL, probe.MimeType, strings.NewReader(probe.Body))
			} else {
				log.Println("Error: unknown method: ", probe.Method)
				continue
			}
			d := time.Since(begin)
			// TODO(jcgregorio) Save the last N responses and present them in a web UI.
			if err == nil && In(resp.StatusCode, probe.Expected) {
				probe.Success.Inc(1)
			} else {
				probe.Failure.Inc(1)
			}
			probe.Latency.Update(d)
		}
	}
}
<commit_msg>Report stats for the prober itself.<commit_after>// Prober is an HTTP prober that periodically sends out HTTP requests to specified
// endpoints and reports if the returned results match the expectations. The results
// of the probe, including latency, are recored in Carbon, which is presumed to run
// on the same machine as the prober. See probers.json as an example of the config
// file format.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/rcrowley/go-metrics"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"time"
)

var (
	config = flag.String("config", "probers.json", "Name of prober config file.")
	prefix = flag.String("prefix", "prober", "Prefix to add to all prober values sent to Carbon.")
	carbon = flag.String("carbon", "localhost:2003", "Address of Carbon server and port.")
)

const (
	SAMPLE_PERIOD = time.Minute
	TIMEOUT       = time.Duration(20 * time.Second)
)

// Probe is a single endpoint we are probing.
type Probe struct {
	// URL is the HTTP URL to probe.
	URL string `json:"url"`

	// Method is the HTTP method to use when probing.
	Method string `json:"method"`

	// Expected is the list of expected HTTP status code, i.e. [200, 201]
	Expected []int `json:"expected"`

	// Body is the body of the request to send if the method is POST.
	Body string `json:"body"`

	// The mimetype of the Body.
	MimeType string `json:"mimetype"`

	Success metrics.Counter
	Failure metrics.Counter
	Latency metrics.Timer
}

// Probes is all the probes that are to be run.
type Probes map[string]*Probe

func readConfigFile(filename string) (*Probes, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("Failed to open config file: %s", err)
	}
	d := json.NewDecoder(file)
	p := &Probes{}
	if err := d.Decode(p); err != nil {
		return nil, fmt.Errorf("Failed to decode JSON in config file: %s", err)
	}
	return p, nil
}

// In returns true if n is found in list.
func In(n int, list []int) bool {
	for _, x := range list {
		if x == n {
			return true
		}
	}
	return false
}

// dialTimeout is a dialer that sets a timeout.
func dialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, TIMEOUT)
}

func main() {
	flag.Parse()
	log.Println("Looking for Carbon server.")
	addr, err := net.ResolveTCPAddr("tcp", *carbon)
	if err != nil {
		log.Fatalln("Failed to resolve the Carbon server: ", err)
	}
	log.Println("Found Carbon server.")

	// We have two sets of metrics, one for the probes and one for the probe
	// server itself.
	serverRegistry := metrics.NewRegistry()
	metrics.RegisterRuntimeMemStats(serverRegistry)
	go metrics.CaptureRuntimeMemStats(serverRegistry, SAMPLE_PERIOD)
	go metrics.Graphite(serverRegistry, SAMPLE_PERIOD, "probeserver", addr)

	probeRegistry := metrics.NewRegistry()
	go metrics.Graphite(probeRegistry, SAMPLE_PERIOD, *prefix, addr)

	// TODO(jcgregorio) Monitor config file and reload if it changes.
	cfg, err := readConfigFile(*config)
	if err != nil {
		log.Fatalln("Failed to read config file: ", err)
	}
	log.Println("Successfully read config file.")
	// Register counters for each probe.
	for name, probe := range *cfg {
		probe.Success = metrics.NewRegisteredCounter(name+".success", probeRegistry)
		probe.Failure = metrics.NewRegisteredCounter(name+".failure", probeRegistry)
		probe.Latency = metrics.NewRegisteredTimer(name+".latency", probeRegistry)
	}
	var resp *http.Response
	var begin time.Time

	// Create a client that uses our dialer with a timeout.
	c := &http.Client{
		Transport: &http.Transport{
			Dial: dialTimeout,
		},
	}
	for _ = range time.Tick(SAMPLE_PERIOD) {
		for name, probe := range *cfg {
			log.Println("Running probe: ", name)
			begin = time.Now()
			if probe.Method == "GET" {
				resp, err = c.Get(probe.URL)
			} else if probe.Method == "POST" {
				resp, err = c.Post(probe.URL, probe.MimeType, strings.NewReader(probe.Body))
			} else {
				log.Println("Error: unknown method: ", probe.Method)
				continue
			}
			resp.Body.Close()
			d := time.Since(begin)
			// TODO(jcgregorio) Save the last N responses and present them in a web UI.
			if err == nil && In(resp.StatusCode, probe.Expected) {
				probe.Success.Inc(1)
			} else {
				probe.Failure.Inc(1)
			}
			probe.Latency.Update(d)
		}
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"strings"

	"github.com/golang/protobuf/proto"
	mesos "github.com/mesos/mesos-go/mesosproto"
)

// Convert param string like "param1=value1;param2=value2" to mesos.Labels
func StringToLabels(s string) *mesos.Labels {
	labels := &mesos.Labels{Labels: make([]*mesos.Label, 0)}
	pairs := strings.Split(s, ";")
	for _, pair := range pairs {
		kv := strings.Split(pair, "=")
		key, value := kv[0], kv[1]
		label := &mesos.Label{Key: proto.String(key), Value: proto.String(value)}
		labels.Labels = append(labels.Labels, label)
	}
	return labels
}
<commit_msg>Fix empty labels.<commit_after>package utils

import (
	"strings"

	"github.com/golang/protobuf/proto"
	mesos "github.com/mesos/mesos-go/mesosproto"
)

// Convert param string like "param1=value1;param2=value2" to mesos.Labels
func StringToLabels(s string) *mesos.Labels {
	labels := &mesos.Labels{Labels: make([]*mesos.Label, 0)}
	if s == "" {
		return labels
	}
	pairs := strings.Split(s, ";")
	for _, pair := range pairs {
		kv := strings.Split(pair, "=")
		key, value := kv[0], kv[1]
		label := &mesos.Label{Key: proto.String(key), Value: proto.String(value)}
		labels.Labels = append(labels.Labels, label)
	}
	return labels
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/foize/go.sgr"
	"github.com/howeyc/fsnotify"
	"go/build"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

var (
	wd string // working directory

	pkg *build.Package

	monAngo *exec.Cmd // command builds ango on file change (uses gomon)

	// last ango generation
	timeLastGenCodeOverwrite time.Time

	// genLock is write-locked when generated source was detected modified unexpectedly
	genLock sync.RWMutex

	// closed on stop
	stopCh = make(chan bool)
	stopWg sync.WaitGroup
)

const exampleAngoFile = "example/chatService.ango"

type CheckWriter struct {
	wr     io.Writer
	filter string
	action func()
}

func (c *CheckWriter) Write(b []byte) (n int, err error) {
	n, err = c.wr.Write(b)
	if strings.Contains(string(b), c.filter) {
		c.action()
	}
	return n, err
}
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	var err error
	fmt.Println("Starting ango dev tool.")

	wd, err = os.Getwd()
	if err != nil {
		fmt.Printf("Error getting wd: %s\n", err)
		stop(1)
		select {}
	}

	pkg, err = build.ImportDir(wd, 0)
	if err != nil {
		fmt.Printf("Error loading package: %s\n", err)
		stop(1)
		select {}
	}

	if pkg.Name != "main" || !pkg.IsCommand() || filepath.Base(wd) != "ango" {
		fmt.Println("Is tool executed from the right directory? (github.com/GeertJohan/ango or a fork)?")
		fmt.Printf("Current package (%s) is invalid.\n", pkg.Name)
		stop(1)
		select {}
	}

	// watch generated source and lock on modification
	go watchGeneratedSource()

	// watch and rebuild/restart example cmd
	go rerunExample()

	// watch and rebuild ango tool
	go rebuildAngo()

	// watch ango tool and templates and re-generate on change
	go watchExampleAngo()

	sigChan := make(chan os.Signal)
	signal.Notify(sigChan, os.Kill, os.Interrupt)
	sig := <-sigChan
	signal.Stop(sigChan)

	fmt.Printf("Received %s, closing...\n", sig)
	if sig == os.Interrupt {
		stop(0)
	} else {
		stop(1)
	}
	select {}
}

func stop(exitCode int) {
	go func() {
		// synced stop
		close(stopCh)
		stopWg.Wait()

		// os.Exit(..)
		os.Exit(exitCode)
	}()
}

func watchGeneratedSource() {
	stopWg.Add(1)
	defer stopWg.Done()

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		fmt.Printf("Error starting watcher: %s\n", err)
		stop(1)
		return
	}
	defer watcher.Close()
	err = watcher.WatchFlags(`example/ango-chatService.gen.go`, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on go generated file: %s\n", err)
		stop(1)
		return
	}
	err = watcher.WatchFlags(`example/http-files/ango-chatService.gen.js`, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on js generated file: %s\n", err)
		stop(1)
		return
	}
	for {
		select {
		case <-stopCh:
			return
		case <-watcher.Event:
			// short timeout consuming another event becuase sublime sometimes saves (modifies) the file twice
			select {
			case <-time.After(100 * time.Millisecond):
			case <-watcher.Event:
			}
			if time.Now().Sub(timeLastGenCodeOverwrite) > time.Duration(3*time.Second) {
				genLock.Lock()
				sgr.Println("[fg-red]Detected modification on generated source. Hit enter to continue devtool (will overwrite changes!).")
				fmt.Scanln()
				timeLastGenCodeOverwrite = time.Now()
				genLock.Unlock()
			}
			select {
			case <-time.After(100 * time.Millisecond):
			case <-watcher.Event:
			}
		case err := <-watcher.Error:
			if err != nil {
				fmt.Printf("Error watching example ango file: %s\n", err)
				stop(1)
				return
			}
		}
	}
}

func rerunExample() {
	stopWg.Add(1)
	defer stopWg.Done()

	cmdRerun := exec.Command("rerun", filepath.Join(pkg.ImportPath, "example"))
	cmdRerun.Stdin = os.Stdin
	cmdRerun.Stdout = sgr.NewColorWriter(os.Stdout, sgr.FgYellow, false)
	cmdRerun.Stderr = sgr.NewColorWriter(os.Stderr, sgr.FgYellow, false)
	err := cmdRerun.Start()
	if err != nil {
		fmt.Printf("Error running rerun example: %s\n", err)
		stop(1)
		return
	}
	<-stopCh
	if cmdRerun.Process != nil {
		cmdRerun.Process.Signal(os.Interrupt)
	}
	err = cmdRerun.Wait()
	if err != nil && err.Error() != "exit status 2" {
		fmt.Printf("Error stopping rerun example: %s\n", err)
		return
	}
}

func rebuildAngo() {
	stopWg.Add(1)
	defer stopWg.Done()

	cmdRerun := exec.Command("rerun", "-build-only", pkg.ImportPath)
	cmdRerun.Stdin = os.Stdin
	cw := &CheckWriter{
		wr:     os.Stderr,
		filter: "build passed",
		action: func() {
			angoExample()
		},
	}
	cmdRerun.Stdout = sgr.NewColorWriter(os.Stdout, sgr.FgCyan, false)
	cmdRerun.Stderr = sgr.NewColorWriter(cw, sgr.FgCyan, false)
	err := cmdRerun.Start()
	if err != nil {
		fmt.Printf("Error running rerun ango build: %s\n", err)
		stop(1)
		return
	}
	<-stopCh
	if cmdRerun.Process != nil {
		cmdRerun.Process.Signal(os.Interrupt)
	}
	err = cmdRerun.Wait()
	if err != nil && err.Error() != "exit status 2" {
		fmt.Printf("Error stopping rerun ango build: %s\n", err)
		return
	}
}

func watchExampleAngo() {
	stopWg.Add(1)
	defer stopWg.Done()

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		fmt.Printf("Error starting watcher: %s\n", err)
		stop(1)
		return
	}
	defer watcher.Close()
	err = watcher.WatchFlags(exampleAngoFile, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on example ango file: %s\n", err)
		stop(1)
		return
	}
	err = watcher.WatchFlags(`templates/ango-service.tmpl.go`, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on go template file: %s\n", err)
		stop(1)
		return
	}
	err = watcher.WatchFlags(`templates/ango-service.tmpl.js`, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on js template file: %s\n", err)
		stop(1)
		return
	}
	for {
		select {
		case <-stopCh:
			return
		case <-watcher.Event:
			// short timeout consuming another event becuase sublime sometimes saves (modifies) the file twice
			select {
			case <-time.After(100 * time.Millisecond):
			case <-watcher.Event:
			}
			angoExample()
			// short timeout consuming another event because somehow ango modifies the template files!?!?!?!?
			select {
			case <-time.After(100 * time.Millisecond):
			case <-watcher.Event:
			}
		case err := <-watcher.Error:
			if err != nil {
				fmt.Printf("Error watching example ango file: %s\n", err)
				stop(1)
				return
			}
		}
	}
}

func angoExample() {
	stopWg.Add(1)
	defer stopWg.Done()
	genLock.RLock()
	defer genLock.RUnlock()
	timeLastGenCodeOverwrite = time.Now()
	fmt.Println("Running ango tool for example/chatService.ango")
	cmdAngoExample := exec.Command(filepath.Join(wd, "ango"), "--verbose", "-i", exampleAngoFile, "--js", "example/http-files", "--go", "example", "--force-overwrite")
	cmdAngoExample.Stdin = os.Stdin
	cmdAngoExample.Stdout = sgr.NewColorWriter(os.Stdout, sgr.FgBlue, false)
	cmdAngoExample.Stderr = sgr.NewColorWriter(os.Stderr, sgr.FgBlue, false)
	err := cmdAngoExample.Run()
	if err != nil {
		fmt.Printf("Error running ango tool: %s\n", err)
	}
}
<commit_msg>Improved synchronization for generate lock on user-modified generated files<commit_after>package main

import (
	"fmt"
	"github.com/foize/go.sgr"
	"github.com/howeyc/fsnotify"
	"go/build"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

var (
	wd string // working directory

	pkg *build.Package

	monAngo *exec.Cmd // command builds ango on file change (uses gomon)

	// last ango generation
	timeLastGenCodeOverwrite time.Time

	// genLock is write-locked when generated source was detected modified unexpectedly
	genLock sync.RWMutex

	// closed on stop
	stopCh = make(chan bool)
	stopWg sync.WaitGroup
)

const exampleAngoFile = "example/chatService.ango"

type CheckWriter struct {
	wr     io.Writer
	filter string
	action func()
}

func (c *CheckWriter) Write(b []byte) (n int, err error) {
	n, err = c.wr.Write(b)
	if strings.Contains(string(b), c.filter) {
		c.action()
	}
	return n, err
}
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	var err error
	fmt.Println("Starting ango dev tool.")

	wd, err = os.Getwd()
	if err != nil {
		fmt.Printf("Error getting wd: %s\n", err)
		stop(1)
		select {}
	}

	pkg, err = build.ImportDir(wd, 0)
	if err != nil {
		fmt.Printf("Error loading package: %s\n", err)
		stop(1)
		select {}
	}

	if pkg.Name != "main" || !pkg.IsCommand() || filepath.Base(wd) != "ango" {
		fmt.Println("Is tool executed from the right directory? (github.com/GeertJohan/ango or a fork)?")
		fmt.Printf("Current package (%s) is invalid.\n", pkg.Name)
		stop(1)
		select {}
	}

	// watch generated source and lock on modification
	go watchGeneratedSource()

	// watch and rebuild/restart example cmd
	go rerunExample()

	// watch and rebuild ango tool
	go rebuildAngo()

	// watch ango tool and templates and re-generate on change
	go watchExampleAngo()

	sigChan := make(chan os.Signal)
	signal.Notify(sigChan, os.Kill, os.Interrupt)
	sig := <-sigChan
	signal.Stop(sigChan)

	fmt.Printf("Received %s, closing...\n", sig)
	if sig == os.Interrupt {
		stop(0)
	} else {
		stop(1)
	}
	select {}
}

func stop(exitCode int) {
	go func() {
		// synced stop
		close(stopCh)
		stopWg.Wait()

		// os.Exit(..)
		os.Exit(exitCode)
	}()
}

func watchGeneratedSource() {
	stopWg.Add(1)
	defer stopWg.Done()

	haveLine := make(chan bool)
	go func() {
		// clear stdin and read line when required
		for {
			fmt.Scanln()
			select {
			case haveLine <- true:
			default:
			}
		}
	}()

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		fmt.Printf("Error starting watcher: %s\n", err)
		stop(1)
		return
	}
	defer watcher.Close()
	err = watcher.WatchFlags(`example/ango-chatService.gen.go`, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on go generated file: %s\n", err)
		stop(1)
		return
	}
	err = watcher.WatchFlags(`example/http-files/ango-chatService.gen.js`, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on js generated file: %s\n", err)
		stop(1)
		return
	}
	for {
		select {
		case <-stopCh:
			return
		case <-watcher.Event:
			// short timeout consuming another event becuase sublime sometimes saves (modifies) the file twice
			select {
			case <-time.After(100 * time.Millisecond):
			case <-watcher.Event:
			}
			if time.Now().Sub(timeLastGenCodeOverwrite) > time.Duration(3*time.Second) {
				genLock.Lock()
				sgr.Println("[fg-red]Detected modification on generated source. Hit enter to continue devtool (will overwrite changes!).")
				<-haveLine
				timeLastGenCodeOverwrite = time.Now()
				genLock.Unlock()
			}
			select {
			case <-time.After(100 * time.Millisecond):
			case <-watcher.Event:
			}
		case err := <-watcher.Error:
			if err != nil {
				fmt.Printf("Error watching example ango file: %s\n", err)
				stop(1)
				return
			}
		}
	}
}

func rerunExample() {
	stopWg.Add(1)
	defer stopWg.Done()

	cmdRerun := exec.Command("rerun", filepath.Join(pkg.ImportPath, "example"))
	cmdRerun.Stdin = os.Stdin
	cmdRerun.Stdout = sgr.NewColorWriter(os.Stdout, sgr.FgYellow, false)
	cmdRerun.Stderr = sgr.NewColorWriter(os.Stderr, sgr.FgYellow, false)
	err := cmdRerun.Start()
	if err != nil {
		fmt.Printf("Error running rerun example: %s\n", err)
		stop(1)
		return
	}
	<-stopCh
	if cmdRerun.Process != nil {
		cmdRerun.Process.Signal(os.Interrupt)
	}
	err = cmdRerun.Wait()
	if err != nil && err.Error() != "exit status 2" {
		fmt.Printf("Error stopping rerun example: %s\n", err)
		return
	}
}

func rebuildAngo() {
	stopWg.Add(1)
	defer stopWg.Done()

	cmdRerun := exec.Command("rerun", "-build-only", pkg.ImportPath)
	cmdRerun.Stdin = os.Stdin
	cw := &CheckWriter{
		wr:     os.Stderr,
		filter: "build passed",
		action: func() {
			angoExample()
		},
	}
	cmdRerun.Stdout = sgr.NewColorWriter(os.Stdout, sgr.FgCyan, false)
	cmdRerun.Stderr = sgr.NewColorWriter(cw, sgr.FgCyan, false)
	err := cmdRerun.Start()
	if err != nil {
		fmt.Printf("Error running rerun ango build: %s\n", err)
		stop(1)
		return
	}
	<-stopCh
	if cmdRerun.Process != nil {
		cmdRerun.Process.Signal(os.Interrupt)
	}
	err = cmdRerun.Wait()
	if err != nil && err.Error() != "exit status 2" {
		fmt.Printf("Error stopping rerun ango build: %s\n", err)
		return
	}
}

func watchExampleAngo() {
	stopWg.Add(1)
	defer stopWg.Done()

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		fmt.Printf("Error starting watcher: %s\n", err)
		stop(1)
		return
	}
	defer watcher.Close()
	err = watcher.WatchFlags(exampleAngoFile, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on example ango file: %s\n", err)
		stop(1)
		return
	}
	err = watcher.WatchFlags(`templates/ango-service.tmpl.go`, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on go template file: %s\n", err)
		stop(1)
		return
	}
	err = watcher.WatchFlags(`templates/ango-service.tmpl.js`, fsnotify.FSN_MODIFY)
	if err != nil {
		fmt.Printf("Error starting watch on js template file: %s\n", err)
		stop(1)
		return
	}
	for {
		select {
		case <-stopCh:
			return
		case <-watcher.Event:
			// short timeout consuming other events becuase sublime sometimes saves (modifies) the file twice
			for {
				select {
				case <-time.After(100 * time.Millisecond):
					goto ango
				case <-watcher.Event:
				}
			}
		ango:
			angoExample()
			// short timeout consuming another event because somehow ango modifies the template files!?!?!?!?
			select {
			case <-time.After(100 * time.Millisecond):
			case <-watcher.Event:
			}
		case err := <-watcher.Error:
			if err != nil {
				fmt.Printf("Error watching example ango file: %s\n", err)
				stop(1)
				return
			}
		}
	}
}

func angoExample() {
	stopWg.Add(1)
	defer stopWg.Done()
	genLock.RLock()
	defer genLock.RUnlock()
	timeLastGenCodeOverwrite = time.Now()
	fmt.Println("Running ango tool for example/chatService.ango")
	cmdAngoExample := exec.Command(filepath.Join(wd, "ango"), "--verbose", "-i", exampleAngoFile, "--js", "example/http-files", "--go", "example", "--force-overwrite")
	cmdAngoExample.Stdin = os.Stdin
	cmdAngoExample.Stdout = sgr.NewColorWriter(os.Stdout, sgr.FgBlue, false)
	cmdAngoExample.Stderr = sgr.NewColorWriter(os.Stderr, sgr.FgBlue, false)
	err := cmdAngoExample.Run()
	if err != nil {
		fmt.Printf("Error running ango tool: %s\n", err)
	}
}
<|endoftext|>


<commit_before>package tools

import (
	"crypto/sha256"
	"encoding/hex"
	"hash"
	"io"

	"github.com/github/git-lfs/errors"
	"github.com/github/git-lfs/progress"
)

type readSeekCloserWrapper struct {
	readSeeker io.ReadSeeker
}

func (r *readSeekCloserWrapper) Read(p []byte) (n int, err error) {
	return r.readSeeker.Read(p)
}

func (r *readSeekCloserWrapper) Seek(offset int64, whence int) (int64, error) {
	return r.readSeeker.Seek(offset, whence)
}

func (r *readSeekCloserWrapper) Close() error {
	return nil
}

// NewReadSeekCloserWrapper wraps an io.ReadSeeker and implements a no-op Close() function
// to make it an io.ReadCloser
func NewReadSeekCloserWrapper(r io.ReadSeeker) io.ReadCloser {
	return &readSeekCloserWrapper{r}
}

// CopyWithCallback copies reader to writer while performing a progress callback
func CopyWithCallback(writer io.Writer, reader io.Reader, totalSize int64, cb progress.CopyCallback) (int64, error) {
	if success, _ := CloneFile(writer, reader); success {
		if cb != nil {
			cb(totalSize, totalSize, 0)
		}
		return totalSize, nil
	}
	if cb == nil {
		return io.Copy(writer, reader)
	}

	cbReader := &progress.CallbackReader{
		C:         cb,
		TotalSize: totalSize,
		Reader:    reader,
	}
	return io.Copy(writer, cbReader)
}

// Get a new Hash instance of the type used to hash LFS content
func NewLfsContentHash() hash.Hash {
	return sha256.New()
}

// HashingReader wraps a reader and calculates the hash of the data as it is read
type HashingReader struct {
	reader io.Reader
	hasher hash.Hash
}

func NewHashingReader(r io.Reader) *HashingReader {
	return &HashingReader{r, NewLfsContentHash()}
}

func NewHashingReaderPreloadHash(r io.Reader, hash hash.Hash) *HashingReader {
	return &HashingReader{r, hash}
}

func (r *HashingReader) Hash() string {
	return hex.EncodeToString(r.hasher.Sum(nil))
}

func (r *HashingReader) Read(b []byte) (int, error) {
	w, err := r.reader.Read(b)
	if err == nil || err == io.EOF {
		_, e := r.hasher.Write(b[0:w])
		if e != nil && err == nil {
			return w, e
		}
	}

	return w, err
}

// RetriableReader wraps a error response of reader as RetriableError()
type RetriableReader struct {
	reader io.Reader
}

func NewRetriableReader(r io.Reader) io.Reader {
	return &RetriableReader{r}
}

func (r *RetriableReader) Read(b []byte) (int, error) {
	n, err := r.reader.Read(b)
	// EOF is a successful response as it is used to signal a graceful end of input
	// c.f. https://git.io/v6riQ
	if err == nil || err == io.EOF {
		return n, err
	}
	return n, errors.NewRetriableError(err)
}
<commit_msg>tools: ensure Retriable errors are wrapped only once<commit_after>package tools

import (
	"crypto/sha256"
	"encoding/hex"
	"hash"
	"io"

	"github.com/github/git-lfs/errors"
	"github.com/github/git-lfs/progress"
)

type readSeekCloserWrapper struct {
	readSeeker io.ReadSeeker
}

func (r *readSeekCloserWrapper) Read(p []byte) (n int, err error) {
	return r.readSeeker.Read(p)
}

func (r *readSeekCloserWrapper) Seek(offset int64, whence int) (int64, error) {
	return r.readSeeker.Seek(offset, whence)
}

func (r *readSeekCloserWrapper) Close() error {
	return nil
}

// NewReadSeekCloserWrapper wraps an io.ReadSeeker and implements a no-op Close() function
// to make it an io.ReadCloser
func NewReadSeekCloserWrapper(r io.ReadSeeker) io.ReadCloser {
	return &readSeekCloserWrapper{r}
}

// CopyWithCallback copies reader to writer while performing a progress callback
func CopyWithCallback(writer io.Writer, reader io.Reader, totalSize int64, cb progress.CopyCallback) (int64, error) {
	if success, _ := CloneFile(writer, reader); success {
		if cb != nil {
			cb(totalSize, totalSize, 0)
		}
		return totalSize, nil
	}
	if cb == nil {
		return io.Copy(writer, reader)
	}

	cbReader := &progress.CallbackReader{
		C:         cb,
		TotalSize: totalSize,
		Reader:    reader,
	}
	return io.Copy(writer, cbReader)
}

// Get a new Hash instance of the type used to hash LFS content
func NewLfsContentHash() hash.Hash {
	return sha256.New()
}

// HashingReader wraps a reader and calculates the hash of the data as it is read
type HashingReader struct {
	reader io.Reader
	hasher hash.Hash
}

func NewHashingReader(r io.Reader) *HashingReader {
	return &HashingReader{r, NewLfsContentHash()}
}

func NewHashingReaderPreloadHash(r io.Reader, hash hash.Hash) *HashingReader {
	return &HashingReader{r, hash}
}

func (r *HashingReader) Hash() string {
	return hex.EncodeToString(r.hasher.Sum(nil))
}

func (r *HashingReader) Read(b []byte) (int, error) {
	w, err := r.reader.Read(b)
	if err == nil || err == io.EOF {
		_, e := r.hasher.Write(b[0:w])
		if e != nil && err == nil {
			return w, e
		}
	}

	return w, err
}

// RetriableReader wraps a error response of reader as RetriableError()
type RetriableReader struct {
	reader io.Reader
}

func NewRetriableReader(r io.Reader) io.Reader {
	return &RetriableReader{r}
}

func (r *RetriableReader) Read(b []byte) (int, error) {
	n, err := r.reader.Read(b)

	// EOF is a successful response as it is used to signal a graceful end
	// of input c.f. https://git.io/v6riQ
	//
	// Otherwise, if the error is non-nill and already retriable (in the
	// case that the underlying reader `r.reader` is itself a
	// `*RetriableReader`, return the error wholesale:
	if err == nil || err == io.EOF || errors.IsRetriableError(err) {
		return n, err
	}

	return n, errors.NewRetriableError(err)
}
<|endoftext|>


<commit_before>package transport

import (
	"encoding/gob"
	"fmt"
	"log"
	"net"
	"pilosa/config"
	"pilosa/core"
	"pilosa/db"

	"tux21b.org/v1/gocql/uuid"
)

type qmessage struct {
	message *db.Message
	host    *uuid.UUID
}

type TcpTransport struct {
	service *core.Service
	port    int
	inbox   chan *db.Message
	outbox  chan qmessage
	stop    chan bool
}

func (self *TcpTransport) Run() {
	log.Println("Initializing TCP transport")
	go self.listen()
	for {
		select {
		case qmessage := <-self.outbox:
			// todo: persistent connections
			process, err := self.service.ProcessMap.GetProcess(qmessage.host)
			if err != nil {
				log.Println("transport/tcp", err)
				return
			}
			host_string := fmt.Sprintf("%s:%d", process.Host(), process.PortTcp())
			conn, err := net.Dial("tcp", host_string)
			encoder := gob.NewEncoder(conn)
			err = encoder.Encode(qmessage.message)
			if err != nil {
				log.Println(err.Error())
				return
			}
			//time.Sleep(1 * time.Second)
		}
	}
}

func (self *TcpTransport) listen() {
	port_string := fmt.Sprintf(":%d", self.port)
	l, e := net.Listen("tcp", port_string)
	if e != nil {
		log.Fatal("Cannot bind to port!", self.port)
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			log.Fatal("Cannot accept message!")
		}
		go self.manage(&conn)
	}
}

func (self *TcpTransport) manage(conn *net.Conn) {
	decoder := gob.NewDecoder(*conn)
	var mess *db.Message
	err := decoder.Decode(&mess)
	if err != nil {
		log.Println("tcp/transport", err.Error())
		return
	}
	self.inbox <- mess
}

func (self *TcpTransport) Close() {
	log.Println("Shutting down TCP transport")
}

func (self *TcpTransport) Send(message *db.Message, host *uuid.UUID) {
	self.outbox <- qmessage{message, host}
}

func (self *TcpTransport) Receive() *db.Message {
	return <-self.inbox
}

func (self *TcpTransport) Push(message *db.Message) {
	self.inbox <- message
}

func NewTcpTransport(service *core.Service) *TcpTransport {
	return &TcpTransport{service, config.GetInt("port_tcp"), make(chan *db.Message), make(chan qmessage), nil}
}
<commit_msg>persistent connection support.<commit_after>package transport

import (
	"encoding/gob"
	"fmt"
	"log"
	"net"
	"pilosa/config"
	"pilosa/core"
	"pilosa/db"
	"github.com/davecgh/go-spew/spew"

	"time"
	"tux21b.org/v1/gocql/uuid"
)

type envelope struct {
	message *db.Message
	host    *uuid.UUID
}

type connection struct {
	transport *TcpTransport
	inbox     chan *db.Message
	outbox    chan *db.Message
	conn      *net.Conn
	process   *uuid.UUID
}

type newconnection struct {
	id         *uuid.UUID
	connection *connection
}

func init() {
	gob.Register(uuid.UUID{})
}

func (self *connection) manage() {
BeginManageConnection:
	for {
		log.Println("manage", self)
		if self.conn == nil {
			process, err := self.transport.service.ProcessMap.GetProcess(self.process)
			if err != nil {
				log.Fatal("transport/tcp: ", err)
				return
			}
			host_string := fmt.Sprintf("%s:%d", process.Host(), process.PortTcp())
			conn, err := net.Dial("tcp", host_string)
			if err != nil {
				log.Println("transport/tcp: error dialing: ", host_string, " Retrying in 2 seconds...")
				time.Sleep(2 * time.Second)
				continue
			}
			self.conn = &conn
			go func() {
				self.outbox <- &db.Message{self.transport.service.Id}
			}()
		}
		encoder := gob.NewEncoder(*self.conn)
		decoder := gob.NewDecoder(*self.conn)
		var exit = make(chan int)
		go func() {
			for {
				var mess *db.Message
				err := decoder.Decode(&mess)
				if err != nil {
					exit <- 1
					return
				}
				self.inbox <- mess
			}
		}()
		for {
			select {
			case message := <-self.outbox:
				err := encoder.Encode(message)
				if err != nil {
					log.Println(err.Error())
					return
				}
			case message := <-self.inbox:
				identifier, ok := message.Data.(uuid.UUID)
				if ok {
					spew.Dump("ok, sending to reg chan", identifier)
					// message is connection registration; bypass inbox and register
					self.process = &identifier
					self.transport.reg <- &newconnection{&identifier, self}
				} else {
					self.transport.inbox <- message
				}
			case <-exit:
				log.Println("decoder stopped!")
				if self.process != nil {
					self.conn = nil
					continue BeginManageConnection
				} else {
					return
				}
			}
		}
	}
}

type TcpTransport struct {
	service     *core.Service
	port        int
	inbox       chan *db.Message
	outbox      chan envelope
	connections map[uuid.UUID]*connection
	reg         chan *newconnection
}

func (self *TcpTransport) Run() {
	log.Println("Initializing TCP transport")
	go self.listen()
	for {
		select {
		case env := <-self.outbox:
			con, ok := self.connections[*(env.host)]
			if !ok {
				con = &connection{self, make(chan *db.Message), make(chan *db.Message), nil, env.host}
				go con.manage()
				self.connections[*env.host] = con
			}
			con.outbox <- env.message
		case nc := <-self.reg:
			self.connections[*nc.id] = nc.connection
		}
	}
}

func (self *TcpTransport) listen() {
	port_string := fmt.Sprintf(":%d", self.port)
	l, e := net.Listen("tcp", port_string)
	if e != nil {
		log.Fatal("Cannot bind to port!", self.port)
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			log.Fatal("Cannot accept message!")
		}
		go self.manage(&conn)
	}
}

func (self *TcpTransport) manage(conn *net.Conn) {
	con := &connection{self, make(chan *db.Message), make(chan *db.Message), conn, nil}
	con.manage()
}

func (self *TcpTransport) Close() {
	log.Println("Shutting down TCP transport")
}

func (self *TcpTransport) Send(message *db.Message, host *uuid.UUID) {
	self.outbox <- envelope{message, host}
}

func (self *TcpTransport) Receive() *db.Message {
	return <-self.inbox
}

func (self *TcpTransport) Push(message *db.Message) {
	self.inbox <- message
}

func NewTcpTransport(service *core.Service) *TcpTransport {
	return &TcpTransport{service, config.GetInt("port_tcp"), make(chan *db.Message, 100), make(chan envelope, 100), make(map[uuid.UUID]*connection), make(chan *newconnection)}
}
<|endoftext|>