<commit_before>package apidef

const Schema = `{
    "type": "object",
    "$schema": "http://json-schema.org/draft-04/schema",
    "id": "http://jsonschema.net",
    "additionalProperties": false,
    "properties": {
        "is_site": {
            "type": "boolean"
        },
        "uptime_tests": {
            "type": "object"
        },
        "expire_analytics_after": {
            "type": "number"
        },
        "id": {
            "type": "string"
        },
        "org_id": {
            "type": "string"
        },
        "api_id": {
            "type": "string"
        },
        "enable_ip_whitelisting": {
            "type": "boolean"
        },
        "enable_ip_blacklisting": {
            "type": "boolean"
        },
        "enable_context_vars": {
            "type": "boolean"
        },
        "strip_auth_data": {
          "type": "boolean"
        },
        "do_not_track": {
            "type": "boolean"
        },
        "enable_jwt": {
            "type": "boolean"
        },
        "use_openid": {
            "type": "boolean"
        },
        "openid_options": {
            "type": "object"
        },
        "use_standard_auth":{
            "type": "boolean"
        },
        "enable_coprocess_auth": {
            "type": "boolean"
        },
        "jwt_skip_kid": {
            "type": "boolean"
        },
        "base_identity_provided_by": {
            "type": "string"
        },
        "disable_rate_limit": {
            "type": "boolean"
        },
        "disable_quota": {
            "type": "boolean"
        },
        "custom_middleware_bundle": {
            "type": "string"
        },
        "jwt_policy_field_name": {
            "type": "string"
        },
        "jwt_signing_method": {
            "type": "string"
        },
        "jwt_source": {
            "type": "string"
        },
        "jwt_identity_base_field": {
            "type": "string"
        },
        "jwt_client_base_field": {
            "type": "string"
        },
        "jwt_disable_issued_at_validation": {
            "type": "boolean"
        },
        "jwt_disable_expires_at_validation": {
            "type": "boolean"
        },
        "jwt_disable_not_before_validation": {
            "type": "boolean"
        },
        "jwt_issued_at_validation_skew": {
            "type": "number"
        },
        "jwt_expires_at_validation_skew": {
            "type": "number"
        },
        "jwt_not_before_validation_skew": {
            "type": "number"
        },
        "jwt_scope_to_policy_mapping": {
            "type": ["object", "null"]
        },
        "use_keyless": {
            "type": "boolean"
        },
        "use_basic_auth": {
            "type": "boolean"
        },
        "use_mutual_tls_auth": {
            "type": "boolean"
        },
        "client_certificates": {
            "type": "array"
        },
        "upstream_certificates": {
            "type": "object"
        },
        "pinned_public_keys": {
            "type": "object"
        },
        "allowed_ips": {
            "type": "array"
        },
        "blacklisted_ips": {
            "type": "array"
        },
        "enable_batch_request_support": {
            "type": "boolean"
        },
        "event_handlers": {
            "type":"object"
        },
        "notifications": {
            "type":"object"
        },
        "use_oauth2": {
            "type": "boolean"
        },
        "oauth_meta": {
            "type":"object"
        },
        "cache_options": {
            "type":"object"
        },
        "tags": {
            "type":"array"
        },
        "tag_headers": {
            "type":"array"
        },
        "basic_auth": {
            "type": "object"
        },
        "CORS": {
            "type":"object"
        },
        "response_processors": {
            "type":"array"
        },
        "auth_provider": {
            "type":"object",
            "properties": {
                "name": {
                    "type": "string",
                    "enum": [""]
                },
                "storage_engine": {
                    "type": "string",
                    "enum": [""]
                }
            }
        },
        "session_provider": {
            "type":"object",
            "properties": {
                "name": {
                    "type": "string",
                    "enum": [""]
                },
                "storage_engine": {
                    "type": "string",
                    "enum": [""]
                }
            }
        },
        "hmac_allowed_clock_skew": {
            "type": "number"
        },
        "dont_set_quota_on_create": {
            "type": "boolean"
            },
        "custom_middleware": {
            "type":"object",
            "properties": {
                "pre": {
                    "type": ["array", "null"]
                },
                "post": {
                    "type": ["array", "null"]
                }
            }
        },
        "session_lifetime": {
            "type": "number"
        },
        "enable_signature_checking": {
            "type": "boolean"
        },
        "active": {
            "type": "boolean",
            "id": "http://jsonschema.net/active"
        },
        "auth": {
            "type": "object",
            "id": "http://jsonschema.net/auth",
            "properties": {
                "auth_header_name": {
                    "type": "string",
                    "id": "http://jsonschema.net/auth/auth_header_name"
                },
                "use_certificate": {
                    "type": "boolean"
                }
            },
            "required": [
                "auth_header_name"
            ]
        },
        "definition": {
            "type": "object",
            "id": "http://jsonschema.net/definition",
            "properties": {
                "key": {
                    "type": "string",
                    "id": "http://jsonschema.net/definition/key"
                },
                "location": {
                    "type": "string",
                    "id": "http://jsonschema.net/definition/location"
                },
                "strip_path": {
                    "type": "boolean",
                    "id": "http://jsonschema.net/definition/location"
                }
            },
            "required": [
                "key",
                "location"
            ]
        },
        "name": {
            "type": "string",
            "id": "http://jsonschema.net/name"
        },
        "slug": {
            "type": "string",
            "pattern": "[a-zA-Z0-9]*",
            "id": "http://jsonschema.net/name"
        },
        "domain": {
            "type": "string"
        },
        "check_host_against_uptime_tests": {
            "type": "boolean"
        },
        "proxy": {
            "type": "object",
            "id": "http://jsonschema.net/proxy",
            "properties": {
                "target_url": {
                    "type": "string",
                    "id": "http://jsonschema.net/proxy/target_url"
                },
                "check_host_against_uptime_tests": {
                    "type": "boolean"
                },
                "preserve_host_header": {
                    "type": "boolean"
                },
                "transport": {
                    "type": "object",
                    "properties": {
                        "ssl_ciphers": {
                            "type": ["array", "null"]
                        },
                        "ssl_min_version": {
                            "type": "number"
                        },
                        "proxy_url": {
                            "type": "string"
                        }
                    }
                }
            },
            "required": [
                "target_url"
            ]
        },
        "hook_references": {
            "type": "object"
        },
        "version_data": {
            "type": "object",
            "id": "http://jsonschema.net/version_data",
            "properties": {
                "not_versioned": {
                    "type": "boolean",
                    "id": "http://jsonschema.net/version_data/not_versioned"
                },
                "default_version":{
                    "type": "string",
                    "id": "http://jsonschema.net/version_data/default_version"
                },
                "versions": {
                    "type": "object",
                    "id": "http://jsonschema.net/version_data/versions",
                    "patternProperties": {
                        "^[a-zA-Z0-9]+$": {
                            "title": "versionInfoProperty",
                            "type": "object",
                            "id": "http://jsonschema.net/access_rights/versionInfoProperty",
                            "properties": {
                                "expires": {
                                    "type": "string",
                                    "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/expires"
                                },
                                "name": {
                                    "type": "string",
                                    "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/name"
                                },
                                "paths": {
                                    "type": "object",
                                    "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/paths",
                                    "properties": {
                                        "black_list": {
                                            "type": "array",
                                            "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/paths/black_list"
                                        },
                                        "ignored": {
                                            "type": "array",
                                            "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/paths/ignored"
                                        },
                                        "white_list": {
                                            "type": "array",
                                            "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/paths/white_list"
                                        }
                                    }
                                }
                            },
                            "required": [
                                "name"
                            ]
                        }
                    }
                }
            },
            "required": [
                "not_versioned",
                "versions"
            ]
        },
        "config_data": {
            "type": "object"
        },
        "global_rate_limit": {
          "type": "object",
           "properties": {
                "rate": {
                    "type": "number"
                },
                "per": {
                    "type": "number"
                }
            }
        }
    },
    "required": [
        "auth",
        "name",
        "proxy",
        "version_data"
    ]
}`
<commit_msg> Fix Schema to make it backward compatible (#2023)<commit_after>package apidef

const Schema = `{
    "type": ["object", "null"],
    "$schema": "http://json-schema.org/draft-04/schema",
    "id": "http://jsonschema.net",
    "additionalProperties": false,
    "properties": {
        "is_site": {
            "type": "boolean"
        },
        "uptime_tests": {
            "type": ["object", "null"]
        },
        "expire_analytics_after": {
            "type": "number"
        },
        "id": {
            "type": "string"
        },
        "org_id": {
            "type": "string"
        },
        "api_id": {
            "type": "string"
        },
        "enable_ip_whitelisting": {
            "type": "boolean"
        },
        "enable_ip_blacklisting": {
            "type": "boolean"
        },
        "enable_context_vars": {
            "type": "boolean"
        },
        "strip_auth_data": {
          "type": "boolean"
        },
        "do_not_track": {
            "type": "boolean"
        },
        "enable_jwt": {
            "type": "boolean"
        },
        "use_openid": {
            "type": "boolean"
        },
        "openid_options": {
            "type": ["object", "null"]
        },
        "use_standard_auth":{
            "type": "boolean"
        },
        "enable_coprocess_auth": {
            "type": "boolean"
        },
        "jwt_skip_kid": {
            "type": "boolean"
        },
        "base_identity_provided_by": {
            "type": "string"
        },
        "disable_rate_limit": {
            "type": "boolean"
        },
        "disable_quota": {
            "type": "boolean"
        },
        "custom_middleware_bundle": {
            "type": "string"
        },
        "jwt_policy_field_name": {
            "type": "string"
        },
        "jwt_signing_method": {
            "type": "string"
        },
        "jwt_source": {
            "type": "string"
        },
        "jwt_identity_base_field": {
            "type": "string"
        },
        "jwt_client_base_field": {
            "type": "string"
        },
        "jwt_disable_issued_at_validation": {
            "type": "boolean"
        },
        "jwt_disable_expires_at_validation": {
            "type": "boolean"
        },
        "jwt_disable_not_before_validation": {
            "type": "boolean"
        },
        "jwt_issued_at_validation_skew": {
            "type": "number"
        },
        "jwt_expires_at_validation_skew": {
            "type": "number"
        },
        "jwt_not_before_validation_skew": {
            "type": "number"
        },
        "jwt_scope_to_policy_mapping": {
            "type": ["object", "null"]
        },
        "use_keyless": {
            "type": "boolean"
        },
        "use_basic_auth": {
            "type": "boolean"
        },
        "use_mutual_tls_auth": {
            "type": "boolean"
        },
        "client_certificates": {
            "type": ["array", "null"]
        },
        "upstream_certificates": {
            "type": ["object", "null"]
        },
        "pinned_public_keys": {
            "type": ["object", "null"]
        },
        "allowed_ips": {
            "type": ["array", "null"]
        },
        "blacklisted_ips": {
            "type": ["array", "null"]
        },
        "enable_batch_request_support": {
            "type": "boolean"
        },
        "event_handlers": {
            "type":["object", "null"]
        },
        "notifications": {
            "type":["object", "null"]
        },
        "use_oauth2": {
            "type": "boolean"
        },
        "oauth_meta": {
            "type":["object", "null"]
        },
        "cache_options": {
            "type":["object", "null"]
        },
        "tags": {
            "type": ["array", "null"]
        },
        "tag_headers": {
            "type": ["array", "null"]
        },
        "basic_auth": {
            "type": ["object", "null"]
        },
        "CORS": {
            "type":["object", "null"]
        },
        "response_processors": {
            "type": ["array", "null"]
        },
        "auth_provider": {
            "type":["object", "null"],
            "properties": {
                "name": {
                    "type": "string",
                    "enum": [""]
                },
                "storage_engine": {
                    "type": "string",
                    "enum": [""]
                }
            }
        },
        "session_provider": {
            "type":["object", "null"],
            "properties": {
                "name": {
                    "type": "string",
                    "enum": [""]
                },
                "storage_engine": {
                    "type": "string",
                    "enum": [""]
                }
            }
        },
        "hmac_allowed_clock_skew": {
            "type": "number"
        },
        "dont_set_quota_on_create": {
            "type": "boolean"
            },
        "custom_middleware": {
            "type":["object", "null"],
            "properties": {
                "pre": {
                    "type": ["array", "null"]
                },
                "post": {
                    "type": ["array", "null"]
                }
            }
        },
        "session_lifetime": {
            "type": "number"
        },
        "enable_signature_checking": {
            "type": "boolean"
        },
        "active": {
            "type": "boolean",
            "id": "http://jsonschema.net/active"
        },
        "auth": {
            "type": ["object", "null"],
            "id": "http://jsonschema.net/auth",
            "properties": {
                "auth_header_name": {
                    "type": "string",
                    "id": "http://jsonschema.net/auth/auth_header_name"
                },
                "use_certificate": {
                    "type": "boolean"
                }
            },
            "required": [
                "auth_header_name"
            ]
        },
        "definition": {
            "type": ["object", "null"],
            "id": "http://jsonschema.net/definition",
            "properties": {
                "key": {
                    "type": "string",
                    "id": "http://jsonschema.net/definition/key"
                },
                "location": {
                    "type": "string",
                    "id": "http://jsonschema.net/definition/location"
                },
                "strip_path": {
                    "type": "boolean",
                    "id": "http://jsonschema.net/definition/location"
                }
            },
            "required": [
                "key",
                "location"
            ]
        },
        "name": {
            "type": "string",
            "id": "http://jsonschema.net/name"
        },
        "slug": {
            "type": "string",
            "pattern": "[a-zA-Z0-9]*",
            "id": "http://jsonschema.net/name"
        },
        "domain": {
            "type": "string"
        },
        "check_host_against_uptime_tests": {
            "type": "boolean"
        },
        "proxy": {
            "type": ["object", "null"],
            "id": "http://jsonschema.net/proxy",
            "properties": {
                "target_url": {
                    "type": "string",
                    "id": "http://jsonschema.net/proxy/target_url"
                },
                "check_host_against_uptime_tests": {
                    "type": "boolean"
                },
                "preserve_host_header": {
                    "type": "boolean"
                },
                "transport": {
                    "type": ["object", "null"],
                    "properties": {
                        "ssl_ciphers": {
                            "type": ["array", "null"]
                        },
                        "ssl_min_version": {
                            "type": "number"
                        },
                        "proxy_url": {
                            "type": "string"
                        }
                    }
                }
            },
            "required": [
                "target_url"
            ]
        },
        "hook_references": {
            "type": ["object", "null"]
        },
        "version_data": {
            "type": ["object", "null"],
            "id": "http://jsonschema.net/version_data",
            "properties": {
                "not_versioned": {
                    "type": "boolean",
                    "id": "http://jsonschema.net/version_data/not_versioned"
                },
                "default_version":{
                    "type": "string",
                    "id": "http://jsonschema.net/version_data/default_version"
                },
                "versions": {
                    "type": ["object", "null"],
                    "id": "http://jsonschema.net/version_data/versions",
                    "patternProperties": {
                        "^[a-zA-Z0-9]+$": {
                            "title": "versionInfoProperty",
                            "type": ["object", "null"],
                            "id": "http://jsonschema.net/access_rights/versionInfoProperty",
                            "properties": {
                                "expires": {
                                    "type": "string",
                                    "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/expires"
                                },
                                "name": {
                                    "type": "string",
                                    "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/name"
                                },
                                "paths": {
                                    "type": ["object", "null"],
                                    "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/paths",
                                    "properties": {
                                        "black_list": {
                                            "type": ["array", "null"],
                                            "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/paths/black_list"
                                        },
                                        "ignored": {
                                            "type": ["array", "null"],
                                            "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/paths/ignored"
                                        },
                                        "white_list": {
                                            "type": ["array", "null"],
                                            "id": "http://jsonschema.net/version_data/versions/versionInfoProperty/paths/white_list"
                                        }
                                    }
                                }
                            },
                            "required": [
                                "name"
                            ]
                        }
                    }
                }
            },
            "required": [
                "not_versioned",
                "versions"
            ]
        },
        "config_data": {
            "type": ["object", "null"]
        },
        "global_rate_limit": {
          "type": ["object", "null"],
           "properties": {
                "rate": {
                    "type": "number"
                },
                "per": {
                    "type": "number"
                }
            }
        }
    },
    "required": [
        "auth",
        "name",
        "proxy",
        "version_data"
    ]
}`
<|endoftext|>


<commit_before>package lessgo

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"path"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/lessgo/lessgo/logs"
	"github.com/lessgo/lessgo/logs/color"
)

// 一旦注册，不可再更改
type (
	ApiMiddleware struct {
		Name          string // 全局唯一
		Desc          string
		DefaultConfig interface{} // 默认配置(JSON格式)
		defaultConfig string      // 默认配置的JSON字符串
		Middleware    func(configJSON string) MiddlewareFunc
		inited        bool // 标记是否已经初始化过
	}
	// 虚拟路由中中间件配置信息
	MiddlewareConfig struct {
		Name   string `json:"name"`   // 全局唯一
		Config string `json:"config"` // JSON格式的配置
	}
)

func (a *ApiMiddleware) CreateMiddlewareFunc(config string) MiddlewareFunc {
	if config == "" {
		return a.Middleware(a.defaultConfig)
	}
	return a.Middleware(config)
}

// 注册中间件
func (a ApiMiddleware) Reg() *ApiMiddleware {
	return a.init()
}

// 初始化中间件，设置id并当Name为空时自动添加Name
func (a *ApiMiddleware) init() *ApiMiddleware {
	if a.inited {
		return getApiMiddleware(a.Name)
	}

	if a.Name == "" {
		v := reflect.ValueOf(a.Middleware)
		a.Name = runtime.FuncForPC(v.Pointer()).Name()
	}

	if m := getApiMiddleware(a.Name); m != nil {
		return m
	}

	if a.DefaultConfig != nil {
		b, _ := json.Marshal(a.DefaultConfig)
		a.defaultConfig = string(b)
	}

	a.inited = true

	setApiMiddleware(a)

	return a
}

var (
	apiMiddlewareMap  = map[string]*ApiMiddleware{}
	apiMiddlewareLock sync.RWMutex
)

func getApiMiddleware(name string) *ApiMiddleware {
	apiMiddlewareLock.RLock()
	defer apiMiddlewareLock.RUnlock()
	return apiMiddlewareMap[name]
}

func setApiMiddleware(vh *ApiMiddleware) {
	apiMiddlewareLock.Lock()
	defer apiMiddlewareLock.Unlock()
	apiMiddlewareMap[vh.Name] = vh
	for i, vh2 := range DefLessgo.apiMiddlewares {
		if vh.Name < vh2.Name {
			list := make([]*ApiMiddleware, len(DefLessgo.apiMiddlewares)+1)
			copy(list, DefLessgo.apiMiddlewares[:i])
			list[i] = vh
			copy(list[i+1:], DefLessgo.apiMiddlewares[i:])
			DefLessgo.apiMiddlewares = list
			return
		}
	}
	DefLessgo.apiMiddlewares = append(DefLessgo.apiMiddlewares, vh)
}

// 检查中间件是否存在
func isExistMiddlewares(middlewareConfigs ...MiddlewareConfig) error {
	var errstring string
	for _, m := range middlewareConfigs {
		_, ok := apiMiddlewareMap[m.Name]
		if !ok {
			errstring += " \"" + m.Name + "\""
		}
	}
	if len(errstring) == 0 {
		return nil
	}
	return fmt.Errorf("Specified below middlewares does not exist: %v\n", errstring)
}

// 根据中间件配置生成中间件
func createMiddlewareFuncs(configs []MiddlewareConfig) []MiddlewareFunc {
	mws := make([]MiddlewareFunc, len(configs))
	for i, mw := range configs {
		mws[i] = apiMiddlewareMap[mw.Name].CreateMiddlewareFunc(mw.Config)
	}
	return mws
}

/*
 * system middleware
 */
func init() {
	(&ApiMiddleware{
		Name:       "检查服务器是否启用",
		Desc:       "检查服务器是否启用",
		Middleware: CheckServer,
	}).init()

	(&ApiMiddleware{
		Name:       "检查是否为访问主页",
		Desc:       "检查是否为访问主页",
		Middleware: CheckHome,
	}).init()

	(&ApiMiddleware{
		Name:       "系统运行日志打印",
		Desc:       "RequestLogger returns a middleware that logs HTTP requests.",
		Middleware: RequestLogger,
	}).init()

	(&ApiMiddleware{
		Name: "捕获运行时恐慌",
		Desc: "Recover returns a middleware which recovers from panics anywhere in the chain and handles the control to the centralized HTTPErrorHandler.",
		DefaultConfig: RecoverConfig{
			StackSize:         4 << 10, // 4 KB
			DisableStackAll:   false,
			DisablePrintStack: false,
		},
		Middleware: Recover,
	}).init()

	(&ApiMiddleware{
		Name:       "设置允许跨域",
		Desc:       "根据配置信息设置允许跨域",
		Middleware: CrossDomain,
	}).init()

	// 系统预设中间件
	PreUse(
		MiddlewareConfig{Name: "检查服务器是否启用"},
		MiddlewareConfig{Name: "检查是否为访问主页"},
		MiddlewareConfig{Name: "系统运行日志打印"},
		MiddlewareConfig{Name: "捕获运行时恐慌"},
		MiddlewareConfig{Name: "设置允许跨域"},
	)
}

// 检查服务器是否启用
func CheckServer(config string) MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			if !ServerEnable() {
				return c.NoContent(http.StatusServiceUnavailable)
			}
			return next(c)
		}
	}
}

// 检查是否为访问主页
func CheckHome(config string) MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			if c.Request().URL().Path() == "/" {
				c.Request().URL().SetPath(GetHome())
			}
			return next(c)
		}
	}
}

// RequestLogger returns a middleware that logs HTTP requests.
func RequestLogger(config string) MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) (err error) {
			req := c.Request()
			res := c.Response()

			remoteAddr := req.RemoteAddress()
			if ip := req.Header().Get(HeaderXRealIP); ip != "" {
				remoteAddr = ip
			} else if ip = req.Header().Get(HeaderXForwardedFor); ip != "" {
				remoteAddr = ip
			} else {
				remoteAddr, _, _ = net.SplitHostPort(remoteAddr)
			}

			start := time.Now()
			if err := next(c); err != nil {
				c.Error(err)
			}
			stop := time.Now()
			method := req.Method()
			path := req.URL().Path()
			if path == "" {
				path = "/"
			}
			size := res.Size()

			n := res.Status()
			code := color.Green(n)
			switch {
			case n >= 500:
				code = color.Red(n)
			case n >= 400:
				code = color.Yellow(n)
			case n >= 300:
				code = color.Cyan(n)
			}

			logs.Debug("%s | %s | %s | %s | %s | %d", remoteAddr, method, path, code, stop.Sub(start), size)
			return nil
		}
	}
}

type (
	// RecoverConfig defines the config for recover middleware.
	RecoverConfig struct {
		// StackSize is the stack size to be printed.
		// Optional with default value as 4k.
		StackSize int

		// DisableStackAll disables formatting stack traces of all other goroutines
		// into buffer after the trace for the current goroutine.
		// Optional with default value as false.
		DisableStackAll bool

		// DisablePrintStack disables printing stack trace.
		// Optional with default value as false.
		DisablePrintStack bool
	}
)

// RecoverWithConfig returns a recover middleware from config.
// See `Recover()`.
func Recover(configJSON string) MiddlewareFunc {
	config := RecoverConfig{}
	json.Unmarshal([]byte(configJSON), &config)

	// Defaults
	if config.StackSize == 0 {
		config.StackSize = 4 << 10
	}

	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			defer func() {
				if r := recover(); r != nil {
					var err error
					switch r := r.(type) {
					case error:
						err = r
					default:
						err = fmt.Errorf("%v", r)
					}
					stack := make([]byte, config.StackSize)
					length := runtime.Stack(stack, !config.DisableStackAll)
					if !config.DisablePrintStack {
						c.Logger().Error("[%s] %s %s", color.Red("PANIC RECOVER"), err, stack[:length])
					}
					c.Error(err)
				}
			}()
			return next(c)
		}
	}
}

// 设置允许跨域
func CrossDomain(config string) MiddlewareFunc {
	return WrapMiddleware(func(c Context) error {
		if AppConfig.CrossDomain {
			c.Response().Header().Set("Access-Control-Allow-Origin", "*")
		}
		return nil
	})
}

func filterTemplate() MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) (err error) {
			ext := path.Ext(c.Request().URL().Path())
			if len(ext) >= 4 && ext[:4] == TPL_EXT {
				return c.NoContent(http.StatusForbidden)
			}
			return next(c)
		}
	}
}

func autoHTMLSuffix() MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) (err error) {
			p := c.Request().URL().Path()
			if p[len(p)-1] != '/' {
				ext := path.Ext(p)
				if ext == "" || ext[0] != '.' {
					c.Request().URL().SetPath(strings.TrimSuffix(p, ext) + STATIC_HTML_EXT + ext)
					c.ParamValues()[0] += STATIC_HTML_EXT
				}
			}
			return next(c)
		}
	}
}
<commit_msg>apimiddleware struct adds id attribute<commit_after>package lessgo

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"path"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/lessgo/lessgo/logs"
	"github.com/lessgo/lessgo/logs/color"
	"github.com/lessgo/lessgo/utils"
)

// 一旦注册，不可再更改
type (
	ApiMiddleware struct {
		Name          string // 全局唯一
		id            string
		Desc          string
		DefaultConfig interface{} // 默认配置(JSON格式)
		defaultConfig string      // 默认配置的JSON字符串
		Middleware    func(configJSON string) MiddlewareFunc
		inited        bool // 标记是否已经初始化过
	}
	// 虚拟路由中中间件配置信息
	MiddlewareConfig struct {
		Name   string `json:"name"`   // 全局唯一
		Config string `json:"config"` // JSON格式的配置
	}
)

func (a *ApiMiddleware) CreateMiddlewareFunc(config string) MiddlewareFunc {
	if config == "" {
		return a.Middleware(a.defaultConfig)
	}
	return a.Middleware(config)
}

// 注册中间件
func (a ApiMiddleware) Reg() *ApiMiddleware {
	return a.init()
}

// 初始化中间件，设置id并当Name为空时自动添加Name
func (a *ApiMiddleware) init() *ApiMiddleware {
	if a.inited {
		return getApiMiddleware(a.Name)
	}

	v := reflect.ValueOf(a.Middleware)
	funcName := runtime.FuncForPC(v.Pointer()).Name()
	if len(a.Name) == 0 {
		if len(a.Desc) > 0 {
			a.Name = a.Desc
		} else {
			a.Name = funcName
		}
	}

	if a.DefaultConfig != nil {
		b, _ := json.Marshal(a.DefaultConfig)
		a.defaultConfig = string(b)
	}

	a.id = utils.MakeHash(a.Name + funcName + a.defaultConfig)

	if m := getApiMiddleware(a.Name); m != nil {
		if m.id == a.id {
			return m
		} else {
			a.Name += "(2)"
			a.id = utils.MakeHash(a.Name + funcName + a.defaultConfig)
		}
	}

	a.inited = true

	setApiMiddleware(a)

	return a
}

var (
	apiMiddlewareMap  = map[string]*ApiMiddleware{}
	apiMiddlewareLock sync.RWMutex
)

func getApiMiddleware(name string) *ApiMiddleware {
	apiMiddlewareLock.RLock()
	defer apiMiddlewareLock.RUnlock()
	return apiMiddlewareMap[name]
}

func setApiMiddleware(vh *ApiMiddleware) {
	apiMiddlewareLock.Lock()
	defer apiMiddlewareLock.Unlock()
	apiMiddlewareMap[vh.Name] = vh
	for i, vh2 := range DefLessgo.apiMiddlewares {
		if vh.Name < vh2.Name {
			list := make([]*ApiMiddleware, len(DefLessgo.apiMiddlewares)+1)
			copy(list, DefLessgo.apiMiddlewares[:i])
			list[i] = vh
			copy(list[i+1:], DefLessgo.apiMiddlewares[i:])
			DefLessgo.apiMiddlewares = list
			return
		}
	}
	DefLessgo.apiMiddlewares = append(DefLessgo.apiMiddlewares, vh)
}

// 检查中间件是否存在
func isExistMiddlewares(middlewareConfigs ...MiddlewareConfig) error {
	var errstring string
	for _, m := range middlewareConfigs {
		_, ok := apiMiddlewareMap[m.Name]
		if !ok {
			errstring += " \"" + m.Name + "\""
		}
	}
	if len(errstring) == 0 {
		return nil
	}
	return fmt.Errorf("Specified below middlewares does not exist: %v\n", errstring)
}

// 根据中间件配置生成中间件
func createMiddlewareFuncs(configs []MiddlewareConfig) []MiddlewareFunc {
	mws := make([]MiddlewareFunc, len(configs))
	for i, mw := range configs {
		mws[i] = apiMiddlewareMap[mw.Name].CreateMiddlewareFunc(mw.Config)
	}
	return mws
}

/*
 * system middleware
 */
func init() {
	(&ApiMiddleware{
		Name:       "检查服务器是否启用",
		Desc:       "检查服务器是否启用",
		Middleware: CheckServer,
	}).init()

	(&ApiMiddleware{
		Name:       "检查是否为访问主页",
		Desc:       "检查是否为访问主页",
		Middleware: CheckHome,
	}).init()

	(&ApiMiddleware{
		Name:       "系统运行日志打印",
		Desc:       "RequestLogger returns a middleware that logs HTTP requests.",
		Middleware: RequestLogger,
	}).init()

	(&ApiMiddleware{
		Name: "捕获运行时恐慌",
		Desc: "Recover returns a middleware which recovers from panics anywhere in the chain and handles the control to the centralized HTTPErrorHandler.",
		DefaultConfig: RecoverConfig{
			StackSize:         4 << 10, // 4 KB
			DisableStackAll:   false,
			DisablePrintStack: false,
		},
		Middleware: Recover,
	}).init()

	(&ApiMiddleware{
		Name:       "设置允许跨域",
		Desc:       "根据配置信息设置允许跨域",
		Middleware: CrossDomain,
	}).init()

	// 系统预设中间件
	PreUse(
		MiddlewareConfig{Name: "检查服务器是否启用"},
		MiddlewareConfig{Name: "检查是否为访问主页"},
		MiddlewareConfig{Name: "系统运行日志打印"},
		MiddlewareConfig{Name: "捕获运行时恐慌"},
		MiddlewareConfig{Name: "设置允许跨域"},
	)
}

// 检查服务器是否启用
func CheckServer(config string) MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			if !ServerEnable() {
				return c.NoContent(http.StatusServiceUnavailable)
			}
			return next(c)
		}
	}
}

// 检查是否为访问主页
func CheckHome(config string) MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			if c.Request().URL().Path() == "/" {
				c.Request().URL().SetPath(GetHome())
			}
			return next(c)
		}
	}
}

// RequestLogger returns a middleware that logs HTTP requests.
func RequestLogger(config string) MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) (err error) {
			req := c.Request()
			res := c.Response()

			remoteAddr := req.RemoteAddress()
			if ip := req.Header().Get(HeaderXRealIP); ip != "" {
				remoteAddr = ip
			} else if ip = req.Header().Get(HeaderXForwardedFor); ip != "" {
				remoteAddr = ip
			} else {
				remoteAddr, _, _ = net.SplitHostPort(remoteAddr)
			}

			start := time.Now()
			if err := next(c); err != nil {
				c.Error(err)
			}
			stop := time.Now()
			method := req.Method()
			path := req.URL().Path()
			if path == "" {
				path = "/"
			}
			size := res.Size()

			n := res.Status()
			code := color.Green(n)
			switch {
			case n >= 500:
				code = color.Red(n)
			case n >= 400:
				code = color.Yellow(n)
			case n >= 300:
				code = color.Cyan(n)
			}

			logs.Debug("%s | %s | %s | %s | %s | %d", remoteAddr, method, path, code, stop.Sub(start), size)
			return nil
		}
	}
}

type (
	// RecoverConfig defines the config for recover middleware.
	RecoverConfig struct {
		// StackSize is the stack size to be printed.
		// Optional with default value as 4k.
		StackSize int

		// DisableStackAll disables formatting stack traces of all other goroutines
		// into buffer after the trace for the current goroutine.
		// Optional with default value as false.
		DisableStackAll bool

		// DisablePrintStack disables printing stack trace.
		// Optional with default value as false.
		DisablePrintStack bool
	}
)

// RecoverWithConfig returns a recover middleware from config.
// See `Recover()`.
func Recover(configJSON string) MiddlewareFunc {
	config := RecoverConfig{}
	json.Unmarshal([]byte(configJSON), &config)

	// Defaults
	if config.StackSize == 0 {
		config.StackSize = 4 << 10
	}

	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			defer func() {
				if r := recover(); r != nil {
					var err error
					switch r := r.(type) {
					case error:
						err = r
					default:
						err = fmt.Errorf("%v", r)
					}
					stack := make([]byte, config.StackSize)
					length := runtime.Stack(stack, !config.DisableStackAll)
					if !config.DisablePrintStack {
						c.Logger().Error("[%s] %s %s", color.Red("PANIC RECOVER"), err, stack[:length])
					}
					c.Error(err)
				}
			}()
			return next(c)
		}
	}
}

// 设置允许跨域
func CrossDomain(config string) MiddlewareFunc {
	return WrapMiddleware(func(c Context) error {
		if AppConfig.CrossDomain {
			c.Response().Header().Set("Access-Control-Allow-Origin", "*")
		}
		return nil
	})
}

func filterTemplate() MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) (err error) {
			ext := path.Ext(c.Request().URL().Path())
			if len(ext) >= 4 && ext[:4] == TPL_EXT {
				return c.NoContent(http.StatusForbidden)
			}
			return next(c)
		}
	}
}

func autoHTMLSuffix() MiddlewareFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) (err error) {
			p := c.Request().URL().Path()
			if p[len(p)-1] != '/' {
				ext := path.Ext(p)
				if ext == "" || ext[0] != '.' {
					c.Request().URL().SetPath(strings.TrimSuffix(p, ext) + STATIC_HTML_EXT + ext)
					c.ParamValues()[0] += STATIC_HTML_EXT
				}
			}
			return next(c)
		}
	}
}
<|endoftext|>


<commit_before>package daemon

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"sync"
	"time"
)

type Config struct {
	Interfaces []string      `type:"arg" min:"1"`
	Interval   time.Duration `help:"the interval between scans"`
	Endpoint   string        `help:"an HTTP POST endpoint (defaults to send to stdout)"`
	SingleCore bool          `help:"only use a single core (defaults to multicore)"`
}

func Run(c Config) {

	//logs to stderr
	log.SetOutput(os.Stderr)

	//get ip
	endpoint, err := url.Parse(c.Endpoint)
	if c.Endpoint == "" {
		endpoint = nil
	} else if err != nil {
		log.Fatal("Invalid endpoint: %s", err)
	}

	cpu := runtime.NumCPU()
	runtime.GOMAXPROCS(cpu)
	log.Printf("whos-home initializing (#%d cores)", cpu)

	//get all in list
	ifaces := []*net.Interface{}
	for _, n := range c.Interfaces {
		iface, err := net.InterfaceByName(n)
		if err != nil {
			log.Printf("could not get interface %s: %s", n, err)
			return
		}
		ifaces = append(ifaces, iface)
	}

	//prepare queue
	queue := make(NodeQueue)

	//in a goroutine, extract all nodes from queue
	go monitor(endpoint, queue)

	//scan all provided interfaces, append all to queue
	var wg sync.WaitGroup
	for _, iface := range ifaces {
		wg.Add(1)
		// Start up a scan on each interface.
		func(iface *net.Interface) {
			defer wg.Done()
			if err := scan(iface, c.Interval, queue); err != nil {
				log.Printf("%s error: %s", iface.Name, err)
			}
		}(iface)
	}

	wg.Wait()
	return
}

func monitor(endpoint *url.URL, queue NodeQueue) {
	//collect unique nodes (thread-safe)
	l := sync.Mutex{}
	nodes := NodeSet{}
	//send out all nodes every 5 seconds
	go func() {
		for {
			l.Lock()
			send(endpoint, nodes)
			for k, _ := range nodes {
				delete(nodes, k)
			}
			l.Unlock()
			time.Sleep(5 * time.Second)
		}
		time.Sleep(time.Second)
	}()
	//fill nodes map forever
	for node := range queue {
		l.Lock()
		nodes[node.MAC.String()] = node.IP.String()
		l.Unlock()
	}
}

var DNScache = map[string]string{}

func send(endpoint *url.URL, nodes NodeSet) {

	if len(nodes) == 0 {
		return
	}

	b, err := json.MarshalIndent(nodes, "", "  ")

	//no endpoint, send to stdout
	if endpoint == nil {
		os.Stdout.Write(b)
		os.Stdout.WriteString("\n")
		return
	}

	req, _ := http.NewRequest("POST", endpoint.String(), bytes.NewReader(b))
	req.Header.Set("Content-Type", "application/json")

	//hook dns
	transport := http.Transport{
		DialTLS: func(network, addr string) (net.Conn, error) {
			if network != "tcp" {
				return nil, errors.New("unsupported network")
			}
			h, p, _ := net.SplitHostPort(addr)
			ip, ok := DNScache[h]
			if !ok {
				ips, err := net.LookupIP(h)
				if err != nil {
					return nil, fmt.Errorf("DNS lookup failed: %s", err)
				}
				ip := ips[0].To4().String()
				log.Printf("DNS lookup: %s -> %s", h, ip)
				DNScache[h] = ip
			}
			return tls.Dial(network, ip+":"+p, &tls.Config{
				ServerName: h,
			})
		},
	}

	resp, err := transport.RoundTrip(req)
	if err != nil {
		log.Printf("send failed: %s", err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		b, _ := ioutil.ReadAll(resp.Body)
		log.Printf("send error: %d: %s", resp.StatusCode, b)
		return
	}
}
<commit_msg>dns caching is now optional<commit_after>package daemon

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"sync"
	"time"
)

type Config struct {
	Interfaces []string      `type:"arg" min:"1"`
	Interval   time.Duration `help:"the interval between scans"`
	Endpoint   string        `help:"an HTTP POST endpoint (defaults to send to stdout)"`
	SingleCore bool          `help:"only use a single core (defaults to multicore)"`
	CacheDNS   bool          `help:"cache dns responses"`
}

var DNScache = map[string]string(nil)

func Run(c Config) {

	//logs to stderr
	log.SetOutput(os.Stderr)

	if c.CacheDNS {
		DNScache = map[string]string{}
	}

	//get ip
	endpoint, err := url.Parse(c.Endpoint)
	if c.Endpoint == "" {
		endpoint = nil
	} else if err != nil {
		log.Fatal("Invalid endpoint: %s", err)
	}

	cpu := runtime.NumCPU()
	runtime.GOMAXPROCS(cpu)
	log.Printf("whos-home initializing (#%d cores)", cpu)

	//get all in list
	ifaces := []*net.Interface{}
	for _, n := range c.Interfaces {
		iface, err := net.InterfaceByName(n)
		if err != nil {
			log.Printf("could not get interface %s: %s", n, err)
			return
		}
		ifaces = append(ifaces, iface)
	}

	//prepare queue
	queue := make(NodeQueue)

	//in a goroutine, extract all nodes from queue
	go monitor(endpoint, queue)

	//scan all provided interfaces, append all to queue
	var wg sync.WaitGroup
	for _, iface := range ifaces {
		wg.Add(1)
		// Start up a scan on each interface.
		func(iface *net.Interface) {
			defer wg.Done()
			if err := scan(iface, c.Interval, queue); err != nil {
				log.Printf("%s error: %s", iface.Name, err)
			}
		}(iface)
	}

	wg.Wait()
	return
}

func monitor(endpoint *url.URL, queue NodeQueue) {
	//collect unique nodes (thread-safe)
	l := sync.Mutex{}
	nodes := NodeSet{}
	//send out all nodes every 5 seconds
	go func() {
		for {
			l.Lock()
			send(endpoint, nodes)
			for k, _ := range nodes {
				delete(nodes, k)
			}
			l.Unlock()
			time.Sleep(5 * time.Second)
		}
		time.Sleep(time.Second)
	}()
	//fill nodes map forever
	for node := range queue {
		l.Lock()
		nodes[node.MAC.String()] = node.IP.String()
		l.Unlock()
	}
}

func send(endpoint *url.URL, nodes NodeSet) {

	if len(nodes) == 0 {
		return
	}

	b, err := json.MarshalIndent(nodes, "", "  ")

	//no endpoint, send to stdout
	if endpoint == nil {
		os.Stdout.Write(b)
		os.Stdout.WriteString("\n")
		return
	}

	req, _ := http.NewRequest("POST", endpoint.String(), bytes.NewReader(b))
	req.Header.Set("Content-Type", "application/json")

	//cache dns?
	var client *http.Client
	if DNScache == nil {
		client = http.DefaultClient
	} else {
		client = dnsCachingClient
	}

	resp, err := client.Do(req)
	if err != nil {
		log.Printf("send failed: %s", err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		b, _ := ioutil.ReadAll(resp.Body)
		log.Printf("send error: %d: %s", resp.StatusCode, b)
		return
	}
}

var dnsCachingClient = &http.Client{
	Transport: &http.Transport{
		DialTLS: func(network, addr string) (net.Conn, error) {
			if network != "tcp" {
				return nil, errors.New("unsupported network")
			}
			h, p, _ := net.SplitHostPort(addr)
			ip, ok := DNScache[h]
			if !ok {
				ips, err := net.LookupIP(h)
				if err != nil {
					return nil, fmt.Errorf("DNS lookup failed: %s", err)
				}
				ip := ips[0].To4().String()
				log.Printf("DNS lookup: %s -> %s", h, ip)
				DNScache[h] = ip
			}
			return tls.Dial(network, ip+":"+p, &tls.Config{
				ServerName: h,
			})
		},
	},
}
<|endoftext|>


<commit_before>package daemon

import (
	"bytes"
	"encoding/json"
	"log"
	"net"
	"net/http"
	"os"
	"runtime"
	"sync"
	"time"
)

type Config struct {
	Interfaces []string      `type:"arg" min:"1"`
	Interval   time.Duration `help:"the interval between scans"`
	Endpoint   string        `help:"an HTTP POST endpoint (defaults to send to stdout)"`
	SingleCore bool          `help:"only use a single core (defaults to multicore)"`
}

func Run(c Config) {

	//logs to stderr
	log.SetOutput(os.Stderr)

	cpu := runtime.NumCPU()
	runtime.GOMAXPROCS(cpu)
	log.Printf("whos-home initializing (#%d cores)", cpu)

	//get all in list
	ifaces := []*net.Interface{}
	for _, n := range c.Interfaces {
		iface, err := net.InterfaceByName(n)
		if err != nil {
			log.Printf("could not get interface %s: %s", n, err)
			return
		}
		ifaces = append(ifaces, iface)
	}

	//prepare queue
	queue := make(NodeQueue)

	//in a goroutine, extract all nodes from queue
	go monitor(c.Endpoint, queue)

	//scan all provided interfaces, append all to queue
	var wg sync.WaitGroup
	for _, iface := range ifaces {
		wg.Add(1)
		// Start up a scan on each interface.
		func(iface *net.Interface) {
			defer wg.Done()
			if err := scan(iface, c.Interval, queue); err != nil {
				log.Printf("%s error: %s", iface.Name, err)
			}
		}(iface)
	}

	wg.Wait()
	return
}

func monitor(endpoint string, queue NodeQueue) {
	//collect unique nodes (thread-safe)
	l := sync.Mutex{}
	nodes := NodeSet{}
	//send out all nodes every 5 seconds
	go func() {
		for {
			l.Lock()
			send(endpoint, nodes)
			for k, _ := range nodes {
				delete(nodes, k)
			}
			l.Unlock()
			time.Sleep(5 * time.Second)
		}
		time.Sleep(time.Second)
	}()
	//fill nodes map forever
	for node := range queue {
		l.Lock()
		nodes[node.MAC.String()] = node.IP.String()
		l.Unlock()
	}
}

func send(endpoint string, nodes NodeSet) {

	if len(nodes) == 0 {
		return
	}

	b, err := json.MarshalIndent(nodes, "", "  ")

	//no endpoint, send to stdout
	if endpoint == "" {
		os.Stdout.Write(b)
		os.Stdout.WriteString("\n")
		return
	}

	resp, err := http.Post(endpoint, "application/json", bytes.NewBuffer(b))
	if err != nil {
		log.Printf("send failed: %s", err)
		return
	}

	if resp.StatusCode != 200 {
		log.Printf("send error: %d", resp.StatusCode)
		return
	}
}
<commit_msg>running into https://github.com/golang/go/issues/3575, added dns caching<commit_after>package daemon

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"sync"
	"time"
)

type Config struct {
	Interfaces []string      `type:"arg" min:"1"`
	Interval   time.Duration `help:"the interval between scans"`
	Endpoint   string        `help:"an HTTP POST endpoint (defaults to send to stdout)"`
	SingleCore bool          `help:"only use a single core (defaults to multicore)"`
}

func Run(c Config) {

	//logs to stderr
	log.SetOutput(os.Stderr)

	//get ip
	endpoint, err := url.Parse(c.Endpoint)
	if c.Endpoint != "" && err != nil {
		log.Fatal("Invalid endpoint: %s", err)
	}

	cpu := runtime.NumCPU()
	runtime.GOMAXPROCS(cpu)
	log.Printf("whos-home initializing (#%d cores)", cpu)

	//get all in list
	ifaces := []*net.Interface{}
	for _, n := range c.Interfaces {
		iface, err := net.InterfaceByName(n)
		if err != nil {
			log.Printf("could not get interface %s: %s", n, err)
			return
		}
		ifaces = append(ifaces, iface)
	}

	//prepare queue
	queue := make(NodeQueue)

	//in a goroutine, extract all nodes from queue
	go monitor(endpoint, queue)

	//scan all provided interfaces, append all to queue
	var wg sync.WaitGroup
	for _, iface := range ifaces {
		wg.Add(1)
		// Start up a scan on each interface.
		func(iface *net.Interface) {
			defer wg.Done()
			if err := scan(iface, c.Interval, queue); err != nil {
				log.Printf("%s error: %s", iface.Name, err)
			}
		}(iface)
	}

	wg.Wait()
	return
}

func monitor(endpoint *url.URL, queue NodeQueue) {
	//collect unique nodes (thread-safe)
	l := sync.Mutex{}
	nodes := NodeSet{}
	//send out all nodes every 5 seconds
	go func() {
		for {
			l.Lock()
			send(endpoint, nodes)
			for k, _ := range nodes {
				delete(nodes, k)
			}
			l.Unlock()
			time.Sleep(5 * time.Second)
		}
		time.Sleep(time.Second)
	}()
	//fill nodes map forever
	for node := range queue {
		l.Lock()
		nodes[node.MAC.String()] = node.IP.String()
		l.Unlock()
	}
}

var DNScache = map[string]string{}

func send(endpoint *url.URL, nodes NodeSet) {

	if len(nodes) == 0 {
		return
	}

	b, err := json.MarshalIndent(nodes, "", "  ")

	//no endpoint, send to stdout
	if endpoint == nil {
		os.Stdout.Write(b)
		os.Stdout.WriteString("\n")
		return
	}

	req, _ := http.NewRequest("POST", endpoint.String(), bytes.NewReader(b))
	req.Header.Set("Content-Type", "application/json")

	//hook dns
	transport := http.Transport{
		DialTLS: func(network, addr string) (net.Conn, error) {
			if network != "tcp" {
				return nil, errors.New("unsupported network")
			}
			h, p, _ := net.SplitHostPort(addr)
			ip, ok := DNScache[h]
			if !ok {
				ips, err := net.LookupIP(h)
				if err != nil {
					return nil, fmt.Errorf("DNS lookup failed: %s", err)
				}
				ip := ips[0].To4().String()
				log.Printf("DNS lookup: %s -> %s", h, ip)
				DNScache[h] = ip
			}
			return tls.Dial(network, ip+":"+p, &tls.Config{
				ServerName: h,
			})
		},
	}

	resp, err := transport.RoundTrip(req)
	if err != nil {
		log.Printf("send failed: %s", err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		b, _ := ioutil.ReadAll(resp.Body)
		log.Printf("send error: %d: %s", resp.StatusCode, b)
		return
	}
}
<|endoftext|>


<commit_before>package chaos_cluster

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"reflect"
	"testing"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/grafana/metrictank/logger"
	"github.com/grafana/metrictank/stacktest/docker"
	"github.com/grafana/metrictank/stacktest/fakemetrics"
	"github.com/grafana/metrictank/stacktest/grafana"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stacktest/track"
	"github.com/grafana/metrictank/test"
	log "github.com/sirupsen/logrus"
)

// TODO: cleanup when ctrl-C go test (teardown all containers)

const numPartitions = 12

var tracker *track.Tracker
var fm *fakemetrics.FakeMetrics

func init() {
	formatter := &logger.TextFormatter{}
	formatter.TimestampFormat = "2006-01-02 15:04:05.000"
	log.SetFormatter(formatter)
	log.SetLevel(log.InfoLevel)
}
func TestMain(m *testing.M) {
	flag.Parse()
	if testing.Short() {
		fmt.Println("skipping chaos cluster test in short mode")
		return
	}
	ctx, cancelFunc := context.WithCancel(context.Background())

	fmt.Println("stopping docker-chaos stack should it be running...")
	cmd := exec.CommandContext(ctx, "docker-compose", "down")
	cmd.Dir = test.Path("docker/docker-chaos")
	err := cmd.Start()
	if err != nil {
		log.Fatal(err.Error())
	}

	fmt.Println("launching docker-chaos stack...")
	cmd = exec.CommandContext(ctx, "docker-compose", "up", "--force-recreate", "-V")
	cmd.Dir = test.Path("docker/docker-chaos")
	cmd.Env = append(cmd.Env, "MT_CLUSTER_MIN_AVAILABLE_SHARDS=12")

	tracker, err = track.NewTracker(cmd, false, false, "launch-stdout", "launch-stderr")
	if err != nil {
		log.Fatal(err.Error())
	}

	err = cmd.Start()
	if err != nil {
		log.Fatal(err.Error())
	}

	retcode := m.Run()
	fm.Close()

	fmt.Println("stopping docker-compose stack...")
	cancelFunc()
	if err := cmd.Wait(); err != nil {
		log.Printf("ERROR: could not cleanly shutdown running docker-compose command: %s", err)
		retcode = 1
	}

	os.Exit(retcode)
}

func TestClusterStartup(t *testing.T) {
	matchers := []track.Matcher{
		{Str: "metrictank0_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank1_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank2_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank3_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank4_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank5_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "grafana.*HTTP Server Listen.*3000"},
	}
	select {
	case <-tracker.Match(matchers):
		fmt.Println("stack now running.")
		fmt.Println("Go to http://localhost:3000 (and login as admin:admin) to see what's going on")
	case <-time.After(time.Second * 40):
		grafana.PostAnnotation("TestClusterStartup:FAIL")
		t.Fatal("timed out while waiting for all metrictank instances to come up")
	}
}

func TestClusterBaseIngestWorkload(t *testing.T) {
	grafana.PostAnnotation("TestClusterBaseIngestWorkload:begin")

	fm = fakemetrics.NewKafka(numPartitions)

	req := graphite.RequestForLocalTestingGraphite("perSecond(metrictank.stats.docker-cluster.*.input.kafka-mdm.metrics_received.counter32)", "-8s")
	resp, ok := graphite.Retry(req, 18, func(resp graphite.Response) bool {
		exp := []string{
			"perSecond(metrictank.stats.docker-cluster.metrictank0.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank1.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank2.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank3.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank4.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank5.input.kafka-mdm.metrics_received.counter32)",
		}
		// avg rate must be 4 (metrics ingested per second by each instance)
		return graphite.ValidateTargets(exp)(resp) && graphite.ValidatorAvgWindowed(8, graphite.Eq(4))(resp)
	})
	if !ok {
		grafana.PostAnnotation("TestClusterBaseIngestWorkload:FAIL")
		t.Fatalf("cluster did not reach a state where each MT instance receives 4 points per second. last response was: %s", spew.Sdump(resp))
	}

	req = graphite.RequestForLocalTestingMT("sum(some.id.of.a.metric.*)", "-16s")
	resp, ok = graphite.Retry(req, 20, graphite.ValidateCorrect(12))
	if !ok {
		grafana.PostAnnotation("TestClusterBaseIngestWorkload:FAIL")
		t.Fatalf("could not query correct result set. sum of 12 series, each valued 1, should result in 12.  last response was: %s", spew.Sdump(resp))
	}
}

func TestQueryWorkload(t *testing.T) {
	grafana.PostAnnotation("TestQueryWorkload:begin")

	results := graphite.CheckMT([]int{6060, 6061, 6062, 6063, 6064, 6065}, "sum(some.id.of.a.metric.*)", "-14s", time.Minute, 6000, graphite.ValidateCorrect(12))

	exp := graphite.CheckResults{
		Valid:   []int{6000},
		Empty:   0,
		Timeout: 0,
		Other:   0,
	}
	if !reflect.DeepEqual(exp, results) {
		grafana.PostAnnotation("TestQueryWorkload:FAIL")
		t.Fatalf("expected only correct results. got %s", spew.Sdump(results))
	}
}

// TestIsolateOneInstance tests what happens during the isolation of one instance, when min-available-shards is 12
// this should happen:
// at all times, all queries to all of the remaining nodes should be successful
// since they have at least 1 instance running for each shard.
// the isolated shard should either return correct replies, or errors (in two cases: when it marks any shards as down,
// but also before it does, but fails to get data via clustered requests from peers)
func TestIsolateOneInstance(t *testing.T) {
	grafana.PostAnnotation("TestIsolateOneInstance:begin")
	numReqMt4 := 1200

	mt4ResultsChan := make(chan graphite.CheckResults, 1)
	otherResultsChan := make(chan graphite.CheckResults, 1)

	go func() {
		mt4ResultsChan <- graphite.CheckMT([]int{6064}, "sum(some.id.of.a.metric.*)", "-15s", time.Minute, numReqMt4, graphite.ValidateCorrect(12), graphite.ValidateCode(503))
	}()
	go func() {
		otherResultsChan <- graphite.CheckMT([]int{6060, 6061, 6062, 6063, 6065}, "sum(some.id.of.a.metric.*)", "-15s", time.Minute, 6000, graphite.ValidateCorrect(12))
	}()

	// now go ahead and isolate for 30s
	docker.Isolate([]string{"metrictank4"}, []string{"metrictank0", "metrictank1", "metrictank2", "metrictank3", "metrictank5"}, "30s")

	// collect results of the minute long experiment
	mt4Results := <-mt4ResultsChan
	otherResults := <-otherResultsChan

	// validate results of isolated node
	if mt4Results.Valid[0]+mt4Results.Valid[1] != numReqMt4 {
		t.Fatalf("expected mt4 to return either correct or erroring responses (total %d). got %s", numReqMt4, spew.Sdump(mt4Results))
	}
	if mt4Results.Valid[1] < numReqMt4*30/100 {
		// the instance is completely down for 30s of the 60s experiment run, but we allow some slack
		t.Fatalf("expected at least 30%% of all mt4 results to succeed. did %d queries. got %s", numReqMt4, spew.Sdump(mt4Results))
	}

	// validate results of other cluster nodes
	exp := graphite.CheckResults{
		Valid:   []int{6000},
		Empty:   0,
		Timeout: 0,
		Other:   0,
	}
	if !reflect.DeepEqual(exp, otherResults) {
		grafana.PostAnnotation("TestIsolateOneInstance:FAIL")
		t.Fatalf("expected only correct results for all cluster nodes. got %s", spew.Sdump(otherResults))
	}
}

func TestHang(t *testing.T) {
	grafana.PostAnnotation("TestHang:begin")
	t.Log("whatever happens, keep hanging for now, so that we can query grafana dashboards still")
	var ch chan struct{}
	<-ch
}

// maybe useful in the future, test also clean exit and rejoin like so:
//stop("metrictank4")
//time.AfterFunc(30*time.Second, func() {
//	start("metrictank4")
//})
<commit_msg>bugfix: wait til docker-compose down is done before docker-compose up!<commit_after>package chaos_cluster

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"reflect"
	"testing"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/grafana/metrictank/logger"
	"github.com/grafana/metrictank/stacktest/docker"
	"github.com/grafana/metrictank/stacktest/fakemetrics"
	"github.com/grafana/metrictank/stacktest/grafana"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stacktest/track"
	"github.com/grafana/metrictank/test"
	log "github.com/sirupsen/logrus"
)

// TODO: cleanup when ctrl-C go test (teardown all containers)

const numPartitions = 12

var tracker *track.Tracker
var fm *fakemetrics.FakeMetrics

func init() {
	formatter := &logger.TextFormatter{}
	formatter.TimestampFormat = "2006-01-02 15:04:05.000"
	log.SetFormatter(formatter)
	log.SetLevel(log.InfoLevel)
}
func TestMain(m *testing.M) {
	flag.Parse()
	if testing.Short() {
		fmt.Println("skipping chaos cluster test in short mode")
		return
	}
	ctx, cancelFunc := context.WithCancel(context.Background())

	fmt.Println("stopping docker-chaos stack should it be running...")
	cmd := exec.CommandContext(ctx, "docker-compose", "down")
	cmd.Dir = test.Path("docker/docker-chaos")
	err := cmd.Start()
	if err != nil {
		log.Fatal(err.Error())
	}
	err = cmd.Wait()
	if err != nil {
		log.Fatal(err.Error())
	}

	fmt.Println("launching docker-chaos stack...")
	cmd = exec.CommandContext(ctx, "docker-compose", "up", "--force-recreate", "-V")
	cmd.Dir = test.Path("docker/docker-chaos")
	cmd.Env = append(cmd.Env, "MT_CLUSTER_MIN_AVAILABLE_SHARDS=12")

	tracker, err = track.NewTracker(cmd, false, false, "launch-stdout", "launch-stderr")
	if err != nil {
		log.Fatal(err.Error())
	}

	err = cmd.Start()
	if err != nil {
		log.Fatal(err.Error())
	}

	retcode := m.Run()
	fm.Close()

	fmt.Println("stopping docker-compose stack...")
	cancelFunc()
	if err := cmd.Wait(); err != nil {
		log.Printf("ERROR: could not cleanly shutdown running docker-compose command: %s", err)
		retcode = 1
	}

	os.Exit(retcode)
}

func TestClusterStartup(t *testing.T) {
	matchers := []track.Matcher{
		{Str: "metrictank0_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank1_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank2_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank3_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank4_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank5_1.*metricIndex initialized.*starting data consumption$"},
		{Str: "grafana.*HTTP Server Listen.*3000"},
	}
	select {
	case <-tracker.Match(matchers):
		fmt.Println("stack now running.")
		fmt.Println("Go to http://localhost:3000 (and login as admin:admin) to see what's going on")
	case <-time.After(time.Second * 40):
		grafana.PostAnnotation("TestClusterStartup:FAIL")
		t.Fatal("timed out while waiting for all metrictank instances to come up")
	}
}

func TestClusterBaseIngestWorkload(t *testing.T) {
	grafana.PostAnnotation("TestClusterBaseIngestWorkload:begin")

	fm = fakemetrics.NewKafka(numPartitions)

	req := graphite.RequestForLocalTestingGraphite("perSecond(metrictank.stats.docker-cluster.*.input.kafka-mdm.metrics_received.counter32)", "-8s")
	resp, ok := graphite.Retry(req, 18, func(resp graphite.Response) bool {
		exp := []string{
			"perSecond(metrictank.stats.docker-cluster.metrictank0.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank1.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank2.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank3.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank4.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank5.input.kafka-mdm.metrics_received.counter32)",
		}
		// avg rate must be 4 (metrics ingested per second by each instance)
		return graphite.ValidateTargets(exp)(resp) && graphite.ValidatorAvgWindowed(8, graphite.Eq(4))(resp)
	})
	if !ok {
		grafana.PostAnnotation("TestClusterBaseIngestWorkload:FAIL")
		t.Fatalf("cluster did not reach a state where each MT instance receives 4 points per second. last response was: %s", spew.Sdump(resp))
	}

	req = graphite.RequestForLocalTestingMT("sum(some.id.of.a.metric.*)", "-16s")
	resp, ok = graphite.Retry(req, 20, graphite.ValidateCorrect(12))
	if !ok {
		grafana.PostAnnotation("TestClusterBaseIngestWorkload:FAIL")
		t.Fatalf("could not query correct result set. sum of 12 series, each valued 1, should result in 12.  last response was: %s", spew.Sdump(resp))
	}
}

func TestQueryWorkload(t *testing.T) {
	grafana.PostAnnotation("TestQueryWorkload:begin")

	results := graphite.CheckMT([]int{6060, 6061, 6062, 6063, 6064, 6065}, "sum(some.id.of.a.metric.*)", "-14s", time.Minute, 6000, graphite.ValidateCorrect(12))

	exp := graphite.CheckResults{
		Valid:   []int{6000},
		Empty:   0,
		Timeout: 0,
		Other:   0,
	}
	if !reflect.DeepEqual(exp, results) {
		grafana.PostAnnotation("TestQueryWorkload:FAIL")
		t.Fatalf("expected only correct results. got %s", spew.Sdump(results))
	}
}

// TestIsolateOneInstance tests what happens during the isolation of one instance, when min-available-shards is 12
// this should happen:
// at all times, all queries to all of the remaining nodes should be successful
// since they have at least 1 instance running for each shard.
// the isolated shard should either return correct replies, or errors (in two cases: when it marks any shards as down,
// but also before it does, but fails to get data via clustered requests from peers)
func TestIsolateOneInstance(t *testing.T) {
	grafana.PostAnnotation("TestIsolateOneInstance:begin")
	numReqMt4 := 1200

	mt4ResultsChan := make(chan graphite.CheckResults, 1)
	otherResultsChan := make(chan graphite.CheckResults, 1)

	go func() {
		mt4ResultsChan <- graphite.CheckMT([]int{6064}, "sum(some.id.of.a.metric.*)", "-15s", time.Minute, numReqMt4, graphite.ValidateCorrect(12), graphite.ValidateCode(503))
	}()
	go func() {
		otherResultsChan <- graphite.CheckMT([]int{6060, 6061, 6062, 6063, 6065}, "sum(some.id.of.a.metric.*)", "-15s", time.Minute, 6000, graphite.ValidateCorrect(12))
	}()

	// now go ahead and isolate for 30s
	docker.Isolate([]string{"metrictank4"}, []string{"metrictank0", "metrictank1", "metrictank2", "metrictank3", "metrictank5"}, "30s")

	// collect results of the minute long experiment
	mt4Results := <-mt4ResultsChan
	otherResults := <-otherResultsChan

	// validate results of isolated node
	if mt4Results.Valid[0]+mt4Results.Valid[1] != numReqMt4 {
		t.Fatalf("expected mt4 to return either correct or erroring responses (total %d). got %s", numReqMt4, spew.Sdump(mt4Results))
	}
	if mt4Results.Valid[1] < numReqMt4*30/100 {
		// the instance is completely down for 30s of the 60s experiment run, but we allow some slack
		t.Fatalf("expected at least 30%% of all mt4 results to succeed. did %d queries. got %s", numReqMt4, spew.Sdump(mt4Results))
	}

	// validate results of other cluster nodes
	exp := graphite.CheckResults{
		Valid:   []int{6000},
		Empty:   0,
		Timeout: 0,
		Other:   0,
	}
	if !reflect.DeepEqual(exp, otherResults) {
		grafana.PostAnnotation("TestIsolateOneInstance:FAIL")
		t.Fatalf("expected only correct results for all cluster nodes. got %s", spew.Sdump(otherResults))
	}
}

func TestHang(t *testing.T) {
	grafana.PostAnnotation("TestHang:begin")
	t.Log("whatever happens, keep hanging for now, so that we can query grafana dashboards still")
	var ch chan struct{}
	<-ch
}

// maybe useful in the future, test also clean exit and rejoin like so:
//stop("metrictank4")
//time.AfterFunc(30*time.Second, func() {
//	start("metrictank4")
//})
<|endoftext|>


<commit_before>package main

import "fmt"

// main creates a square matrix of size (n*n) and fills it in a circular fashion
func main() {
	var gridDim int
	fmt.Print("Enter circular array size: ")
	fmt.Scan(&gridDim)

	var cellValue int = 1
	var col1, col2, row1, row2 int = 0, gridDim - 1, 0, gridDim - 1
	var gridLen int = gridDim * gridDim

	// allocate composed 2d array
	grid := make([][]int, gridDim)
	for i := range grid {
		grid[i] = make([]int, gridDim)
	}

	for cellValue <= gridLen {
		for i := col1; i <= col2; i++ {
			grid[row1][i] = cellValue
			cellValue++
		}

		for j := row1 + 1; j <= row2; j++ {
			grid[j][col2] = cellValue
			cellValue++
		}

		for i := col2 - 1; i >= col1; i-- {
			grid[row2][i] = cellValue
			cellValue++
		}

		for j := row2 - 1; j >= row1+1; j-- {
			grid[j][col1] = cellValue
			cellValue++
		}

		col1++
		col2--
		row1++
		row2--
	}

	/* printing the circular matrix */
	println("Here is your circular matrix is:")
	for i := 0; i < gridDim; i++ {
		for j := 0; j < gridDim; j++ {
			fmt.Printf("%d\t", grid[i][j])
		}
		println()
	}

}<commit_msg>Renamed vars to enhance clarity, added some newlines<commit_after>package main

import "fmt"

// main creates a square matrix of size (n*n) and fills it in a circular fashion
func main() {
	var gridDim int
	fmt.Print("\nEnter circular matrix size: ")
	fmt.Scan(&gridDim)

	var cellValue int = 1
	var col_begin, col_end, row_begin, row_end int = 0, gridDim - 1, 0, gridDim - 1
	var gridLen int = gridDim * gridDim

	// allocate composed 2d array
	grid := make([][]int, gridDim)
	for i := range grid {
		grid[i] = make([]int, gridDim)
	}

	for cellValue <= gridLen {
		for i := col_begin; i <= col_end; i++ {
			grid[row_begin][i] = cellValue
			cellValue++
		}

		for j := row_begin + 1; j <= row_end; j++ {
			grid[j][col_end] = cellValue
			cellValue++
		}

		for i := col_end - 1; i >= col_begin; i-- {
			grid[row_end][i] = cellValue
			cellValue++
		}

		for j := row_end - 1; j >= row_begin +1; j-- {
			grid[j][col_begin] = cellValue
			cellValue++
		}

		col_begin++
		col_end--
		row_begin++
		row_end--
	}

	/* printing the circular matrix */
	println("\nYour circular matrix is:")
	for i := 0; i < gridDim; i++ {
		for j := 0; j < gridDim; j++ {
			fmt.Printf("%d\t", grid[i][j])
		}
		println()
	}

}<|endoftext|>


<commit_before>package daemon

import (
	"context"
	"fmt"

	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/policy"
	"github.com/weaveworks/flux/resource"
	"github.com/weaveworks/flux/update"
)

func (d *Daemon) pollForNewImages(logger log.Logger) {
	logger.Log("msg", "polling images")

	ctx := context.Background()

	candidateServices, err := d.getUnlockedAutomatedResources(ctx)
	if err != nil {
		logger.Log("error", errors.Wrap(err, "getting unlocked automated resources"))
		return
	}
	if len(candidateServices) == 0 {
		logger.Log("msg", "no automated services")
		return
	}
	// Find images to check
	services, err := d.Cluster.SomeControllers(candidateServices.IDs())
	if err != nil {
		logger.Log("error", errors.Wrap(err, "checking services for new images"))
		return
	}
	// Check the latest available image(s) for each service
	imageRepos, err := update.FetchImageRepos(d.Registry, clusterContainers(services), logger)
	if err != nil {
		logger.Log("error", errors.Wrap(err, "fetching image updates"))
		return
	}

	changes := &update.Automated{}
	for _, service := range services {
		var p policy.Set
		if resource, ok := candidateServices[service.ID]; ok {
			p = resource.Policy()
		}
	containers:
		for _, container := range service.ContainersOrNil() {
			currentImageID := container.Image
			pattern := policy.GetTagPattern(p, container.Name)
			repo := currentImageID.Name
			logger := log.With(logger, "service", service.ID, "container", container.Name, "repo", repo, "pattern", pattern, "current", currentImageID)

			filteredImages := imageRepos.GetRepoImages(repo).FilterAndSort(pattern)

			if latest, ok := filteredImages.Latest(); ok && latest.ID != currentImageID {
				if latest.ID.Tag == "" {
					logger.Log("warning", "untagged image in available images", "action", "skip container")
					continue containers
				}
				currentCreatedAt := ""
				for _, info := range filteredImages {
					if info.CreatedAt.IsZero() {
						logger.Log("warning", "image with zero created timestamp", "image", info.ID, "action", "skip container")
						continue containers
					}
					if info.ID == currentImageID {
						currentCreatedAt = info.CreatedAt.String()
					}
				}
				if currentCreatedAt == "" {
					currentCreatedAt = "filtered out or missing"
					logger.Log("warning", "current image not in filtered images", "action", "proceed anyway")
				}
				newImage := currentImageID.WithNewTag(latest.ID.Tag)
				changes.Add(service.ID, container, newImage)
				logger.Log("info", "added update to automation run", "new", newImage, "reason", fmt.Sprintf("latest %s (%s) > current %s (%s)", latest.ID.Tag, latest.CreatedAt, currentImageID.Tag, currentCreatedAt))
			}
		}
	}

	if len(changes.Changes) > 0 {
		d.UpdateManifests(ctx, update.Spec{Type: update.Auto, Spec: changes})
	}
}

type resources map[flux.ResourceID]resource.Resource

func (r resources) IDs() (ids []flux.ResourceID) {
	for k, _ := range r {
		ids = append(ids, k)
	}
	return ids
}

// getUnlockedAutomatedServices returns all the resources that are
// both automated, and not locked.
func (d *Daemon) getUnlockedAutomatedResources(ctx context.Context) (resources, error) {
	resources, _, err := d.getResources(ctx)
	if err != nil {
		return nil, err
	}

	result := map[flux.ResourceID]resource.Resource{}
	for _, resource := range resources {
		policies := resource.Policy()
		if policies.Has(policy.Automated) && !policies.Has(policy.Locked) {
			result[resource.ResourceID()] = resource
		}
	}
	return result, nil
}
<commit_msg>Take ignore policy into account while fetching<commit_after>package daemon

import (
	"context"
	"fmt"

	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/policy"
	"github.com/weaveworks/flux/resource"
	"github.com/weaveworks/flux/update"
)

func (d *Daemon) pollForNewImages(logger log.Logger) {
	logger.Log("msg", "polling images")

	ctx := context.Background()

	candidateServices, err := d.getAllowedAutomatedResources(ctx)
	if err != nil {
		logger.Log("error", errors.Wrap(err, "getting unlocked automated resources"))
		return
	}
	if len(candidateServices) == 0 {
		logger.Log("msg", "no automated services")
		return
	}
	// Find images to check
	services, err := d.Cluster.SomeControllers(candidateServices.IDs())
	if err != nil {
		logger.Log("error", errors.Wrap(err, "checking services for new images"))
		return
	}
	// Check the latest available image(s) for each service
	imageRepos, err := update.FetchImageRepos(d.Registry, clusterContainers(services), logger)
	if err != nil {
		logger.Log("error", errors.Wrap(err, "fetching image updates"))
		return
	}

	changes := &update.Automated{}
	for _, service := range services {
		var p policy.Set
		if resource, ok := candidateServices[service.ID]; ok {
			p = resource.Policy()
		}
	containers:
		for _, container := range service.ContainersOrNil() {
			currentImageID := container.Image
			pattern := policy.GetTagPattern(p, container.Name)
			repo := currentImageID.Name
			logger := log.With(logger, "service", service.ID, "container", container.Name, "repo", repo, "pattern", pattern, "current", currentImageID)

			filteredImages := imageRepos.GetRepoImages(repo).FilterAndSort(pattern)

			if latest, ok := filteredImages.Latest(); ok && latest.ID != currentImageID {
				if latest.ID.Tag == "" {
					logger.Log("warning", "untagged image in available images", "action", "skip container")
					continue containers
				}
				currentCreatedAt := ""
				for _, info := range filteredImages {
					if info.CreatedAt.IsZero() {
						logger.Log("warning", "image with zero created timestamp", "image", info.ID, "action", "skip container")
						continue containers
					}
					if info.ID == currentImageID {
						currentCreatedAt = info.CreatedAt.String()
					}
				}
				if currentCreatedAt == "" {
					currentCreatedAt = "filtered out or missing"
					logger.Log("warning", "current image not in filtered images", "action", "proceed anyway")
				}
				newImage := currentImageID.WithNewTag(latest.ID.Tag)
				changes.Add(service.ID, container, newImage)
				logger.Log("info", "added update to automation run", "new", newImage, "reason", fmt.Sprintf("latest %s (%s) > current %s (%s)", latest.ID.Tag, latest.CreatedAt, currentImageID.Tag, currentCreatedAt))
			}
		}
	}

	if len(changes.Changes) > 0 {
		d.UpdateManifests(ctx, update.Spec{Type: update.Auto, Spec: changes})
	}
}

type resources map[flux.ResourceID]resource.Resource

func (r resources) IDs() (ids []flux.ResourceID) {
	for k, _ := range r {
		ids = append(ids, k)
	}
	return ids
}

// getAllowedAutomatedResources returns all the resources that are
// automated but do not have policies set to restrain them from
// getting updated.
func (d *Daemon) getAllowedAutomatedResources(ctx context.Context) (resources, error) {
	resources, _, err := d.getResources(ctx)
	if err != nil {
		return nil, err
	}

	result := map[flux.ResourceID]resource.Resource{}
	for _, resource := range resources {
		policies := resource.Policy()
		if policies.Has(policy.Automated) && !policies.Has(policy.Locked) && !policies.Has(policy.Ignore) {
			result[resource.ResourceID()] = resource
		}
	}
	return result, nil
}
<|endoftext|>