<commit_before>// Copyright 2018 The Kubernetes Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package app

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/spf13/cobra"
	corev1 "k8s.io/api/core/v1"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/healthz"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/kubernetes-incubator/metrics-server/pkg/apiserver"
	genericmetrics "github.com/kubernetes-incubator/metrics-server/pkg/apiserver/generic"
	"github.com/kubernetes-incubator/metrics-server/pkg/manager"
	"github.com/kubernetes-incubator/metrics-server/pkg/provider/sink"
	"github.com/kubernetes-incubator/metrics-server/pkg/sources"
	"github.com/kubernetes-incubator/metrics-server/pkg/sources/summary"
)

// NewCommandStartMetricsServer provides a CLI handler for the metrics server entrypoint
func NewCommandStartMetricsServer(out, errOut io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewMetricsServerOptions()

	cmd := &cobra.Command{
		Short: "Launch metrics-server",
		Long:  "Launch metrics-server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Run(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	flags := cmd.Flags()
	flags.DurationVar(&o.MetricResolution, "metric-resolution", o.MetricResolution, "The resolution at which metrics-server will retain metrics.")

	flags.BoolVar(&o.InsecureKubeletTLS, "kubelet-insecure-tls", o.InsecureKubeletTLS, "Do not verify CA of serving certificates presented by Kubelets.  For testing purposes only.")
	flags.BoolVar(&o.DeprecatedCompletelyInsecureKubelet, "deprecated-kubelet-completely-insecure", o.DeprecatedCompletelyInsecureKubelet, "Do not use any encryption, authorization, or authentication when communicating with the Kubelet.")
	flags.IntVar(&o.KubeletPort, "kubelet-port", o.KubeletPort, "The port to use to connect to Kubelets (defaults to 10250)")
	flags.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "The path to the kubeconfig used to connect to the Kubernetes API server and the Kubelets (defaults to in-cluster config)")
	flags.StringSliceVar(&o.KubeletPreferredAddressTypes, "kubelet-preferred-address-types", o.KubeletPreferredAddressTypes, "The priority of node address types to use when determining which address to use to connect to a particular node")

	flags.MarkDeprecated("deprecated-kubelet-completely-insecure", "This is rarely the right option, since it leaves kubelet communication completely insecure.  If you encounter auth errors, make sure you've enabled token webhook auth on the Kubelet, and if you're in a test cluster with self-signed Kubelet certificates, consider using kubelet-insecure-tls instead.")

	o.SecureServing.AddFlags(flags)
	o.Authentication.AddFlags(flags)
	o.Authorization.AddFlags(flags)
	o.Features.AddFlags(flags)

	return cmd
}

type MetricsServerOptions struct {
	// genericoptions.ReccomendedOptions - EtcdOptions
	SecureServing  *genericoptions.SecureServingOptionsWithLoopback
	Authentication *genericoptions.DelegatingAuthenticationOptions
	Authorization  *genericoptions.DelegatingAuthorizationOptions
	Features       *genericoptions.FeatureOptions

	Kubeconfig string

	// Only to be used to for testing
	DisableAuthForTesting bool

	MetricResolution time.Duration

	KubeletPort                  int
	InsecureKubeletTLS           bool
	KubeletPreferredAddressTypes []string

	DeprecatedCompletelyInsecureKubelet bool
}

// NewMetricsServerOptions constructs a new set of default options for metrics-server.
func NewMetricsServerOptions() *MetricsServerOptions {
	o := &MetricsServerOptions{
		SecureServing:  genericoptions.WithLoopback(genericoptions.NewSecureServingOptions()),
		Authentication: genericoptions.NewDelegatingAuthenticationOptions(),
		Authorization:  genericoptions.NewDelegatingAuthorizationOptions(),
		Features:       genericoptions.NewFeatureOptions(),

		MetricResolution:             60 * time.Second,
		KubeletPort:                  10250,
		KubeletPreferredAddressTypes: make([]string, len(summary.DefaultAddressTypePriority)),
	}

	for i, addrType := range summary.DefaultAddressTypePriority {
		o.KubeletPreferredAddressTypes[i] = string(addrType)
	}

	return o
}

func (o MetricsServerOptions) Config() (*apiserver.Config, error) {
	if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	serverConfig := genericapiserver.NewConfig(genericmetrics.Codecs)
	if err := o.SecureServing.ApplyTo(serverConfig); err != nil {
		return nil, err
	}

	if !o.DisableAuthForTesting {
		if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil {
			return nil, err
		}
		if err := o.Authorization.ApplyTo(&serverConfig.Authorization); err != nil {
			return nil, err
		}
	}

	return &apiserver.Config{
		GenericConfig:  serverConfig,
		ProviderConfig: genericmetrics.ProviderConfig{},
	}, nil
}

func (o MetricsServerOptions) Run(stopCh <-chan struct{}) error {
	// grab the config for the API server
	config, err := o.Config()
	if err != nil {
		return err
	}
	config.GenericConfig.EnableMetrics = true

	// set up the client config
	var clientConfig *rest.Config
	if len(o.Kubeconfig) > 0 {
		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.Kubeconfig}
		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

		clientConfig, err = loader.ClientConfig()
	} else {
		clientConfig, err = rest.InClusterConfig()
	}
	if err != nil {
		return fmt.Errorf("unable to construct lister client config: %v", err)
	}

	// set up the informers
	kubeClient, err := kubernetes.NewForConfig(clientConfig)
	if err != nil {
		return fmt.Errorf("unable to construct lister client: %v", err)
	}
	// we should never need to resync, since we're not worried about missing events,
	// and resync is actually for regular interval-based reconciliation these days,
	// so set the default resync interval to 0
	informerFactory := informers.NewSharedInformerFactory(kubeClient, 0)

	// set up the source manager
	kubeletConfig := summary.GetKubeletConfig(clientConfig, o.KubeletPort, o.InsecureKubeletTLS, o.DeprecatedCompletelyInsecureKubelet)
	kubeletClient, err := summary.KubeletClientFor(kubeletConfig)
	if err != nil {
		return fmt.Errorf("unable to construct a client to connect to the kubelets: %v", err)
	}

	// set up an address resolver according to the user's priorities
	addrPriority := make([]corev1.NodeAddressType, len(o.KubeletPreferredAddressTypes))
	for i, addrType := range o.KubeletPreferredAddressTypes {
		addrPriority[i] = corev1.NodeAddressType(addrType)
	}
	addrResolver := summary.NewPriorityNodeAddressResolver(addrPriority)

	sourceProvider := summary.NewSummaryProvider(informerFactory.Core().V1().Nodes().Lister(), kubeletClient, addrResolver)
	scrapeTimeout := time.Duration(float64(o.MetricResolution) * 0.90) // scrape timeout is 90% of the scrape interval
	sources.RegisterDurationMetrics(scrapeTimeout)
	sourceManager := sources.NewSourceManager(sourceProvider, scrapeTimeout)

	// set up the in-memory sink and provider
	metricSink, metricsProvider := sink.NewSinkProvider()

	// set up the general manager
	manager.RegisterDurationMetrics(o.MetricResolution)
	mgr := manager.NewManager(sourceManager, metricSink, o.MetricResolution)

	// inject the providers into the config
	config.ProviderConfig.Node = metricsProvider
	config.ProviderConfig.Pod = metricsProvider

	// complete the config to get an API server
	server, err := config.Complete(informerFactory).New()
	if err != nil {
		return err
	}

	// add health checks
	server.AddHealthzChecks(healthz.NamedCheck("healthz", mgr.CheckHealth))

	// run everything (the apiserver runs the shared informer factory for us)
	mgr.RunUntil(stopCh)
	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<commit_msg>Delete duplicate "Default:" description in command help<commit_after>// Copyright 2018 The Kubernetes Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package app

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/spf13/cobra"
	corev1 "k8s.io/api/core/v1"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/healthz"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/kubernetes-incubator/metrics-server/pkg/apiserver"
	genericmetrics "github.com/kubernetes-incubator/metrics-server/pkg/apiserver/generic"
	"github.com/kubernetes-incubator/metrics-server/pkg/manager"
	"github.com/kubernetes-incubator/metrics-server/pkg/provider/sink"
	"github.com/kubernetes-incubator/metrics-server/pkg/sources"
	"github.com/kubernetes-incubator/metrics-server/pkg/sources/summary"
)

// NewCommandStartMetricsServer provides a CLI handler for the metrics server entrypoint
func NewCommandStartMetricsServer(out, errOut io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewMetricsServerOptions()

	cmd := &cobra.Command{
		Short: "Launch metrics-server",
		Long:  "Launch metrics-server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Run(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	flags := cmd.Flags()
	flags.DurationVar(&o.MetricResolution, "metric-resolution", o.MetricResolution, "The resolution at which metrics-server will retain metrics.")

	flags.BoolVar(&o.InsecureKubeletTLS, "kubelet-insecure-tls", o.InsecureKubeletTLS, "Do not verify CA of serving certificates presented by Kubelets.  For testing purposes only.")
	flags.BoolVar(&o.DeprecatedCompletelyInsecureKubelet, "deprecated-kubelet-completely-insecure", o.DeprecatedCompletelyInsecureKubelet, "Do not use any encryption, authorization, or authentication when communicating with the Kubelet.")
	flags.IntVar(&o.KubeletPort, "kubelet-port", o.KubeletPort, "The port to use to connect to Kubelets.")
	flags.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "The path to the kubeconfig used to connect to the Kubernetes API server and the Kubelets (defaults to in-cluster config)")
	flags.StringSliceVar(&o.KubeletPreferredAddressTypes, "kubelet-preferred-address-types", o.KubeletPreferredAddressTypes, "The priority of node address types to use when determining which address to use to connect to a particular node")

	flags.MarkDeprecated("deprecated-kubelet-completely-insecure", "This is rarely the right option, since it leaves kubelet communication completely insecure.  If you encounter auth errors, make sure you've enabled token webhook auth on the Kubelet, and if you're in a test cluster with self-signed Kubelet certificates, consider using kubelet-insecure-tls instead.")

	o.SecureServing.AddFlags(flags)
	o.Authentication.AddFlags(flags)
	o.Authorization.AddFlags(flags)
	o.Features.AddFlags(flags)

	return cmd
}

type MetricsServerOptions struct {
	// genericoptions.ReccomendedOptions - EtcdOptions
	SecureServing  *genericoptions.SecureServingOptionsWithLoopback
	Authentication *genericoptions.DelegatingAuthenticationOptions
	Authorization  *genericoptions.DelegatingAuthorizationOptions
	Features       *genericoptions.FeatureOptions

	Kubeconfig string

	// Only to be used to for testing
	DisableAuthForTesting bool

	MetricResolution time.Duration

	KubeletPort                  int
	InsecureKubeletTLS           bool
	KubeletPreferredAddressTypes []string

	DeprecatedCompletelyInsecureKubelet bool
}

// NewMetricsServerOptions constructs a new set of default options for metrics-server.
func NewMetricsServerOptions() *MetricsServerOptions {
	o := &MetricsServerOptions{
		SecureServing:  genericoptions.WithLoopback(genericoptions.NewSecureServingOptions()),
		Authentication: genericoptions.NewDelegatingAuthenticationOptions(),
		Authorization:  genericoptions.NewDelegatingAuthorizationOptions(),
		Features:       genericoptions.NewFeatureOptions(),

		MetricResolution:             60 * time.Second,
		KubeletPort:                  10250,
		KubeletPreferredAddressTypes: make([]string, len(summary.DefaultAddressTypePriority)),
	}

	for i, addrType := range summary.DefaultAddressTypePriority {
		o.KubeletPreferredAddressTypes[i] = string(addrType)
	}

	return o
}

func (o MetricsServerOptions) Config() (*apiserver.Config, error) {
	if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	serverConfig := genericapiserver.NewConfig(genericmetrics.Codecs)
	if err := o.SecureServing.ApplyTo(serverConfig); err != nil {
		return nil, err
	}

	if !o.DisableAuthForTesting {
		if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil {
			return nil, err
		}
		if err := o.Authorization.ApplyTo(&serverConfig.Authorization); err != nil {
			return nil, err
		}
	}

	return &apiserver.Config{
		GenericConfig:  serverConfig,
		ProviderConfig: genericmetrics.ProviderConfig{},
	}, nil
}

func (o MetricsServerOptions) Run(stopCh <-chan struct{}) error {
	// grab the config for the API server
	config, err := o.Config()
	if err != nil {
		return err
	}
	config.GenericConfig.EnableMetrics = true

	// set up the client config
	var clientConfig *rest.Config
	if len(o.Kubeconfig) > 0 {
		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.Kubeconfig}
		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

		clientConfig, err = loader.ClientConfig()
	} else {
		clientConfig, err = rest.InClusterConfig()
	}
	if err != nil {
		return fmt.Errorf("unable to construct lister client config: %v", err)
	}

	// set up the informers
	kubeClient, err := kubernetes.NewForConfig(clientConfig)
	if err != nil {
		return fmt.Errorf("unable to construct lister client: %v", err)
	}
	// we should never need to resync, since we're not worried about missing events,
	// and resync is actually for regular interval-based reconciliation these days,
	// so set the default resync interval to 0
	informerFactory := informers.NewSharedInformerFactory(kubeClient, 0)

	// set up the source manager
	kubeletConfig := summary.GetKubeletConfig(clientConfig, o.KubeletPort, o.InsecureKubeletTLS, o.DeprecatedCompletelyInsecureKubelet)
	kubeletClient, err := summary.KubeletClientFor(kubeletConfig)
	if err != nil {
		return fmt.Errorf("unable to construct a client to connect to the kubelets: %v", err)
	}

	// set up an address resolver according to the user's priorities
	addrPriority := make([]corev1.NodeAddressType, len(o.KubeletPreferredAddressTypes))
	for i, addrType := range o.KubeletPreferredAddressTypes {
		addrPriority[i] = corev1.NodeAddressType(addrType)
	}
	addrResolver := summary.NewPriorityNodeAddressResolver(addrPriority)

	sourceProvider := summary.NewSummaryProvider(informerFactory.Core().V1().Nodes().Lister(), kubeletClient, addrResolver)
	scrapeTimeout := time.Duration(float64(o.MetricResolution) * 0.90) // scrape timeout is 90% of the scrape interval
	sources.RegisterDurationMetrics(scrapeTimeout)
	sourceManager := sources.NewSourceManager(sourceProvider, scrapeTimeout)

	// set up the in-memory sink and provider
	metricSink, metricsProvider := sink.NewSinkProvider()

	// set up the general manager
	manager.RegisterDurationMetrics(o.MetricResolution)
	mgr := manager.NewManager(sourceManager, metricSink, o.MetricResolution)

	// inject the providers into the config
	config.ProviderConfig.Node = metricsProvider
	config.ProviderConfig.Pod = metricsProvider

	// complete the config to get an API server
	server, err := config.Complete(informerFactory).New()
	if err != nil {
		return err
	}

	// add health checks
	server.AddHealthzChecks(healthz.NamedCheck("healthz", mgr.CheckHealth))

	// run everything (the apiserver runs the shared informer factory for us)
	mgr.RunUntil(stopCh)
	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<|endoftext|>


<commit_before>package martini

import (
	"fmt"
	"net/http"
	"reflect"
	"regexp"
	"strconv"
	"sync"
)

// Params is a map of name/value pairs for named routes. An instance of martini.Params is available to be injected into any route handler.
type Params map[string]string

// Router is Martini's de-facto routing interface. Supports HTTP verbs, stacked handlers, and dependency injection.
type Router interface {
	Routes

	// Group adds a group where related routes can be added.
	Group(string, func(Router), ...Handler)
	// Get adds a route for a HTTP GET request to the specified matching pattern.
	Get(string, ...Handler) Route
	// Patch adds a route for a HTTP PATCH request to the specified matching pattern.
	Patch(string, ...Handler) Route
	// Post adds a route for a HTTP POST request to the specified matching pattern.
	Post(string, ...Handler) Route
	// Put adds a route for a HTTP PUT request to the specified matching pattern.
	Put(string, ...Handler) Route
	// Delete adds a route for a HTTP DELETE request to the specified matching pattern.
	Delete(string, ...Handler) Route
	// Options adds a route for a HTTP OPTIONS request to the specified matching pattern.
	Options(string, ...Handler) Route
	// Head adds a route for a HTTP HEAD request to the specified matching pattern.
	Head(string, ...Handler) Route
	// Any adds a route for any HTTP method request to the specified matching pattern.
	Any(string, ...Handler) Route
	// AddRoute adds a route for a given HTTP method request to the specified matching pattern.
	AddRoute(string, string, ...Handler) Route

	// NotFound sets the handlers that are called when a no route matches a request. Throws a basic 404 by default.
	NotFound(...Handler)

	// Handle is the entry point for routing. This is used as a martini.Handler
	Handle(http.ResponseWriter, *http.Request, Context)

	// GetMatchedRoute will return a Martini route if a can be found
	// given a METHOD and path. If none can be found, nil is returned
	GetMatchedRoute(string, string) route
}

type router struct {
	routes     []*route
	notFounds  []Handler
	groups     []group
	routesLock sync.RWMutex
}

type group struct {
	pattern  string
	handlers []Handler
}

// NewRouter creates a new Router instance.
// If you aren't using ClassicMartini, then you can add Routes as a
// service with:
//
//	m := martini.New()
//	r := martini.NewRouter()
//	m.MapTo(r, (*martini.Routes)(nil))
//
// If you are using ClassicMartini, then this is done for you.
func NewRouter() Router {
	return &router{notFounds: []Handler{http.NotFound}, groups: make([]group, 0)}
}

func (r *router) Group(pattern string, fn func(Router), h ...Handler) {
	r.groups = append(r.groups, group{pattern, h})
	fn(r)
	r.groups = r.groups[:len(r.groups)-1]
}

func (r *router) Get(pattern string, h ...Handler) Route {
	return r.addRoute("GET", pattern, h)
}

func (r *router) Patch(pattern string, h ...Handler) Route {
	return r.addRoute("PATCH", pattern, h)
}

func (r *router) Post(pattern string, h ...Handler) Route {
	return r.addRoute("POST", pattern, h)
}

func (r *router) Put(pattern string, h ...Handler) Route {
	return r.addRoute("PUT", pattern, h)
}

func (r *router) Delete(pattern string, h ...Handler) Route {
	return r.addRoute("DELETE", pattern, h)
}

func (r *router) Options(pattern string, h ...Handler) Route {
	return r.addRoute("OPTIONS", pattern, h)
}

func (r *router) Head(pattern string, h ...Handler) Route {
	return r.addRoute("HEAD", pattern, h)
}

func (r *router) Any(pattern string, h ...Handler) Route {
	return r.addRoute("*", pattern, h)
}

func (r *router) AddRoute(method, pattern string, h ...Handler) Route {
	return r.addRoute(method, pattern, h)
}

func (r *router) Handle(res http.ResponseWriter, req *http.Request, context Context) {
	for _, route := range r.getRoutes() {
		ok, vals := route.Match(req.Method, req.URL.Path)
		if ok {
			params := Params(vals)
			context.Map(params)
			route.Handle(context, res)
			return
		}
	}

	// no routes exist, 404
	c := &routeContext{context, 0, r.notFounds}
	context.MapTo(c, (*Context)(nil))
	c.run()
}

// GetMatchedRoute will return a Martini route if a can be found
// given a METHOD and path. If none can be found, nil is returned
func (r *router) GetMatchedRoute(method string, path string) route {
	for _, route := range r.routes {
		ok, _ := route.Match(method, path)
		if ok {
			return route
		}
	}

	return nil
}

func (r *router) NotFound(handler ...Handler) {
	r.notFounds = handler
}

func (r *router) addRoute(method string, pattern string, handlers []Handler) *route {
	if len(r.groups) > 0 {
		groupPattern := ""
		h := make([]Handler, 0)
		for _, g := range r.groups {
			groupPattern += g.pattern
			h = append(h, g.handlers...)
		}

		pattern = groupPattern + pattern
		h = append(h, handlers...)
		handlers = h
	}

	route := newRoute(method, pattern, handlers)
	route.Validate()
	r.appendRoute(route)
	return route
}

func (r *router) appendRoute(rt *route) {
	r.routesLock.Lock()
	defer r.routesLock.Unlock()
	r.routes = append(r.routes, rt)
}

func (r *router) getRoutes() []*route {
	r.routesLock.RLock()
	defer r.routesLock.RUnlock()
	return r.routes[:]
}

func (r *router) findRoute(name string) *route {
	for _, route := range r.getRoutes() {
		if route.name == name {
			return route
		}
	}

	return nil
}

// Route is an interface representing a Route in Martini's routing layer.
type Route interface {
	// URLWith returns a rendering of the Route's url with the given string params.
	URLWith([]string) string
	// Name sets a name for the route.
	Name(string)
	// GetName returns the name of the route.
	GetName() string
	// Pattern returns the pattern of the route.
	Pattern() string
	// Method returns the method of the route.
	Method() string
}

type route struct {
	method   string
	regex    *regexp.Regexp
	handlers []Handler
	pattern  string
	name     string
}

func newRoute(method string, pattern string, handlers []Handler) *route {
	route := route{method, nil, handlers, pattern, ""}
	r := regexp.MustCompile(`:[^/#?()\.\\]+`)
	pattern = r.ReplaceAllStringFunc(pattern, func(m string) string {
		return fmt.Sprintf(`(?P<%s>[^/#?]+)`, m[1:])
	})
	r2 := regexp.MustCompile(`\*\*`)
	var index int
	pattern = r2.ReplaceAllStringFunc(pattern, func(m string) string {
		index++
		return fmt.Sprintf(`(?P<_%d>[^#?]*)`, index)
	})
	pattern += `\/?`
	route.regex = regexp.MustCompile(pattern)
	return &route
}

func (r route) MatchMethod(method string) bool {
	return r.method == "*" || method == r.method || (method == "HEAD" && r.method == "GET")
}

func (r route) Match(method string, path string) (bool, map[string]string) {
	// add Any method matching support
	if !r.MatchMethod(method) {
		return false, nil
	}

	matches := r.regex.FindStringSubmatch(path)
	if len(matches) > 0 && matches[0] == path {
		params := make(map[string]string)
		for i, name := range r.regex.SubexpNames() {
			if len(name) > 0 {
				params[name] = matches[i]
			}
		}
		return true, params
	}
	return false, nil
}

func (r *route) Validate() {
	for _, handler := range r.handlers {
		validateHandler(handler)
	}
}

func (r *route) Handle(c Context, res http.ResponseWriter) {
	context := &routeContext{c, 0, r.handlers}
	c.MapTo(context, (*Context)(nil))
	c.MapTo(r, (*Route)(nil))
	context.run()
}

// URLWith returns the url pattern replacing the parameters for its values
func (r *route) URLWith(args []string) string {
	if len(args) > 0 {
		reg := regexp.MustCompile(`:[^/#?()\.\\]+|\(\?P<[a-zA-Z0-9]+>.*\)`)
		argCount := len(args)
		i := 0
		url := reg.ReplaceAllStringFunc(r.pattern, func(m string) string {
			var val interface{}
			if i < argCount {
				val = args[i]
			} else {
				val = m
			}
			i += 1
			return fmt.Sprintf(`%v`, val)
		})

		return url
	}
	return r.pattern
}

func (r *route) Name(name string) {
	r.name = name
}

func (r *route) GetName() string {
	return r.name
}

func (r *route) Pattern() string {
	return r.pattern
}

func (r *route) Method() string {
	return r.method
}

// Routes is a helper service for Martini's routing layer.
type Routes interface {
	// URLFor returns a rendered URL for the given route. Optional params can be passed to fulfill named parameters in the route.
	URLFor(name string, params ...interface{}) string
	// MethodsFor returns an array of methods available for the path
	MethodsFor(path string) []string
	// All returns an array with all the routes in the router.
	All() []Route
}

// URLFor returns the url for the given route name.
func (r *router) URLFor(name string, params ...interface{}) string {
	route := r.findRoute(name)

	if route == nil {
		panic("route not found")
	}

	var args []string
	for _, param := range params {
		switch v := param.(type) {
		case int:
			args = append(args, strconv.FormatInt(int64(v), 10))
		case string:
			args = append(args, v)
		default:
			if v != nil {
				panic("Arguments passed to URLFor must be integers or strings")
			}
		}
	}

	return route.URLWith(args)
}

func (r *router) All() []Route {
	routes := r.getRoutes()
	var ri = make([]Route, len(routes))

	for i, route := range routes {
		ri[i] = Route(route)
	}

	return ri
}

func hasMethod(methods []string, method string) bool {
	for _, v := range methods {
		if v == method {
			return true
		}
	}
	return false
}

// MethodsFor returns all methods available for path
func (r *router) MethodsFor(path string) []string {
	methods := []string{}
	for _, route := range r.getRoutes() {
		matches := route.regex.FindStringSubmatch(path)
		if len(matches) > 0 && matches[0] == path && !hasMethod(methods, route.method) {
			methods = append(methods, route.method)
		}
	}
	return methods
}

type routeContext struct {
	Context
	index    int
	handlers []Handler
}

func (r *routeContext) Next() {
	r.index += 1
	r.run()
}

func (r *routeContext) run() {
	for r.index < len(r.handlers) {
		handler := r.handlers[r.index]
		vals, err := r.Invoke(handler)
		if err != nil {
			panic(err)
		}
		r.index += 1

		// if the handler returned something, write it to the http response
		if len(vals) > 0 {
			ev := r.Get(reflect.TypeOf(ReturnHandler(nil)))
			handleReturn := ev.Interface().(ReturnHandler)
			handleReturn(r, vals)
		}

		if r.Written() {
			return
		}
	}
}
<commit_msg>fixing return variable being pointer<commit_after>package martini

import (
	"fmt"
	"net/http"
	"reflect"
	"regexp"
	"strconv"
	"sync"
)

// Params is a map of name/value pairs for named routes. An instance of martini.Params is available to be injected into any route handler.
type Params map[string]string

// Router is Martini's de-facto routing interface. Supports HTTP verbs, stacked handlers, and dependency injection.
type Router interface {
	Routes

	// Group adds a group where related routes can be added.
	Group(string, func(Router), ...Handler)
	// Get adds a route for a HTTP GET request to the specified matching pattern.
	Get(string, ...Handler) Route
	// Patch adds a route for a HTTP PATCH request to the specified matching pattern.
	Patch(string, ...Handler) Route
	// Post adds a route for a HTTP POST request to the specified matching pattern.
	Post(string, ...Handler) Route
	// Put adds a route for a HTTP PUT request to the specified matching pattern.
	Put(string, ...Handler) Route
	// Delete adds a route for a HTTP DELETE request to the specified matching pattern.
	Delete(string, ...Handler) Route
	// Options adds a route for a HTTP OPTIONS request to the specified matching pattern.
	Options(string, ...Handler) Route
	// Head adds a route for a HTTP HEAD request to the specified matching pattern.
	Head(string, ...Handler) Route
	// Any adds a route for any HTTP method request to the specified matching pattern.
	Any(string, ...Handler) Route
	// AddRoute adds a route for a given HTTP method request to the specified matching pattern.
	AddRoute(string, string, ...Handler) Route

	// NotFound sets the handlers that are called when a no route matches a request. Throws a basic 404 by default.
	NotFound(...Handler)

	// Handle is the entry point for routing. This is used as a martini.Handler
	Handle(http.ResponseWriter, *http.Request, Context)

	// GetMatchedRoute will return a Martini route if a can be found
	// given a METHOD and path. If none can be found, nil is returned
	GetMatchedRoute(string, string) *route
}

type router struct {
	routes     []*route
	notFounds  []Handler
	groups     []group
	routesLock sync.RWMutex
}

type group struct {
	pattern  string
	handlers []Handler
}

// NewRouter creates a new Router instance.
// If you aren't using ClassicMartini, then you can add Routes as a
// service with:
//
//	m := martini.New()
//	r := martini.NewRouter()
//	m.MapTo(r, (*martini.Routes)(nil))
//
// If you are using ClassicMartini, then this is done for you.
func NewRouter() Router {
	return &router{notFounds: []Handler{http.NotFound}, groups: make([]group, 0)}
}

func (r *router) Group(pattern string, fn func(Router), h ...Handler) {
	r.groups = append(r.groups, group{pattern, h})
	fn(r)
	r.groups = r.groups[:len(r.groups)-1]
}

func (r *router) Get(pattern string, h ...Handler) Route {
	return r.addRoute("GET", pattern, h)
}

func (r *router) Patch(pattern string, h ...Handler) Route {
	return r.addRoute("PATCH", pattern, h)
}

func (r *router) Post(pattern string, h ...Handler) Route {
	return r.addRoute("POST", pattern, h)
}

func (r *router) Put(pattern string, h ...Handler) Route {
	return r.addRoute("PUT", pattern, h)
}

func (r *router) Delete(pattern string, h ...Handler) Route {
	return r.addRoute("DELETE", pattern, h)
}

func (r *router) Options(pattern string, h ...Handler) Route {
	return r.addRoute("OPTIONS", pattern, h)
}

func (r *router) Head(pattern string, h ...Handler) Route {
	return r.addRoute("HEAD", pattern, h)
}

func (r *router) Any(pattern string, h ...Handler) Route {
	return r.addRoute("*", pattern, h)
}

func (r *router) AddRoute(method, pattern string, h ...Handler) Route {
	return r.addRoute(method, pattern, h)
}

func (r *router) Handle(res http.ResponseWriter, req *http.Request, context Context) {
	for _, route := range r.getRoutes() {
		ok, vals := route.Match(req.Method, req.URL.Path)
		if ok {
			params := Params(vals)
			context.Map(params)
			route.Handle(context, res)
			return
		}
	}

	// no routes exist, 404
	c := &routeContext{context, 0, r.notFounds}
	context.MapTo(c, (*Context)(nil))
	c.run()
}

// GetMatchedRoute will return a Martini route if a can be found
// given a METHOD and path. If none can be found, nil is returned
func (r *router) GetMatchedRoute(method string, path string) *route {
	for _, route := range r.routes {
		ok, _ := route.Match(method, path)
		if ok {
			return route
		}
	}

	return nil
}

func (r *router) NotFound(handler ...Handler) {
	r.notFounds = handler
}

func (r *router) addRoute(method string, pattern string, handlers []Handler) *route {
	if len(r.groups) > 0 {
		groupPattern := ""
		h := make([]Handler, 0)
		for _, g := range r.groups {
			groupPattern += g.pattern
			h = append(h, g.handlers...)
		}

		pattern = groupPattern + pattern
		h = append(h, handlers...)
		handlers = h
	}

	route := newRoute(method, pattern, handlers)
	route.Validate()
	r.appendRoute(route)
	return route
}

func (r *router) appendRoute(rt *route) {
	r.routesLock.Lock()
	defer r.routesLock.Unlock()
	r.routes = append(r.routes, rt)
}

func (r *router) getRoutes() []*route {
	r.routesLock.RLock()
	defer r.routesLock.RUnlock()
	return r.routes[:]
}

func (r *router) findRoute(name string) *route {
	for _, route := range r.getRoutes() {
		if route.name == name {
			return route
		}
	}

	return nil
}

// Route is an interface representing a Route in Martini's routing layer.
type Route interface {
	// URLWith returns a rendering of the Route's url with the given string params.
	URLWith([]string) string
	// Name sets a name for the route.
	Name(string)
	// GetName returns the name of the route.
	GetName() string
	// Pattern returns the pattern of the route.
	Pattern() string
	// Method returns the method of the route.
	Method() string
}

type route struct {
	method   string
	regex    *regexp.Regexp
	handlers []Handler
	pattern  string
	name     string
}

func newRoute(method string, pattern string, handlers []Handler) *route {
	route := route{method, nil, handlers, pattern, ""}
	r := regexp.MustCompile(`:[^/#?()\.\\]+`)
	pattern = r.ReplaceAllStringFunc(pattern, func(m string) string {
		return fmt.Sprintf(`(?P<%s>[^/#?]+)`, m[1:])
	})
	r2 := regexp.MustCompile(`\*\*`)
	var index int
	pattern = r2.ReplaceAllStringFunc(pattern, func(m string) string {
		index++
		return fmt.Sprintf(`(?P<_%d>[^#?]*)`, index)
	})
	pattern += `\/?`
	route.regex = regexp.MustCompile(pattern)
	return &route
}

func (r route) MatchMethod(method string) bool {
	return r.method == "*" || method == r.method || (method == "HEAD" && r.method == "GET")
}

func (r route) Match(method string, path string) (bool, map[string]string) {
	// add Any method matching support
	if !r.MatchMethod(method) {
		return false, nil
	}

	matches := r.regex.FindStringSubmatch(path)
	if len(matches) > 0 && matches[0] == path {
		params := make(map[string]string)
		for i, name := range r.regex.SubexpNames() {
			if len(name) > 0 {
				params[name] = matches[i]
			}
		}
		return true, params
	}
	return false, nil
}

func (r *route) Validate() {
	for _, handler := range r.handlers {
		validateHandler(handler)
	}
}

func (r *route) Handle(c Context, res http.ResponseWriter) {
	context := &routeContext{c, 0, r.handlers}
	c.MapTo(context, (*Context)(nil))
	c.MapTo(r, (*Route)(nil))
	context.run()
}

// URLWith returns the url pattern replacing the parameters for its values
func (r *route) URLWith(args []string) string {
	if len(args) > 0 {
		reg := regexp.MustCompile(`:[^/#?()\.\\]+|\(\?P<[a-zA-Z0-9]+>.*\)`)
		argCount := len(args)
		i := 0
		url := reg.ReplaceAllStringFunc(r.pattern, func(m string) string {
			var val interface{}
			if i < argCount {
				val = args[i]
			} else {
				val = m
			}
			i += 1
			return fmt.Sprintf(`%v`, val)
		})

		return url
	}
	return r.pattern
}

func (r *route) Name(name string) {
	r.name = name
}

func (r *route) GetName() string {
	return r.name
}

func (r *route) Pattern() string {
	return r.pattern
}

func (r *route) Method() string {
	return r.method
}

// Routes is a helper service for Martini's routing layer.
type Routes interface {
	// URLFor returns a rendered URL for the given route. Optional params can be passed to fulfill named parameters in the route.
	URLFor(name string, params ...interface{}) string
	// MethodsFor returns an array of methods available for the path
	MethodsFor(path string) []string
	// All returns an array with all the routes in the router.
	All() []Route
}

// URLFor returns the url for the given route name.
func (r *router) URLFor(name string, params ...interface{}) string {
	route := r.findRoute(name)

	if route == nil {
		panic("route not found")
	}

	var args []string
	for _, param := range params {
		switch v := param.(type) {
		case int:
			args = append(args, strconv.FormatInt(int64(v), 10))
		case string:
			args = append(args, v)
		default:
			if v != nil {
				panic("Arguments passed to URLFor must be integers or strings")
			}
		}
	}

	return route.URLWith(args)
}

func (r *router) All() []Route {
	routes := r.getRoutes()
	var ri = make([]Route, len(routes))

	for i, route := range routes {
		ri[i] = Route(route)
	}

	return ri
}

func hasMethod(methods []string, method string) bool {
	for _, v := range methods {
		if v == method {
			return true
		}
	}
	return false
}

// MethodsFor returns all methods available for path
func (r *router) MethodsFor(path string) []string {
	methods := []string{}
	for _, route := range r.getRoutes() {
		matches := route.regex.FindStringSubmatch(path)
		if len(matches) > 0 && matches[0] == path && !hasMethod(methods, route.method) {
			methods = append(methods, route.method)
		}
	}
	return methods
}

type routeContext struct {
	Context
	index    int
	handlers []Handler
}

func (r *routeContext) Next() {
	r.index += 1
	r.run()
}

func (r *routeContext) run() {
	for r.index < len(r.handlers) {
		handler := r.handlers[r.index]
		vals, err := r.Invoke(handler)
		if err != nil {
			panic(err)
		}
		r.index += 1

		// if the handler returned something, write it to the http response
		if len(vals) > 0 {
			ev := r.Get(reflect.TypeOf(ReturnHandler(nil)))
			handleReturn := ev.Interface().(ReturnHandler)
			handleReturn(r, vals)
		}

		if r.Written() {
			return
		}
	}
}
<|endoftext|>


<commit_before>package mns

import (
	"bytes"
	"github.com/gogap/ali_mns"
	"github.com/rs/xid"
	"strings"
	"sync"
	"time"

	"github.com/gogap/spirit"
)

const (
	mnsSenderURN = "urn:spirit-contrib:sender:mns"

	mnsSenderFlowMetadata            = "urn:spirit-contrib:sender:mns#flow"
	mnsSenderQueueURNMetadata        = "urn:spirit-contrib:sender:mns#queue_urn"
	mnsSenderQueueParallelIdMetadata = "urn:spirit-contrib:sender:mns#parallel"
)

var _ spirit.TranslatorSender = new(MNSSender)

type _Deliveries struct {
	Deliveries []spirit.Delivery
	Error      error
}

type MNSSenderConfig struct {
	URL             string `json:"url"`
	AccessKeyId     string `json:"access_key_id"`
	AccessKeySecret string `json:"access_key_secret"`
	DelaySeconds    int    `json:"delay_seconds"`
	Priority        int    `json:"priority"`
	ProxyAddress    string `json:"proxy_address"`
}

type MNSSender struct {
	statusLocker sync.Mutex
	terminaled   chan bool
	conf         MNSSenderConfig

	status spirit.Status

	mnsClient    map[string]ali_mns.AliMNSQueue
	clientLocker sync.Mutex

	translator spirit.OutputTranslator

	getter spirit.DeliveryGetter
}

func init() {
	spirit.RegisterSender(mnsSenderURN, NewMNSSender)
}

func NewMNSSender(config spirit.Map) (sender spirit.Sender, err error) {
	conf := MNSSenderConfig{}
	if err = config.ToObject(&conf); err != nil {
		return
	}

	if conf.URL == "" {
		err = ErrAliURLIsEmpty
		return
	}

	if conf.AccessKeyId == "" {
		err = ErrAliAccessKeyIdIsEmpty
		return
	}

	if conf.AccessKeySecret == "" {
		err = ErrAliAccessKeySecretIsEmpty
		return
	}

	if conf.Priority == 0 {
		conf.Priority = 8
	}

	if conf.DelaySeconds > 604800 {
		conf.DelaySeconds = 604800
	}

	if conf.DelaySeconds < 0 {
		conf.DelaySeconds = 0
	}

	sender = &MNSSender{
		conf:       conf,
		mnsClient:  make(map[string]ali_mns.AliMNSQueue),
		terminaled: make(chan bool),
	}

	return
}

func (p *MNSSender) Start() (err error) {
	p.statusLocker.Lock()
	defer p.statusLocker.Unlock()

	if p.status == spirit.StatusRunning {
		err = spirit.ErrSenderAlreadyRunning
		return
	}

	spirit.Logger().WithField("actor", spirit.ActorSender).
		WithField("urn", mnsSenderURN).
		WithField("event", "start").
		Infoln("enter start")

	if p.getter == nil {
		err = spirit.ErrSenderDeliveryGetterIsNil
		return
	}

	p.terminaled = make(chan bool)

	p.status = spirit.StatusRunning

	go func() {
		for {
			if deliveries, err := p.getter.Get(); err != nil {
				spirit.Logger().
					WithField("actor", spirit.ActorSender).
					WithField("urn", mnsSenderURN).
					WithField("event", "get delivery from getter").
					Errorln(err)
			} else {
				for _, delivery := range deliveries {
					if metadata := delivery.Metadata(); metadata != nil {
						flow := MNSFlowMetadata{}
						if err := metadata.Object(mnsSenderFlowMetadata, &flow); err != nil {
							spirit.Logger().
								WithField("actor", spirit.ActorSender).
								WithField("urn", mnsSenderURN).
								WithField("event", "convert metadata to mns_flow failed").
								Errorln(err)

							continue
						}
						p.callFlow(delivery, &flow)
					} else {
						spirit.Logger().
							WithField("actor", spirit.ActorSender).
							WithField("urn", mnsSenderURN).
							WithField("event", "get metadata").
							Warnln("metadata not exist")

						continue
					}
				}
			}

			select {
			case signal := <-p.terminaled:
				{
					if signal == true {
						spirit.Logger().WithField("actor", spirit.ActorSender).
							WithField("urn", mnsSenderURN).
							WithField("event", "terminal").
							Debugln("terminal singal received")
						return
					}
				}
			case <-time.After(time.Microsecond * 10):
				{
					continue
				}
			}
		}
	}()

	return
}

func (p *MNSSender) Stop() (err error) {
	p.statusLocker.Lock()
	defer p.statusLocker.Unlock()

	if p.status == spirit.StatusStopped {
		err = spirit.ErrSenderDidNotRunning
		return
	}

	spirit.Logger().WithField("actor", spirit.ActorSender).
		WithField("urn", mnsSenderURN).
		WithField("event", "stop").
		Infoln("enter stop")

	p.terminaled <- true

	close(p.terminaled)

	spirit.Logger().WithField("actor", spirit.ActorSender).
		WithField("urn", mnsSenderURN).
		WithField("event", "stop").
		Infoln("stopped")

	return
}

func (p *MNSSender) Status() spirit.Status {
	return p.status
}

func (p *MNSSender) SetDeliveryGetter(getter spirit.DeliveryGetter) (err error) {
	p.getter = getter
	return
}

func (p *MNSSender) getMNSClient(queue string) ali_mns.AliMNSQueue {
	p.clientLocker.Lock()
	defer p.clientLocker.Unlock()

	if q, exist := p.mnsClient[queue]; exist {
		return q
	}

	cli := ali_mns.NewAliMNSClient(p.conf.URL, p.conf.AccessKeyId, p.conf.AccessKeySecret)
	if p.conf.ProxyAddress != "" {
		cli.SetProxy(p.conf.ProxyAddress)
	}

	q := ali_mns.NewMNSQueue(queue, cli)

	p.mnsClient[queue] = q

	return q
}

func (p *MNSSender) callFlow(delivery spirit.Delivery, flowMetadata *MNSFlowMetadata) {
	if flowMetadata == nil {
		return
	}

	sendDeliveryFunc := func(ignoreSendError bool, delivery spirit.Delivery, queueURN map[string]string, queues ...string) (err error) {
		if queues == nil || len(queues) == 0 {
			return
		}

		backupURN := delivery.URN()
		backupParallel := new(MNSParallelFlowMetadata)
		delivery.Metadata().Object(mnsSenderQueueParallelIdMetadata, backupParallel)

		// recover
		defer func() {
			delivery.SetURN(backupURN)
			delivery.SetMetadata(mnsSenderQueueParallelIdMetadata, backupParallel)
		}()

		for _, q := range queues {

			if q == "" {
				continue
			}

			parallelQueues := strings.Split(q, "||")
			parallelCount := len(parallelQueues)
			parallelQueuePid := ""

			if parallelCount > 0 {
				parallelQueuePid = xid.New().String()
			}

			for index, queue := range parallelQueues {

				// recover
				delivery.SetURN(backupURN)
				delivery.SetMetadata(mnsSenderQueueParallelIdMetadata, backupParallel)

				urn := ""
				if queueURN != nil {
					urn, _ = queueURN[queue]
				}

				buf := bytes.Buffer{}
				delivery.SetURN(urn)

				if parallelCount > 0 {
					delivery.SetMetadata(mnsSenderQueueParallelIdMetadata, MNSParallelFlowMetadata{Id: parallelQueuePid, Index: index, Count: parallelCount})
				} else {
					delivery.DeleteMetadata(mnsSenderQueueParallelIdMetadata)
				}

				if err = p.translator.Out(&buf, delivery); err != nil {
					spirit.Logger().
						WithField("actor", spirit.ActorSender).
						WithField("urn", mnsSenderURN).
						WithField("event", "translate delivery").
						Errorln(err)
					return
				} else {
					reqData := ali_mns.MessageSendRequest{
						MessageBody:  ali_mns.Base64Bytes(buf.Bytes()),
						DelaySeconds: int64(p.conf.DelaySeconds),
						Priority:     int64(p.conf.Priority),
					}

					client := p.getMNSClient(queue)
					if _, err = client.SendMessage(reqData); err != nil {

						spirit.Logger().
							WithField("actor", spirit.ActorSender).
							WithField("urn", mnsSenderURN).
							WithField("event", "send mns message").
							Errorln(err)

						if ignoreSendError {
							err = nil
							continue
						}
						return
					}
				}
			}

		}
		return
	}

	currentQueueURN := map[string]string{}
	if err := delivery.Metadata().Object(mnsSenderQueueURNMetadata, &currentQueueURN); err != nil {
		spirit.Logger().
			WithField("actor", spirit.ActorSender).
			WithField("urn", mnsSenderURN).
			WithField("event", "get queue urn").
			Errorln(err)
	}

	if delivery.Payload().LastError() != nil {
		// run error flow

		if flowMetadata.Error != nil && len(flowMetadata.Error) > 0 {

			queueURN := map[string]string{}

			for _, queue := range flowMetadata.Error {
				if urn, exist := currentQueueURN[queue]; exist {
					queueURN[queue] = urn
				}
			}

			delivery.SetMetadata(mnsSenderFlowMetadata, nil)
			sendDeliveryFunc(true, delivery, queueURN, flowMetadata.Error...)
		}

	} else {
		// run normal flow
		if flowMetadata.Normal != nil && len(flowMetadata.Normal) > 0 {
			nextQueue := flowMetadata.Normal[flowMetadata.CurrentFlowId]

			queueURN := map[string]string{}

			if len(flowMetadata.Normal) > flowMetadata.CurrentFlowId {
				tmpRemainQueues := flowMetadata.Normal[flowMetadata.CurrentFlowId:]

				remainQueues := []string{}
				for _, queue := range tmpRemainQueues {
					queues := strings.Split(queue, "||")
					remainQueues = append(remainQueues, queues...)
				}

				for _, queue := range remainQueues {
					if urn, exist := currentQueueURN[queue]; exist {
						queueURN[queue] = urn
					}
				}
			}

			flowMetadata.CurrentFlowId += 1
			defer func() { flowMetadata.CurrentFlowId -= 1 }()

			delivery.SetMetadata(mnsSenderFlowMetadata, flowMetadata)
			sendDeliveryFunc(false, delivery, queueURN, nextQueue)
		}
	}
}

func (p *MNSSender) SetTranslator(translator spirit.OutputTranslator) (err error) {
	p.translator = translator
	return
}
<commit_msg>move metadata of parallel to payload context<commit_after>package mns

import (
	"bytes"
	"github.com/gogap/ali_mns"
	"github.com/rs/xid"
	"strings"
	"sync"
	"time"

	"github.com/gogap/spirit"
)

const (
	mnsSenderURN = "urn:spirit-contrib:sender:mns"

	mnsSenderFlowMetadata           = "urn:spirit-contrib:sender:mns#flow"
	mnsSenderQueueURNMetadata       = "urn:spirit-contrib:sender:mns#queue_urn"
	mnsSenderQueueParallelIdContext = "urn:spirit-contrib:sender:mns#parallel"
)

var _ spirit.TranslatorSender = new(MNSSender)

type _Deliveries struct {
	Deliveries []spirit.Delivery
	Error      error
}

type MNSSenderConfig struct {
	URL             string `json:"url"`
	AccessKeyId     string `json:"access_key_id"`
	AccessKeySecret string `json:"access_key_secret"`
	DelaySeconds    int    `json:"delay_seconds"`
	Priority        int    `json:"priority"`
	ProxyAddress    string `json:"proxy_address"`
}

type MNSSender struct {
	statusLocker sync.Mutex
	terminaled   chan bool
	conf         MNSSenderConfig

	status spirit.Status

	mnsClient    map[string]ali_mns.AliMNSQueue
	clientLocker sync.Mutex

	translator spirit.OutputTranslator

	getter spirit.DeliveryGetter
}

func init() {
	spirit.RegisterSender(mnsSenderURN, NewMNSSender)
}

func NewMNSSender(config spirit.Map) (sender spirit.Sender, err error) {
	conf := MNSSenderConfig{}
	if err = config.ToObject(&conf); err != nil {
		return
	}

	if conf.URL == "" {
		err = ErrAliURLIsEmpty
		return
	}

	if conf.AccessKeyId == "" {
		err = ErrAliAccessKeyIdIsEmpty
		return
	}

	if conf.AccessKeySecret == "" {
		err = ErrAliAccessKeySecretIsEmpty
		return
	}

	if conf.Priority == 0 {
		conf.Priority = 8
	}

	if conf.DelaySeconds > 604800 {
		conf.DelaySeconds = 604800
	}

	if conf.DelaySeconds < 0 {
		conf.DelaySeconds = 0
	}

	sender = &MNSSender{
		conf:       conf,
		mnsClient:  make(map[string]ali_mns.AliMNSQueue),
		terminaled: make(chan bool),
	}

	return
}

func (p *MNSSender) Start() (err error) {
	p.statusLocker.Lock()
	defer p.statusLocker.Unlock()

	if p.status == spirit.StatusRunning {
		err = spirit.ErrSenderAlreadyRunning
		return
	}

	spirit.Logger().WithField("actor", spirit.ActorSender).
		WithField("urn", mnsSenderURN).
		WithField("event", "start").
		Infoln("enter start")

	if p.getter == nil {
		err = spirit.ErrSenderDeliveryGetterIsNil
		return
	}

	p.terminaled = make(chan bool)

	p.status = spirit.StatusRunning

	go func() {
		for {
			if deliveries, err := p.getter.Get(); err != nil {
				spirit.Logger().
					WithField("actor", spirit.ActorSender).
					WithField("urn", mnsSenderURN).
					WithField("event", "get delivery from getter").
					Errorln(err)
			} else {
				for _, delivery := range deliveries {
					if metadata := delivery.Metadata(); metadata != nil {
						flow := MNSFlowMetadata{}
						if err := metadata.Object(mnsSenderFlowMetadata, &flow); err != nil {
							spirit.Logger().
								WithField("actor", spirit.ActorSender).
								WithField("urn", mnsSenderURN).
								WithField("event", "convert metadata to mns_flow failed").
								Errorln(err)

							continue
						}
						p.callFlow(delivery, &flow)
					} else {
						spirit.Logger().
							WithField("actor", spirit.ActorSender).
							WithField("urn", mnsSenderURN).
							WithField("event", "get metadata").
							Warnln("metadata not exist")

						continue
					}
				}
			}

			select {
			case signal := <-p.terminaled:
				{
					if signal == true {
						spirit.Logger().WithField("actor", spirit.ActorSender).
							WithField("urn", mnsSenderURN).
							WithField("event", "terminal").
							Debugln("terminal singal received")
						return
					}
				}
			case <-time.After(time.Microsecond * 10):
				{
					continue
				}
			}
		}
	}()

	return
}

func (p *MNSSender) Stop() (err error) {
	p.statusLocker.Lock()
	defer p.statusLocker.Unlock()

	if p.status == spirit.StatusStopped {
		err = spirit.ErrSenderDidNotRunning
		return
	}

	spirit.Logger().WithField("actor", spirit.ActorSender).
		WithField("urn", mnsSenderURN).
		WithField("event", "stop").
		Infoln("enter stop")

	p.terminaled <- true

	close(p.terminaled)

	spirit.Logger().WithField("actor", spirit.ActorSender).
		WithField("urn", mnsSenderURN).
		WithField("event", "stop").
		Infoln("stopped")

	return
}

func (p *MNSSender) Status() spirit.Status {
	return p.status
}

func (p *MNSSender) SetDeliveryGetter(getter spirit.DeliveryGetter) (err error) {
	p.getter = getter
	return
}

func (p *MNSSender) getMNSClient(queue string) ali_mns.AliMNSQueue {
	p.clientLocker.Lock()
	defer p.clientLocker.Unlock()

	if q, exist := p.mnsClient[queue]; exist {
		return q
	}

	cli := ali_mns.NewAliMNSClient(p.conf.URL, p.conf.AccessKeyId, p.conf.AccessKeySecret)
	if p.conf.ProxyAddress != "" {
		cli.SetProxy(p.conf.ProxyAddress)
	}

	q := ali_mns.NewMNSQueue(queue, cli)

	p.mnsClient[queue] = q

	return q
}

func (p *MNSSender) callFlow(delivery spirit.Delivery, flowMetadata *MNSFlowMetadata) {
	if flowMetadata == nil {
		return
	}

	sendDeliveryFunc := func(ignoreSendError bool, delivery spirit.Delivery, queueURN map[string]string, queues ...string) (err error) {
		if queues == nil || len(queues) == 0 {
			return
		}

		backupURN := delivery.URN()
		backupParallel := new(MNSParallelFlowMetadata)
		delivery.Payload().Context().Object(mnsSenderQueueParallelIdContext, backupParallel)

		// recover
		defer func() {
			delivery.SetURN(backupURN)
			delivery.Payload().SetContext(mnsSenderQueueParallelIdContext, backupParallel)
		}()

		for _, q := range queues {

			if q == "" {
				continue
			}

			parallelQueues := strings.Split(q, "||")
			parallelCount := len(parallelQueues)
			parallelQueuePid := ""

			if parallelCount > 0 {
				parallelQueuePid = xid.New().String()
			}

			for index, queue := range parallelQueues {

				// recover
				delivery.SetURN(backupURN)
				delivery.Payload().SetContext(mnsSenderQueueParallelIdContext, backupParallel)

				urn := ""
				if queueURN != nil {
					urn, _ = queueURN[queue]
				}

				buf := bytes.Buffer{}
				delivery.SetURN(urn)

				if parallelCount > 0 {
					delivery.Payload().SetContext(mnsSenderQueueParallelIdContext, MNSParallelFlowMetadata{Id: parallelQueuePid, Index: index, Count: parallelCount})
				}

				if err = p.translator.Out(&buf, delivery); err != nil {
					spirit.Logger().
						WithField("actor", spirit.ActorSender).
						WithField("urn", mnsSenderURN).
						WithField("event", "translate delivery").
						Errorln(err)
					return
				} else {
					reqData := ali_mns.MessageSendRequest{
						MessageBody:  ali_mns.Base64Bytes(buf.Bytes()),
						DelaySeconds: int64(p.conf.DelaySeconds),
						Priority:     int64(p.conf.Priority),
					}

					client := p.getMNSClient(queue)
					if _, err = client.SendMessage(reqData); err != nil {

						spirit.Logger().
							WithField("actor", spirit.ActorSender).
							WithField("urn", mnsSenderURN).
							WithField("event", "send mns message").
							Errorln(err)

						if ignoreSendError {
							err = nil
							continue
						}
						return
					}
				}
			}

		}
		return
	}

	currentQueueURN := map[string]string{}
	if err := delivery.Metadata().Object(mnsSenderQueueURNMetadata, &currentQueueURN); err != nil {
		spirit.Logger().
			WithField("actor", spirit.ActorSender).
			WithField("urn", mnsSenderURN).
			WithField("event", "get queue urn").
			Errorln(err)
	}

	if delivery.Payload().LastError() != nil {
		// run error flow

		if flowMetadata.Error != nil && len(flowMetadata.Error) > 0 {

			queueURN := map[string]string{}

			for _, queue := range flowMetadata.Error {
				if urn, exist := currentQueueURN[queue]; exist {
					queueURN[queue] = urn
				}
			}

			delivery.SetMetadata(mnsSenderFlowMetadata, nil)
			sendDeliveryFunc(true, delivery, queueURN, flowMetadata.Error...)
		}

	} else {
		// run normal flow
		if flowMetadata.Normal != nil && len(flowMetadata.Normal) > 0 {
			nextQueue := flowMetadata.Normal[flowMetadata.CurrentFlowId]

			queueURN := map[string]string{}

			if len(flowMetadata.Normal) > flowMetadata.CurrentFlowId {
				tmpRemainQueues := flowMetadata.Normal[flowMetadata.CurrentFlowId:]

				remainQueues := []string{}
				for _, queue := range tmpRemainQueues {
					queues := strings.Split(queue, "||")
					remainQueues = append(remainQueues, queues...)
				}

				for _, queue := range remainQueues {
					if urn, exist := currentQueueURN[queue]; exist {
						queueURN[queue] = urn
					}
				}
			}

			flowMetadata.CurrentFlowId += 1
			defer func() { flowMetadata.CurrentFlowId -= 1 }()

			delivery.SetMetadata(mnsSenderFlowMetadata, flowMetadata)
			sendDeliveryFunc(false, delivery, queueURN, nextQueue)
		}
	}
}

func (p *MNSSender) SetTranslator(translator spirit.OutputTranslator) (err error) {
	p.translator = translator
	return
}
<|endoftext|>


<commit_before>package pushover

import (
	"github.com/TIBCOSoftware/flogo-lib/flow/activity"
	"github.com/op/go-logging"
	"github.com/toorop/pushover"
	"time"
)

// log is the default package logger
var log = logging.MustGetLogger("activity-pushover")

const (
	ivUserKey = "userKey"
	ivAuthToken = "authToken"
	ivMessage = "message"
	ivMessageTitle = "messageTitle"
	ivDevice = "device"
	ivSound = "sound"
	ivURL = "url"
	ivURLTitle = "urlTitle"
	ivPriority = "priority"
	ovStatus = "status"
)

// MyActivity is a stub for your Activity implementation
type PushoverActivity struct {
	metadata *activity.Metadata
}

// init create & register activity
func init() {
	md := activity.NewMetadata(jsonMetadata)
	activity.Register(&PushoverActivity{metadata: md})
}

// Metadata implements activity.Activity.Metadata
func (a *PushoverActivity) Metadata() *activity.Metadata {
	return a.metadata
}

// Eval implements activity.Activity.Eval
func (a *PushoverActivity) Eval(context activity.Context) (done bool, err error) {

	userKey := context.GetInput(ivUserKey)
	authToken := context.GetInput(ivAuthToken)
	message := context.GetInput(ivMessage)
	messageTitle := context.GetInput(ivMessageTitle)
	device := context.GetInput(ivDevice)
	sound := context.GetInput(ivSound)
	url := context.GetInput(ivURL)
	urlTitle := context.GetInput(ivURLTitle)
	priority := context.GetInput(ivPriority)

	// Check if mandatory credentials are set in config
	if userKey == nil || authToken == nil {
		log.Error("Missing Pushover Credentials")
		err := activity.NewError("Credential Config not specified")
		return false, err
	}

	// Check if there is a message to send
	if message == nil {
		log.Error("No Message to Send")
		context.SetOutput(ovStatus, "NO_MSG")
		return true, nil
	}

	// Now let's try to connect
	po, err := pushover.NewPushover(authToken.(string), userKey.(string))
	if err != nil {
		log.Error("PushOver Connection Error : ", err)
		context.SetOutput(ovStatus, "CONNECT_ERR")
		return true, nil
	}

	m := pushover.Message{}

	m.Message = message.(string)
	if priority != nil {
		m.Priority = priority.(int)
	}
	if messageTitle != nil {
		m.Title = messageTitle.(string)
	}
	if device != nil {
		m.Device = device.(string)
	}
	if url != nil {
		m.Url = url.(string)
	}
	if urlTitle != nil {
		m.UrlTitle = urlTitle.(string)
	}
	m.Timestamp = time.Now().Unix()
	if sound != nil {
		m.Sound = sound.(string)
	}

	if preq, presp, err := po.Push(&m); err != nil {
		log.Error("Push Message Error : ", err)
		log.Error("Sent : ", preq)
		log.Error("Received : ", presp)
		context.SetOutput(ovStatus, "PUSH_ERR")
		return true, nil
	}

	context.SetOutput(ovStatus, "OK")

	return true, nil
}
<commit_msg>fix issue with priority attribute<commit_after>package pushover

import (
	"github.com/TIBCOSoftware/flogo-lib/flow/activity"
	"github.com/op/go-logging"
	"github.com/toorop/pushover"
	"time"
)

// log is the default package logger
var log = logging.MustGetLogger("activity-pushover")

const (
	ivUserKey = "userKey"
	ivAuthToken = "authToken"
	ivMessage = "message"
	ivMessageTitle = "messageTitle"
	ivDevice = "device"
	ivSound = "sound"
	ivURL = "url"
	ivURLTitle = "urlTitle"
	ivPriority = "priority"
	ovStatus = "status"
)

// MyActivity is a stub for your Activity implementation
type PushoverActivity struct {
	metadata *activity.Metadata
}

// init create & register activity
func init() {
	md := activity.NewMetadata(jsonMetadata)
	activity.Register(&PushoverActivity{metadata: md})
}

// Metadata implements activity.Activity.Metadata
func (a *PushoverActivity) Metadata() *activity.Metadata {
	return a.metadata
}

// Eval implements activity.Activity.Eval
func (a *PushoverActivity) Eval(context activity.Context) (done bool, err error) {

	userKey := context.GetInput(ivUserKey)
	authToken := context.GetInput(ivAuthToken)
	message := context.GetInput(ivMessage)
	messageTitle := context.GetInput(ivMessageTitle)
	device := context.GetInput(ivDevice)
	sound := context.GetInput(ivSound)
	url := context.GetInput(ivURL)
	urlTitle := context.GetInput(ivURLTitle)
	priority := context.GetInput(ivPriority).(int)

	// Check if mandatory credentials are set in config
	if userKey == nil || authToken == nil {
		log.Error("Missing Pushover Credentials")
		err := activity.NewError("Credential Config not specified")
		return false, err
	}

	// Check if there is a message to send
	if message == nil {
		log.Error("No Message to Send")
		context.SetOutput(ovStatus, "NO_MSG")
		return true, nil
	}

	// Now let's try to connect
	po, err := pushover.NewPushover(authToken.(string), userKey.(string))
	if err != nil {
		log.Error("PushOver Connection Error : ", err)
		context.SetOutput(ovStatus, "CONNECT_ERR")
		return true, nil
	}

	m := pushover.Message{}

	m.Message = message.(string)
	if priority != nil {
		m.Priority = priority
	} else {
		m.Priority = 0
	}
	if messageTitle != nil {
		m.Title = messageTitle.(string)
	}
	if device != nil {
		m.Device = device.(string)
	}
	if url != nil {
		m.Url = url.(string)
	}
	if urlTitle != nil {
		m.UrlTitle = urlTitle.(string)
	}
	m.Timestamp = time.Now().Unix()
	if sound != nil {
		m.Sound = sound.(string)
	}

	if preq, presp, err := po.Push(&m); err != nil {
		log.Error("Push Message Error : ", err)
		log.Error("Sent : ", preq)
		log.Error("Received : ", presp)
		context.SetOutput(ovStatus, "PUSH_ERR")
		return true, nil
	}

	context.SetOutput(ovStatus, "OK")

	return true, nil
}
<|endoftext|>


<commit_before>package balancer

import (
	"fmt"
	"math/rand"
	"net"
	"sort"

	"github.com/getlantern/golog"
)

var (
	log = golog.LoggerFor("balancer")
)

var (
	emptyDialers = []*dialer{}
)

// Balancer balances connections established by one or more Dialers.
type Balancer struct {
	dialers []*dialer
}

// New creates a new Balancer using the supplied Dialers.
func New(dialers ...*Dialer) *Balancer {
	dhs := make([]*dialer, 0, len(dialers))
	for _, d := range dialers {
		dl := &dialer{Dialer: d}
		dl.start()
		dhs = append(dhs, dl)
	}
	// Sort dialers by QOS (ascending) for later selection
	sort.Sort(byQOSAscending(dhs))
	return &Balancer{
		dialers: dhs,
	}
}

// DialQOS dials network, addr using one of the currently active configured
// Dialers. It attempts to use a Dialer whose QOS is higher than targetQOS, but
// will use the highest QOS Dialer(s) if none meet targetQOS. When multiple
// Dialers meet the targetQOS, load is distributed amongst them randomly based
// on their relative Weights.
//
// If a Dialer fails to connect, Dial will keep falling back through the
// remaining Dialers until it either manages to connect, or runs out of dialers
// in which case it returns an error.
func (b *Balancer) DialQOS(network, addr string, targetQOS int) (net.Conn, error) {
	dialers := b.getDialers()
	for {
		if len(dialers) == 0 {
			return nil, fmt.Errorf("No dialers left to try")
		}
		var d *dialer
		d, dialers = randomDialer(dialers, targetQOS)
		if d == nil {
			return nil, fmt.Errorf("No dialers left")
		}
		if d.Label != "" {
			log.Debugf("Dialing %s://%s with %s", network, addr, d.Label)
		}
		conn, err := d.Dial(network, addr)
		if err != nil {
			log.Tracef("Unable to dial: %s", err)
			d.onError(err)
			continue
		}
		return conn, nil
	}
}

// Dial is like DialQOS with a targetQOS of 0.
func (b *Balancer) Dial(network, addr string) (net.Conn, error) {
	return b.DialQOS(network, addr, 0)
}

// Close closes this Balancer, stopping all background processing. You must call
// Close to avoid leaking goroutines.
func (b *Balancer) Close() {
	for _, d := range b.dialers {
		d.stop()
	}
}

func (b *Balancer) getDialers() []*dialer {
	result := make([]*dialer, len(b.dialers))
	copy(result, b.dialers)
	return result
}

func randomDialer(dialers []*dialer, targetQOS int) (chosen *dialer, others []*dialer) {
	// Weed out inactive dialers and those with too low QOS, preferring higher
	// QOS
	filtered := make([]*dialer, 0)
	highestQOS := 0
	for _, d := range dialers {
		if !d.isActive() {
			log.Trace("Excluding inactive dialer")
			continue
		}

		highestQOS = d.QOS // don't need to compare since dialers are already sorted by QOS (ascending)
		if d.QOS >= targetQOS {
			log.Tracef("Including dialer with QOS %d meeting targetQOS %d", d.QOS, targetQOS)
			filtered = append(filtered, d)
		}
	}

	if len(filtered) == 0 {
		log.Trace("No dialers meet targetQOS, finding remaining with highest QOS")
		for _, d := range dialers {
			if !d.isActive() {
				log.Trace("Excluding inactive dialer")
				continue
			}

			if d.QOS == highestQOS {
				filtered = append(filtered, d)
			}
		}
	}

	if len(filtered) == 0 {
		log.Trace("Still no dialers!")
		return nil, nil
	}

	totalWeights := 0
	for _, d := range filtered {
		totalWeights += d.Weight
	}

	// Pick a random server using a target value between 0 and the total weights
	t := rand.Intn(totalWeights)
	aw := 0
	for _, d := range filtered {
		aw += d.Weight
		if aw > t {
			log.Trace("Reached random target value, using this dialer")
			return d, withoutDialer(dialers, d)
		}
	}

	// We should never reach this
	panic("No dialer found!")
}

func withoutDialer(dialers []*dialer, d *dialer) []*dialer {
	for i, existing := range dialers {
		if existing == d {
			return without(dialers, i)
		}
	}
	log.Tracef("Dialer not found for removal: %s", d)
	return dialers
}

func without(dialers []*dialer, i int) []*dialer {
	if len(dialers) == 1 {
		return emptyDialers
	} else if i == len(dialers)-1 {
		return dialers[:i]
	} else {
		c := make([]*dialer, len(dialers)-1)
		copy(c[:i], dialers[:i])
		copy(c[i:], dialers[i+1:])
		return c
	}
}

// byQOSAscending implements sort.Interface for []*dialer based on the QOS
// (ascending)
type byQOSAscending []*dialer

func (a byQOSAscending) Len() int           { return len(a) }
func (a byQOSAscending) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byQOSAscending) Less(i, j int) bool { return a[i].QOS < a[j].QOS }
<commit_msg>Removed unnecessary copy<commit_after>package balancer

import (
	"fmt"
	"math/rand"
	"net"
	"sort"

	"github.com/getlantern/golog"
)

var (
	log = golog.LoggerFor("balancer")
)

var (
	emptyDialers = []*dialer{}
)

// Balancer balances connections established by one or more Dialers.
type Balancer struct {
	dialers []*dialer
}

// New creates a new Balancer using the supplied Dialers.
func New(dialers ...*Dialer) *Balancer {
	dhs := make([]*dialer, 0, len(dialers))
	for _, d := range dialers {
		dl := &dialer{Dialer: d}
		dl.start()
		dhs = append(dhs, dl)
	}
	// Sort dialers by QOS (ascending) for later selection
	sort.Sort(byQOSAscending(dhs))
	return &Balancer{
		dialers: dhs,
	}
}

// DialQOS dials network, addr using one of the currently active configured
// Dialers. It attempts to use a Dialer whose QOS is higher than targetQOS, but
// will use the highest QOS Dialer(s) if none meet targetQOS. When multiple
// Dialers meet the targetQOS, load is distributed amongst them randomly based
// on their relative Weights.
//
// If a Dialer fails to connect, Dial will keep falling back through the
// remaining Dialers until it either manages to connect, or runs out of dialers
// in which case it returns an error.
func (b *Balancer) DialQOS(network, addr string, targetQOS int) (net.Conn, error) {
	dialers := b.dialers
	for {
		if len(dialers) == 0 {
			return nil, fmt.Errorf("No dialers left to try")
		}
		var d *dialer
		d, dialers = randomDialer(dialers, targetQOS)
		if d == nil {
			return nil, fmt.Errorf("No dialers left")
		}
		if d.Label != "" {
			log.Debugf("Dialing %s://%s with %s", network, addr, d.Label)
		}
		conn, err := d.Dial(network, addr)
		if err != nil {
			log.Tracef("Unable to dial: %s", err)
			d.onError(err)
			continue
		}
		return conn, nil
	}
}

// Dial is like DialQOS with a targetQOS of 0.
func (b *Balancer) Dial(network, addr string) (net.Conn, error) {
	return b.DialQOS(network, addr, 0)
}

// Close closes this Balancer, stopping all background processing. You must call
// Close to avoid leaking goroutines.
func (b *Balancer) Close() {
	for _, d := range b.dialers {
		d.stop()
	}
}

func randomDialer(dialers []*dialer, targetQOS int) (chosen *dialer, others []*dialer) {
	// Weed out inactive dialers and those with too low QOS, preferring higher
	// QOS
	filtered := make([]*dialer, 0)
	highestQOS := 0
	for _, d := range dialers {
		if !d.isActive() {
			log.Trace("Excluding inactive dialer")
			continue
		}

		highestQOS = d.QOS // don't need to compare since dialers are already sorted by QOS (ascending)
		if d.QOS >= targetQOS {
			log.Tracef("Including dialer with QOS %d meeting targetQOS %d", d.QOS, targetQOS)
			filtered = append(filtered, d)
		}
	}

	if len(filtered) == 0 {
		log.Trace("No dialers meet targetQOS, finding remaining with highest QOS")
		for _, d := range dialers {
			if !d.isActive() {
				log.Trace("Excluding inactive dialer")
				continue
			}

			if d.QOS == highestQOS {
				filtered = append(filtered, d)
			}
		}
	}

	if len(filtered) == 0 {
		log.Trace("Still no dialers!")
		return nil, nil
	}

	totalWeights := 0
	for _, d := range filtered {
		totalWeights += d.Weight
	}

	// Pick a random server using a target value between 0 and the total weights
	t := rand.Intn(totalWeights)
	aw := 0
	for _, d := range filtered {
		aw += d.Weight
		if aw > t {
			log.Trace("Reached random target value, using this dialer")
			return d, withoutDialer(dialers, d)
		}
	}

	// We should never reach this
	panic("No dialer found!")
}

func withoutDialer(dialers []*dialer, d *dialer) []*dialer {
	for i, existing := range dialers {
		if existing == d {
			return without(dialers, i)
		}
	}
	log.Tracef("Dialer not found for removal: %s", d)
	return dialers
}

func without(dialers []*dialer, i int) []*dialer {
	if len(dialers) == 1 {
		return emptyDialers
	} else if i == len(dialers)-1 {
		return dialers[:i]
	} else {
		c := make([]*dialer, len(dialers)-1)
		copy(c[:i], dialers[:i])
		copy(c[i:], dialers[i+1:])
		return c
	}
}

// byQOSAscending implements sort.Interface for []*dialer based on the QOS
// (ascending)
type byQOSAscending []*dialer

func (a byQOSAscending) Len() int           { return len(a) }
func (a byQOSAscending) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byQOSAscending) Less(i, j int) bool { return a[i].QOS < a[j].QOS }
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"os"
	"os/signal"
)

// interruptChannel is used to receive SIGINT (Ctrl+C) signals.
var interruptChannel chan os.Signal

// addHandlerChannel is used to add an interrupt handler to the list of handlers
// to be invoked on SIGINT (Ctrl+C) signals.
var addHandlerChannel = make(chan func())

// mainInterruptHandler listens for SIGINT (Ctrl+C) signals on the
// interruptChannel and invokes the registered interruptCallbacks accordingly.
// It also listens for callback registration.  It must be run as a goroutine.
func mainInterruptHandler() {
	// interruptCallbacks is a list of callbacks to invoke when a
	// SIGINT (Ctrl+C) is received.
	var interruptCallbacks []func()

	// isShutdown is a flag which is used to indicate whether or not
	// the shutdown signal has already been received and hence any future
	// attempts to add a new interrupt handler should invoke them
	// immediately.
	var isShutdown bool

	for {
		select {
		case <-interruptChannel:
			// Ignore more than one shutdown signal.
			if isShutdown {
				btcdLog.Infof("Received SIGINT (Ctrl+C).  " +
					"Already shutting down...")
				continue
			}

			isShutdown = true
			btcdLog.Infof("Received SIGINT (Ctrl+C).  Shutting down...")

			// Run handlers in LIFO order.
			for i := range interruptCallbacks {
				idx := len(interruptCallbacks) - 1 - i
				callback := interruptCallbacks[idx]
				callback()
			}

			// Signal the main goroutine to shutdown.
			go func() {
				shutdownChannel <- struct{}{}
			}()

		case handler := <-addHandlerChannel:
			// The shutdown signal has already been received, so
			// just invoke and new handlers immediately.
			if isShutdown {
				handler()
			}

			interruptCallbacks = append(interruptCallbacks, handler)
		}
	}
}

// addInterruptHandler adds a handler to call when a SIGINT (Ctrl+C) is
// received.
func addInterruptHandler(handler func()) {
	// Create the channel and start the main interrupt handler which invokes
	// all other callbacks and exits if not already done.
	if interruptChannel == nil {
		interruptChannel = make(chan os.Signal, 1)
		signal.Notify(interruptChannel, os.Interrupt)
		go mainInterruptHandler()
	}

	addHandlerChannel <- handler
}
<commit_msg>Catching SIGTERM signal<commit_after>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"os"
	"os/signal"
	"syscall"
)

// interruptChannel is used to receive SIGINT (Ctrl+C) signals.
var interruptChannel chan os.Signal

// addHandlerChannel is used to add an interrupt handler to the list of handlers
// to be invoked on SIGINT (Ctrl+C) signals.
var addHandlerChannel = make(chan func())

// mainInterruptHandler listens for SIGINT (Ctrl+C) signals on the
// interruptChannel and invokes the registered interruptCallbacks accordingly.
// It also listens for callback registration.  It must be run as a goroutine.
func mainInterruptHandler() {
	// interruptCallbacks is a list of callbacks to invoke when a
	// SIGINT (Ctrl+C) is received.
	var interruptCallbacks []func()

	// isShutdown is a flag which is used to indicate whether or not
	// the shutdown signal has already been received and hence any future
	// attempts to add a new interrupt handler should invoke them
	// immediately.
	var isShutdown bool

	for {
		select {
		case <-interruptChannel:
			// Ignore more than one shutdown signal.
			if isShutdown {
				btcdLog.Infof("Received SIGINT (Ctrl+C).  " +
					"Already shutting down...")
				continue
			}

			isShutdown = true
			btcdLog.Infof("Received SIGINT (Ctrl+C).  Shutting down...")

			// Run handlers in LIFO order.
			for i := range interruptCallbacks {
				idx := len(interruptCallbacks) - 1 - i
				callback := interruptCallbacks[idx]
				callback()
			}

			// Signal the main goroutine to shutdown.
			go func() {
				shutdownChannel <- struct{}{}
			}()

		case handler := <-addHandlerChannel:
			// The shutdown signal has already been received, so
			// just invoke and new handlers immediately.
			if isShutdown {
				handler()
			}

			interruptCallbacks = append(interruptCallbacks, handler)
		}
	}
}

// addInterruptHandler adds a handler to call when a SIGINT (Ctrl+C) is
// received.
func addInterruptHandler(handler func()) {
	// Create the channel and start the main interrupt handler which invokes
	// all other callbacks and exits if not already done.
	if interruptChannel == nil {
		interruptChannel = make(chan os.Signal, 1)
		signal.Notify(interruptChannel, os.Interrupt, syscall.SIGTERM)
		go mainInterruptHandler()
	}

	addHandlerChannel <- handler
}
<|endoftext|>


<commit_before>package webface

import (
	"bitbucket.org/kardianos/osext"
	"encoding/json"
	"github.com/gorilla/mux"
	"github.com/tsukanov/steaminfo-go/storage"
	"html/template"
	"log"
	"net"
	"net/http"
	"net/http/fcgi"
	"strconv"
)

func makeRouter() *mux.Router {
	r := mux.NewRouter()
	r.HandleFunc("/", homeHandler)
	r.HandleFunc("/history/{appid:[0-9]+}.json", historyHandler)
	r.HandleFunc("/search", searchHandler)
	r.HandleFunc("/about/", aboutHandler)
	return r
}

func homeHandler(w http.ResponseWriter, r *http.Request) {
	exeloc, err := osext.ExecutableFolder()
	t, err := template.ParseFiles(
		exeloc+"webface/templates/base.html",
		exeloc+"webface/templates/home.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	err = t.Execute(w, r.URL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

func historyHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	appId, err := strconv.Atoi(vars["appid"])
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	name, err := storage.GetName(appId)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	history, err := storage.AllUsageHistory(appId)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	type jason struct {
		Name    string     `json:"name"`
		History [][2]int64 `json:"history"`
	}
	result := jason{
		Name:    name,
		History: history,
	}
	b, err := json.Marshal(result)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
}

func searchHandler(w http.ResponseWriter, r *http.Request) {
	queries := r.URL.Query()
	query := queries["q"][0]
	results, err := storage.Search(query)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	b, err := json.Marshal(results)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	exeloc, err := osext.ExecutableFolder()
	t, err := template.ParseFiles(
		exeloc+"webface/templates/base.html",
		exeloc+"webface/templates/about.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	err = t.Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

// Start starts FastCGI server at 127.0.0.1:9000
func Start() {
	log.Println("Starting server...")
	l, err := net.Listen("tcp", "127.0.0.1:9000")
	if err != nil {
		log.Fatal("Failed to start server!", err)
	}
	fcgi.Serve(l, makeRouter())
}

// StartDev starts a development server at localhost:8080
func StartDev() {
	log.Println("Starting development server (localhost:8080)...")
	http.ListenAndServe(":8080", makeRouter())
}
<commit_msg>Fixed webface crash that occured if query was not passed to searchHandler. Closes #1.<commit_after>package webface

import (
	"bitbucket.org/kardianos/osext"
	"encoding/json"
	"github.com/gorilla/mux"
	"github.com/tsukanov/steaminfo-go/storage"
	"html/template"
	"log"
	"net"
	"net/http"
	"net/http/fcgi"
	"strconv"
)

func makeRouter() *mux.Router {
	r := mux.NewRouter()
	r.HandleFunc("/", homeHandler)
	r.HandleFunc("/history/{appid:[0-9]+}.json", historyHandler)
	r.HandleFunc("/search", searchHandler)
	r.HandleFunc("/about/", aboutHandler)
	return r
}

func homeHandler(w http.ResponseWriter, r *http.Request) {
	exeloc, err := osext.ExecutableFolder()
	t, err := template.ParseFiles(
		exeloc+"webface/templates/base.html",
		exeloc+"webface/templates/home.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	err = t.Execute(w, r.URL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

func historyHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	appId, err := strconv.Atoi(vars["appid"])
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	name, err := storage.GetName(appId)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	history, err := storage.AllUsageHistory(appId)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	type jason struct {
		Name    string     `json:"name"`
		History [][2]int64 `json:"history"`
	}
	result := jason{
		Name:    name,
		History: history,
	}
	b, err := json.Marshal(result)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
}

func searchHandler(w http.ResponseWriter, r *http.Request) {
	queries := r.URL.Query()
	query, ok := queries["q"]
	if !ok {
		http.Error(w, "No query", http.StatusBadRequest)
		return
	}
	results, err := storage.Search(query[0])
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	b, err := json.Marshal(results)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	exeloc, err := osext.ExecutableFolder()
	t, err := template.ParseFiles(
		exeloc+"webface/templates/base.html",
		exeloc+"webface/templates/about.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	err = t.Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

// Start starts FastCGI server at 127.0.0.1:9000
func Start() {
	log.Println("Starting server...")
	l, err := net.Listen("tcp", "127.0.0.1:9000")
	if err != nil {
		log.Fatal("Failed to start server!", err)
	}
	fcgi.Serve(l, makeRouter())
}

// StartDev starts a development server at localhost:8080
func StartDev() {
	log.Println("Starting development server (localhost:8080)...")
	http.ListenAndServe(":8080", makeRouter())
}
<|endoftext|>


<commit_before><commit_msg>Removed json package<commit_after><|endoftext|>


<commit_before>package brats_test

import (
	"fmt"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

func extractAzIpsMap(regex *regexp.Regexp, contents string) map[string][]string {
	out := map[string][]string{
		"z1": {},
		"z2": {},
	}

	instances := regex.FindAllStringSubmatch(contents, -1)
	Expect(instances).ToNot(BeNil())
	for _, q := range instances {
		out[q[1]] = append(out[q[1]], q[2])
	}

	return out
}

func mustGetLatestDnsVersions() []int {
	session, err := gexec.Start(exec.Command(
		boshBinaryPath, "-n",
		"-d", deploymentName,
		"ssh",
		"-c", "sudo cat /var/vcap/instance/dns/records.json",
	), GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())
	Eventually(session, time.Minute).Should(gexec.Exit(0))

	trimmedOutput := strings.TrimSpace(string(session.Out.Contents()))

	results := extractDnsVersionsList(trimmedOutput)
	Expect(len(results)).To(BeNumerically(">", 0))

	return results
}

var versionSegmentsPattern = regexp.MustCompile(`"version":(\d+)`)

func extractDnsVersionsList(sshContents string) []int {
	matches := versionSegmentsPattern.FindAllStringSubmatch(sshContents, -1)
	Expect(matches).ToNot(BeNil())
	results := make([]int, len(matches))

	for i, match := range matches {
		Expect(len(match)).To(Equal(2))
		value, err := strconv.Atoi(match[1])
		Expect(err).ToNot(HaveOccurred())

		results[i] = value
	}
	return results
}

var _ = Describe("BoshDns", func() {
	var (
		manifestPath              string
		linkedTemplateReleasePath string
	)

	BeforeEach(func() {
		startInnerBosh()

		session, err := gexec.Start(exec.Command(boshBinaryPath, "-n", "upload-stemcell", candidateWardenLinuxStemcellPath), GinkgoWriter, GinkgoWriter)
		Expect(err).ToNot(HaveOccurred())
		Eventually(session, 5*time.Minute).Should(gexec.Exit(0))

		manifestPath, err = filepath.Abs("../assets/dns-with-templates-manifest.yml")
		Expect(err).NotTo(HaveOccurred())

		linkedTemplateReleasePath, err = filepath.Abs("../assets/linked-templates-release")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(stopInnerBosh)

	Context("having enabled short dns addresses", func() {
		BeforeEach(func() {
			opFilePath, err := filepath.Abs("../assets/op-enable-short-dns-addresses.yml")
			Expect(err).NotTo(HaveOccurred())

			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "deploy",
				"-n",
				"-d", deploymentName,
				manifestPath,
				"-o", opFilePath,
				"-v", fmt.Sprintf("dns-release-path=%s", dnsReleasePath),
				"-v", fmt.Sprintf("linked-template-release-path=%s", linkedTemplateReleasePath),
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, 15*time.Minute).Should(gexec.Exit(0))
		})

		It("can find instances using the address helper with short names", func() {
			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "-n",
				"-d", deploymentName,
				"instances",
				"--column", "instance",
				"--column", "az",
				"--column", "ips",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			instanceList := session.Out.Contents()

			matchExpression := regexp.MustCompile(`provider\S+\s+(z1|z2)\s+(\S+)`)
			knownProviders := extractAzIpsMap(matchExpression, string(instanceList))

			session, err = gexec.Start(exec.Command(boshBinaryPath,
				"-d", deploymentName,
				"run-errand", "query-all",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			Expect(session.Out).To(gbytes.Say("ANSWER: 3"))

			output := string(session.Out.Contents())

			for _, ips := range knownProviders {
				for _, ip := range ips {
					Expect(output).To(MatchRegexp(`q-s0\.g-\d+\.bosh\.\s+\d+\s+IN\s+A\s+%s`, ip))
				}
			}
		})

		PIt("can find instances using the address helper with short names by network and instance ID", func() {
			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "-n",
				"-d", deploymentName,
				"instances",
				"--column", "instance",
				"--column", "az",
				"--column", "ips",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			instanceList := session.Out.Contents()

			matchExpression := regexp.MustCompile(`provider\S+\s+(z1)\s+(\S+)`)
			knownProviders := extractAzIpsMap(matchExpression, string(instanceList))

			session, err = gexec.Start(exec.Command(boshBinaryPath,
				"-d", deploymentName,
				"run-errand", "query-individual-instance",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			Expect(session.Out).To(gbytes.Say("ANSWER: 1"))

			output := string(session.Out.Contents())

			ip := knownProviders["z1"][0]
			Expect(output).To(MatchRegexp(`q-n\d+m\d+\.g-\d\.bosh\.\s+\d+\s+IN\s+A\s+%s`, ip))
		})
	})

	Context("When deploying vms across different azs", func() {
		BeforeEach(func() {
			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "deploy",
				"-n",
				"-d", deploymentName,
				manifestPath,
				"-v", fmt.Sprintf("dns-release-path=%s", dnsReleasePath),
				"-v", fmt.Sprintf("linked-template-release-path=%s", linkedTemplateReleasePath),
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, 15*time.Minute).Should(gexec.Exit(0))
		})

		It("can find instances using the address helper", func() {
			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "-n",
				"-d", deploymentName,
				"instances",
				"--column", "instance",
				"--column", "az",
				"--column", "ips",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			instanceList := session.Out.Contents()

			By("finding instances in all AZs", func() {
				matchExpression := regexp.MustCompile(`provider\S+\s+(z1|z2)\s+(\S+)`)
				knownProviders := extractAzIpsMap(matchExpression, string(instanceList))

				session, err = gexec.Start(exec.Command(boshBinaryPath,
					"-d", deploymentName,
					"run-errand", "query-all",
				), GinkgoWriter, GinkgoWriter)
				Expect(err).ToNot(HaveOccurred())
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				Expect(session.Out).To(gbytes.Say("ANSWER: 3"))
				output := string(session.Out.Contents())

				for _, ips := range knownProviders {
					for _, ip := range ips {
						Expect(output).To(ContainSubstring(ip))
					}
				}
			})

			By("finding instances filtering by AZ", func() {
				matchExpression := regexp.MustCompile(`provider\S+\s+(z1)\s+(\S+)`)
				knownProviders := extractAzIpsMap(matchExpression, string(instanceList))

				session, err = gexec.Start(exec.Command(boshBinaryPath,
					"-d", deploymentName,
					"run-errand", "query-with-az-filter",
				), GinkgoWriter, GinkgoWriter)
				Expect(err).ToNot(HaveOccurred())
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				Expect(session.Out).To(gbytes.Say("ANSWER: 2"))
				output := string(session.Out.Contents())

				for _, ips := range knownProviders {
					for _, ip := range ips {
						Expect(output).To(ContainSubstring(ip))
					}
				}
			})
		})

		It("can force a new DNS blob to propagate to ALL vms", func() {
			versionPerInstance := mustGetLatestDnsVersions()
			previousMax := -1
			for _, version := range versionPerInstance {
				if previousMax < version {
					previousMax = version
				}
			}

			session, err := gexec.Start(exec.Command("ssh",
				fmt.Sprintf("%s@%s", innerDirectorUser, innerDirectorIP),
				"-i", innerBoshJumpboxPrivateKeyPath,
				"-oStrictHostKeyChecking=no",
				"sudo /var/vcap/jobs/director/bin/sync_dns_ctl force"),
				GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, 2*time.Minute).Should(gexec.Exit(0))

			newVersionPerInstance := mustGetLatestDnsVersions()
			firstNewVersion := newVersionPerInstance[0]
			Expect(firstNewVersion).To(BeNumerically(">", previousMax))
			for _, version := range newVersionPerInstance {
				Expect(version).To(Equal(firstNewVersion))
			}
		})
	})
})
<commit_msg>Add n filter to expected output for dig results<commit_after>package brats_test

import (
	"fmt"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

func extractAzIpsMap(regex *regexp.Regexp, contents string) map[string][]string {
	out := map[string][]string{
		"z1": {},
		"z2": {},
	}

	instances := regex.FindAllStringSubmatch(contents, -1)
	Expect(instances).ToNot(BeNil())
	for _, q := range instances {
		out[q[1]] = append(out[q[1]], q[2])
	}

	return out
}

func mustGetLatestDnsVersions() []int {
	session, err := gexec.Start(exec.Command(
		boshBinaryPath, "-n",
		"-d", deploymentName,
		"ssh",
		"-c", "sudo cat /var/vcap/instance/dns/records.json",
	), GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())
	Eventually(session, time.Minute).Should(gexec.Exit(0))

	trimmedOutput := strings.TrimSpace(string(session.Out.Contents()))

	results := extractDnsVersionsList(trimmedOutput)
	Expect(len(results)).To(BeNumerically(">", 0))

	return results
}

var versionSegmentsPattern = regexp.MustCompile(`"version":(\d+)`)

func extractDnsVersionsList(sshContents string) []int {
	matches := versionSegmentsPattern.FindAllStringSubmatch(sshContents, -1)
	Expect(matches).ToNot(BeNil())
	results := make([]int, len(matches))

	for i, match := range matches {
		Expect(len(match)).To(Equal(2))
		value, err := strconv.Atoi(match[1])
		Expect(err).ToNot(HaveOccurred())

		results[i] = value
	}
	return results
}

var _ = Describe("BoshDns", func() {
	var (
		manifestPath              string
		linkedTemplateReleasePath string
	)

	BeforeEach(func() {
		startInnerBosh()

		session, err := gexec.Start(exec.Command(boshBinaryPath, "-n", "upload-stemcell", candidateWardenLinuxStemcellPath), GinkgoWriter, GinkgoWriter)
		Expect(err).ToNot(HaveOccurred())
		Eventually(session, 5*time.Minute).Should(gexec.Exit(0))

		manifestPath, err = filepath.Abs("../assets/dns-with-templates-manifest.yml")
		Expect(err).NotTo(HaveOccurred())

		linkedTemplateReleasePath, err = filepath.Abs("../assets/linked-templates-release")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(stopInnerBosh)

	Context("having enabled short dns addresses", func() {
		BeforeEach(func() {
			opFilePath, err := filepath.Abs("../assets/op-enable-short-dns-addresses.yml")
			Expect(err).NotTo(HaveOccurred())

			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "deploy",
				"-n",
				"-d", deploymentName,
				manifestPath,
				"-o", opFilePath,
				"-v", fmt.Sprintf("dns-release-path=%s", dnsReleasePath),
				"-v", fmt.Sprintf("linked-template-release-path=%s", linkedTemplateReleasePath),
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, 15*time.Minute).Should(gexec.Exit(0))
		})

		It("can find instances using the address helper with short names", func() {
			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "-n",
				"-d", deploymentName,
				"instances",
				"--column", "instance",
				"--column", "az",
				"--column", "ips",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			instanceList := session.Out.Contents()

			matchExpression := regexp.MustCompile(`provider\S+\s+(z1|z2)\s+(\S+)`)
			knownProviders := extractAzIpsMap(matchExpression, string(instanceList))

			session, err = gexec.Start(exec.Command(boshBinaryPath,
				"-d", deploymentName,
				"run-errand", "query-all",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			Expect(session.Out).To(gbytes.Say("ANSWER: 3"))

			output := string(session.Out.Contents())

			for _, ips := range knownProviders {
				for _, ip := range ips {
					Expect(output).To(MatchRegexp(`q-n\d+s0\.g-\d+\.bosh\.\s+\d+\s+IN\s+A\s+%s`, ip))
				}
			}
		})

		PIt("can find instances using the address helper with short names by network and instance ID", func() {
			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "-n",
				"-d", deploymentName,
				"instances",
				"--column", "instance",
				"--column", "az",
				"--column", "ips",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			instanceList := session.Out.Contents()

			matchExpression := regexp.MustCompile(`provider\S+\s+(z1)\s+(\S+)`)
			knownProviders := extractAzIpsMap(matchExpression, string(instanceList))

			session, err = gexec.Start(exec.Command(boshBinaryPath,
				"-d", deploymentName,
				"run-errand", "query-individual-instance",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			Expect(session.Out).To(gbytes.Say("ANSWER: 1"))

			output := string(session.Out.Contents())

			ip := knownProviders["z1"][0]
			Expect(output).To(MatchRegexp(`q-n\d+m\d+\.g-\d\.bosh\.\s+\d+\s+IN\s+A\s+%s`, ip))
		})
	})

	Context("When deploying vms across different azs", func() {
		BeforeEach(func() {
			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "deploy",
				"-n",
				"-d", deploymentName,
				manifestPath,
				"-v", fmt.Sprintf("dns-release-path=%s", dnsReleasePath),
				"-v", fmt.Sprintf("linked-template-release-path=%s", linkedTemplateReleasePath),
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, 15*time.Minute).Should(gexec.Exit(0))
		})

		It("can find instances using the address helper", func() {
			session, err := gexec.Start(exec.Command(
				boshBinaryPath, "-n",
				"-d", deploymentName,
				"instances",
				"--column", "instance",
				"--column", "az",
				"--column", "ips",
			), GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, time.Minute).Should(gexec.Exit(0))

			instanceList := session.Out.Contents()

			By("finding instances in all AZs", func() {
				matchExpression := regexp.MustCompile(`provider\S+\s+(z1|z2)\s+(\S+)`)
				knownProviders := extractAzIpsMap(matchExpression, string(instanceList))

				session, err = gexec.Start(exec.Command(boshBinaryPath,
					"-d", deploymentName,
					"run-errand", "query-all",
				), GinkgoWriter, GinkgoWriter)
				Expect(err).ToNot(HaveOccurred())
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				Expect(session.Out).To(gbytes.Say("ANSWER: 3"))
				output := string(session.Out.Contents())

				for _, ips := range knownProviders {
					for _, ip := range ips {
						Expect(output).To(ContainSubstring(ip))
					}
				}
			})

			By("finding instances filtering by AZ", func() {
				matchExpression := regexp.MustCompile(`provider\S+\s+(z1)\s+(\S+)`)
				knownProviders := extractAzIpsMap(matchExpression, string(instanceList))

				session, err = gexec.Start(exec.Command(boshBinaryPath,
					"-d", deploymentName,
					"run-errand", "query-with-az-filter",
				), GinkgoWriter, GinkgoWriter)
				Expect(err).ToNot(HaveOccurred())
				Eventually(session, time.Minute).Should(gexec.Exit(0))

				Expect(session.Out).To(gbytes.Say("ANSWER: 2"))
				output := string(session.Out.Contents())

				for _, ips := range knownProviders {
					for _, ip := range ips {
						Expect(output).To(ContainSubstring(ip))
					}
				}
			})
		})

		It("can force a new DNS blob to propagate to ALL vms", func() {
			versionPerInstance := mustGetLatestDnsVersions()
			previousMax := -1
			for _, version := range versionPerInstance {
				if previousMax < version {
					previousMax = version
				}
			}

			session, err := gexec.Start(exec.Command("ssh",
				fmt.Sprintf("%s@%s", innerDirectorUser, innerDirectorIP),
				"-i", innerBoshJumpboxPrivateKeyPath,
				"-oStrictHostKeyChecking=no",
				"sudo /var/vcap/jobs/director/bin/sync_dns_ctl force"),
				GinkgoWriter, GinkgoWriter)
			Expect(err).ToNot(HaveOccurred())
			Eventually(session, 2*time.Minute).Should(gexec.Exit(0))

			newVersionPerInstance := mustGetLatestDnsVersions()
			firstNewVersion := newVersionPerInstance[0]
			Expect(firstNewVersion).To(BeNumerically(">", previousMax))
			for _, version := range newVersionPerInstance {
				Expect(version).To(Equal(firstNewVersion))
			}
		})
	})
})
<|endoftext|>