<commit_before>/*
 * Copyright 2019 The Knative Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package filter

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"sync/atomic"
	"time"

	cloudevents "github.com/cloudevents/sdk-go"
	cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http"
	"go.uber.org/zap"
	eventingv1alpha1 "knative.dev/eventing/pkg/apis/eventing/v1alpha1"
	"knative.dev/eventing/pkg/broker"
	eventinglisters "knative.dev/eventing/pkg/client/listers/eventing/v1alpha1"
	"knative.dev/eventing/pkg/logging"
	"knative.dev/eventing/pkg/reconciler/trigger/path"
	"knative.dev/pkg/tracing"
)

const (
	writeTimeout = 15 * time.Minute

	passFilter FilterResult = "pass"
	failFilter FilterResult = "fail"
	noFilter   FilterResult = "no_filter"
)

// Handler parses Cloud Events, determines if they pass a filter, and sends them to a subscriber.
type Handler struct {
	logger        *zap.Logger
	triggerLister eventinglisters.TriggerNamespaceLister
	ceClient      cloudevents.Client
	reporter      StatsReporter
	isReady       *atomic.Value
}

// FilterResult has the result of the filtering operation.
type FilterResult string

// NewHandler creates a new Handler and its associated MessageReceiver. The caller is responsible for
// Start()ing the returned Handler.
func NewHandler(logger *zap.Logger, triggerLister eventinglisters.TriggerNamespaceLister, reporter StatsReporter) (*Handler, error) {
	httpTransport, err := cloudevents.NewHTTPTransport(cloudevents.WithBinaryEncoding(), cehttp.WithMiddleware(tracing.HTTPSpanMiddleware))
	if err != nil {
		return nil, err
	}

	ceClient, err := cloudevents.NewClient(httpTransport, cloudevents.WithTimeNow(), cloudevents.WithUUIDs())
	if err != nil {
		return nil, err
	}

	r := &Handler{
		logger:        logger,
		triggerLister: triggerLister,
		ceClient:      ceClient,
		reporter:      reporter,
		isReady:       &atomic.Value{},
	}
	r.isReady.Store(false)

	httpTransport.Handler = http.NewServeMux()
	httpTransport.Handler.HandleFunc("/healthz", r.healthZ)
	httpTransport.Handler.HandleFunc("/readyz", r.readyZ)

	return r, nil
}

func (r *Handler) healthZ(writer http.ResponseWriter, _ *http.Request) {
	writer.WriteHeader(http.StatusOK)
}

func (r *Handler) readyZ(writer http.ResponseWriter, _ *http.Request) {
	if r.isReady == nil || !r.isReady.Load().(bool) {
		http.Error(writer, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
		return
	}
	writer.WriteHeader(http.StatusOK)
}

// Start begins to receive messages for the handler.
//
// Only HTTP POST requests to the root path (/) are accepted. If other paths or
// methods are needed, use the HandleRequest method directly with another HTTP
// server.
//
// This method will block until a message is received on the stop channel.
func (r *Handler) Start(ctx context.Context) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	errCh := make(chan error, 1)
	go func() {
		errCh <- r.ceClient.StartReceiver(ctx, r.serveHTTP)
	}()

	// We are ready.
	r.isReady.Store(true)

	// Stop either if the receiver stops (sending to errCh) or if stopCh is closed.
	select {
	case err := <-errCh:
		return err
	case <-ctx.Done():
		break
	}

	// No longer ready.
	r.isReady.Store(false)

	// stopCh has been closed, we need to gracefully shutdown h.ceClient. cancel() will start its
	// shutdown, if it hasn't finished in a reasonable amount of time, just return an error.
	cancel()
	select {
	case err := <-errCh:
		return err
	case <-time.After(writeTimeout):
		return errors.New("timeout shutting down ceClient")
	}
}

func (r *Handler) serveHTTP(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error {
	tctx := cloudevents.HTTPTransportContextFrom(ctx)
	if tctx.Method != http.MethodPost {
		resp.Status = http.StatusMethodNotAllowed
		return nil
	}

	// tctx.URI is actually the path...
	triggerRef, err := path.Parse(tctx.URI)
	if err != nil {
		r.logger.Info("Unable to parse path as a trigger", zap.Error(err), zap.String("path", tctx.URI))
		return errors.New("unable to parse path as a Trigger")
	}

	// Remove the TTL attribute that is used by the Broker.
	originalV3 := event.Context.AsV03()
	ttl, ttlKey := broker.GetTTL(event.Context)
	if ttl == nil {
		// Only messages sent by the Broker should be here. If the attribute isn't here, then the
		// event wasn't sent by the Broker, so we can drop it.
		r.logger.Warn("No TTL seen, dropping", zap.Any("triggerRef", triggerRef), zap.Any("event", event))
		// This doesn't return an error because normally this function is called by a Channel, which
		// will retry all non-2XX responses. If we return an error from this function, then the
		// framework returns a 500 to the caller, so the Channel would send this repeatedly.
		return nil
	}
	delete(originalV3.Extensions, ttlKey)
	event.Context = originalV3

	r.logger.Debug("Received message", zap.Any("triggerRef", triggerRef))

	responseEvent, err := r.sendEvent(ctx, tctx, triggerRef, &event)
	if err != nil {
		r.logger.Error("Error sending the event", zap.Error(err))
		return err
	}

	resp.Status = http.StatusAccepted
	if responseEvent == nil {
		return nil
	}

	// Reattach the TTL (with the same value) to the response event before sending it to the Broker.
	responseEvent.Context, err = broker.SetTTL(responseEvent.Context, ttl)
	if err != nil {
		return err
	}
	resp.Event = responseEvent
	resp.Context = &cloudevents.HTTPTransportResponseContext{
		Header: broker.ExtractPassThroughHeaders(tctx),
	}

	return nil
}

// sendEvent sends an event to a subscriber if the trigger filter passes.
func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportContext, trigger path.NamespacedNameUID, event *cloudevents.Event) (*cloudevents.Event, error) {
	t, err := r.getTrigger(ctx, trigger)
	if err != nil {
		r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger))
		return nil, err
	}

	reportArgs := &ReportArgs{
		ns:           t.Namespace,
		trigger:      t.Name,
		broker:       t.Spec.Broker,
		filterType:   triggerFilterAttribute(t.Spec.Filter, "type"),
		filterSource: triggerFilterAttribute(t.Spec.Filter, "source"),
	}

	subscriberURIString := t.Status.SubscriberURI
	if subscriberURIString == "" {
		err = errors.New("unable to read subscriberURI")
		// Record the event count.
		r.reporter.ReportEventCount(reportArgs, http.StatusNotFound)
		return nil, err
	}
	// We could just send the request to this URI regardless, but let's just check to see if it well
	// formed first, that way we can generate better error message if it isn't.
	subscriberURI, err := url.Parse(subscriberURIString)
	if err != nil {
		r.logger.Error("Unable to parse subscriberURI", zap.Error(err), zap.String("subscriberURIString", subscriberURIString))
		// Record the event count.
		r.reporter.ReportEventCount(reportArgs, http.StatusInternalServerError)
		return nil, err
	}

	// Check if the event should be sent.
	filterResult := r.shouldSendEvent(ctx, &t.Spec, event)

	if filterResult == failFilter {
		r.logger.Debug("Event did not pass filter", zap.Any("triggerRef", trigger))
		// Record the event count.
		r.reporter.ReportEventCount(reportArgs, http.StatusExpectationFailed)
		return nil, nil
	}

	// Record the event processing time. This might be off if the receiver and the filter pods are running in
	// different nodes with different clocks.
	var arrivalTimeStr string
	if extErr := event.ExtensionAs(broker.EventArrivalTime, &arrivalTimeStr); extErr == nil {
		arrivalTime, err := time.Parse(time.RFC3339, arrivalTimeStr)
		if err == nil {
			r.reporter.ReportEventProcessingTime(reportArgs, time.Since(arrivalTime))
		}
	}

	start := time.Now()
	sendingCTX := broker.SendingContext(ctx, tctx, subscriberURI)
	rctx, replyEvent, err := r.ceClient.Send(sendingCTX, *event)
	rtctx := cloudevents.HTTPTransportContextFrom(rctx)
	// Record the dispatch time.
	r.reporter.ReportEventDispatchTime(reportArgs, rtctx.StatusCode, time.Since(start))
	// Record the event count.
	r.reporter.ReportEventCount(reportArgs, rtctx.StatusCode)
	return replyEvent, err
}

func (r *Handler) getTrigger(ctx context.Context, ref path.NamespacedNameUID) (*eventingv1alpha1.Trigger, error) {
	t, err := r.triggerLister.Get(ref.Name)
	if err != nil {
		return nil, err
	}
	if t.UID != ref.UID {
		return nil, fmt.Errorf("trigger had a different UID. From ref '%s'. From Kubernetes '%s'", ref.UID, t.UID)
	}
	return t, nil
}

// shouldSendEvent determines whether event 'event' should be sent based on the triggerSpec 'ts'.
// Currently it supports exact matching on event context attributes and extension attributes.
// If no filter is present, shouldSendEvent returns passFilter.
func (r *Handler) shouldSendEvent(ctx context.Context, ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) FilterResult {
	// No filter specified, default to passing everything.
	if ts.Filter == nil || (ts.Filter.DeprecatedSourceAndType == nil && ts.Filter.Attributes == nil) {
		return noFilter
	}

	attrs := map[string]string{}
	// Since the filters cannot distinguish presence, filtering for an empty
	// string is impossible.
	if ts.Filter.DeprecatedSourceAndType != nil {
		attrs["type"] = ts.Filter.DeprecatedSourceAndType.Type
		attrs["source"] = ts.Filter.DeprecatedSourceAndType.Source
	} else if ts.Filter.Attributes != nil {
		attrs = map[string]string(*ts.Filter.Attributes)
	}

	return r.filterEventByAttributes(ctx, attrs, event)
}

func (r *Handler) filterEventByAttributes(ctx context.Context, attrs map[string]string, event *cloudevents.Event) FilterResult {
	// Set standard context attributes. The attributes available may not be
	// exactly the same as the attributes defined in the current version of the
	// CloudEvents spec.
	ce := map[string]interface{}{
		"specversion":         event.SpecVersion(),
		"type":                event.Type(),
		"source":              event.Source(),
		"subject":             event.Subject(),
		"id":                  event.ID(),
		"time":                event.Time().String(),
		"schemaurl":           event.SchemaURL(),
		"datacontenttype":     event.DataContentType(),
		"datamediatype":       event.DataMediaType(),
		"datacontentencoding": event.DataContentEncoding(),
	}
	ext := event.Extensions()
	if ext != nil {
		for k, v := range ext {
			ce[k] = v
		}
	}

	for k, v := range attrs {
		var value interface{}
		value, ok := ce[k]
		// If the attribute does not exist in the event, return false.
		if !ok {
			logging.FromContext(ctx).Debug("Attribute not found", zap.String("attribute", k))
			return failFilter
		}
		// If the attribute is not set to any and is different than the one from the event, return false.
		if v != eventingv1alpha1.TriggerAnyFilter && v != value {
			logging.FromContext(ctx).Debug("Attribute had non-matching value", zap.String("attribute", k), zap.String("filter", v), zap.Any("received", value))
			return failFilter
		}
	}
	return passFilter
}

// triggerFilterAttribute returns the filter attribute value for a given `attributeName`. If it doesn't not exist,
// returns the any value filter.
func triggerFilterAttribute(filter *eventingv1alpha1.TriggerFilter, attributeName string) string {
	attributeValue := eventingv1alpha1.TriggerAnyFilter
	if filter != nil {
		if filter.DeprecatedSourceAndType != nil {
			if attributeName == "type" {
				attributeValue = filter.DeprecatedSourceAndType.Type
			} else if attributeName == "source" {
				attributeValue = filter.DeprecatedSourceAndType.Source
			}
		} else if filter.Attributes != nil {
			attrs := map[string]string(*filter.Attributes)
			if v, ok := attrs[attributeName]; ok {
				attributeValue = v
			}
		}
	}
	return attributeValue
}
<commit_msg>Do not trace readiness requests. (#1844)<commit_after>/*
 * Copyright 2019 The Knative Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package filter

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"sync/atomic"
	"time"

	cloudevents "github.com/cloudevents/sdk-go"
	cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http"
	"go.uber.org/zap"
	eventingv1alpha1 "knative.dev/eventing/pkg/apis/eventing/v1alpha1"
	"knative.dev/eventing/pkg/broker"
	eventinglisters "knative.dev/eventing/pkg/client/listers/eventing/v1alpha1"
	"knative.dev/eventing/pkg/logging"
	"knative.dev/eventing/pkg/reconciler/trigger/path"
	"knative.dev/pkg/tracing"
)

const (
	writeTimeout = 15 * time.Minute

	passFilter FilterResult = "pass"
	failFilter FilterResult = "fail"
	noFilter   FilterResult = "no_filter"

	// readyz is the HTTP path that will be used for readiness checks.
	readyz = "/readyz"
)

// Handler parses Cloud Events, determines if they pass a filter, and sends them to a subscriber.
type Handler struct {
	logger        *zap.Logger
	triggerLister eventinglisters.TriggerNamespaceLister
	ceClient      cloudevents.Client
	reporter      StatsReporter
	isReady       *atomic.Value
}

// FilterResult has the result of the filtering operation.
type FilterResult string

// NewHandler creates a new Handler and its associated MessageReceiver. The caller is responsible for
// Start()ing the returned Handler.
func NewHandler(logger *zap.Logger, triggerLister eventinglisters.TriggerNamespaceLister, reporter StatsReporter) (*Handler, error) {
	httpTransport, err := cloudevents.NewHTTPTransport(cloudevents.WithBinaryEncoding(), cehttp.WithMiddleware(tracing.HTTPSpanIgnoringPaths(readyz)))
	if err != nil {
		return nil, err
	}

	ceClient, err := cloudevents.NewClient(httpTransport, cloudevents.WithTimeNow(), cloudevents.WithUUIDs())
	if err != nil {
		return nil, err
	}

	r := &Handler{
		logger:        logger,
		triggerLister: triggerLister,
		ceClient:      ceClient,
		reporter:      reporter,
		isReady:       &atomic.Value{},
	}
	r.isReady.Store(false)

	httpTransport.Handler = http.NewServeMux()
	httpTransport.Handler.HandleFunc("/healthz", r.healthZ)
	httpTransport.Handler.HandleFunc(readyz, r.readyZ)

	return r, nil
}

func (r *Handler) healthZ(writer http.ResponseWriter, _ *http.Request) {
	writer.WriteHeader(http.StatusOK)
}

func (r *Handler) readyZ(writer http.ResponseWriter, _ *http.Request) {
	if r.isReady == nil || !r.isReady.Load().(bool) {
		http.Error(writer, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
		return
	}
	writer.WriteHeader(http.StatusOK)
}

// Start begins to receive messages for the handler.
//
// Only HTTP POST requests to the root path (/) are accepted. If other paths or
// methods are needed, use the HandleRequest method directly with another HTTP
// server.
//
// This method will block until a message is received on the stop channel.
func (r *Handler) Start(ctx context.Context) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	errCh := make(chan error, 1)
	go func() {
		errCh <- r.ceClient.StartReceiver(ctx, r.serveHTTP)
	}()

	// We are ready.
	r.isReady.Store(true)

	// Stop either if the receiver stops (sending to errCh) or if stopCh is closed.
	select {
	case err := <-errCh:
		return err
	case <-ctx.Done():
		break
	}

	// No longer ready.
	r.isReady.Store(false)

	// stopCh has been closed, we need to gracefully shutdown h.ceClient. cancel() will start its
	// shutdown, if it hasn't finished in a reasonable amount of time, just return an error.
	cancel()
	select {
	case err := <-errCh:
		return err
	case <-time.After(writeTimeout):
		return errors.New("timeout shutting down ceClient")
	}
}

func (r *Handler) serveHTTP(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error {
	tctx := cloudevents.HTTPTransportContextFrom(ctx)
	if tctx.Method != http.MethodPost {
		resp.Status = http.StatusMethodNotAllowed
		return nil
	}

	// tctx.URI is actually the path...
	triggerRef, err := path.Parse(tctx.URI)
	if err != nil {
		r.logger.Info("Unable to parse path as a trigger", zap.Error(err), zap.String("path", tctx.URI))
		return errors.New("unable to parse path as a Trigger")
	}

	// Remove the TTL attribute that is used by the Broker.
	originalV3 := event.Context.AsV03()
	ttl, ttlKey := broker.GetTTL(event.Context)
	if ttl == nil {
		// Only messages sent by the Broker should be here. If the attribute isn't here, then the
		// event wasn't sent by the Broker, so we can drop it.
		r.logger.Warn("No TTL seen, dropping", zap.Any("triggerRef", triggerRef), zap.Any("event", event))
		// This doesn't return an error because normally this function is called by a Channel, which
		// will retry all non-2XX responses. If we return an error from this function, then the
		// framework returns a 500 to the caller, so the Channel would send this repeatedly.
		return nil
	}
	delete(originalV3.Extensions, ttlKey)
	event.Context = originalV3

	r.logger.Debug("Received message", zap.Any("triggerRef", triggerRef))

	responseEvent, err := r.sendEvent(ctx, tctx, triggerRef, &event)
	if err != nil {
		r.logger.Error("Error sending the event", zap.Error(err))
		return err
	}

	resp.Status = http.StatusAccepted
	if responseEvent == nil {
		return nil
	}

	// Reattach the TTL (with the same value) to the response event before sending it to the Broker.
	responseEvent.Context, err = broker.SetTTL(responseEvent.Context, ttl)
	if err != nil {
		return err
	}
	resp.Event = responseEvent
	resp.Context = &cloudevents.HTTPTransportResponseContext{
		Header: broker.ExtractPassThroughHeaders(tctx),
	}

	return nil
}

// sendEvent sends an event to a subscriber if the trigger filter passes.
func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportContext, trigger path.NamespacedNameUID, event *cloudevents.Event) (*cloudevents.Event, error) {
	t, err := r.getTrigger(ctx, trigger)
	if err != nil {
		r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger))
		return nil, err
	}

	reportArgs := &ReportArgs{
		ns:           t.Namespace,
		trigger:      t.Name,
		broker:       t.Spec.Broker,
		filterType:   triggerFilterAttribute(t.Spec.Filter, "type"),
		filterSource: triggerFilterAttribute(t.Spec.Filter, "source"),
	}

	subscriberURIString := t.Status.SubscriberURI
	if subscriberURIString == "" {
		err = errors.New("unable to read subscriberURI")
		// Record the event count.
		r.reporter.ReportEventCount(reportArgs, http.StatusNotFound)
		return nil, err
	}
	// We could just send the request to this URI regardless, but let's just check to see if it well
	// formed first, that way we can generate better error message if it isn't.
	subscriberURI, err := url.Parse(subscriberURIString)
	if err != nil {
		r.logger.Error("Unable to parse subscriberURI", zap.Error(err), zap.String("subscriberURIString", subscriberURIString))
		// Record the event count.
		r.reporter.ReportEventCount(reportArgs, http.StatusInternalServerError)
		return nil, err
	}

	// Check if the event should be sent.
	filterResult := r.shouldSendEvent(ctx, &t.Spec, event)

	if filterResult == failFilter {
		r.logger.Debug("Event did not pass filter", zap.Any("triggerRef", trigger))
		// Record the event count.
		r.reporter.ReportEventCount(reportArgs, http.StatusExpectationFailed)
		return nil, nil
	}

	// Record the event processing time. This might be off if the receiver and the filter pods are running in
	// different nodes with different clocks.
	var arrivalTimeStr string
	if extErr := event.ExtensionAs(broker.EventArrivalTime, &arrivalTimeStr); extErr == nil {
		arrivalTime, err := time.Parse(time.RFC3339, arrivalTimeStr)
		if err == nil {
			r.reporter.ReportEventProcessingTime(reportArgs, time.Since(arrivalTime))
		}
	}

	start := time.Now()
	sendingCTX := broker.SendingContext(ctx, tctx, subscriberURI)
	rctx, replyEvent, err := r.ceClient.Send(sendingCTX, *event)
	rtctx := cloudevents.HTTPTransportContextFrom(rctx)
	// Record the dispatch time.
	r.reporter.ReportEventDispatchTime(reportArgs, rtctx.StatusCode, time.Since(start))
	// Record the event count.
	r.reporter.ReportEventCount(reportArgs, rtctx.StatusCode)
	return replyEvent, err
}

func (r *Handler) getTrigger(ctx context.Context, ref path.NamespacedNameUID) (*eventingv1alpha1.Trigger, error) {
	t, err := r.triggerLister.Get(ref.Name)
	if err != nil {
		return nil, err
	}
	if t.UID != ref.UID {
		return nil, fmt.Errorf("trigger had a different UID. From ref '%s'. From Kubernetes '%s'", ref.UID, t.UID)
	}
	return t, nil
}

// shouldSendEvent determines whether event 'event' should be sent based on the triggerSpec 'ts'.
// Currently it supports exact matching on event context attributes and extension attributes.
// If no filter is present, shouldSendEvent returns passFilter.
func (r *Handler) shouldSendEvent(ctx context.Context, ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) FilterResult {
	// No filter specified, default to passing everything.
	if ts.Filter == nil || (ts.Filter.DeprecatedSourceAndType == nil && ts.Filter.Attributes == nil) {
		return noFilter
	}

	attrs := map[string]string{}
	// Since the filters cannot distinguish presence, filtering for an empty
	// string is impossible.
	if ts.Filter.DeprecatedSourceAndType != nil {
		attrs["type"] = ts.Filter.DeprecatedSourceAndType.Type
		attrs["source"] = ts.Filter.DeprecatedSourceAndType.Source
	} else if ts.Filter.Attributes != nil {
		attrs = map[string]string(*ts.Filter.Attributes)
	}

	return r.filterEventByAttributes(ctx, attrs, event)
}

func (r *Handler) filterEventByAttributes(ctx context.Context, attrs map[string]string, event *cloudevents.Event) FilterResult {
	// Set standard context attributes. The attributes available may not be
	// exactly the same as the attributes defined in the current version of the
	// CloudEvents spec.
	ce := map[string]interface{}{
		"specversion":         event.SpecVersion(),
		"type":                event.Type(),
		"source":              event.Source(),
		"subject":             event.Subject(),
		"id":                  event.ID(),
		"time":                event.Time().String(),
		"schemaurl":           event.SchemaURL(),
		"datacontenttype":     event.DataContentType(),
		"datamediatype":       event.DataMediaType(),
		"datacontentencoding": event.DataContentEncoding(),
	}
	ext := event.Extensions()
	if ext != nil {
		for k, v := range ext {
			ce[k] = v
		}
	}

	for k, v := range attrs {
		var value interface{}
		value, ok := ce[k]
		// If the attribute does not exist in the event, return false.
		if !ok {
			logging.FromContext(ctx).Debug("Attribute not found", zap.String("attribute", k))
			return failFilter
		}
		// If the attribute is not set to any and is different than the one from the event, return false.
		if v != eventingv1alpha1.TriggerAnyFilter && v != value {
			logging.FromContext(ctx).Debug("Attribute had non-matching value", zap.String("attribute", k), zap.String("filter", v), zap.Any("received", value))
			return failFilter
		}
	}
	return passFilter
}

// triggerFilterAttribute returns the filter attribute value for a given `attributeName`. If it doesn't not exist,
// returns the any value filter.
func triggerFilterAttribute(filter *eventingv1alpha1.TriggerFilter, attributeName string) string {
	attributeValue := eventingv1alpha1.TriggerAnyFilter
	if filter != nil {
		if filter.DeprecatedSourceAndType != nil {
			if attributeName == "type" {
				attributeValue = filter.DeprecatedSourceAndType.Type
			} else if attributeName == "source" {
				attributeValue = filter.DeprecatedSourceAndType.Source
			}
		} else if filter.Attributes != nil {
			attrs := map[string]string(*filter.Attributes)
			if v, ok := attrs[attributeName]; ok {
				attributeValue = v
			}
		}
	}
	return attributeValue
}
<|endoftext|>


<commit_before>/*
	rulehuntersrv - A server to find rules in data based on user specified goals
	Copyright (C) 2016 vLife Systems Ltd <http://vlifesystems.com>

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program; see the file COPYING.  If not, see
	<http://www.gnu.org/licenses/>.
*/

// Package quitter handles stopping go routines cleanly
package quitter

import (
	"os"
	"sync"
)

type Quitter struct {
	shouldQuit bool
	waitGroup  *sync.WaitGroup
}

func New() *Quitter {
	return &Quitter{
		shouldQuit: false,
		waitGroup:  &sync.WaitGroup{},
	}
}

// Add adds a go routine to wait for
func (q *Quitter) Add() {
	q.waitGroup.Add(1)
}

// Done indicates that a go routine has finished
func (q *Quitter) Done() {
	q.waitGroup.Done()
}

// Quit indicates to all the go routines that they should quit, it then waits
// for them to finish. Once they have all finished if killProcess is true
// then the os.Interrupt signal is sent to stop the process.
func (q *Quitter) Quit(killProcess bool) {
	q.shouldQuit = true
	q.waitGroup.Wait()
	if killProcess {
		p, err := os.FindProcess(os.Getpid())
		if err != nil {
			panic("Can't find process to Quit")
		}
		p.Signal(os.Interrupt)
	}
}

// ShouldQuit returns if a go routine should quit
func (q *Quitter) ShouldQuit() bool {
	return q.shouldQuit
}
<commit_msg>Add check of error for Signal in Quitter<commit_after>/*
	rulehuntersrv - A server to find rules in data based on user specified goals
	Copyright (C) 2016 vLife Systems Ltd <http://vlifesystems.com>

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program; see the file COPYING.  If not, see
	<http://www.gnu.org/licenses/>.
*/

// Package quitter handles stopping go routines cleanly
package quitter

import (
	"fmt"
	"os"
	"sync"
)

type Quitter struct {
	shouldQuit bool
	waitGroup  *sync.WaitGroup
}

func New() *Quitter {
	return &Quitter{
		shouldQuit: false,
		waitGroup:  &sync.WaitGroup{},
	}
}

// Add adds a go routine to wait for
func (q *Quitter) Add() {
	q.waitGroup.Add(1)
}

// Done indicates that a go routine has finished
func (q *Quitter) Done() {
	q.waitGroup.Done()
}

// Quit indicates to all the go routines that they should quit, it then waits
// for them to finish. Once they have all finished if killProcess is true
// then the os.Interrupt signal is sent to stop the process.
func (q *Quitter) Quit(killProcess bool) {
	q.shouldQuit = true
	q.waitGroup.Wait()
	if killProcess {
		p, err := os.FindProcess(os.Getpid())
		if err != nil {
			panic("Can't find process to Quit")
		}
		if err := p.Signal(os.Interrupt); err != nil {
			panic(fmt.Sprintf("Can't send signal: %s", err))
		}
	}
}

// ShouldQuit returns if a go routine should quit
func (q *Quitter) ShouldQuit() bool {
	return q.shouldQuit
}
<|endoftext|>


<commit_before>package lispy

func Button(li *Lispy) string {
	const htmlstr = `{{$li := .}}<button{{if .Exist "value"}} value="{{.GetDel "value"|html}}"{{end}}{{if .Exist "name"}} name="{{.GetDel "name"|html}}"{{end}}{{if .Exist "type"}} type="{{.GetDel "type"|html}}"{{end}}{{if ExistRes "disabled"}} disabled{{end}}{{if ExistRes "autofocus"}} autofocus{{end}}{{if ExistRes "formnovalidate"}} formnovalidate{{end}}{{range .GetNames}} {{.|attr}}="{{$li.Get .|html}}"{{end}} />`
	return li.HtmlRender(htmlstr)
}
<commit_msg>Improvement to button!<commit_after>package lispy

func Button(li *Lispy) string {
	str := `<button`

	if li.Exist("value") {
		str += ` value="` + li.GetDel("value") + `"`
	}

	if li.Exist("name") {
		str += ` name="` + li.GetDel("name") + `"`
	}

	if li.Exist("type") {
		str += ` type="` + li.GetDel("type") + `"`
	}

	if li.ExistRes("disabled") {
		str += ` disabled`
	}

	if li.ExistRes("autofocus") {
		str += ` autofocus`
	}

	if li.ExistRes("formnovalidate") {
		str += ` formnovalidate`
	}

	str += li.GetParam()

	str += `/>`

	return str
}
<|endoftext|>


<commit_before>package consul

import (
	"context"
	"crypto/x509"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/armon/go-metrics"
	"github.com/armon/go-metrics/prometheus"
	"github.com/hashicorp/go-hclog"

	"github.com/hashicorp/consul/agent/connect"
	"github.com/hashicorp/consul/agent/connect/ca"
	"github.com/hashicorp/consul/logging"
	"github.com/hashicorp/consul/tlsutil"
)

var metricsKeyMeshRootCAExpiry = []string{"mesh", "active-root-ca", "expiry"}
var metricsKeyMeshActiveSigningCAExpiry = []string{"mesh", "active-signing-ca", "expiry"}

var CertExpirationGauges = []prometheus.GaugeDefinition{
	{
		Name: metricsKeyMeshRootCAExpiry,
		Help: "Seconds until the service mesh root certificate expires. Updated every hour",
	},
	{
		Name: metricsKeyMeshActiveSigningCAExpiry,
		Help: "Seconds until the service mesh signing certificate expires. Updated every hour",
	},
	{
		Name: metricsKeyAgentTLSCertExpiry,
		Help: "Seconds until the agent tls certificate expires. Updated every hour",
	},
}

func rootCAExpiryMonitor(s *Server) CertExpirationMonitor {
	return CertExpirationMonitor{
		Key: metricsKeyMeshRootCAExpiry,
		Labels: []metrics.Label{
			{Name: "datacenter", Value: s.config.Datacenter},
		},
		Logger: s.logger.Named(logging.Connect),
		Query: func() (time.Duration, error) {
			return getRootCAExpiry(s)
		},
	}
}

func getRootCAExpiry(s *Server) (time.Duration, error) {
	state := s.fsm.State()
	_, root, err := state.CARootActive(nil)
	switch {
	case err != nil:
		return 0, fmt.Errorf("failed to retrieve root CA: %w", err)
	case root == nil:
		return 0, fmt.Errorf("no active root CA")
	}

	return time.Until(root.NotAfter), nil
}

func signingCAExpiryMonitor(s *Server) CertExpirationMonitor {
	isPrimary := s.config.Datacenter == s.config.PrimaryDatacenter
	if isPrimary {
		return CertExpirationMonitor{
			Key: metricsKeyMeshActiveSigningCAExpiry,
			Labels: []metrics.Label{
				{Name: "datacenter", Value: s.config.Datacenter},
			},
			Logger: s.logger.Named(logging.Connect),
			Query: func() (time.Duration, error) {
				provider, _ := s.caManager.getCAProvider()

				if _, ok := provider.(ca.PrimaryUsesIntermediate); ok {
					return getActiveIntermediateExpiry(s)
				}
				return getRootCAExpiry(s)
			},
		}
	}

	return CertExpirationMonitor{
		Key: metricsKeyMeshActiveSigningCAExpiry,
		Labels: []metrics.Label{
			{Name: "datacenter", Value: s.config.Datacenter},
		},
		Logger: s.logger.Named(logging.Connect),
		Query: func() (time.Duration, error) {
			return getActiveIntermediateExpiry(s)
		},
	}
}

func getActiveIntermediateExpiry(s *Server) (time.Duration, error) {
	state := s.fsm.State()
	_, root, err := state.CARootActive(nil)
	if err != nil {
		return 0, err
	}

	// the CA used in a secondary DC is the active intermediate,
	// which is the last in the IntermediateCerts stack
	if len(root.IntermediateCerts) == 0 {
		return 0, errors.New("no intermediate available")
	}
	cert, err := connect.ParseCert(root.IntermediateCerts[len(root.IntermediateCerts)-1])
	if err != nil {
		return 0, err
	}
	return time.Until(cert.NotAfter), nil
}

type CertExpirationMonitor struct {
	Key    []string
	Labels []metrics.Label
	Logger hclog.Logger
	// Query is called at each interval. It should return the duration until the
	// certificate expires, or an error if the query failed.
	Query func() (time.Duration, error)
}

const certExpirationMonitorInterval = time.Hour

func (m CertExpirationMonitor) Monitor(ctx context.Context) error {
	ticker := time.NewTicker(certExpirationMonitorInterval)
	defer ticker.Stop()

	logger := m.Logger.With("metric", strings.Join(m.Key, "."))

	for {
		select {
		case <-ctx.Done():
			return nil
		case <-ticker.C:
			d, err := m.Query()
			if err != nil {
				logger.Warn("failed to emit certificate expiry metric", "error", err)
				continue
			}

			if d < 24*time.Hour {
				logger.Warn("certificate will expire soon",
					"time_to_expiry", d, "expiration", time.Now().Add(d))
			}

			expiry := d / time.Second
			metrics.SetGaugeWithLabels(m.Key, float32(expiry), m.Labels)
		}
	}
}

var metricsKeyAgentTLSCertExpiry = []string{"agent", "tls", "cert", "expiry"}

// AgentTLSCertExpirationMonitor returns a CertExpirationMonitor which will
// monitor the expiration of the certificate used for agent TLS.
func AgentTLSCertExpirationMonitor(c *tlsutil.Configurator, logger hclog.Logger, dc string) CertExpirationMonitor {
	return CertExpirationMonitor{
		Key: metricsKeyAgentTLSCertExpiry,
		Labels: []metrics.Label{
			{Name: "node", Value: c.Base().NodeName},
			{Name: "datacenter", Value: dc},
		},
		Logger: logger,
		Query: func() (time.Duration, error) {
			raw := c.Cert()
			if raw == nil {
				return 0, fmt.Errorf("tls not enabled")
			}

			cert, err := x509.ParseCertificate(raw.Certificate[0])
			if err != nil {
				return 0, fmt.Errorf("failed to parse agent tls cert: %w", err)
			}
			return time.Until(cert.NotAfter), nil
		},
	}
}
<commit_msg>telemetry: improve cert expiry metrics<commit_after>package consul

import (
	"context"
	"crypto/x509"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/armon/go-metrics"
	"github.com/armon/go-metrics/prometheus"
	"github.com/hashicorp/go-hclog"

	"github.com/hashicorp/consul/agent/connect"
	"github.com/hashicorp/consul/agent/connect/ca"
	"github.com/hashicorp/consul/logging"
	"github.com/hashicorp/consul/tlsutil"
)

var metricsKeyMeshRootCAExpiry = []string{"mesh", "active-root-ca", "expiry"}
var metricsKeyMeshActiveSigningCAExpiry = []string{"mesh", "active-signing-ca", "expiry"}

var CertExpirationGauges = []prometheus.GaugeDefinition{
	{
		Name: metricsKeyMeshRootCAExpiry,
		Help: "Seconds until the service mesh root certificate expires. Updated every hour",
	},
	{
		Name: metricsKeyMeshActiveSigningCAExpiry,
		Help: "Seconds until the service mesh signing certificate expires. Updated every hour",
	},
	{
		Name: metricsKeyAgentTLSCertExpiry,
		Help: "Seconds until the agent tls certificate expires. Updated every hour",
	},
}

func rootCAExpiryMonitor(s *Server) CertExpirationMonitor {
	return CertExpirationMonitor{
		Key: metricsKeyMeshRootCAExpiry,
		Labels: []metrics.Label{
			{Name: "datacenter", Value: s.config.Datacenter},
		},
		Logger: s.logger.Named(logging.Connect),
		Query: func() (time.Duration, error) {
			return getRootCAExpiry(s)
		},
	}
}

func getRootCAExpiry(s *Server) (time.Duration, error) {
	state := s.fsm.State()
	_, root, err := state.CARootActive(nil)
	switch {
	case err != nil:
		return 0, fmt.Errorf("failed to retrieve root CA: %w", err)
	case root == nil:
		return 0, fmt.Errorf("no active root CA")
	}

	return time.Until(root.NotAfter), nil
}

func signingCAExpiryMonitor(s *Server) CertExpirationMonitor {
	isPrimary := s.config.Datacenter == s.config.PrimaryDatacenter
	if isPrimary {
		return CertExpirationMonitor{
			Key: metricsKeyMeshActiveSigningCAExpiry,
			Labels: []metrics.Label{
				{Name: "datacenter", Value: s.config.Datacenter},
			},
			Logger: s.logger.Named(logging.Connect),
			Query: func() (time.Duration, error) {
				provider, _ := s.caManager.getCAProvider()

				if _, ok := provider.(ca.PrimaryUsesIntermediate); ok {
					return getActiveIntermediateExpiry(s)
				}
				return getRootCAExpiry(s)
			},
		}
	}

	return CertExpirationMonitor{
		Key: metricsKeyMeshActiveSigningCAExpiry,
		Labels: []metrics.Label{
			{Name: "datacenter", Value: s.config.Datacenter},
		},
		Logger: s.logger.Named(logging.Connect),
		Query: func() (time.Duration, error) {
			return getActiveIntermediateExpiry(s)
		},
	}
}

func getActiveIntermediateExpiry(s *Server) (time.Duration, error) {
	state := s.fsm.State()
	_, root, err := state.CARootActive(nil)
	switch {
	case err != nil:
		return 0, fmt.Errorf("failed to retrieve root CA: %w", err)
	case root == nil:
		return 0, fmt.Errorf("no active root CA")
	}

	// the CA used in a secondary DC is the active intermediate,
	// which is the last in the IntermediateCerts stack
	if len(root.IntermediateCerts) == 0 {
		return 0, errors.New("no intermediate available")
	}
	cert, err := connect.ParseCert(root.IntermediateCerts[len(root.IntermediateCerts)-1])
	if err != nil {
		return 0, err
	}
	return time.Until(cert.NotAfter), nil
}

type CertExpirationMonitor struct {
	Key    []string
	Labels []metrics.Label
	Logger hclog.Logger
	// Query is called at each interval. It should return the duration until the
	// certificate expires, or an error if the query failed.
	Query func() (time.Duration, error)
}

const certExpirationMonitorInterval = time.Hour

func (m CertExpirationMonitor) Monitor(ctx context.Context) error {
	ticker := time.NewTicker(certExpirationMonitorInterval)
	defer ticker.Stop()

	logger := m.Logger.With("metric", strings.Join(m.Key, "."))

	fn := func() {
		d, err := m.Query()
		if err != nil {
			logger.Warn("failed to emit certificate expiry metric", "error", err)
			return
		}

		if d < 24*time.Hour {
			logger.Warn("certificate will expire soon",
				"time_to_expiry", d, "expiration", time.Now().Add(d))
		}

		expiry := d / time.Second
		metrics.SetGaugeWithLabels(m.Key, float32(expiry), m.Labels)
	}

	// emit the metric immediately so that if a cert was just updated the
	// new metric will be updated to the new expiration time.
	fn()

	for {
		select {
		case <-ctx.Done():
			return nil
		case <-ticker.C:
			fn()
		}
	}
}

var metricsKeyAgentTLSCertExpiry = []string{"agent", "tls", "cert", "expiry"}

// AgentTLSCertExpirationMonitor returns a CertExpirationMonitor which will
// monitor the expiration of the certificate used for agent TLS.
func AgentTLSCertExpirationMonitor(c *tlsutil.Configurator, logger hclog.Logger, dc string) CertExpirationMonitor {
	return CertExpirationMonitor{
		Key: metricsKeyAgentTLSCertExpiry,
		Labels: []metrics.Label{
			{Name: "node", Value: c.Base().NodeName},
			{Name: "datacenter", Value: dc},
		},
		Logger: logger,
		Query: func() (time.Duration, error) {
			raw := c.Cert()
			if raw == nil {
				return 0, fmt.Errorf("tls not enabled")
			}

			cert, err := x509.ParseCertificate(raw.Certificate[0])
			if err != nil {
				return 0, fmt.Errorf("failed to parse agent tls cert: %w", err)
			}
			return time.Until(cert.NotAfter), nil
		},
	}
}
<|endoftext|>


<commit_before>package filepathfilter

import (
	"runtime"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestPatternMatch(t *testing.T) {
	for _, wildcard := range []string{"*", "*.*"} {
		assertPatternMatch(t, wildcard,
			"a",
			"a/",
			"a.a",
			"a/b",
			"a/b/",
			"a/b.b",
			"a/b/c",
			"a/b/c/",
			"a/b/c.c",
		)
	}

	assertPatternMatch(t, "filename.txt", "filename.txt")
	assertPatternMatch(t, "*.txt", "filename.txt")
	refutePatternMatch(t, "*.tx", "filename.txt")
	assertPatternMatch(t, "f*.txt", "filename.txt")
	refutePatternMatch(t, "g*.txt", "filename.txt")
	assertPatternMatch(t, "file*", "filename.txt")
	refutePatternMatch(t, "file", "filename.txt")

	// With no path separators, should match in subfolders
	assertPatternMatch(t, "*.txt", "sub/filename.txt")
	refutePatternMatch(t, "*.tx", "sub/filename.txt")
	assertPatternMatch(t, "f*.txt", "sub/filename.txt")
	refutePatternMatch(t, "g*.txt", "sub/filename.txt")
	assertPatternMatch(t, "file*", "sub/filename.txt")
	refutePatternMatch(t, "file", "sub/filename.txt")

	// matches only in subdir
	assertPatternMatch(t, "sub/*.txt", "sub/filename.txt")
	refutePatternMatch(t, "sub/*.txt",
		"top/sub/filename.txt",
		"sub/filename.dat",
		"other/filename.txt",
	)

	// Needs wildcard for exact filename
	assertPatternMatch(t, "**/filename.txt", "sub/sub/sub/filename.txt")

	// Should not match dots to subparts
	refutePatternMatch(t, "*.ign", "sub/shouldignoreme.txt")

	// Path specific
	assertPatternMatch(t, "sub",
		"sub/",
		"sub",
		"sub/filename.txt",
		"top/sub/",
		"top/sub",
		"top/sub/filename.txt",
	)

	assertPatternMatch(t, "sub/", "sub/filename.txt", "top/sub/filename.txt")
	assertPatternMatch(t, "/sub", "sub/", "sub", "sub/filename.txt")
	assertPatternMatch(t, "/sub/", "sub/filename.txt")
	refutePatternMatch(t, "/sub", "subfilename.txt", "top/sub/", "top/sub", "top/sub/filename.txt")
	refutePatternMatch(t, "sub", "subfilename.txt")
	refutePatternMatch(t, "sub/", "subfilename.txt")
	refutePatternMatch(t, "/sub/", "subfilename.txt", "top/sub/filename.txt")

	// nested path
	assertPatternMatch(t, "top/sub",
		"top/sub/filename.txt",
		"top/sub/",
		"top/sub",
		"root/top/sub/filename.txt",
		"root/top/sub/",
		"root/top/sub",
	)
	assertPatternMatch(t, "top/sub/", "top/sub/filename.txt")
	assertPatternMatch(t, "top/sub/", "root/top/sub/filename.txt")

	assertPatternMatch(t, "/top/sub", "top/sub/", "top/sub", "top/sub/filename.txt")
	assertPatternMatch(t, "/top/sub/", "top/sub/filename.txt")

	refutePatternMatch(t, "top/sub", "top/subfilename.txt")
	refutePatternMatch(t, "top/sub/", "top/subfilename.txt")
	refutePatternMatch(t, "/top/sub",
		"top/subfilename.txt",
		"root/top/sub/filename.txt",
		"root/top/sub/",
		"root/top/sub",
	)

	refutePatternMatch(t, "/top/sub/",
		"root/top/sub/filename.txt",
		"top/subfilename.txt",
	)

	// Absolute
	assertPatternMatch(t, "*.dat", "/path/to/sub/.git/test.dat")
	assertPatternMatch(t, "**/.git", "/path/to/sub/.git")

	// Match anything
	assertPatternMatch(t, ".", "path.txt")
	assertPatternMatch(t, "./", "path.txt")
	assertPatternMatch(t, ".\\", "path.txt")
}

func assertPatternMatch(t *testing.T, pattern string, filenames ...string) {
	p := NewPattern(pattern)
	for _, filename := range filenames {
		assert.True(t, p.Match(filename), "%q should match pattern %q", filename, pattern)
	}
}

func refutePatternMatch(t *testing.T, pattern string, filenames ...string) {
	p := NewPattern(pattern)
	for _, filename := range filenames {
		assert.False(t, p.Match(filename), "%q should not match pattern %q", filename, pattern)
	}
}

type filterTest struct {
	expectedResult  bool
	expectedPattern string
	includes        []string
	excludes        []string
}

type filterPrefixTest struct {
	expected bool
	prefixes []string
	includes []string
	excludes []string
}

func (c *filterPrefixTest) Assert(t *testing.T) {
	f := New(c.platformIncludes(), c.platformExcludes())

	prefixes := c.prefixes
	if runtime.GOOS == "windows" {
		prefixes = toWindowsPaths(prefixes)
	}

	for _, prefix := range prefixes {
		assert.Equal(t, c.expected, f.HasPrefix(prefix),
			"expected=%v, prefix=%s", c.expected, prefix)
	}

}

func (c *filterPrefixTest) platformIncludes() []string {
	if runtime.GOOS == "windows" {
		return toWindowsPaths(c.includes)
	}
	return c.includes
}

func (c *filterPrefixTest) platformExcludes() []string {
	if runtime.GOOS == "windows" {
		return toWindowsPaths(c.excludes)
	}
	return c.excludes
}

func toWindowsPaths(paths []string) []string {
	var out []string
	for _, path := range paths {
		out = append(out, strings.Replace(path, "/", "\\", -1))
	}

	return out
}

func TestFilterHasPrefix(t *testing.T) {
	prefixes := []string{"foo", "foo/", "foo/bar", "foo/bar/baz", "foo/bar/baz/"}
	for desc, c := range map[string]*filterPrefixTest{
		"empty filter":              {true, prefixes, nil, nil},
		"path prefix pattern":       {true, prefixes, []string{"/foo/bar/baz"}, nil},
		"path pattern":              {true, prefixes, []string{"foo/bar/baz"}, nil},
		"simple ext pattern":        {true, prefixes, []string{"*.dat"}, nil},
		"pathless wildcard pattern": {true, prefixes, []string{"foo*.dat"}, nil},
		"double wildcard pattern":   {true, prefixes, []string{"foo/**/baz"}, nil},
		"include other dir":         {false, prefixes, []string{"other"}, nil},

		"exclude pattern":                   {true, prefixes, nil, []string{"other"}},
		"exclude simple ext pattern":        {true, prefixes, nil, []string{"*.dat"}},
		"exclude pathless wildcard pattern": {true, prefixes, nil, []string{"foo*.dat"}},
	} {
		t.Run(desc, c.Assert)
	}

	prefixes = []string{"foo", "foo/", "foo/bar"}
	for desc, c := range map[string]*filterPrefixTest{
		"exclude path prefix pattern":     {true, prefixes, nil, []string{"/foo/bar/baz"}},
		"exclude path pattern":            {true, prefixes, nil, []string{"foo/bar/baz"}},
		"exclude double wildcard pattern": {true, prefixes, nil, []string{"foo/**/baz"}},
	} {
		t.Run(desc, c.Assert)
	}

	prefixes = []string{"foo/bar/baz", "foo/bar/baz/"}
	for desc, c := range map[string]*filterPrefixTest{
		"exclude path prefix pattern": {false, prefixes, nil, []string{"/foo/bar/baz"}},
		"exclude path pattern":        {false, prefixes, nil, []string{"foo/bar/baz"}},
	} {
		t.Run(desc, c.Assert)
	}

	prefixes = []string{"foo/bar/baz", "foo/test/baz"}
	for desc, c := range map[string]*filterPrefixTest{
		"exclude double wildcard pattern": {false, prefixes, nil, []string{"foo/**/baz"}},
	} {
		t.Run(desc, c.Assert)
	}
}

func TestFilterAllows(t *testing.T) {
	cases := []filterTest{
		// Null case
		filterTest{true, "", nil, nil},
		// Inclusion
		filterTest{true, "*.dat", []string{"*.dat"}, nil},
		filterTest{true, "file*.dat", []string{"file*.dat"}, nil},
		filterTest{true, "file*", []string{"file*"}, nil},
		filterTest{true, "*name.dat", []string{"*name.dat"}, nil},
		filterTest{false, "", []string{"/*.dat"}, nil},
		filterTest{false, "", []string{"otherfolder/*.dat"}, nil},
		filterTest{false, "", []string{"*.nam"}, nil},
		filterTest{true, "test/filename.dat", []string{"test/filename.dat"}, nil},
		filterTest{true, "test/filename.dat", []string{"test/filename.dat"}, nil},
		filterTest{false, "", []string{"blank", "something", "foo"}, nil},
		filterTest{false, "", []string{"test/notfilename.dat"}, nil},
		filterTest{true, "test", []string{"test"}, nil},
		filterTest{true, "test/*", []string{"test/*"}, nil},
		filterTest{false, "", []string{"nottest"}, nil},
		filterTest{false, "", []string{"nottest/*"}, nil},
		filterTest{true, "test/fil*", []string{"test/fil*"}, nil},
		filterTest{false, "", []string{"test/g*"}, nil},
		filterTest{true, "tes*/*", []string{"tes*/*"}, nil},
		filterTest{true, "[Tt]est/[Ff]ilename.dat", []string{"[Tt]est/[Ff]ilename.dat"}, nil},
		// Exclusion
		filterTest{false, "*.dat", nil, []string{"*.dat"}},
		filterTest{false, "file*.dat", nil, []string{"file*.dat"}},
		filterTest{false, "file*", nil, []string{"file*"}},
		filterTest{false, "*name.dat", nil, []string{"*name.dat"}},
		filterTest{true, "", nil, []string{"/*.dat"}},
		filterTest{true, "", nil, []string{"otherfolder/*.dat"}},
		filterTest{false, "test/filename.dat", nil, []string{"test/filename.dat"}},
		filterTest{false, "test/filename.dat", nil, []string{"blank", "something", "test/filename.dat", "foo"}},
		filterTest{true, "", nil, []string{"blank", "something", "foo"}},
		filterTest{true, "", nil, []string{"test/notfilename.dat"}},
		filterTest{false, "test", nil, []string{"test"}},
		filterTest{false, "test/*", nil, []string{"test/*"}},
		filterTest{true, "", nil, []string{"nottest"}},
		filterTest{true, "", nil, []string{"nottest/*"}},
		filterTest{false, "test/fil*", nil, []string{"test/fil*"}},
		filterTest{true, "", nil, []string{"test/g*"}},
		filterTest{false, "tes*/*", nil, []string{"tes*/*"}},
		filterTest{false, "[Tt]est/[Ff]ilename.dat", nil, []string{"[Tt]est/[Ff]ilename.dat"}},

		// // Both
		filterTest{true, "test/filename.dat", []string{"test/filename.dat"}, []string{"test/notfilename.dat"}},
		filterTest{false, "test/filename.dat", []string{"test"}, []string{"test/filename.dat"}},
		filterTest{true, "test/*", []string{"test/*"}, []string{"test/notfile*"}},
		filterTest{false, "test/file*", []string{"test/*"}, []string{"test/file*"}},
		filterTest{false, "test/filename.dat", []string{"another/*", "test/*"}, []string{"test/notfilename.dat", "test/filename.dat"}},
	}

	for _, c := range cases {
		if runtime.GOOS == "windows" {
			c.expectedPattern = strings.Replace(c.expectedPattern, "/", "\\", -1)
		}

		filter := New(c.includes, c.excludes)

		r1 := filter.Allows("test/filename.dat")
		pattern, r2 := filter.AllowsPattern("test/filename.dat")

		assert.Equal(t, r1, r2,
			"filepathfilter: expected Allows() and AllowsPattern() to return identical result")

		assert.Equal(t, c.expectedResult, r2, "includes: %v excludes: %v", c.includes, c.excludes)
		assert.Equal(t, c.expectedPattern, pattern,
			"filepathfilter: expected pattern match of: %q, got: %q",
			c.expectedPattern, pattern)

		if runtime.GOOS == "windows" {
			// also test with \ path separators, tolerate mixed separators
			for i, inc := range c.includes {
				c.includes[i] = strings.Replace(inc, "/", "\\", -1)
			}
			for i, ex := range c.excludes {
				c.excludes[i] = strings.Replace(ex, "/", "\\", -1)
			}

			filter = New(c.includes, c.excludes)

			r1 = filter.Allows("test/filename.dat")
			pattern, r2 = filter.AllowsPattern("test/filename.dat")

			assert.Equal(t, r1, r2,
				"filepathfilter: expected Allows() and AllowsPattern() to return identical result")

			assert.Equal(t, c.expectedResult, r1, c)
			assert.Equal(t, c.expectedPattern, pattern,
				"filepathfilter: expected pattern match of: %q, got: %q",
				c.expectedPattern, pattern)
		}
	}
}

func TestFilterReportsIncludePatterns(t *testing.T) {
	filter := New([]string{"*.foo", "*.bar"}, nil)

	assert.Equal(t, []string{"*.foo", "*.bar"}, filter.Include())
}

func TestFilterReportsExcludePatterns(t *testing.T) {
	filter := New(nil, []string{"*.baz", "*.quux"})

	assert.Equal(t, []string{"*.baz", "*.quux"}, filter.Exclude())
}
<commit_msg>filepathfilter: convert paths on windows<commit_after>package filepathfilter

import (
	"runtime"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestPatternMatch(t *testing.T) {
	for _, wildcard := range []string{"*", "*.*"} {
		assertPatternMatch(t, wildcard,
			"a",
			"a/",
			"a.a",
			"a/b",
			"a/b/",
			"a/b.b",
			"a/b/c",
			"a/b/c/",
			"a/b/c.c",
		)
	}

	assertPatternMatch(t, "filename.txt", "filename.txt")
	assertPatternMatch(t, "*.txt", "filename.txt")
	refutePatternMatch(t, "*.tx", "filename.txt")
	assertPatternMatch(t, "f*.txt", "filename.txt")
	refutePatternMatch(t, "g*.txt", "filename.txt")
	assertPatternMatch(t, "file*", "filename.txt")
	refutePatternMatch(t, "file", "filename.txt")

	// With no path separators, should match in subfolders
	assertPatternMatch(t, "*.txt", "sub/filename.txt")
	refutePatternMatch(t, "*.tx", "sub/filename.txt")
	assertPatternMatch(t, "f*.txt", "sub/filename.txt")
	refutePatternMatch(t, "g*.txt", "sub/filename.txt")
	assertPatternMatch(t, "file*", "sub/filename.txt")
	refutePatternMatch(t, "file", "sub/filename.txt")

	// matches only in subdir
	assertPatternMatch(t, "sub/*.txt", "sub/filename.txt")
	refutePatternMatch(t, "sub/*.txt",
		"top/sub/filename.txt",
		"sub/filename.dat",
		"other/filename.txt",
	)

	// Needs wildcard for exact filename
	assertPatternMatch(t, "**/filename.txt", "sub/sub/sub/filename.txt")

	// Should not match dots to subparts
	refutePatternMatch(t, "*.ign", "sub/shouldignoreme.txt")

	// Path specific
	assertPatternMatch(t, "sub",
		"sub/",
		"sub",
		"sub/filename.txt",
		"top/sub/",
		"top/sub",
		"top/sub/filename.txt",
	)

	assertPatternMatch(t, "sub/", "sub/filename.txt", "top/sub/filename.txt")
	assertPatternMatch(t, "/sub", "sub/", "sub", "sub/filename.txt")
	assertPatternMatch(t, "/sub/", "sub/filename.txt")
	refutePatternMatch(t, "/sub", "subfilename.txt", "top/sub/", "top/sub", "top/sub/filename.txt")
	refutePatternMatch(t, "sub", "subfilename.txt")
	refutePatternMatch(t, "sub/", "subfilename.txt")
	refutePatternMatch(t, "/sub/", "subfilename.txt", "top/sub/filename.txt")

	// nested path
	assertPatternMatch(t, "top/sub",
		"top/sub/filename.txt",
		"top/sub/",
		"top/sub",
		"root/top/sub/filename.txt",
		"root/top/sub/",
		"root/top/sub",
	)
	assertPatternMatch(t, "top/sub/", "top/sub/filename.txt")
	assertPatternMatch(t, "top/sub/", "root/top/sub/filename.txt")

	assertPatternMatch(t, "/top/sub", "top/sub/", "top/sub", "top/sub/filename.txt")
	assertPatternMatch(t, "/top/sub/", "top/sub/filename.txt")

	refutePatternMatch(t, "top/sub", "top/subfilename.txt")
	refutePatternMatch(t, "top/sub/", "top/subfilename.txt")
	refutePatternMatch(t, "/top/sub",
		"top/subfilename.txt",
		"root/top/sub/filename.txt",
		"root/top/sub/",
		"root/top/sub",
	)

	refutePatternMatch(t, "/top/sub/",
		"root/top/sub/filename.txt",
		"top/subfilename.txt",
	)

	// Absolute
	assertPatternMatch(t, "*.dat", "/path/to/sub/.git/test.dat")
	assertPatternMatch(t, "**/.git", "/path/to/sub/.git")

	// Match anything
	assertPatternMatch(t, ".", "path.txt")
	assertPatternMatch(t, "./", "path.txt")
	assertPatternMatch(t, ".\\", "path.txt")
}

func assertPatternMatch(t *testing.T, pattern string, filenames ...string) {
	p := NewPattern(pattern)
	for _, filename := range toWindowsPaths(filenames) {
		assert.True(t, p.Match(filename), "%q should match pattern %q", filename, pattern)
	}
}

func refutePatternMatch(t *testing.T, pattern string, filenames ...string) {
	p := NewPattern(pattern)
	for _, filename := range toWindowsPaths(filenames) {
		assert.False(t, p.Match(filename), "%q should not match pattern %q", filename, pattern)
	}
}

type filterTest struct {
	expectedResult  bool
	expectedPattern string
	includes        []string
	excludes        []string
}

type filterPrefixTest struct {
	expected bool
	prefixes []string
	includes []string
	excludes []string
}

func (c *filterPrefixTest) Assert(t *testing.T) {
	f := New(c.platformIncludes(), c.platformExcludes())

	prefixes := c.prefixes
	if runtime.GOOS == "windows" {
		prefixes = toWindowsPaths(prefixes)
	}

	for _, prefix := range prefixes {
		assert.Equal(t, c.expected, f.HasPrefix(prefix),
			"expected=%v, prefix=%s", c.expected, prefix)
	}

}

func (c *filterPrefixTest) platformIncludes() []string {
	return toWindowsPaths(c.includes)
}

func (c *filterPrefixTest) platformExcludes() []string {
	return toWindowsPaths(c.excludes)
}

func toWindowsPaths(paths []string) []string {
	if runtime.GOOS != "windows" {
		return paths
	}

	out := make([]string, len(paths))
	for i, path := range paths {
		out[i] = strings.Replace(path, "/", "\\", -1)
	}

	return out
}

func TestFilterHasPrefix(t *testing.T) {
	prefixes := []string{"foo", "foo/", "foo/bar", "foo/bar/baz", "foo/bar/baz/"}
	for desc, c := range map[string]*filterPrefixTest{
		"empty filter":              {true, prefixes, nil, nil},
		"path prefix pattern":       {true, prefixes, []string{"/foo/bar/baz"}, nil},
		"path pattern":              {true, prefixes, []string{"foo/bar/baz"}, nil},
		"simple ext pattern":        {true, prefixes, []string{"*.dat"}, nil},
		"pathless wildcard pattern": {true, prefixes, []string{"foo*.dat"}, nil},
		"double wildcard pattern":   {true, prefixes, []string{"foo/**/baz"}, nil},
		"include other dir":         {false, prefixes, []string{"other"}, nil},

		"exclude pattern":                   {true, prefixes, nil, []string{"other"}},
		"exclude simple ext pattern":        {true, prefixes, nil, []string{"*.dat"}},
		"exclude pathless wildcard pattern": {true, prefixes, nil, []string{"foo*.dat"}},
	} {
		t.Run(desc, c.Assert)
	}

	prefixes = []string{"foo", "foo/", "foo/bar"}
	for desc, c := range map[string]*filterPrefixTest{
		"exclude path prefix pattern":     {true, prefixes, nil, []string{"/foo/bar/baz"}},
		"exclude path pattern":            {true, prefixes, nil, []string{"foo/bar/baz"}},
		"exclude double wildcard pattern": {true, prefixes, nil, []string{"foo/**/baz"}},
	} {
		t.Run(desc, c.Assert)
	}

	prefixes = []string{"foo/bar/baz", "foo/bar/baz/"}
	for desc, c := range map[string]*filterPrefixTest{
		"exclude path prefix pattern": {false, prefixes, nil, []string{"/foo/bar/baz"}},
		"exclude path pattern":        {false, prefixes, nil, []string{"foo/bar/baz"}},
	} {
		t.Run(desc, c.Assert)
	}

	prefixes = []string{"foo/bar/baz", "foo/test/baz"}
	for desc, c := range map[string]*filterPrefixTest{
		"exclude double wildcard pattern": {false, prefixes, nil, []string{"foo/**/baz"}},
	} {
		t.Run(desc, c.Assert)
	}
}

func TestFilterAllows(t *testing.T) {
	cases := []filterTest{
		// Null case
		filterTest{true, "", nil, nil},
		// Inclusion
		filterTest{true, "*.dat", []string{"*.dat"}, nil},
		filterTest{true, "file*.dat", []string{"file*.dat"}, nil},
		filterTest{true, "file*", []string{"file*"}, nil},
		filterTest{true, "*name.dat", []string{"*name.dat"}, nil},
		filterTest{false, "", []string{"/*.dat"}, nil},
		filterTest{false, "", []string{"otherfolder/*.dat"}, nil},
		filterTest{false, "", []string{"*.nam"}, nil},
		filterTest{true, "test/filename.dat", []string{"test/filename.dat"}, nil},
		filterTest{true, "test/filename.dat", []string{"test/filename.dat"}, nil},
		filterTest{false, "", []string{"blank", "something", "foo"}, nil},
		filterTest{false, "", []string{"test/notfilename.dat"}, nil},
		filterTest{true, "test", []string{"test"}, nil},
		filterTest{true, "test/*", []string{"test/*"}, nil},
		filterTest{false, "", []string{"nottest"}, nil},
		filterTest{false, "", []string{"nottest/*"}, nil},
		filterTest{true, "test/fil*", []string{"test/fil*"}, nil},
		filterTest{false, "", []string{"test/g*"}, nil},
		filterTest{true, "tes*/*", []string{"tes*/*"}, nil},
		filterTest{true, "[Tt]est/[Ff]ilename.dat", []string{"[Tt]est/[Ff]ilename.dat"}, nil},
		// Exclusion
		filterTest{false, "*.dat", nil, []string{"*.dat"}},
		filterTest{false, "file*.dat", nil, []string{"file*.dat"}},
		filterTest{false, "file*", nil, []string{"file*"}},
		filterTest{false, "*name.dat", nil, []string{"*name.dat"}},
		filterTest{true, "", nil, []string{"/*.dat"}},
		filterTest{true, "", nil, []string{"otherfolder/*.dat"}},
		filterTest{false, "test/filename.dat", nil, []string{"test/filename.dat"}},
		filterTest{false, "test/filename.dat", nil, []string{"blank", "something", "test/filename.dat", "foo"}},
		filterTest{true, "", nil, []string{"blank", "something", "foo"}},
		filterTest{true, "", nil, []string{"test/notfilename.dat"}},
		filterTest{false, "test", nil, []string{"test"}},
		filterTest{false, "test/*", nil, []string{"test/*"}},
		filterTest{true, "", nil, []string{"nottest"}},
		filterTest{true, "", nil, []string{"nottest/*"}},
		filterTest{false, "test/fil*", nil, []string{"test/fil*"}},
		filterTest{true, "", nil, []string{"test/g*"}},
		filterTest{false, "tes*/*", nil, []string{"tes*/*"}},
		filterTest{false, "[Tt]est/[Ff]ilename.dat", nil, []string{"[Tt]est/[Ff]ilename.dat"}},

		// // Both
		filterTest{true, "test/filename.dat", []string{"test/filename.dat"}, []string{"test/notfilename.dat"}},
		filterTest{false, "test/filename.dat", []string{"test"}, []string{"test/filename.dat"}},
		filterTest{true, "test/*", []string{"test/*"}, []string{"test/notfile*"}},
		filterTest{false, "test/file*", []string{"test/*"}, []string{"test/file*"}},
		filterTest{false, "test/filename.dat", []string{"another/*", "test/*"}, []string{"test/notfilename.dat", "test/filename.dat"}},
	}

	for _, c := range cases {
		if runtime.GOOS == "windows" {
			c.expectedPattern = strings.Replace(c.expectedPattern, "/", "\\", -1)
		}

		filter := New(c.includes, c.excludes)

		r1 := filter.Allows("test/filename.dat")
		pattern, r2 := filter.AllowsPattern("test/filename.dat")

		assert.Equal(t, r1, r2,
			"filepathfilter: expected Allows() and AllowsPattern() to return identical result")

		assert.Equal(t, c.expectedResult, r2, "includes: %v excludes: %v", c.includes, c.excludes)
		assert.Equal(t, c.expectedPattern, pattern,
			"filepathfilter: expected pattern match of: %q, got: %q",
			c.expectedPattern, pattern)

		if runtime.GOOS == "windows" {
			// also test with \ path separators, tolerate mixed separators
			for i, inc := range c.includes {
				c.includes[i] = strings.Replace(inc, "/", "\\", -1)
			}
			for i, ex := range c.excludes {
				c.excludes[i] = strings.Replace(ex, "/", "\\", -1)
			}

			filter = New(c.includes, c.excludes)

			r1 = filter.Allows("test/filename.dat")
			pattern, r2 = filter.AllowsPattern("test/filename.dat")

			assert.Equal(t, r1, r2,
				"filepathfilter: expected Allows() and AllowsPattern() to return identical result")

			assert.Equal(t, c.expectedResult, r1, c)
			assert.Equal(t, c.expectedPattern, pattern,
				"filepathfilter: expected pattern match of: %q, got: %q",
				c.expectedPattern, pattern)
		}
	}
}

func TestFilterReportsIncludePatterns(t *testing.T) {
	filter := New([]string{"*.foo", "*.bar"}, nil)

	assert.Equal(t, []string{"*.foo", "*.bar"}, filter.Include())
}

func TestFilterReportsExcludePatterns(t *testing.T) {
	filter := New(nil, []string{"*.baz", "*.quux"})

	assert.Equal(t, []string{"*.baz", "*.quux"}, filter.Exclude())
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"fmt"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/golang/glog"
)

type MinionController struct {
	cloud           cloudprovider.Interface
	matchRE         string
	staticResources *api.NodeResources
	minions         []string
	kubeClient      client.Interface
}

// NewMinionController returns a new minion controller to sync instances from cloudprovider.
func NewMinionController(
	cloud cloudprovider.Interface,
	matchRE string,
	minions []string,
	staticResources *api.NodeResources,
	kubeClient client.Interface) *MinionController {
	return &MinionController{
		cloud:           cloud,
		matchRE:         matchRE,
		minions:         minions,
		staticResources: staticResources,
		kubeClient:      kubeClient,
	}
}

// Run starts syncing instances from cloudprovider periodically, or create initial minion list.
func (s *MinionController) Run(period time.Duration) {
	if s.cloud != nil && len(s.matchRE) > 0 {
		go util.Forever(func() {
			if err := s.SyncCloud(); err != nil {
				glog.Errorf("Error syncing cloud: %v", err)
			}
		}, period)
	} else {
		go s.SyncStatic(period)
	}
}

// SyncStatic registers list of machines from command line flag. It returns after successful
// registration of all machines.
func (s *MinionController) SyncStatic(period time.Duration) error {
	registered := util.NewStringSet()
	for {
		for _, minionID := range s.minions {
			if registered.Has(minionID) {
				continue
			}
			_, err := s.kubeClient.Minions().Create(&api.Minion{
				ObjectMeta: api.ObjectMeta{Name: minionID},
				Spec: api.NodeSpec{
					Capacity: s.staticResources.Capacity,
				},
			})
			if err == nil {
				registered.Insert(minionID)
			}
		}
		if registered.Len() == len(s.minions) {
			return nil
		}
		time.Sleep(period)
	}
	return nil
}

// SyncCloud syncs list of instances from cloudprovider to master etcd registry.
func (s *MinionController) SyncCloud() error {
	matches, err := s.cloudMinions()
	if err != nil {
		return err
	}
	minions, err := s.kubeClient.Minions().List()
	if err != nil {
		return err
	}
	minionMap := make(map[string]*api.Minion)
	for _, minion := range minions.Items {
		minionMap[minion.Name] = &minion
	}

	// Create or delete minions from registry.
	for _, minion := range matches.Items {
		if _, ok := minionMap[minion.Name]; !ok {
			glog.Infof("Create minion in registry: %s", minion.Name)
			_, err = s.kubeClient.Minions().Create(&minion)
			if err != nil {
				glog.Errorf("Create minion error: %s", minion.Name)
			}
		}
		delete(minionMap, minion.Name)
	}

	for minionID := range minionMap {
		glog.Infof("Delete minion from registry: %s", minionID)
		err = s.kubeClient.Minions().Delete(minionID)
		if err != nil {
			glog.Errorf("Delete minion error: %s", minionID)
		}
	}
	return nil
}

// cloudMinions constructs and returns api.MinionList from cloudprovider.
func (s *MinionController) cloudMinions() (*api.MinionList, error) {
	instances, ok := s.cloud.Instances()
	if !ok {
		return nil, fmt.Errorf("cloud doesn't support instances")
	}
	matches, err := instances.List(s.matchRE)
	if err != nil {
		return nil, err
	}
	result := &api.MinionList{
		Items: make([]api.Minion, len(matches)),
	}
	for i := range matches {
		result.Items[i].Name = matches[i]
		resources, err := instances.GetNodeResources(matches[i])
		if err != nil {
			return nil, err
		}
		if resources == nil {
			resources = s.staticResources
		}
		if resources != nil {
			result.Items[i].Spec.Capacity = resources.Capacity
		}
	}
	return result, nil
}
<commit_msg>Query hostIP for instances<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"fmt"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/golang/glog"
)

type MinionController struct {
	cloud           cloudprovider.Interface
	matchRE         string
	staticResources *api.NodeResources
	minions         []string
	kubeClient      client.Interface
}

// NewMinionController returns a new minion controller to sync instances from cloudprovider.
func NewMinionController(
	cloud cloudprovider.Interface,
	matchRE string,
	minions []string,
	staticResources *api.NodeResources,
	kubeClient client.Interface) *MinionController {
	return &MinionController{
		cloud:           cloud,
		matchRE:         matchRE,
		minions:         minions,
		staticResources: staticResources,
		kubeClient:      kubeClient,
	}
}

// Run starts syncing instances from cloudprovider periodically, or create initial minion list.
func (s *MinionController) Run(period time.Duration) {
	if s.cloud != nil && len(s.matchRE) > 0 {
		go util.Forever(func() {
			if err := s.SyncCloud(); err != nil {
				glog.Errorf("Error syncing cloud: %v", err)
			}
		}, period)
	} else {
		go s.SyncStatic(period)
	}
}

// SyncStatic registers list of machines from command line flag. It returns after successful
// registration of all machines.
func (s *MinionController) SyncStatic(period time.Duration) error {
	registered := util.NewStringSet()
	for {
		for _, minionID := range s.minions {
			if registered.Has(minionID) {
				continue
			}
			_, err := s.kubeClient.Minions().Create(&api.Minion{
				ObjectMeta: api.ObjectMeta{Name: minionID},
				Spec: api.NodeSpec{
					Capacity: s.staticResources.Capacity,
				},
			})
			if err == nil {
				registered.Insert(minionID)
			}
		}
		if registered.Len() == len(s.minions) {
			return nil
		}
		time.Sleep(period)
	}
	return nil
}

// SyncCloud syncs list of instances from cloudprovider to master etcd registry.
func (s *MinionController) SyncCloud() error {
	matches, err := s.cloudMinions()
	if err != nil {
		return err
	}
	minions, err := s.kubeClient.Minions().List()
	if err != nil {
		return err
	}
	minionMap := make(map[string]*api.Minion)
	for _, minion := range minions.Items {
		minionMap[minion.Name] = &minion
	}

	// Create or delete minions from registry.
	for _, minion := range matches.Items {
		if _, ok := minionMap[minion.Name]; !ok {
			glog.Infof("Create minion in registry: %s", minion.Name)
			_, err = s.kubeClient.Minions().Create(&minion)
			if err != nil {
				glog.Errorf("Create minion error: %s", minion.Name)
			}
		}
		delete(minionMap, minion.Name)
	}

	for minionID := range minionMap {
		glog.Infof("Delete minion from registry: %s", minionID)
		err = s.kubeClient.Minions().Delete(minionID)
		if err != nil {
			glog.Errorf("Delete minion error: %s", minionID)
		}
	}
	return nil
}

// cloudMinions constructs and returns api.MinionList from cloudprovider.
func (s *MinionController) cloudMinions() (*api.MinionList, error) {
	instances, ok := s.cloud.Instances()
	if !ok {
		return nil, fmt.Errorf("cloud doesn't support instances")
	}
	matches, err := instances.List(s.matchRE)
	if err != nil {
		return nil, err
	}
	result := &api.MinionList{
		Items: make([]api.Minion, len(matches)),
	}
	for i := range matches {
		result.Items[i].Name = matches[i]
		hostIP, err := instances.IPAddress(matches[i])
		if err != nil {
			glog.Errorf("error getting instance ip address for %s: %v", matches[i], err)
		} else {
			result.Items[i].Status.HostIP = hostIP.String()
		}
		resources, err := instances.GetNodeResources(matches[i])
		if err != nil {
			return nil, err
		}
		if resources == nil {
			resources = s.staticResources
		}
		if resources != nil {
			result.Items[i].Spec.Capacity = resources.Capacity
		}
	}
	return result, nil
}
<|endoftext|>


<commit_before>// Package agent runs readers, writers, and HTTP server.
package agent

import (
	"encoding/json"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	resourced_config "github.com/resourced/resourced/config"
	"github.com/resourced/resourced/executors"
	"github.com/resourced/resourced/host"
	"github.com/resourced/resourced/libnet"
	"github.com/resourced/resourced/libstring"
	"github.com/resourced/resourced/libtime"
	"github.com/resourced/resourced/readers"
	"github.com/resourced/resourced/storage"
	"github.com/resourced/resourced/writers"
	"github.com/resourced/resourced/wstrafficker"
	"github.com/satori/go.uuid"
)

// New is the constructor for Agent struct.
func New() (*Agent, error) {
	agent := &Agent{}

	agent.ID = uuid.NewV4().String()

	err := agent.setConfigs()
	if err != nil {
		return nil, err
	}

	err = agent.setAllowedNetworks()
	if err != nil {
		return nil, err
	}

	err = agent.setTags()
	if err != nil {
		return nil, err
	}

	err = agent.setWSTrafficker()
	if err != nil {
		return nil, err
	}

	err = agent.setStorages()
	if err != nil {
		return nil, err
	}

	return agent, err
}

// Agent struct carries most of the functionality of ResourceD.
// It collects information through readers and serve them up as HTTP+JSON.
type Agent struct {
	ID               string
	Tags             map[string]string
	Configs          *resourced_config.Configs
	GeneralConfig    resourced_config.GeneralConfig
	MetadataStorages *storage.MetadataStorages
	DbPath           string
	Db               *storage.Storage
	AllowedNetworks  []*net.IPNet
	WSTrafficker     *wstrafficker.WSTrafficker
}

func (a *Agent) IsTLS() bool {
	if a.GeneralConfig.HTTPS.CertFile != "" && a.GeneralConfig.HTTPS.KeyFile != "" {
		return true
	}
	return false
}

func (a *Agent) setAllowedNetworks() error {
	allowedNetworks, err := libnet.ParseCIDRs(a.GeneralConfig.AllowedNetworks)
	if err != nil {
		return err
	}

	a.AllowedNetworks = allowedNetworks
	return nil
}

// pathWithPrefix prepends the short version of config.Kind to path.
func (a *Agent) pathWithPrefix(config resourced_config.Config) string {
	if config.Kind == "reader" {
		return a.pathWithKindPrefix("r", config)
	} else if config.Kind == "writer" {
		return a.pathWithKindPrefix("w", config)
	} else if config.Kind == "executor" {
		return a.pathWithKindPrefix("x", config)
	}
	return config.Path
}

// pathWithKindPrefix is common function called by pathWithReaderPrefix or pathWithWriterPrefix
func (a *Agent) pathWithKindPrefix(kind string, input interface{}) string {
	prefix := "/" + kind

	switch v := input.(type) {
	case resourced_config.Config:
		return prefix + v.Path
	case string:
		if strings.HasPrefix(v, prefix+"/") {
			return v
		} else {
			return prefix + v
		}
	}
	return ""
}

// Run executes a reader/writer config.
// Run will save reader data as JSON in local db.
func (a *Agent) Run(config resourced_config.Config) (output []byte, err error) {
	if config.GoStruct != "" && config.Kind == "reader" {
		output, err = a.runGoStructReader(config)
	} else if config.GoStruct != "" && config.Kind == "writer" {
		output, err = a.runGoStructWriter(config)
	} else if config.GoStruct != "" && config.Kind == "executor" {
		output, err = a.runGoStructExecutor(config)
	}

	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error":              err.Error(),
			"config.GoStruct":    config.GoStruct,
			"config.Path":        config.Path,
			"config.Interval":    config.Interval,
			"config.Kind":        config.Kind,
			"config.ReaderPaths": fmt.Sprintf("%s", config.ReaderPaths),
		}).Error("Failed to execute runGoStructReader/runGoStructWriter/runGoStructExecutor")
	}

	err = a.saveRun(config, output, err)

	return output, err
}

// initGoStructReader initialize and return IReader.
func (a *Agent) initGoStructReader(config resourced_config.Config) (readers.IReader, error) {
	return readers.NewGoStructByConfig(config)
}

// initGoStructWriter initialize and return IWriter.
func (a *Agent) initGoStructWriter(config resourced_config.Config) (writers.IWriter, error) {
	writer, err := writers.NewGoStructByConfig(config)
	if err != nil {
		return nil, err
	}

	// Set configs data.
	writer.SetConfigs(a.Configs)

	// Get readers data.
	readersData := make(map[string][]byte)

	for _, readerPath := range config.ReaderPaths {
		readerJsonBytes, err := a.GetRunByPath(a.pathWithKindPrefix("r", readerPath))
		if err == nil {
			readersData[readerPath] = readerJsonBytes
		}
	}

	writer.SetReadersDataInBytes(readersData)

	return writer, err
}

// initResourcedMasterWriter initialize ResourceD Master specific IWriter.
func (a *Agent) initResourcedMasterWriter(config resourced_config.Config) (writers.IWriter, error) {
	var apiPath string

	if config.GoStruct == "ResourcedMasterHost" {
		apiPath = "/api/hosts"
	} else if config.GoStruct == "ResourcedMasterExecutors" {
		apiPath = "/api/executors"
	}

	urlFromConfigInterface, ok := config.GoStructFields["Url"]
	if !ok || urlFromConfigInterface == nil {
		config.GoStructFields["Url"] = a.GeneralConfig.ResourcedMaster.URL + apiPath

	} else {
		urlFromConfig := urlFromConfigInterface.(string)
		if !strings.HasSuffix(urlFromConfig, apiPath) {
			config.GoStructFields["Url"] = a.GeneralConfig.ResourcedMaster.URL + apiPath
		}
	}

	return a.initGoStructWriter(config)
}

// initGoStructExecutor initialize and return IExecutor.
func (a *Agent) initGoStructExecutor(config resourced_config.Config) (executors.IExecutor, error) {
	executor, err := executors.NewGoStructByConfig(config)
	if err != nil {
		return nil, err
	}

	executor.SetReadersDataInBytes(a.Db.Data)
	executor.SetTags(a.Tags)
	executor.SetMetadataStorages(a.MetadataStorages)

	return executor, nil
}

// runGoStruct executes Run() fom IReader/IWriter/IExecutor and returns the output.
// Note that IWriter and IExecutor also implement IReader.
func (a *Agent) runGoStruct(readerOrWriterOrExecutor readers.IReader) ([]byte, error) {
	err := readerOrWriterOrExecutor.Run()
	if err != nil {
		errData := make(map[string]string)
		errData["Error"] = err.Error()
		return json.Marshal(errData)
	}

	return readerOrWriterOrExecutor.ToJson()
}

// runGoStructReader executes IReader and returns the output.
func (a *Agent) runGoStructReader(config resourced_config.Config) ([]byte, error) {
	// Initialize IReader
	reader, err := a.initGoStructReader(config)
	if err != nil {
		return nil, err
	}

	return a.runGoStruct(reader)
}

// runGoStructWriter executes IWriter and returns error if exists.
func (a *Agent) runGoStructWriter(config resourced_config.Config) ([]byte, error) {
	var writer writers.IWriter
	var err error

	// Initialize IWriter
	if strings.HasPrefix(config.GoStruct, "ResourcedMaster") {
		writer, err = a.initResourcedMasterWriter(config)
		if err != nil {
			return nil, err
		}

	} else {
		writer, err = a.initGoStructWriter(config)
		if err != nil {
			return nil, err
		}
	}

	err = writer.GenerateData()
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error":              err.Error(),
			"config.GoStruct":    config.GoStruct,
			"config.Path":        config.Path,
			"config.Interval":    config.Interval,
			"config.Kind":        config.Kind,
			"config.ReaderPaths": fmt.Sprintf("%s", config.ReaderPaths),
		}).Error("Failed to execute writer.GenerateData()")

		return nil, err
	}

	return a.runGoStruct(writer)
}

// runGoStructExecutor executes IExecutor and returns the output.
func (a *Agent) runGoStructExecutor(config resourced_config.Config) ([]byte, error) {
	var executor executors.IExecutor
	var err error

	// Initialize IExecutor
	executor, err = a.initGoStructExecutor(config)
	if err != nil {
		return nil, err
	}

	return a.runGoStruct(executor)
}

// commonData gathers common information for every reader and writer.
func (a *Agent) commonData(config resourced_config.Config) map[string]interface{} {
	record := make(map[string]interface{})
	record["UnixNano"] = time.Now().UnixNano()
	record["Path"] = config.Path

	if config.Interval == "" {
		config.Interval = "1m"
	}
	record["Interval"] = config.Interval

	if config.GoStruct != "" {
		record["GoStruct"] = config.GoStruct
	}

	return record
}

// hostData builds host related information.
func (a *Agent) hostData() (*host.Host, error) {
	h, err := host.NewHostByHostname()
	if err != nil {
		return nil, err
	}

	h.Tags = a.Tags

	return h, nil
}

// saveRun gathers basic, host, and reader/witer information and save them into local storage.
func (a *Agent) saveRun(config resourced_config.Config, output []byte, err error) error {
	// Do not perform save if config.Path is empty.
	if config.Path == "" {
		return nil
	}

	record := a.commonData(config)

	host, err := a.hostData()
	if err != nil {
		return err
	}
	record["Host"] = host

	if err == nil {
		runData := new(interface{})
		err = json.Unmarshal(output, &runData)
		if err != nil {
			return err
		}
		record["Data"] = runData

	} else {
		errMap := make(map[string]string)
		errMap["Error"] = err.Error()
		record["Data"] = errMap
	}

	recordInJson, err := json.Marshal(record)
	if err != nil {
		return err
	}

	a.Db.Set(a.pathWithPrefix(config), recordInJson)

	return err
}

// GetRun returns the JSON data stored in local storage given Config struct.
func (a *Agent) GetRun(config resourced_config.Config) ([]byte, error) {
	return a.GetRunByPath(a.pathWithPrefix(config))
}

// GetRunByPath returns JSON data stored in local storage given path string.
func (a *Agent) GetRunByPath(path string) ([]byte, error) {
	return a.Db.Get(path), nil
}

// RunForever executes Run() in an infinite loop with a sleep of config.Interval.
func (a *Agent) RunForever(config resourced_config.Config) {
	go func(a *Agent, config resourced_config.Config) {
		for {
			a.Run(config)
			libtime.SleepString(config.Interval)
		}
	}(a, config)
}

// RunAllForever executes all readers & writers in an infinite loop.
func (a *Agent) RunAllForever() {
	for _, config := range a.Configs.Readers {
		a.RunForever(config)
	}
	for _, config := range a.Configs.Writers {
		a.RunForever(config)
	}
	for _, config := range a.Configs.Executors {
		a.RunForever(config)
	}
}

// Check if a given IP:PORT is part of an allowed CIDR
func (a *Agent) IsAllowed(address string) bool {
	// Allow all if we allowed networks is not set
	if len(a.AllowedNetworks) == 0 {
		return true
	}

	ip := libstring.GetIP(address)
	if ip == nil {
		return false
	}

	// Check if IP is in one of our allowed networks
	for _, network := range a.AllowedNetworks {
		if network.Contains(ip) {
			return true
		}
	}

	return false
}
<commit_msg>Report executors by hostname<commit_after>// Package agent runs readers, writers, and HTTP server.
package agent

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	resourced_config "github.com/resourced/resourced/config"
	"github.com/resourced/resourced/executors"
	"github.com/resourced/resourced/host"
	"github.com/resourced/resourced/libnet"
	"github.com/resourced/resourced/libstring"
	"github.com/resourced/resourced/libtime"
	"github.com/resourced/resourced/readers"
	"github.com/resourced/resourced/storage"
	"github.com/resourced/resourced/writers"
	"github.com/resourced/resourced/wstrafficker"
	"github.com/satori/go.uuid"
)

// New is the constructor for Agent struct.
func New() (*Agent, error) {
	agent := &Agent{}

	agent.ID = uuid.NewV4().String()

	err := agent.setConfigs()
	if err != nil {
		return nil, err
	}

	err = agent.setAllowedNetworks()
	if err != nil {
		return nil, err
	}

	err = agent.setTags()
	if err != nil {
		return nil, err
	}

	err = agent.setWSTrafficker()
	if err != nil {
		return nil, err
	}

	err = agent.setStorages()
	if err != nil {
		return nil, err
	}

	return agent, err
}

// Agent struct carries most of the functionality of ResourceD.
// It collects information through readers and serve them up as HTTP+JSON.
type Agent struct {
	ID               string
	Tags             map[string]string
	Configs          *resourced_config.Configs
	GeneralConfig    resourced_config.GeneralConfig
	MetadataStorages *storage.MetadataStorages
	DbPath           string
	Db               *storage.Storage
	AllowedNetworks  []*net.IPNet
	WSTrafficker     *wstrafficker.WSTrafficker
}

func (a *Agent) IsTLS() bool {
	if a.GeneralConfig.HTTPS.CertFile != "" && a.GeneralConfig.HTTPS.KeyFile != "" {
		return true
	}
	return false
}

func (a *Agent) setAllowedNetworks() error {
	allowedNetworks, err := libnet.ParseCIDRs(a.GeneralConfig.AllowedNetworks)
	if err != nil {
		return err
	}

	a.AllowedNetworks = allowedNetworks
	return nil
}

// pathWithPrefix prepends the short version of config.Kind to path.
func (a *Agent) pathWithPrefix(config resourced_config.Config) string {
	if config.Kind == "reader" {
		return a.pathWithKindPrefix("r", config)
	} else if config.Kind == "writer" {
		return a.pathWithKindPrefix("w", config)
	} else if config.Kind == "executor" {
		return a.pathWithKindPrefix("x", config)
	}
	return config.Path
}

// pathWithKindPrefix is common function called by pathWithReaderPrefix or pathWithWriterPrefix
func (a *Agent) pathWithKindPrefix(kind string, input interface{}) string {
	prefix := "/" + kind

	switch v := input.(type) {
	case resourced_config.Config:
		return prefix + v.Path
	case string:
		if strings.HasPrefix(v, prefix+"/") {
			return v
		} else {
			return prefix + v
		}
	}
	return ""
}

// Run executes a reader/writer config.
// Run will save reader data as JSON in local db.
func (a *Agent) Run(config resourced_config.Config) (output []byte, err error) {
	if config.GoStruct != "" && config.Kind == "reader" {
		output, err = a.runGoStructReader(config)
	} else if config.GoStruct != "" && config.Kind == "writer" {
		output, err = a.runGoStructWriter(config)
	} else if config.GoStruct != "" && config.Kind == "executor" {
		output, err = a.runGoStructExecutor(config)
	}

	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error":              err.Error(),
			"config.GoStruct":    config.GoStruct,
			"config.Path":        config.Path,
			"config.Interval":    config.Interval,
			"config.Kind":        config.Kind,
			"config.ReaderPaths": fmt.Sprintf("%s", config.ReaderPaths),
		}).Error("Failed to execute runGoStructReader/runGoStructWriter/runGoStructExecutor")
	}

	err = a.saveRun(config, output, err)

	return output, err
}

// initGoStructReader initialize and return IReader.
func (a *Agent) initGoStructReader(config resourced_config.Config) (readers.IReader, error) {
	return readers.NewGoStructByConfig(config)
}

// initGoStructWriter initialize and return IWriter.
func (a *Agent) initGoStructWriter(config resourced_config.Config) (writers.IWriter, error) {
	writer, err := writers.NewGoStructByConfig(config)
	if err != nil {
		return nil, err
	}

	// Set configs data.
	writer.SetConfigs(a.Configs)

	// Get readers data.
	readersData := make(map[string][]byte)

	for _, readerPath := range config.ReaderPaths {
		readerJsonBytes, err := a.GetRunByPath(a.pathWithKindPrefix("r", readerPath))
		if err == nil {
			readersData[readerPath] = readerJsonBytes
		}
	}

	writer.SetReadersDataInBytes(readersData)

	return writer, err
}

// initResourcedMasterWriter initialize ResourceD Master specific IWriter.
func (a *Agent) initResourcedMasterWriter(config resourced_config.Config) (writers.IWriter, error) {
	var apiPath string

	if config.GoStruct == "ResourcedMasterHost" {
		apiPath = "/api/hosts"
	} else if config.GoStruct == "ResourcedMasterExecutors" {
		hostname, err := os.Hostname()
		if err != nil {
			return nil, err
		}
		apiPath = "/api/executors/" + hostname
	}

	urlFromConfigInterface, ok := config.GoStructFields["Url"]
	if !ok || urlFromConfigInterface == nil { // Check if Url is not defined in config
		config.GoStructFields["Url"] = a.GeneralConfig.ResourcedMaster.URL + apiPath

	} else { // Check if Url does not contain apiPath
		urlFromConfig := urlFromConfigInterface.(string)
		if !strings.HasSuffix(urlFromConfig, apiPath) {
			config.GoStructFields["Url"] = a.GeneralConfig.ResourcedMaster.URL + apiPath
		}
	}

	// Check if username is not defined
	// If so, set GeneralConfig.ResourcedMaster.AccessToken as default
	usernameFromConfigInterface, ok := config.GoStructFields["Username"]
	if !ok || usernameFromConfigInterface == nil {
		config.GoStructFields["Username"] = a.GeneralConfig.ResourcedMaster.AccessToken

	}

	return a.initGoStructWriter(config)
}

// initGoStructExecutor initialize and return IExecutor.
func (a *Agent) initGoStructExecutor(config resourced_config.Config) (executors.IExecutor, error) {
	executor, err := executors.NewGoStructByConfig(config)
	if err != nil {
		return nil, err
	}

	executor.SetReadersDataInBytes(a.Db.Data)
	executor.SetTags(a.Tags)
	executor.SetMetadataStorages(a.MetadataStorages)

	return executor, nil
}

// runGoStruct executes Run() fom IReader/IWriter/IExecutor and returns the output.
// Note that IWriter and IExecutor also implement IReader.
func (a *Agent) runGoStruct(readerOrWriterOrExecutor readers.IReader) ([]byte, error) {
	err := readerOrWriterOrExecutor.Run()
	if err != nil {
		errData := make(map[string]string)
		errData["Error"] = err.Error()
		return json.Marshal(errData)
	}

	return readerOrWriterOrExecutor.ToJson()
}

// runGoStructReader executes IReader and returns the output.
func (a *Agent) runGoStructReader(config resourced_config.Config) ([]byte, error) {
	// Initialize IReader
	reader, err := a.initGoStructReader(config)
	if err != nil {
		return nil, err
	}

	return a.runGoStruct(reader)
}

// runGoStructWriter executes IWriter and returns error if exists.
func (a *Agent) runGoStructWriter(config resourced_config.Config) ([]byte, error) {
	var writer writers.IWriter
	var err error

	// Initialize IWriter
	if strings.HasPrefix(config.GoStruct, "ResourcedMaster") {
		writer, err = a.initResourcedMasterWriter(config)
		if err != nil {
			return nil, err
		}

	} else {
		writer, err = a.initGoStructWriter(config)
		if err != nil {
			return nil, err
		}
	}

	err = writer.GenerateData()
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error":              err.Error(),
			"config.GoStruct":    config.GoStruct,
			"config.Path":        config.Path,
			"config.Interval":    config.Interval,
			"config.Kind":        config.Kind,
			"config.ReaderPaths": fmt.Sprintf("%s", config.ReaderPaths),
		}).Error("Failed to execute writer.GenerateData()")

		return nil, err
	}

	return a.runGoStruct(writer)
}

// runGoStructExecutor executes IExecutor and returns the output.
func (a *Agent) runGoStructExecutor(config resourced_config.Config) ([]byte, error) {
	var executor executors.IExecutor
	var err error

	// Initialize IExecutor
	executor, err = a.initGoStructExecutor(config)
	if err != nil {
		return nil, err
	}

	return a.runGoStruct(executor)
}

// commonData gathers common information for every reader and writer.
func (a *Agent) commonData(config resourced_config.Config) map[string]interface{} {
	record := make(map[string]interface{})
	record["UnixNano"] = time.Now().UnixNano()
	record["Path"] = config.Path

	if config.Interval == "" {
		config.Interval = "1m"
	}
	record["Interval"] = config.Interval

	if config.GoStruct != "" {
		record["GoStruct"] = config.GoStruct
	}

	return record
}

// hostData builds host related information.
func (a *Agent) hostData() (*host.Host, error) {
	h, err := host.NewHostByHostname()
	if err != nil {
		return nil, err
	}

	h.Tags = a.Tags

	return h, nil
}

// saveRun gathers basic, host, and reader/witer information and save them into local storage.
func (a *Agent) saveRun(config resourced_config.Config, output []byte, err error) error {
	// Do not perform save if config.Path is empty.
	if config.Path == "" {
		return nil
	}

	record := a.commonData(config)

	host, err := a.hostData()
	if err != nil {
		return err
	}
	record["Host"] = host

	if err == nil {
		runData := new(interface{})
		err = json.Unmarshal(output, &runData)
		if err != nil {
			return err
		}
		record["Data"] = runData

	} else {
		errMap := make(map[string]string)
		errMap["Error"] = err.Error()
		record["Data"] = errMap
	}

	recordInJson, err := json.Marshal(record)
	if err != nil {
		return err
	}

	a.Db.Set(a.pathWithPrefix(config), recordInJson)

	return err
}

// GetRun returns the JSON data stored in local storage given Config struct.
func (a *Agent) GetRun(config resourced_config.Config) ([]byte, error) {
	return a.GetRunByPath(a.pathWithPrefix(config))
}

// GetRunByPath returns JSON data stored in local storage given path string.
func (a *Agent) GetRunByPath(path string) ([]byte, error) {
	return a.Db.Get(path), nil
}

// RunForever executes Run() in an infinite loop with a sleep of config.Interval.
func (a *Agent) RunForever(config resourced_config.Config) {
	go func(a *Agent, config resourced_config.Config) {
		for {
			a.Run(config)
			libtime.SleepString(config.Interval)
		}
	}(a, config)
}

// RunAllForever executes all readers & writers in an infinite loop.
func (a *Agent) RunAllForever() {
	for _, config := range a.Configs.Readers {
		a.RunForever(config)
	}
	for _, config := range a.Configs.Writers {
		a.RunForever(config)
	}
	for _, config := range a.Configs.Executors {
		a.RunForever(config)
	}
}

// Check if a given IP:PORT is part of an allowed CIDR
func (a *Agent) IsAllowed(address string) bool {
	// Allow all if we allowed networks is not set
	if len(a.AllowedNetworks) == 0 {
		return true
	}

	ip := libstring.GetIP(address)
	if ip == nil {
		return false
	}

	// Check if IP is in one of our allowed networks
	for _, network := range a.AllowedNetworks {
		if network.Contains(ip) {
			return true
		}
	}

	return false
}
<|endoftext|>