<commit_before>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"encoding/json"
	"strings"

	pipelinev1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	"github.com/tidwall/gjson"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"knative.dev/pkg/apis"
)

// apiVersion for Tekton core types
const tektonAPIVersion = GroupName + "/v1alpha1"

// allowedTemplate types are the resource types identified by apiVersion + kind
// that can be templated using TriggerResourceTemplates
// TODO: Replace static restrictions here with SubjectAccessReview create checks
var allowedTemplateTypes = map[string][]string{
	tektonAPIVersion: {"pipelineresource", "pipelinerun", "taskrun", "pipeline", "clustertask", "task", "condition"},
}

// Check that TriggerTemplate may be validated and defaulted.
var _ apis.Validatable = (*TriggerTemplate)(nil)
var _ apis.Defaultable = (*TriggerTemplate)(nil)

// TriggerTemplateSpec holds the desired state of TriggerTemplate
type TriggerTemplateSpec struct {
	Params            []pipelinev1.ParamSpec    `json:"params,omitempty"`
	ResourceTemplates []TriggerResourceTemplate `json:"resourcetemplates,omitempty"`
}

// TriggerResourceTemplate describes a resource to create
type TriggerResourceTemplate struct {
	json.RawMessage `json:",inline"`
}

// TriggerTemplateStatus describes the desired state of TriggerTemplate
type TriggerTemplateStatus struct{}

// TriggerTemplate takes parameters and uses them to create CRDs
//
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:openapi-gen=true
type TriggerTemplate struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`
	// Spec holds the desired state of the TriggerTemplate from the client
	// +optional
	Spec TriggerTemplateSpec `json:"spec"`
	// +optional
	Status TriggerTemplateStatus `json:"status"`
}

// TriggerTemplateList contains a list of TriggerTemplate
//
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type TriggerTemplateList struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`
	Items           []TriggerTemplate `json:"items"`
}

// getAPIVersionAndKind returns the apiVersion and Kind for the resourceTemplate
// Missing fields are represented by empty strings
func (trt *TriggerResourceTemplate) getAPIVersionAndKind() (string, string) {
	apiVersion := gjson.GetBytes(trt.RawMessage, "apiVersion").String()
	kind := gjson.GetBytes(trt.RawMessage, "kind").String()
	return apiVersion, kind
}

// IsAllowedType returns true if the resourceTemplate has an apiVersion
// and kind field set to one of the allowed ones.
func (trt *TriggerResourceTemplate) IsAllowedType() bool {
	apiVersion, kind := trt.getAPIVersionAndKind()

	if kinds, ok := allowedTemplateTypes[apiVersion]; ok {
		for _, allowedKind := range kinds {
			if strings.ToLower(kind) == allowedKind {
				return true
			}
		}
	}
	return false
}
<commit_msg>Avoid using gjson to parse a json.RawMessage<commit_after>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"bytes"
	"encoding/json"
	"strings"

	pipelinev1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"knative.dev/pkg/apis"
)

// apiVersion for Tekton core types
const tektonAPIVersion = GroupName + "/v1alpha1"

// allowedTemplate types are the resource types identified by apiVersion + kind
// that can be templated using TriggerResourceTemplates
// TODO: Replace static restrictions here with SubjectAccessReview create checks
var allowedTemplateTypes = map[string][]string{
	tektonAPIVersion: {"pipelineresource", "pipelinerun", "taskrun", "pipeline", "clustertask", "task", "condition"},
}

// Check that TriggerTemplate may be validated and defaulted.
var _ apis.Validatable = (*TriggerTemplate)(nil)
var _ apis.Defaultable = (*TriggerTemplate)(nil)

// TriggerTemplateSpec holds the desired state of TriggerTemplate
type TriggerTemplateSpec struct {
	Params            []pipelinev1.ParamSpec    `json:"params,omitempty"`
	ResourceTemplates []TriggerResourceTemplate `json:"resourcetemplates,omitempty"`
}

// TriggerResourceTemplate describes a resource to create
type TriggerResourceTemplate struct {
	json.RawMessage `json:",inline"`
}

// TriggerTemplateStatus describes the desired state of TriggerTemplate
type TriggerTemplateStatus struct{}

// TriggerTemplate takes parameters and uses them to create CRDs
//
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:openapi-gen=true
type TriggerTemplate struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`
	// Spec holds the desired state of the TriggerTemplate from the client
	// +optional
	Spec TriggerTemplateSpec `json:"spec"`
	// +optional
	Status TriggerTemplateStatus `json:"status"`
}

// TriggerTemplateList contains a list of TriggerTemplate
//
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type TriggerTemplateList struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`
	Items           []TriggerTemplate `json:"items"`
}

// getAPIVersionAndKind returns the apiVersion and Kind for the resourceTemplate
// Missing fields are represented by empty strings
func (trt *TriggerResourceTemplate) getAPIVersionAndKind() (string, string) {
	var tm metav1.TypeMeta
	if err := json.NewDecoder(bytes.NewReader(trt.RawMessage)).Decode(&tm); err != nil {
		return "", ""
	}
	return tm.APIVersion, tm.Kind
}

// IsAllowedType returns true if the resourceTemplate has an apiVersion
// and kind field set to one of the allowed ones.
func (trt *TriggerResourceTemplate) IsAllowedType() bool {
	apiVersion, kind := trt.getAPIVersionAndKind()

	if kinds, ok := allowedTemplateTypes[apiVersion]; ok {
		for _, allowedKind := range kinds {
			if strings.ToLower(kind) == allowedKind {
				return true
			}
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package golang provides the Match Making Function service for Open Match golang harness.
package golang

import (
	"context"
	"io"

	structpb "github.com/golang/protobuf/ptypes/struct"
	"github.com/sirupsen/logrus"
	"open-match.dev/open-match/internal/config"
	"open-match.dev/open-match/internal/rpc"
	"open-match.dev/open-match/pkg/pb"
)

var (
	logger = logrus.WithFields(logrus.Fields{
		"app":       "openmatch",
		"component": "matchfunction.harness.golang",
	})
)

// MatchFunction is the function signature for the Match Making Function (MMF) to be implemented by the user.
// The harness will pass the Rosters and PlayerPool for the match profile to this
// function and it will return the Rosters to be populated in the proposal.
// Input:
//  - MatchFunctionParams:
//			A structure that defines the resources that are available to the match function.
//			Developers can choose to add context to the structure such that match function has the ability
//			to cancel a stream response/request or to limit match function by sharing a static and protected view.
type MatchFunction func(*MatchFunctionParams) ([]*pb.Match, error)

// matchFunctionService implements pb.MatchFunctionServer, the server generated
// by compiling the protobuf, by fulfilling the pb.MatchFunctionServer interface.
type matchFunctionService struct {
	cfg           config.View
	function      MatchFunction
	mmlogicClient pb.MmLogicClient
}

// MatchFunctionParams : This harness example defines a protected view for the match function.
//  - logger:
//			A logger used to generate error/debug logs
//  - profileName
//			The name of profile
//  - properties:
//			'Properties' of a MatchObject.
//  - rosters:
//			An array of Rosters. By convention, your input Roster contains players already in
//          the match, and the names of pools to search when trying to fill an empty slot.
//  - poolNameToTickets:
//			A map that contains mappings from pool name to a list of tickets that satisfied the filters in the pool
type MatchFunctionParams struct {
	Logger            *logrus.Entry
	ProfileName       string
	Properties        *structpb.Struct
	Rosters           []*pb.Roster
	PoolNameToTickets map[string][]*pb.Ticket
}

// Run is this harness's implementation of the gRPC call defined in api/matchfunction.proto.
func (s *matchFunctionService) Run(req *pb.RunRequest, stream pb.MatchFunction_RunServer) error {
	poolNameToTickets, err := s.getMatchManifest(stream.Context(), req)
	if err != nil {
		return err
	}

	// The matchfunction takes in some half-filled/empty rosters, a property bag, and a map[poolNames]tickets to generate match proposals
	mfParams := &MatchFunctionParams{
		Logger: logrus.WithFields(logrus.Fields{
			"app":       "openmatch",
			"component": "matchfunction.implementation",
		}),
		ProfileName:       req.GetProfile().GetName(),
		Properties:        req.GetProfile().GetProperties(),
		Rosters:           req.GetProfile().GetRosters(),
		PoolNameToTickets: poolNameToTickets,
	}
	// Run the customize match function!
	proposals, err := s.function(mfParams)
	if err != nil {
		return err
	}
	logger.WithFields(logrus.Fields{
		"proposals": proposals,
	}).Trace("proposals returned by match function")

	for _, proposal := range proposals {
		if err := stream.Send(&pb.RunResponse{Proposal: proposal}); err != nil {
			return err
		}
	}

	return nil
}

func newMatchFunctionService(cfg config.View, fs *FunctionSettings) (*matchFunctionService, error) {
	conn, err := rpc.GRPCClientFromConfig(cfg, "api.mmlogic")
	if err != nil {
		logger.Errorf("Failed to get MMLogic connection, %v.", err)
		return nil, err
	}

	mmfService := &matchFunctionService{cfg: cfg, function: fs.Func, mmlogicClient: pb.NewMmLogicClient(conn)}
	return mmfService, nil
}

// getMatchManifest fetches all the data needed from the mmlogic API.
func (s *matchFunctionService) getMatchManifest(ctx context.Context, req *pb.RunRequest) (map[string][]*pb.Ticket, error) {
	poolNameToTickets := make(map[string][]*pb.Ticket)
	filterPools := req.GetProfile().GetPools()

	for _, pool := range filterPools {
		qtClient, err := s.mmlogicClient.QueryTickets(ctx, &pb.QueryTicketsRequest{Pool: pool})
		if err != nil {
			logger.WithError(err).Error("Failed to get queryTicketClient from mmlogic.")
			return nil, err
		}

		// Aggregate tickets by poolName
		poolTickets := make([]*pb.Ticket, 0)
		for {
			qtResponse, err := qtClient.Recv()
			if err == io.EOF {
				logger.Trace("Received all results from the queryTicketClient.")
				// Break when all results are received
				break
			}

			if err != nil {
				logger.WithError(err).Error("Failed to receive a response from the queryTicketClient.")
				return nil, err
			}
			poolTickets = append(poolTickets, qtResponse.Tickets...)
		}
		poolNameToTickets[pool.GetName()] = poolTickets
	}

	return poolNameToTickets, nil
}
<commit_msg>Have mmf harness pass extension instead of properties (#865)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package golang provides the Match Making Function service for Open Match golang harness.
package golang

import (
	"context"
	"io"

	"github.com/golang/protobuf/ptypes/any"
	"github.com/sirupsen/logrus"
	"open-match.dev/open-match/internal/config"
	"open-match.dev/open-match/internal/rpc"
	"open-match.dev/open-match/pkg/pb"
)

var (
	logger = logrus.WithFields(logrus.Fields{
		"app":       "openmatch",
		"component": "matchfunction.harness.golang",
	})
)

// MatchFunction is the function signature for the Match Making Function (MMF) to be implemented by the user.
// The harness will pass the Rosters and PlayerPool for the match profile to this
// function and it will return the Rosters to be populated in the proposal.
// Input:
//  - MatchFunctionParams:
//			A structure that defines the resources that are available to the match function.
//			Developers can choose to add context to the structure such that match function has the ability
//			to cancel a stream response/request or to limit match function by sharing a static and protected view.
type MatchFunction func(*MatchFunctionParams) ([]*pb.Match, error)

// matchFunctionService implements pb.MatchFunctionServer, the server generated
// by compiling the protobuf, by fulfilling the pb.MatchFunctionServer interface.
type matchFunctionService struct {
	cfg           config.View
	function      MatchFunction
	mmlogicClient pb.MmLogicClient
}

// MatchFunctionParams is a protected view for the match function.
type MatchFunctionParams struct {
	// Logger is used to generate error/debug logs
	Logger *logrus.Entry

	// 'Name' from the MatchProfile.
	ProfileName string

	// 'Extension' from the MatchProfile.
	Extension *any.Any

	// An array of Rosters. By convention, your input Roster contains players already in
	// the match, and the names of pools to search when trying to fill an empty slot.
	Rosters []*pb.Roster

	// A map that contains mappings from pool name to a list of tickets that satisfied the filters in the pool
	PoolNameToTickets map[string][]*pb.Ticket
}

// Run is this harness's implementation of the gRPC call defined in api/matchfunction.proto.
func (s *matchFunctionService) Run(req *pb.RunRequest, stream pb.MatchFunction_RunServer) error {
	poolNameToTickets, err := s.getMatchManifest(stream.Context(), req)
	if err != nil {
		return err
	}

	// The matchfunction takes in some half-filled/empty rosters, a property bag, and a map[poolNames]tickets to generate match proposals
	mfParams := &MatchFunctionParams{
		Logger: logrus.WithFields(logrus.Fields{
			"app":       "openmatch",
			"component": "matchfunction.implementation",
		}),
		ProfileName:       req.GetProfile().GetName(),
		Extension:         req.GetProfile().GetExtension(),
		Rosters:           req.GetProfile().GetRosters(),
		PoolNameToTickets: poolNameToTickets,
	}
	// Run the customize match function!
	proposals, err := s.function(mfParams)
	if err != nil {
		return err
	}
	logger.WithFields(logrus.Fields{
		"proposals": proposals,
	}).Trace("proposals returned by match function")

	for _, proposal := range proposals {
		if err := stream.Send(&pb.RunResponse{Proposal: proposal}); err != nil {
			return err
		}
	}

	return nil
}

func newMatchFunctionService(cfg config.View, fs *FunctionSettings) (*matchFunctionService, error) {
	conn, err := rpc.GRPCClientFromConfig(cfg, "api.mmlogic")
	if err != nil {
		logger.Errorf("Failed to get MMLogic connection, %v.", err)
		return nil, err
	}

	mmfService := &matchFunctionService{cfg: cfg, function: fs.Func, mmlogicClient: pb.NewMmLogicClient(conn)}
	return mmfService, nil
}

// getMatchManifest fetches all the data needed from the mmlogic API.
func (s *matchFunctionService) getMatchManifest(ctx context.Context, req *pb.RunRequest) (map[string][]*pb.Ticket, error) {
	poolNameToTickets := make(map[string][]*pb.Ticket)
	filterPools := req.GetProfile().GetPools()

	for _, pool := range filterPools {
		qtClient, err := s.mmlogicClient.QueryTickets(ctx, &pb.QueryTicketsRequest{Pool: pool})
		if err != nil {
			logger.WithError(err).Error("Failed to get queryTicketClient from mmlogic.")
			return nil, err
		}

		// Aggregate tickets by poolName
		poolTickets := make([]*pb.Ticket, 0)
		for {
			qtResponse, err := qtClient.Recv()
			if err == io.EOF {
				logger.Trace("Received all results from the queryTicketClient.")
				// Break when all results are received
				break
			}

			if err != nil {
				logger.WithError(err).Error("Failed to receive a response from the queryTicketClient.")
				return nil, err
			}
			poolTickets = append(poolTickets, qtResponse.Tickets...)
		}
		poolNameToTickets[pool.GetName()] = poolTickets
	}

	return poolNameToTickets, nil
}
<|endoftext|>


<commit_before>package pgx

type ConnectionPool struct {
	connectionChannel chan *Connection
	parameters        ConnectionParameters // options used when establishing connection
	MaxConnections    int
}

// options: options used by Connect
// MaxConnections: max simultaneous connections to use (currently all are immediately connected)
func NewConnectionPool(parameters ConnectionParameters, MaxConnections int) (p *ConnectionPool, err error) {
	p = new(ConnectionPool)
	p.connectionChannel = make(chan *Connection, MaxConnections)
	p.MaxConnections = MaxConnections

	p.parameters = parameters

	for i := 0; i < p.MaxConnections; i++ {
		var c *Connection
		c, err = Connect(p.parameters)
		if err != nil {
			return
		}
		p.connectionChannel <- c
	}

	return
}

func (p *ConnectionPool) Acquire() (c *Connection) {
	c = <-p.connectionChannel
	return
}

func (p *ConnectionPool) Release(c *Connection) {
	p.connectionChannel <- c
}

func (p *ConnectionPool) Close() {
	for i := 0; i < p.MaxConnections; i++ {
		c := <-p.connectionChannel
		_ = c.Close()
	}
}
<commit_msg>Add one shot queries against connection pool<commit_after>package pgx

type ConnectionPool struct {
	connectionChannel chan *Connection
	parameters        ConnectionParameters // options used when establishing connection
	MaxConnections    int
}

// options: options used by Connect
// MaxConnections: max simultaneous connections to use (currently all are immediately connected)
func NewConnectionPool(parameters ConnectionParameters, MaxConnections int) (p *ConnectionPool, err error) {
	p = new(ConnectionPool)
	p.connectionChannel = make(chan *Connection, MaxConnections)
	p.MaxConnections = MaxConnections

	p.parameters = parameters

	for i := 0; i < p.MaxConnections; i++ {
		var c *Connection
		c, err = Connect(p.parameters)
		if err != nil {
			return
		}
		p.connectionChannel <- c
	}

	return
}

func (p *ConnectionPool) Acquire() (c *Connection) {
	c = <-p.connectionChannel
	return
}

func (p *ConnectionPool) Release(c *Connection) {
	p.connectionChannel <- c
}

func (p *ConnectionPool) Close() {
	for i := 0; i < p.MaxConnections; i++ {
		c := <-p.connectionChannel
		_ = c.Close()
	}
}

// Acquires a connection, delegates the call to that connection, and releases the connection
func (p *ConnectionPool) SelectFunc(sql string, onDataRow func(*DataRowReader) error, arguments ...interface{}) (err error) {
	c := p.Acquire()
	defer p.Release(c)

	return c.SelectFunc(sql, onDataRow, arguments...)
}

// Acquires a connection, delegates the call to that connection, and releases the connection
func (p *ConnectionPool) SelectRows(sql string, arguments ...interface{}) (rows []map[string]interface{}, err error) {
	c := p.Acquire()
	defer p.Release(c)

	return c.SelectRows(sql, arguments...)
}

// Acquires a connection, delegates the call to that connection, and releases the connection
func (p *ConnectionPool) SelectRow(sql string, arguments ...interface{}) (row map[string]interface{}, err error) {
	c := p.Acquire()
	defer p.Release(c)

	return c.SelectRow(sql, arguments...)
}

// Acquires a connection, delegates the call to that connection, and releases the connection
func (p *ConnectionPool) SelectValue(sql string, arguments ...interface{}) (v interface{}, err error) {
	c := p.Acquire()
	defer p.Release(c)

	return c.SelectValue(sql, arguments...)
}

// Acquires a connection, delegates the call to that connection, and releases the connection
func (p *ConnectionPool) SelectValues(sql string, arguments ...interface{}) (values []interface{}, err error) {
	c := p.Acquire()
	defer p.Release(c)

	return c.SelectValues(sql, arguments...)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"gopkg.in/alecthomas/kingpin.v2"

	consul_api "github.com/hashicorp/consul/api"
	consul "github.com/hashicorp/consul/consul/structs"
	cleanhttp "github.com/hashicorp/go-cleanhttp"
)

const (
	namespace = "consul"
)

var (
	up = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "up"),
		"Was the last query of Consul successful.",
		nil, nil,
	)
	clusterServers = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "raft_peers"),
		"How many peers (servers) are in the Raft cluster.",
		nil, nil,
	)
	clusterLeader = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "raft_leader"),
		"Does Raft cluster have a leader (according to this node).",
		nil, nil,
	)
	nodeCount = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "serf_lan_members"),
		"How many members are in the cluster.",
		nil, nil,
	)
	serviceCount = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "catalog_services"),
		"How many services are in the cluster.",
		nil, nil,
	)
	serviceNodesHealthy = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "catalog_service_node_healthy"),
		"Is this service healthy on this node?",
		[]string{"service_id", "node", "service_name"}, nil,
	)
	nodeChecks = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "health_node_status"),
		"Status of health checks associated with a node.",
		[]string{"check", "node", "status"}, nil,
	)
	serviceChecks = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "health_service_status"),
		"Status of health checks associated with a service.",
		[]string{"check", "node", "service_id", "service_name", "status"}, nil,
	)
	keyValues = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "catalog_kv"),
		"The values for selected keys in Consul's key/value catalog. Keys with non-numeric values are omitted.",
		[]string{"key"}, nil,
	)
)

// Exporter collects Consul stats from the given server and exports them using
// the prometheus metrics package.
type Exporter struct {
	client        *consul_api.Client
	kvPrefix      string
	kvFilter      *regexp.Regexp
	healthSummary bool
}

type consulOpts struct {
	uri        string
	caFile     string
	certFile   string
	keyFile    string
	serverName string
	timeout    time.Duration
}

// NewExporter returns an initialized Exporter.
func NewExporter(opts consulOpts, kvPrefix, kvFilter string, healthSummary bool) (*Exporter, error) {
	uri := opts.uri
	if !strings.Contains(uri, "://") {
		uri = "http://" + uri
	}
	u, err := url.Parse(uri)
	if err != nil {
		return nil, fmt.Errorf("invalid consul URL: %s", err)
	}
	if u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
		return nil, fmt.Errorf("invalid consul URL: %s", uri)
	}

	tlsConfig, err := consul_api.SetupTLSConfig(&consul_api.TLSConfig{
		Address:  opts.serverName,
		CAFile:   opts.caFile,
		CertFile: opts.certFile,
		KeyFile:  opts.keyFile,
	})
	if err != nil {
		return nil, err
	}
	transport := cleanhttp.DefaultPooledTransport()
	transport.TLSClientConfig = tlsConfig

	config := consul_api.DefaultConfig()
	config.Address = u.Host
	config.Scheme = u.Scheme
	config.HttpClient.Timeout = opts.timeout
	config.HttpClient.Transport = transport

	client, err := consul_api.NewClient(config)
	if err != nil {
		return nil, err
	}

	// Init our exporter.
	return &Exporter{
		client:        client,
		kvPrefix:      kvPrefix,
		kvFilter:      regexp.MustCompile(kvFilter),
		healthSummary: healthSummary,
	}, nil
}

// Describe describes all the metrics ever exported by the Consul exporter. It
// implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- up
	ch <- clusterServers
	ch <- clusterLeader
	ch <- nodeCount
	ch <- serviceCount
	ch <- serviceNodesHealthy
	ch <- nodeChecks
	ch <- serviceChecks
	ch <- keyValues
}

// Collect fetches the stats from configured Consul location and delivers them
// as Prometheus metrics. It implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	// How many peers are in the Consul cluster?
	peers, err := e.client.Status().Peers()
	if err != nil {
		ch <- prometheus.MustNewConstMetric(
			up, prometheus.GaugeValue, 0,
		)
		log.Errorf("Can't query consul: %v", err)
		return
	}

	// We'll use peers to decide that we're up.
	ch <- prometheus.MustNewConstMetric(
		up, prometheus.GaugeValue, 1,
	)
	ch <- prometheus.MustNewConstMetric(
		clusterServers, prometheus.GaugeValue, float64(len(peers)),
	)

	leader, err := e.client.Status().Leader()
	if err != nil {
		log.Errorf("Can't query consul: %v", err)
	}
	if len(leader) == 0 {
		ch <- prometheus.MustNewConstMetric(
			clusterLeader, prometheus.GaugeValue, 0,
		)
	} else {
		ch <- prometheus.MustNewConstMetric(
			clusterLeader, prometheus.GaugeValue, 1,
		)
	}

	// How many nodes are registered?
	nodes, _, err := e.client.Catalog().Nodes(&consul_api.QueryOptions{})
	if err != nil {
		// FIXME: How should we handle a partial failure like this?
	} else {
		ch <- prometheus.MustNewConstMetric(
			nodeCount, prometheus.GaugeValue, float64(len(nodes)),
		)
	}

	// Query for the full list of services.
	serviceNames, _, err := e.client.Catalog().Services(&consul_api.QueryOptions{})
	if err != nil {
		// FIXME: How should we handle a partial failure like this?
		return
	}
	ch <- prometheus.MustNewConstMetric(
		serviceCount, prometheus.GaugeValue, float64(len(serviceNames)),
	)

	if e.healthSummary {
		e.collectHealthSummary(ch, serviceNames)
	}

	checks, _, err := e.client.Health().State("any", &consul_api.QueryOptions{})
	if err != nil {
		log.Errorf("Failed to query service health: %v", err)
		return
	}

	for _, hc := range checks {
		var passing, warning, critical, maintenance float64

		switch hc.Status {
		case consul.HealthPassing:
			passing = 1
		case consul.HealthWarning:
			warning = 1
		case consul.HealthCritical:
			critical = 1
		case consul.HealthMaint:
			maintenance = 1
		}

		if hc.ServiceID == "" {
			ch <- prometheus.MustNewConstMetric(
				nodeChecks, prometheus.GaugeValue, passing, hc.CheckID, hc.Node, consul.HealthPassing,
			)
			ch <- prometheus.MustNewConstMetric(
				nodeChecks, prometheus.GaugeValue, warning, hc.CheckID, hc.Node, consul.HealthWarning,
			)
			ch <- prometheus.MustNewConstMetric(
				nodeChecks, prometheus.GaugeValue, critical, hc.CheckID, hc.Node, consul.HealthCritical,
			)
			ch <- prometheus.MustNewConstMetric(
				nodeChecks, prometheus.GaugeValue, maintenance, hc.CheckID, hc.Node, consul.HealthMaint,
			)
		} else {
			ch <- prometheus.MustNewConstMetric(
				serviceChecks, prometheus.GaugeValue, passing, hc.CheckID, hc.Node, hc.ServiceID, hc.ServiceName, consul.HealthPassing,
			)
			ch <- prometheus.MustNewConstMetric(
				serviceChecks, prometheus.GaugeValue, warning, hc.CheckID, hc.Node, hc.ServiceID, hc.ServiceName, consul.HealthWarning,
			)
			ch <- prometheus.MustNewConstMetric(
				serviceChecks, prometheus.GaugeValue, critical, hc.CheckID, hc.Node, hc.ServiceID, hc.ServiceName, consul.HealthCritical,
			)
			ch <- prometheus.MustNewConstMetric(
				serviceChecks, prometheus.GaugeValue, maintenance, hc.CheckID, hc.Node, hc.ServiceID, hc.ServiceName, consul.HealthMaint,
			)
		}
	}

	e.collectKeyValues(ch)
}

// collectHealthSummary collects health information about every node+service
// combination. It will cause one lookup query per service.
func (e *Exporter) collectHealthSummary(ch chan<- prometheus.Metric, serviceNames map[string][]string) {
	for s := range serviceNames {
		service, _, err := e.client.Health().Service(s, "", false, &consul_api.QueryOptions{})
		if err != nil {
			log.Errorf("Failed to query service health: %v", err)
			continue
		}

		for _, entry := range service {
			// We have a Node, a Service, and one or more Checks. Our
			// service-node combo is passing if all checks have a `status`
			// of "passing."
			passing := 1.
			for _, hc := range entry.Checks {
				if hc.Status != consul.HealthPassing {
					passing = 0
					break
				}
			}
			ch <- prometheus.MustNewConstMetric(
				serviceNodesHealthy, prometheus.GaugeValue, passing, entry.Service.ID, entry.Node.Node, entry.Service.Service,
			)
		}
	}
}

func (e *Exporter) collectKeyValues(ch chan<- prometheus.Metric) {
	if e.kvPrefix == "" {
		return
	}

	kv := e.client.KV()
	pairs, _, err := kv.List(e.kvPrefix, &consul_api.QueryOptions{})
	if err != nil {
		log.Errorf("Error fetching key/values: %s", err)
		return
	}

	for _, pair := range pairs {
		if e.kvFilter.MatchString(pair.Key) {
			val, err := strconv.ParseFloat(string(pair.Value), 64)
			if err == nil {
				ch <- prometheus.MustNewConstMetric(
					keyValues, prometheus.GaugeValue, val, pair.Key,
				)
			}
		}
	}
}

func init() {
	prometheus.MustRegister(version.NewCollector("consul_exporter"))
}

func main() {
	var (
		listenAddress = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9107").String()
		metricsPath   = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
		healthSummary = kingpin.Flag("consul.health-summary", "Generate a health summary for each service instance. Needs n+1 queries to collect all information.").Default("true").Bool()
		kvPrefix      = kingpin.Flag("kv.prefix", "Prefix from which to expose key/value pairs.").Default("").String()
		kvFilter      = kingpin.Flag("kv.filter", "Regex that determines which keys to expose.").Default(".*").String()

		opts = consulOpts{}
	)
	kingpin.Flag("consul.server", "HTTP API address of a Consul server or agent. (prefix with https:// to connect over HTTPS)").Default("http://localhost:8500").StringVar(&opts.uri)
	kingpin.Flag("consul.ca-file", "File path to a PEM-encoded certificate authority used to validate the authenticity of a server certificate.").Default("").StringVar(&opts.caFile)
	kingpin.Flag("consul.cert-file", "File path to a PEM-encoded certificate used with the private key to verify the exporter's authenticity.").Default("").StringVar(&opts.certFile)
	kingpin.Flag("consul.key-file", "File path to a PEM-encoded private key used with the certificate to verify the exporter's authenticity.").Default("").StringVar(&opts.keyFile)
	kingpin.Flag("consul.server-name", "When provided, this overrides the hostname for the TLS certificate. It can be used to ensure that the certificate name matches the hostname we declare.").Default("").StringVar(&opts.serverName)
	kingpin.Flag("consul.timeout", "Timeout on HTTP requests to consul.").Default("200ms").DurationVar(&opts.timeout)

	log.AddFlags(kingpin.CommandLine)
	kingpin.Version(version.Print("consul_exporter"))
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()

	log.Infoln("Starting consul_exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	exporter, err := NewExporter(opts, *kvPrefix, *kvFilter, *healthSummary)
	if err != nil {
		log.Fatalln(err)
	}
	prometheus.MustRegister(exporter)

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
             <title>Consul Exporter</title>

Consul Exporter

<a href='` + *metricsPath + `'>Metrics

             `))
	})

	log.Infoln("Listening on", *listenAddress)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<commit_msg>Fetch health summaries in parallel<commit_after>package main

import (
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"gopkg.in/alecthomas/kingpin.v2"

	consul_api "github.com/hashicorp/consul/api"
	consul "github.com/hashicorp/consul/consul/structs"
	cleanhttp "github.com/hashicorp/go-cleanhttp"
)

const (
	namespace = "consul"
)

var (
	up = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "up"),
		"Was the last query of Consul successful.",
		nil, nil,
	)
	clusterServers = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "raft_peers"),
		"How many peers (servers) are in the Raft cluster.",
		nil, nil,
	)
	clusterLeader = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "raft_leader"),
		"Does Raft cluster have a leader (according to this node).",
		nil, nil,
	)
	nodeCount = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "serf_lan_members"),
		"How many members are in the cluster.",
		nil, nil,
	)
	serviceCount = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "catalog_services"),
		"How many services are in the cluster.",
		nil, nil,
	)
	serviceNodesHealthy = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "catalog_service_node_healthy"),
		"Is this service healthy on this node?",
		[]string{"service_id", "node", "service_name"}, nil,
	)
	nodeChecks = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "health_node_status"),
		"Status of health checks associated with a node.",
		[]string{"check", "node", "status"}, nil,
	)
	serviceChecks = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "health_service_status"),
		"Status of health checks associated with a service.",
		[]string{"check", "node", "service_id", "service_name", "status"}, nil,
	)
	keyValues = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", "catalog_kv"),
		"The values for selected keys in Consul's key/value catalog. Keys with non-numeric values are omitted.",
		[]string{"key"}, nil,
	)
)

// Exporter collects Consul stats from the given server and exports them using
// the prometheus metrics package.
type Exporter struct {
	client        *consul_api.Client
	kvPrefix      string
	kvFilter      *regexp.Regexp
	healthSummary bool
}

type consulOpts struct {
	uri        string
	caFile     string
	certFile   string
	keyFile    string
	serverName string
	timeout    time.Duration
}

// NewExporter returns an initialized Exporter.
func NewExporter(opts consulOpts, kvPrefix, kvFilter string, healthSummary bool) (*Exporter, error) {
	uri := opts.uri
	if !strings.Contains(uri, "://") {
		uri = "http://" + uri
	}
	u, err := url.Parse(uri)
	if err != nil {
		return nil, fmt.Errorf("invalid consul URL: %s", err)
	}
	if u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
		return nil, fmt.Errorf("invalid consul URL: %s", uri)
	}

	tlsConfig, err := consul_api.SetupTLSConfig(&consul_api.TLSConfig{
		Address:  opts.serverName,
		CAFile:   opts.caFile,
		CertFile: opts.certFile,
		KeyFile:  opts.keyFile,
	})
	if err != nil {
		return nil, err
	}
	transport := cleanhttp.DefaultPooledTransport()
	transport.TLSClientConfig = tlsConfig

	config := consul_api.DefaultConfig()
	config.Address = u.Host
	config.Scheme = u.Scheme
	config.HttpClient.Timeout = opts.timeout
	config.HttpClient.Transport = transport

	client, err := consul_api.NewClient(config)
	if err != nil {
		return nil, err
	}

	// Init our exporter.
	return &Exporter{
		client:        client,
		kvPrefix:      kvPrefix,
		kvFilter:      regexp.MustCompile(kvFilter),
		healthSummary: healthSummary,
	}, nil
}

// Describe describes all the metrics ever exported by the Consul exporter. It
// implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- up
	ch <- clusterServers
	ch <- clusterLeader
	ch <- nodeCount
	ch <- serviceCount
	ch <- serviceNodesHealthy
	ch <- nodeChecks
	ch <- serviceChecks
	ch <- keyValues
}

// Collect fetches the stats from configured Consul location and delivers them
// as Prometheus metrics. It implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	// How many peers are in the Consul cluster?
	peers, err := e.client.Status().Peers()
	if err != nil {
		ch <- prometheus.MustNewConstMetric(
			up, prometheus.GaugeValue, 0,
		)
		log.Errorf("Can't query consul: %v", err)
		return
	}

	// We'll use peers to decide that we're up.
	ch <- prometheus.MustNewConstMetric(
		up, prometheus.GaugeValue, 1,
	)
	ch <- prometheus.MustNewConstMetric(
		clusterServers, prometheus.GaugeValue, float64(len(peers)),
	)

	leader, err := e.client.Status().Leader()
	if err != nil {
		log.Errorf("Can't query consul: %v", err)
	}
	if len(leader) == 0 {
		ch <- prometheus.MustNewConstMetric(
			clusterLeader, prometheus.GaugeValue, 0,
		)
	} else {
		ch <- prometheus.MustNewConstMetric(
			clusterLeader, prometheus.GaugeValue, 1,
		)
	}

	// How many nodes are registered?
	nodes, _, err := e.client.Catalog().Nodes(&consul_api.QueryOptions{})
	if err != nil {
		// FIXME: How should we handle a partial failure like this?
	} else {
		ch <- prometheus.MustNewConstMetric(
			nodeCount, prometheus.GaugeValue, float64(len(nodes)),
		)
	}

	// Query for the full list of services.
	serviceNames, _, err := e.client.Catalog().Services(&consul_api.QueryOptions{})
	if err != nil {
		// FIXME: How should we handle a partial failure like this?
		return
	}
	ch <- prometheus.MustNewConstMetric(
		serviceCount, prometheus.GaugeValue, float64(len(serviceNames)),
	)

	if e.healthSummary {
		e.collectHealthSummary(ch, serviceNames)
	}

	checks, _, err := e.client.Health().State("any", &consul_api.QueryOptions{})
	if err != nil {
		log.Errorf("Failed to query service health: %v", err)
		return
	}

	for _, hc := range checks {
		var passing, warning, critical, maintenance float64

		switch hc.Status {
		case consul.HealthPassing:
			passing = 1
		case consul.HealthWarning:
			warning = 1
		case consul.HealthCritical:
			critical = 1
		case consul.HealthMaint:
			maintenance = 1
		}

		if hc.ServiceID == "" {
			ch <- prometheus.MustNewConstMetric(
				nodeChecks, prometheus.GaugeValue, passing, hc.CheckID, hc.Node, consul.HealthPassing,
			)
			ch <- prometheus.MustNewConstMetric(
				nodeChecks, prometheus.GaugeValue, warning, hc.CheckID, hc.Node, consul.HealthWarning,
			)
			ch <- prometheus.MustNewConstMetric(
				nodeChecks, prometheus.GaugeValue, critical, hc.CheckID, hc.Node, consul.HealthCritical,
			)
			ch <- prometheus.MustNewConstMetric(
				nodeChecks, prometheus.GaugeValue, maintenance, hc.CheckID, hc.Node, consul.HealthMaint,
			)
		} else {
			ch <- prometheus.MustNewConstMetric(
				serviceChecks, prometheus.GaugeValue, passing, hc.CheckID, hc.Node, hc.ServiceID, hc.ServiceName, consul.HealthPassing,
			)
			ch <- prometheus.MustNewConstMetric(
				serviceChecks, prometheus.GaugeValue, warning, hc.CheckID, hc.Node, hc.ServiceID, hc.ServiceName, consul.HealthWarning,
			)
			ch <- prometheus.MustNewConstMetric(
				serviceChecks, prometheus.GaugeValue, critical, hc.CheckID, hc.Node, hc.ServiceID, hc.ServiceName, consul.HealthCritical,
			)
			ch <- prometheus.MustNewConstMetric(
				serviceChecks, prometheus.GaugeValue, maintenance, hc.CheckID, hc.Node, hc.ServiceID, hc.ServiceName, consul.HealthMaint,
			)
		}
	}

	e.collectKeyValues(ch)
}

// collectHealthSummary collects health information about every node+service
// combination. It will cause one lookup query per service.
func (e *Exporter) collectHealthSummary(ch chan<- prometheus.Metric, serviceNames map[string][]string) {
	var wg sync.WaitGroup

	for s := range serviceNames {
		wg.Add(1)
		go func(s string) {
			defer wg.Done()
			e.collectOneHealthSummary(ch, s)
		}(s)
	}

	wg.Wait()
}

func (e *Exporter) collectOneHealthSummary(ch chan<- prometheus.Metric, serviceName string) error {
	log.Debugf("Fetching health summary for: %s", serviceName)

	service, _, err := e.client.Health().Service(serviceName, "", false, &consul_api.QueryOptions{})
	if err != nil {
		log.Errorf("Failed to query service health: %v", err)
		return err
	}

	for _, entry := range service {
		// We have a Node, a Service, and one or more Checks. Our
		// service-node combo is passing if all checks have a `status`
		// of "passing."
		passing := 1.
		for _, hc := range entry.Checks {
			if hc.Status != consul.HealthPassing {
				passing = 0
				break
			}
		}
		ch <- prometheus.MustNewConstMetric(
			serviceNodesHealthy, prometheus.GaugeValue, passing, entry.Service.ID, entry.Node.Node, entry.Service.Service,
		)
	}
	return nil
}

func (e *Exporter) collectKeyValues(ch chan<- prometheus.Metric) {
	if e.kvPrefix == "" {
		return
	}

	kv := e.client.KV()
	pairs, _, err := kv.List(e.kvPrefix, &consul_api.QueryOptions{})
	if err != nil {
		log.Errorf("Error fetching key/values: %s", err)
		return
	}

	for _, pair := range pairs {
		if e.kvFilter.MatchString(pair.Key) {
			val, err := strconv.ParseFloat(string(pair.Value), 64)
			if err == nil {
				ch <- prometheus.MustNewConstMetric(
					keyValues, prometheus.GaugeValue, val, pair.Key,
				)
			}
		}
	}
}

func init() {
	prometheus.MustRegister(version.NewCollector("consul_exporter"))
}

func main() {
	var (
		listenAddress = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9107").String()
		metricsPath   = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
		healthSummary = kingpin.Flag("consul.health-summary", "Generate a health summary for each service instance. Needs n+1 queries to collect all information.").Default("true").Bool()
		kvPrefix      = kingpin.Flag("kv.prefix", "Prefix from which to expose key/value pairs.").Default("").String()
		kvFilter      = kingpin.Flag("kv.filter", "Regex that determines which keys to expose.").Default(".*").String()

		opts = consulOpts{}
	)
	kingpin.Flag("consul.server", "HTTP API address of a Consul server or agent. (prefix with https:// to connect over HTTPS)").Default("http://localhost:8500").StringVar(&opts.uri)
	kingpin.Flag("consul.ca-file", "File path to a PEM-encoded certificate authority used to validate the authenticity of a server certificate.").Default("").StringVar(&opts.caFile)
	kingpin.Flag("consul.cert-file", "File path to a PEM-encoded certificate used with the private key to verify the exporter's authenticity.").Default("").StringVar(&opts.certFile)
	kingpin.Flag("consul.key-file", "File path to a PEM-encoded private key used with the certificate to verify the exporter's authenticity.").Default("").StringVar(&opts.keyFile)
	kingpin.Flag("consul.server-name", "When provided, this overrides the hostname for the TLS certificate. It can be used to ensure that the certificate name matches the hostname we declare.").Default("").StringVar(&opts.serverName)
	kingpin.Flag("consul.timeout", "Timeout on HTTP requests to consul.").Default("200ms").DurationVar(&opts.timeout)

	log.AddFlags(kingpin.CommandLine)
	kingpin.Version(version.Print("consul_exporter"))
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()

	log.Infoln("Starting consul_exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	exporter, err := NewExporter(opts, *kvPrefix, *kvFilter, *healthSummary)
	if err != nil {
		log.Fatalln(err)
	}
	prometheus.MustRegister(exporter)

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
             <title>Consul Exporter</title>

Consul Exporter

<a href='` + *metricsPath + `'>Metrics

             `))
	})

	log.Infoln("Listening on", *listenAddress)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package boomer provides commands to run load tests and display results.
package boomer

import (
	"crypto/tls"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/rakyll/pb"
)

// client is the http.Client that will be used to make all requests
// to the destination.
var client *http.Client

type result struct {
	err           error
	statusCode    int
	duration      time.Duration
	contentLength int64
}

type ReqOpts struct {
	Method   string
	URL      string
	Header   http.Header
	Body     string
	Username string
	Password string
	ReadAll  bool
}

// Creates a req object from req options
func (r *ReqOpts) Request() *http.Request {
	req, _ := http.NewRequest(r.Method, r.URL, strings.NewReader(r.Body))
	req.Header = r.Header
	if r.Username != "" && r.Password != "" {
		req.SetBasicAuth(r.Username, r.Password)
	}
	return req
}

type Boomer struct {
	// Req represents the options of the request to be made.
	// TODO(jbd): Make it work with an http.Request instead.
	Req *ReqOpts

	// N is the total number of requests to make.
	N int

	// C is the concurrency level, the number of concurrent workers to run.
	C int

	// Timeout in seconds.
	Timeout int

	// Qps is the rate limit.
	Qps int

	// AllowInsecure is an option to allow insecure TLS/SSL certificates.
	AllowInsecure bool

	// DisableCompression is an option to disable compression in response
	DisableCompression bool

	// DisableKeepAlives is an option to prevents re-use of TCP connections between different HTTP requests
	DisableKeepAlives bool

	// Output represents the output type. If "csv" is provided, the
	// output will be dumped as a csv stream.
	Output string

	// ProxyAddr is the address of HTTP proxy server in the format on "host:port".
	// Optional.
	ProxyAddr *url.URL

	bar     *pb.ProgressBar
	results chan *result
}

func (b *Boomer) startProgress() {
	if b.Output == "" {
		return
	}
	b.bar = pb.New(b.N)
	b.bar.Format("Bom !")
	b.bar.Start()
}

func (b *Boomer) finalizeProgress() {
	if b.Output == "" {
		return
	}
	b.bar.Finish()
}

func (b *Boomer) incProgress() {
	if b.Output == "" {
		return
	}
	b.bar.Increment()
}

// Run makes all the requests, prints the summary. It blocks until
// all work is done.
func (b *Boomer) Run() {
	b.results = make(chan *result, b.N)
	b.startProgress()

	start := time.Now()
	b.runWorkers()
	b.finalizeProgress()

	newReport(b.N, b.results, b.Output, time.Now().Sub(start)).finalize()
	close(b.results)
}

func (b *Boomer) runWorker(wg *sync.WaitGroup, ch chan *http.Request, readAll bool) {
	for req := range ch {
		s := time.Now()

		var code int
		var size int64

		resp, err := client.Do(req)
		if err == nil {
			size = resp.ContentLength
			code = resp.StatusCode
			if readAll {
				_, err = io.Copy(ioutil.Discard, resp.Body)
			}
			resp.Body.Close()
		}

		wg.Done()
		b.incProgress()
		b.results <- &result{
			statusCode:    code,
			duration:      time.Now().Sub(s),
			err:           err,
			contentLength: size,
		}
	}
}

func (b *Boomer) runWorkers() {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: b.AllowInsecure,
		},
		DisableCompression: b.DisableCompression,
		DisableKeepAlives:  b.DisableKeepAlives,
		// TODO(jbd): Add dial timeout.
		TLSHandshakeTimeout: time.Duration(b.Timeout) * time.Millisecond,
		Proxy:               http.ProxyURL(b.ProxyAddr),
	}
	client = &http.Client{Transport: tr}

	var wg sync.WaitGroup
	wg.Add(b.N)

	var throttle <-chan time.Time
	if b.Qps > 0 {
		throttle = time.Tick(time.Duration(1e6/(b.Qps)) * time.Microsecond)
	}
	jobs := make(chan *http.Request, b.N)
	for i := 0; i < b.C; i++ {
		go func() {
			b.runWorker(&wg, jobs, b.Req.ReadAll)
		}()
	}
	for i := 0; i < b.N; i++ {
		if b.Qps > 0 {
			<-throttle
		}
		jobs <- b.Req.Request()
	}
	close(jobs)
	wg.Wait()
}
<commit_msg>Don't report progress in csv mode.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package boomer provides commands to run load tests and display results.
package boomer

import (
	"crypto/tls"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/rakyll/pb"
)

// client is the http.Client that will be used to make all requests
// to the destination.
var client *http.Client

type result struct {
	err           error
	statusCode    int
	duration      time.Duration
	contentLength int64
}

type ReqOpts struct {
	Method   string
	URL      string
	Header   http.Header
	Body     string
	Username string
	Password string
	ReadAll  bool
}

// Creates a req object from req options
func (r *ReqOpts) Request() *http.Request {
	req, _ := http.NewRequest(r.Method, r.URL, strings.NewReader(r.Body))
	req.Header = r.Header
	if r.Username != "" && r.Password != "" {
		req.SetBasicAuth(r.Username, r.Password)
	}
	return req
}

type Boomer struct {
	// Req represents the options of the request to be made.
	// TODO(jbd): Make it work with an http.Request instead.
	Req *ReqOpts

	// N is the total number of requests to make.
	N int

	// C is the concurrency level, the number of concurrent workers to run.
	C int

	// Timeout in seconds.
	Timeout int

	// Qps is the rate limit.
	Qps int

	// AllowInsecure is an option to allow insecure TLS/SSL certificates.
	AllowInsecure bool

	// DisableCompression is an option to disable compression in response
	DisableCompression bool

	// DisableKeepAlives is an option to prevents re-use of TCP connections between different HTTP requests
	DisableKeepAlives bool

	// Output represents the output type. If "csv" is provided, the
	// output will be dumped as a csv stream.
	Output string

	// ProxyAddr is the address of HTTP proxy server in the format on "host:port".
	// Optional.
	ProxyAddr *url.URL

	bar     *pb.ProgressBar
	results chan *result
}

func (b *Boomer) startProgress() {
	if b.Output != "" {
		return
	}
	b.bar = pb.New(b.N)
	b.bar.Format("Bom !")
	b.bar.Start()
}

func (b *Boomer) finalizeProgress() {
	if b.Output != "" {
		return
	}
	b.bar.Finish()
}

func (b *Boomer) incProgress() {
	if b.Output != "" {
		return
	}
	b.bar.Increment()
}

// Run makes all the requests, prints the summary. It blocks until
// all work is done.
func (b *Boomer) Run() {
	b.results = make(chan *result, b.N)
	b.startProgress()

	start := time.Now()
	b.runWorkers()
	b.finalizeProgress()

	newReport(b.N, b.results, b.Output, time.Now().Sub(start)).finalize()
	close(b.results)
}

func (b *Boomer) runWorker(wg *sync.WaitGroup, ch chan *http.Request, readAll bool) {
	for req := range ch {
		s := time.Now()

		var code int
		var size int64

		resp, err := client.Do(req)
		if err == nil {
			size = resp.ContentLength
			code = resp.StatusCode
			if readAll {
				_, err = io.Copy(ioutil.Discard, resp.Body)
			}
			resp.Body.Close()
		}

		wg.Done()
		b.incProgress()
		b.results <- &result{
			statusCode:    code,
			duration:      time.Now().Sub(s),
			err:           err,
			contentLength: size,
		}
	}
}

func (b *Boomer) runWorkers() {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: b.AllowInsecure,
		},
		DisableCompression: b.DisableCompression,
		DisableKeepAlives:  b.DisableKeepAlives,
		// TODO(jbd): Add dial timeout.
		TLSHandshakeTimeout: time.Duration(b.Timeout) * time.Millisecond,
		Proxy:               http.ProxyURL(b.ProxyAddr),
	}
	client = &http.Client{Transport: tr}

	var wg sync.WaitGroup
	wg.Add(b.N)

	var throttle <-chan time.Time
	if b.Qps > 0 {
		throttle = time.Tick(time.Duration(1e6/(b.Qps)) * time.Microsecond)
	}
	jobs := make(chan *http.Request, b.N)
	for i := 0; i < b.C; i++ {
		go func() {
			b.runWorker(&wg, jobs, b.Req.ReadAll)
		}()
	}
	for i := 0; i < b.N; i++ {
		if b.Qps > 0 {
			<-throttle
		}
		jobs <- b.Req.Request()
	}
	close(jobs)
	wg.Wait()
}
<|endoftext|>


<commit_before>package admin

import (
	"time"

	"github.com/jinzhu/now"
	"github.com/qor/qor-example/app/models"
	"github.com/qor/qor-example/db"
)

func initFuncMap() {
	Admin.RegisterFuncMap("latest_orders", latestOrders)
	Admin.RegisterFuncMap("last_week_orders", lastWeekOrderData)
	Admin.RegisterFuncMap("last_week_users", lastWeekUserData)
}

func latestOrders() (orders []models.Order) {
	Admin.Config.DB.Limit(5).Find(&orders)
	return
}

type ChartData struct {
	Total string
	Date  time.Time
}

/*
date format 2015-01-23
*/
func GetChartData(table, start, end string) (res []ChartData) {
	startdate, err := time.Parse("2006-01-02", start)
	if err != nil {
		return
	}
	enddate, err := time.Parse("2006-01-02", end)
	if err != nil || enddate.UnixNano() < startdate.UnixNano() {
		enddate = now.EndOfDay()
	} else {
		enddate.AddDate(0, 0, 1)
	}
	db.DB.Table(table).Where("created_at > ? AND created_at < ?", startdate, enddate).Select("date(created_at) as date, count(1) as total").Group("date(created_at)").Scan(&res)
	return
}

func lastWeekOrderData() (res []ChartData) {
	res = GetChartData("orders", time.Now().AddDate(0, 0, -6).Format("2006-01-02"), time.Now().Format("2006-01-02"))
	return
}

func lastWeekUserData() (res []ChartData) {
	res = GetChartData("users", time.Now().AddDate(0, 0, -6).Format("2006-01-02"), time.Now().Format("2006-01-02"))
	return
}
<commit_msg>Show latest orders first in dashboard<commit_after>package admin

import (
	"time"

	"github.com/jinzhu/now"
	"github.com/qor/qor-example/app/models"
	"github.com/qor/qor-example/db"
)

func initFuncMap() {
	Admin.RegisterFuncMap("latest_orders", latestOrders)
	Admin.RegisterFuncMap("last_week_orders", lastWeekOrderData)
	Admin.RegisterFuncMap("last_week_users", lastWeekUserData)
}

func latestOrders() (orders []models.Order) {
	Admin.Config.DB.Order("id desc").Limit(5).Find(&orders)
	return
}

type ChartData struct {
	Total string
	Date  time.Time
}

/*
date format 2015-01-23
*/
func GetChartData(table, start, end string) (res []ChartData) {
	startdate, err := time.Parse("2006-01-02", start)
	if err != nil {
		return
	}
	enddate, err := time.Parse("2006-01-02", end)
	if err != nil || enddate.UnixNano() < startdate.UnixNano() {
		enddate = now.EndOfDay()
	} else {
		enddate.AddDate(0, 0, 1)
	}
	db.DB.Table(table).Where("created_at > ? AND created_at < ?", startdate, enddate).Select("date(created_at) as date, count(1) as total").Group("date(created_at)").Scan(&res)
	return
}

func lastWeekOrderData() (res []ChartData) {
	res = GetChartData("orders", time.Now().AddDate(0, 0, -6).Format("2006-01-02"), time.Now().Format("2006-01-02"))
	return
}

func lastWeekUserData() (res []ChartData) {
	res = GetChartData("users", time.Now().AddDate(0, 0, -6).Format("2006-01-02"), time.Now().Format("2006-01-02"))
	return
}
<|endoftext|>


<commit_before>package controller

import (
	"encoding/json"
	"html/template"
	"log"
	"net/http"
	"strings"

	"github.com/longkai/xiaolongtongxue.com/context"
	"github.com/longkai/xiaolongtongxue.com/github"
	"github.com/longkai/xiaolongtongxue.com/medium"
	"github.com/longkai/xiaolongtongxue.com/repo"
)

const (
	pageSize = 7
)

var (
	conf       context.Conf
	repository repo.Repo
	templs     *template.Template
)

// Ctrl main controller.
func Ctrl(_conf context.Conf) {
	conf = _conf

	repository = repo.NewRepo(conf.RepoDir, conf.SkipDirs, conf.GlobDocs,
		conf.Github.User, conf.Github.Repo, medium.NewMedium(conf))

	github.Init("/api/github/hook", conf.RepoDir, conf.Github.HookSecret,
		conf.Github.AccessToken, func(a, m, d []string) {
			repository.Batch(a, m, d)
		})

	templs = template.Must(template.New("templ").Funcs(map[string]interface{}{
		"bg": func(bg string) string {
			lo := strings.ToLower(bg)
			if strings.HasPrefix(lo, "//") || strings.HasPrefix(lo, "http://") || strings.HasPrefix(lo, "https://") {
				return bg
			}
			// TODO: hard code, local resource use cdn
			return "//cdn.jsdelivr.net/gh/longkai/essays" + bg
		},
	}).ParseGlob("templ/*"))

	// Global handler.
	http.HandleFunc("/", handle)
}

func handle(w http.ResponseWriter, r *http.Request) {
	switch r.URL.Path {
	case "/": // Home page.
		home(w, r)
	case "/list": // Pagination.
		list(w, r)
	default: // Otherwise it should be an entry or its static resources request.
		entry(w, r)
	}
}

func home(w http.ResponseWriter, r *http.Request) {
	v := repository.List("/", pageSize)
	data := &struct {
		List repo.Docs
		Meta interface{}
	}{v, conf.Meta}

	if err := templs.ExecuteTemplate(w, "index.html", data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func list(w http.ResponseWriter, r *http.Request) {
	key := r.URL.Query().Get("since")
	// If the key is not found, return the latest ones.
	v := repository.List(key, pageSize)
	b, err := json.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Write(b)
}

// entry bundles entry and its static resources into a same handler,
// since we use the same directory layout, for doc and its static files(images).
// If entry is not found in repository, fallback to its static resources.
func entry(w http.ResponseWriter, r *http.Request) {
	// Compatible with old URL scheme, i.e., `/a/b/title/` to `/a/b/title`.
	p := r.URL.Path
	if strings.HasSuffix(p, "/") {
		http.Redirect(w, r, p[:len(p)-1], http.StatusMovedPermanently)
		return
	}
	// User-defined redirection mapping.
	if predir, ok := conf.Redirects[p]; ok {
		log.Printf("redirect %q to %q", p, predir)
		http.Redirect(w, r, predir, http.StatusMovedPermanently)
		return
	}
	// Try document first.
	doc, err := repository.Get(p)
	switch e := err.(type) {
	case nil:
		data := &struct {
			A    repo.Doc
			Meta interface{}
		}{doc, conf.Meta}
		// w.Header().Add("Cache-Control", "max-age=7200, public")
		if err = templs.ExecuteTemplate(w, "entry.html", data); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	case repo.NotFoundError:
		http.NotFound(w, r)
	default: // general error
		http.Error(w, e.Error(), http.StatusInternalServerError)
	}
}
<commit_msg>drop medium support<commit_after>package controller

import (
	"encoding/json"
	"html/template"
	"log"
	"net/http"
	"strings"

	"github.com/longkai/xiaolongtongxue.com/context"
	"github.com/longkai/xiaolongtongxue.com/github"
	"github.com/longkai/xiaolongtongxue.com/repo"
)

const (
	pageSize = 7
)

var (
	conf       context.Conf
	repository repo.Repo
	templs     *template.Template
)

// Ctrl main controller.
func Ctrl(_conf context.Conf) {
	conf = _conf

	repository = repo.NewRepo(conf.RepoDir, conf.SkipDirs, conf.GlobDocs,
		conf.Github.User, conf.Github.Repo)

	github.Init("/api/github/hook", conf.RepoDir, conf.Github.HookSecret,
		conf.Github.AccessToken, func(a, m, d []string) {
			repository.Batch(a, m, d)
		})

	templs = template.Must(template.New("templ").Funcs(map[string]interface{}{
		"bg": func(bg string) string {
			lo := strings.ToLower(bg)
			if strings.HasPrefix(lo, "//") || strings.HasPrefix(lo, "http://") || strings.HasPrefix(lo, "https://") {
				return bg
			}
			// TODO: hard code, local resource use cdn
			return "//cdn.jsdelivr.net/gh/longkai/essays" + bg
		},
	}).ParseGlob("templ/*"))

	// Global handler.
	http.HandleFunc("/", handle)
}

func handle(w http.ResponseWriter, r *http.Request) {
	switch r.URL.Path {
	case "/": // Home page.
		home(w, r)
	case "/list": // Pagination.
		list(w, r)
	default: // Otherwise it should be an entry or its static resources request.
		entry(w, r)
	}
}

func home(w http.ResponseWriter, r *http.Request) {
	v := repository.List("/", pageSize)
	data := &struct {
		List repo.Docs
		Meta interface{}
	}{v, conf.Meta}

	if err := templs.ExecuteTemplate(w, "index.html", data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func list(w http.ResponseWriter, r *http.Request) {
	key := r.URL.Query().Get("since")
	// If the key is not found, return the latest ones.
	v := repository.List(key, pageSize)
	b, err := json.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Write(b)
}

// entry bundles entry and its static resources into a same handler,
// since we use the same directory layout, for doc and its static files(images).
// If entry is not found in repository, fallback to its static resources.
func entry(w http.ResponseWriter, r *http.Request) {
	// Compatible with old URL scheme, i.e., `/a/b/title/` to `/a/b/title`.
	p := r.URL.Path
	if strings.HasSuffix(p, "/") {
		http.Redirect(w, r, p[:len(p)-1], http.StatusMovedPermanently)
		return
	}
	// User-defined redirection mapping.
	if predir, ok := conf.Redirects[p]; ok {
		log.Printf("redirect %q to %q", p, predir)
		http.Redirect(w, r, predir, http.StatusMovedPermanently)
		return
	}
	// Try document first.
	doc, err := repository.Get(p)
	switch e := err.(type) {
	case nil:
		data := &struct {
			A    repo.Doc
			Meta interface{}
		}{doc, conf.Meta}
		// w.Header().Add("Cache-Control", "max-age=7200, public")
		if err = templs.ExecuteTemplate(w, "entry.html", data); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	case repo.NotFoundError:
		http.NotFound(w, r)
	default: // general error
		http.Error(w, e.Error(), http.StatusInternalServerError)
	}
}
<|endoftext|>


<commit_before>package generatorControllers

import (
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"eaciit/wfdemo-git/web/helper"
	"os"
	"time"

	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	"github.com/eaciit/toolkit"
)

type GenDataPeriod struct {
	*BaseController
}

func (d *GenDataPeriod) Generate(base *BaseController) {
	d.BaseController = base
	conn, e := PrepareConnection()
	if e != nil {
		toolkit.Println("Scada Summary : " + e.Error())
		os.Exit(0)
	}

	// reset data
	// #faisal
	// remove this function
	d.BaseController.Ctx.DeleteMany(NewLatestDataPeriod(), dbox.Ne("projectname", ""))

	projects, _ := helper.GetProjectList()

	for _, proj := range projects {
		projectName := proj.Value
		scadaResults := make([]time.Time, 2)
		dgrResults := make([]time.Time, 2)
		alarmResults := make([]time.Time, 2)
		jmrResults := make([]time.Time, 2)
		metResults := make([]time.Time, 2)
		durationResults := make([]time.Time, 2)
		scadaAnomalyresults := make([]time.Time, 2)
		alarmOverlappingresults := make([]time.Time, 2)
		alarmScadaAnomalyresults := make([]time.Time, 2)
		scadaHFDResult := make([]time.Time, 2)

		scadaResults[0], scadaResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		dgrResults[0], dgrResults[1], e = getDataDateAvailable(conn, new(DGRModel).TableName(), "dateinfo.dateid", dbox.Eq("site", projectName))
		alarmResults[0], alarmResults[1], e = getDataDateAvailable(conn, new(Alarm).TableName(), "startdate", dbox.Eq("farm", projectName))
		alarmOverlappingresults[0], alarmOverlappingresults[1], e = getDataDateAvailable(conn, new(AlarmOverlapping).TableName(), "startdate", dbox.Eq("farm", projectName))
		alarmScadaAnomalyresults[0], alarmScadaAnomalyresults[1], e = getDataDateAvailable(conn, new(AlarmScadaAnomaly).TableName(), "startdate", dbox.Eq("farm", projectName))
		jmrResults[0], jmrResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "dateinfo.dateid", dbox.Eq("projectname", projectName))
		metResults[0], metResults[1], e = getDataDateAvailable(conn, new(MetTower).TableName(), "timestamp", dbox.Eq("project", projectName))
		durationResults[0], durationResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", false), dbox.Eq("projectname", projectName)))
		scadaAnomalyresults[0], scadaAnomalyresults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", true), dbox.Eq("projectname", projectName)))
		scadaHFDResult[0], scadaHFDResult[1], e = getDataDateAvailable(conn, new(ScadaDataHFD).TableName(), "timestamp", dbox.Eq("projectname", projectName))

		availdatedata := struct {
			ScadaData         []time.Time
			DGRData           []time.Time
			Alarm             []time.Time
			JMR               []time.Time
			MET               []time.Time
			Duration          []time.Time
			ScadaAnomaly      []time.Time
			AlarmOverlapping  []time.Time
			AlarmScadaAnomaly []time.Time
			ScadaDataHFD      []time.Time
		}{
			ScadaData:         scadaResults,
			DGRData:           dgrResults,
			Alarm:             alarmResults,
			JMR:               jmrResults,
			MET:               metResults,
			Duration:          durationResults,
			ScadaAnomaly:      scadaAnomalyresults,
			AlarmOverlapping:  alarmOverlappingresults,
			AlarmScadaAnomaly: alarmScadaAnomalyresults,
			ScadaDataHFD:      scadaHFDResult,
		}

		mdl := NewLatestDataPeriod()
		mdl.Type = "ScadaData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaData

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "DGRData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.DGRData

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "Alarm"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Alarm

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "JMR"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.JMR

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "MET"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.MET

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "Duration"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Duration

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "ScadaAnomaly"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaAnomaly

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "AlarmOverlapping"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.AlarmOverlapping

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "AlarmScadaAnomaly"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.AlarmScadaAnomaly

		mdl = NewLatestDataPeriod()
		mdl.Type = "ScadaDataHFD"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaDataHFD

		d.BaseController.Ctx.Insert(mdl)
	}

}

func getDataDateAvailable(conn dbox.IConnection, collectionName string, timestampColumn string, where *dbox.Filter) (min time.Time, max time.Time, err error) {
	q := conn.
		NewQuery().
		From(collectionName)

	if where != nil {
		q.Where(where)
	}

	csr, err := q.
		Aggr(dbox.AggrMin, "$"+timestampColumn, "min").
		Aggr(dbox.AggrMax, "$"+timestampColumn, "max").
		Group("enable").
		Cursor(nil)

	defer csr.Close()

	if err != nil {
		csr.Close()
		return
	}

	data := []toolkit.M{}
	err = csr.Fetch(&data, 0, false)

	if err != nil || len(data) == 0 {
		csr.Close()
		return
	}

	min = data[0].Get("min").(time.Time)
	max = data[0].Get("max").(time.Time)

	csr.Close()
	return
}
<commit_msg>adding data available date for warning on summary generator<commit_after>package generatorControllers

import (
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"eaciit/wfdemo-git/web/helper"
	"os"
	"time"

	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	"github.com/eaciit/toolkit"
)

type GenDataPeriod struct {
	*BaseController
}

func (d *GenDataPeriod) Generate(base *BaseController) {
	d.BaseController = base
	conn, e := PrepareConnection()
	if e != nil {
		toolkit.Println("Scada Summary : " + e.Error())
		os.Exit(0)
	}

	// reset data
	// #faisal
	// remove this function
	d.BaseController.Ctx.DeleteMany(NewLatestDataPeriod(), dbox.Ne("projectname", ""))

	projects, _ := helper.GetProjectList()

	for _, proj := range projects {
		projectName := proj.Value
		scadaResults := make([]time.Time, 2)
		dgrResults := make([]time.Time, 2)
		alarmResults := make([]time.Time, 2)
		jmrResults := make([]time.Time, 2)
		metResults := make([]time.Time, 2)
		durationResults := make([]time.Time, 2)
		scadaAnomalyresults := make([]time.Time, 2)
		alarmOverlappingresults := make([]time.Time, 2)
		alarmScadaAnomalyresults := make([]time.Time, 2)
		scadaHFDResult := make([]time.Time, 2)
		warningResult := make([]time.Time, 2)

		scadaResults[0], scadaResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		dgrResults[0], dgrResults[1], e = getDataDateAvailable(conn, new(DGRModel).TableName(), "dateinfo.dateid", dbox.Eq("site", projectName))
		alarmResults[0], alarmResults[1], e = getDataDateAvailable(conn, new(Alarm).TableName(), "startdate", dbox.Eq("farm", projectName))
		alarmOverlappingresults[0], alarmOverlappingresults[1], e = getDataDateAvailable(conn, new(AlarmOverlapping).TableName(), "startdate", dbox.Eq("farm", projectName))
		alarmScadaAnomalyresults[0], alarmScadaAnomalyresults[1], e = getDataDateAvailable(conn, new(AlarmScadaAnomaly).TableName(), "startdate", dbox.Eq("farm", projectName))
		jmrResults[0], jmrResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "dateinfo.dateid", dbox.Eq("projectname", projectName))
		metResults[0], metResults[1], e = getDataDateAvailable(conn, new(MetTower).TableName(), "timestamp", dbox.Eq("project", projectName))
		durationResults[0], durationResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", false), dbox.Eq("projectname", projectName)))
		scadaAnomalyresults[0], scadaAnomalyresults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", true), dbox.Eq("projectname", projectName)))
		scadaHFDResult[0], scadaHFDResult[1], e = getDataDateAvailable(conn, new(ScadaDataHFD).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		warningResult[0], warningResult[1], e = getDataDateAvailable(conn, new(ScadaDataHFD).TableName(), "timeend", dbox.Eq("projectname", projectName))

		availdatedata := struct {
			ScadaData         []time.Time
			DGRData           []time.Time
			Alarm             []time.Time
			JMR               []time.Time
			MET               []time.Time
			Duration          []time.Time
			ScadaAnomaly      []time.Time
			AlarmOverlapping  []time.Time
			AlarmScadaAnomaly []time.Time
			ScadaDataHFD      []time.Time
			Warning           []time.Time
		}{
			ScadaData:         scadaResults,
			DGRData:           dgrResults,
			Alarm:             alarmResults,
			JMR:               jmrResults,
			MET:               metResults,
			Duration:          durationResults,
			ScadaAnomaly:      scadaAnomalyresults,
			AlarmOverlapping:  alarmOverlappingresults,
			AlarmScadaAnomaly: alarmScadaAnomalyresults,
			ScadaDataHFD:      scadaHFDResult,
			Warning:           warningResult,
		}

		mdl := NewLatestDataPeriod()
		mdl.Type = "ScadaData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaData

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "DGRData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.DGRData

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "Alarm"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Alarm

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "JMR"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.JMR

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "MET"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.MET

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "Duration"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Duration

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "ScadaAnomaly"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaAnomaly

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "AlarmOverlapping"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.AlarmOverlapping

		d.BaseController.Ctx.Insert(mdl)

		mdl = NewLatestDataPeriod()
		mdl.Type = "AlarmScadaAnomaly"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.AlarmScadaAnomaly

		mdl = NewLatestDataPeriod()
		mdl.Type = "ScadaDataHFD"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaDataHFD

		mdl = NewLatestDataPeriod()
		mdl.Type = "Warning"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Warning

		d.BaseController.Ctx.Insert(mdl)
	}

}

func getDataDateAvailable(conn dbox.IConnection, collectionName string, timestampColumn string, where *dbox.Filter) (min time.Time, max time.Time, err error) {
	q := conn.
		NewQuery().
		From(collectionName)

	if where != nil {
		q.Where(where)
	}

	csr, err := q.
		Aggr(dbox.AggrMin, "$"+timestampColumn, "min").
		Aggr(dbox.AggrMax, "$"+timestampColumn, "max").
		Group("enable").
		Cursor(nil)

	defer csr.Close()

	if err != nil {
		csr.Close()
		return
	}

	data := []toolkit.M{}
	err = csr.Fetch(&data, 0, false)

	if err != nil || len(data) == 0 {
		csr.Close()
		return
	}

	min = data[0].Get("min").(time.Time)
	max = data[0].Get("max").(time.Time)

	csr.Close()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/lonelycode/tykcommon"
	"net/http"
)

// VersionCheck will check whether the version of the requested API the request is accessing has any restrictions on URL endpoints
type VersionCheck struct {
	TykMiddleware
}

// New lets you do any initialisations for the object can be done here
func (v *VersionCheck) New() {}

// GetConfig retrieves the configuration from the API config
func (v *VersionCheck) GetConfig() (interface{}, error) {
	return nil, nil
}

// ProcessRequest will run any checks on the request on the way through the system, return an error to have the chain fail
func (v *VersionCheck) ProcessRequest(w http.ResponseWriter, r *http.Request, configuration interface{}) (error, int) {
	// Check versioning, blacklist, whitelist and ignored status
	requestValid, stat, meta := v.TykMiddleware.Spec.IsRequestValid(r)
	if requestValid == false {
		// Fire a versioning failure event
		go v.TykMiddleware.FireEvent(EVENT_VersionFailure,
			EVENT_VersionFailureMeta{
				EventMetaDefault: EventMetaDefault{Message: "Attempted access to disallowed version / path."},
				Path:             r.URL.Path,
				Origin:           r.RemoteAddr,
				Key:              "",
				Reason:           string(stat),
			})
		return errors.New(string(stat)), 409
	}

	if stat == StatusOkAndIgnore {
		handler := SuccessHandler{v.TykMiddleware}
		// Skip all other execution
		handler.ServeHTTP(w, r)
		return nil, 666
	}

	if stat == StatusRedirectFlowByReply {
		// Reply with some alternate data
		thisMeta := meta.(tykcommon.EndpointMethodMeta)
		responseMessage := []byte(thisMeta.Data)
		for header, value := range thisMeta.Headers {
			w.Header().Add(header, value)
		}

		w.WriteHeader(thisMeta.Code)
		fmt.Fprintf(w, string(responseMessage))
		return nil, 666
	}

	return nil, 200
}
<commit_msg>Minor refactoring<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/lonelycode/tykcommon"
	"net/http"
)

// VersionCheck will check whether the version of the requested API the request is accessing has any restrictions on URL endpoints
type VersionCheck struct {
	TykMiddleware
}

// New lets you do any initialisations for the object can be done here
func (v *VersionCheck) New() {}

// GetConfig retrieves the configuration from the API config
func (v *VersionCheck) GetConfig() (interface{}, error) {
	return nil, nil
}

func (v *VersionCheck) DoMockReply(w http.ResponseWriter, meta interface{}) () {
    // Reply with some alternate data
    thisMeta := meta.(tykcommon.EndpointMethodMeta)
    responseMessage := []byte(thisMeta.Data)
    for header, value := range thisMeta.Headers {
        w.Header().Add(header, value)
    }

    w.WriteHeader(thisMeta.Code)
    fmt.Fprintf(w, string(responseMessage))
    return
}

// ProcessRequest will run any checks on the request on the way through the system, return an error to have the chain fail
func (v *VersionCheck) ProcessRequest(w http.ResponseWriter, r *http.Request, configuration interface{}) (error, int) {
	// Check versioning, blacklist, whitelist and ignored status
	requestValid, stat, meta := v.TykMiddleware.Spec.IsRequestValid(r)
	if requestValid == false {
		// Fire a versioning failure event
		go v.TykMiddleware.FireEvent(EVENT_VersionFailure,
			EVENT_VersionFailureMeta{
				EventMetaDefault: EventMetaDefault{Message: "Attempted access to disallowed version / path."},
				Path:             r.URL.Path,
				Origin:           r.RemoteAddr,
				Key:              "",
				Reason:           string(stat),
			})
		return errors.New(string(stat)), 409
	}

    // We handle redirects before ignores in case we aren't using a whitelist
    if stat == StatusRedirectFlowByReply {
		v.DoMockReply(w, meta)
		return nil, 666
	}

	if stat == StatusOkAndIgnore {
		handler := SuccessHandler{v.TykMiddleware}
		// Skip all other execution
		handler.ServeHTTP(w, r)
		return nil, 666
	}

	return nil, 200
}
<|endoftext|>


<commit_before>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package webhook

import (
	"encoding/json"
	"fmt"
	"strings"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/markup"
	api "code.gitea.io/gitea/modules/structs"
)

type (
	// TelegramPayload represents
	TelegramPayload struct {
		Message           string `json:"text"`
		ParseMode         string `json:"parse_mode"`
		DisableWebPreview bool   `json:"disable_web_page_preview"`
	}

	// TelegramMeta contains the telegram metadata
	TelegramMeta struct {
		BotToken string `json:"bot_token"`
		ChatID   string `json:"chat_id"`
	}
)

// GetTelegramHook returns telegram metadata
func GetTelegramHook(w *models.Webhook) *TelegramMeta {
	s := &TelegramMeta{}
	if err := json.Unmarshal([]byte(w.Meta), s); err != nil {
		log.Error("webhook.GetTelegramHook(%d): %v", w.ID, err)
	}
	return s
}

// SetSecret sets the telegram secret
func (p *TelegramPayload) SetSecret(_ string) {}

// JSONPayload Marshals the TelegramPayload to json
func (p *TelegramPayload) JSONPayload() ([]byte, error) {
	p.ParseMode = "HTML"
	p.DisableWebPreview = true
	p.Message = markup.Sanitize(p.Message)
	data, err := json.MarshalIndent(p, "", "  ")
	if err != nil {
		return []byte{}, err
	}
	return data, nil
}

func getTelegramCreatePayload(p *api.CreatePayload) (*TelegramPayload, error) {
	// created tag/branch
	refName := git.RefEndName(p.Ref)
	title := fmt.Sprintf(`[<a href="%s">%s</a>] %s <a href="%s">%s</a> created`, p.Repo.HTMLURL, p.Repo.FullName, p.RefType,
		p.Repo.HTMLURL+"/src/"+refName, refName)

	return &TelegramPayload{
		Message: title,
	}, nil
}

func getTelegramDeletePayload(p *api.DeletePayload) (*TelegramPayload, error) {
	// created tag/branch
	refName := git.RefEndName(p.Ref)
	title := fmt.Sprintf(`[<a href="%s">%s</a>] %s <a href="%s">%s</a> deleted`, p.Repo.HTMLURL, p.Repo.FullName, p.RefType,
		p.Repo.HTMLURL+"/src/"+refName, refName)

	return &TelegramPayload{
		Message: title,
	}, nil
}

func getTelegramForkPayload(p *api.ForkPayload) (*TelegramPayload, error) {
	title := fmt.Sprintf(`%s is forked to <a href="%s">%s</a>`, p.Forkee.FullName, p.Repo.HTMLURL, p.Repo.FullName)

	return &TelegramPayload{
		Message: title,
	}, nil
}

func getTelegramPushPayload(p *api.PushPayload) (*TelegramPayload, error) {
	var (
		branchName = git.RefEndName(p.Ref)
		commitDesc string
	)

	var titleLink string
	if len(p.Commits) == 1 {
		commitDesc = "1 new commit"
		titleLink = p.Commits[0].URL
	} else {
		commitDesc = fmt.Sprintf("%d new commits", len(p.Commits))
		titleLink = p.CompareURL
	}
	if titleLink == "" {
		titleLink = p.Repo.HTMLURL + "/src/" + branchName
	}
	title := fmt.Sprintf(`[<a href="%s">%s</a>:<a href="%s">%s</a>] %s`, p.Repo.HTMLURL, p.Repo.FullName, titleLink, branchName, commitDesc)

	var text string
	// for each commit, generate attachment text
	for i, commit := range p.Commits {
		var authorName string
		if commit.Author != nil {
			authorName = " - " + commit.Author.Name
		}
		text += fmt.Sprintf(`[<a href="%s">%s</a>] %s`, commit.URL, commit.ID[:7],
			strings.TrimRight(commit.Message, "\r\n")) + authorName
		// add linebreak to each commit but the last
		if i < len(p.Commits)-1 {
			text += "\n"
		}
	}

	return &TelegramPayload{
		Message: title + "\n" + text,
	}, nil
}

func getTelegramIssuesPayload(p *api.IssuePayload) (*TelegramPayload, error) {
	text, _, attachmentText, _ := getIssuesPayloadInfo(p, htmlLinkFormatter, true)

	return &TelegramPayload{
		Message: text + "\n\n" + attachmentText,
	}, nil
}

func getTelegramIssueCommentPayload(p *api.IssueCommentPayload) (*TelegramPayload, error) {
	text, _, _ := getIssueCommentPayloadInfo(p, htmlLinkFormatter, true)

	return &TelegramPayload{
		Message: text + "\n" + p.Comment.Body,
	}, nil
}

func getTelegramPullRequestPayload(p *api.PullRequestPayload) (*TelegramPayload, error) {
	text, _, attachmentText, _ := getPullRequestPayloadInfo(p, htmlLinkFormatter, true)

	return &TelegramPayload{
		Message: text + "\n" + attachmentText,
	}, nil
}

func getTelegramRepositoryPayload(p *api.RepositoryPayload) (*TelegramPayload, error) {
	var title string
	switch p.Action {
	case api.HookRepoCreated:
		title = fmt.Sprintf(`[<a href="%s">%s</a>] Repository created`, p.Repository.HTMLURL, p.Repository.FullName)
		return &TelegramPayload{
			Message: title,
		}, nil
	case api.HookRepoDeleted:
		title = fmt.Sprintf("[%s] Repository deleted", p.Repository.FullName)
		return &TelegramPayload{
			Message: title,
		}, nil
	}
	return nil, nil
}

func getTelegramReleasePayload(p *api.ReleasePayload) (*TelegramPayload, error) {
	text, _ := getReleasePayloadInfo(p, htmlLinkFormatter, true)

	return &TelegramPayload{
		Message: text + "\n",
	}, nil
}

// GetTelegramPayload converts a telegram webhook into a TelegramPayload
func GetTelegramPayload(p api.Payloader, event models.HookEventType, meta string) (*TelegramPayload, error) {
	s := new(TelegramPayload)

	switch event {
	case models.HookEventCreate:
		return getTelegramCreatePayload(p.(*api.CreatePayload))
	case models.HookEventDelete:
		return getTelegramDeletePayload(p.(*api.DeletePayload))
	case models.HookEventFork:
		return getTelegramForkPayload(p.(*api.ForkPayload))
	case models.HookEventIssues:
		return getTelegramIssuesPayload(p.(*api.IssuePayload))
	case models.HookEventIssueComment:
		return getTelegramIssueCommentPayload(p.(*api.IssueCommentPayload))
	case models.HookEventPush:
		return getTelegramPushPayload(p.(*api.PushPayload))
	case models.HookEventPullRequest:
		return getTelegramPullRequestPayload(p.(*api.PullRequestPayload))
	case models.HookEventRepository:
		return getTelegramRepositoryPayload(p.(*api.RepositoryPayload))
	case models.HookEventRelease:
		return getTelegramReleasePayload(p.(*api.ReleasePayload))
	}

	return s, nil
}
<commit_msg>Add PR review webhook to Telegram (#9653)<commit_after>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package webhook

import (
	"encoding/json"
	"fmt"
	"strings"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/markup"
	api "code.gitea.io/gitea/modules/structs"
)

type (
	// TelegramPayload represents
	TelegramPayload struct {
		Message           string `json:"text"`
		ParseMode         string `json:"parse_mode"`
		DisableWebPreview bool   `json:"disable_web_page_preview"`
	}

	// TelegramMeta contains the telegram metadata
	TelegramMeta struct {
		BotToken string `json:"bot_token"`
		ChatID   string `json:"chat_id"`
	}
)

// GetTelegramHook returns telegram metadata
func GetTelegramHook(w *models.Webhook) *TelegramMeta {
	s := &TelegramMeta{}
	if err := json.Unmarshal([]byte(w.Meta), s); err != nil {
		log.Error("webhook.GetTelegramHook(%d): %v", w.ID, err)
	}
	return s
}

// SetSecret sets the telegram secret
func (p *TelegramPayload) SetSecret(_ string) {}

// JSONPayload Marshals the TelegramPayload to json
func (p *TelegramPayload) JSONPayload() ([]byte, error) {
	p.ParseMode = "HTML"
	p.DisableWebPreview = true
	p.Message = markup.Sanitize(p.Message)
	data, err := json.MarshalIndent(p, "", "  ")
	if err != nil {
		return []byte{}, err
	}
	return data, nil
}

func getTelegramCreatePayload(p *api.CreatePayload) (*TelegramPayload, error) {
	// created tag/branch
	refName := git.RefEndName(p.Ref)
	title := fmt.Sprintf(`[<a href="%s">%s</a>] %s <a href="%s">%s</a> created`, p.Repo.HTMLURL, p.Repo.FullName, p.RefType,
		p.Repo.HTMLURL+"/src/"+refName, refName)

	return &TelegramPayload{
		Message: title,
	}, nil
}

func getTelegramDeletePayload(p *api.DeletePayload) (*TelegramPayload, error) {
	// created tag/branch
	refName := git.RefEndName(p.Ref)
	title := fmt.Sprintf(`[<a href="%s">%s</a>] %s <a href="%s">%s</a> deleted`, p.Repo.HTMLURL, p.Repo.FullName, p.RefType,
		p.Repo.HTMLURL+"/src/"+refName, refName)

	return &TelegramPayload{
		Message: title,
	}, nil
}

func getTelegramForkPayload(p *api.ForkPayload) (*TelegramPayload, error) {
	title := fmt.Sprintf(`%s is forked to <a href="%s">%s</a>`, p.Forkee.FullName, p.Repo.HTMLURL, p.Repo.FullName)

	return &TelegramPayload{
		Message: title,
	}, nil
}

func getTelegramPushPayload(p *api.PushPayload) (*TelegramPayload, error) {
	var (
		branchName = git.RefEndName(p.Ref)
		commitDesc string
	)

	var titleLink string
	if len(p.Commits) == 1 {
		commitDesc = "1 new commit"
		titleLink = p.Commits[0].URL
	} else {
		commitDesc = fmt.Sprintf("%d new commits", len(p.Commits))
		titleLink = p.CompareURL
	}
	if titleLink == "" {
		titleLink = p.Repo.HTMLURL + "/src/" + branchName
	}
	title := fmt.Sprintf(`[<a href="%s">%s</a>:<a href="%s">%s</a>] %s`, p.Repo.HTMLURL, p.Repo.FullName, titleLink, branchName, commitDesc)

	var text string
	// for each commit, generate attachment text
	for i, commit := range p.Commits {
		var authorName string
		if commit.Author != nil {
			authorName = " - " + commit.Author.Name
		}
		text += fmt.Sprintf(`[<a href="%s">%s</a>] %s`, commit.URL, commit.ID[:7],
			strings.TrimRight(commit.Message, "\r\n")) + authorName
		// add linebreak to each commit but the last
		if i < len(p.Commits)-1 {
			text += "\n"
		}
	}

	return &TelegramPayload{
		Message: title + "\n" + text,
	}, nil
}

func getTelegramIssuesPayload(p *api.IssuePayload) (*TelegramPayload, error) {
	text, _, attachmentText, _ := getIssuesPayloadInfo(p, htmlLinkFormatter, true)

	return &TelegramPayload{
		Message: text + "\n\n" + attachmentText,
	}, nil
}

func getTelegramIssueCommentPayload(p *api.IssueCommentPayload) (*TelegramPayload, error) {
	text, _, _ := getIssueCommentPayloadInfo(p, htmlLinkFormatter, true)

	return &TelegramPayload{
		Message: text + "\n" + p.Comment.Body,
	}, nil
}

func getTelegramPullRequestPayload(p *api.PullRequestPayload) (*TelegramPayload, error) {
	text, _, attachmentText, _ := getPullRequestPayloadInfo(p, htmlLinkFormatter, true)

	return &TelegramPayload{
		Message: text + "\n" + attachmentText,
	}, nil
}

func getTelegramPullRequestApprovalPayload(p *api.PullRequestPayload, event models.HookEventType) (*TelegramPayload, error) {
	var text, attachmentText string
	switch p.Action {
	case api.HookIssueSynchronized:
		action, err := parseHookPullRequestEventType(event)
		if err != nil {
			return nil, err
		}

		text = fmt.Sprintf("[%s] Pull request review %s: #%d %s", p.Repository.FullName, action, p.Index, p.PullRequest.Title)
		attachmentText = p.Review.Content

	}

	return &TelegramPayload{
		Message: text + "\n" + attachmentText,
	}, nil
}

func getTelegramRepositoryPayload(p *api.RepositoryPayload) (*TelegramPayload, error) {
	var title string
	switch p.Action {
	case api.HookRepoCreated:
		title = fmt.Sprintf(`[<a href="%s">%s</a>] Repository created`, p.Repository.HTMLURL, p.Repository.FullName)
		return &TelegramPayload{
			Message: title,
		}, nil
	case api.HookRepoDeleted:
		title = fmt.Sprintf("[%s] Repository deleted", p.Repository.FullName)
		return &TelegramPayload{
			Message: title,
		}, nil
	}
	return nil, nil
}

func getTelegramReleasePayload(p *api.ReleasePayload) (*TelegramPayload, error) {
	text, _ := getReleasePayloadInfo(p, htmlLinkFormatter, true)

	return &TelegramPayload{
		Message: text + "\n",
	}, nil
}

// GetTelegramPayload converts a telegram webhook into a TelegramPayload
func GetTelegramPayload(p api.Payloader, event models.HookEventType, meta string) (*TelegramPayload, error) {
	s := new(TelegramPayload)

	switch event {
	case models.HookEventCreate:
		return getTelegramCreatePayload(p.(*api.CreatePayload))
	case models.HookEventDelete:
		return getTelegramDeletePayload(p.(*api.DeletePayload))
	case models.HookEventFork:
		return getTelegramForkPayload(p.(*api.ForkPayload))
	case models.HookEventIssues:
		return getTelegramIssuesPayload(p.(*api.IssuePayload))
	case models.HookEventIssueComment:
		return getTelegramIssueCommentPayload(p.(*api.IssueCommentPayload))
	case models.HookEventPush:
		return getTelegramPushPayload(p.(*api.PushPayload))
	case models.HookEventPullRequest:
		return getTelegramPullRequestPayload(p.(*api.PullRequestPayload))
	case models.HookEventPullRequestRejected, models.HookEventPullRequestApproved, models.HookEventPullRequestComment:
		return getTelegramPullRequestApprovalPayload(p.(*api.PullRequestPayload), event)
	case models.HookEventRepository:
		return getTelegramRepositoryPayload(p.(*api.RepositoryPayload))
	case models.HookEventRelease:
		return getTelegramReleasePayload(p.(*api.ReleasePayload))
	}

	return s, nil
}
<|endoftext|>


<commit_before>package overlay

import (
	"fmt"
	"net"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/driverapi"
	"github.com/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
)

// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
	if err := validateID(nid, eid); err != nil {
		return err
	}

	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("could not find network with id %s", nid)
	}

	ep := n.endpoint(eid)
	if ep == nil {
		return fmt.Errorf("could not find endpoint with id %s", eid)
	}

	s := n.getSubnetforIP(ep.addr)
	if s == nil {
		return fmt.Errorf("could not find subnet for endpoint %s", eid)
	}

	if err := n.obtainVxlanID(s); err != nil {
		return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
	}

	if err := n.joinSandbox(); err != nil {
		return fmt.Errorf("network sandbox join failed: %v", err)
	}

	if err := n.joinSubnetSandbox(s); err != nil {
		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
	}

	// joinSubnetSandbox gets called when an endpoint comes up on a new subnet in the
	// overlay network. Hence the Endpoint count should be updated outside joinSubnetSandbox
	n.incEndpointCount()

	sbox := n.sandbox()

	overlayIfName, containerIfName, err := createVethPair()
	if err != nil {
		return err
	}

	ep.ifName = overlayIfName

	// Set the container interface and its peer MTU to 1450 to allow
	// for 50 bytes vxlan encap (inner eth header(14) + outer IP(20) +
	// outer UDP(8) + vxlan header(8))
	veth, err := netlink.LinkByName(overlayIfName)
	if err != nil {
		return fmt.Errorf("cound not find link by name %s: %v", overlayIfName, err)
	}
	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
	if err != nil {
		return err
	}

	if err := sbox.AddInterface(overlayIfName, "veth",
		sbox.InterfaceOptions().Master(s.brName)); err != nil {
		return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)
	}

	veth, err = netlink.LinkByName(containerIfName)
	if err != nil {
		return fmt.Errorf("could not find link by name %s: %v", containerIfName, err)
	}
	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
	if err != nil {
		return err
	}

	if err := netlink.LinkSetHardwareAddr(veth, ep.mac); err != nil {
		return fmt.Errorf("could not set mac address (%v) to the container interface: %v", ep.mac, err)
	}

	for _, sub := range n.subnets {
		if sub == s {
			continue
		}
		if err := jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil {
			log.Errorf("Adding subnet %s static route in network %q failed\n", s.subnetIP, n.id)
		}
	}

	if iNames := jinfo.InterfaceName(); iNames != nil {
		err = iNames.SetNames(containerIfName, "eth")
		if err != nil {
			return err
		}
	}

	d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac,
		net.ParseIP(d.bindAddress), true)
	d.pushLocalEndpointEvent("join", nid, eid)

	return nil
}

// Leave method is invoked when a Sandbox detaches from an endpoint.
func (d *driver) Leave(nid, eid string) error {
	if err := validateID(nid, eid); err != nil {
		return err
	}

	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("could not find network with id %s", nid)
	}

	ep := n.endpoint(eid)

	if ep == nil {
		return types.InternalMaskableErrorf("could not find endpoint with id %s", eid)
	}

	if d.notifyCh != nil {
		d.notifyCh <- ovNotify{
			action: "leave",
			nid:    nid,
			eid:    eid,
		}
	}

	n.leaveSandbox()

	return nil
}
<commit_msg>Fix veth leak in overlay driver<commit_after>package overlay

import (
	"fmt"
	"net"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/driverapi"
	"github.com/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
)

// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
	if err := validateID(nid, eid); err != nil {
		return err
	}

	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("could not find network with id %s", nid)
	}

	ep := n.endpoint(eid)
	if ep == nil {
		return fmt.Errorf("could not find endpoint with id %s", eid)
	}

	s := n.getSubnetforIP(ep.addr)
	if s == nil {
		return fmt.Errorf("could not find subnet for endpoint %s", eid)
	}

	if err := n.obtainVxlanID(s); err != nil {
		return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
	}

	if err := n.joinSandbox(); err != nil {
		return fmt.Errorf("network sandbox join failed: %v", err)
	}

	if err := n.joinSubnetSandbox(s); err != nil {
		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
	}

	// joinSubnetSandbox gets called when an endpoint comes up on a new subnet in the
	// overlay network. Hence the Endpoint count should be updated outside joinSubnetSandbox
	n.incEndpointCount()

	sbox := n.sandbox()

	overlayIfName, containerIfName, err := createVethPair()
	if err != nil {
		return err
	}

	ep.ifName = containerIfName

	// Set the container interface and its peer MTU to 1450 to allow
	// for 50 bytes vxlan encap (inner eth header(14) + outer IP(20) +
	// outer UDP(8) + vxlan header(8))
	veth, err := netlink.LinkByName(overlayIfName)
	if err != nil {
		return fmt.Errorf("cound not find link by name %s: %v", overlayIfName, err)
	}
	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
	if err != nil {
		return err
	}

	if err := sbox.AddInterface(overlayIfName, "veth",
		sbox.InterfaceOptions().Master(s.brName)); err != nil {
		return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)
	}

	veth, err = netlink.LinkByName(containerIfName)
	if err != nil {
		return fmt.Errorf("could not find link by name %s: %v", containerIfName, err)
	}
	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
	if err != nil {
		return err
	}

	if err := netlink.LinkSetHardwareAddr(veth, ep.mac); err != nil {
		return fmt.Errorf("could not set mac address (%v) to the container interface: %v", ep.mac, err)
	}

	for _, sub := range n.subnets {
		if sub == s {
			continue
		}
		if err := jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil {
			log.Errorf("Adding subnet %s static route in network %q failed\n", s.subnetIP, n.id)
		}
	}

	if iNames := jinfo.InterfaceName(); iNames != nil {
		err = iNames.SetNames(containerIfName, "eth")
		if err != nil {
			return err
		}
	}

	d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac,
		net.ParseIP(d.bindAddress), true)
	d.pushLocalEndpointEvent("join", nid, eid)

	return nil
}

// Leave method is invoked when a Sandbox detaches from an endpoint.
func (d *driver) Leave(nid, eid string) error {
	if err := validateID(nid, eid); err != nil {
		return err
	}

	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("could not find network with id %s", nid)
	}

	ep := n.endpoint(eid)

	if ep == nil {
		return types.InternalMaskableErrorf("could not find endpoint with id %s", eid)
	}

	if d.notifyCh != nil {
		d.notifyCh <- ovNotify{
			action: "leave",
			nid:    nid,
			eid:    eid,
		}
	}

	n.leaveSandbox()

	return nil
}
<|endoftext|>