<commit_before>package hammertime

import (
	"fmt"
	"time"

	kitlog "github.com/go-kit/kit/log"
	core_v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/record"

	"github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
	"github.com/sapcc/kubernikus/pkg/controller/base"
	"github.com/sapcc/kubernikus/pkg/controller/config"
	"github.com/sapcc/kubernikus/pkg/controller/metrics"
	"github.com/sapcc/kubernikus/pkg/controller/nodeobservatory"
)

type hammertimeController struct {
	nodeObervatory *nodeobservatory.NodeObservatory
	client         kubernetes.Interface
	timeout        time.Duration
	logger         kitlog.Logger
	recorder       record.EventRecorder
}

func New(syncPeriod time.Duration, timeout time.Duration, factories config.Factories, clients config.Clients, recorder record.EventRecorder, logger kitlog.Logger) base.Controller {

	logger = kitlog.With(logger, "controller", "hammertime")

	controller := hammertimeController{
		nodeObervatory: factories.NodesObservatory.NodeInformer(),
		client:         clients.Kubernetes,
		timeout:        timeout,
		logger:         logger,
		recorder:       recorder,
	}

	return base.NewPollingController(syncPeriod, factories.Kubernikus.Kubernikus().V1().Klusters(), &controller, logger)
}

func (hc *hammertimeController) Reconcile(kluster *v1.Kluster) error {
	logger := kitlog.With(hc.logger, "kluster", kluster.GetName())

	lister, err := hc.nodeObervatory.GetListerForKluster(kluster)
	if err != nil {
		return fmt.Errorf("Failed to get node lister: %s", err)
	}
	nodes, err := lister.List(labels.Everything())
	if err != nil {
		return fmt.Errorf("listing nodes failed: %s", err)
	}

	if len(nodes) < 2 {
		return nil
	}
	//var oldestHeartbeat time.Time = time.Now()
	var newestHearbeat time.Time = time.Time{}
	for _, node := range nodes {
		ready := nodeReadyCondition(node)
		if ready == nil {
			continue
		}
		if ready.LastHeartbeatTime.After(newestHearbeat) {
			newestHearbeat = ready.LastHeartbeatTime.Time
		}
	}

	timeout_exeeded := time.Now().Sub(newestHearbeat) > hc.timeout
	if timeout_exeeded {
		metrics.HammertimeStatus.WithLabelValues(kluster.Name).Set(1)
	} else {
		metrics.HammertimeStatus.WithLabelValues(kluster.Name).Set(0)
	}

	return hc.scaleDeployment(kluster, timeout_exeeded, logger)

}

func (hc *hammertimeController) scaleDeployment(kluster *v1.Kluster, disable bool, logger kitlog.Logger) error {

	deploymentClient := hc.client.ExtensionsV1beta1().Deployments(kluster.Namespace)

	deploymentName := fmt.Sprintf("%s-cmanager", kluster.GetName())
	scale, err := deploymentClient.GetScale(deploymentName, meta_v1.GetOptions{})
	if apierrors.IsNotFound(err) {
		deploymentName = fmt.Sprintf("%s-controller-manager", kluster.GetName())
		scale, err = deploymentClient.GetScale(deploymentName, meta_v1.GetOptions{})
	}
	if err != nil {
		return fmt.Errorf("Failed to get deployment scale: %s", err)
	}

	if scale.Spec.Replicas > 0 {
		if disable {
			scale.Spec.Replicas = 0
			_, err = deploymentClient.UpdateScale(deploymentName, scale)
			logger.Log("msg", "Scaling down", "deployment", deploymentName, "err", err)
		}
	} else {
		if !disable {
			scale.Spec.Replicas = 1
			_, err = deploymentClient.UpdateScale(deploymentName, scale)
			logger.Log("msg", "Scaling up", "deployment", deploymentName, "err", err)
		}
	}
	return err
}

func nodeReadyCondition(node *core_v1.Node) *core_v1.NodeCondition {
	for _, c := range node.Status.Conditions {
		if c.Type == core_v1.NodeReady {
			return &c
		}
	}
	return nil
}
<commit_msg>Enable hammertime controller only in phase running<commit_after>package hammertime

import (
	"fmt"
	"time"

	kitlog "github.com/go-kit/kit/log"
	core_v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/record"

	"github.com/sapcc/kubernikus/pkg/api/models"
	"github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
	"github.com/sapcc/kubernikus/pkg/controller/base"
	"github.com/sapcc/kubernikus/pkg/controller/config"
	"github.com/sapcc/kubernikus/pkg/controller/metrics"
	"github.com/sapcc/kubernikus/pkg/controller/nodeobservatory"
)

type hammertimeController struct {
	nodeObervatory *nodeobservatory.NodeObservatory
	client         kubernetes.Interface
	timeout        time.Duration
	logger         kitlog.Logger
	recorder       record.EventRecorder
}

func New(syncPeriod time.Duration, timeout time.Duration, factories config.Factories, clients config.Clients, recorder record.EventRecorder, logger kitlog.Logger) base.Controller {

	logger = kitlog.With(logger, "controller", "hammertime")

	controller := hammertimeController{
		nodeObervatory: factories.NodesObservatory.NodeInformer(),
		client:         clients.Kubernetes,
		timeout:        timeout,
		logger:         logger,
		recorder:       recorder,
	}

	return base.NewPollingController(syncPeriod, factories.Kubernikus.Kubernikus().V1().Klusters(), &controller, logger)
}

func (hc *hammertimeController) Reconcile(kluster *v1.Kluster) error {
	logger := kitlog.With(hc.logger, "kluster", kluster.GetName())

	lister, err := hc.nodeObervatory.GetListerForKluster(kluster)
	if err != nil {
		return fmt.Errorf("Failed to get node lister: %s", err)
	}
	nodes, err := lister.List(labels.Everything())
	if err != nil {
		return fmt.Errorf("listing nodes failed: %s", err)
	}

	// No Hammertime if kluster is not in state Running and has at least 2 nodes
	if len(nodes) < 2 || kluster.Status.Phase != models.KlusterPhaseRunning {
		metrics.HammertimeStatus.WithLabelValues(kluster.Name).Set(0)
		return hc.scaleDeployment(kluster, false, logger)
	}

	//var oldestHeartbeat time.Time = time.Now()
	var newestHearbeat time.Time = time.Time{}
	for _, node := range nodes {
		ready := nodeReadyCondition(node)
		if ready == nil {
			continue
		}
		if ready.LastHeartbeatTime.After(newestHearbeat) {
			newestHearbeat = ready.LastHeartbeatTime.Time
		}
	}

	timeout_exeeded := time.Now().Sub(newestHearbeat) > hc.timeout
	if timeout_exeeded {
		metrics.HammertimeStatus.WithLabelValues(kluster.Name).Set(1)
	} else {
		metrics.HammertimeStatus.WithLabelValues(kluster.Name).Set(0)
	}

	return hc.scaleDeployment(kluster, timeout_exeeded, logger)

}

func (hc *hammertimeController) scaleDeployment(kluster *v1.Kluster, disable bool, logger kitlog.Logger) error {

	deploymentClient := hc.client.ExtensionsV1beta1().Deployments(kluster.Namespace)

	deploymentName := fmt.Sprintf("%s-cmanager", kluster.GetName())
	scale, err := deploymentClient.GetScale(deploymentName, meta_v1.GetOptions{})
	if apierrors.IsNotFound(err) {
		deploymentName = fmt.Sprintf("%s-controller-manager", kluster.GetName())
		scale, err = deploymentClient.GetScale(deploymentName, meta_v1.GetOptions{})
	}
	if err != nil {
		return fmt.Errorf("Failed to get deployment scale: %s", err)
	}

	if scale.Spec.Replicas > 0 {
		if disable {
			scale.Spec.Replicas = 0
			_, err = deploymentClient.UpdateScale(deploymentName, scale)
			logger.Log("msg", "Scaling down", "deployment", deploymentName, "err", err)
		}
	} else {
		if !disable {
			scale.Spec.Replicas = 1
			_, err = deploymentClient.UpdateScale(deploymentName, scale)
			logger.Log("msg", "Scaling up", "deployment", deploymentName, "err", err)
		}
	}
	return err
}

func nodeReadyCondition(node *core_v1.Node) *core_v1.NodeCondition {
	for _, c := range node.Status.Conditions {
		if c.Type == core_v1.NodeReady {
			return &c
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mon for the Ceph monitors.
package mon

import (
	"fmt"
	"time"

	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/daemon/ceph/client"
	"github.com/rook/rook/pkg/daemon/ceph/mon"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	// HealthCheckInterval interval to check the mons to be in quorum
	HealthCheckInterval = 45 * time.Second
	// MonOutTimeout the duration to wait before removing/failover to a new mon pod
	MonOutTimeout = 300 * time.Second
)

// HealthChecker check health for the monitors
type HealthChecker struct {
	monCluster *Cluster
}

// NewHealthChecker creates a new HealthChecker object
func NewHealthChecker(monCluster *Cluster) *HealthChecker {
	return &HealthChecker{
		monCluster: monCluster,
	}
}

// Check periodically the health of the monitors
func (hc *HealthChecker) Check(stopCh chan struct{}) {
	for {
		select {
		case <-stopCh:
			logger.Infof("Stopping monitoring of cluster in namespace %s", hc.monCluster.Namespace)
			return

		case <-time.After(HealthCheckInterval):
			logger.Debugf("checking health of mons")
			err := hc.monCluster.checkHealth()
			if err != nil {
				logger.Infof("failed to check mon health. %+v", err)
			}
		}
	}
}

func (c *Cluster) checkHealth() error {
	logger.Debugf("Checking health for mons. %+v", c.clusterInfo)

	// connect to the mons
	// get the status and check for quorum
	status, err := client.GetMonStatus(c.context, c.clusterInfo.Name, true)
	if err != nil {
		return fmt.Errorf("failed to get mon status. %+v", err)
	}
	logger.Debugf("Mon status: %+v", status)

	// Source of thruth of which mons should exist is our *clusterInfo*
	monsNotFound := map[string]interface{}{}
	for _, mon := range c.clusterInfo.Monitors {
		monsNotFound[mon.Name] = struct{}{}
	}

	// first handle mons that are not in quorum but in the cecph mon map
	//failover the unhealthy mons
	for _, mon := range status.MonMap.Mons {
		inQuorum := monInQuorum(mon, status.Quorum)
		// if the mon is in quorum remove it from our check for "existence"
		//else see below condition
		if _, ok := monsNotFound[mon.Name]; ok {
			delete(monsNotFound, mon.Name)
		} else {
			// when the mon isn't in the clusterInfo, but is in qorum and there are
			//enough mons, remove it else remove it on the next run
			if inQuorum && len(status.MonMap.Mons) > c.Size {
				logger.Warningf("mon %s not in source of truth but in quorum, removing", mon.Name)
				c.removeMon(mon.Name)
			} else {
				logger.Warningf(
					"mon %s not in source of truth and not in quorum, not enough mons to remove now (wanted: %d, current: %d)",
					mon.Name,
					c.Size,
					len(status.MonMap.Mons),
				)
			}
		}

		if inQuorum {
			logger.Debugf("mon %s found in quorum", mon.Name)
			// delete the "timeout" for a mon if the pod is in quorum again
			if _, ok := c.monTimeoutList[mon.Name]; ok {
				delete(c.monTimeoutList, mon.Name)
			}
		} else {
			logger.Warningf("mon %s NOT found in quorum. %+v", mon.Name, status)

			// If not yet set, add the current time, for the timeout
			// calculation, to the list
			if _, ok := c.monTimeoutList[mon.Name]; !ok {
				c.monTimeoutList[mon.Name] = time.Now()
			}

			// when the timeout for the mon has been reached, continue to the
			// normal failover/delete mon pod part of the code
			if time.Since(c.monTimeoutList[mon.Name]) <= MonOutTimeout {
				logger.Warningf("mon %s NOT found in quorum, STILL in mon out timeout", mon.Name)
				continue
			}

			c.failMon(len(status.MonMap.Mons), mon.Name)
			// only deal with one unhealthy mon per health check
			return nil
		}
	}

	// after all unhealthy mons have been removed/failovered
	//handle all mons that haven't been in the Ceph mon map
	for mon := range monsNotFound {
		logger.Warningf("mon %s NOT found in ceph mon map, failover", mon)
		c.failMon(len(c.clusterInfo.Monitors), mon)
		// only deal with one "not found in ceph mon map" mon per health check
		return nil
	}

	// check if there are more than two mons running on the same node, failover one mon in that case
	done, err := c.checkMonsOnSameNode()
	if done || err != nil {
		return err
	}

	done, err = c.checkMonsOnValidNodes()
	if done || err != nil {
		return err
	}

	return nil
}

func (c *Cluster) checkMonsOnSameNode() (bool, error) {
	nodesUsed := map[string]struct{}{}
	for name, node := range c.mapping.Node {
		// when the node is already in the list we have more than one mon on that node
		if _, ok := nodesUsed[node.Name]; ok {
			// get list of available nodes for mons
			availableNodes, _, err := c.getAvailableMonNodes()
			if err != nil {
				return true, fmt.Errorf("failed to get available mon nodes. %+v", err)
			}
			// if there are enough nodes for one mon "that is too much" to be failovered,
			// fail it over to an other node
			if len(availableNodes) > 0 {
				logger.Infof("rebalance: enough nodes available %d to failover mon %s", len(availableNodes), name)
				c.failMon(len(c.clusterInfo.Monitors), name)
			} else {
				logger.Debugf("rebalance: not enough nodes available to failover mon %s", name)
			}

			// deal with one mon too much on a node at a time
			return true, nil
		}
		nodesUsed[node.Name] = struct{}{}
	}
	return false, nil
}

func (c *Cluster) checkMonsOnValidNodes() (bool, error) {
	for mon, nInfo := range c.mapping.Node {
		// get node to use for validNode() func
		node, err := c.context.Clientset.CoreV1().Nodes().Get(nInfo.Name, metav1.GetOptions{})
		if err != nil {
			return true, err
		}
		// check if node the mon is on is still valid
		if !validNode(*node, c.placement) {
			logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon)
			c.failoverMon(mon)
			return true, nil
		}
		logger.Debugf("node %s with mon %s is still valid", nInfo.Name, mon)
	}
	return false, nil
}

// failMon monCount is compared against c.Size (wanted mon count)
func (c *Cluster) failMon(monCount int, name string) {
	if monCount > c.Size {
		// no need to create a new mon since we have an extra
		if err := c.removeMon(name); err != nil {
			logger.Errorf("failed to remove mon %s. %+v", name, err)
		}
	} else {
		// bring up a new mon to replace the unhealthy mon
		if err := c.failoverMon(name); err != nil {
			logger.Errorf("failed to failover mon %s. %+v", name, err)
		}
	}
}

func (c *Cluster) failoverMon(name string) error {
	logger.Infof("Failing over monitor %s", name)

	// Start a new monitor
	m := &monConfig{Name: fmt.Sprintf("%s%d", appName, c.maxMonID+1), Port: int32(mon.DefaultPort)}
	logger.Infof("starting new mon %s", m.Name)

	// Create the service endpoint
	serviceIP, err := c.createService(m)
	if err != nil {
		return fmt.Errorf("failed to create mon service. %+v", err)
	}

	mConf := []*monConfig{m}

	// Assign the pod to a node
	if err = c.assignMons(mConf); err != nil {
		return fmt.Errorf("failed to assign pods to mons. %+v", err)
	}

	if c.HostNetwork {
		node, ok := c.mapping.Node[m.Name]
		if !ok {
			return fmt.Errorf("mon %s doesn't exist in assignment map", m.Name)
		}
		m.PublicIP = node.Address
	} else {
		m.PublicIP = serviceIP
	}
	c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.PublicIP, m.Port)

	// Start the pod
	if err = c.startPods(mConf); err != nil {
		return fmt.Errorf("failed to start new mon %s. %+v", m.Name, err)
	}

	// Only increment the max mon id if the new pod started successfully
	c.maxMonID++

	return c.removeMon(name)
}

func (c *Cluster) removeMon(name string) error {
	logger.Infof("ensuring removal of unhealthy monitor %s", name)

	// Remove the mon pod if it is still there
	var gracePeriod int64
	propagation := metav1.DeletePropagationForeground
	options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
	if err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Delete(name, options); err != nil {
		if errors.IsNotFound(err) {
			logger.Infof("dead mon %s was already gone", name)
		} else {
			return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
		}
	}

	// Remove the bad monitor from quorum
	if err := removeMonitorFromQuorum(c.context, c.clusterInfo.Name, name); err != nil {
		return fmt.Errorf("failed to remove mon %s from quorum. %+v", name, err)
	}
	delete(c.clusterInfo.Monitors, name)
	// check if a mapping exists for the mon
	if _, ok := c.mapping.Node[name]; ok {
		nodeName := c.mapping.Node[name].Name
		delete(c.mapping.Node, name)
		// if node->port "mapping" has been created, decrease or delete it
		if port, ok := c.mapping.Port[nodeName]; ok {
			if port == mon.DefaultPort {
				delete(c.mapping.Port, nodeName)
			}
			// don't clean up if a node port is higher than the default port, other
			//mons could be on the same node with > DefaultPort ports, decreasing could
			//cause port collsions
			// This can be solved by using a map[nodeName][]int32 for the ports to
			//even better check which ports are open for the HostNetwork mode
		}
	}

	// Remove the service endpoint
	if err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(name, options); err != nil {
		if errors.IsNotFound(err) {
			logger.Infof("dead mon service %s was already gone", name)
		} else {
			return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
		}
	}

	if err := c.saveMonConfig(); err != nil {
		return fmt.Errorf("failed to save mon config after failing over mon %s. %+v", name, err)
	}

	// make sure to rewrite the config so NO new connections are made to the removed mon
	if err := WriteConnectionConfig(c.context, c.clusterInfo); err != nil {
		return fmt.Errorf("failed to write connection config after failing over mon %s. %+v", name, err)
	}

	return nil
}

func removeMonitorFromQuorum(context *clusterd.Context, clusterName, name string) error {
	logger.Debugf("removing monitor %s", name)
	args := []string{"mon", "remove", name}
	if _, err := client.ExecuteCephCommand(context, clusterName, args); err != nil {
		return fmt.Errorf("mon %s remove failed: %+v", name, err)
	}

	logger.Infof("removed monitor %s", name)
	return nil
}
<commit_msg>Log when mon is back in quorum and exceeded timeout<commit_after>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mon for the Ceph monitors.
package mon

import (
	"fmt"
	"time"

	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/daemon/ceph/client"
	"github.com/rook/rook/pkg/daemon/ceph/mon"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	// HealthCheckInterval interval to check the mons to be in quorum
	HealthCheckInterval = 45 * time.Second
	// MonOutTimeout the duration to wait before removing/failover to a new mon pod
	MonOutTimeout = 300 * time.Second
)

// HealthChecker check health for the monitors
type HealthChecker struct {
	monCluster *Cluster
}

// NewHealthChecker creates a new HealthChecker object
func NewHealthChecker(monCluster *Cluster) *HealthChecker {
	return &HealthChecker{
		monCluster: monCluster,
	}
}

// Check periodically the health of the monitors
func (hc *HealthChecker) Check(stopCh chan struct{}) {
	for {
		select {
		case <-stopCh:
			logger.Infof("Stopping monitoring of cluster in namespace %s", hc.monCluster.Namespace)
			return

		case <-time.After(HealthCheckInterval):
			logger.Debugf("checking health of mons")
			err := hc.monCluster.checkHealth()
			if err != nil {
				logger.Infof("failed to check mon health. %+v", err)
			}
		}
	}
}

func (c *Cluster) checkHealth() error {
	logger.Debugf("Checking health for mons. %+v", c.clusterInfo)

	// connect to the mons
	// get the status and check for quorum
	status, err := client.GetMonStatus(c.context, c.clusterInfo.Name, true)
	if err != nil {
		return fmt.Errorf("failed to get mon status. %+v", err)
	}
	logger.Debugf("Mon status: %+v", status)

	// Source of thruth of which mons should exist is our *clusterInfo*
	monsNotFound := map[string]interface{}{}
	for _, mon := range c.clusterInfo.Monitors {
		monsNotFound[mon.Name] = struct{}{}
	}

	// first handle mons that are not in quorum but in the cecph mon map
	//failover the unhealthy mons
	for _, mon := range status.MonMap.Mons {
		inQuorum := monInQuorum(mon, status.Quorum)
		// if the mon is in quorum remove it from our check for "existence"
		//else see below condition
		if _, ok := monsNotFound[mon.Name]; ok {
			delete(monsNotFound, mon.Name)
		} else {
			// when the mon isn't in the clusterInfo, but is in qorum and there are
			//enough mons, remove it else remove it on the next run
			if inQuorum && len(status.MonMap.Mons) > c.Size {
				logger.Warningf("mon %s not in source of truth but in quorum, removing", mon.Name)
				c.removeMon(mon.Name)
			} else {
				logger.Warningf(
					"mon %s not in source of truth and not in quorum, not enough mons to remove now (wanted: %d, current: %d)",
					mon.Name,
					c.Size,
					len(status.MonMap.Mons),
				)
			}
		}

		if inQuorum {
			logger.Debugf("mon %s found in quorum", mon.Name)
			// delete the "timeout" for a mon if the pod is in quorum again
			if _, ok := c.monTimeoutList[mon.Name]; ok {
				delete(c.monTimeoutList, mon.Name)
				logger.Infof("mon %s is back in quorum, removed from mon out timeout list", mon.Name)
			}
		} else {
			logger.Debugf("mon %s NOT found in quorum. Mon status: %+v", mon.Name, status)

			// If not yet set, add the current time, for the timeout
			// calculation, to the list
			if _, ok := c.monTimeoutList[mon.Name]; !ok {
				c.monTimeoutList[mon.Name] = time.Now()
			}

			// when the timeout for the mon has been reached, continue to the
			// normal failover/delete mon pod part of the code
			if time.Since(c.monTimeoutList[mon.Name]) <= MonOutTimeout {
				logger.Warningf("mon %s not found in quorum, still in mon out timeout", mon.Name)
				continue
			}

			logger.Warningf("mon %s NOT found in quorum and timeout exceeded, mon will be failed over", mon.Name)
			c.failMon(len(status.MonMap.Mons), mon.Name)
			// only deal with one unhealthy mon per health check
			return nil
		}
	}

	// after all unhealthy mons have been removed/failovered
	//handle all mons that haven't been in the Ceph mon map
	for mon := range monsNotFound {
		logger.Warningf("mon %s NOT found in ceph mon map, failover", mon)
		c.failMon(len(c.clusterInfo.Monitors), mon)
		// only deal with one "not found in ceph mon map" mon per health check
		return nil
	}

	// check if there are more than two mons running on the same node, failover one mon in that case
	done, err := c.checkMonsOnSameNode()
	if done || err != nil {
		return err
	}

	done, err = c.checkMonsOnValidNodes()
	if done || err != nil {
		return err
	}

	return nil
}

func (c *Cluster) checkMonsOnSameNode() (bool, error) {
	nodesUsed := map[string]struct{}{}
	for name, node := range c.mapping.Node {
		// when the node is already in the list we have more than one mon on that node
		if _, ok := nodesUsed[node.Name]; ok {
			// get list of available nodes for mons
			availableNodes, _, err := c.getAvailableMonNodes()
			if err != nil {
				return true, fmt.Errorf("failed to get available mon nodes. %+v", err)
			}
			// if there are enough nodes for one mon "that is too much" to be failovered,
			// fail it over to an other node
			if len(availableNodes) > 0 {
				logger.Infof("rebalance: enough nodes available %d to failover mon %s", len(availableNodes), name)
				c.failMon(len(c.clusterInfo.Monitors), name)
			} else {
				logger.Debugf("rebalance: not enough nodes available to failover mon %s", name)
			}

			// deal with one mon too much on a node at a time
			return true, nil
		}
		nodesUsed[node.Name] = struct{}{}
	}
	return false, nil
}

func (c *Cluster) checkMonsOnValidNodes() (bool, error) {
	for mon, nInfo := range c.mapping.Node {
		// get node to use for validNode() func
		node, err := c.context.Clientset.CoreV1().Nodes().Get(nInfo.Name, metav1.GetOptions{})
		if err != nil {
			return true, err
		}
		// check if node the mon is on is still valid
		if !validNode(*node, c.placement) {
			logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon)
			c.failoverMon(mon)
			return true, nil
		}
		logger.Debugf("node %s with mon %s is still valid", nInfo.Name, mon)
	}
	return false, nil
}

// failMon monCount is compared against c.Size (wanted mon count)
func (c *Cluster) failMon(monCount int, name string) {
	if monCount > c.Size {
		// no need to create a new mon since we have an extra
		if err := c.removeMon(name); err != nil {
			logger.Errorf("failed to remove mon %s. %+v", name, err)
		}
	} else {
		// bring up a new mon to replace the unhealthy mon
		if err := c.failoverMon(name); err != nil {
			logger.Errorf("failed to failover mon %s. %+v", name, err)
		}
	}
}

func (c *Cluster) failoverMon(name string) error {
	logger.Infof("Failing over monitor %s", name)

	// Start a new monitor
	m := &monConfig{Name: fmt.Sprintf("%s%d", appName, c.maxMonID+1), Port: int32(mon.DefaultPort)}
	logger.Infof("starting new mon %s", m.Name)

	// Create the service endpoint
	serviceIP, err := c.createService(m)
	if err != nil {
		return fmt.Errorf("failed to create mon service. %+v", err)
	}

	mConf := []*monConfig{m}

	// Assign the pod to a node
	if err = c.assignMons(mConf); err != nil {
		return fmt.Errorf("failed to assign pods to mons. %+v", err)
	}

	if c.HostNetwork {
		node, ok := c.mapping.Node[m.Name]
		if !ok {
			return fmt.Errorf("mon %s doesn't exist in assignment map", m.Name)
		}
		m.PublicIP = node.Address
	} else {
		m.PublicIP = serviceIP
	}
	c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.PublicIP, m.Port)

	// Start the pod
	if err = c.startPods(mConf); err != nil {
		return fmt.Errorf("failed to start new mon %s. %+v", m.Name, err)
	}

	// Only increment the max mon id if the new pod started successfully
	c.maxMonID++

	return c.removeMon(name)
}

func (c *Cluster) removeMon(name string) error {
	logger.Infof("ensuring removal of unhealthy monitor %s", name)

	// Remove the mon pod if it is still there
	var gracePeriod int64
	propagation := metav1.DeletePropagationForeground
	options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
	if err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Delete(name, options); err != nil {
		if errors.IsNotFound(err) {
			logger.Infof("dead mon %s was already gone", name)
		} else {
			return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
		}
	}

	// Remove the bad monitor from quorum
	if err := removeMonitorFromQuorum(c.context, c.clusterInfo.Name, name); err != nil {
		return fmt.Errorf("failed to remove mon %s from quorum. %+v", name, err)
	}
	delete(c.clusterInfo.Monitors, name)
	// check if a mapping exists for the mon
	if _, ok := c.mapping.Node[name]; ok {
		nodeName := c.mapping.Node[name].Name
		delete(c.mapping.Node, name)
		// if node->port "mapping" has been created, decrease or delete it
		if port, ok := c.mapping.Port[nodeName]; ok {
			if port == mon.DefaultPort {
				delete(c.mapping.Port, nodeName)
			}
			// don't clean up if a node port is higher than the default port, other
			//mons could be on the same node with > DefaultPort ports, decreasing could
			//cause port collsions
			// This can be solved by using a map[nodeName][]int32 for the ports to
			//even better check which ports are open for the HostNetwork mode
		}
	}

	// Remove the service endpoint
	if err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(name, options); err != nil {
		if errors.IsNotFound(err) {
			logger.Infof("dead mon service %s was already gone", name)
		} else {
			return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
		}
	}

	if err := c.saveMonConfig(); err != nil {
		return fmt.Errorf("failed to save mon config after failing over mon %s. %+v", name, err)
	}

	// make sure to rewrite the config so NO new connections are made to the removed mon
	if err := WriteConnectionConfig(c.context, c.clusterInfo); err != nil {
		return fmt.Errorf("failed to write connection config after failing over mon %s. %+v", name, err)
	}

	return nil
}

func removeMonitorFromQuorum(context *clusterd.Context, clusterName, name string) error {
	logger.Debugf("removing monitor %s", name)
	args := []string{"mon", "remove", name}
	if _, err := client.ExecuteCephCommand(context, clusterName, args); err != nil {
		return fmt.Errorf("mon %s remove failed: %+v", name, err)
	}

	logger.Infof("removed monitor %s", name)
	return nil
}
<|endoftext|>


<commit_before>/*
 * Drop and recreate database objects. Used for testing and creating a new
 * deployment. Must be run after createdb/main.go. Also serves as table
 * documentation.
 */
package main

import (
    "log"
    "database/sql"
    _ "github.com/lib/pq"
)

func main() {
    db, err := sql.Open("postgres", "user=recipes dbname=recipes sslmode=disable")
    if err != nil {
        log.Println(err)
        log.Fatal("ERROR: connection params are invalid")
    }
    err = db.Ping()
    if err != nil {
        log.Println(err)
        log.Fatal("ERROR: failed to connect to the DB")
    }

    log.Println("dropping old objects")
    wrap_sql(db, "DROP TABLE IF EXISTS linked_recipes")
    wrap_sql(db, "DROP TABLE IF EXISTS tags CASCADE")
    wrap_sql(db, "DROP TABLE IF EXISTS recipes CASCADE")
    wrap_sql(db, "DROP TABLE IF EXISTS users")

    log.Println("creating new objects")

    wrap_sql(db, `CREATE TABLE users (
        id              serial PRIMARY KEY,
        email           text NOT NULL,
        name            text,
        role            text NOT NULL,
        token           text,
        creation_date   timestamp WITH TIME ZONE NOT NULL
                            DEFAULT CURRENT_TIMESTAMP,
        lastlog         timestamp WITH TIME ZONE
    )`)
    wrap_sql(db, `CREATE TABLE recipes (
        id          serial PRIMARY KEY,
        revision    integer NOT NULL DEFAULT 0,
        amount      text NOT NULL DEFAULT '',
        author_id   integer NOT NULL REFERENCES users(id),
        directions  jsonb NOT NULL DEFAULT '[]',
        ingredients jsonb NOT NULL DEFAULT '[]',
        notes       text NOT NULL DEFAULT '',
        oven        text NOT NULL DEFAULT '',
        source      text NOT NULL DEFAULT '',
        summary     text NOT NULL DEFAULT '',
        time        text NOT NULL DEFAULT '',
        title       text NOT NULL
    )`)
    wrap_sql(db, `CREATE TABLE tags (
        recipe_id       integer NOT NULL REFERENCES recipes(id),
        tag             text NOT NULL,
        UNIQUE(recipe_id, tag)
    )`)
    wrap_sql(db, `CREATE TABLE linked_recipes (
        src     integer REFERENCES recipes(id) NOT NULL,
        dest    integer REFERENCES recipes(id) NOT NULL,
        CONSTRAINT must_be_different CHECK ( src != dest ),
        UNIQUE (src, dest)
    )`)

    log.Println("complete")
}

func wrap_sql(db *sql.DB, s string) {
    _, err := db.Query(s)
    if err != nil {
        log.Println("error during:", s)
        log.Println(err)
        log.Fatal()
    }
}
<commit_msg>Add missing CASCADE statements to reset_db script<commit_after>/*
 * Drop and recreate database objects. Used for testing and creating a new
 * deployment. Must be run after createdb/main.go. Also serves as table
 * documentation.
 */
package main

import (
    "log"
    "database/sql"
    _ "github.com/lib/pq"
)

func main() {
    db, err := sql.Open("postgres", "user=recipes dbname=recipes sslmode=disable")
    if err != nil {
        log.Println(err)
        log.Fatal("ERROR: connection params are invalid")
    }
    err = db.Ping()
    if err != nil {
        log.Println(err)
        log.Fatal("ERROR: failed to connect to the DB")
    }

    log.Println("dropping old objects")
    wrap_sql(db, "DROP TABLE IF EXISTS linked_recipes")
    wrap_sql(db, "DROP TABLE IF EXISTS tags CASCADE")
    wrap_sql(db, "DROP TABLE IF EXISTS recipes CASCADE")
    wrap_sql(db, "DROP TABLE IF EXISTS users")

    log.Println("creating new objects")

    wrap_sql(db, `CREATE TABLE users (
        id              serial PRIMARY KEY,
        email           text NOT NULL,
        name            text,
        role            text NOT NULL,
        token           text,
        creation_date   timestamp WITH TIME ZONE NOT NULL
                            DEFAULT CURRENT_TIMESTAMP,
        lastlog         timestamp WITH TIME ZONE
    )`)
    wrap_sql(db, `CREATE TABLE recipes (
        id          serial PRIMARY KEY,
        revision    integer NOT NULL DEFAULT 0,
        amount      text NOT NULL DEFAULT '',
        author_id   integer NOT NULL REFERENCES users(id),
        directions  jsonb NOT NULL DEFAULT '[]',
        ingredients jsonb NOT NULL DEFAULT '[]',
        notes       text NOT NULL DEFAULT '',
        oven        text NOT NULL DEFAULT '',
        source      text NOT NULL DEFAULT '',
        summary     text NOT NULL DEFAULT '',
        time        text NOT NULL DEFAULT '',
        title       text NOT NULL
    )`)
    wrap_sql(db, `CREATE TABLE tags (
        recipe_id       integer REFERENCES recipes(id) ON DELETE CASCADE NOT NULL,
        tag             text NOT NULL,
        UNIQUE(recipe_id, tag)
    )`)
    wrap_sql(db, `CREATE TABLE linked_recipes (
        src     integer REFERENCES recipes(id) ON DELETE CASCADE NOT NULL,
        dest    integer REFERENCES recipes(id) ON DELETE CASCADE NOT NULL,
        CONSTRAINT must_be_different CHECK ( src != dest ),
        UNIQUE (src, dest)
    )`)

    log.Println("complete")
}

func wrap_sql(db *sql.DB, s string) {
    _, err := db.Query(s)
    if err != nil {
        log.Println("error during:", s)
        log.Println(err)
        log.Fatal()
    }
}
<|endoftext|>


<commit_before>// Google Image Search functionality
package gis

import (
	"fmt"
	"gesture/core"
	"gesture/util"
	"math/rand"
	neturl "net/url"
	"strings"
	"time"
)

func Create(bot *core.Gobot) {
	bot.ListenFor("^gis (.*)", func(msg core.Message, matches []string) error {
		link, err := search(matches[1])
		if err == nil {
			msg.Ftfy(link)
		}
		return err
	})
}

// these structs really tie the room together, man
type gisResult struct {
	Url string
}
type gisResponse struct {
	ResponseData struct {
		Results *[]gisResult // use a pointer here b/c sometimes the results are null :(
	}
}

// Search queries google for some images, and then randomly selects one
func search(search string) (string, error) {
	searchUrl := "http://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + neturl.QueryEscape(search)
	var gisResponse gisResponse
	if err := util.UnmarshalUrl(searchUrl, &gisResponse); err != nil {
		return "", err
	}

	if gisResponse.ResponseData.Results == nil {
		return "", fmt.Errorf("No results were returned for query %s", search)
	}

	results := *gisResponse.ResponseData.Results

	if len(results) > 0 {

		// start a goroutine to determine image info for each response result
		imageUrlCh := make(chan string)
		errorsCh := make(chan error)
		for _, resultUrl := range results {
			go getImageInfo(resultUrl.Url, imageUrlCh, errorsCh)
		}

		// until a timeout is met, build a collection of urls
		totalResults := len(results)
		remainingResults := totalResults
		urls := make([]string, 0, totalResults)
		errors := make([]error, 0, totalResults)
		timeout := time.After(500 * time.Millisecond)

	SEARCH:
		for remainingResults > 0 {
			select {
			case url := <-imageUrlCh:
				urls = append(urls, url)
				remainingResults--
			case err := <-errorsCh:
				errors = append(errors, err)
				remainingResults--
			case <-timeout:
				break SEARCH
			}
		}
		if len(urls) == 0 {
			return "", fmt.Errorf("No image could be found for \"%s\"", search)
		}
		return urls[rand.Intn(len(urls))], nil

	}
	return "", fmt.Errorf("No image could be found for \"%s\"", search)
}

// getImageInfo looks at the header info for the url, and if it is an image, it sends an imageInfo on the channel
func getImageInfo(url string, ch chan<- string, failures chan<- error) {
	imageUrl, contentType, err := util.ResponseHeaderContentType(url)
	if err == nil && strings.HasPrefix(contentType, "image/") {
		url, err := ensureSuffix(imageUrl, "."+contentType[len("image/"):])
		if err != nil {
			failures <- err
		} else {
			ch <- url
		}
	} else {
		failures <- fmt.Errorf("Not an image: %s", url)
	}
}

// ensureSuffix ensures a url ends with suffixes like .jpg, .png, etc
func ensureSuffix(url, suffix string) (string, error) {
	var err error
	url, err = neturl.QueryUnescape(url)
	if err != nil {
		return "", err
	}
	lowerSuffix := strings.ToLower(suffix)
	lowerUrl := strings.ToLower(url)
	if lowerSuffix == ".jpeg" && strings.HasSuffix(lowerUrl, ".jpg") {
		return url, nil
	}
	if lowerSuffix == ".jpg" && strings.HasSuffix(lowerUrl, ".jpeg") {
		return url, nil
	}
	if strings.HasSuffix(lowerUrl, lowerSuffix) {
		return url, nil
	}
	if strings.Contains(url, "?") {
		return url + "&lol=lol" + suffix, nil
	}
	return url + "?lol=lol" + suffix, nil
}
<commit_msg>Escape gis query results<commit_after>// Google Image Search functionality
package gis

import (
	"fmt"
	"gesture/core"
	"gesture/util"
	"math/rand"
	neturl "net/url"
	"strings"
	"time"
)

func Create(bot *core.Gobot) {
	bot.ListenFor("^gis (.*)", func(msg core.Message, matches []string) error {
		link, err := search(matches[1])
		if err == nil {
			msg.Ftfy(neturl.QueryEscape(link))
		}
		return err
	})
}

// these structs really tie the room together, man
type gisResult struct {
	Url string
}
type gisResponse struct {
	ResponseData struct {
		Results *[]gisResult // use a pointer here b/c sometimes the results are null :(
	}
}

// Search queries google for some images, and then randomly selects one
func search(search string) (string, error) {
	searchUrl := "http://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + neturl.QueryEscape(search)
	var gisResponse gisResponse
	if err := util.UnmarshalUrl(searchUrl, &gisResponse); err != nil {
		return "", err
	}

	if gisResponse.ResponseData.Results == nil {
		return "", fmt.Errorf("No results were returned for query %s", search)
	}

	results := *gisResponse.ResponseData.Results

	if len(results) > 0 {

		// start a goroutine to determine image info for each response result
		imageUrlCh := make(chan string)
		errorsCh := make(chan error)
		for _, resultUrl := range results {
			go getImageInfo(resultUrl.Url, imageUrlCh, errorsCh)
		}

		// until a timeout is met, build a collection of urls
		totalResults := len(results)
		remainingResults := totalResults
		urls := make([]string, 0, totalResults)
		errors := make([]error, 0, totalResults)
		timeout := time.After(500 * time.Millisecond)

	SEARCH:
		for remainingResults > 0 {
			select {
			case url := <-imageUrlCh:
				urls = append(urls, url)
				remainingResults--
			case err := <-errorsCh:
				errors = append(errors, err)
				remainingResults--
			case <-timeout:
				break SEARCH
			}
		}
		if len(urls) == 0 {
			return "", fmt.Errorf("No image could be found for \"%s\"", search)
		}
		return urls[rand.Intn(len(urls))], nil

	}
	return "", fmt.Errorf("No image could be found for \"%s\"", search)
}

// getImageInfo looks at the header info for the url, and if it is an image, it sends an imageInfo on the channel
func getImageInfo(url string, ch chan<- string, failures chan<- error) {
	imageUrl, contentType, err := util.ResponseHeaderContentType(url)
	if err == nil && strings.HasPrefix(contentType, "image/") {
		url, err := ensureSuffix(imageUrl, "."+contentType[len("image/"):])
		if err != nil {
			failures <- err
		} else {
			ch <- url
		}
	} else {
		failures <- fmt.Errorf("Not an image: %s", url)
	}
}

// ensureSuffix ensures a url ends with suffixes like .jpg, .png, etc
func ensureSuffix(url, suffix string) (string, error) {
	var err error
	url, err = neturl.QueryUnescape(url)
	if err != nil {
		return "", err
	}
	lowerSuffix := strings.ToLower(suffix)
	lowerUrl := strings.ToLower(url)
	if lowerSuffix == ".jpeg" && strings.HasSuffix(lowerUrl, ".jpg") {
		return url, nil
	}
	if lowerSuffix == ".jpg" && strings.HasSuffix(lowerUrl, ".jpeg") {
		return url, nil
	}
	if strings.HasSuffix(lowerUrl, lowerSuffix) {
		return url, nil
	}
	if strings.Contains(url, "?") {
		return url + "&lol=lol" + suffix, nil
	}
	return url + "?lol=lol" + suffix, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package factory can set up a scheduler. This code is here instead of
// plugin/cmd/scheduler for both testability and reuse.
package factory

import (
	"math/rand"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	algorithm "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler"
	schedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api"

	"github.com/golang/glog"
)

// ConfigFactory knows how to fill out a scheduler config with its support functions.
type ConfigFactory struct {
	Client *client.Client
	// queue for pods that need scheduling
	PodQueue *cache.FIFO
	// a means to list all known scheduled pods.
	ScheduledPodLister *cache.StoreToPodLister
	// a means to list all known scheduled pods and pods assumed to have been scheduled.
	PodLister algorithm.PodLister
	// a means to list all minions
	NodeLister *cache.StoreToNodeLister
	// a means to list all services
	ServiceLister *cache.StoreToServiceLister

	modeler scheduler.SystemModeler
}

// Initializes the factory.
func NewConfigFactory(client *client.Client) *ConfigFactory {
	c := &ConfigFactory{
		Client:             client,
		PodQueue:           cache.NewFIFO(cache.MetaNamespaceKeyFunc),
		ScheduledPodLister: &cache.StoreToPodLister{cache.NewStore(cache.MetaNamespaceKeyFunc)},
		NodeLister:         &cache.StoreToNodeLister{cache.NewStore(cache.MetaNamespaceKeyFunc)},
		ServiceLister:      &cache.StoreToServiceLister{cache.NewStore(cache.MetaNamespaceKeyFunc)},
	}
	modeler := scheduler.NewSimpleModeler(&cache.StoreToPodLister{c.PodQueue}, c.ScheduledPodLister)
	c.modeler = modeler
	c.PodLister = modeler.PodLister()
	return c
}

// Create creates a scheduler with the default algorithm provider.
func (f *ConfigFactory) Create() (*scheduler.Config, error) {
	return f.CreateFromProvider(DefaultProvider)
}

// Creates a scheduler from the name of a registered algorithm provider.
func (f *ConfigFactory) CreateFromProvider(providerName string) (*scheduler.Config, error) {
	glog.V(2).Infof("creating scheduler from algorithm provider '%v'", providerName)
	provider, err := GetAlgorithmProvider(providerName)
	if err != nil {
		return nil, err
	}

	return f.CreateFromKeys(provider.FitPredicateKeys, provider.PriorityFunctionKeys)
}

// Creates a scheduler from the configuration file
func (f *ConfigFactory) CreateFromConfig(policy schedulerapi.Policy) (*scheduler.Config, error) {
	glog.V(2).Infof("creating scheduler from configuration: %v", policy)

	predicateKeys := util.NewStringSet()
	for _, predicate := range policy.Predicates {
		glog.V(2).Infof("Registering predicate: %s", predicate.Name)
		predicateKeys.Insert(RegisterCustomFitPredicate(predicate))
	}

	priorityKeys := util.NewStringSet()
	for _, priority := range policy.Priorities {
		glog.V(2).Infof("Registering priority: %s", priority.Name)
		priorityKeys.Insert(RegisterCustomPriorityFunction(priority))
	}

	return f.CreateFromKeys(predicateKeys, priorityKeys)
}

// Creates a scheduler from a set of registered fit predicate keys and priority keys.
func (f *ConfigFactory) CreateFromKeys(predicateKeys, priorityKeys util.StringSet) (*scheduler.Config, error) {
	glog.V(2).Infof("creating scheduler with fit predicates '%v' and priority functions '%v", predicateKeys, priorityKeys)
	pluginArgs := PluginFactoryArgs{
		PodLister:     f.PodLister,
		ServiceLister: f.ServiceLister,
		NodeLister:    f.NodeLister,
	}
	predicateFuncs, err := getFitPredicateFunctions(predicateKeys, pluginArgs)
	if err != nil {
		return nil, err
	}

	priorityConfigs, err := getPriorityFunctionConfigs(priorityKeys, pluginArgs)
	if err != nil {
		return nil, err
	}

	// Watch and queue pods that need scheduling.
	cache.NewReflector(f.createUnassignedPodLW(), &api.Pod{}, f.PodQueue, 0).Run()

	// Watch and cache all running pods. Scheduler needs to find all pods
	// so it knows where it's safe to place a pod. Cache this locally.
	cache.NewReflector(f.createAssignedPodLW(), &api.Pod{}, f.ScheduledPodLister.Store, 0).Run()

	// Watch minions.
	// Minions may be listed frequently, so provide a local up-to-date cache.
	if false {
		// Disable this code until minions support watches. Note when this code is enabled,
		// we need to make sure minion ListWatcher has proper FieldSelector.
		cache.NewReflector(f.createMinionLW(), &api.Node{}, f.NodeLister.Store, 0).Run()
	} else {
		cache.NewPoller(f.pollMinions, 10*time.Second, f.NodeLister.Store).Run()
	}

	// Watch and cache all service objects. Scheduler needs to find all pods
	// created by the same service, so that it can spread them correctly.
	// Cache this locally.
	cache.NewReflector(f.createServiceLW(), &api.Service{}, f.ServiceLister.Store, 0).Run()

	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	algo := algorithm.NewGenericScheduler(predicateFuncs, priorityConfigs, f.PodLister, r)

	podBackoff := podBackoff{
		perPodBackoff: map[string]*backoffEntry{},
		clock:         realClock{},

		defaultDuration: 1 * time.Second,
		maxDuration:     60 * time.Second,
	}

	return &scheduler.Config{
		Modeler:      f.modeler,
		MinionLister: f.NodeLister,
		Algorithm:    algo,
		Binder:       &binder{f.Client},
		NextPod: func() *api.Pod {
			pod := f.PodQueue.Pop().(*api.Pod)
			glog.V(2).Infof("About to try and schedule pod %v", pod.Name)
			return pod
		},
		Error:    f.makeDefaultErrorFunc(&podBackoff, f.PodQueue),
		Recorder: record.FromSource(api.EventSource{Component: "scheduler"}),
	}, nil
}

// Returns a cache.ListWatch that finds all pods that need to be
// scheduled.
func (factory *ConfigFactory) createUnassignedPodLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(factory.Client, "pods", api.NamespaceAll, labels.Set{getHostFieldLabel(factory.Client.APIVersion()): ""}.AsSelector())
}

func parseSelectorOrDie(s string) labels.Selector {
	selector, err := labels.ParseSelector(s)
	if err != nil {
		panic(err)
	}
	return selector
}

// Returns a cache.ListWatch that finds all pods that are
// already scheduled.
// TODO: return a ListerWatcher interface instead?
func (factory *ConfigFactory) createAssignedPodLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(factory.Client, "pods", api.NamespaceAll,
		parseSelectorOrDie(getHostFieldLabel(factory.Client.APIVersion())+"!="))
}

// createMinionLW returns a cache.ListWatch that gets all changes to minions.
func (factory *ConfigFactory) createMinionLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(factory.Client, "minions", api.NamespaceAll, parseSelectorOrDie(""))
}

// Lists all minions and filter out unhealthy ones, then returns
// an enumerator for cache.Poller.
func (factory *ConfigFactory) pollMinions() (cache.Enumerator, error) {
	allNodes := &api.NodeList{}
	err := factory.Client.Get().Resource("minions").Do().Into(allNodes)
	if err != nil {
		return nil, err
	}
	nodes := &api.NodeList{
		TypeMeta: allNodes.TypeMeta,
		ListMeta: allNodes.ListMeta,
	}
	for _, node := range allNodes.Items {
		conditionMap := make(map[api.NodeConditionType]*api.NodeCondition)
		for i := range node.Status.Conditions {
			cond := node.Status.Conditions[i]
			conditionMap[cond.Type] = &cond
		}
		if condition, ok := conditionMap[api.NodeSchedulable]; ok {
			if condition.Status != api.ConditionFull {
				continue
			}
		}
		if condition, ok := conditionMap[api.NodeReady]; ok {
			if condition.Status == api.ConditionFull {
				nodes.Items = append(nodes.Items, node)
			}
		} else if condition, ok := conditionMap[api.NodeReachable]; ok {
			if condition.Status == api.ConditionFull {
				nodes.Items = append(nodes.Items, node)
			}
		} else {
			// If no condition is set, we get unknown node condition. In such cases,
			// we add nodes unconditionally.
			nodes.Items = append(nodes.Items, node)
		}
	}
	return &nodeEnumerator{nodes}, nil
}

// Returns a cache.ListWatch that gets all changes to services.
func (factory *ConfigFactory) createServiceLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(factory.Client, "services", api.NamespaceAll, parseSelectorOrDie(""))
}

func (factory *ConfigFactory) makeDefaultErrorFunc(backoff *podBackoff, podQueue *cache.FIFO) func(pod *api.Pod, err error) {
	return func(pod *api.Pod, err error) {
		glog.Errorf("Error scheduling %v %v: %v; retrying", pod.Namespace, pod.Name, err)
		backoff.gc()
		// Retry asynchronously.
		// Note that this is extremely rudimentary and we need a more real error handling path.
		go func() {
			defer util.HandleCrash()
			podID := pod.Name
			podNamespace := pod.Namespace
			backoff.wait(podID)
			// Get the pod again; it may have changed/been scheduled already.
			pod = &api.Pod{}
			err := factory.Client.Get().Namespace(podNamespace).Resource("pods").Name(podID).Do().Into(pod)
			if err != nil {
				glog.Errorf("Error getting pod %v for retry: %v; abandoning", podID, err)
				return
			}
			if pod.Status.Host == "" {
				podQueue.Add(pod)
			}
		}()
	}
}

func getHostFieldLabel(apiVersion string) string {
	switch apiVersion {
	case "v1beta1", "v1beta2":
		return "DesiredState.Host"
	default:
		return "spec.host"
	}
}

// nodeEnumerator allows a cache.Poller to enumerate items in an api.NodeList
type nodeEnumerator struct {
	*api.NodeList
}

// Len returns the number of items in the node list.
func (ne *nodeEnumerator) Len() int {
	if ne.NodeList == nil {
		return 0
	}
	return len(ne.Items)
}

// Get returns the item (and ID) with the particular index.
func (ne *nodeEnumerator) Get(index int) interface{} {
	return &ne.Items[index]
}

type binder struct {
	*client.Client
}

// Bind just does a POST binding RPC.
func (b *binder) Bind(binding *api.Binding) error {
	glog.V(2).Infof("Attempting to bind %v to %v", binding.Name, binding.Target.Name)
	ctx := api.WithNamespace(api.NewContext(), binding.Namespace)
	return b.Post().Namespace(api.NamespaceValue(ctx)).Resource("bindings").Body(binding).Do().Error()
	// TODO: use Pods interface for binding once clusters are upgraded
	// return b.Pods(binding.Namespace).Bind(binding)
}

type clock interface {
	Now() time.Time
}

type realClock struct{}

func (realClock) Now() time.Time {
	return time.Now()
}

type backoffEntry struct {
	backoff    time.Duration
	lastUpdate time.Time
}

type podBackoff struct {
	perPodBackoff   map[string]*backoffEntry
	lock            sync.Mutex
	clock           clock
	defaultDuration time.Duration
	maxDuration     time.Duration
}

func (p *podBackoff) getEntry(podID string) *backoffEntry {
	p.lock.Lock()
	defer p.lock.Unlock()
	entry, ok := p.perPodBackoff[podID]
	if !ok {
		entry = &backoffEntry{backoff: p.defaultDuration}
		p.perPodBackoff[podID] = entry
	}
	entry.lastUpdate = p.clock.Now()
	return entry
}

func (p *podBackoff) getBackoff(podID string) time.Duration {
	entry := p.getEntry(podID)
	duration := entry.backoff
	entry.backoff *= 2
	if entry.backoff > p.maxDuration {
		entry.backoff = p.maxDuration
	}
	glog.V(4).Infof("Backing off %s for pod %s", duration.String(), podID)
	return duration
}

func (p *podBackoff) wait(podID string) {
	time.Sleep(p.getBackoff(podID))
}

func (p *podBackoff) gc() {
	p.lock.Lock()
	defer p.lock.Unlock()
	now := p.clock.Now()
	for podID, entry := range p.perPodBackoff {
		if now.Sub(entry.lastUpdate) > p.maxDuration {
			delete(p.perPodBackoff, podID)
		}
	}
}
<commit_msg>Fill NodeInfo in CreateFromKey method in scheduler<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package factory can set up a scheduler. This code is here instead of
// plugin/cmd/scheduler for both testability and reuse.
package factory

import (
	"math/rand"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	algorithm "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler"
	schedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api"

	"github.com/golang/glog"
)

// ConfigFactory knows how to fill out a scheduler config with its support functions.
type ConfigFactory struct {
	Client *client.Client
	// queue for pods that need scheduling
	PodQueue *cache.FIFO
	// a means to list all known scheduled pods.
	ScheduledPodLister *cache.StoreToPodLister
	// a means to list all known scheduled pods and pods assumed to have been scheduled.
	PodLister algorithm.PodLister
	// a means to list all minions
	NodeLister *cache.StoreToNodeLister
	// a means to list all services
	ServiceLister *cache.StoreToServiceLister

	modeler scheduler.SystemModeler
}

// Initializes the factory.
func NewConfigFactory(client *client.Client) *ConfigFactory {
	c := &ConfigFactory{
		Client:             client,
		PodQueue:           cache.NewFIFO(cache.MetaNamespaceKeyFunc),
		ScheduledPodLister: &cache.StoreToPodLister{cache.NewStore(cache.MetaNamespaceKeyFunc)},
		NodeLister:         &cache.StoreToNodeLister{cache.NewStore(cache.MetaNamespaceKeyFunc)},
		ServiceLister:      &cache.StoreToServiceLister{cache.NewStore(cache.MetaNamespaceKeyFunc)},
	}
	modeler := scheduler.NewSimpleModeler(&cache.StoreToPodLister{c.PodQueue}, c.ScheduledPodLister)
	c.modeler = modeler
	c.PodLister = modeler.PodLister()
	return c
}

// Create creates a scheduler with the default algorithm provider.
func (f *ConfigFactory) Create() (*scheduler.Config, error) {
	return f.CreateFromProvider(DefaultProvider)
}

// Creates a scheduler from the name of a registered algorithm provider.
func (f *ConfigFactory) CreateFromProvider(providerName string) (*scheduler.Config, error) {
	glog.V(2).Infof("creating scheduler from algorithm provider '%v'", providerName)
	provider, err := GetAlgorithmProvider(providerName)
	if err != nil {
		return nil, err
	}

	return f.CreateFromKeys(provider.FitPredicateKeys, provider.PriorityFunctionKeys)
}

// Creates a scheduler from the configuration file
func (f *ConfigFactory) CreateFromConfig(policy schedulerapi.Policy) (*scheduler.Config, error) {
	glog.V(2).Infof("creating scheduler from configuration: %v", policy)

	predicateKeys := util.NewStringSet()
	for _, predicate := range policy.Predicates {
		glog.V(2).Infof("Registering predicate: %s", predicate.Name)
		predicateKeys.Insert(RegisterCustomFitPredicate(predicate))
	}

	priorityKeys := util.NewStringSet()
	for _, priority := range policy.Priorities {
		glog.V(2).Infof("Registering priority: %s", priority.Name)
		priorityKeys.Insert(RegisterCustomPriorityFunction(priority))
	}

	return f.CreateFromKeys(predicateKeys, priorityKeys)
}

// Creates a scheduler from a set of registered fit predicate keys and priority keys.
func (f *ConfigFactory) CreateFromKeys(predicateKeys, priorityKeys util.StringSet) (*scheduler.Config, error) {
	glog.V(2).Infof("creating scheduler with fit predicates '%v' and priority functions '%v", predicateKeys, priorityKeys)
	pluginArgs := PluginFactoryArgs{
		PodLister:     f.PodLister,
		ServiceLister: f.ServiceLister,
		NodeLister:    f.NodeLister,
		NodeInfo:      f.NodeLister,
	}
	predicateFuncs, err := getFitPredicateFunctions(predicateKeys, pluginArgs)
	if err != nil {
		return nil, err
	}

	priorityConfigs, err := getPriorityFunctionConfigs(priorityKeys, pluginArgs)
	if err != nil {
		return nil, err
	}

	// Watch and queue pods that need scheduling.
	cache.NewReflector(f.createUnassignedPodLW(), &api.Pod{}, f.PodQueue, 0).Run()

	// Watch and cache all running pods. Scheduler needs to find all pods
	// so it knows where it's safe to place a pod. Cache this locally.
	cache.NewReflector(f.createAssignedPodLW(), &api.Pod{}, f.ScheduledPodLister.Store, 0).Run()

	// Watch minions.
	// Minions may be listed frequently, so provide a local up-to-date cache.
	if false {
		// Disable this code until minions support watches. Note when this code is enabled,
		// we need to make sure minion ListWatcher has proper FieldSelector.
		cache.NewReflector(f.createMinionLW(), &api.Node{}, f.NodeLister.Store, 0).Run()
	} else {
		cache.NewPoller(f.pollMinions, 10*time.Second, f.NodeLister.Store).Run()
	}

	// Watch and cache all service objects. Scheduler needs to find all pods
	// created by the same service, so that it can spread them correctly.
	// Cache this locally.
	cache.NewReflector(f.createServiceLW(), &api.Service{}, f.ServiceLister.Store, 0).Run()

	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	algo := algorithm.NewGenericScheduler(predicateFuncs, priorityConfigs, f.PodLister, r)

	podBackoff := podBackoff{
		perPodBackoff: map[string]*backoffEntry{},
		clock:         realClock{},

		defaultDuration: 1 * time.Second,
		maxDuration:     60 * time.Second,
	}

	return &scheduler.Config{
		Modeler:      f.modeler,
		MinionLister: f.NodeLister,
		Algorithm:    algo,
		Binder:       &binder{f.Client},
		NextPod: func() *api.Pod {
			pod := f.PodQueue.Pop().(*api.Pod)
			glog.V(2).Infof("About to try and schedule pod %v", pod.Name)
			return pod
		},
		Error:    f.makeDefaultErrorFunc(&podBackoff, f.PodQueue),
		Recorder: record.FromSource(api.EventSource{Component: "scheduler"}),
	}, nil
}

// Returns a cache.ListWatch that finds all pods that need to be
// scheduled.
func (factory *ConfigFactory) createUnassignedPodLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(factory.Client, "pods", api.NamespaceAll, labels.Set{getHostFieldLabel(factory.Client.APIVersion()): ""}.AsSelector())
}

func parseSelectorOrDie(s string) labels.Selector {
	selector, err := labels.ParseSelector(s)
	if err != nil {
		panic(err)
	}
	return selector
}

// Returns a cache.ListWatch that finds all pods that are
// already scheduled.
// TODO: return a ListerWatcher interface instead?
func (factory *ConfigFactory) createAssignedPodLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(factory.Client, "pods", api.NamespaceAll,
		parseSelectorOrDie(getHostFieldLabel(factory.Client.APIVersion())+"!="))
}

// createMinionLW returns a cache.ListWatch that gets all changes to minions.
func (factory *ConfigFactory) createMinionLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(factory.Client, "minions", api.NamespaceAll, parseSelectorOrDie(""))
}

// Lists all minions and filter out unhealthy ones, then returns
// an enumerator for cache.Poller.
func (factory *ConfigFactory) pollMinions() (cache.Enumerator, error) {
	allNodes := &api.NodeList{}
	err := factory.Client.Get().Resource("minions").Do().Into(allNodes)
	if err != nil {
		return nil, err
	}
	nodes := &api.NodeList{
		TypeMeta: allNodes.TypeMeta,
		ListMeta: allNodes.ListMeta,
	}
	for _, node := range allNodes.Items {
		conditionMap := make(map[api.NodeConditionType]*api.NodeCondition)
		for i := range node.Status.Conditions {
			cond := node.Status.Conditions[i]
			conditionMap[cond.Type] = &cond
		}
		if condition, ok := conditionMap[api.NodeSchedulable]; ok {
			if condition.Status != api.ConditionFull {
				continue
			}
		}
		if condition, ok := conditionMap[api.NodeReady]; ok {
			if condition.Status == api.ConditionFull {
				nodes.Items = append(nodes.Items, node)
			}
		} else if condition, ok := conditionMap[api.NodeReachable]; ok {
			if condition.Status == api.ConditionFull {
				nodes.Items = append(nodes.Items, node)
			}
		} else {
			// If no condition is set, we get unknown node condition. In such cases,
			// we add nodes unconditionally.
			nodes.Items = append(nodes.Items, node)
		}
	}
	return &nodeEnumerator{nodes}, nil
}

// Returns a cache.ListWatch that gets all changes to services.
func (factory *ConfigFactory) createServiceLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(factory.Client, "services", api.NamespaceAll, parseSelectorOrDie(""))
}

func (factory *ConfigFactory) makeDefaultErrorFunc(backoff *podBackoff, podQueue *cache.FIFO) func(pod *api.Pod, err error) {
	return func(pod *api.Pod, err error) {
		glog.Errorf("Error scheduling %v %v: %v; retrying", pod.Namespace, pod.Name, err)
		backoff.gc()
		// Retry asynchronously.
		// Note that this is extremely rudimentary and we need a more real error handling path.
		go func() {
			defer util.HandleCrash()
			podID := pod.Name
			podNamespace := pod.Namespace
			backoff.wait(podID)
			// Get the pod again; it may have changed/been scheduled already.
			pod = &api.Pod{}
			err := factory.Client.Get().Namespace(podNamespace).Resource("pods").Name(podID).Do().Into(pod)
			if err != nil {
				glog.Errorf("Error getting pod %v for retry: %v; abandoning", podID, err)
				return
			}
			if pod.Status.Host == "" {
				podQueue.Add(pod)
			}
		}()
	}
}

func getHostFieldLabel(apiVersion string) string {
	switch apiVersion {
	case "v1beta1", "v1beta2":
		return "DesiredState.Host"
	default:
		return "spec.host"
	}
}

// nodeEnumerator allows a cache.Poller to enumerate items in an api.NodeList
type nodeEnumerator struct {
	*api.NodeList
}

// Len returns the number of items in the node list.
func (ne *nodeEnumerator) Len() int {
	if ne.NodeList == nil {
		return 0
	}
	return len(ne.Items)
}

// Get returns the item (and ID) with the particular index.
func (ne *nodeEnumerator) Get(index int) interface{} {
	return &ne.Items[index]
}

type binder struct {
	*client.Client
}

// Bind just does a POST binding RPC.
func (b *binder) Bind(binding *api.Binding) error {
	glog.V(2).Infof("Attempting to bind %v to %v", binding.Name, binding.Target.Name)
	ctx := api.WithNamespace(api.NewContext(), binding.Namespace)
	return b.Post().Namespace(api.NamespaceValue(ctx)).Resource("bindings").Body(binding).Do().Error()
	// TODO: use Pods interface for binding once clusters are upgraded
	// return b.Pods(binding.Namespace).Bind(binding)
}

type clock interface {
	Now() time.Time
}

type realClock struct{}

func (realClock) Now() time.Time {
	return time.Now()
}

type backoffEntry struct {
	backoff    time.Duration
	lastUpdate time.Time
}

type podBackoff struct {
	perPodBackoff   map[string]*backoffEntry
	lock            sync.Mutex
	clock           clock
	defaultDuration time.Duration
	maxDuration     time.Duration
}

func (p *podBackoff) getEntry(podID string) *backoffEntry {
	p.lock.Lock()
	defer p.lock.Unlock()
	entry, ok := p.perPodBackoff[podID]
	if !ok {
		entry = &backoffEntry{backoff: p.defaultDuration}
		p.perPodBackoff[podID] = entry
	}
	entry.lastUpdate = p.clock.Now()
	return entry
}

func (p *podBackoff) getBackoff(podID string) time.Duration {
	entry := p.getEntry(podID)
	duration := entry.backoff
	entry.backoff *= 2
	if entry.backoff > p.maxDuration {
		entry.backoff = p.maxDuration
	}
	glog.V(4).Infof("Backing off %s for pod %s", duration.String(), podID)
	return duration
}

func (p *podBackoff) wait(podID string) {
	time.Sleep(p.getBackoff(podID))
}

func (p *podBackoff) gc() {
	p.lock.Lock()
	defer p.lock.Unlock()
	now := p.clock.Now()
	for podID, entry := range p.perPodBackoff {
		if now.Sub(entry.lastUpdate) > p.maxDuration {
			delete(p.perPodBackoff, podID)
		}
	}
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
)

// Kubernetes represents the config object for the plugin
type Kubernetes struct {
	URL string

	// Bearer Token authorization file path
	BearerToken string `toml:"bearer_token"`

	// Path to CA file
	SSLCA string `toml:"ssl_ca"`
	// Path to host cert file
	SSLCert string `toml:"ssl_cert"`
	// Path to cert key file
	SSLKey string `toml:"ssl_key"`
	// Use SSL but skip chain & host verification
	InsecureSkipVerify bool

	RoundTripper http.RoundTripper
}

var sampleConfig = `
  ## URL for the kubelet
  url = "http://1.1.1.1:10255"

  ## Use bearer token for authorization
  # bearer_token = /path/to/bearer/token

  ## Optional SSL Config
  # ssl_ca = /path/to/cafile
  # ssl_cert = /path/to/certfile
  # ssl_key = /path/to/keyfile
  ## Use SSL but skip chain & host verification
  # insecure_skip_verify = false
`

const (
	summaryEndpoint = `%s/stats/summary`
)

func init() {
	inputs.Add("kubernetes", func() telegraf.Input {
		return &Kubernetes{}
	})
}

//SampleConfig returns a sample config
func (k *Kubernetes) SampleConfig() string {
	return sampleConfig
}

//Description returns the description of this plugin
func (k *Kubernetes) Description() string {
	return "Read metrics from the kubernetes kubelet api"
}

//Gather collects kubernetes metrics from a given URL
func (k *Kubernetes) Gather(acc telegraf.Accumulator) error {
	var wg sync.WaitGroup
	wg.Add(1)
	go func(k *Kubernetes) {
		defer wg.Done()
		acc.AddError(k.gatherSummary(k.URL, acc))
	}(k)
	wg.Wait()
	return nil
}

func buildURL(endpoint string, base string) (*url.URL, error) {
	u := fmt.Sprintf(endpoint, base)
	addr, err := url.Parse(u)
	if err != nil {
		return nil, fmt.Errorf("Unable to parse address '%s': %s", u, err)
	}
	return addr, nil
}

func (k *Kubernetes) gatherSummary(baseURL string, acc telegraf.Accumulator) error {
	url := fmt.Sprintf("%s/stats/summary", baseURL)
	var req, err = http.NewRequest("GET", url, nil)
	var token []byte
	var resp *http.Response

	tlsCfg, err := internal.GetTLSConfig(k.SSLCert, k.SSLKey, k.SSLCA, k.InsecureSkipVerify)
	if err != nil {
		return err
	}

	if k.RoundTripper == nil {
		k.RoundTripper = &http.Transport{
			TLSHandshakeTimeout:   5 * time.Second,
			TLSClientConfig:       tlsCfg,
			ResponseHeaderTimeout: time.Duration(3 * time.Second),
		}
	}

	if k.BearerToken != "" {
		token, err = ioutil.ReadFile(k.BearerToken)
		if err != nil {
			return err
		}
		req.Header.Set("Authorization", "Bearer "+string(token))
	}

	resp, err = k.RoundTripper.RoundTrip(req)
	if err != nil {
		return fmt.Errorf("error making HTTP request to %s: %s", url, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("%s returned HTTP status %s", url, resp.Status)
	}

	summaryMetrics := &SummaryMetrics{}
	err = json.NewDecoder(resp.Body).Decode(summaryMetrics)
	if err != nil {
		return fmt.Errorf(`Error parsing response: %s`, err)
	}
	buildSystemContainerMetrics(summaryMetrics, acc)
	buildNodeMetrics(summaryMetrics, acc)
	buildPodMetrics(summaryMetrics, acc)
	return nil
}

func buildSystemContainerMetrics(summaryMetrics *SummaryMetrics, acc telegraf.Accumulator) {
	for _, container := range summaryMetrics.Node.SystemContainers {
		tags := map[string]string{
			"node_name":      summaryMetrics.Node.NodeName,
			"container_name": container.Name,
		}
		fields := make(map[string]interface{})
		fields["cpu_usage_nanocores"] = container.CPU.UsageNanoCores
		fields["cpu_usage_core_nanoseconds"] = container.CPU.UsageCoreNanoSeconds
		fields["memory_usage_bytes"] = container.Memory.UsageBytes
		fields["memory_working_set_bytes"] = container.Memory.WorkingSetBytes
		fields["memory_rss_bytes"] = container.Memory.RSSBytes
		fields["memory_page_faults"] = container.Memory.PageFaults
		fields["memory_major_page_faults"] = container.Memory.MajorPageFaults
		fields["rootfs_available_bytes"] = container.RootFS.AvailableBytes
		fields["rootfs_capacity_bytes"] = container.RootFS.CapacityBytes
		fields["logsfs_avaialble_bytes"] = container.LogsFS.AvailableBytes
		fields["logsfs_capacity_bytes"] = container.LogsFS.CapacityBytes
		acc.AddFields("kubernetes_system_container", fields, tags)
	}
}

func buildNodeMetrics(summaryMetrics *SummaryMetrics, acc telegraf.Accumulator) {
	tags := map[string]string{
		"node_name": summaryMetrics.Node.NodeName,
	}
	fields := make(map[string]interface{})
	fields["cpu_usage_nanocores"] = summaryMetrics.Node.CPU.UsageNanoCores
	fields["cpu_usage_core_nanoseconds"] = summaryMetrics.Node.CPU.UsageCoreNanoSeconds
	fields["memory_available_bytes"] = summaryMetrics.Node.Memory.AvailableBytes
	fields["memory_usage_bytes"] = summaryMetrics.Node.Memory.UsageBytes
	fields["memory_working_set_bytes"] = summaryMetrics.Node.Memory.WorkingSetBytes
	fields["memory_rss_bytes"] = summaryMetrics.Node.Memory.RSSBytes
	fields["memory_page_faults"] = summaryMetrics.Node.Memory.PageFaults
	fields["memory_major_page_faults"] = summaryMetrics.Node.Memory.MajorPageFaults
	fields["network_rx_bytes"] = summaryMetrics.Node.Network.RXBytes
	fields["network_rx_errors"] = summaryMetrics.Node.Network.RXErrors
	fields["network_tx_bytes"] = summaryMetrics.Node.Network.TXBytes
	fields["network_tx_errors"] = summaryMetrics.Node.Network.TXErrors
	fields["fs_available_bytes"] = summaryMetrics.Node.FileSystem.AvailableBytes
	fields["fs_capacity_bytes"] = summaryMetrics.Node.FileSystem.CapacityBytes
	fields["fs_used_bytes"] = summaryMetrics.Node.FileSystem.UsedBytes
	fields["runtime_image_fs_available_bytes"] = summaryMetrics.Node.Runtime.ImageFileSystem.AvailableBytes
	fields["runtime_image_fs_capacity_bytes"] = summaryMetrics.Node.Runtime.ImageFileSystem.CapacityBytes
	fields["runtime_image_fs_used_bytes"] = summaryMetrics.Node.Runtime.ImageFileSystem.UsedBytes
	acc.AddFields("kubernetes_node", fields, tags)
}

func buildPodMetrics(summaryMetrics *SummaryMetrics, acc telegraf.Accumulator) {
	for _, pod := range summaryMetrics.Pods {
		for _, container := range pod.Containers {
			tags := map[string]string{
				"node_name":      summaryMetrics.Node.NodeName,
				"namespace":      pod.PodRef.Namespace,
				"container_name": container.Name,
				"pod_name":       pod.PodRef.Name,
			}
			fields := make(map[string]interface{})
			fields["cpu_usage_nanocores"] = container.CPU.UsageNanoCores
			fields["cpu_usage_core_nanoseconds"] = container.CPU.UsageCoreNanoSeconds
			fields["memory_usage_bytes"] = container.Memory.UsageBytes
			fields["memory_working_set_bytes"] = container.Memory.WorkingSetBytes
			fields["memory_rss_bytes"] = container.Memory.RSSBytes
			fields["memory_page_faults"] = container.Memory.PageFaults
			fields["memory_major_page_faults"] = container.Memory.MajorPageFaults
			fields["rootfs_available_bytes"] = container.RootFS.AvailableBytes
			fields["rootfs_capacity_bytes"] = container.RootFS.CapacityBytes
			fields["rootfs_used_bytes"] = container.RootFS.UsedBytes
			fields["logsfs_avaialble_bytes"] = container.LogsFS.AvailableBytes
			fields["logsfs_capacity_bytes"] = container.LogsFS.CapacityBytes
			fields["logsfs_used_bytes"] = container.LogsFS.UsedBytes
			acc.AddFields("kubernetes_pod_container", fields, tags)
		}

		for _, volume := range pod.Volumes {
			tags := map[string]string{
				"node_name":   summaryMetrics.Node.NodeName,
				"pod_name":    pod.PodRef.Name,
				"namespace":   pod.PodRef.Namespace,
				"volume_name": volume.Name,
			}
			fields := make(map[string]interface{})
			fields["available_bytes"] = volume.AvailableBytes
			fields["capacity_bytes"] = volume.CapacityBytes
			fields["used_bytes"] = volume.UsedBytes
			acc.AddFields("kubernetes_pod_volume", fields, tags)
		}

		tags := map[string]string{
			"node_name": summaryMetrics.Node.NodeName,
			"pod_name":  pod.PodRef.Name,
			"namespace": pod.PodRef.Namespace,
		}
		fields := make(map[string]interface{})
		fields["rx_bytes"] = pod.Network.RXBytes
		fields["rx_errors"] = pod.Network.RXErrors
		fields["tx_bytes"] = pod.Network.TXBytes
		fields["tx_errors"] = pod.Network.TXErrors
		acc.AddFields("kubernetes_pod_network", fields, tags)
	}
}
<commit_msg>Add timeout option for kubernetes (#3211)<commit_after>package kubernetes

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
)

// Kubernetes represents the config object for the plugin
type Kubernetes struct {
	URL string

	// Bearer Token authorization file path
	BearerToken string `toml:"bearer_token"`

	// Path to CA file
	SSLCA string `toml:"ssl_ca"`
	// Path to host cert file
	SSLCert string `toml:"ssl_cert"`
	// Path to cert key file
	SSLKey string `toml:"ssl_key"`
	// Use SSL but skip chain & host verification
	InsecureSkipVerify bool

	// HTTP Timeout specified as a string - 3s, 1m, 1h
	ResponseTimeout internal.Duration

	RoundTripper http.RoundTripper
}

var sampleConfig = `
  ## URL for the kubelet
  url = "http://1.1.1.1:10255"

  ## Use bearer token for authorization
  # bearer_token = /path/to/bearer/token

  ## Set response_timeout (default 5 seconds)
  # response_timeout = "5s"

  ## Optional SSL Config
  # ssl_ca = /path/to/cafile
  # ssl_cert = /path/to/certfile
  # ssl_key = /path/to/keyfile
  ## Use SSL but skip chain & host verification
  # insecure_skip_verify = false
`

const (
	summaryEndpoint = `%s/stats/summary`
)

func init() {
	inputs.Add("kubernetes", func() telegraf.Input {
		return &Kubernetes{}
	})
}

//SampleConfig returns a sample config
func (k *Kubernetes) SampleConfig() string {
	return sampleConfig
}

//Description returns the description of this plugin
func (k *Kubernetes) Description() string {
	return "Read metrics from the kubernetes kubelet api"
}

//Gather collects kubernetes metrics from a given URL
func (k *Kubernetes) Gather(acc telegraf.Accumulator) error {
	var wg sync.WaitGroup
	wg.Add(1)
	go func(k *Kubernetes) {
		defer wg.Done()
		acc.AddError(k.gatherSummary(k.URL, acc))
	}(k)
	wg.Wait()
	return nil
}

func buildURL(endpoint string, base string) (*url.URL, error) {
	u := fmt.Sprintf(endpoint, base)
	addr, err := url.Parse(u)
	if err != nil {
		return nil, fmt.Errorf("Unable to parse address '%s': %s", u, err)
	}
	return addr, nil
}

func (k *Kubernetes) gatherSummary(baseURL string, acc telegraf.Accumulator) error {
	url := fmt.Sprintf("%s/stats/summary", baseURL)
	var req, err = http.NewRequest("GET", url, nil)
	var token []byte
	var resp *http.Response

	tlsCfg, err := internal.GetTLSConfig(k.SSLCert, k.SSLKey, k.SSLCA, k.InsecureSkipVerify)
	if err != nil {
		return err
	}

	if k.RoundTripper == nil {
		// Set default values
		if k.ResponseTimeout.Duration < time.Second {
			k.ResponseTimeout.Duration = time.Second * 5
		}
		if err != nil {
			return err
		}
		k.RoundTripper = &http.Transport{
			TLSHandshakeTimeout:   5 * time.Second,
			TLSClientConfig:       tlsCfg,
			ResponseHeaderTimeout: k.ResponseTimeout.Duration,
		}
	}

	if k.BearerToken != "" {
		token, err = ioutil.ReadFile(k.BearerToken)
		if err != nil {
			return err
		}
		req.Header.Set("Authorization", "Bearer "+string(token))
	}

	resp, err = k.RoundTripper.RoundTrip(req)
	if err != nil {
		return fmt.Errorf("error making HTTP request to %s: %s", url, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("%s returned HTTP status %s", url, resp.Status)
	}

	summaryMetrics := &SummaryMetrics{}
	err = json.NewDecoder(resp.Body).Decode(summaryMetrics)
	if err != nil {
		return fmt.Errorf(`Error parsing response: %s`, err)
	}
	buildSystemContainerMetrics(summaryMetrics, acc)
	buildNodeMetrics(summaryMetrics, acc)
	buildPodMetrics(summaryMetrics, acc)
	return nil
}

func buildSystemContainerMetrics(summaryMetrics *SummaryMetrics, acc telegraf.Accumulator) {
	for _, container := range summaryMetrics.Node.SystemContainers {
		tags := map[string]string{
			"node_name":      summaryMetrics.Node.NodeName,
			"container_name": container.Name,
		}
		fields := make(map[string]interface{})
		fields["cpu_usage_nanocores"] = container.CPU.UsageNanoCores
		fields["cpu_usage_core_nanoseconds"] = container.CPU.UsageCoreNanoSeconds
		fields["memory_usage_bytes"] = container.Memory.UsageBytes
		fields["memory_working_set_bytes"] = container.Memory.WorkingSetBytes
		fields["memory_rss_bytes"] = container.Memory.RSSBytes
		fields["memory_page_faults"] = container.Memory.PageFaults
		fields["memory_major_page_faults"] = container.Memory.MajorPageFaults
		fields["rootfs_available_bytes"] = container.RootFS.AvailableBytes
		fields["rootfs_capacity_bytes"] = container.RootFS.CapacityBytes
		fields["logsfs_avaialble_bytes"] = container.LogsFS.AvailableBytes
		fields["logsfs_capacity_bytes"] = container.LogsFS.CapacityBytes
		acc.AddFields("kubernetes_system_container", fields, tags)
	}
}

func buildNodeMetrics(summaryMetrics *SummaryMetrics, acc telegraf.Accumulator) {
	tags := map[string]string{
		"node_name": summaryMetrics.Node.NodeName,
	}
	fields := make(map[string]interface{})
	fields["cpu_usage_nanocores"] = summaryMetrics.Node.CPU.UsageNanoCores
	fields["cpu_usage_core_nanoseconds"] = summaryMetrics.Node.CPU.UsageCoreNanoSeconds
	fields["memory_available_bytes"] = summaryMetrics.Node.Memory.AvailableBytes
	fields["memory_usage_bytes"] = summaryMetrics.Node.Memory.UsageBytes
	fields["memory_working_set_bytes"] = summaryMetrics.Node.Memory.WorkingSetBytes
	fields["memory_rss_bytes"] = summaryMetrics.Node.Memory.RSSBytes
	fields["memory_page_faults"] = summaryMetrics.Node.Memory.PageFaults
	fields["memory_major_page_faults"] = summaryMetrics.Node.Memory.MajorPageFaults
	fields["network_rx_bytes"] = summaryMetrics.Node.Network.RXBytes
	fields["network_rx_errors"] = summaryMetrics.Node.Network.RXErrors
	fields["network_tx_bytes"] = summaryMetrics.Node.Network.TXBytes
	fields["network_tx_errors"] = summaryMetrics.Node.Network.TXErrors
	fields["fs_available_bytes"] = summaryMetrics.Node.FileSystem.AvailableBytes
	fields["fs_capacity_bytes"] = summaryMetrics.Node.FileSystem.CapacityBytes
	fields["fs_used_bytes"] = summaryMetrics.Node.FileSystem.UsedBytes
	fields["runtime_image_fs_available_bytes"] = summaryMetrics.Node.Runtime.ImageFileSystem.AvailableBytes
	fields["runtime_image_fs_capacity_bytes"] = summaryMetrics.Node.Runtime.ImageFileSystem.CapacityBytes
	fields["runtime_image_fs_used_bytes"] = summaryMetrics.Node.Runtime.ImageFileSystem.UsedBytes
	acc.AddFields("kubernetes_node", fields, tags)
}

func buildPodMetrics(summaryMetrics *SummaryMetrics, acc telegraf.Accumulator) {
	for _, pod := range summaryMetrics.Pods {
		for _, container := range pod.Containers {
			tags := map[string]string{
				"node_name":      summaryMetrics.Node.NodeName,
				"namespace":      pod.PodRef.Namespace,
				"container_name": container.Name,
				"pod_name":       pod.PodRef.Name,
			}
			fields := make(map[string]interface{})
			fields["cpu_usage_nanocores"] = container.CPU.UsageNanoCores
			fields["cpu_usage_core_nanoseconds"] = container.CPU.UsageCoreNanoSeconds
			fields["memory_usage_bytes"] = container.Memory.UsageBytes
			fields["memory_working_set_bytes"] = container.Memory.WorkingSetBytes
			fields["memory_rss_bytes"] = container.Memory.RSSBytes
			fields["memory_page_faults"] = container.Memory.PageFaults
			fields["memory_major_page_faults"] = container.Memory.MajorPageFaults
			fields["rootfs_available_bytes"] = container.RootFS.AvailableBytes
			fields["rootfs_capacity_bytes"] = container.RootFS.CapacityBytes
			fields["rootfs_used_bytes"] = container.RootFS.UsedBytes
			fields["logsfs_avaialble_bytes"] = container.LogsFS.AvailableBytes
			fields["logsfs_capacity_bytes"] = container.LogsFS.CapacityBytes
			fields["logsfs_used_bytes"] = container.LogsFS.UsedBytes
			acc.AddFields("kubernetes_pod_container", fields, tags)
		}

		for _, volume := range pod.Volumes {
			tags := map[string]string{
				"node_name":   summaryMetrics.Node.NodeName,
				"pod_name":    pod.PodRef.Name,
				"namespace":   pod.PodRef.Namespace,
				"volume_name": volume.Name,
			}
			fields := make(map[string]interface{})
			fields["available_bytes"] = volume.AvailableBytes
			fields["capacity_bytes"] = volume.CapacityBytes
			fields["used_bytes"] = volume.UsedBytes
			acc.AddFields("kubernetes_pod_volume", fields, tags)
		}

		tags := map[string]string{
			"node_name": summaryMetrics.Node.NodeName,
			"pod_name":  pod.PodRef.Name,
			"namespace": pod.PodRef.Namespace,
		}
		fields := make(map[string]interface{})
		fields["rx_bytes"] = pod.Network.RXBytes
		fields["rx_errors"] = pod.Network.RXErrors
		fields["tx_bytes"] = pod.Network.TXBytes
		fields["tx_errors"] = pod.Network.TXErrors
		acc.AddFields("kubernetes_pod_network", fields, tags)
	}
}
<|endoftext|>


<commit_before>package Golf

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"testing"
)

func makeTestHTTPRequest(body io.Reader, method, url string) *http.Request {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil
	}
	return req
}

func makeTestContext(method, url string) *Context {
	r := makeTestHTTPRequest(nil, method, url)
	w := httptest.NewRecorder()
	app := New()
	return NewContext(r, w, app)
}

func TestContextCreate(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/foo/bar/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	if ctx == nil {
		t.Errorf("Can not create context.")
	}
}

func TestCookieSet(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/foo/bar/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	ctx.SetCookie("foo", "bar", 0)
	ctx.Send()
	if w.HeaderMap.Get("Set-Cookie") != `foo=bar; Path=/` {
		t.Errorf("Cookie test failed: %q != %q", w.HeaderMap.Get("Set-Cookie"), `foo=bar; Path=/`)
	}
}

func TestCookieSetWithExpire(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/foo/bar/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	ctx.SetCookie("foo", "bar", 3600)
	ctx.Send()
	rawCookie := w.HeaderMap.Get("Set-Cookie")
	rawRequest := fmt.Sprintf("GET / HTTP/1.0\r\nCookie: %s\r\n\r\n", rawCookie)
	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(rawRequest)))
	if err == nil {
		cookies := req.Cookies()
		cookie := cookies[3]
		if cookie.Value != "3600" {
			t.Errorf("Can not set cookie with expiration correctly.")
		}
	}
}

func TestTemplateLoader(t *testing.T) {
	ctx := makeTestContext("GET", "/")
	ctx.Loader("admin")
	if ctx.templateLoader != "admin" {
		t.Errorf("Can not set templateLoader for Context.")
	}
}

func TestQuery(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/search?q=foo&p=bar")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	q, err := ctx.Query("q")
	if err != nil {
		t.Errorf("Can not retrieve a query.")
	} else {
		if q != "foo" {
			t.Errorf("Can not retrieve the correct query `q`.")
		}
	}
	p, err := ctx.Query("p")
	if err != nil {
		t.Errorf("Can not retrieve a query.")
	} else {
		if p != "bar" {
			t.Errorf("Can not retrieve the correct query `p`.")
		}
	}
}

func TestQueries(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/search?myarray=value1&myarray=value2&myarray=value3")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	q, err := ctx.Query("myarray", 2)
	if err != nil {
		t.Errorf("Can not retrieve a query.")
	}
	if q != "value3" {
		t.Errorf("Can not correctly retrive a query.")
	}
}

func TestQueryNotFound(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/search?myarray=value1&myarray=value2&myarray=value3")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	q, err := ctx.Query("query")
	if err == nil || q != "" {
		t.Errorf("Can not raise error when query not found.")
	}
}

func makeNewContext(method, url string) *Context {
	r := makeTestHTTPRequest(nil, method, url)
	w := httptest.NewRecorder()
	app := New()
	return NewContext(r, w, app)
}

func TestRedirection(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	ctx.Redirect("/foo")
	ctx.Send()
	if w.HeaderMap.Get("Location") != `/foo` {
		t.Errorf("Can not perform a 301 redirection.")
	}
}

func TestWrite(t *testing.T) {
	ctx := makeNewContext("GET", "/foo")
	ctx.Write("hello world")
	if !reflect.DeepEqual(ctx.Body, []byte("hello world")) {
		t.Errorf("Context.Write failed.")
	}
}

func TestAbort(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	ctx.Abort(500)
	if w.Code != 500 || !ctx.IsSent {
		t.Errorf("Can not abort a context.")
	}
}

func TestRenderFromString(t *testing.T) {
	cases := []struct {
		src    string
		args   map[string]interface{}
		output string
	}{
		{
			"foo {{.Title}} bar",
			map[string]interface{}{"Title": "Hello World"},
			"foo Hello World bar",
		},
	}

	for _, c := range cases {
		r := makeTestHTTPRequest(nil, "GET", "/")
		w := httptest.NewRecorder()
		app := New()
		ctx := NewContext(r, w, app)
		ctx.RenderFromString(c.src, c.args)
		ctx.Send()
		if w.Body.String() != c.output {
			t.Errorf("Can not render from string correctly: %v != %v", w.Body.String(), c.output)
		}
	}
}

func TestJSON(t *testing.T) {
	cases := []struct {
		input  map[string]interface{}
		output string
	}{
		{
			map[string]interface{}{"status": "success", "code": 200},
			`{"code":200,"status":"success"}`,
		},
	}

	for _, c := range cases {
		r := makeTestHTTPRequest(nil, "GET", "/")
		w := httptest.NewRecorder()
		app := New()
		ctx := NewContext(r, w, app)
		ctx.JSON(c.input)
		ctx.Send()
		if w.Body.String() != c.output {
			t.Errorf("Can not return JSON correctly: %v != %v", w.Body.String(), c.output)
		}
		if w.HeaderMap.Get("Content-Type") != `application/json` {
			t.Errorf("Content-Type didn't set properly when calling Context.JSON.")
		}
	}
}
<commit_msg>[test] Improved test cases for sessions and XSRF protection.<commit_after>package Golf

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"testing"
)

func makeTestHTTPRequest(body io.Reader, method, url string) *http.Request {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil
	}
	return req
}

func makeTestContext(method, url string) (*Context, *Application, *http.Request, *httptest.ResponseRecorder) {
	r := makeTestHTTPRequest(nil, method, url)
	w := httptest.NewRecorder()
	app := New()
	return NewContext(r, w, app), app, r, w
}

func TestContextCreate(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/foo/bar/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	if ctx == nil {
		t.Errorf("Could not create context.")
	}
}

func TestCookieSet(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/foo/bar/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	ctx.SetCookie("foo", "bar", 0)
	ctx.Send()
	if w.HeaderMap.Get("Set-Cookie") != `foo=bar; Path=/` {
		t.Errorf("Cookie test failed: %q != %q", w.HeaderMap.Get("Set-Cookie"), `foo=bar; Path=/`)
	}
}

func TestCookieSetWithExpire(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/foo/bar/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	ctx.SetCookie("foo", "bar", 3600)
	ctx.Send()
	rawCookie := w.HeaderMap.Get("Set-Cookie")
	rawRequest := fmt.Sprintf("GET / HTTP/1.0\r\nCookie: %s\r\n\r\n", rawCookie)
	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(rawRequest)))
	if err == nil {
		cookies := req.Cookies()
		cookie := cookies[3]
		if cookie.Value != "3600" {
			t.Errorf("Could not set cookie with expiration correctly.")
		}
	}
}

func TestSessionWithInvalidSid(t *testing.T) {
	ctx, app, r, w := makeTestContext("GET", "/foo/bar/")
	app.SessionManager = NewMemorySessionManager()
	ctx.retrieveSession()
	if ctx.Session == nil {
		t.Errorf("Could not retrieve session when there is no sid.")
	}
	r.AddCookie(&http.Cookie{Name: "sid", Value: "abc"})
	ctx = NewContext(r, w, app)
	ctx.retrieveSession()
	if ctx.Session == nil {
		t.Errorf("Could not retrieve session when sid is not valid.")
	}
}

func TestSession(t *testing.T) {
	_, app, r, w := makeTestContext("GET", "/foo/")
	app.SessionManager = NewMemorySessionManager()
	app.MiddlewareChain = NewChain(SessionMiddleware)
	var (
		firstSid string
	)
	app.Get("/foo/", func(ctx *Context) {
		if ctx.Session == nil {
			t.Errorf("Could not retrieve session.")
		}
		firstSid = ctx.Session.SessionID()
		ctx.Write("success")
	})
	app.ServeHTTP(w, r)
	app.Get("/bar/", func(ctx *Context) {
		if ctx.Session.SessionID() != firstSid {
			t.Errorf("Could not retrieve correct session from the same user.")
		}
		ctx.Write("success")
	})
	_, _, r, w = makeTestContext("GET", "/bar/")
	r.AddCookie(&http.Cookie{Name: "sid", Value: firstSid})
	app.ServeHTTP(w, r)
}

func TestXSRFProtectionWithoutCookie(t *testing.T) {
	ctx, app, _, _ := makeTestContext("GET", "/foo/bar/")
	app.Config.Set("xsrf_cookies", true)
	if ctx.getRawXSRFToken() != "" {
		t.Errorf("Should not retrieve raw XSRF token when there is no `_xsrf` cookie set.")
	}
}

func TestXSRFProtectionDisabled(t *testing.T) {
	_, app, r, w := makeTestContext("POST", "/foo/bar/")
	app.MiddlewareChain = NewChain(XSRFProtectionMiddleware)
	app.Post("/foo/bar/", func(ctx *Context) {
		ctx.Write("success")
	})
	app.ServeHTTP(w, r)

	if w.Code == 403 {
		t.Errorf("Should not check XSRF")
	}

	if w.Body.String() != "success" {
		t.Errorf("Did not returned expected result.")
	}
}

func TestTemplateLoader(t *testing.T) {
	ctx, _, _, _ := makeTestContext("GET", "/")
	ctx.Loader("admin")
	if ctx.templateLoader != "admin" {
		t.Errorf("Could not set templateLoader for Context.")
	}
}

func TestQuery(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/search?q=foo&p=bar")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	q, err := ctx.Query("q")
	if err != nil {
		t.Errorf("Could not retrieve a query.")
	} else {
		if q != "foo" {
			t.Errorf("Could not retrieve the correct query `q`.")
		}
	}
	p, err := ctx.Query("p")
	if err != nil {
		t.Errorf("Could not retrieve a query.")
	} else {
		if p != "bar" {
			t.Errorf("Could not retrieve the correct query `p`.")
		}
	}
}

func TestQueries(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/search?myarray=value1&myarray=value2&myarray=value3")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	q, err := ctx.Query("myarray", 2)
	if err != nil {
		t.Errorf("Could not retrieve a query.")
	}
	if q != "value3" {
		t.Errorf("Could not correctly retrive a query.")
	}
}

func TestQueryNotFound(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/search?myarray=value1&myarray=value2&myarray=value3")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	q, err := ctx.Query("query")
	if err == nil || q != "" {
		t.Errorf("Could not raise error when query not found.")
	}
}

func makeNewContext(method, url string) *Context {
	r := makeTestHTTPRequest(nil, method, url)
	w := httptest.NewRecorder()
	app := New()
	return NewContext(r, w, app)
}

func TestRedirection(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	ctx.Redirect("/foo")
	ctx.Send()
	if w.HeaderMap.Get("Location") != `/foo` {
		t.Errorf("Could not perform a 301 redirection.")
	}
}

func TestWrite(t *testing.T) {
	ctx := makeNewContext("GET", "/foo")
	ctx.Write("hello world")
	if !reflect.DeepEqual(ctx.Body, []byte("hello world")) {
		t.Errorf("Context.Write failed.")
	}
}

func TestAbort(t *testing.T) {
	r := makeTestHTTPRequest(nil, "GET", "/")
	w := httptest.NewRecorder()
	app := New()
	ctx := NewContext(r, w, app)
	ctx.Abort(500)
	if w.Code != 500 || !ctx.IsSent {
		t.Errorf("Could not abort a context.")
	}
}

func TestRenderFromString(t *testing.T) {
	cases := []struct {
		src    string
		args   map[string]interface{}
		output string
	}{
		{
			"foo {{.Title}} bar",
			map[string]interface{}{"Title": "Hello World"},
			"foo Hello World bar",
		},
	}

	for _, c := range cases {
		r := makeTestHTTPRequest(nil, "GET", "/")
		w := httptest.NewRecorder()
		app := New()
		ctx := NewContext(r, w, app)
		ctx.RenderFromString(c.src, c.args)
		ctx.Send()
		if w.Body.String() != c.output {
			t.Errorf("Could not render from string correctly: %v != %v", w.Body.String(), c.output)
		}
	}
}

func TestJSON(t *testing.T) {
	cases := []struct {
		input  map[string]interface{}
		output string
	}{
		{
			map[string]interface{}{"status": "success", "code": 200},
			`{"code":200,"status":"success"}`,
		},
	}

	for _, c := range cases {
		r := makeTestHTTPRequest(nil, "GET", "/")
		w := httptest.NewRecorder()
		app := New()
		ctx := NewContext(r, w, app)
		ctx.JSON(c.input)
		ctx.Send()
		if w.Body.String() != c.output {
			t.Errorf("Could not return JSON correctly: %v != %v", w.Body.String(), c.output)
		}
		if w.HeaderMap.Get("Content-Type") != `application/json` {
			t.Errorf("Content-Type didn't set properly when calling Context.JSON.")
		}
	}
}
<|endoftext|>


<commit_before>package espsdk

import (
	"encoding/json"
)

type Contribution struct {
	CameraShotDate       string   `json:"camera_shot_date,omitempty"`
	CollectionCode       string   `json:"collection_code,omitempty"`
	ContentProviderName  string   `json:"content_provider_name,omitempty"`
	ContentProviderTitle string   `json:"content_provider_title,omitempty"`
	CountryOfShoot       string   `json:"country_of_shoot,omitempty"`
	CreditLine           string   `json:"credit_line,omitempty"`
	ExternalFileLocation string   `json:"external_file_location,omitempty"`
	FileName             string   `json:"file_name,omitempty"`
	FilePath             string   `json:"file_path,omitempty"`
	Headline             string   `json:"headline,omitempty"`
	Id                   string   `json:"id,omitempty"`
	IptcCategory         string   `json:"iptc_category,omitempty"`
	IptcSubjects         []string `json:"iptc_subjects,omitempty"`
	MimeType             string   `json:"mime_type,omitempty"`
	ParentSource         string   `json:"parent_source,omitempty"`
	RecordedDate         string   `json:"recorded_date,omitempty"`
	RiskCategory         string   `json:"risk_category,omitempty"`
	ShotSpeed            string   `json:"shot_speed,omitempty"`
	SiteDestination      []string `json:"site_destination,omitempty"`
	Source               string   `json:"source,omitempty"`
	SubmittedToReviewAt  string   `json:"submitted_to_review_at,omitempty"`
	UploadBucket         string   `json:"upload_bucket,omitempty"`
	UploadId             string   `json:"upload_id,omitempty"`
}

func (c Contribution) Marshal() ([]byte, error) {
	return json.MarshalIndent(c, "", "  ")
}
<commit_msg>enable Contribution update (PUT) and DELETE<commit_after>package espsdk

import (
	"encoding/json"
)

type Contribution struct {
	CameraShotDate       string   `json:"camera_shot_date,omitempty"`
	CollectionCode       string   `json:"collection_code,omitempty"`
	ContentProviderName  string   `json:"content_provider_name,omitempty"`
	ContentProviderTitle string   `json:"content_provider_title,omitempty"`
	CountryOfShoot       string   `json:"country_of_shoot,omitempty"`
	CreditLine           string   `json:"credit_line,omitempty"`
	ExternalFileLocation string   `json:"external_file_location,omitempty"`
	FileName             string   `json:"file_name,omitempty"`
	FilePath             string   `json:"file_path,omitempty"`
	Headline             string   `json:"headline,omitempty"`
	Id                   string   `json:"id,omitempty"`
	IptcCategory         string   `json:"iptc_category,omitempty"`
	IptcSubjects         []string `json:"iptc_subjects,omitempty"`
	MimeType             string   `json:"mime_type,omitempty"`
	ParentSource         string   `json:"parent_source,omitempty"`
	RecordedDate         string   `json:"recorded_date,omitempty"`
	RiskCategory         string   `json:"risk_category,omitempty"`
	ShotSpeed            string   `json:"shot_speed,omitempty"`
	SiteDestination      []string `json:"site_destination,omitempty"`
	Source               string   `json:"source,omitempty"`
	SubmittedToReviewAt  string   `json:"submitted_to_review_at,omitempty"`
	UploadBucket         string   `json:"upload_bucket,omitempty"`
	UploadId             string   `json:"upload_id,omitempty"`
}

func (c Contribution) Marshal() ([]byte, error) {
	return json.MarshalIndent(c, "", "  ")
}

type ContributionUpdate struct {
	Contribution Contribution `json:"contribution"`
}

type ContributionDeleteParams struct {
	SubmissionBatchID string `json:"submission_batch_id,omitempty"`
	ContributionID    string `json:"id,omitempty"`
}

func (c ContributionDeleteParams) Marshal() ([]byte, error) {
	return json.MarshalIndent(c, "", "  ")
}

func (c ContributionUpdate) Marshal() ([]byte, error) {
	return json.MarshalIndent(c, "", "  ")
}
<|endoftext|>