<commit_before>package jobs

import (
	"container/list"
	"context"
	"fmt"
	"sync"
	"sync/atomic"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/prefixer"
	multierror "github.com/hashicorp/go-multierror"
)

type (
	// memQueue is a queue in-memory implementation of the Queue interface.
	memQueue struct {
		MaxCapacity int
		Jobs        chan *Job

		list *list.List
		run  bool
		jmu  sync.RWMutex
	}

	// memBroker is an in-memory broker implementation of the Broker interface.
	memBroker struct {
		queues       map[string]*memQueue
		workers      []*Worker
		workersTypes []string
		running      uint32
		closed       chan struct{}
	}
)

// newMemQueue creates and a new in-memory queue.
func newMemQueue(workerType string) *memQueue {
	return &memQueue{
		list: list.New(),
		Jobs: make(chan *Job),
	}
}

// Enqueue into the queue
func (q *memQueue) Enqueue(job *Job) error {
	q.jmu.Lock()
	defer q.jmu.Unlock()
	q.list.PushBack(job.Clone())
	if !q.run {
		q.run = true
		go q.send()
	}
	return nil
}

func (q *memQueue) send() {
	for {
		q.jmu.Lock()
		e := q.list.Front()
		if e == nil {
			q.run = false
			q.jmu.Unlock()
			return
		}
		q.list.Remove(e)
		q.jmu.Unlock()
		q.Jobs <- e.Value.(*Job)
	}
}

// Len returns the length of the queue
func (q *memQueue) Len() int {
	q.jmu.RLock()
	defer q.jmu.RUnlock()
	return q.list.Len()
}

// NewMemBroker creates a new in-memory broker system.
//
// The in-memory implementation of the job system has the specifity that
// workers are actually launched by the broker at its creation.
func NewMemBroker() Broker {
	return &memBroker{
		queues: make(map[string]*memQueue),
		closed: make(chan struct{}),
	}
}

func (b *memBroker) StartWorkers(ws WorkersList) error {
	if !atomic.CompareAndSwapUint32(&b.running, 0, 1) {
		return ErrClosed
	}

	for _, conf := range ws {
		b.workersTypes = append(b.workersTypes, conf.WorkerType)
		if conf.Concurrency <= 0 {
			continue
		}
		q := newMemQueue(conf.WorkerType)
		w := NewWorker(conf)
		b.queues[conf.WorkerType] = q
		b.workers = append(b.workers, w)
		if err := w.Start(q.Jobs); err != nil {
			return err
		}
	}

	if len(b.workers) > 0 {
		joblog.Infof("Started in-memory broker for %d workers type", len(b.workers))
	}

	// XXX for retro-compat
	if slots := config.GetConfig().Jobs.NbWorkers; len(b.workers) > 0 && slots > 0 {
		joblog.Warnf("Limiting the number of total concurrent workers to %d", slots)
		joblog.Warnf("Please update your configuration file to avoid a hard limit")
		setNbSlots(slots)
	}

	return nil
}

func (b *memBroker) ShutdownWorkers(ctx context.Context) error {
	if !atomic.CompareAndSwapUint32(&b.running, 1, 0) {
		return ErrClosed
	}
	if len(b.workers) == 0 {
		return nil
	}
	fmt.Print("  shutting down in-memory broker...")
	errs := make(chan error)
	for _, w := range b.workers {
		go func(w *Worker) { errs <- w.Shutdown(ctx) }(w)
	}
	var errm error
	for i := 0; i < len(b.workers); i++ {
		if err := <-errs; err != nil {
			errm = multierror.Append(errm, err)
		}
	}
	if errm != nil {
		fmt.Println("failed:", errm)
	} else {
		fmt.Println("ok.")
	}
	return errm
}

// PushJob will produce a new Job with the given options and enqueue the job in
// the proper queue.
func (b *memBroker) PushJob(db prefixer.Prefixer, req *JobRequest) (*Job, error) {
	if atomic.LoadUint32(&b.running) == 0 {
		return nil, ErrClosed
	}

	workerType := req.WorkerType
	var worker *Worker
	for _, w := range b.workers {
		if w.Type == workerType {
			worker = w
			break
		}
	}
	if worker == nil {
		return nil, ErrUnknownWorker
	}
	if worker.Conf.AdminOnly && !req.Admin {
		return nil, ErrUnknownWorker
	}
	if worker.Conf.BeforeHook != nil {
		if ok, err := worker.Conf.BeforeHook(req); !ok || err != nil {
			return nil, err
		}
	}

	job := NewJob(db, req)
	if err := job.Create(); err != nil {
		return nil, err
	}
	q := b.queues[workerType]
	if err := q.Enqueue(job); err != nil {
		return nil, err
	}
	return job, nil
}

// WorkerQueueLen returns the size of the number of elements in queue of the
// specified worker type.
func (b *memBroker) WorkerQueueLen(workerType string) (int, error) {
	q, ok := b.queues[workerType]
	if !ok {
		return 0, ErrUnknownWorker
	}
	return q.Len(), nil
}

func (b *memBroker) WorkersTypes() []string {
	return b.workersTypes
}

var (
	_ Broker = &memBroker{}
)
<commit_msg>Fix a panic on shutdown with the mem broker<commit_after>package jobs

import (
	"container/list"
	"context"
	"fmt"
	"sync"
	"sync/atomic"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/prefixer"
	multierror "github.com/hashicorp/go-multierror"
)

type (
	// memQueue is a queue in-memory implementation of the Queue interface.
	memQueue struct {
		MaxCapacity int
		Jobs        chan *Job
		closed      chan struct{}

		list *list.List
		run  bool
		jmu  sync.RWMutex
	}

	// memBroker is an in-memory broker implementation of the Broker interface.
	memBroker struct {
		queues       map[string]*memQueue
		workers      []*Worker
		workersTypes []string
		running      uint32
	}
)

// newMemQueue creates and a new in-memory queue.
func newMemQueue(workerType string) *memQueue {
	return &memQueue{
		list:   list.New(),
		Jobs:   make(chan *Job),
		closed: make(chan struct{}),
	}
}

// Enqueue into the queue
func (q *memQueue) Enqueue(job *Job) error {
	q.jmu.Lock()
	defer q.jmu.Unlock()
	q.list.PushBack(job.Clone())
	if !q.run {
		q.run = true
		go q.send()
	}
	return nil
}

func (q *memQueue) send() {
	for {
		q.jmu.Lock()
		e := q.list.Front()
		if e == nil || !q.run {
			q.run = false
			q.jmu.Unlock()
			return
		}
		q.list.Remove(e)
		q.jmu.Unlock()
		select {
		case <-q.closed:
			return
		case q.Jobs <- e.Value.(*Job):
		}
	}
}

func (q *memQueue) close() {
	q.jmu.Lock()
	defer q.jmu.Unlock()
	if !q.run {
		return
	}
	q.run = false
	go func() { q.closed <- struct{}{} }()
}

// Len returns the length of the queue
func (q *memQueue) Len() int {
	q.jmu.RLock()
	defer q.jmu.RUnlock()
	return q.list.Len()
}

// NewMemBroker creates a new in-memory broker system.
//
// The in-memory implementation of the job system has the specifity that
// workers are actually launched by the broker at its creation.
func NewMemBroker() Broker {
	return &memBroker{
		queues: make(map[string]*memQueue),
	}
}

func (b *memBroker) StartWorkers(ws WorkersList) error {
	if !atomic.CompareAndSwapUint32(&b.running, 0, 1) {
		return ErrClosed
	}

	for _, conf := range ws {
		b.workersTypes = append(b.workersTypes, conf.WorkerType)
		if conf.Concurrency <= 0 {
			continue
		}
		q := newMemQueue(conf.WorkerType)
		w := NewWorker(conf)
		b.queues[conf.WorkerType] = q
		b.workers = append(b.workers, w)
		if err := w.Start(q.Jobs); err != nil {
			return err
		}
	}

	if len(b.workers) > 0 {
		joblog.Infof("Started in-memory broker for %d workers type", len(b.workers))
	}

	// XXX for retro-compat
	if slots := config.GetConfig().Jobs.NbWorkers; len(b.workers) > 0 && slots > 0 {
		joblog.Warnf("Limiting the number of total concurrent workers to %d", slots)
		joblog.Warnf("Please update your configuration file to avoid a hard limit")
		setNbSlots(slots)
	}

	return nil
}

func (b *memBroker) ShutdownWorkers(ctx context.Context) error {
	if !atomic.CompareAndSwapUint32(&b.running, 1, 0) {
		return ErrClosed
	}
	if len(b.workers) == 0 {
		return nil
	}

	fmt.Print("  shutting down in-memory broker...")

	for _, q := range b.queues {
		q.close()
	}

	errs := make(chan error)
	for _, w := range b.workers {
		go func(w *Worker) { errs <- w.Shutdown(ctx) }(w)
	}
	var errm error
	for i := 0; i < len(b.workers); i++ {
		if err := <-errs; err != nil {
			errm = multierror.Append(errm, err)
		}
	}

	if errm != nil {
		fmt.Println("failed:", errm)
	} else {
		fmt.Println("ok.")
	}
	return errm
}

// PushJob will produce a new Job with the given options and enqueue the job in
// the proper queue.
func (b *memBroker) PushJob(db prefixer.Prefixer, req *JobRequest) (*Job, error) {
	if atomic.LoadUint32(&b.running) == 0 {
		return nil, ErrClosed
	}

	workerType := req.WorkerType
	var worker *Worker
	for _, w := range b.workers {
		if w.Type == workerType {
			worker = w
			break
		}
	}
	if worker == nil {
		return nil, ErrUnknownWorker
	}
	if worker.Conf.AdminOnly && !req.Admin {
		return nil, ErrUnknownWorker
	}
	if worker.Conf.BeforeHook != nil {
		if ok, err := worker.Conf.BeforeHook(req); !ok || err != nil {
			return nil, err
		}
	}

	job := NewJob(db, req)
	if err := job.Create(); err != nil {
		return nil, err
	}
	q := b.queues[workerType]
	if err := q.Enqueue(job); err != nil {
		return nil, err
	}
	return job, nil
}

// WorkerQueueLen returns the size of the number of elements in queue of the
// specified worker type.
func (b *memBroker) WorkerQueueLen(workerType string) (int, error) {
	q, ok := b.queues[workerType]
	if !ok {
		return 0, ErrUnknownWorker
	}
	return q.Len(), nil
}

func (b *memBroker) WorkersTypes() []string {
	return b.workersTypes
}

var (
	_ Broker = &memBroker{}
)
<|endoftext|>


<commit_before>package knative

import (
	"fmt"
	"os"
	"time"

	log "github.com/sirupsen/logrus"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	"knative.dev/serving/pkg/apis/networking"
	networkingv1alpha1 "knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	networkingClientSet "knative.dev/serving/pkg/client/clientset/versioned/typed/networking/v1alpha1"
	servingClientSet "knative.dev/serving/pkg/client/clientset/versioned/typed/serving/v1alpha1"
	"knative.dev/serving/pkg/reconciler"
)

const (
	internalServiceName     = "kourier-internal"
	externalServiceName     = "kourier-external"
	namespaceEnv            = "KOURIER_NAMESPACE"
	kourierIngressClassName = "kourier.ingress.networking.knative.dev"
)

type KNativeClient struct {
	ServingClient    *servingClientSet.ServingV1alpha1Client
	NetworkingClient *networkingClientSet.NetworkingV1alpha1Client
	KourierNamespace string
}

func NewKnativeClient(config *rest.Config) KNativeClient {
	kourierNamespace := os.Getenv(namespaceEnv)
	if kourierNamespace == "" {
		log.Info("Env KOURIER_NAMESPACE empty, using default: \"knative-serving\"")
		kourierNamespace = "knative-serving"
	}

	servingClient, err := servingClientSet.NewForConfig(config)
	if err != nil {
		panic(err)
	}
	networkingClient, err := networkingClientSet.NewForConfig(config)
	if err != nil {
		panic(err)
	}
	return KNativeClient{ServingClient: servingClient, NetworkingClient: networkingClient, KourierNamespace: kourierNamespace}
}

func (kNativeClient *KNativeClient) Services(namespace string) (*v1alpha1.ServiceList, error) {
	return kNativeClient.ServingClient.Services(namespace).List(v1.ListOptions{})
}

// IngressAccessors returns both the Ingresses and ClusterIngresses that have
// the Kourier ingressClass in the Annotations
func (kNativeClient *KNativeClient) IngressAccessors() ([]networkingv1alpha1.IngressAccessor, error) {
	ingresses, err := kNativeClient.ingresses()
	if err != nil {
		return nil, err
	}

	clusterIngresses, err := kNativeClient.clusterIngresses()
	if err != nil {
		return nil, err
	}

	var ingressList []networkingv1alpha1.IngressAccessor

	for i := range ingresses {
		ingressList = append(ingressList, networkingv1alpha1.IngressAccessor(&ingresses[i]))
	}

	for i := range clusterIngresses {
		ingressList = append(ingressList, networkingv1alpha1.IngressAccessor(&clusterIngresses[i]))
	}

	return filterByIngressClass(ingressList), nil
}

// Pushes an event to the "events" queue received when theres a change in a ClusterIngress is added/deleted/updated.
func (kNativeClient *KNativeClient) WatchChangesInClusterIngress(namespace string, eventsQueue *workqueue.Type, stopChan <-chan struct{}) {

	restClient := kNativeClient.NetworkingClient.RESTClient()

	watchlist := cache.NewListWatchFromClient(restClient, "clusteringresses", namespace,
		fields.Everything())

	_, controller := cache.NewInformer(
		watchlist,
		&networkingv1alpha1.ClusterIngress{},
		time.Second*30, //TODO: Review resync time and adjust.
		cache.FilteringResourceEventHandler{
			FilterFunc: kourierIngressClassNameFilterFunc(),
			Handler: cache.ResourceEventHandlerFuncs{
				AddFunc: func(obj interface{}) {
					eventsQueue.Add(struct{}{})
				},

				DeleteFunc: func(obj interface{}) {
					eventsQueue.Add(struct{}{})
				},

				UpdateFunc: func(oldObj, newObj interface{}) {
					if oldObj != newObj {
						eventsQueue.Add(struct{}{})
					}
				},
			},
		},
	)

	controller.Run(stopChan)
}

// Pushes an event to the "events" queue received when theres a change in a Ingress is added/deleted/updated.
func (kNativeClient *KNativeClient) WatchChangesInIngress(namespace string, eventsQueue *workqueue.Type, stopChan <-chan struct{}) {

	restClient := kNativeClient.NetworkingClient.RESTClient()

	watchlist := cache.NewListWatchFromClient(restClient, "ingresses", namespace,
		fields.Everything())

	_, controller := cache.NewInformer(
		watchlist,
		&networkingv1alpha1.Ingress{},
		time.Second*30, //TODO: Review resync time and adjust.
		cache.FilteringResourceEventHandler{
			FilterFunc: kourierIngressClassNameFilterFunc(),
			Handler: cache.ResourceEventHandlerFuncs{
				AddFunc: func(obj interface{}) {
					eventsQueue.Add(struct{}{})
				},

				DeleteFunc: func(obj interface{}) {
					eventsQueue.Add(struct{}{})
				},

				UpdateFunc: func(oldObj, newObj interface{}) {
					if oldObj != newObj {
						eventsQueue.Add(struct{}{})
					}
				},
			},
		},
	)

	controller.Run(stopChan)
}

func (kNativeClient *KNativeClient) MarkIngressReady(ingress networkingv1alpha1.IngressAccessor) error {
	// TODO: Improve. Currently once we go trough the generation of the envoy cache, we mark the objects as Ready,
	//  but that is not exactly true, it can take a while until envoy exposes the routes. Is there a way to get a "callback" from envoy?
	var err error
	status := ingress.GetStatus()
	if ingress.GetGeneration() != status.ObservedGeneration || !ingress.GetStatus().IsReady() {

		internalDomain := internalServiceName + "." + kNativeClient.KourierNamespace + ".svc.cluster.local"
		externalDomain := externalServiceName + "." + kNativeClient.KourierNamespace + ".svc.cluster.local"

		status.InitializeConditions()
		status.MarkLoadBalancerReady(
			[]networkingv1alpha1.LoadBalancerIngressStatus{
				{
					DomainInternal: externalDomain,
				},
			},
			[]networkingv1alpha1.LoadBalancerIngressStatus{
				{
					DomainInternal: externalDomain,
				},
			},
			[]networkingv1alpha1.LoadBalancerIngressStatus{
				{
					DomainInternal: internalDomain,
				},
			})
		status.MarkNetworkConfigured()
		status.ObservedGeneration = ingress.GetGeneration()
		status.ObservedGeneration = ingress.GetGeneration()
		ingress.SetStatus(*status)

		// Handle both types of ingresses
		switch ingress.(type) {
		case *networkingv1alpha1.ClusterIngress:
			in := ingress.(*networkingv1alpha1.ClusterIngress)
			_, err = kNativeClient.NetworkingClient.ClusterIngresses().UpdateStatus(in)
			return err
		case *networkingv1alpha1.Ingress:
			in := ingress.(*networkingv1alpha1.Ingress)
			_, err = kNativeClient.NetworkingClient.Ingresses(ingress.GetNamespace()).UpdateStatus(in)
			return err
		default:
			return fmt.Errorf("can't update object, not Ingress or ClusterIngress")
		}
	}
	return nil
}

func kourierIngressClassNameFilterFunc() func(interface{}) bool {
	return reconciler.AnnotationFilterFunc(
		networking.IngressClassAnnotationKey,
		kourierIngressClassName,
		true,
	)
}

func (kNativeClient *KNativeClient) clusterIngresses() ([]networkingv1alpha1.ClusterIngress, error) {
	list, err := kNativeClient.NetworkingClient.ClusterIngresses().List(v1.ListOptions{})

	if err != nil {
		return nil, err
	}

	return list.Items, err
}

func (kNativeClient *KNativeClient) ingresses() ([]networkingv1alpha1.Ingress, error) {
	list, err := kNativeClient.NetworkingClient.Ingresses(v1.NamespaceAll).List(v1.ListOptions{})

	if err != nil {
		return nil, err
	}

	return list.Items, err
}

func filterByIngressClass(ingressAccessors []networkingv1alpha1.IngressAccessor) []networkingv1alpha1.IngressAccessor {
	var res []networkingv1alpha1.IngressAccessor

	for _, ingressAccessor := range ingressAccessors {
		ingressClass := ingressClass(ingressAccessor)

		if ingressClass == kourierIngressClassName {
			res = append(res, ingressAccessor)
		}
	}

	return res
}

func ingressClass(ingressAccessor networkingv1alpha1.IngressAccessor) string {
	return ingressAccessor.GetObjectMeta().GetAnnotations()[networking.IngressClassAnnotationKey]
}
<commit_msg>Use internal/external domain for status based on visibility<commit_after>package knative

import (
	"fmt"
	"os"
	"time"

	log "github.com/sirupsen/logrus"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	"knative.dev/serving/pkg/apis/networking"
	networkingv1alpha1 "knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	networkingClientSet "knative.dev/serving/pkg/client/clientset/versioned/typed/networking/v1alpha1"
	servingClientSet "knative.dev/serving/pkg/client/clientset/versioned/typed/serving/v1alpha1"
	"knative.dev/serving/pkg/reconciler"
)

const (
	internalServiceName     = "kourier-internal"
	externalServiceName     = "kourier-external"
	namespaceEnv            = "KOURIER_NAMESPACE"
	kourierIngressClassName = "kourier.ingress.networking.knative.dev"
)

type KNativeClient struct {
	ServingClient    *servingClientSet.ServingV1alpha1Client
	NetworkingClient *networkingClientSet.NetworkingV1alpha1Client
	KourierNamespace string
}

func NewKnativeClient(config *rest.Config) KNativeClient {
	kourierNamespace := os.Getenv(namespaceEnv)
	if kourierNamespace == "" {
		log.Info("Env KOURIER_NAMESPACE empty, using default: \"knative-serving\"")
		kourierNamespace = "knative-serving"
	}

	servingClient, err := servingClientSet.NewForConfig(config)
	if err != nil {
		panic(err)
	}
	networkingClient, err := networkingClientSet.NewForConfig(config)
	if err != nil {
		panic(err)
	}
	return KNativeClient{ServingClient: servingClient, NetworkingClient: networkingClient, KourierNamespace: kourierNamespace}
}

func (kNativeClient *KNativeClient) Services(namespace string) (*v1alpha1.ServiceList, error) {
	return kNativeClient.ServingClient.Services(namespace).List(v1.ListOptions{})
}

// IngressAccessors returns both the Ingresses and ClusterIngresses that have
// the Kourier ingressClass in the Annotations
func (kNativeClient *KNativeClient) IngressAccessors() ([]networkingv1alpha1.IngressAccessor, error) {
	ingresses, err := kNativeClient.ingresses()
	if err != nil {
		return nil, err
	}

	clusterIngresses, err := kNativeClient.clusterIngresses()
	if err != nil {
		return nil, err
	}

	var ingressList []networkingv1alpha1.IngressAccessor

	for i := range ingresses {
		ingressList = append(ingressList, networkingv1alpha1.IngressAccessor(&ingresses[i]))
	}

	for i := range clusterIngresses {
		ingressList = append(ingressList, networkingv1alpha1.IngressAccessor(&clusterIngresses[i]))
	}

	return filterByIngressClass(ingressList), nil
}

// Pushes an event to the "events" queue received when theres a change in a ClusterIngress is added/deleted/updated.
func (kNativeClient *KNativeClient) WatchChangesInClusterIngress(namespace string, eventsQueue *workqueue.Type, stopChan <-chan struct{}) {

	restClient := kNativeClient.NetworkingClient.RESTClient()

	watchlist := cache.NewListWatchFromClient(restClient, "clusteringresses", namespace,
		fields.Everything())

	_, controller := cache.NewInformer(
		watchlist,
		&networkingv1alpha1.ClusterIngress{},
		time.Second*30, //TODO: Review resync time and adjust.
		cache.FilteringResourceEventHandler{
			FilterFunc: kourierIngressClassNameFilterFunc(),
			Handler: cache.ResourceEventHandlerFuncs{
				AddFunc: func(obj interface{}) {
					eventsQueue.Add(struct{}{})
				},

				DeleteFunc: func(obj interface{}) {
					eventsQueue.Add(struct{}{})
				},

				UpdateFunc: func(oldObj, newObj interface{}) {
					if oldObj != newObj {
						eventsQueue.Add(struct{}{})
					}
				},
			},
		},
	)

	controller.Run(stopChan)
}

// Pushes an event to the "events" queue received when theres a change in a Ingress is added/deleted/updated.
func (kNativeClient *KNativeClient) WatchChangesInIngress(namespace string, eventsQueue *workqueue.Type, stopChan <-chan struct{}) {

	restClient := kNativeClient.NetworkingClient.RESTClient()

	watchlist := cache.NewListWatchFromClient(restClient, "ingresses", namespace,
		fields.Everything())

	_, controller := cache.NewInformer(
		watchlist,
		&networkingv1alpha1.Ingress{},
		time.Second*30, //TODO: Review resync time and adjust.
		cache.FilteringResourceEventHandler{
			FilterFunc: kourierIngressClassNameFilterFunc(),
			Handler: cache.ResourceEventHandlerFuncs{
				AddFunc: func(obj interface{}) {
					eventsQueue.Add(struct{}{})
				},

				DeleteFunc: func(obj interface{}) {
					eventsQueue.Add(struct{}{})
				},

				UpdateFunc: func(oldObj, newObj interface{}) {
					if oldObj != newObj {
						eventsQueue.Add(struct{}{})
					}
				},
			},
		},
	)

	controller.Run(stopChan)
}

func (kNativeClient *KNativeClient) MarkIngressReady(ingress networkingv1alpha1.IngressAccessor) error {
	// TODO: Improve. Currently once we go trough the generation of the envoy cache, we mark the objects as Ready,
	//  but that is not exactly true, it can take a while until envoy exposes the routes. Is there a way to get a "callback" from envoy?
	var err error
	status := ingress.GetStatus()
	if ingress.GetGeneration() != status.ObservedGeneration || !ingress.GetStatus().IsReady() {

		internalDomain := internalServiceName + "." + kNativeClient.KourierNamespace + ".svc.cluster.local"
		externalDomain := externalServiceName + "." + kNativeClient.KourierNamespace + ".svc.cluster.local"

		status.InitializeConditions()
		var domain string

		if ingress.GetSpec().Visibility == networkingv1alpha1.IngressVisibilityClusterLocal {
			domain = internalDomain
		} else {
			domain = externalDomain
		}

		status.MarkLoadBalancerReady(
			[]networkingv1alpha1.LoadBalancerIngressStatus{
				{
					DomainInternal: domain,
				},
			},
			[]networkingv1alpha1.LoadBalancerIngressStatus{
				{
					DomainInternal: externalDomain,
				},
			},
			[]networkingv1alpha1.LoadBalancerIngressStatus{
				{
					DomainInternal: internalDomain,
				},
			})
		status.MarkNetworkConfigured()
		status.ObservedGeneration = ingress.GetGeneration()
		status.ObservedGeneration = ingress.GetGeneration()
		ingress.SetStatus(*status)

		// Handle both types of ingresses
		switch ingress.(type) {
		case *networkingv1alpha1.ClusterIngress:
			in := ingress.(*networkingv1alpha1.ClusterIngress)
			_, err = kNativeClient.NetworkingClient.ClusterIngresses().UpdateStatus(in)
			return err
		case *networkingv1alpha1.Ingress:
			in := ingress.(*networkingv1alpha1.Ingress)
			_, err = kNativeClient.NetworkingClient.Ingresses(ingress.GetNamespace()).UpdateStatus(in)
			return err
		default:
			return fmt.Errorf("can't update object, not Ingress or ClusterIngress")
		}
	}
	return nil
}

func kourierIngressClassNameFilterFunc() func(interface{}) bool {
	return reconciler.AnnotationFilterFunc(
		networking.IngressClassAnnotationKey,
		kourierIngressClassName,
		true,
	)
}

func (kNativeClient *KNativeClient) clusterIngresses() ([]networkingv1alpha1.ClusterIngress, error) {
	list, err := kNativeClient.NetworkingClient.ClusterIngresses().List(v1.ListOptions{})

	if err != nil {
		return nil, err
	}

	return list.Items, err
}

func (kNativeClient *KNativeClient) ingresses() ([]networkingv1alpha1.Ingress, error) {
	list, err := kNativeClient.NetworkingClient.Ingresses(v1.NamespaceAll).List(v1.ListOptions{})

	if err != nil {
		return nil, err
	}

	return list.Items, err
}

func filterByIngressClass(ingressAccessors []networkingv1alpha1.IngressAccessor) []networkingv1alpha1.IngressAccessor {
	var res []networkingv1alpha1.IngressAccessor

	for _, ingressAccessor := range ingressAccessors {
		ingressClass := ingressClass(ingressAccessor)

		if ingressClass == kourierIngressClassName {
			res = append(res, ingressAccessor)
		}
	}

	return res
}

func ingressClass(ingressAccessor networkingv1alpha1.IngressAccessor) string {
	return ingressAccessor.GetObjectMeta().GetAnnotations()[networking.IngressClassAnnotationKey]
}
<|endoftext|>


<commit_before>package metrics

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/alecthomas/units"
	"github.com/matt-deboer/kuill/pkg/auth"
	"github.com/matt-deboer/kuill/pkg/clients"
	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Provider gathers metrics from the underlying kubernetes cluster
type Provider struct {
	summary     *Summaries
	mutex       sync.RWMutex
	kubeClients *clients.KubeClients
	killSwitch  chan struct{}
}

// NewMetricsProvider returns a Provider capable of returning metrics for the cluster
func NewMetricsProvider(kubeClients *clients.KubeClients) (*Provider, error) {

	m := &Provider{
		kubeClients: kubeClients,
		killSwitch:  make(chan struct{}, 1),
	}

	m.summary = m.summarize()

	go func() {
		for {
			select {
			case _ = <-m.killSwitch:
				return
			default:
				time.Sleep(15 * time.Second)

				summary := m.summarize()

				m.mutex.Lock()
				m.summary = summary
				m.mutex.Unlock()
			}
		}
	}()

	return m, nil
}

// Close shuts down the metrics provider's background worker(s)
func (m *Provider) Close() {
	m.killSwitch <- struct{}{}
}

// GetMetrics returns a summarized metrics bundle for the entire cluster
func (m *Provider) GetMetrics(w http.ResponseWriter, r *http.Request, context auth.Context) {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	data, err := json.Marshal(m.summary)
	if err != nil {
		log.Error(err)
		http.Error(w, "Failed to marshall metrics", http.StatusInternalServerError)
	} else {
		w.Header().Set("Content-Type", "application/json")
		w.Write(data)
	}
}

func (m *Provider) summarize() *Summaries {

	summary := &Summaries{
		Namespace: make(map[string]*Summary),
		Node:      make(map[string]*Summary),
		Pod:       make(map[string]*Summary),
	}
	aggregates := make(map[string]uint64)
	namespaces := make(map[string]bool)

	for _, node := range m.listNodes() {
		nodeName := node.ObjectMeta.Name

		nodeSummary := m.readNodeSummary(fmt.Sprintf("%s/api/v1/proxy/nodes/%s:10255/stats/summary",
			m.kubeClients.BaseURL, nodeName))
		if nodeSummary != nil {

			convertedSummary, convertedByNs := convertSummary(nodeSummary, node, summary)
			// aggregate at the cluster level
			aggregates["Cluster:totalMillicores"] += convertedSummary.CPU.Total
			aggregates["Cluster:memTotalBytes"] += convertedSummary.Memory.Total
			aggregates["Cluster:usageNanoCores"] += nodeSummary.Node.CPU.UsageNanoCores
			aggregates["Cluster:memUsageBytes"] += nodeSummary.Node.Memory.UsageBytes
			aggregates["Cluster:networkTxBytes"] += nodeSummary.Node.Network.TxBytes
			aggregates["Cluster:networkRxBytes"] += nodeSummary.Node.Network.RxBytes
			aggregates["Cluster:networkSeconds"] += uint64(nodeSummary.Node.Network.Time.Sub(nodeSummary.Node.StartTime).Seconds())
			aggregates["Cluster:fsCapacityBytes"] += nodeSummary.Node.Fs.CapacityBytes
			aggregates["Cluster:fsUsedBytes"] += nodeSummary.Node.Fs.UsedBytes

			// aggregate at the namespace level
			for ns, nsStat := range convertedByNs {
				aggregates["Namespace:"+ns+":fsUsedBytes"] += nsStat.Disk.Usage
				aggregates["Namespace:"+ns+":fsCapacityBytes"] += nsStat.Disk.Total
			}

			for _, podStats := range nodeSummary.Pods {
				ns := podStats.PodRef.Namespace
				aggregates["Namespace:"+ns+":pods"]++
				aggregates["Cluster:pods"]++
				convertedSummary.Pods.Usage++
				namespaces[ns] = true
				for _, volStats := range podStats.VolumeStats {
					aggregates["Cluster:volCapacityBytes"] += volStats.CapacityBytes
					aggregates["Cluster:volUsedBytes"] += volStats.UsedBytes
					aggregates["Namespace:"+ns+":volCapacityBytes"] += volStats.CapacityBytes
					aggregates["Namespace:"+ns+":volUsedBytes"] += volStats.UsedBytes
				}
				for _, c := range podStats.Containers {
					aggregates["Namespace:"+ns+":containers"]++
					aggregates["Cluster:containers"]++
					convertedSummary.Containers.Usage++
					if c.CPU != nil {
						aggregates["Namespace:"+ns+":usageCoreNanoSeconds"] += c.CPU.UsageCoreNanoSeconds
						aggregates["Namespace:"+ns+":usageNanoCores"] += c.CPU.UsageNanoCores
					}
					if c.Memory != nil {
						aggregates["Namespace:"+ns+":memAvailableBytes"] += c.Memory.AvailableBytes
						aggregates["Namespace:"+ns+":memUsageBytes"] += c.Memory.UsageBytes
						aggregates["Namespace:"+ns+":memRSSBytes"] += c.Memory.RSSBytes
						aggregates["Namespace:"+ns+":memWorkingSetBytes"] += c.Memory.WorkingSetBytes
					}
				}
				if podStats.Network != nil {
					aggregates["Namespace:"+ns+":networkTxBytes"] += podStats.Network.TxBytes
					aggregates["Namespace:"+ns+":networkRxBytes"] += podStats.Network.RxBytes
					aggregates["Namespace:"+ns+":networkSeconds"] += uint64(podStats.Network.Time.Unix() - podStats.StartTime.Unix())
				}
			}

			summary.Node[node.ObjectMeta.Name] = convertedSummary
		}
	}

	summary.Cluster = makeSummary("Cluster", aggregates)
	for ns := range namespaces {
		prefix := "Namespace:" + ns
		nsSummary := makeSummary(prefix, aggregates)

		// To be replaced by resource quota for the NS, should one exist
		nsSummary.CPU.Total = summary.Cluster.CPU.Total
		nsSummary.CPU.Ratio = safeDivideFloat(float64(nsSummary.CPU.Usage), float64(nsSummary.CPU.Total))
		nsSummary.Memory.Total = summary.Cluster.Memory.Total
		nsSummary.Memory.Ratio = safeDivideFloat(float64(nsSummary.Memory.Usage), float64(nsSummary.Memory.Total))

		summary.Namespace[ns] = nsSummary
	}

	return summary
}

func safeGet(ref *uint64) uint64 {
	if ref != nil {
		return *ref
	}
	return 0
}

func safeDivideInt(dividend, divisor uint64) uint64 {
	if divisor == 0 {
		return 0
	}
	return dividend / divisor
}

func safeDivideFloat(dividend, divisor float64) float64 {
	if divisor == 0.0 {
		return 0.0
	}
	return dividend / divisor
}

func convertSummary(summary *KubeletStatsSummary, node v1.Node, summaries *Summaries) (*Summary, map[string]*Summary) {

	summaryByNs := make(map[string]*Summary)
	networkSeconds := uint64(summary.Node.Network.Time.Unix() - summary.Node.StartTime.Unix())
	volCapacityBytes := uint64(0)
	volUsageBytes := uint64(0)
	for _, podStats := range summary.Pods {

		podSummary := convertPodSummary(podStats)
		podKey := fmt.Sprintf("%s/%s", podStats.PodRef.Namespace, podStats.PodRef.Name)
		summaries.Pod[podKey] = podSummary
		nsSummary, ok := summaryByNs[podStats.PodRef.Namespace]
		if !ok {
			nsSummary = &Summary{Disk: &SummaryStat{Units: "bytes"}}
			summaryByNs[podStats.PodRef.Namespace] = nsSummary
		}
		nsSummary.Disk.Usage += podSummary.Disk.Usage
		nsSummary.Disk.Total += podSummary.Disk.Total

		volUsageBytes += podSummary.Volumes.Usage
		volCapacityBytes += podSummary.Volumes.Total
	}

	totalCPUCores, err := strconv.Atoi(node.Status.Allocatable.Cpu().String())
	if err != nil {
		log.Errorf("Failed to parse allocatable.cpu; %v", err)
	}
	allocatableBytesString := node.Status.Allocatable.Memory().String()
	if strings.HasSuffix(allocatableBytesString, "i") {
		allocatableBytesString += "B"
	}
	totalMemoryBytes, err := units.ParseStrictBytes(allocatableBytesString)
	if err != nil {
		log.Errorf("Failed to parse allocatable.memory; %v", err)
	}

	if summary.Node.CPU == nil {
		summary.Node.CPU = &CPUStats{}
	}

	if summary.Node.Memory == nil {
		summary.Node.Memory = &MemoryStats{}
	}

	if summary.Node.Fs == nil {
		summary.Node.Fs = &FsStats{}
	}

	return &Summary{
		CPU: newSummaryStat(
			summary.Node.CPU.UsageNanoCores/1000000,
			uint64(totalCPUCores*1000),
			"millicores"),
		Memory: newSummaryStat(
			summary.Node.Memory.UsageBytes,
			uint64(totalMemoryBytes),
			"bytes"),
		Disk: newSummaryStat(
			summary.Node.Fs.UsedBytes,
			summary.Node.Fs.CapacityBytes,
			"bytes"),
		Volumes: newSummaryStat(
			volUsageBytes,
			volCapacityBytes,
			"bytes"),
		NetRx: newSummaryStat(
			safeDivideInt(summary.Node.Network.RxBytes, networkSeconds),
			1,
			"bytes/sec"),
		NetTx: newSummaryStat(
			safeDivideInt(summary.Node.Network.TxBytes, networkSeconds),
			1,
			"bytes/sec"),
		Pods: &SummaryStat{
			Usage: 0,
		},
		Containers: &SummaryStat{
			Usage: 0,
		},
	}, summaryByNs
}

func safeDivide(dividend, divisor uint64) uint64 {
	if divisor == 0 {
		return 0
	}
	return dividend / divisor
}

func convertPodSummary(pod PodStats) *Summary {

	networkSeconds := uint64(pod.Network.Time.Unix() - pod.StartTime.Unix())
	summary := &Summary{
		CPU:     &SummaryStat{Units: "millicores"},
		Memory:  &SummaryStat{Units: "bytes"},
		Volumes: &SummaryStat{Units: "bytes"},
		Disk:    &SummaryStat{Units: "bytes"},
		NetRx: newSummaryStat(
			safeDivideInt(pod.Network.RxBytes, networkSeconds),
			1,
			"bytes/sec"),
		NetTx: newSummaryStat(
			safeDivideInt(pod.Network.TxBytes, networkSeconds),
			1,
			"bytes/sec"),
		Pods: &SummaryStat{
			Usage: 1,
		},
		Containers: &SummaryStat{
			Usage: uint64(len(pod.Containers)),
		},
	}

	for _, volStats := range pod.VolumeStats {
		summary.Volumes.Usage += volStats.CapacityBytes
		summary.Volumes.Total += volStats.UsedBytes
	}

	for _, container := range pod.Containers {
		if container.CPU != nil {
			summary.CPU.Usage += container.CPU.UsageNanoCores / 1000000
		}
		if container.Memory != nil {
			summary.Memory.Usage += container.Memory.UsageBytes
		}
		if container.Rootfs != nil {
			summary.Disk.Usage += container.Rootfs.UsedBytes
		}
	}
	return summary
}

func makeSummary(prefix string, aggregates map[string]uint64) *Summary {

	usageNanoCores := aggregates[prefix+":usageNanoCores"]
	totalMillicores := aggregates[prefix+":totalMillicores"]
	memUsageBytes := aggregates[prefix+":memUsageBytes"]
	memTotalBytes := aggregates[prefix+":memTotalBytes"]
	networkTxBytes := aggregates[prefix+":networkTxBytes"]
	networkRxBytes := aggregates[prefix+":networkRxBytes"]
	// We create a fake timespan here with the current time as the endpoint;
	// the important part is that the duration is correct
	networkSeconds := aggregates[prefix+":networkSeconds"]
	if networkSeconds == 0 {
		networkSeconds = math.MaxUint64
	}

	fsCapacityBytes := aggregates[prefix+":fsCapacityBytes"]
	fsUsedBytes := aggregates[prefix+":fsUsedBytes"]
	volCapacityBytes := aggregates[prefix+":volCapacityBytes"]
	volUsedBytes := aggregates[prefix+":volUsedBytes"]
	podCount := aggregates[prefix+":pods"]
	containerCount := aggregates[prefix+":containers"]

	return &Summary{

		CPU: newSummaryStat(
			usageNanoCores/1000000,
			totalMillicores,
			"millicores"),
		Memory: newSummaryStat(
			memUsageBytes,
			memTotalBytes,
			"bytes"),
		Disk: newSummaryStat(
			fsUsedBytes,
			fsCapacityBytes,
			"bytes"),
		Volumes: newSummaryStat(
			volUsedBytes,
			volCapacityBytes,
			"bytes"),
		NetRx: newSummaryStat(
			safeDivideInt(networkRxBytes, networkSeconds),
			1,
			"bytes/sec"),
		NetTx: newSummaryStat(
			safeDivideInt(networkTxBytes, networkSeconds),
			1,
			"bytes/sec"),
		Pods: &SummaryStat{
			Usage: podCount,
		},
		Containers: &SummaryStat{
			Usage: containerCount,
		},
	}
}

func (m *Provider) readNodeSummary(path string) *KubeletStatsSummary {

	req, err := http.NewRequest("GET", fmt.Sprintf("%s", path), nil)
	if err != nil {
		log.Errorf("Failed to create request for path %s: %v", path, err)
		return nil
	}
	if len(m.kubeClients.BearerToken) > 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(m.kubeClients.BearerToken)))
	}

	resp, err := m.kubeClients.HTTP.Do(req)
	if err == nil {
		if resp.StatusCode != http.StatusOK {
			log.Errorf("Failed to read node summary response from '%s'; %s: %v", path, resp.Status, err)
		}
		b, err := ioutil.ReadAll(resp.Body)
		if err == nil {
			// var data stats.Summary
			var data KubeletStatsSummary
			if err = json.Unmarshal(b, &data); err == nil {
				return &data
			}
			log.Errorf("Failed to unmarshal node summary response from '%s'; %v", path, string(b))
		} else {
			log.Errorf("Failed to read node summary response from '%s'; %v", path, err)
		}
	} else {
		log.Errorf("Failed to read node summary from '%s': %v", path, err)
	}
	return nil
}

func (m *Provider) listNodes() []v1.Node {
	nodes, err := m.kubeClients.Standard.Core().Nodes().List(meta_v1.ListOptions{})
	if err == nil {
		return nodes.Items
	}
	log.Error(err)
	return []v1.Node{}
}
<commit_msg>account for possible nil network stats<commit_after>package metrics

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/alecthomas/units"
	"github.com/matt-deboer/kuill/pkg/auth"
	"github.com/matt-deboer/kuill/pkg/clients"
	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Provider gathers metrics from the underlying kubernetes cluster
type Provider struct {
	summary     *Summaries
	mutex       sync.RWMutex
	kubeClients *clients.KubeClients
	killSwitch  chan struct{}
}

// NewMetricsProvider returns a Provider capable of returning metrics for the cluster
func NewMetricsProvider(kubeClients *clients.KubeClients) (*Provider, error) {

	m := &Provider{
		kubeClients: kubeClients,
		killSwitch:  make(chan struct{}, 1),
	}

	m.summary = m.summarize()

	go func() {
		for {
			select {
			case _ = <-m.killSwitch:
				return
			default:
				time.Sleep(15 * time.Second)

				summary := m.summarize()

				m.mutex.Lock()
				m.summary = summary
				m.mutex.Unlock()
			}
		}
	}()

	return m, nil
}

// Close shuts down the metrics provider's background worker(s)
func (m *Provider) Close() {
	m.killSwitch <- struct{}{}
}

// GetMetrics returns a summarized metrics bundle for the entire cluster
func (m *Provider) GetMetrics(w http.ResponseWriter, r *http.Request, context auth.Context) {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	data, err := json.Marshal(m.summary)
	if err != nil {
		log.Error(err)
		http.Error(w, "Failed to marshall metrics", http.StatusInternalServerError)
	} else {
		w.Header().Set("Content-Type", "application/json")
		w.Write(data)
	}
}

func (m *Provider) summarize() *Summaries {

	summary := &Summaries{
		Namespace: make(map[string]*Summary),
		Node:      make(map[string]*Summary),
		Pod:       make(map[string]*Summary),
	}
	aggregates := make(map[string]uint64)
	namespaces := make(map[string]bool)

	for _, node := range m.listNodes() {
		nodeName := node.ObjectMeta.Name

		nodeSummary := m.readNodeSummary(fmt.Sprintf("%s/api/v1/proxy/nodes/%s:10255/stats/summary",
			m.kubeClients.BaseURL, nodeName))
		if nodeSummary != nil {

			convertedSummary, convertedByNs := convertSummary(nodeSummary, node, summary)
			// aggregate at the cluster level
			aggregates["Cluster:totalMillicores"] += convertedSummary.CPU.Total
			aggregates["Cluster:memTotalBytes"] += convertedSummary.Memory.Total
			aggregates["Cluster:usageNanoCores"] += nodeSummary.Node.CPU.UsageNanoCores
			aggregates["Cluster:memUsageBytes"] += nodeSummary.Node.Memory.UsageBytes
			aggregates["Cluster:networkTxBytes"] += nodeSummary.Node.Network.TxBytes
			aggregates["Cluster:networkRxBytes"] += nodeSummary.Node.Network.RxBytes
			aggregates["Cluster:networkSeconds"] += uint64(nodeSummary.Node.Network.Time.Sub(nodeSummary.Node.StartTime).Seconds())
			aggregates["Cluster:fsCapacityBytes"] += nodeSummary.Node.Fs.CapacityBytes
			aggregates["Cluster:fsUsedBytes"] += nodeSummary.Node.Fs.UsedBytes

			// aggregate at the namespace level
			for ns, nsStat := range convertedByNs {
				aggregates["Namespace:"+ns+":fsUsedBytes"] += nsStat.Disk.Usage
				aggregates["Namespace:"+ns+":fsCapacityBytes"] += nsStat.Disk.Total
			}

			for _, podStats := range nodeSummary.Pods {
				ns := podStats.PodRef.Namespace
				aggregates["Namespace:"+ns+":pods"]++
				aggregates["Cluster:pods"]++
				convertedSummary.Pods.Usage++
				namespaces[ns] = true
				for _, volStats := range podStats.VolumeStats {
					aggregates["Cluster:volCapacityBytes"] += volStats.CapacityBytes
					aggregates["Cluster:volUsedBytes"] += volStats.UsedBytes
					aggregates["Namespace:"+ns+":volCapacityBytes"] += volStats.CapacityBytes
					aggregates["Namespace:"+ns+":volUsedBytes"] += volStats.UsedBytes
				}
				for _, c := range podStats.Containers {
					aggregates["Namespace:"+ns+":containers"]++
					aggregates["Cluster:containers"]++
					convertedSummary.Containers.Usage++
					if c.CPU != nil {
						aggregates["Namespace:"+ns+":usageCoreNanoSeconds"] += c.CPU.UsageCoreNanoSeconds
						aggregates["Namespace:"+ns+":usageNanoCores"] += c.CPU.UsageNanoCores
					}
					if c.Memory != nil {
						aggregates["Namespace:"+ns+":memAvailableBytes"] += c.Memory.AvailableBytes
						aggregates["Namespace:"+ns+":memUsageBytes"] += c.Memory.UsageBytes
						aggregates["Namespace:"+ns+":memRSSBytes"] += c.Memory.RSSBytes
						aggregates["Namespace:"+ns+":memWorkingSetBytes"] += c.Memory.WorkingSetBytes
					}
				}
				if podStats.Network != nil {
					aggregates["Namespace:"+ns+":networkTxBytes"] += podStats.Network.TxBytes
					aggregates["Namespace:"+ns+":networkRxBytes"] += podStats.Network.RxBytes
					aggregates["Namespace:"+ns+":networkSeconds"] += uint64(podStats.Network.Time.Unix() - podStats.StartTime.Unix())
				}
			}

			summary.Node[node.ObjectMeta.Name] = convertedSummary
		}
	}

	summary.Cluster = makeSummary("Cluster", aggregates)
	for ns := range namespaces {
		prefix := "Namespace:" + ns
		nsSummary := makeSummary(prefix, aggregates)

		// To be replaced by resource quota for the NS, should one exist
		nsSummary.CPU.Total = summary.Cluster.CPU.Total
		nsSummary.CPU.Ratio = safeDivideFloat(float64(nsSummary.CPU.Usage), float64(nsSummary.CPU.Total))
		nsSummary.Memory.Total = summary.Cluster.Memory.Total
		nsSummary.Memory.Ratio = safeDivideFloat(float64(nsSummary.Memory.Usage), float64(nsSummary.Memory.Total))

		summary.Namespace[ns] = nsSummary
	}

	return summary
}

func safeGet(ref *uint64) uint64 {
	if ref != nil {
		return *ref
	}
	return 0
}

func safeDivideInt(dividend, divisor uint64) uint64 {
	if divisor == 0 {
		return 0
	}
	return dividend / divisor
}

func safeDivideFloat(dividend, divisor float64) float64 {
	if divisor == 0.0 {
		return 0.0
	}
	return dividend / divisor
}

func convertSummary(summary *KubeletStatsSummary, node v1.Node, summaries *Summaries) (*Summary, map[string]*Summary) {

	summaryByNs := make(map[string]*Summary)
	networkSeconds := uint64(0)
	if summary.Node.Network != nil {
		networkSeconds = uint64(summary.Node.Network.Time.Unix() - summary.Node.StartTime.Unix())
	}
	volCapacityBytes := uint64(0)
	volUsageBytes := uint64(0)
	for _, podStats := range summary.Pods {

		podSummary := convertPodSummary(podStats)
		podKey := fmt.Sprintf("%s/%s", podStats.PodRef.Namespace, podStats.PodRef.Name)
		summaries.Pod[podKey] = podSummary
		nsSummary, ok := summaryByNs[podStats.PodRef.Namespace]
		if !ok {
			nsSummary = &Summary{Disk: &SummaryStat{Units: "bytes"}}
			summaryByNs[podStats.PodRef.Namespace] = nsSummary
		}
		nsSummary.Disk.Usage += podSummary.Disk.Usage
		nsSummary.Disk.Total += podSummary.Disk.Total

		volUsageBytes += podSummary.Volumes.Usage
		volCapacityBytes += podSummary.Volumes.Total
	}

	totalCPUCores, err := strconv.Atoi(node.Status.Allocatable.Cpu().String())
	if err != nil {
		log.Errorf("Failed to parse allocatable.cpu; %v", err)
	}
	allocatableBytesString := node.Status.Allocatable.Memory().String()
	if strings.HasSuffix(allocatableBytesString, "i") {
		allocatableBytesString += "B"
	}
	totalMemoryBytes, err := units.ParseStrictBytes(allocatableBytesString)
	if err != nil {
		log.Errorf("Failed to parse allocatable.memory; %v", err)
	}

	if summary.Node.CPU == nil {
		summary.Node.CPU = &CPUStats{}
	}

	if summary.Node.Memory == nil {
		summary.Node.Memory = &MemoryStats{}
	}

	if summary.Node.Fs == nil {
		summary.Node.Fs = &FsStats{}
	}

	return &Summary{
		CPU: newSummaryStat(
			summary.Node.CPU.UsageNanoCores/1000000,
			uint64(totalCPUCores*1000),
			"millicores"),
		Memory: newSummaryStat(
			summary.Node.Memory.UsageBytes,
			uint64(totalMemoryBytes),
			"bytes"),
		Disk: newSummaryStat(
			summary.Node.Fs.UsedBytes,
			summary.Node.Fs.CapacityBytes,
			"bytes"),
		Volumes: newSummaryStat(
			volUsageBytes,
			volCapacityBytes,
			"bytes"),
		NetRx: newSummaryStat(
			safeDivideInt(summary.Node.Network.RxBytes, networkSeconds),
			1,
			"bytes/sec"),
		NetTx: newSummaryStat(
			safeDivideInt(summary.Node.Network.TxBytes, networkSeconds),
			1,
			"bytes/sec"),
		Pods: &SummaryStat{
			Usage: 0,
		},
		Containers: &SummaryStat{
			Usage: 0,
		},
	}, summaryByNs
}

func safeDivide(dividend, divisor uint64) uint64 {
	if divisor == 0 {
		return 0
	}
	return dividend / divisor
}

func convertPodSummary(pod PodStats) *Summary {

	networkSeconds := uint64(pod.Network.Time.Unix() - pod.StartTime.Unix())
	summary := &Summary{
		CPU:     &SummaryStat{Units: "millicores"},
		Memory:  &SummaryStat{Units: "bytes"},
		Volumes: &SummaryStat{Units: "bytes"},
		Disk:    &SummaryStat{Units: "bytes"},
		NetRx: newSummaryStat(
			safeDivideInt(pod.Network.RxBytes, networkSeconds),
			1,
			"bytes/sec"),
		NetTx: newSummaryStat(
			safeDivideInt(pod.Network.TxBytes, networkSeconds),
			1,
			"bytes/sec"),
		Pods: &SummaryStat{
			Usage: 1,
		},
		Containers: &SummaryStat{
			Usage: uint64(len(pod.Containers)),
		},
	}

	for _, volStats := range pod.VolumeStats {
		summary.Volumes.Usage += volStats.CapacityBytes
		summary.Volumes.Total += volStats.UsedBytes
	}

	for _, container := range pod.Containers {
		if container.CPU != nil {
			summary.CPU.Usage += container.CPU.UsageNanoCores / 1000000
		}
		if container.Memory != nil {
			summary.Memory.Usage += container.Memory.UsageBytes
		}
		if container.Rootfs != nil {
			summary.Disk.Usage += container.Rootfs.UsedBytes
		}
	}
	return summary
}

func makeSummary(prefix string, aggregates map[string]uint64) *Summary {

	usageNanoCores := aggregates[prefix+":usageNanoCores"]
	totalMillicores := aggregates[prefix+":totalMillicores"]
	memUsageBytes := aggregates[prefix+":memUsageBytes"]
	memTotalBytes := aggregates[prefix+":memTotalBytes"]
	networkTxBytes := aggregates[prefix+":networkTxBytes"]
	networkRxBytes := aggregates[prefix+":networkRxBytes"]
	// We create a fake timespan here with the current time as the endpoint;
	// the important part is that the duration is correct
	networkSeconds := aggregates[prefix+":networkSeconds"]
	if networkSeconds == 0 {
		networkSeconds = math.MaxUint64
	}

	fsCapacityBytes := aggregates[prefix+":fsCapacityBytes"]
	fsUsedBytes := aggregates[prefix+":fsUsedBytes"]
	volCapacityBytes := aggregates[prefix+":volCapacityBytes"]
	volUsedBytes := aggregates[prefix+":volUsedBytes"]
	podCount := aggregates[prefix+":pods"]
	containerCount := aggregates[prefix+":containers"]

	return &Summary{

		CPU: newSummaryStat(
			usageNanoCores/1000000,
			totalMillicores,
			"millicores"),
		Memory: newSummaryStat(
			memUsageBytes,
			memTotalBytes,
			"bytes"),
		Disk: newSummaryStat(
			fsUsedBytes,
			fsCapacityBytes,
			"bytes"),
		Volumes: newSummaryStat(
			volUsedBytes,
			volCapacityBytes,
			"bytes"),
		NetRx: newSummaryStat(
			safeDivideInt(networkRxBytes, networkSeconds),
			1,
			"bytes/sec"),
		NetTx: newSummaryStat(
			safeDivideInt(networkTxBytes, networkSeconds),
			1,
			"bytes/sec"),
		Pods: &SummaryStat{
			Usage: podCount,
		},
		Containers: &SummaryStat{
			Usage: containerCount,
		},
	}
}

func (m *Provider) readNodeSummary(path string) *KubeletStatsSummary {

	req, err := http.NewRequest("GET", fmt.Sprintf("%s", path), nil)
	if err != nil {
		log.Errorf("Failed to create request for path %s: %v", path, err)
		return nil
	}
	if len(m.kubeClients.BearerToken) > 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(m.kubeClients.BearerToken)))
	}

	resp, err := m.kubeClients.HTTP.Do(req)
	if err == nil {
		if resp.StatusCode != http.StatusOK {
			log.Errorf("Failed to read node summary response from '%s'; %s: %v", path, resp.Status, err)
		}
		b, err := ioutil.ReadAll(resp.Body)
		if err == nil {
			// var data stats.Summary
			var data KubeletStatsSummary
			if err = json.Unmarshal(b, &data); err == nil {
				return &data
			}
			log.Errorf("Failed to unmarshal node summary response from '%s'; %v", path, string(b))
		} else {
			log.Errorf("Failed to read node summary response from '%s'; %v", path, err)
		}
	} else {
		log.Errorf("Failed to read node summary from '%s': %v", path, err)
	}
	return nil
}

func (m *Provider) listNodes() []v1.Node {
	nodes, err := m.kubeClients.Standard.Core().Nodes().List(meta_v1.ListOptions{})
	if err == nil {
		return nodes.Items
	}
	log.Error(err)
	return []v1.Node{}
}
<|endoftext|>


<commit_before>// Package pidfile provides structure and helper functions to create and remove
// PID file. A PID file is usually a file used to store the process ID of a
// running process.
package pidfile

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
)

// PIDFile is a file used to store the process ID of a running process.
type PIDFile struct {
	path string
}

func checkPIDFileAlreadyExists(path string) error {
	if pidByte, err := ioutil.ReadFile(path); err == nil {
		pidString := strings.TrimSpace(string(pidByte))
		if pid, err := strconv.Atoi(pidString); err == nil {
			if processExists(pid) {
				return fmt.Errorf("pid file found, ensure docker is not running or delete %s", path)
			}
		}
	}
	return nil
}

// New creates a PIDfile using the specified path.
func New(path string) (*PIDFile, error) {
	if err := checkPIDFileAlreadyExists(path); err != nil {
		return nil, err
	}
	if err := ioutil.WriteFile(path, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil {
		return nil, err
	}

	return &PIDFile{path: path}, nil
}

// Remove removes the PIDFile.
func (file PIDFile) Remove() error {
	if err := os.Remove(file.path); err != nil {
		return err
	}
	return nil
}
<commit_msg>mkdirall on the PID file path<commit_after>// Package pidfile provides structure and helper functions to create and remove
// PID file. A PID file is usually a file used to store the process ID of a
// running process.
package pidfile

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/docker/docker/pkg/system"
)

// PIDFile is a file used to store the process ID of a running process.
type PIDFile struct {
	path string
}

func checkPIDFileAlreadyExists(path string) error {
	if pidByte, err := ioutil.ReadFile(path); err == nil {
		pidString := strings.TrimSpace(string(pidByte))
		if pid, err := strconv.Atoi(pidString); err == nil {
			if processExists(pid) {
				return fmt.Errorf("pid file found, ensure docker is not running or delete %s", path)
			}
		}
	}
	return nil
}

// New creates a PIDfile using the specified path.
func New(path string) (*PIDFile, error) {
	if err := checkPIDFileAlreadyExists(path); err != nil {
		return nil, err
	}
	// Note MkdirAll returns nil if a directory already exists
	if err := system.MkdirAll(filepath.Dir(path), os.FileMode(0755)); err != nil {
		return nil, err
	}
	if err := ioutil.WriteFile(path, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil {
		return nil, err
	}

	return &PIDFile{path: path}, nil
}

// Remove removes the PIDFile.
func (file PIDFile) Remove() error {
	if err := os.Remove(file.path); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package plugins

import (
	"encoding/json"
	"errors"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
)

var (
	DataSources  map[string]DataSourcePlugin
	Panels       map[string]PanelPlugin
	StaticRoutes []*StaticRootConfig
)

type PluginScanner struct {
	pluginPath string
	errors     []error
}

func Init() error {
	DataSources = make(map[string]DataSourcePlugin)
	StaticRoutes = make([]*StaticRootConfig, 0)
	Panels = make(map[string]PanelPlugin)

	scan(path.Join(setting.StaticRootPath, "app/plugins"))
	scan(path.Join(setting.PluginsPath))
	checkExternalPluginPaths()

	return nil
}

func checkExternalPluginPaths() error {
	for _, section := range setting.Cfg.Sections() {
		if strings.HasPrefix(section.Name(), "plugin.") {
			path := section.Key("path").String()
			if path != "" {
				log.Info("Plugin: Scaning dir %s", path)
				scan(path)
			}
		}
	}
	return nil
}

func scan(pluginDir string) error {
	scanner := &PluginScanner{
		pluginPath: pluginDir,
	}

	if err := util.Walk(pluginDir, true, true, scanner.walker); err != nil {
		return err
	}

	if len(scanner.errors) > 0 {
		return errors.New("Some plugins failed to load")
	}

	return nil
}

func (scanner *PluginScanner) walker(currentPath string, f os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	if f.IsDir() {
		return nil
	}

	if f.Name() == "plugin.json" {
		err := scanner.loadPluginJson(currentPath)
		if err != nil {
			log.Error(3, "Failed to load plugin json file: %v,  err: %v", currentPath, err)
			scanner.errors = append(scanner.errors, err)
		}
	}
	return nil
}

func addStaticRoot(staticRootConfig *StaticRootConfig, currentDir string) {
	if staticRootConfig != nil {
		staticRootConfig.Path = path.Join(currentDir, staticRootConfig.Path)
		StaticRoutes = append(StaticRoutes, staticRootConfig)
	}
}

func (scanner *PluginScanner) loadPluginJson(pluginJsonFilePath string) error {
	currentDir := filepath.Dir(pluginJsonFilePath)
	reader, err := os.Open(pluginJsonFilePath)
	if err != nil {
		return err
	}

	defer reader.Close()

	jsonParser := json.NewDecoder(reader)

	pluginJson := make(map[string]interface{})
	if err := jsonParser.Decode(&pluginJson); err != nil {
		return err
	}

	pluginType, exists := pluginJson["pluginType"]
	if !exists {
		return errors.New("Did not find pluginType property in plugin.json")
	}

	if pluginType == "datasource" {
		p := DataSourcePlugin{}
		reader.Seek(0, 0)
		if err := jsonParser.Decode(&p); err != nil {
			return err
		}

		if p.Type == "" {
			return errors.New("Did not find type property in plugin.json")
		}

		DataSources[p.Type] = p
		addStaticRoot(p.StaticRootConfig, currentDir)
	}

	if pluginType == "panel" {
		p := PanelPlugin{}
		reader.Seek(0, 0)
		if err := jsonParser.Decode(&p); err != nil {
			return err
		}

		if p.Type == "" {
			return errors.New("Did not find type property in plugin.json")
		}

		Panels[p.Type] = p
		addStaticRoot(p.StaticRootConfig, currentDir)
	}

	return nil
}
<commit_msg>feat(plugins): adds warn log for scanning folders<commit_after>package plugins

import (
	"encoding/json"
	"errors"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
)

var (
	DataSources  map[string]DataSourcePlugin
	Panels       map[string]PanelPlugin
	StaticRoutes []*StaticRootConfig
)

type PluginScanner struct {
	pluginPath string
	errors     []error
}

func Init() error {
	DataSources = make(map[string]DataSourcePlugin)
	StaticRoutes = make([]*StaticRootConfig, 0)
	Panels = make(map[string]PanelPlugin)

	scan(path.Join(setting.StaticRootPath, "app/plugins"))
	scan(path.Join(setting.PluginsPath))
	checkExternalPluginPaths()

	return nil
}

func checkExternalPluginPaths() error {
	for _, section := range setting.Cfg.Sections() {
		if strings.HasPrefix(section.Name(), "plugin.") {
			path := section.Key("path").String()
			if path != "" {
				log.Info("Plugin: Scaning dir %s", path)
				scan(path)
			}
		}
	}
	return nil
}

func scan(pluginDir string) error {
	scanner := &PluginScanner{
		pluginPath: pluginDir,
	}

	if err := util.Walk(pluginDir, true, true, scanner.walker); err != nil {
		log.Warn("Failed to scan dir \"%v\" error: %s", pluginDir, err)
		return err
	}

	if len(scanner.errors) > 0 {
		return errors.New("Some plugins failed to load")
	}

	return nil
}

func (scanner *PluginScanner) walker(currentPath string, f os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	if f.IsDir() {
		return nil
	}

	if f.Name() == "plugin.json" {
		err := scanner.loadPluginJson(currentPath)
		if err != nil {
			log.Error(3, "Failed to load plugin json file: %v,  err: %v", currentPath, err)
			scanner.errors = append(scanner.errors, err)
		}
	}
	return nil
}

func addStaticRoot(staticRootConfig *StaticRootConfig, currentDir string) {
	if staticRootConfig != nil {
		staticRootConfig.Path = path.Join(currentDir, staticRootConfig.Path)
		StaticRoutes = append(StaticRoutes, staticRootConfig)
	}
}

func (scanner *PluginScanner) loadPluginJson(pluginJsonFilePath string) error {
	currentDir := filepath.Dir(pluginJsonFilePath)
	reader, err := os.Open(pluginJsonFilePath)
	if err != nil {
		return err
	}

	defer reader.Close()

	jsonParser := json.NewDecoder(reader)

	pluginJson := make(map[string]interface{})
	if err := jsonParser.Decode(&pluginJson); err != nil {
		return err
	}

	pluginType, exists := pluginJson["pluginType"]
	if !exists {
		return errors.New("Did not find pluginType property in plugin.json")
	}

	if pluginType == "datasource" {
		p := DataSourcePlugin{}
		reader.Seek(0, 0)
		if err := jsonParser.Decode(&p); err != nil {
			return err
		}

		if p.Type == "" {
			return errors.New("Did not find type property in plugin.json")
		}

		DataSources[p.Type] = p
		addStaticRoot(p.StaticRootConfig, currentDir)
	}

	if pluginType == "panel" {
		p := PanelPlugin{}
		reader.Seek(0, 0)
		if err := jsonParser.Decode(&p); err != nil {
			return err
		}

		if p.Type == "" {
			return errors.New("Did not find type property in plugin.json")
		}

		Panels[p.Type] = p
		addStaticRoot(p.StaticRootConfig, currentDir)
	}

	return nil
}
<|endoftext|>


<commit_before>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.0.0-beta4"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 5.0.1-dev (#1172)<commit_after>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.0.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>package prometheus

import (
	"fmt"
	"math"
	"strconv"
	"strings"

	"github.com/kubeflow/arena/pkg/apis/types"
	log "github.com/sirupsen/logrus"
	"k8s.io/client-go/kubernetes"
)

type JobGpuMetric map[string]types.PodGpuMetric

func (m *JobGpuMetric) SetPodMetric(metric types.GpuMetricInfo) {
	v, err := strconv.ParseFloat(metric.Value, 64)
	if err != nil {
		return
	}
	metricMap := *m
	if _, ok := metricMap[metric.PodName]; !ok {
		metricMap[metric.PodName] = types.PodGpuMetric{}
	}

	podMetric := metricMap[metric.PodName]
	if _, ok := podMetric[metric.Id]; !ok {
		podMetric[metric.Id] = &types.GpuMetric{}
	}
	podGPUMetric := podMetric[metric.Id]
	switch metric.MetricName {
	case "nvidia_gpu_duty_cycle":
		podGPUMetric.GpuDutyCycle = v
	case "nvidia_gpu_memory_used_bytes":
		podGPUMetric.GpuMemoryUsed = v
	case "nvidia_gpu_memory_total_bytes":
		v = math.Trunc(v/(1024*1024*1024)) * (1024 * 1024 * 1024)
		podGPUMetric.GpuMemoryTotal = v
	}
}

func (m JobGpuMetric) GetPodMetrics(podName string) types.PodGpuMetric {
	metricMap := m
	if podMetrics, ok := metricMap[podName]; ok {
		return podMetrics
	}
	return nil
}

func GetPodsGpuInfo(client *kubernetes.Clientset, podNames []string) (JobGpuMetric, error) {
	jobMetric := &JobGpuMetric{}
	query := fmt.Sprintf(types.POD_METRIC_TMP, strings.Join(types.GPU_METRIC_LIST, "|"), strings.Join(podNames, "|"))
	gpuMetrics, err := QueryPrometheusMetrics(client, query)
	if err != nil {
		return nil, err
	}
	if gpuMetrics == nil {
		return nil, fmt.Errorf("failed to get gpu metrics,because the result of querying is null")
	}
	for _, metric := range gpuMetrics {
		jobMetric.SetPodMetric(metric)
	}
	return *jobMetric, nil
}

func getMetricAverage(metrics []types.GpuMetricInfo) float64 {
	var result float64
	result = 0
	for _, metric := range metrics {
		v, _ := strconv.ParseFloat(metric.Value, 64)
		result = result + v
	}
	if result == 0 {
		return result
	}
	result = result / float64(len(metrics))
	return result
}

// {__name__=~"nvidia_gpu_duty_cycle|nvidia_gpu_memory_used_bytes|nvidia_gpu_memory_total_bytes", pod_name=~"tf-distributed-test-ps-0|tf-distributed-test-worker-0"}

func GetNodeGPUMetrics(client *kubernetes.Clientset, nodeNames []string) (map[string]types.NodeGpuMetric, error) {

	query := fmt.Sprintf(types.NODE_METRIC_TMP, strings.Join(types.GPU_METRIC_LIST, "|"), strings.Join(nodeNames, "|"))
	gpuMetrics, err := QueryPrometheusMetrics(client, query)
	if err != nil {
		return nil, err
	}
	if gpuMetrics == nil {
		return nil, fmt.Errorf("failed to get node gpu metrics,because the result of querying is null")
	}
	return generateNodeGPUMetrics(gpuMetrics), nil
}

func generateNodeGPUMetrics(metrics []types.GpuMetricInfo) map[string]types.NodeGpuMetric {
	nodeMetrics := map[string]types.NodeGpuMetric{}
	shareModeUsedGPUMemory := map[string]map[string][]float64{}
	for _, metric := range metrics {
		if metric.PodNamespace == "" {
			continue
		}
		v, err := strconv.ParseFloat(metric.Value, 64)
		if err != nil {
			log.Debugf("failed to parse gpu duty cycle,reason: %v", err)
			continue
		}
		if nodeMetrics[metric.NodeName] == nil {
			nodeMetrics[metric.NodeName] = types.NodeGpuMetric{}
		}

		if nodeMetrics[metric.NodeName][metric.Id] == nil {
			nodeMetrics[metric.NodeName][metric.Id] = &types.AdvancedGpuMetric{
				Id:       metric.Id,
				UUID:     metric.GPUUID,
				PodNames: []string{},
			}
		}
		if shareModeUsedGPUMemory[metric.NodeName] == nil {
			shareModeUsedGPUMemory[metric.NodeName] = map[string][]float64{}
		}
		if shareModeUsedGPUMemory[metric.NodeName][metric.Id] == nil {
			shareModeUsedGPUMemory[metric.NodeName][metric.Id] = []float64{}
		}
		switch metric.MetricName {
		case "nvidia_gpu_duty_cycle":
			nodeMetrics[metric.NodeName][metric.Id].GpuDutyCycle = v
		case "nvidia_gpu_memory_used_bytes":
			nodeMetrics[metric.NodeName][metric.Id].GpuMemoryUsed = v
			if metric.AllocateMode == "share" {
				shareModeUsedGPUMemory[metric.NodeName][metric.Id] = append(shareModeUsedGPUMemory[metric.NodeName][metric.Id], v)
			}
		case "nvidia_gpu_memory_total_bytes":
			v = math.Trunc(v/(1024*1024*1024)) * (1024 * 1024 * 1024)
			nodeMetrics[metric.NodeName][metric.Id].GpuMemoryTotal = v
		}
		if metric.PodName != "" {
			podName := fmt.Sprintf("%v/%v", metric.PodNamespace, metric.PodName)
			nodeMetrics[metric.NodeName][metric.Id].PodNames = append(nodeMetrics[metric.NodeName][metric.Id].PodNames, podName)
		}
	}
	for nodeName, allUsedGPUMemory := range shareModeUsedGPUMemory {
		for gpuId, usedGPUMemory := range allUsedGPUMemory {
			if len(usedGPUMemory) > 0 {
				total := float64(0)
				for _, v := range usedGPUMemory {
					total += v
				}
				nodeMetrics[nodeName][gpuId].GpuMemoryUsed = total
			}
		}
	}
	return nodeMetrics
}
<commit_msg>fix bug: cannot display total gpu memory (#561)<commit_after>package prometheus

import (
	"fmt"
	"math"
	"strconv"
	"strings"

	"github.com/kubeflow/arena/pkg/apis/types"
	log "github.com/sirupsen/logrus"
	"k8s.io/client-go/kubernetes"
)

type JobGpuMetric map[string]types.PodGpuMetric

func (m *JobGpuMetric) SetPodMetric(metric types.GpuMetricInfo) {
	v, err := strconv.ParseFloat(metric.Value, 64)
	if err != nil {
		return
	}
	metricMap := *m
	if _, ok := metricMap[metric.PodName]; !ok {
		metricMap[metric.PodName] = types.PodGpuMetric{}
	}

	podMetric := metricMap[metric.PodName]
	if _, ok := podMetric[metric.Id]; !ok {
		podMetric[metric.Id] = &types.GpuMetric{}
	}
	podGPUMetric := podMetric[metric.Id]
	switch metric.MetricName {
	case "nvidia_gpu_duty_cycle":
		podGPUMetric.GpuDutyCycle = v
	case "nvidia_gpu_memory_used_bytes":
		podGPUMetric.GpuMemoryUsed = v
	case "nvidia_gpu_memory_total_bytes":
		v = math.Trunc(v/(1024*1024*1024)) * (1024 * 1024 * 1024)
		podGPUMetric.GpuMemoryTotal = v
	}
}

func (m JobGpuMetric) GetPodMetrics(podName string) types.PodGpuMetric {
	metricMap := m
	if podMetrics, ok := metricMap[podName]; ok {
		return podMetrics
	}
	return nil
}

func GetPodsGpuInfo(client *kubernetes.Clientset, podNames []string) (JobGpuMetric, error) {
	jobMetric := &JobGpuMetric{}
	query := fmt.Sprintf(types.POD_METRIC_TMP, strings.Join(types.GPU_METRIC_LIST, "|"), strings.Join(podNames, "|"))
	gpuMetrics, err := QueryPrometheusMetrics(client, query)
	if err != nil {
		return nil, err
	}
	if gpuMetrics == nil {
		return nil, fmt.Errorf("failed to get gpu metrics,because the result of querying is null")
	}
	for _, metric := range gpuMetrics {
		jobMetric.SetPodMetric(metric)
	}
	return *jobMetric, nil
}

func getMetricAverage(metrics []types.GpuMetricInfo) float64 {
	var result float64
	result = 0
	for _, metric := range metrics {
		v, _ := strconv.ParseFloat(metric.Value, 64)
		result = result + v
	}
	if result == 0 {
		return result
	}
	result = result / float64(len(metrics))
	return result
}

// {__name__=~"nvidia_gpu_duty_cycle|nvidia_gpu_memory_used_bytes|nvidia_gpu_memory_total_bytes", pod_name=~"tf-distributed-test-ps-0|tf-distributed-test-worker-0"}

func GetNodeGPUMetrics(client *kubernetes.Clientset, nodeNames []string) (map[string]types.NodeGpuMetric, error) {

	query := fmt.Sprintf(types.NODE_METRIC_TMP, strings.Join(types.GPU_METRIC_LIST, "|"), strings.Join(nodeNames, "|"))
	gpuMetrics, err := QueryPrometheusMetrics(client, query)
	if err != nil {
		return nil, err
	}
	if gpuMetrics == nil {
		return nil, fmt.Errorf("failed to get node gpu metrics,because the result of querying is null")
	}
	return generateNodeGPUMetrics(gpuMetrics), nil
}

func generateNodeGPUMetrics(metrics []types.GpuMetricInfo) map[string]types.NodeGpuMetric {
	nodeMetrics := map[string]types.NodeGpuMetric{}
	shareModeUsedGPUMemory := map[string]map[string][]float64{}
	for _, metric := range metrics {
		//if metric.PodNamespace == "" {
		//	continue
		//}
		v, err := strconv.ParseFloat(metric.Value, 64)
		if err != nil {
			log.Debugf("failed to parse gpu duty cycle,reason: %v", err)
			continue
		}
		if nodeMetrics[metric.NodeName] == nil {
			nodeMetrics[metric.NodeName] = types.NodeGpuMetric{}
		}

		if nodeMetrics[metric.NodeName][metric.Id] == nil {
			nodeMetrics[metric.NodeName][metric.Id] = &types.AdvancedGpuMetric{
				Id:       metric.Id,
				UUID:     metric.GPUUID,
				PodNames: []string{},
			}
		}
		if shareModeUsedGPUMemory[metric.NodeName] == nil {
			shareModeUsedGPUMemory[metric.NodeName] = map[string][]float64{}
		}
		if shareModeUsedGPUMemory[metric.NodeName][metric.Id] == nil {
			shareModeUsedGPUMemory[metric.NodeName][metric.Id] = []float64{}
		}
		switch metric.MetricName {
		case "nvidia_gpu_duty_cycle":
			nodeMetrics[metric.NodeName][metric.Id].GpuDutyCycle = v
		case "nvidia_gpu_memory_used_bytes":
			nodeMetrics[metric.NodeName][metric.Id].GpuMemoryUsed = v
			if metric.AllocateMode == "share" {
				shareModeUsedGPUMemory[metric.NodeName][metric.Id] = append(shareModeUsedGPUMemory[metric.NodeName][metric.Id], v)
			}
		case "nvidia_gpu_memory_total_bytes":
			v = math.Trunc(v/(1024*1024*1024)) * (1024 * 1024 * 1024)
			nodeMetrics[metric.NodeName][metric.Id].GpuMemoryTotal = v
		}
		if metric.PodNamespace != "" && metric.PodName != "" {
			podName := fmt.Sprintf("%v/%v", metric.PodNamespace, metric.PodName)
			nodeMetrics[metric.NodeName][metric.Id].PodNames = append(nodeMetrics[metric.NodeName][metric.Id].PodNames, podName)
		}
	}
	for nodeName, allUsedGPUMemory := range shareModeUsedGPUMemory {
		for gpuId, usedGPUMemory := range allUsedGPUMemory {
			if len(usedGPUMemory) > 0 {
				total := float64(0)
				for _, v := range usedGPUMemory {
					total += v
				}
				nodeMetrics[nodeName][gpuId].GpuMemoryUsed = total
			}
		}
	}
	return nodeMetrics
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"

	"github.com/gorilla/mux"
)

type Build struct {
	Id  int    `json:"id"`
	Key string `json:"key"`
}

type Log struct {
	Id      int `json:"id"`
	BuildId int `json:"buildId"`

	Type                 string `json:"type"`
	ContentType          string `json:"contentType"`
	ContentTypeParameter string `json:"contentTypeParameter"`
	Size                 int64  `json:"size"`
}

func (s *Server) handleNewBuild(w http.ResponseWriter, r *http.Request) {
	qs := r.URL.Query()
	key := qs.Get("key")
	if key == "" {
		http.Error(w, `"key" is required`, 400)
		return
	}

	id, err := s.buildlog.Create(key)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	b := Build{
		Id:  id,
		Key: key,
	}
	err = json.NewEncoder(w).Encode(b)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handleGetBuild(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	b := Build{
		Id:  build.Id,
		Key: build.Key,
	}

	err = json.NewEncoder(w).Encode(b)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handlePostLog(w http.ResponseWriter, r *http.Request) {
	qs := r.URL.Query()
	t := qs.Get("type")
	if t == "" {
		http.Error(w, `"type" is required`, 400)
		return
	}

	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	ct := r.Header["Content-Type"][0]

	lid, err := build.Log(t, ct, r.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	l := Log{
		Id: lid,
	}

	err = json.NewEncoder(w).Encode(l)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handleGetLogMetadata(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}
	buildId, err := strconv.Atoi(vars["buildId"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(buildId)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	data, err := build.GetLog(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if data == nil {
		http.Error(w, "Not found", 404)
		return
	}

	jsonData := Log{
		Id:                   data.Id,
		BuildId:              data.Build.Id,
		Type:                 data.Type,
		ContentType:          data.ContentType,
		ContentTypeParameter: data.ContentTypeParameter,
		Size:                 data.Size,
	}
	err = json.NewEncoder(w).Encode(jsonData)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handleGetLog(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}
	buildId, err := strconv.Atoi(vars["buildId"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(buildId)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	data, err := build.GetLog(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if data == nil {
		http.Error(w, "Not found", 404)
		return
	}

	dr, err := data.Read()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	ct := createContentType(data.ContentType, data.ContentTypeParameter)
	w.Header().Set("Content-Type", ct)
	w.Header().Set("Content-Length", fmt.Sprintf("%d", data.Size))
	io.CopyN(w, dr, data.Size)
}

func createContentType(contentType, parameter string) string {
	var b bytes.Buffer
	b.WriteString(contentType)
	if parameter != "" {
		b.WriteString("; ")
		b.WriteString(parameter)
	}
	return b.String()
}
<commit_msg>Only return id and known information when submitting log<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"

	"github.com/gorilla/mux"
)

type Build struct {
	Id  int    `json:"id"`
	Key string `json:"key"`
}

type Log struct {
	Id      int    `json:"id"`
	BuildId int    `json:"buildId"`
	Type    string `json:"type"`
}

type LogMetadata struct {
	Id      int `json:"id"`
	BuildId int `json:"buildId"`

	Type                 string `json:"type"`
	ContentType          string `json:"contentType"`
	ContentTypeParameter string `json:"contentTypeParameter"`
	Size                 int64  `json:"size"`
}

func (s *Server) handleNewBuild(w http.ResponseWriter, r *http.Request) {
	qs := r.URL.Query()
	key := qs.Get("key")
	if key == "" {
		http.Error(w, `"key" is required`, 400)
		return
	}

	id, err := s.buildlog.Create(key)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	b := Build{
		Id:  id,
		Key: key,
	}
	err = json.NewEncoder(w).Encode(b)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handleGetBuild(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	b := Build{
		Id:  build.Id,
		Key: build.Key,
	}

	err = json.NewEncoder(w).Encode(b)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handlePostLog(w http.ResponseWriter, r *http.Request) {
	qs := r.URL.Query()
	t := qs.Get("type")
	if t == "" {
		http.Error(w, `"type" is required`, 400)
		return
	}

	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	ct := r.Header["Content-Type"][0]

	lid, err := build.Log(t, ct, r.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	l := Log{
		Id:      lid,
		BuildId: build.Id,
		Type:    t,
	}

	err = json.NewEncoder(w).Encode(l)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handleGetLogMetadata(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}
	buildId, err := strconv.Atoi(vars["buildId"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(buildId)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	data, err := build.GetLog(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if data == nil {
		http.Error(w, "Not found", 404)
		return
	}

	jsonData := LogMetadata{
		Id:                   data.Id,
		BuildId:              data.Build.Id,
		Type:                 data.Type,
		ContentType:          data.ContentType,
		ContentTypeParameter: data.ContentTypeParameter,
		Size:                 data.Size,
	}
	err = json.NewEncoder(w).Encode(jsonData)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handleGetLog(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}
	buildId, err := strconv.Atoi(vars["buildId"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(buildId)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	data, err := build.GetLog(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if data == nil {
		http.Error(w, "Not found", 404)
		return
	}

	dr, err := data.Read()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	ct := createContentType(data.ContentType, data.ContentTypeParameter)
	w.Header().Set("Content-Type", ct)
	w.Header().Set("Content-Length", fmt.Sprintf("%d", data.Size))
	io.CopyN(w, dr, data.Size)
}

func createContentType(contentType, parameter string) string {
	var b bytes.Buffer
	b.WriteString(contentType)
	if parameter != "" {
		b.WriteString("; ")
		b.WriteString(parameter)
	}
	return b.String()
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/mux"
	"io"
)

type buildRequest struct {
	Key  string `json:"key"`
	Name string `json:"name"`

	Status string `json:"status"`
}

type Build struct {
	Id int `json:"id"`

	Key  string `json:"key"`
	Name string `json:"name"`

	Status   string   `json:"status"`
	Started  *iso8601 `json:"started"`
	Finished *iso8601 `json:"finished"`
}

func validateBuildRequest(req *buildRequest) error {
	if req.Key == "" {
		return fmt.Errorf(`"key" is required`)
	}

	if req.Status == "" {
		req.Status = "STARTED"
	}

	return nil
}

func (s *Server) handleNewBuild(w http.ResponseWriter, r *http.Request) {
	var req buildRequest

	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		http.Error(w, err.Error(), 400)
		return
	}

	err = validateBuildRequest(&req)
	if err != nil {
		http.Error(w, err.Error(), 400)
		return
	}

	build, err := s.buildlog.Create(req.Key)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	started := iso8601(time.Now())
	b := Build{
		Id:       build.Id,
		Key:      req.Key,
		Name:     req.Name,
		Status:   req.Status,
		Started:  &started,
		Finished: nil,
	}
	err = json.NewEncoder(w).Encode(b)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handleGetBuild(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	var started *iso8601 = nil
	var finished *iso8601 = nil
	if build.Started != nil {
		t := iso8601(*build.Started)
		started = &t
	}
	if build.Finished != nil {
		t := iso8601(*build.Finished)
		finished = &t
	}
	b := Build{
		Id:       build.Id,
		Key:      build.Key,
		Name:     build.Name,
		Status:   build.Status,
		Started:  started,
		Finished: finished,
	}

	err = json.NewEncoder(w).Encode(b)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handlePatchBuild(w http.ResponseWriter, r *http.Request) {
	var req buildRequest

	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		http.Error(w, err.Error(), 400)
		return
	}
}

func (s *Server) handlePutBuild(w http.ResponseWriter, r *http.Request) {
	mr, err := r.MultipartReader()
	if err != nil {
		http.Error(w, err.Error(), 400)
		return
	}

	for {
		p, err := mr.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			http.Error(w, err.Error(), 400)
			return
		}

		fmt.Printf("Part: name=%s, filename=%s, type=%s\n", p.FormName(), p.FileName(), p.Header.Get("Content-Type"))
	}
}
<commit_msg>Fix invalid response code<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/mux"
	"io"
)

type buildRequest struct {
	Key  string `json:"key"`
	Name string `json:"name"`

	Status string `json:"status"`
}

type Build struct {
	Id int `json:"id"`

	Key  string `json:"key"`
	Name string `json:"name"`

	Status   string   `json:"status"`
	Started  *iso8601 `json:"started"`
	Finished *iso8601 `json:"finished"`
}

func validateBuildRequest(req *buildRequest) error {
	if req.Key == "" {
		return fmt.Errorf(`"key" is required`)
	}

	if req.Status == "" {
		req.Status = "STARTED"
	}

	return nil
}

func (s *Server) handleNewBuild(w http.ResponseWriter, r *http.Request) {
	var req buildRequest

	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		http.Error(w, err.Error(), 400)
		return
	}

	err = validateBuildRequest(&req)
	if err != nil {
		http.Error(w, err.Error(), 400)
		return
	}

	build, err := s.buildlog.Create(req.Key)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	started := iso8601(time.Now())
	b := Build{
		Id:       build.Id,
		Key:      req.Key,
		Name:     req.Name,
		Status:   req.Status,
		Started:  &started,
		Finished: nil,
	}
	err = json.NewEncoder(w).Encode(b)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handleGetBuild(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, "Not found", 404)
		return
	}

	build, err := s.buildlog.Get(id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if build == nil {
		http.Error(w, "Not found", 404)
		return
	}

	var started *iso8601 = nil
	var finished *iso8601 = nil
	if build.Started != nil {
		t := iso8601(*build.Started)
		started = &t
	}
	if build.Finished != nil {
		t := iso8601(*build.Finished)
		finished = &t
	}
	b := Build{
		Id:       build.Id,
		Key:      build.Key,
		Name:     build.Name,
		Status:   build.Status,
		Started:  started,
		Finished: finished,
	}

	err = json.NewEncoder(w).Encode(b)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handlePatchBuild(w http.ResponseWriter, r *http.Request) {
	var req buildRequest

	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (s *Server) handlePutBuild(w http.ResponseWriter, r *http.Request) {
	mr, err := r.MultipartReader()
	if err != nil {
		http.Error(w, err.Error(), 400)
		return
	}

	for {
		p, err := mr.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			http.Error(w, err.Error(), 400)
			return
		}

		fmt.Printf("Part: name=%s, filename=%s, type=%s\n", p.FormName(), p.FileName(), p.Header.Get("Content-Type"))
	}
}
<|endoftext|>