repo_description: Heterogeneous GPU Sharing on Kubernetes
repo_topics: ascend cambricon cncf device-plugin gpu-acceleration gpu-management gpu-shareable gpu-virtualization hygon iluvatar kubernetes kubernetes-gpu-cluster metax mthreads nvidia vgpu vgpu-hypervisor
code: /*
  Copyright 2024 The HAMi Authors.
  
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
  
      http://www.apache.org/licenses/LICENSE-2.0
  
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
  */
  
  package main
  
  import (
  	"fmt"
  	"os"
  	"strings"
  	"time"
  	"unicode/utf8"
  
  	"github.com/Project-HAMi/HAMi/pkg/device"
  	dp "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
  	nv "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
  	"github.com/Project-HAMi/HAMi/pkg/monitor/nvidia"
  	"github.com/Project-HAMi/HAMi/pkg/util"
  
  	"github.com/NVIDIA/go-nvml/pkg/nvml"
  	"github.com/prometheus/client_golang/prometheus"
  	corev1 "k8s.io/api/core/v1"
  	"k8s.io/apimachinery/pkg/labels"
  	"k8s.io/client-go/informers"
  	listerscorev1 "k8s.io/client-go/listers/core/v1"
  	"k8s.io/klog/v2"
  )
  
  // ClusterManager is an example for a system that might have been built without
  // Prometheus in mind. It models a central manager of jobs running in a
  // cluster. Thus, we implement a custom Collector called
  // ClusterManagerCollector, which collects information from a ClusterManager
  // using its provided methods and turns them into Prometheus Metrics for
  // collection.
  //
  // An additional challenge is that multiple instances of the ClusterManager are
  // run within the same binary, each in charge of a different zone. We need to
  // make use of wrapping Registerers to be able to register each
  // ClusterManagerCollector instance with Prometheus.
  type ClusterManager struct {
  	Zone string
  	// Contains many more fields not listed in this example.
  	PodLister       listerscorev1.PodLister
  	containerLister *nvidia.ContainerLister
  	LegacyMetrics   bool
  }
  
  // ReallyExpensiveAssessmentOfTheSystemState is a mock for the data gathering a
  // real cluster manager would have to do. Since it may actually be really
  // expensive, it must only be called once per collection. This implementation,
  // obviously, only returns some made-up data.
  func (c *ClusterManager) ReallyExpensiveAssessmentOfTheSystemState() (
  	oomCountByHost map[string]int, ramUsageByHost map[string]float64,
  ) {
  	// Just example fake data.
  	oomCountByHost = map[string]int{
  		"foo.example.org": 42,
  		"bar.example.org": 2001,
  	}
  	ramUsageByHost = map[string]float64{
  		"foo.example.org": 6.023e23,
  		"bar.example.org": 3.14,
  	}
  	return
  }
  
  // ClusterManagerCollector implements the Collector interface.
  type ClusterManagerCollector struct {
  	ClusterManager *ClusterManager
  }
  
  // Descriptors used by the ClusterManagerCollector below.
  // Metric and label names follow Prometheus naming best practices:
  // https://prometheus.io/docs/practices/naming/
  var (
  	hostGPUdesc = prometheus.NewDesc(
  		"hami_host_gpu_memory_used_bytes",
  		"GPU device memory usage in bytes",
  		[]string{"device_index", "device_uuid", "device_type"}, nil,
  	)
  
  	hostGPUUtilizationdesc = prometheus.NewDesc(
  		"hami_host_gpu_utilization_ratio",
  		"GPU core utilization ratio (0-100)",
  		[]string{"device_index", "device_uuid", "device_type"}, nil,
  	)
  
  	ctrvGPUdesc = prometheus.NewDesc(
  		"hami_vgpu_memory_used_bytes",
  		"vGPU device memory usage in bytes",
  		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil,
  	)
  
  	ctrvGPUlimitdesc = prometheus.NewDesc(
  		"hami_vgpu_memory_limit_bytes",
  		"vGPU device memory limit in bytes",
  		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil,
  	)
  	ctrDeviceMemorydesc = prometheus.NewDesc(
  		"hami_container_device_memory_bytes",
  		`Container device memory usage breakdown in bytes (The label "context_size", "module_size", "buffer_size" and "offset" will be deprecated in v2.10.0, use hami_vgpu_memory_context_bytes, hami_vgpu_memory_module_bytes and hami_vgpu_memory_buffer_bytes instead)`,
  		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid", "context_size", "module_size", "buffer_size", "offset"}, nil,
  	)
  	ctrDeviceUtilizationdesc = prometheus.NewDesc(
  		"hami_container_device_utilization_ratio",
  		"Container device SM utilization ratio",
  		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil,
  	)
  	ctrDeviceLastKernelDesc = prometheus.NewDesc(
  		"hami_container_last_kernel_elapsed_seconds",
  		"Seconds since last kernel execution in container",
  		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil,
  	)
  	ctrDeviceMigInfo = prometheus.NewDesc(
  		"hami_mig_device_info",
  		"MIG device information for container",
  		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid", "instance_id"}, nil,
  	)
  	ctrDeviceMemoryContextDesc = prometheus.NewDesc(
  		"hami_vgpu_memory_context_bytes",
  		"Container device memory context size in bytes",
  		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil,
  	)
  
  	ctrDeviceMemoryModuleDesc = prometheus.NewDesc(
  		"hami_vgpu_memory_module_bytes",
  		"Container device memory module size in bytes",
  		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil,
  	)
  
  	ctrDeviceMemoryBufferDesc = prometheus.NewDesc(
  		"hami_vgpu_memory_buffer_bytes",
  		"Container device memory buffer size in bytes",
  		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil,
  	)
  )
  
  // Legacy metric descriptors (populated only when --legacy-metrics is enabled).
  var (
  	legacyHostGPUdesc              *prometheus.Desc
  	legacyHostGPUUtilizationdesc   *prometheus.Desc
  	legacyCtrvGPUdesc              *prometheus.Desc
  	legacyCtrvGPUlimitdesc         *prometheus.Desc
  	legacyCtrDeviceMemorydesc      *prometheus.Desc
  	legacyCtrDeviceUtilizationdesc *prometheus.Desc
  	legacyCtrDeviceLastKernelDesc  *prometheus.Desc
  	legacyCtrDeviceMigInfo         *prometheus.Desc
  )
  
  func initLegacyDescriptors() {
  	legacyHostGPUdesc = prometheus.NewDesc(
  		"HostGPUMemoryUsage",
  		"GPU device memory usage",
  		[]string{"deviceidx", "deviceuuid", "devicetype"}, nil,
  	)
  	legacyHostGPUUtilizationdesc = prometheus.NewDesc(
  		"HostCoreUtilization",
  		"GPU core utilization",
  		[]string{"deviceidx", "deviceuuid", "devicetype"}, nil,
  	)
  	legacyCtrvGPUdesc = prometheus.NewDesc(
  		"vGPU_device_memory_usage_in_bytes",
  		"vGPU device usage",
  		[]string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil,
  	)
  	legacyCtrvGPUlimitdesc = prometheus.NewDesc(
  		"vGPU_device_memory_limit_in_bytes",
  		"vGPU device limit",
  		[]string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil,
  	)
  	legacyCtrDeviceMemorydesc = prometheus.NewDesc(
  		"Device_memory_desc_of_container",
  		"Container device memory description",
  		[]string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "context", "module", "data", "offset"}, nil,
  	)
  	legacyCtrDeviceUtilizationdesc = prometheus.NewDesc(
  		"Device_utilization_desc_of_container",
  		"Container device utilization description",
  		[]string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil,
  	)
  	legacyCtrDeviceLastKernelDesc = prometheus.NewDesc(
  		"Device_last_kernel_of_container",
  		"Container device last kernel description",
  		[]string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil,
  	)
  	legacyCtrDeviceMigInfo = prometheus.NewDesc(
  		"MigInfo",
  		"Mig device information for container",
  		[]string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "instanceid"}, nil,
  	)
  }
  
  func sendLegacyMetric(ch chan<- prometheus.Metric, desc *prometheus.Desc, valueType prometheus.ValueType, value float64, labels ...string) {
  	if desc == nil {
  		return
  	}
  	if err := sendMetric(ch, desc, valueType, value, labels...); err != nil {
  		klog.V(4).Infof("Failed to send legacy metric: %v", err)
  	}
  }
  
  // Describe is implemented with DescribeByCollect. That's possible because the
  // Collect method will always return the same two metrics with the same two
  // descriptors.
  func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
  	ch <- hostGPUdesc
  	ch <- ctrvGPUdesc
  	ch <- ctrvGPUlimitdesc
  	ch <- hostGPUUtilizationdesc
  	ch <- ctrDeviceMemorydesc
  	ch <- ctrDeviceUtilizationdesc
  	ch <- ctrDeviceMemoryContextDesc
  	ch <- ctrDeviceMemoryModuleDesc
  	ch <- ctrDeviceMemoryBufferDesc
  
  	if cc.ClusterManager.LegacyMetrics {
  		ch <- legacyHostGPUdesc
  		ch <- legacyHostGPUUtilizationdesc
  		ch <- legacyCtrvGPUdesc
  		ch <- legacyCtrvGPUlimitdesc
  		ch <- legacyCtrDeviceMemorydesc
  		ch <- legacyCtrDeviceUtilizationdesc
  		ch <- legacyCtrDeviceLastKernelDesc
  		ch <- legacyCtrDeviceMigInfo
  	}
  }
  
  //func parseidstr(podusage string) (string, string, error) {
  //	tmp := strings.Split(podusage, "_")
  //	if len(tmp) > 1 {
  //		return tmp[0], tmp[1], nil
  //	} else {
  //		return "", "", errors.New("parse error")
  //	}
  //}
  //
  //func gettotalusage(usage podusage, vidx int) (deviceMemory, error) {
  //	added := deviceMemory{
  //		bufferSize:  0,
  //		contextSize: 0,
  //		moduleSize:  0,
  //		offset:      0,
  //		total:       0,
  //	}
  //	for _, val := range usage.sr.procs {
  //		added.bufferSize += val.used[vidx].bufferSize
  //		added.contextSize += val.used[vidx].contextSize
  //		added.moduleSize += val.used[vidx].moduleSize
  //		added.offset += val.used[vidx].offset
  //		added.total += val.used[vidx].total
  //	}
  //	return added, nil
  //}
  //
  //func getTotalUtilization(usage podusage, vidx int) deviceUtilization {
  //	added := deviceUtilization{
  //		decUtil: 0,
  //		encUtil: 0,
  //		smUtil:  0,
  //	}
  //	for _, val := range usage.sr.procs {
  //		added.decUtil += val.deviceUtil[vidx].decUtil
  //		added.encUtil += val.deviceUtil[vidx].encUtil
  //		added.smUtil += val.deviceUtil[vidx].smUtil
  //	}
  //	return added
  //}
  
  // Collect first triggers the ReallyExpensiveAssessmentOfTheSystemState. Then it
  // creates constant metrics for each host on the fly based on the returned data.
  //
  // Note that Collect could be called concurrently, so we depend on
  // ReallyExpensiveAssessmentOfTheSystemState to be concurrency-safe.
  func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
  	klog.Info("Starting to collect metrics for vGPUMonitor")
  
  	// Collect GPU information
  	if err := cc.collectGPUInfo(ch); err != nil {
  		klog.Errorf("Failed to collect GPU info: %v", err)
  		// Decide whether to continue or return based on business requirements
  	}
  
  	// Collect Pod and Container information
  	if err := cc.collectPodAndContainerInfo(ch); err != nil {
  		klog.Errorf("Failed to collect Pod and Container info: %v", err)
  		// Decide whether to continue or return based on business requirements
  	}
  
  	// Collect Pod and Container Mig information
  	if err := cc.collectPodAndContainerMigInfo(ch); err != nil {
  		klog.Errorf("Failed to collect Pod and Container Mig info: %v", err)
  		// Decide whether to continue or return based on business requirements
  	}
  
  	klog.Info("Finished collecting metrics for vGPUMonitor")
  }
  
  func (cc ClusterManagerCollector) collectGPUInfo(ch chan<- prometheus.Metric) error {
  	if err := cc.initNVML(); err != nil {
  		return err
  	}
  	defer nvml.Shutdown()
  
  	devnum, err := cc.getDeviceCount()
  	if err != nil {
  		return err
  	}
  
  	for ii := range devnum {
  		if err := cc.collectGPUDeviceMetrics(ch, ii); err != nil {
  			klog.Error("Failed to collect metrics for GPU device ", ii, ": ", err)
  		}
  	}
  
  	return nil
  }
  
  func (cc ClusterManagerCollector) initNVML() error {
  	nvret := nvml.Init()
  	if nvret != nvml.SUCCESS {
  		return fmt.Errorf("nvml Init err: %s", nvml.ErrorString(nvret))
  	}
  	return nil
  }
  
  func (cc ClusterManagerCollector) getDeviceCount() (int, error) {
  	devnum, nvret := nvml.DeviceGetCount()
  	if nvret != nvml.SUCCESS {
  		return 0, fmt.Errorf("nvml GetDeviceCount err: %s", nvml.ErrorString(nvret))
  	}
  	return devnum, nil
  }
  
  func (cc ClusterManagerCollector) collectGPUDeviceMetrics(ch chan<- prometheus.Metric, index int) error {
  	hdev, nvret := nvml.DeviceGetHandleByIndex(index)
  	if nvret != nvml.SUCCESS {
  		return fmt.Errorf("nvml DeviceGetHandleByIndex err: %s", nvml.ErrorString(nvret))
  	}
  
  	if err := cc.collectGPUMemoryMetrics(ch, hdev, index); err != nil {
  		return err
  	}
  
  	if err := cc.collectGPUUtilizationMetrics(ch, hdev, index); err != nil {
  		return err
  	}
  
  	return nil
  }
  
  func (cc ClusterManagerCollector) collectGPUMemoryMetrics(ch chan<- prometheus.Metric, hdev nvml.Device, index int) error {
  	memory, ret := hdev.GetMemoryInfo()
  	if ret == nvml.ERROR_NOT_SUPPORTED {
  		klog.V(3).Infof("Memory metrics not supported for device %d (unified memory architecture), skipping", index)
  		return nil
  	}
  	if ret != nvml.SUCCESS {
  		return fmt.Errorf("nvml get memory error ret=%d", ret)
  	}
  
  	uuid, nvret := hdev.GetUUID()
  	if nvret != nvml.SUCCESS {
  		return fmt.Errorf("nvml GetUUID err: %s", nvml.ErrorString(nvret))
  	}
  
  	deviceName, nvret := hdev.GetName()
  	if nvret != nvml.SUCCESS {
  		return fmt.Errorf("nvml GetName err: %s", nvml.ErrorString(nvret))
  	}
  
  	deviceName = "NVIDIA-" + deviceName
  
  	ch <- prometheus.MustNewConstMetric(
  		hostGPUdesc,
  		prometheus.GaugeValue,
  		float64(memory.Used),
  		fmt.Sprint(index), uuid, deviceName,
  	)
  
  	sendLegacyMetric(ch, legacyHostGPUdesc, prometheus.GaugeValue, float64(memory.Used),
  		fmt.Sprint(index), uuid, deviceName,
  	)
  
  	return nil
  }
  
  func (cc ClusterManagerCollector) collectGPUUtilizationMetrics(ch chan<- prometheus.Metric, hdev nvml.Device, index int) error {
  	util, nvret := hdev.GetUtilizationRates()
  	if nvret != nvml.SUCCESS {
  		return fmt.Errorf("nvml GetUtilizationRates err: %s", nvml.ErrorString(nvret))
  	}
  
  	uuid, nvret := hdev.GetUUID()
  	if nvret != nvml.SUCCESS {
  		return fmt.Errorf("nvml GetUUID err: %s", nvml.ErrorString(nvret))
  	}
  
  	deviceName, nvret := hdev.GetName()
  	if nvret != nvml.SUCCESS {
  		return fmt.Errorf("nvml GetName err: %s", nvml.ErrorString(nvret))
  	}
  
  	deviceName = "NVIDIA-" + deviceName
  
  	ch <- prometheus.MustNewConstMetric(
  		hostGPUUtilizationdesc,
  		prometheus.GaugeValue,
  		float64(util.Gpu),
  		fmt.Sprint(index), uuid, deviceName,
  	)
  
  	sendLegacyMetric(ch, legacyHostGPUUtilizationdesc, prometheus.GaugeValue, float64(util.Gpu),
  		fmt.Sprint(index), uuid, deviceName,
  	)
  
  	return nil
  }
  
  func (cc ClusterManagerCollector) collectPodAndContainerInfo(ch chan<- prometheus.Metric) error {
  	nodeName := os.Getenv(util.NodeNameEnvName)
  	if nodeName == "" {
  		return fmt.Errorf("node name environment variable %s is not set", util.NodeNameEnvName)
  	}
  
  	pods, err := cc.ClusterManager.PodLister.List(labels.SelectorFromSet(labels.Set{util.AssignedNodeAnnotations: nodeName}))
  	if err != nil {
  		klog.Errorf("Failed to list pods for node %s: %v", nodeName, err)
  		return fmt.Errorf("failed to list pods: %w", err)
  	}
  
  	containers := cc.ClusterManager.containerLister.ListContainers()
  	containerMap := make(map[string][]*nvidia.ContainerUsage) // podUID -> containers
  	for _, c := range containers {
  		if c.Info != nil && c.PodUID != "" {
  			containerMap[c.PodUID] = append(containerMap[c.PodUID], c)
  		}
  	}
  
  	nowSec := time.Now().Unix()
  
  	// Iterate through each Pod
  	for _, pod := range pods {
  		podContainers, found := containerMap[string(pod.UID)]
  		if !found {
  			klog.V(5).Infof("No containers found for pod %s/%s", pod.Namespace, pod.Name)
  			continue
  		}
  
  		klog.V(5).Infof("Processing Pod %s/%s", pod.Namespace, pod.Name)
  
  		// Iterate through each container in the Pod
  		for _, ctr := range pod.Spec.Containers {
  			// Find the matching container
  			for _, c := range podContainers {
  				if c.ContainerName == ctr.Name {
  					klog.V(5).Infof("Processing Container %s in Pod %s/%s", ctr.Name, pod.Namespace, pod.Name)
  					if err := cc.collectContainerMetrics(ch, pod, ctr, c, nowSec); err != nil {
  						klog.Errorf("Failed to collect metrics for container %s in Pod %s/%s: %v", ctr.Name, pod.Namespace, pod.Name, err)
  					}
  					break // Exit the inner loop after finding the matching container
  				}
  			}
  		}
  	}
  
  	klog.V(4).Infof("Finished collecting metrics for %d pods", len(pods))
  	return nil
  }
  
  func (cc ClusterManagerCollector) isPodUIDMatched(pod *corev1.Pod, podUID string) bool {
  	if pod == nil {
  		return false
  	}
  	return string(pod.UID) == podUID
  }
  
  func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.Metric, pod *corev1.Pod, ctr corev1.Container, c *nvidia.ContainerUsage, nowSec int64) error {
  	// Validate inputs
  	if c == nil || c.Info == nil {
  		klog.Errorf("Container or ContainerInfo is nil for Pod %s/%s, Container %s", pod.Namespace, pod.Name, ctr.Name)
  		return fmt.Errorf("container or container info is nil")
  	}
  
  	// Iterate through each device
  	for i := range c.Info.DeviceNum() {
  		uuid := c.Info.DeviceUUID(i)
  		if len(uuid) < 40 {
  			klog.Errorf("Invalid UUID length for device %d in Pod %s/%s, Container %s", i, pod.Namespace, pod.Name, ctr.Name)
  			return fmt.Errorf("invalid UUID length for device %d", i)
  		}
  		uuid = uuid[0:40] // Ensure UUID is truncated to 40 characters
  		if !utf8.ValidString(uuid) {
  			klog.Warningf("Device %d in Pod %s/%s, Container %s has invalid UTF-8 UUID (shared memory not yet initialised); skipping until next scrape", i, pod.Namespace, pod.Name, ctr.Name)
  			continue
  		}
  
  		// Collect device metrics
  		memoryTotal := c.Info.DeviceMemoryTotal(i)
  		memoryLimit := c.Info.DeviceMemoryLimit(i)
  		memoryContextSize := c.Info.DeviceMemoryContextSize(i)
  		memoryModuleSize := c.Info.DeviceMemoryModuleSize(i)
  		memoryBufferSize := c.Info.DeviceMemoryBufferSize(i)
  		smUtil := c.Info.DeviceSmUtil(i)
  		lastKernelTime := c.Info.LastKernelTime()
  
  		labels := []string{pod.Namespace, pod.Name, ctr.Name, fmt.Sprint(i), uuid}
  
  		if err := sendMetric(ch, ctrvGPUdesc, prometheus.GaugeValue, float64(memoryTotal), labels...); err != nil {
  			klog.Errorf("Failed to send memoryTotal metric: %v", err)
  			return err
  		}
  		sendLegacyMetric(ch, legacyCtrvGPUdesc, prometheus.GaugeValue, float64(memoryTotal), labels...)
  
  		if err := sendMetric(ch, ctrvGPUlimitdesc, prometheus.GaugeValue, float64(memoryLimit), labels...); err != nil {
  			klog.Errorf("Failed to send memoryLimit metric: %v", err)
  			return err
  		}
  		sendLegacyMetric(ch, legacyCtrvGPUlimitdesc, prometheus.GaugeValue, float64(memoryLimit), labels...)
  
  		memoryOffset := memoryTotal - memoryContextSize - memoryModuleSize - memoryBufferSize
  		memoryLabels := append(labels, fmt.Sprint(memoryContextSize), fmt.Sprint(memoryModuleSize), fmt.Sprint(memoryBufferSize), fmt.Sprint(memoryOffset))
  		if err := sendMetric(ch, ctrDeviceMemorydesc, prometheus.GaugeValue, float64(memoryTotal), memoryLabels...); err != nil {
  			klog.Errorf("Failed to send device memory desc: %v", err)
  			return err
  		}
  		sendLegacyMetric(ch, legacyCtrDeviceMemorydesc, prometheus.GaugeValue, float64(memoryTotal), memoryLabels...)
  
  		if err := sendMetric(ch, ctrDeviceUtilizationdesc, prometheus.GaugeValue, float64(smUtil), labels...); err != nil {
  			klog.Errorf("Failed to send device utilization desc: %v", err)
  			return err
  		}
  		sendLegacyMetric(ch, legacyCtrDeviceUtilizationdesc, prometheus.GaugeValue, float64(smUtil), labels...)
  
  		if err := sendMetric(ch, ctrDeviceMemoryContextDesc, prometheus.GaugeValue, float64(memoryContextSize), labels...); err != nil {
  			klog.Errorf("Failed to send Device Memory context size metric: %v", err)
  			return err
  		}
  		if err := sendMetric(ch, ctrDeviceMemoryModuleDesc, prometheus.GaugeValue, float64(memoryModuleSize), labels...); err != nil {
  			klog.Errorf("Failed to send Device Memory module size metric: %v", err)
  			return err
  		}
  		if err := sendMetric(ch, ctrDeviceMemoryBufferDesc, prometheus.GaugeValue, float64(memoryBufferSize), labels...); err != nil {
  			klog.Errorf("Failed to send Device Memory buffer size metric: %v", err)
  			return err
  		}
  
  		if lastKernelTime > 0 {
  			lastSec := max(nowSec-lastKernelTime, 0)
  			if err := sendMetric(ch, ctrDeviceLastKernelDesc, prometheus.GaugeValue, float64(lastSec), labels...); err != nil {
  				klog.Errorf("Failed to send last kernel time metric: %v", err)
  				return err
  			}
  			sendLegacyMetric(ch, legacyCtrDeviceLastKernelDesc, prometheus.GaugeValue, float64(lastSec), labels...)
  		}
  	}
  
  	klog.V(5).Infof("Successfully collected metrics for Pod %s/%s, Container %s", pod.Namespace, pod.Name, ctr.Name)
  	return nil
  }
  
  func (cc ClusterManagerCollector) collectPodAndContainerMigInfo(ch chan<- prometheus.Metric) error {
  	nodeName := os.Getenv(util.NodeNameEnvName)
  	if nodeName == "" {
  		return fmt.Errorf("node name environment variable %s is not set", util.NodeNameEnvName)
  	}
  
  	pods, err := cc.ClusterManager.PodLister.List(labels.SelectorFromSet(labels.Set{util.AssignedNodeAnnotations: nodeName}))
  	if err != nil {
  		klog.Errorf("Failed to list pods for node %s: %v", nodeName, err)
  		return fmt.Errorf("failed to list pods: %w", err)
  	}
  	for _, pod := range pods {
  		pdevices, err := device.DecodePodDevices(device.SupportDevices, pod.Annotations)
  		if err != nil {
  			return fmt.Errorf("failed to decode pod devices: %w", err)
  		}
  		for ctrIdx, container := range pod.Spec.Containers {
  			for ctrDevIdx, ctrDevices := range pdevices[nv.NvidiaGPUDevice] {
  				if len(ctrDevices) == 0 || ctrIdx != ctrDevIdx {
  					continue
  				}
  				for _, ctrDev := range ctrDevices {
  					if strings.Contains(ctrDev.UUID, "[") {
  						uuid := strings.Split(ctrDev.UUID, "[")[0]
  						_, idx, err := device.ExtractMigTemplatesFromUUID(ctrDev.UUID)
  						if err != nil {
  							klog.Errorf("Failed to get mig template for device %s in Pod %s/%s, container %s: %v", ctrDev.UUID, pod.Namespace, pod.Name, container.Name, err)
  							continue
  						}
  						gpuInstanceId, err := dp.GetMigGpuInstanceIdFromIndex(ctrDev.UUID, idx)
  						if err != nil {
  							klog.Errorf("Failed to get mig InstanceId for device %s in Pod %s/%s, container %s: %v", ctrDev.UUID, pod.Namespace, pod.Name, container.Name, err)
  							continue
  						}
  						labels := []string{pod.Namespace, pod.Name, container.Name, fmt.Sprint(idx), uuid, fmt.Sprint(gpuInstanceId)}
  						if err := sendMetric(ch, ctrDeviceMigInfo, prometheus.GaugeValue, 1, labels...); err != nil {
  							klog.Errorf("Failed to send mig info metric for device %s in Pod %s/%s, container %s: %v", ctrDev.UUID, pod.Namespace, pod.Name, container.Name, err)
  							return err
  						}
  						sendLegacyMetric(ch, legacyCtrDeviceMigInfo, prometheus.GaugeValue, 1, labels...)
  					}
  				}
  			}
  		}
  	}
  	return nil
  }
  
  func sendMetric(ch chan<- prometheus.Metric, desc *prometheus.Desc, valueType prometheus.ValueType, value float64, labels ...string) error {
  	metric, err := prometheus.NewConstMetric(desc, valueType, value, labels...)
  	if err != nil {
  		return fmt.Errorf("failed to create metric: %w", err)
  	}
  	ch <- metric
  	return nil
  }
  
  // NewClusterManager first creates a Prometheus-ignorant ClusterManager
  // instance. Then, it creates a ClusterManagerCollector for the just created
  // ClusterManager. Finally, it registers the ClusterManagerCollector with a
  // wrapping Registerer that adds the zone as a label. In this way, the metrics
  // collected by different ClusterManagerCollectors do not collide.
  func NewClusterManager(zone string, reg prometheus.Registerer, containerLister *nvidia.ContainerLister, legacyMetrics bool) *ClusterManager {
  	if legacyMetrics {
  		initLegacyDescriptors()
  	}
  	c := &ClusterManager{
  		Zone:            zone,
  		containerLister: containerLister,
  		LegacyMetrics:   legacyMetrics,
  	}
  
  	informerFactory := informers.NewSharedInformerFactoryWithOptions(containerLister.Clientset(), time.Hour*1)
  	c.PodLister = informerFactory.Core().V1().Pods().Lister()
  	stopCh := make(chan struct{})
  	informerFactory.Start(stopCh)
  
  	cc := ClusterManagerCollector{ClusterManager: c}
  	prometheus.WrapRegistererWith(prometheus.Labels{"zone": zone}, reg).MustRegister(cc)
  	return c
  }
  