mirror of
https://github.com/prometheus-community/windows_exporter.git
synced 2026-10-17 10:09:05 +02:00
656 lines
21 KiB
Go
656 lines
21 KiB
Go
// SPDX-License-Identifier: Apache-2.0
|
|
//
|
|
// Copyright The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//go:build windows
|
|
|
|
package gpu
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"github.com/alecthomas/kingpin/v2"
|
|
"github.com/prometheus-community/windows_exporter/internal/headers/cfgmgr32"
|
|
"github.com/prometheus-community/windows_exporter/internal/headers/gdi32"
|
|
"github.com/prometheus-community/windows_exporter/internal/labelcache"
|
|
"github.com/prometheus-community/windows_exporter/internal/mi"
|
|
"github.com/prometheus-community/windows_exporter/internal/pdh"
|
|
"github.com/prometheus-community/windows_exporter/internal/types"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
)
|
|
|
|
const Name = "gpu"
|
|
|
|
type Config struct{}
|
|
|
|
//nolint:gochecknoglobals
|
|
var ConfigDefaults = Config{}
|
|
|
|
// deviceCacheRefreshInterval limits how often the GPU device cache is
|
|
// rebuilt when a performance counter instance references an unknown LUID.
|
|
const deviceCacheRefreshInterval = 30 * time.Second
|
|
|
|
type Collector struct {
|
|
config Config
|
|
logger *slog.Logger
|
|
|
|
// gpuDeviceCache maps the LUID ("0x%08X_0x%08X") of each adapter to its device information.
|
|
// LUIDs change if a driver is updated or a device is restarted, so the cache is
|
|
// refreshed whenever a performance counter instance references an unknown LUID.
|
|
gpuDeviceCache map[string]gpuDevice
|
|
gpuDeviceCacheLastRefresh time.Time
|
|
|
|
// discoverGPUDevices enumerates the GPU adapters. If nil, gdi32.GetGPUDevices is used.
|
|
// It is replaced in tests to make the discovery deterministic.
|
|
discoverGPUDevices func() ([]gdi32.GPUDevice, error)
|
|
|
|
// labels keeps the label pairs of the series of the last scrape, and
|
|
// instances the parsed counter instance names, so neither is built again
|
|
// while the GPU process or engine stays.
|
|
labels labelcache.Cache
|
|
instances instanceCache
|
|
|
|
// GPU Engine
|
|
gpuEnginePerfDataCollector *pdh.Collector[gpuEnginePerfDataCounterValues]
|
|
gpuEnginePerfDataObject []gpuEnginePerfDataCounterValues
|
|
|
|
gpuInfo *prometheus.Desc
|
|
gpuEngineRunningTime *prometheus.Desc
|
|
|
|
gpuSharedSystemMemorySize *prometheus.Desc
|
|
gpuDedicatedSystemMemorySize *prometheus.Desc
|
|
gpuDedicatedVideoMemorySize *prometheus.Desc
|
|
|
|
// GPU Adapter Memory
|
|
gpuAdapterMemoryPerfDataCollector *pdh.Collector[gpuAdapterMemoryPerfDataCounterValues]
|
|
gpuAdapterMemoryPerfDataObject []gpuAdapterMemoryPerfDataCounterValues
|
|
|
|
gpuAdapterMemoryDedicatedUsage *prometheus.Desc
|
|
gpuAdapterMemorySharedUsage *prometheus.Desc
|
|
gpuAdapterMemoryTotalCommitted *prometheus.Desc
|
|
|
|
// GPU Local Adapter Memory
|
|
gpuLocalAdapterMemoryPerfDataCollector *pdh.Collector[gpuLocalAdapterMemoryPerfDataCounterValues]
|
|
gpuLocalAdapterMemoryPerfDataObject []gpuLocalAdapterMemoryPerfDataCounterValues
|
|
|
|
gpuLocalAdapterMemoryUsage *prometheus.Desc
|
|
|
|
// GPU Non Local Adapter Memory
|
|
gpuNonLocalAdapterMemoryPerfDataCollector *pdh.Collector[gpuNonLocalAdapterMemoryPerfDataCounterValues]
|
|
gpuNonLocalAdapterMemoryPerfDataObject []gpuNonLocalAdapterMemoryPerfDataCounterValues
|
|
|
|
gpuNonLocalAdapterMemoryUsage *prometheus.Desc
|
|
|
|
// GPU Process Memory
|
|
gpuProcessMemoryPerfDataCollector *pdh.Collector[gpuProcessMemoryPerfDataCounterValues]
|
|
gpuProcessMemoryPerfDataObject []gpuProcessMemoryPerfDataCounterValues
|
|
|
|
gpuProcessMemoryDedicatedUsage *prometheus.Desc
|
|
gpuProcessMemoryLocalUsage *prometheus.Desc
|
|
gpuProcessMemoryNonLocalUsage *prometheus.Desc
|
|
gpuProcessMemorySharedUsage *prometheus.Desc
|
|
gpuProcessMemoryTotalCommitted *prometheus.Desc
|
|
|
|
// GPU sensors
|
|
sensorMetrics sensorMetrics
|
|
}
|
|
|
|
type gpuDevice struct {
|
|
gdi32 gdi32.GPUDevice
|
|
cfgmgr32 cfgmgr32.Device
|
|
sensors gpuSensors
|
|
ID string
|
|
|
|
// skip marks adapters that are known, but not exposed, e.g. software devices
|
|
// like the Microsoft Basic Render Driver. Keeping them in the cache prevents
|
|
// their performance counter instances from triggering a cache refresh.
|
|
skip bool
|
|
}
|
|
|
|
func New(config *Config) *Collector {
|
|
if config == nil {
|
|
config = &ConfigDefaults
|
|
}
|
|
|
|
c := &Collector{
|
|
config: *config,
|
|
}
|
|
|
|
return c
|
|
}
|
|
|
|
func NewWithFlags(_ *kingpin.Application) *Collector {
|
|
return &Collector{}
|
|
}
|
|
|
|
func (c *Collector) GetName() string {
|
|
return Name
|
|
}
|
|
|
|
func (c *Collector) Close() error {
|
|
c.gpuEnginePerfDataCollector.Close()
|
|
c.gpuAdapterMemoryPerfDataCollector.Close()
|
|
c.gpuLocalAdapterMemoryPerfDataCollector.Close()
|
|
c.gpuNonLocalAdapterMemoryPerfDataCollector.Close()
|
|
c.gpuProcessMemoryPerfDataCollector.Close()
|
|
c.labels.Reset()
|
|
c.instances.reset()
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error {
|
|
var err error
|
|
|
|
c.logger = logger.With(slog.String("collector", Name))
|
|
|
|
c.gpuInfo = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "info"),
|
|
"A metric with a constant '1' value labeled with GPU device information.",
|
|
[]string{"luid", "device_id", "name", "bus_number", "device_number", "function_number", "driver_version", "wddm_version", "architecture"},
|
|
nil,
|
|
)
|
|
|
|
c.gpuSharedSystemMemorySize = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "shared_system_memory_size_bytes"),
|
|
"The size, in bytes, of memory from system memory that can be shared by many users.",
|
|
[]string{"luid", "device_id"},
|
|
nil,
|
|
)
|
|
c.gpuDedicatedSystemMemorySize = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "dedicated_system_memory_size_bytes"),
|
|
"The size, in bytes, of memory that is dedicated from system memory.",
|
|
[]string{"luid", "device_id"},
|
|
nil,
|
|
)
|
|
c.gpuDedicatedVideoMemorySize = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "dedicated_video_memory_size_bytes"),
|
|
"The size, in bytes, of memory that is dedicated from video memory.",
|
|
[]string{"luid", "device_id"},
|
|
nil,
|
|
)
|
|
|
|
c.gpuEngineRunningTime = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "engine_time_seconds"),
|
|
"Total running time of the GPU engine in seconds.",
|
|
[]string{"process_id", "luid", "device_id", "phys", "eng", "engtype"},
|
|
nil,
|
|
)
|
|
|
|
c.gpuAdapterMemoryDedicatedUsage = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "adapter_memory_dedicated_bytes"),
|
|
"Dedicated GPU memory usage in bytes.",
|
|
[]string{"luid", "device_id", "phys"},
|
|
nil,
|
|
)
|
|
c.gpuAdapterMemorySharedUsage = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "adapter_memory_shared_bytes"),
|
|
"Shared GPU memory usage in bytes.",
|
|
[]string{"luid", "device_id", "phys"},
|
|
nil,
|
|
)
|
|
c.gpuAdapterMemoryTotalCommitted = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "adapter_memory_committed_bytes"),
|
|
"Total committed GPU memory in bytes.",
|
|
[]string{"luid", "device_id", "phys"},
|
|
nil,
|
|
)
|
|
|
|
c.gpuLocalAdapterMemoryUsage = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "local_adapter_memory_bytes"),
|
|
"Local adapter memory usage in bytes.",
|
|
[]string{"luid", "device_id", "phys", "part"},
|
|
nil,
|
|
)
|
|
|
|
c.gpuNonLocalAdapterMemoryUsage = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "non_local_adapter_memory_bytes"),
|
|
"Non-local adapter memory usage in bytes.",
|
|
[]string{"luid", "device_id", "phys", "part"},
|
|
nil,
|
|
)
|
|
|
|
c.gpuProcessMemoryDedicatedUsage = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "process_memory_dedicated_bytes"),
|
|
"Dedicated process memory usage in bytes.",
|
|
[]string{"process_id", "luid", "device_id", "phys"},
|
|
nil,
|
|
)
|
|
c.gpuProcessMemoryLocalUsage = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "process_memory_local_bytes"),
|
|
"Local process memory usage in bytes.",
|
|
[]string{"process_id", "luid", "device_id", "phys"},
|
|
nil,
|
|
)
|
|
c.gpuProcessMemoryNonLocalUsage = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "process_memory_non_local_bytes"),
|
|
"Non-local process memory usage in bytes.",
|
|
[]string{"process_id", "luid", "device_id", "phys"},
|
|
nil,
|
|
)
|
|
c.gpuProcessMemorySharedUsage = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "process_memory_shared_bytes"),
|
|
"Shared process memory usage in bytes.",
|
|
[]string{"process_id", "luid", "device_id", "phys"},
|
|
nil,
|
|
)
|
|
c.gpuProcessMemoryTotalCommitted = prometheus.NewDesc(
|
|
prometheus.BuildFQName(types.Namespace, Name, "process_memory_committed_bytes"),
|
|
"Total committed process memory in bytes.",
|
|
[]string{"process_id", "luid", "device_id", "phys"},
|
|
nil,
|
|
)
|
|
|
|
c.sensorMetrics = newSensorMetrics()
|
|
|
|
errs := make([]error, 0)
|
|
|
|
c.gpuEnginePerfDataCollector, err = pdh.NewCollector[gpuEnginePerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Engine", pdh.InstancesAll)
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("failed to create GPU Engine perf data collector: %w", err))
|
|
}
|
|
|
|
c.gpuAdapterMemoryPerfDataCollector, err = pdh.NewCollector[gpuAdapterMemoryPerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Adapter Memory", pdh.InstancesAll)
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("failed to create GPU Adapter Memory perf data collector: %w", err))
|
|
}
|
|
|
|
c.gpuLocalAdapterMemoryPerfDataCollector, err = pdh.NewCollector[gpuLocalAdapterMemoryPerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Local Adapter Memory", pdh.InstancesAll)
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("failed to create GPU Local Adapter Memory perf data collector: %w", err))
|
|
}
|
|
|
|
c.gpuNonLocalAdapterMemoryPerfDataCollector, err = pdh.NewCollector[gpuNonLocalAdapterMemoryPerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Non Local Adapter Memory", pdh.InstancesAll)
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("failed to create GPU Non Local Adapter Memory perf data collector: %w", err))
|
|
}
|
|
|
|
c.gpuProcessMemoryPerfDataCollector, err = pdh.NewCollector[gpuProcessMemoryPerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Process Memory", pdh.InstancesAll)
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("failed to create GPU Process Memory perf data collector: %w", err))
|
|
}
|
|
|
|
// GPU discovery problems are not fatal. The collector keeps working with the
|
|
// adapters that could be discovered and retries on unknown LUIDs during Collect.
|
|
c.refreshGPUDevices()
|
|
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
// refreshGPUDevices rebuilds the GPU device cache.
|
|
// Problems with single adapters are logged and do not stop the discovery of the remaining adapters.
|
|
func (c *Collector) refreshGPUDevices() {
|
|
c.gpuDeviceCacheLastRefresh = time.Now()
|
|
|
|
discoverGPUDevices := c.discoverGPUDevices
|
|
if discoverGPUDevices == nil {
|
|
discoverGPUDevices = gdi32.GetGPUDevices
|
|
}
|
|
|
|
gpus, err := discoverGPUDevices()
|
|
if err != nil {
|
|
c.logger.Warn("failed to discover some GPU devices",
|
|
slog.Any("err", err),
|
|
)
|
|
|
|
// Keep the previous cache, if the discovery failed completely.
|
|
if len(gpus) == 0 {
|
|
return
|
|
}
|
|
}
|
|
|
|
gpuDeviceCache := make(map[string]gpuDevice, len(gpus))
|
|
|
|
for _, gpu := range gpus {
|
|
luidKey := fmt.Sprintf("0x%08X_0x%08X", gpu.LUID.HighPart, gpu.LUID.LowPart)
|
|
|
|
// Skip software devices like the Microsoft Basic Render Driver.
|
|
if gpu.AdapterString == "" || gpu.IsSoftwareDevice() || isBasicRenderDriver(gpu) {
|
|
gpuDeviceCache[luidKey] = gpuDevice{gdi32: gpu, ID: gpu.DeviceID, skip: true}
|
|
|
|
c.logger.Debug("Skipping GPU device",
|
|
slog.String("name", gpu.AdapterString),
|
|
slog.String("luid", luidKey),
|
|
slog.String("device_id", gpu.DeviceID),
|
|
slog.Uint64("adapter_type", uint64(gpu.AdapterType)),
|
|
)
|
|
|
|
continue
|
|
}
|
|
|
|
device := c.newGPUDevice(gpu, luidKey)
|
|
device.sensors = discoverSensors(c.logger.With(slog.String("luid", luidKey)), gpu)
|
|
gpuDeviceCache[luidKey] = device
|
|
|
|
c.logger.Debug("Found GPU device",
|
|
slog.String("name", gpu.AdapterString),
|
|
slog.String("luid", luidKey),
|
|
slog.String("device_id", device.ID),
|
|
slog.Uint64("bus_number", uint64(gpu.BusNumber)),
|
|
slog.Uint64("device_number", uint64(gpu.DeviceNumber)),
|
|
slog.Uint64("function_number", uint64(gpu.FunctionNumber)),
|
|
)
|
|
}
|
|
|
|
c.gpuDeviceCache = gpuDeviceCache
|
|
}
|
|
|
|
// newGPUDevice resolves the PnP device instance ID of the adapter.
|
|
// If the device instance can't be found, the PnP device ID reported by gdi32 is used as ID.
|
|
func (c *Collector) newGPUDevice(gpu gdi32.GPUDevice, luidKey string) gpuDevice {
|
|
device := gpuDevice{
|
|
gdi32: gpu,
|
|
ID: gpu.DeviceID,
|
|
}
|
|
|
|
cfgmgr32Devs, err := cfgmgr32.GetDevicesInstanceIDs(gpu.DeviceID)
|
|
|
|
for _, dev := range cfgmgr32Devs {
|
|
if dev.BusNumber == gpu.BusNumber && dev.DeviceNumber == gpu.DeviceNumber && dev.FunctionNumber == gpu.FunctionNumber {
|
|
device.cfgmgr32 = dev
|
|
device.ID = dev.InstanceID
|
|
|
|
return device
|
|
}
|
|
}
|
|
|
|
attrs := []any{
|
|
slog.String("name", gpu.AdapterString),
|
|
slog.String("luid", luidKey),
|
|
slog.String("device_id", gpu.DeviceID),
|
|
}
|
|
|
|
if err != nil {
|
|
attrs = append(attrs, slog.Any("err", err))
|
|
}
|
|
|
|
c.logger.Warn("failed to find matching PnP device instance for GPU device, using the PnP device ID as device_id", attrs...)
|
|
|
|
return device
|
|
}
|
|
|
|
// getGPUDevice returns the cached device for the given LUID.
|
|
// If the LUID is unknown, e.g. because a driver was updated, a device was restarted
|
|
// or a device was hot-plugged, the cache is refreshed at most once per deviceCacheRefreshInterval.
|
|
// Devices marked as skip are not returned.
|
|
func (c *Collector) getGPUDevice(luid string) (gpuDevice, bool) {
|
|
if luid == "" {
|
|
return gpuDevice{}, false
|
|
}
|
|
|
|
device, ok := c.gpuDeviceCache[luid]
|
|
if !ok && time.Since(c.gpuDeviceCacheLastRefresh) >= deviceCacheRefreshInterval {
|
|
c.logger.Debug("unknown GPU LUID, refreshing GPU device cache",
|
|
slog.String("luid", luid),
|
|
)
|
|
|
|
c.refreshGPUDevices()
|
|
|
|
device, ok = c.gpuDeviceCache[luid]
|
|
}
|
|
|
|
return device, ok && !device.skip
|
|
}
|
|
|
|
func (c *Collector) Collect(ch chan<- prometheus.Metric, _ time.Duration) error {
|
|
errs := make([]error, 0)
|
|
|
|
scrape := c.labels.Begin()
|
|
defer scrape.Commit()
|
|
|
|
instances := c.instances.begin()
|
|
defer instances.commit()
|
|
|
|
if err := c.collectGpuEngineMetrics(ch, scrape, instances); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
if err := c.collectGpuAdapterMemoryMetrics(ch, scrape, instances); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
if err := c.collectGpuLocalAdapterMemoryMetrics(ch, scrape, instances); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
if err := c.collectGpuNonLocalAdapterMemoryMetrics(ch, scrape, instances); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
if err := c.collectGpuProcessMemoryMetrics(ch, scrape, instances); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
|
|
// Collected last, since the metrics above may refresh the GPU device cache.
|
|
c.collectGpuInfo(ch)
|
|
c.collectGpuSensorMetrics(ch)
|
|
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
func (c *Collector) collectGpuInfo(ch chan<- prometheus.Metric) {
|
|
for luid, gpu := range c.gpuDeviceCache {
|
|
if gpu.skip {
|
|
continue
|
|
}
|
|
|
|
ch <- prometheus.MustNewConstMetric(
|
|
c.gpuInfo,
|
|
prometheus.GaugeValue,
|
|
1.0,
|
|
luid,
|
|
gpu.ID,
|
|
gpu.gdi32.AdapterString,
|
|
gpu.gdi32.BusNumber.String(),
|
|
gpu.gdi32.DeviceNumber.String(),
|
|
gpu.gdi32.FunctionNumber.String(),
|
|
gpu.sensors.driverVersion,
|
|
gpu.sensors.wddmVersion,
|
|
gpu.sensors.architecture,
|
|
)
|
|
|
|
ch <- prometheus.MustNewConstMetric(
|
|
c.gpuSharedSystemMemorySize,
|
|
prometheus.GaugeValue,
|
|
float64(gpu.gdi32.SharedSystemMemorySize),
|
|
luid, gpu.ID,
|
|
)
|
|
|
|
ch <- prometheus.MustNewConstMetric(
|
|
c.gpuDedicatedSystemMemorySize,
|
|
prometheus.GaugeValue,
|
|
float64(gpu.gdi32.DedicatedSystemMemorySize),
|
|
luid, gpu.ID,
|
|
)
|
|
|
|
ch <- prometheus.MustNewConstMetric(
|
|
c.gpuDedicatedVideoMemorySize,
|
|
prometheus.GaugeValue,
|
|
float64(gpu.gdi32.DedicatedVideoMemorySize),
|
|
luid, gpu.ID,
|
|
)
|
|
}
|
|
}
|
|
|
|
func (c *Collector) collectGpuEngineMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error {
|
|
// Collect the GPU Engine perf data.
|
|
if err := c.gpuEnginePerfDataCollector.Collect(&c.gpuEnginePerfDataObject); err != nil {
|
|
return fmt.Errorf("failed to collect GPU Engine perf data: %w", err)
|
|
}
|
|
|
|
// Iterate over the GPU Engine perf data and aggregate the values.
|
|
for _, data := range c.gpuEnginePerfDataObject {
|
|
instance := instances.parse(data.Name)
|
|
|
|
device, ok := c.getGPUDevice(instance.Luid)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuEngineRunningTime,
|
|
prometheus.CounterValue,
|
|
data.RunningTime/10_000_000,
|
|
instance.Pid, instance.Luid, device.ID, instance.Phys, instance.Eng, instance.Engtype,
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *Collector) collectGpuAdapterMemoryMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error {
|
|
// Collect the GPU Adapter Memory perf data.
|
|
if err := c.gpuAdapterMemoryPerfDataCollector.Collect(&c.gpuAdapterMemoryPerfDataObject); err != nil {
|
|
return fmt.Errorf("failed to collect GPU Adapter Memory perf data: %w", err)
|
|
}
|
|
|
|
for _, data := range c.gpuAdapterMemoryPerfDataObject {
|
|
instance := instances.parse(data.Name)
|
|
|
|
device, ok := c.getGPUDevice(instance.Luid)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuAdapterMemoryDedicatedUsage,
|
|
prometheus.GaugeValue,
|
|
data.DedicatedUsage,
|
|
instance.Luid, device.ID, instance.Phys,
|
|
)
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuAdapterMemorySharedUsage,
|
|
prometheus.GaugeValue,
|
|
data.SharedUsage,
|
|
instance.Luid, device.ID, instance.Phys,
|
|
)
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuAdapterMemoryTotalCommitted,
|
|
prometheus.GaugeValue,
|
|
data.TotalCommitted,
|
|
instance.Luid, device.ID, instance.Phys,
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *Collector) collectGpuLocalAdapterMemoryMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error {
|
|
// Collect the GPU Local Adapter Memory perf data.
|
|
if err := c.gpuLocalAdapterMemoryPerfDataCollector.Collect(&c.gpuLocalAdapterMemoryPerfDataObject); err != nil {
|
|
return fmt.Errorf("failed to collect GPU Local Adapter Memory perf data: %w", err)
|
|
}
|
|
|
|
for _, data := range c.gpuLocalAdapterMemoryPerfDataObject {
|
|
instance := instances.parse(data.Name)
|
|
|
|
device, ok := c.getGPUDevice(instance.Luid)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuLocalAdapterMemoryUsage,
|
|
prometheus.GaugeValue,
|
|
data.LocalUsage,
|
|
instance.Luid, device.ID, instance.Phys, instance.Part,
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *Collector) collectGpuNonLocalAdapterMemoryMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error {
|
|
// Collect the GPU Non Local Adapter Memory perf data.
|
|
if err := c.gpuNonLocalAdapterMemoryPerfDataCollector.Collect(&c.gpuNonLocalAdapterMemoryPerfDataObject); err != nil {
|
|
return fmt.Errorf("failed to collect GPU Non Local Adapter Memory perf data: %w", err)
|
|
}
|
|
|
|
for _, data := range c.gpuNonLocalAdapterMemoryPerfDataObject {
|
|
instance := instances.parse(data.Name)
|
|
|
|
device, ok := c.getGPUDevice(instance.Luid)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuNonLocalAdapterMemoryUsage,
|
|
prometheus.GaugeValue,
|
|
data.NonLocalUsage,
|
|
instance.Luid, device.ID, instance.Phys, instance.Part,
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *Collector) collectGpuProcessMemoryMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error {
|
|
// Collect the GPU Process Memory perf data.
|
|
if err := c.gpuProcessMemoryPerfDataCollector.Collect(&c.gpuProcessMemoryPerfDataObject); err != nil {
|
|
return fmt.Errorf("failed to collect GPU Process Memory perf data: %w", err)
|
|
}
|
|
|
|
for _, data := range c.gpuProcessMemoryPerfDataObject {
|
|
instance := instances.parse(data.Name)
|
|
|
|
device, ok := c.getGPUDevice(instance.Luid)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuProcessMemoryDedicatedUsage,
|
|
prometheus.GaugeValue,
|
|
data.DedicatedUsage,
|
|
instance.Pid, instance.Luid, device.ID, instance.Phys,
|
|
)
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuProcessMemoryLocalUsage,
|
|
prometheus.GaugeValue,
|
|
data.LocalUsage,
|
|
instance.Pid, instance.Luid, device.ID, instance.Phys,
|
|
)
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuProcessMemoryNonLocalUsage,
|
|
prometheus.GaugeValue,
|
|
data.NonLocalUsage,
|
|
instance.Pid, instance.Luid, device.ID, instance.Phys,
|
|
)
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuProcessMemorySharedUsage,
|
|
prometheus.GaugeValue,
|
|
data.SharedUsage,
|
|
instance.Pid, instance.Luid, device.ID, instance.Phys,
|
|
)
|
|
|
|
ch <- scrape.MustNewConstMetric(
|
|
c.gpuProcessMemoryTotalCommitted,
|
|
prometheus.GaugeValue,
|
|
data.TotalCommitted,
|
|
instance.Pid, instance.Luid, device.ID, instance.Phys,
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|