From 15f290c8c85b0ca044179521b39cc7795fd6c073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Fri, 9 Oct 2026 09:12:12 +0200 Subject: [PATCH] gpu: add vendor-neutral sensor metrics from dxgkrnl (#2571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Claude Opus 5.5 Signed-off-by: Jan-Otto Kröpke --- docs/collector.gpu.md | 74 ++++- internal/collector/gpu/gpu.go | 16 +- internal/collector/gpu/sensors.go | 425 ++++++++++++++++++++++++ internal/collector/gpu/sensors_test.go | 53 +++ internal/headers/gdi32/gdi32.go | 14 + internal/headers/gdi32/perfdata.go | 163 +++++++++ internal/headers/gdi32/perfdata_test.go | 86 +++++ internal/headers/gdi32/types.go | 67 ++++ 8 files changed, 883 insertions(+), 15 deletions(-) create mode 100644 internal/collector/gpu/sensors.go create mode 100644 internal/collector/gpu/sensors_test.go create mode 100644 internal/headers/gdi32/perfdata.go create mode 100644 internal/headers/gdi32/perfdata_test.go diff --git a/docs/collector.gpu.md b/docs/collector.gpu.md index 3da22978..0322e31e 100644 --- a/docs/collector.gpu.md +++ b/docs/collector.gpu.md @@ -1,12 +1,12 @@ -# gpu collector +# GPU collector -The gpu collector exposes metrics about GPU usage and memory consumption, both at the adapter (physical GPU) and +The GPU collector exposes metrics about GPU usage and memory consumption, both at the adapter (physical GPU) and per-process level. | | | |---------------------|--------------------------------------| | Metric name prefix | `gpu` | -| Data source | Perflib | +| Data source | Perflib, D3DKMT (dxgkrnl) | | Counters | GPU Engine, GPU Adapter, GPU Process | | Enabled by default? | No | @@ -20,17 +20,43 @@ These metrics are available on supported versions of Windows with compatible GPU ### Adapter-level Metrics -| Name | Description | Type | Labels | -|--------------------------------------------------|------------------------------------------------------------------------------------|-------|-----------------------------------------------------------------| -| `windows_gpu_info` | A metric with a constant '1' value labeled with gpu device information. | gauge | `bus_number`,`device_id`,`function_number`,`luid`,`name`,`phys` | -| `windows_gpu_dedicated_system_memory_size_bytes` | The size, in bytes, of memory that is dedicated from system memory. | gauge | `device_id`,`luid` | -| `windows_gpu_dedicated_video_memory_size_bytes` | The size, in bytes, of memory that is dedicated from video memory. | gauge | `device_id`,`luid` | -| `windows_gpu_shared_system_memory_size_bytes` | The size, in bytes, of memory from system memory that can be shared by many users. | gauge | `device_id`,`luid` | -| `windows_gpu_adapter_memory_committed_bytes` | Total committed GPU memory in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | -| `windows_gpu_adapter_memory_dedicated_bytes` | Dedicated GPU memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | -| `windows_gpu_adapter_memory_shared_bytes` | Shared GPU memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | -| `windows_gpu_local_adapter_memory_bytes` | Local adapter memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys`,`part` | -| `windows_gpu_non_local_adapter_memory_bytes` | Non-local adapter memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys`,`part` | +| Name | Description | Type | Labels | +|--------------------------------------------------|------------------------------------------------------------------------------------|-------|----------------------------------------------------------------------------------------------------------------| +| `windows_gpu_info` | A metric with a constant '1' value labeled with GPU device information. | gauge | `architecture`,`bus_number`,`device_id`,`driver_version`,`function_number`,`luid`,`name`,`phys`,`wddm_version` | +| `windows_gpu_dedicated_system_memory_size_bytes` | The size, in bytes, of memory that is dedicated from system memory. | gauge | `device_id`,`luid` | +| `windows_gpu_dedicated_video_memory_size_bytes` | The size, in bytes, of memory that is dedicated from video memory. | gauge | `device_id`,`luid` | +| `windows_gpu_shared_system_memory_size_bytes` | The size, in bytes, of memory from system memory that can be shared by many users. | gauge | `device_id`,`luid` | +| `windows_gpu_adapter_memory_committed_bytes` | Total committed GPU memory in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_adapter_memory_dedicated_bytes` | Dedicated GPU memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_adapter_memory_shared_bytes` | Shared GPU memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_local_adapter_memory_bytes` | Local adapter memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys`,`part` | +| `windows_gpu_non_local_adapter_memory_bytes` | Non-local adapter memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys`,`part` | + +The `driver_version`, `wddm_version` and `architecture` labels of `windows_gpu_info` are empty if the driver does not report them. + +### Sensor Metrics + +The sensor metrics are read from the graphics kernel (dxgkrnl) with `D3DKMTQueryAdapterInfo`, the same source Task Manager +uses. They are vendor-neutral, but depend on WDDM 2.4 or newer and on driver support. Drivers report 0 for values they do +not support, so a metric is only exposed if the driver reported a non-zero value or capability for it at startup. Microsoft +software adapters, like the Microsoft Basic Render Driver, are skipped and never queried. Query failures are logged at debug +level and never fail the scrape. + +| Name | Description | Type | Labels | +|--------------------------------------------|---------------------------------------------------------------------------------------|---------|---------------------------------| +| `windows_gpu_temperature_celsius` | Main temperature sensor reading of the physical GPU in degrees Celsius | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_temperature_warning_celsius` | Temperature in degrees Celsius at which the physical GPU starts throttling | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_temperature_max_celsius` | Maximum temperature in degrees Celsius before the physical GPU takes damage | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_fan_speed_rpm` | Current speed of the main fan in revolutions per minute (0 while the fan is stopped) | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_fan_speed_max_rpm` | Maximum speed of the main fan in revolutions per minute | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_power_usage_ratio` | Current power draw as a ratio (0–1) of the maximum power (TDP) of the physical GPU | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_memory_frequency_hertz` | Current clock frequency of the GPU memory in hertz | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_memory_frequency_max_hertz` | Maximum clock frequency of the GPU memory in hertz, while not overclocked | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_engine_frequency_hertz` | Current clock frequency of the GPU engine in hertz | gauge | `device_id`,`luid`,`phys`,`eng` | +| `windows_gpu_engine_frequency_max_hertz` | Maximum clock frequency of the GPU engine in hertz, while not overclocked | gauge | `device_id`,`luid`,`phys`,`eng` | + +The engine frequency metrics are only exposed for engines that report a maximum frequency, typically the 3D engine. The +`eng` label matches the `eng` label of `windows_gpu_engine_time_seconds`. ### Per-process Metrics @@ -49,6 +75,9 @@ These metrics are available on supported versions of Windows with compatible GPU * `eng`: GPU engine index (e.g., "0", "1", ...) * `engtype`: GPU engine type (e.g., "3D", "Copy", "VideoDecode", etc.) * `process_id`: Process ID +* `driver_version`: Kernel mode driver version (e.g., "32.0.15.8266") +* `wddm_version`: WDDM version of the driver (e.g., "3.2") +* `architecture`: GPU architecture as reported by the driver (e.g., "Pascal") ## Example Metric @@ -112,6 +141,12 @@ Show GPU engine time with process owner and command line: windows_gpu_engine_time_seconds * on(process_id) group_left(owner, cmdline) windows_process_info ``` +**Show the engine clock as a ratio of its maximum clock:** + +```promql +windows_gpu_engine_frequency_hertz / windows_gpu_engine_frequency_max_hertz +``` + ## Alerting Examples **prometheus.rules** @@ -129,6 +164,16 @@ windows_gpu_engine_time_seconds * on(process_id) group_left(owner, cmdline) wind annotations: summary: "High GPU Utilization (process {{ $labels.process_id }})" description: "Process is using more than 80% of GPU resources\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" +# Alert on GPUs running within 5 °C of their throttling temperature +- alert: GpuTemperatureHigh + expr: | + windows_gpu_temperature_celsius > windows_gpu_temperature_warning_celsius - 5 + for: 5m + labels: + severity: warning + annotations: + summary: "GPU temperature high (luid {{ $labels.luid }})" + description: "GPU is close to its throttling temperature\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" ``` ## Notes @@ -137,6 +182,7 @@ windows_gpu_engine_time_seconds * on(process_id) group_left(owner, cmdline) wind * Adapter-level metrics provide an overview of total GPU memory usage. * For overall GPU utilization, aggregate per-process metrics in Prometheus using queries such as `sum()`. * The collector relies on Windows performance counters; ensure your system and drivers support these counters. +* Sensor metrics (temperature, fan, power, clocks) require WDDM 2.4 or newer and a driver that reports these values. ## Enabling the Collector diff --git a/internal/collector/gpu/gpu.go b/internal/collector/gpu/gpu.go index 2e2e3b5a..aa4e90d9 100644 --- a/internal/collector/gpu/gpu.go +++ b/internal/collector/gpu/gpu.go @@ -41,6 +41,7 @@ var ConfigDefaults = Config{} type Collector struct { config Config + logger *slog.Logger gpuDeviceCache map[string]gpuDevice @@ -84,11 +85,15 @@ type Collector struct { gpuProcessMemoryNonLocalUsage *prometheus.Desc gpuProcessMemorySharedUsage *prometheus.Desc gpuProcessMemoryTotalCommitted *prometheus.Desc + + // GPU sensors + sensorMetrics sensorMetrics } type gpuDevice struct { gdi32 gdi32.GPUDevice cfgmgr32 cfgmgr32.Device + sensors gpuSensors ID string } @@ -125,10 +130,12 @@ func (c *Collector) Close() error { func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error { var err error + c.logger = logger.With(slog.String("collector", Name)) + c.gpuInfo = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "info"), "A metric with a constant '1' value labeled with gpu device information.", - []string{"luid", "device_id", "name", "bus_number", "phys", "function_number"}, + []string{"luid", "device_id", "name", "bus_number", "phys", "function_number", "driver_version", "wddm_version", "architecture"}, nil, ) @@ -222,6 +229,8 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error { nil, ) + c.sensorMetrics = newSensorMetrics() + errs := make([]error, 0) c.gpuEnginePerfDataCollector, err = pdh.NewCollector[gpuEnginePerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Engine", pdh.InstancesAll) @@ -297,6 +306,7 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error { c.gpuDeviceCache[luidKey] = gpuDevice{ gdi32: gpu, cfgmgr32: cfgmgr32Dev, + sensors: discoverSensors(c.logger.With(slog.String("luid", luidKey)), gpu), ID: deviceID, } @@ -319,6 +329,7 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric, _ time.Duration) error errs := make([]error, 0) c.collectGpuInfo(ch) + c.collectGpuSensorMetrics(ch) if err := c.collectGpuEngineMetrics(ch); err != nil { errs = append(errs, err) @@ -355,6 +366,9 @@ func (c *Collector) collectGpuInfo(ch chan<- prometheus.Metric) { gpu.gdi32.BusNumber.String(), gpu.gdi32.DeviceNumber.String(), gpu.gdi32.FunctionNumber.String(), + gpu.sensors.driverVersion, + gpu.sensors.wddmVersion, + gpu.sensors.architecture, ) ch <- prometheus.MustNewConstMetric( diff --git a/internal/collector/gpu/sensors.go b/internal/collector/gpu/sensors.go new file mode 100644 index 00000000..f40cc81f --- /dev/null +++ b/internal/collector/gpu/sensors.go @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package gpu + +import ( + "log/slog" + "strconv" + "strings" + + "github.com/prometheus-community/windows_exporter/internal/headers/gdi32" + "github.com/prometheus-community/windows_exporter/internal/types" + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/sys/windows" +) + +// maxNodes limits the probing of engine nodes. Probing stops at the first node ordinal the driver rejects. +const maxNodes = 64 + +// sensorMetrics holds the descriptors of the sensor metrics, read from dxgkrnl via D3DKMTQueryAdapterInfo. +type sensorMetrics struct { + temperature *prometheus.Desc + temperatureWarning *prometheus.Desc + temperatureMax *prometheus.Desc + fanSpeed *prometheus.Desc + fanSpeedMax *prometheus.Desc + powerUsage *prometheus.Desc + memoryFrequency *prometheus.Desc + memoryFrequencyMax *prometheus.Desc + engineFrequency *prometheus.Desc + engineFrequencyMax *prometheus.Desc +} + +// gpuSensors holds the static adapter data and the sensors the driver reported as supported during discovery. +type gpuSensors struct { + driverVersion string + wddmVersion string + architecture string + + physicalAdapters []physicalAdapterSensors +} + +type physicalAdapterSensors struct { + index uint32 + phys string + + caps gdi32.D3DKMT_ADAPTER_PERFDATACAPS + + // perfData is true if KMTQAITYPE_ADAPTERPERFDATA is supported. + perfData bool + temperature bool + fanSpeed bool + powerUsage bool + memoryFrequency bool + + // nodes are the engine node ordinals that report a frequency. + nodes []uint32 +} + +func (s *gpuSensors) hasDynamicSensors() bool { + for _, physicalAdapter := range s.physicalAdapters { + if physicalAdapter.perfData || len(physicalAdapter.nodes) > 0 { + return true + } + } + + return false +} + +func newSensorMetrics() sensorMetrics { + labels := []string{"luid", "device_id", "phys"} + + return sensorMetrics{ + temperature: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "temperature_celsius"), + "Main temperature sensor reading of the physical GPU in degrees Celsius.", + labels, + nil, + ), + temperatureWarning: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "temperature_warning_celsius"), + "Temperature in degrees Celsius at which the physical GPU starts throttling.", + labels, + nil, + ), + temperatureMax: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "temperature_max_celsius"), + "Maximum temperature in degrees Celsius before the physical GPU takes damage.", + labels, + nil, + ), + fanSpeed: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "fan_speed_rpm"), + "Current speed of the main fan of the physical GPU in revolutions per minute.", + labels, + nil, + ), + fanSpeedMax: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "fan_speed_max_rpm"), + "Maximum speed of the main fan of the physical GPU in revolutions per minute.", + labels, + nil, + ), + powerUsage: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "power_usage_ratio"), + "Current power draw of the physical GPU as a ratio of its maximum power (TDP).", + labels, + nil, + ), + memoryFrequency: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "memory_frequency_hertz"), + "Current clock frequency of the physical GPU memory in hertz.", + labels, + nil, + ), + memoryFrequencyMax: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "memory_frequency_max_hertz"), + "Maximum clock frequency of the physical GPU memory in hertz, while not overclocked.", + labels, + nil, + ), + engineFrequency: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "engine_frequency_hertz"), + "Current clock frequency of the GPU engine in hertz.", + append(labels, "eng"), + nil, + ), + engineFrequencyMax: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "engine_frequency_max_hertz"), + "Maximum clock frequency of the GPU engine in hertz, while not overclocked.", + append(labels, "eng"), + nil, + ), + } +} + +// isSoftwareAdapter reports whether the adapter is a Microsoft software or virtual adapter, +// like the Microsoft Basic Render Driver (PCI\VEN_1414&DEV_008C). These adapters have no +// sensors and reject the perf data queries with STATUS_INVALID_PARAMETER. +func isSoftwareAdapter(device gdi32.GPUDevice) bool { + return device.AdapterString == "" || strings.HasPrefix(device.DeviceID, `PCI\VEN_1414&`) +} + +// discoverSensors reads the static adapter data and probes which sensors the driver supports. +// The queries depend on WDDM 2.4+ and driver support. Failures are logged at debug level and +// mark the sensor as unsupported. +func discoverSensors(logger *slog.Logger, device gdi32.GPUDevice) gpuSensors { + var sensors gpuSensors + + // Skip software adapters explicitly, so they are never queried, even if the + // device discovery stops filtering them out. + if isSoftwareAdapter(device) { + return sensors + } + + hAdapter, err := gdi32.OpenAdapterFromLUID(device.LUID) + if err != nil { + logger.Debug("failed to open GPU adapter for sensor discovery", slog.Any("err", err)) + + return sensors + } + + defer func() { + if err := gdi32.CloseAdapter(hAdapter); err != nil { + logger.Debug("failed to close GPU adapter", slog.Any("err", err)) + } + }() + + if version, err := gdi32.QueryKMDDriverVersion(hAdapter); err != nil { + logger.Debug("GPU driver version not supported", slog.Any("err", err)) + } else { + sensors.driverVersion = gdi32.FormatKMDDriverVersion(version) + } + + if version, err := gdi32.QueryWDDMVersion(hAdapter); err != nil { + logger.Debug("GPU WDDM version not supported", slog.Any("err", err)) + } else { + sensors.wddmVersion = gdi32.FormatWDDMVersion(version) + } + + physicalAdapterCount, err := gdi32.QueryPhysicalAdapterCount(hAdapter) + if err != nil || physicalAdapterCount == 0 { + logger.Debug("GPU physical adapter count not supported, assuming 1", slog.Any("err", err)) + + physicalAdapterCount = 1 + } + + for index := range physicalAdapterCount { + physicalAdapter := discoverPhysicalAdapterSensors(logger.With(slog.Uint64("phys", uint64(index))), hAdapter, index) + + if index == 0 { + if version, err := gdi32.QueryGPUVersion(hAdapter, index); err != nil { + logger.Debug("GPU version not supported", slog.Any("err", err)) + } else { + sensors.architecture = windows.UTF16ToString(version.GpuArchitecture[:]) + } + } + + sensors.physicalAdapters = append(sensors.physicalAdapters, physicalAdapter) + } + + return sensors +} + +func discoverPhysicalAdapterSensors(logger *slog.Logger, hAdapter gdi32.D3DKMT_HANDLE, index uint32) physicalAdapterSensors { + physicalAdapter := physicalAdapterSensors{ + index: index, + phys: strconv.FormatUint(uint64(index), 10), + } + + caps, err := gdi32.QueryAdapterPerfDataCaps(hAdapter, index) + if err != nil { + logger.Debug("GPU adapter perf data caps not supported", slog.Any("err", err)) + } else { + physicalAdapter.caps = caps + } + + // Drivers report 0 for unsupported values, so a sensor counts as supported + // if either the caps or the first reading are non-zero. + perfData, err := gdi32.QueryAdapterPerfData(hAdapter, index) + if err != nil { + logger.Debug("GPU adapter perf data not supported", slog.Any("err", err)) + } else { + physicalAdapter.perfData = true + physicalAdapter.temperature = perfData.Temperature > 0 || caps.TemperatureWarning > 0 || caps.TemperatureMax > 0 + physicalAdapter.fanSpeed = perfData.FanRPM > 0 || caps.MaxFanRPM > 0 + physicalAdapter.powerUsage = perfData.Power > 0 + physicalAdapter.memoryFrequency = perfData.MemoryFrequency > 0 || perfData.MaxMemoryFrequency > 0 + } + + for node := range uint32(maxNodes) { + nodePerfData, err := gdi32.QueryNodePerfData(hAdapter, index, node) + if err != nil { + // The driver rejects node ordinals past the last node. + logger.Debug("GPU node perf data not supported", + slog.Uint64("eng", uint64(node)), + slog.Any("err", err), + ) + + break + } + + if nodePerfData.MaxFrequency > 0 { + physicalAdapter.nodes = append(physicalAdapter.nodes, node) + } + } + + return physicalAdapter +} + +func (c *Collector) collectGpuSensorMetrics(ch chan<- prometheus.Metric) { + for luid, device := range c.gpuDeviceCache { + // The caps were read during discovery and need no adapter handle. + for _, physicalAdapter := range device.sensors.physicalAdapters { + c.collectPhysicalAdapterCaps(ch, []string{luid, device.ID, physicalAdapter.phys}, physicalAdapter.caps) + } + + if !device.sensors.hasDynamicSensors() { + continue + } + + logger := c.logger.With(slog.String("luid", luid)) + + hAdapter, err := gdi32.OpenAdapterFromLUID(device.gdi32.LUID) + if err != nil { + logger.Debug("failed to open GPU adapter", slog.Any("err", err)) + + continue + } + + for _, physicalAdapter := range device.sensors.physicalAdapters { + c.collectPhysicalAdapterSensorMetrics(ch, logger, hAdapter, luid, device.ID, physicalAdapter) + } + + if err := gdi32.CloseAdapter(hAdapter); err != nil { + logger.Debug("failed to close GPU adapter", slog.Any("err", err)) + } + } +} + +func (c *Collector) collectPhysicalAdapterCaps(ch chan<- prometheus.Metric, labels []string, caps gdi32.D3DKMT_ADAPTER_PERFDATACAPS) { + if caps.TemperatureWarning > 0 { + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.temperatureWarning, + prometheus.GaugeValue, + float64(caps.TemperatureWarning)/10, + labels..., + ) + } + + if caps.TemperatureMax > 0 { + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.temperatureMax, + prometheus.GaugeValue, + float64(caps.TemperatureMax)/10, + labels..., + ) + } + + if caps.MaxFanRPM > 0 { + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.fanSpeedMax, + prometheus.GaugeValue, + float64(caps.MaxFanRPM), + labels..., + ) + } +} + +func (c *Collector) collectPhysicalAdapterSensorMetrics( + ch chan<- prometheus.Metric, + logger *slog.Logger, + hAdapter gdi32.D3DKMT_HANDLE, + luid, deviceID string, + physicalAdapter physicalAdapterSensors, +) { + labels := []string{luid, deviceID, physicalAdapter.phys} + + if physicalAdapter.perfData { + perfData, err := gdi32.QueryAdapterPerfData(hAdapter, physicalAdapter.index) + if err != nil { + logger.Debug("failed to query GPU adapter perf data", + slog.String("phys", physicalAdapter.phys), + slog.Any("err", err), + ) + } else { + c.collectAdapterPerfData(ch, physicalAdapter, perfData, labels) + } + } + + for _, node := range physicalAdapter.nodes { + nodePerfData, err := gdi32.QueryNodePerfData(hAdapter, physicalAdapter.index, node) + if err != nil { + logger.Debug("failed to query GPU node perf data", + slog.String("phys", physicalAdapter.phys), + slog.Uint64("eng", uint64(node)), + slog.Any("err", err), + ) + + continue + } + + eng := strconv.FormatUint(uint64(node), 10) + + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.engineFrequency, + prometheus.GaugeValue, + float64(nodePerfData.Frequency), + append(labels, eng)..., + ) + + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.engineFrequencyMax, + prometheus.GaugeValue, + float64(nodePerfData.MaxFrequency), + append(labels, eng)..., + ) + } +} + +func (c *Collector) collectAdapterPerfData( + ch chan<- prometheus.Metric, + physicalAdapter physicalAdapterSensors, + perfData gdi32.D3DKMT_ADAPTER_PERFDATA, + labels []string, +) { + if physicalAdapter.temperature { + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.temperature, + prometheus.GaugeValue, + float64(perfData.Temperature)/10, + labels..., + ) + } + + if physicalAdapter.fanSpeed { + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.fanSpeed, + prometheus.GaugeValue, + float64(perfData.FanRPM), + labels..., + ) + } + + if physicalAdapter.powerUsage { + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.powerUsage, + prometheus.GaugeValue, + float64(perfData.Power)/1000, + labels..., + ) + } + + if physicalAdapter.memoryFrequency { + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.memoryFrequency, + prometheus.GaugeValue, + float64(perfData.MemoryFrequency), + labels..., + ) + + if perfData.MaxMemoryFrequency > 0 { + ch <- prometheus.MustNewConstMetric( + c.sensorMetrics.memoryFrequencyMax, + prometheus.GaugeValue, + float64(perfData.MaxMemoryFrequency), + labels..., + ) + } + } +} diff --git a/internal/collector/gpu/sensors_test.go b/internal/collector/gpu/sensors_test.go new file mode 100644 index 00000000..29ce585a --- /dev/null +++ b/internal/collector/gpu/sensors_test.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package gpu + +import ( + "testing" + + "github.com/prometheus-community/windows_exporter/internal/headers/gdi32" + "github.com/stretchr/testify/require" +) + +func TestIsSoftwareAdapter(t *testing.T) { + t.Parallel() + + for name, tc := range map[string]struct { + device gdi32.GPUDevice + expected bool + }{ + "Microsoft Basic Render Driver": { + device: gdi32.GPUDevice{AdapterString: "Microsoft Basic Render Driver", DeviceID: `PCI\VEN_1414&DEV_008C&SUBSYS_00000000&REV_00`}, + expected: true, + }, + "empty adapter string": { + device: gdi32.GPUDevice{DeviceID: `PCI\VEN_0000&DEV_0000&SUBSYS_00000000&REV_00`}, + expected: true, + }, + "NVIDIA": { + device: gdi32.GPUDevice{AdapterString: "NVIDIA GeForce GTX 1070", DeviceID: `PCI\VEN_10DE&DEV_1B81&SUBSYS_61733842&REV_A1`}, + expected: false, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.expected, isSoftwareAdapter(tc.device)) + }) + } +} diff --git a/internal/headers/gdi32/gdi32.go b/internal/headers/gdi32/gdi32.go index c6b12a10..aae72f2d 100644 --- a/internal/headers/gdi32/gdi32.go +++ b/internal/headers/gdi32/gdi32.go @@ -36,6 +36,20 @@ const ( KMTQAITYPE_ADAPTERREGISTRYINFO = 8 // KMTQAITYPE_PHYSICALADAPTERDEVICEIDS pPrivateDriverData points to a D3DKMT_QUERY_DEVICE_IDS structure that specifies the device ID(s) of the physical adapters. Supported starting with Windows 10 (WDDM 2.0). KMTQAITYPE_PHYSICALADAPTERDEVICEIDS = 31 + // KMTQAITYPE_DRIVERVERSION pPrivateDriverData points to a D3DKMT_DRIVERVERSION value that indicates the WDDM version of the display miniport driver. + KMTQAITYPE_DRIVERVERSION = 13 + // KMTQAITYPE_PHYSICALADAPTERCOUNT pPrivateDriverData points to a UINT that receives the number of physical adapters in the LDA chain. Supported starting with Windows 10 (WDDM 2.0). + KMTQAITYPE_PHYSICALADAPTERCOUNT = 30 + // KMTQAITYPE_KMD_DRIVER_VERSION pPrivateDriverData points to a LARGE_INTEGER that receives the kernel mode driver version. Supported starting with WDDM 2.2. + KMTQAITYPE_KMD_DRIVER_VERSION = 49 + // KMTQAITYPE_NODEPERFDATA pPrivateDriverData points to a D3DKMT_NODE_PERFDATA structure. Supported starting with WDDM 2.4. + KMTQAITYPE_NODEPERFDATA = 61 + // KMTQAITYPE_ADAPTERPERFDATA pPrivateDriverData points to a D3DKMT_ADAPTER_PERFDATA structure. Supported starting with WDDM 2.4. + KMTQAITYPE_ADAPTERPERFDATA = 62 + // KMTQAITYPE_ADAPTERPERFDATA_CAPS pPrivateDriverData points to a D3DKMT_ADAPTER_PERFDATACAPS structure. Supported starting with WDDM 2.4. + KMTQAITYPE_ADAPTERPERFDATA_CAPS = 63 + // KMTQUITYPE_GPUVERSION pPrivateDriverData points to a D3DKMT_GPUVERSION structure. Supported starting with WDDM 2.4. + KMTQUITYPE_GPUVERSION = 64 ) var ErrNoGPUDevices = errors.New("no GPU devices found") diff --git a/internal/headers/gdi32/perfdata.go b/internal/headers/gdi32/perfdata.go new file mode 100644 index 00000000..1896dbc1 --- /dev/null +++ b/internal/headers/gdi32/perfdata.go @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package gdi32 + +import ( + "fmt" + "unsafe" + + "golang.org/x/sys/windows" +) + +// OpenAdapterFromLUID opens a handle to the adapter with the given LUID. +// The handle must be released with CloseAdapter. +func OpenAdapterFromLUID(luid windows.LUID) (D3DKMT_HANDLE, error) { + openAdapter := D3DKMT_OPENADAPTERFROMLUID{ + AdapterLUID: luid, + } + + if err := D3DKMTOpenAdapterFromLuid(&openAdapter); err != nil { + return 0, err + } + + return openAdapter.HAdapter, nil +} + +// CloseAdapter closes a handle opened by OpenAdapterFromLUID. +func CloseAdapter(hAdapter D3DKMT_HANDLE) error { + return D3DKMTCloseAdapter(&D3DKMT_CLOSEADAPTER{ + HAdapter: hAdapter, + }) +} + +// queryAdapterInfo calls D3DKMTQueryAdapterInfo with data as private driver data. +// data is used as input and output. +func queryAdapterInfo[T any](hAdapter D3DKMT_HANDLE, queryType int32, data *T) error { + query := D3DKMT_QUERYADAPTERINFO{ + hAdapter: hAdapter, + queryType: queryType, + pPrivateDriverData: unsafe.Pointer(data), + privateDriverDataSize: uint32(unsafe.Sizeof(*data)), + } + + return D3DKMTQueryAdapterInfo(&query) +} + +// QueryPhysicalAdapterCount returns the number of physical adapters in the LDA chain of the adapter. +func QueryPhysicalAdapterCount(hAdapter D3DKMT_HANDLE) (uint32, error) { + var count uint32 + + if err := queryAdapterInfo(hAdapter, KMTQAITYPE_PHYSICALADAPTERCOUNT, &count); err != nil { + return 0, fmt.Errorf("physical adapter count: %w", err) + } + + return count, nil +} + +// QueryWDDMVersion returns the WDDM version of the display miniport driver +// as D3DKMT_DRIVERVERSION value, e.g. 3200 for WDDM 3.2. +func QueryWDDMVersion(hAdapter D3DKMT_HANDLE) (uint32, error) { + var version int32 + + if err := queryAdapterInfo(hAdapter, KMTQAITYPE_DRIVERVERSION, &version); err != nil { + return 0, fmt.Errorf("WDDM version: %w", err) + } + + return uint32(version), nil +} + +// QueryKMDDriverVersion returns the kernel mode driver version, packed as four 16-bit parts. +func QueryKMDDriverVersion(hAdapter D3DKMT_HANDLE) (uint64, error) { + var version uint64 + + if err := queryAdapterInfo(hAdapter, KMTQAITYPE_KMD_DRIVER_VERSION, &version); err != nil { + return 0, fmt.Errorf("KMD driver version: %w", err) + } + + return version, nil +} + +// QueryAdapterPerfData returns the current performance data of a physical adapter. +func QueryAdapterPerfData(hAdapter D3DKMT_HANDLE, physicalAdapterIndex uint32) (D3DKMT_ADAPTER_PERFDATA, error) { + data := D3DKMT_ADAPTER_PERFDATA{ + PhysicalAdapterIndex: physicalAdapterIndex, + } + + if err := queryAdapterInfo(hAdapter, KMTQAITYPE_ADAPTERPERFDATA, &data); err != nil { + return data, fmt.Errorf("adapter perf data: %w", err) + } + + return data, nil +} + +// QueryAdapterPerfDataCaps returns the performance data capabilities of a physical adapter. +func QueryAdapterPerfDataCaps(hAdapter D3DKMT_HANDLE, physicalAdapterIndex uint32) (D3DKMT_ADAPTER_PERFDATACAPS, error) { + data := D3DKMT_ADAPTER_PERFDATACAPS{ + PhysicalAdapterIndex: physicalAdapterIndex, + } + + if err := queryAdapterInfo(hAdapter, KMTQAITYPE_ADAPTERPERFDATA_CAPS, &data); err != nil { + return data, fmt.Errorf("adapter perf data caps: %w", err) + } + + return data, nil +} + +// QueryNodePerfData returns the current performance data of an engine (node) of a physical adapter. +// The node ordinal matches the eng_N part of the GPU Engine performance counter instances. +// Querying a node ordinal past the last node returns an error. +func QueryNodePerfData(hAdapter D3DKMT_HANDLE, physicalAdapterIndex, nodeOrdinal uint32) (D3DKMT_NODE_PERFDATA, error) { + data := D3DKMT_NODE_PERFDATA{ + NodeOrdinal: nodeOrdinal, + PhysicalAdapterIndex: physicalAdapterIndex, + } + + if err := queryAdapterInfo(hAdapter, KMTQAITYPE_NODEPERFDATA, &data); err != nil { + return data, fmt.Errorf("node %d perf data: %w", nodeOrdinal, err) + } + + return data, nil +} + +// QueryGPUVersion returns the BIOS version and architecture of a physical adapter. +func QueryGPUVersion(hAdapter D3DKMT_HANDLE, physicalAdapterIndex uint32) (D3DKMT_GPUVERSION, error) { + data := D3DKMT_GPUVERSION{ + PhysicalAdapterIndex: physicalAdapterIndex, + } + + if err := queryAdapterInfo(hAdapter, KMTQUITYPE_GPUVERSION, &data); err != nil { + return data, fmt.Errorf("GPU version: %w", err) + } + + return data, nil +} + +// FormatKMDDriverVersion formats a version returned by QueryKMDDriverVersion as a.b.c.d. +func FormatKMDDriverVersion(version uint64) string { + return fmt.Sprintf("%d.%d.%d.%d", + uint16(version>>48), + uint16(version>>32), + uint16(version>>16), + uint16(version), + ) +} + +// FormatWDDMVersion formats a version returned by QueryWDDMVersion as major.minor, e.g. 3200 as 3.2. +func FormatWDDMVersion(version uint32) string { + return fmt.Sprintf("%d.%d", version/1000, version%1000/100) +} diff --git a/internal/headers/gdi32/perfdata_test.go b/internal/headers/gdi32/perfdata_test.go new file mode 100644 index 00000000..d2314379 --- /dev/null +++ b/internal/headers/gdi32/perfdata_test.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package gdi32_test + +import ( + "testing" + + "github.com/prometheus-community/windows_exporter/internal/headers/gdi32" + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +// TestPerfDataQueries calls the perf data queries on every adapter. +// The queries depend on WDDM 2.4+ and driver support, so failures are only logged. +func TestPerfDataQueries(t *testing.T) { + devices, err := gdi32.GetGPUDevices() + if len(devices) == 0 { + t.Skipf("no GPU devices found: %v", err) + } + + for _, device := range devices { + t.Run(device.AdapterString, func(t *testing.T) { + hAdapter, err := gdi32.OpenAdapterFromLUID(device.LUID) + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, gdi32.CloseAdapter(hAdapter)) + }) + + count, err := gdi32.QueryPhysicalAdapterCount(hAdapter) + t.Logf("physical adapter count: %d, err: %v", count, err) + + wddmVersion, err := gdi32.QueryWDDMVersion(hAdapter) + t.Logf("WDDM version: %s, err: %v", gdi32.FormatWDDMVersion(wddmVersion), err) + + driverVersion, err := gdi32.QueryKMDDriverVersion(hAdapter) + t.Logf("driver version: %s, err: %v", gdi32.FormatKMDDriverVersion(driverVersion), err) + + version, err := gdi32.QueryGPUVersion(hAdapter, 0) + t.Logf("BIOS version: %q, architecture: %q, err: %v", + windows.UTF16ToString(version.BiosVersion[:]), windows.UTF16ToString(version.GpuArchitecture[:]), err) + + caps, err := gdi32.QueryAdapterPerfDataCaps(hAdapter, 0) + t.Logf("adapter perf data caps: %+v, err: %v", caps, err) + + perfData, err := gdi32.QueryAdapterPerfData(hAdapter, 0) + t.Logf("adapter perf data: %+v, err: %v", perfData, err) + + for node := range uint32(64) { + nodePerfData, err := gdi32.QueryNodePerfData(hAdapter, 0, node) + if err != nil { + t.Logf("node %d: %v", node, err) + + break + } + + require.Equal(t, node, nodePerfData.NodeOrdinal) + t.Logf("node perf data: %+v", nodePerfData) + } + }) + } +} + +func TestFormatKMDDriverVersion(t *testing.T) { + require.Equal(t, "32.0.15.8266", gdi32.FormatKMDDriverVersion(32<<48|0<<32|15<<16|8266)) +} + +func TestFormatWDDMVersion(t *testing.T) { + require.Equal(t, "3.2", gdi32.FormatWDDMVersion(3200)) + require.Equal(t, "1.1", gdi32.FormatWDDMVersion(1105)) +} diff --git a/internal/headers/gdi32/types.go b/internal/headers/gdi32/types.go index ef73534d..c8546726 100644 --- a/internal/headers/gdi32/types.go +++ b/internal/headers/gdi32/types.go @@ -96,3 +96,70 @@ type GPUDevice struct { DeviceNumber win32.UINT FunctionNumber win32.UINT } + +// D3DKMT_NODE_PERFDATA is the output of KMTQAITYPE_NODEPERFDATA. +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/d3dkmthk/ns-d3dkmthk-_d3dkmt_node_perfdata +type D3DKMT_NODE_PERFDATA struct { + NodeOrdinal uint32 + PhysicalAdapterIndex uint32 + // Frequency is the current clock frequency of the engine in hertz. + Frequency uint64 + // MaxFrequency is the maximum clock frequency of the engine in hertz, while not overclocked. + MaxFrequency uint64 + // MaxFrequencyOC is the maximum clock frequency of the engine in hertz, while overclocked. + MaxFrequencyOC uint64 + // Voltage is the current voltage of the engine in milli volts. + Voltage win32.ULONG + VoltageMax win32.ULONG + VoltageMaxOC win32.ULONG + // MaxTransitionLatency is the maximum transition latency to change the frequency in 100 nanoseconds. + MaxTransitionLatency uint64 +} + +// D3DKMT_ADAPTER_PERFDATA is the output of KMTQAITYPE_ADAPTERPERFDATA. +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/d3dkmthk/ns-d3dkmthk-_d3dkmt_adapter_perfdata +type D3DKMT_ADAPTER_PERFDATA struct { + PhysicalAdapterIndex uint32 + // MemoryFrequency is the clock frequency of the memory in hertz. + MemoryFrequency uint64 + // MaxMemoryFrequency is the max clock frequency of the memory in hertz, while not overclocked. + MaxMemoryFrequency uint64 + // MaxMemoryFrequencyOC is the max clock frequency of the memory in hertz, while overclocked. + MaxMemoryFrequencyOC uint64 + // MemoryBandwidth is the total amount of memory transferred in bytes. + MemoryBandwidth uint64 + // PCIEBandwidth is the total amount of memory transferred over PCIe in bytes. + PCIEBandwidth uint64 + // FanRPM is the current rpm of the main fan. + FanRPM win32.ULONG + // Power is the current power draw of the adapter in tenths of a percent (1 = 0.1%). + Power win32.ULONG + // Temperature is the main temperature sensor reading in tenths of a degree Celsius (1 = 0.1 °C). + Temperature win32.ULONG + // PowerStateOverride is 1 if the GPU is powered on, otherwise 0. + PowerStateOverride uint8 +} + +// D3DKMT_ADAPTER_PERFDATACAPS is the output of KMTQAITYPE_ADAPTERPERFDATA_CAPS. +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/d3dkmthk/ns-d3dkmthk-_d3dkmt_adapter_perfdatacaps +type D3DKMT_ADAPTER_PERFDATACAPS struct { + PhysicalAdapterIndex uint32 + // MaxMemoryBandwidth is the max memory bandwidth in bytes for 1 second. + MaxMemoryBandwidth uint64 + // MaxPCIEBandwidth is the max PCIe bandwidth in bytes for 1 second. + MaxPCIEBandwidth uint64 + // MaxFanRPM is the max fan rpm. + MaxFanRPM win32.ULONG + // TemperatureMax is the max temperature before damage, in tenths of a degree Celsius. + TemperatureMax win32.ULONG + // TemperatureWarning is the temperature at which the GPU starts throttling, in tenths of a degree Celsius. + TemperatureWarning win32.ULONG +} + +// D3DKMT_GPUVERSION is the output of KMTQAITYPE_GPUVERSION. +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/d3dkmthk/ns-d3dkmthk-_d3dkmt_gpuversion +type D3DKMT_GPUVERSION struct { + PhysicalAdapterIndex uint32 + BiosVersion [32]uint16 + GpuArchitecture [32]uint16 +}