diff --git a/docs/collector.gpu.md b/docs/collector.gpu.md index 0322e31e..080710c3 100644 --- a/docs/collector.gpu.md +++ b/docs/collector.gpu.md @@ -1,14 +1,19 @@ # GPU collector -The GPU collector exposes metrics about GPU usage and memory consumption, both at the adapter (physical GPU) and +The `gpu` collector exposes metrics about GPU usage and memory consumption, both at the adapter (physical GPU) and per-process level. -| | | -|---------------------|--------------------------------------| -| Metric name prefix | `gpu` | -| Data source | Perflib, D3DKMT (dxgkrnl) | -| Counters | GPU Engine, GPU Adapter, GPU Process | -| Enabled by default? | No | +| | | +|---------------------|------------------------------------------------------------------------------------------------------------------| +| Metric name prefix | `gpu` | +| Data source | Perflib, D3DKMT (gdi32.dll), Configuration Manager (cfgmgr32.dll) | +| Counters | GPU Engine, GPU Adapter Memory, GPU Local Adapter Memory, GPU Non Local Adapter Memory, GPU Process Memory | +| Enabled by default? | No | + +GPU adapters are discovered through the D3DKMT API. Software adapters like the Microsoft Basic Render Driver are not +exposed. If an adapter can't be fully discovered, a warning is logged and the collector keeps working with the +remaining adapters. The list of adapters is refreshed when a performance counter references an unknown adapter, e.g. +after a driver update, a device restart or when an external GPU is attached (at most once every 30 seconds). ## Flags @@ -20,17 +25,17 @@ These metrics are available on supported versions of Windows with compatible GPU ### Adapter-level Metrics -| Name | Description | Type | Labels | -|--------------------------------------------------|------------------------------------------------------------------------------------|-------|----------------------------------------------------------------------------------------------------------------| -| `windows_gpu_info` | A metric with a constant '1' value labeled with GPU device information. | gauge | `architecture`,`bus_number`,`device_id`,`driver_version`,`function_number`,`luid`,`name`,`phys`,`wddm_version` | -| `windows_gpu_dedicated_system_memory_size_bytes` | The size, in bytes, of memory that is dedicated from system memory. | gauge | `device_id`,`luid` | -| `windows_gpu_dedicated_video_memory_size_bytes` | The size, in bytes, of memory that is dedicated from video memory. | gauge | `device_id`,`luid` | -| `windows_gpu_shared_system_memory_size_bytes` | The size, in bytes, of memory from system memory that can be shared by many users. | gauge | `device_id`,`luid` | -| `windows_gpu_adapter_memory_committed_bytes` | Total committed GPU memory in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | -| `windows_gpu_adapter_memory_dedicated_bytes` | Dedicated GPU memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | -| `windows_gpu_adapter_memory_shared_bytes` | Shared GPU memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | -| `windows_gpu_local_adapter_memory_bytes` | Local adapter memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys`,`part` | -| `windows_gpu_non_local_adapter_memory_bytes` | Non-local adapter memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys`,`part` | +| Name | Description | Type | Labels | +|--------------------------------------------------|------------------------------------------------------------------------------------|-------|-------------------------------------------------------------------------------------------------------------------------| +| `windows_gpu_info` | A metric with a constant '1' value labeled with GPU device information. | gauge | `architecture`,`bus_number`,`device_id`,`device_number`,`driver_version`,`function_number`,`luid`,`name`,`wddm_version` | +| `windows_gpu_dedicated_system_memory_size_bytes` | The size, in bytes, of memory that is dedicated from system memory. | gauge | `device_id`,`luid` | +| `windows_gpu_dedicated_video_memory_size_bytes` | The size, in bytes, of memory that is dedicated from video memory. | gauge | `device_id`,`luid` | +| `windows_gpu_shared_system_memory_size_bytes` | The size, in bytes, of memory from system memory that can be shared by many users. | gauge | `device_id`,`luid` | +| `windows_gpu_adapter_memory_committed_bytes` | Total committed GPU memory in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_adapter_memory_dedicated_bytes` | Dedicated GPU memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_adapter_memory_shared_bytes` | Shared GPU memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys` | +| `windows_gpu_local_adapter_memory_bytes` | Local adapter memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys`,`part` | +| `windows_gpu_non_local_adapter_memory_bytes` | Non-local adapter memory usage in bytes per physical GPU | gauge | `device_id`,`luid`,`phys`,`part` | The `driver_version`, `wddm_version` and `architecture` labels of `windows_gpu_info` are empty if the driver does not report them. @@ -38,9 +43,9 @@ The `driver_version`, `wddm_version` and `architecture` labels of `windows_gpu_i The sensor metrics are read from the graphics kernel (dxgkrnl) with `D3DKMTQueryAdapterInfo`, the same source Task Manager uses. They are vendor-neutral, but depend on WDDM 2.4 or newer and on driver support. Drivers report 0 for values they do -not support, so a metric is only exposed if the driver reported a non-zero value or capability for it at startup. Microsoft -software adapters, like the Microsoft Basic Render Driver, are skipped and never queried. Query failures are logged at debug -level and never fail the scrape. +not support, so a metric is only exposed if the driver reported a non-zero value or capability for it when the adapter was +discovered. Software adapters, like the Microsoft Basic Render Driver, are skipped and never queried. Query failures are +logged at debug level and never fail the scrape. | Name | Description | Type | Labels | |--------------------------------------------|---------------------------------------------------------------------------------------|---------|---------------------------------| @@ -71,7 +76,15 @@ The engine frequency metrics are only exposed for engines that report a maximum ## Metric Labels -* `luid`,`phys`: Physical GPU index (e.g., "0") +* `luid`: Locally unique identifier of the GPU adapter (e.g., "0x00000000_0x0001136A"), formatted as + `_`. It is assigned by Windows and identifies the adapter in all metrics of this collector. Use it + to join metrics with `windows_gpu_info`. The LUID is not stable: it changes when the driver is updated, the device is + restarted or the system reboots. +* `device_id`: PnP device instance ID of the GPU adapter (e.g., "PCI\VEN_10DE&DEV_1B81&SUBSYS_61733842&REV_A1\4&1d81e16&0&0019"). + If the device instance can't be resolved, the PnP device ID without the instance part is used. +* `phys`: Physical adapter index within a (linked) logical adapter (e.g., "0") +* `bus_number`,`device_number`,`function_number`: PCI address of the GPU adapter (`windows_gpu_info` only) +* `part`: Memory segment index (e.g., "0") * `eng`: GPU engine index (e.g., "0", "1", ...) * `engtype`: GPU engine type (e.g., "3D", "Copy", "VideoDecode", etc.) * `process_id`: Process ID @@ -83,10 +96,16 @@ The engine frequency metrics are only exposed for engines that report a maximum These are basic queries to help you get started with GPU monitoring on Windows using Prometheus. -**Show GPU information for a specific physical GPU (0):** +**Show GPU information:** ```promql -windows_gpu_info{bus_number="8",device_id="PCI\\VEN_10DE&DEV_1B81&SUBSYS_61733842&REV_A1",function_number="0",luid="0x00000000_0x00010F8A",name="NVIDIA GeForce GTX 1070",phys="0"} 1 +windows_gpu_info{architecture="Pascal",bus_number="8",device_id="PCI\\VEN_10DE&DEV_1B81&SUBSYS_61733842&REV_A1\\4&1d81e16&0&0019",device_number="0",driver_version="32.0.15.8266",function_number="0",luid="0x00000000_0x0001136A",name="NVIDIA GeForce GTX 1070",wddm_version="3.2"} 1 +``` + +**Add the GPU name to a metric:** + +```promql +windows_gpu_adapter_memory_dedicated_bytes * on(luid) group_left(name) windows_gpu_info ``` **Show total dedicated GPU memory (in bytes) usage on GPU 0:** @@ -114,7 +133,9 @@ sum by (phys, process_id) ( **Show dedicated GPU memory per process:** ```promql -windows_gpu_adapter_memory_dedicated_bytes +sum by (luid, process_id) ( + windows_gpu_process_memory_dedicated_bytes +) ``` ## Useful Queries diff --git a/internal/collector/gpu/gpu.go b/internal/collector/gpu/gpu.go index aa4e90d9..4ca71a81 100644 --- a/internal/collector/gpu/gpu.go +++ b/internal/collector/gpu/gpu.go @@ -39,11 +39,23 @@ type Config struct{} //nolint:gochecknoglobals var ConfigDefaults = Config{} +// deviceCacheRefreshInterval limits how often the GPU device cache is +// rebuilt when a performance counter instance references an unknown LUID. +const deviceCacheRefreshInterval = 30 * time.Second + type Collector struct { config Config logger *slog.Logger - gpuDeviceCache map[string]gpuDevice + // gpuDeviceCache maps the LUID ("0x%08X_0x%08X") of each adapter to its device information. + // LUIDs change if a driver is updated or a device is restarted, so the cache is + // refreshed whenever a performance counter instance references an unknown LUID. + gpuDeviceCache map[string]gpuDevice + gpuDeviceCacheLastRefresh time.Time + + // discoverGPUDevices enumerates the GPU adapters. If nil, gdi32.GetGPUDevices is used. + // It is replaced in tests to make the discovery deterministic. + discoverGPUDevices func() ([]gdi32.GPUDevice, error) // GPU Engine gpuEnginePerfDataCollector *pdh.Collector[gpuEnginePerfDataCounterValues] @@ -95,6 +107,11 @@ type gpuDevice struct { cfgmgr32 cfgmgr32.Device sensors gpuSensors ID string + + // skip marks adapters that are known, but not exposed, e.g. software devices + // like the Microsoft Basic Render Driver. Keeping them in the cache prevents + // their performance counter instances from triggering a cache refresh. + skip bool } func New(config *Config) *Collector { @@ -134,8 +151,8 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error { c.gpuInfo = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "info"), - "A metric with a constant '1' value labeled with gpu device information.", - []string{"luid", "device_id", "name", "bus_number", "phys", "function_number", "driver_version", "wddm_version", "architecture"}, + "A metric with a constant '1' value labeled with GPU device information.", + []string{"luid", "device_id", "name", "bus_number", "device_number", "function_number", "driver_version", "wddm_version", "architecture"}, nil, ) @@ -160,7 +177,7 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error { c.gpuEngineRunningTime = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "engine_time_seconds"), - "Total running time of the GPU in seconds.", + "Total running time of the GPU engine in seconds.", []string{"process_id", "luid", "device_id", "phys", "eng", "engtype"}, nil, ) @@ -258,79 +275,131 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error { errs = append(errs, fmt.Errorf("failed to create GPU Process Memory perf data collector: %w", err)) } - gpus, err := gdi32.GetGPUDevices() - if err != nil { - errs = append(errs, fmt.Errorf("failed to get GPU devices: %w", err)) + // GPU discovery problems are not fatal. The collector keeps working with the + // adapters that could be discovered and retries on unknown LUIDs during Collect. + c.refreshGPUDevices() + + return errors.Join(errs...) +} + +// refreshGPUDevices rebuilds the GPU device cache. +// Problems with single adapters are logged and do not stop the discovery of the remaining adapters. +func (c *Collector) refreshGPUDevices() { + c.gpuDeviceCacheLastRefresh = time.Now() + + discoverGPUDevices := c.discoverGPUDevices + if discoverGPUDevices == nil { + discoverGPUDevices = gdi32.GetGPUDevices } + gpus, err := discoverGPUDevices() + if err != nil { + c.logger.Warn("failed to discover some GPU devices", + slog.Any("err", err), + ) + + // Keep the previous cache, if the discovery failed completely. + if len(gpus) == 0 { + return + } + } + + gpuDeviceCache := make(map[string]gpuDevice, len(gpus)) + for _, gpu := range gpus { - if gpu.AdapterString == "" { - continue - } - - // Skip Microsoft Basic Render Driver - // https://devicehunt.com/view/type/pci/vendor/1414/device/008C - if gpu.DeviceID == `PCI\VEN_1414&DEV_008C&SUBSYS_00000000&REV_00` { - continue - } - - if c.gpuDeviceCache == nil { - c.gpuDeviceCache = make(map[string]gpuDevice) - } - luidKey := fmt.Sprintf("0x%08X_0x%08X", gpu.LUID.HighPart, gpu.LUID.LowPart) - deviceID := gpu.DeviceID + // Skip software devices like the Microsoft Basic Render Driver. + if gpu.AdapterString == "" || gpu.IsSoftwareDevice() || isBasicRenderDriver(gpu) { + gpuDeviceCache[luidKey] = gpuDevice{gdi32: gpu, ID: gpu.DeviceID, skip: true} - cfgmgr32Devs, err := cfgmgr32.GetDevicesInstanceIDs(gpu.DeviceID) - if err != nil { - errs = append(errs, fmt.Errorf("failed to get device instance IDs for device ID %s: %w", gpu.DeviceID, err)) + c.logger.Debug("Skipping GPU device", + slog.String("name", gpu.AdapterString), + slog.String("luid", luidKey), + slog.String("device_id", gpu.DeviceID), + slog.Uint64("adapter_type", uint64(gpu.AdapterType)), + ) + + continue } - var cfgmgr32Dev cfgmgr32.Device + device := c.newGPUDevice(gpu, luidKey) + device.sensors = discoverSensors(c.logger.With(slog.String("luid", luidKey)), gpu) + gpuDeviceCache[luidKey] = device - for _, dev := range cfgmgr32Devs { - if dev.BusNumber == gpu.BusNumber && dev.DeviceNumber == gpu.DeviceNumber && dev.FunctionNumber == gpu.FunctionNumber { - cfgmgr32Dev = dev - - break - } - } - - if cfgmgr32Dev.InstanceID == "" { - errs = append(errs, fmt.Errorf("failed to find matching device for device ID %s", gpu.DeviceID)) - } else { - deviceID = cfgmgr32Dev.InstanceID - } - - c.gpuDeviceCache[luidKey] = gpuDevice{ - gdi32: gpu, - cfgmgr32: cfgmgr32Dev, - sensors: discoverSensors(c.logger.With(slog.String("luid", luidKey)), gpu), - ID: deviceID, - } - - logger.Debug("Found GPU device", - slog.String("collector", Name), + c.logger.Debug("Found GPU device", slog.String("name", gpu.AdapterString), slog.String("luid", luidKey), - slog.String("device_id", deviceID), - slog.String("name", gpu.AdapterString), + slog.String("device_id", device.ID), slog.Uint64("bus_number", uint64(gpu.BusNumber)), slog.Uint64("device_number", uint64(gpu.DeviceNumber)), slog.Uint64("function_number", uint64(gpu.FunctionNumber)), ) } - return errors.Join(errs...) + c.gpuDeviceCache = gpuDeviceCache +} + +// newGPUDevice resolves the PnP device instance ID of the adapter. +// If the device instance can't be found, the PnP device ID reported by gdi32 is used as ID. +func (c *Collector) newGPUDevice(gpu gdi32.GPUDevice, luidKey string) gpuDevice { + device := gpuDevice{ + gdi32: gpu, + ID: gpu.DeviceID, + } + + cfgmgr32Devs, err := cfgmgr32.GetDevicesInstanceIDs(gpu.DeviceID) + + for _, dev := range cfgmgr32Devs { + if dev.BusNumber == gpu.BusNumber && dev.DeviceNumber == gpu.DeviceNumber && dev.FunctionNumber == gpu.FunctionNumber { + device.cfgmgr32 = dev + device.ID = dev.InstanceID + + return device + } + } + + attrs := []any{ + slog.String("name", gpu.AdapterString), + slog.String("luid", luidKey), + slog.String("device_id", gpu.DeviceID), + } + + if err != nil { + attrs = append(attrs, slog.Any("err", err)) + } + + c.logger.Warn("failed to find matching PnP device instance for GPU device, using the PnP device ID as device_id", attrs...) + + return device +} + +// getGPUDevice returns the cached device for the given LUID. +// If the LUID is unknown, e.g. because a driver was updated, a device was restarted +// or a device was hot-plugged, the cache is refreshed at most once per deviceCacheRefreshInterval. +// Devices marked as skip are not returned. +func (c *Collector) getGPUDevice(luid string) (gpuDevice, bool) { + if luid == "" { + return gpuDevice{}, false + } + + device, ok := c.gpuDeviceCache[luid] + if !ok && time.Since(c.gpuDeviceCacheLastRefresh) >= deviceCacheRefreshInterval { + c.logger.Debug("unknown GPU LUID, refreshing GPU device cache", + slog.String("luid", luid), + ) + + c.refreshGPUDevices() + + device, ok = c.gpuDeviceCache[luid] + } + + return device, ok && !device.skip } func (c *Collector) Collect(ch chan<- prometheus.Metric, _ time.Duration) error { errs := make([]error, 0) - c.collectGpuInfo(ch) - c.collectGpuSensorMetrics(ch) - if err := c.collectGpuEngineMetrics(ch); err != nil { errs = append(errs, err) } @@ -351,11 +420,19 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric, _ time.Duration) error errs = append(errs, err) } + // Collected last, since the metrics above may refresh the GPU device cache. + c.collectGpuInfo(ch) + c.collectGpuSensorMetrics(ch) + return errors.Join(errs...) } func (c *Collector) collectGpuInfo(ch chan<- prometheus.Metric) { for luid, gpu := range c.gpuDeviceCache { + if gpu.skip { + continue + } + ch <- prometheus.MustNewConstMetric( c.gpuInfo, prometheus.GaugeValue, @@ -404,7 +481,7 @@ func (c *Collector) collectGpuEngineMetrics(ch chan<- prometheus.Metric) error { for _, data := range c.gpuEnginePerfDataObject { instance := parseGPUCounterInstanceString(data.Name) - device, ok := c.gpuDeviceCache[instance.Luid] + device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } @@ -429,7 +506,7 @@ func (c *Collector) collectGpuAdapterMemoryMetrics(ch chan<- prometheus.Metric) for _, data := range c.gpuAdapterMemoryPerfDataObject { instance := parseGPUCounterInstanceString(data.Name) - device, ok := c.gpuDeviceCache[instance.Luid] + device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } @@ -468,7 +545,7 @@ func (c *Collector) collectGpuLocalAdapterMemoryMetrics(ch chan<- prometheus.Met for _, data := range c.gpuLocalAdapterMemoryPerfDataObject { instance := parseGPUCounterInstanceString(data.Name) - device, ok := c.gpuDeviceCache[instance.Luid] + device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } @@ -493,7 +570,7 @@ func (c *Collector) collectGpuNonLocalAdapterMemoryMetrics(ch chan<- prometheus. for _, data := range c.gpuNonLocalAdapterMemoryPerfDataObject { instance := parseGPUCounterInstanceString(data.Name) - device, ok := c.gpuDeviceCache[instance.Luid] + device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } @@ -518,7 +595,7 @@ func (c *Collector) collectGpuProcessMemoryMetrics(ch chan<- prometheus.Metric) for _, data := range c.gpuProcessMemoryPerfDataObject { instance := parseGPUCounterInstanceString(data.Name) - device, ok := c.gpuDeviceCache[instance.Luid] + device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } diff --git a/internal/collector/gpu/gpu_internal_test.go b/internal/collector/gpu/gpu_internal_test.go new file mode 100644 index 00000000..49ab19eb --- /dev/null +++ b/internal/collector/gpu/gpu_internal_test.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package gpu + +import ( + "errors" + "log/slog" + "testing" + "time" + + "github.com/prometheus-community/windows_exporter/internal/headers/gdi32" + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +// Device IDs of the test adapters. No real device instance matches them, +// so the hardware adapter keeps its PnP device ID as ID. +const ( + testHardwareDeviceID = `PCI\VEN_FFFF&DEV_FFFF&SUBSYS_00000000&REV_00` + testBasicRenderDeviceID = `PCI\VEN_1414&DEV_008C&SUBSYS_00000000&REV_00` +) + +//nolint:gochecknoglobals +var ( + testHardwareDevice = gdi32.GPUDevice{ + AdapterString: "Test GPU", + LUID: windows.LUID{LowPart: 0x20}, + DeviceID: testHardwareDeviceID, + AdapterType: gdi32.D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED | gdi32.D3DKMT_ADAPTERTYPE_DISPLAY_SUPPORTED, + } + testSoftwareDevice = gdi32.GPUDevice{ + AdapterString: "Test Software Adapter", + LUID: windows.LUID{LowPart: 0x10}, + DeviceID: `PCI\VEN_FFFF&DEV_0001&SUBSYS_00000000&REV_00`, + AdapterType: gdi32.D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED | gdi32.D3DKMT_ADAPTERTYPE_SOFTWARE_DEVICE, + } + // The Microsoft Basic Render Driver is not flagged as software device on every system, + // e.g. Windows Server 2022. + testBasicRenderDevice = gdi32.GPUDevice{ + AdapterString: "Microsoft Basic Render Driver", + LUID: windows.LUID{LowPart: 0x30}, + DeviceID: testBasicRenderDeviceID, + AdapterType: gdi32.D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED, + } +) + +func newTestCollector(cache map[string]gpuDevice, lastRefresh time.Time, discover func() ([]gdi32.GPUDevice, error)) *Collector { + return &Collector{ + logger: slog.New(slog.DiscardHandler), + gpuDeviceCache: cache, + gpuDeviceCacheLastRefresh: lastRefresh, + discoverGPUDevices: discover, + } +} + +func noDiscovery(t *testing.T) func() ([]gdi32.GPUDevice, error) { + t.Helper() + + return func() ([]gdi32.GPUDevice, error) { + t.Error("unexpected GPU device discovery") + + return nil, nil + } +} + +func TestGetGPUDevice(t *testing.T) { + t.Parallel() + + expired := time.Now().Add(-2 * deviceCacheRefreshInterval) + + t.Run("known device", func(t *testing.T) { + t.Parallel() + + c := newTestCollector(map[string]gpuDevice{"known": {ID: "known"}}, time.Now(), noDiscovery(t)) + + device, ok := c.getGPUDevice("known") + require.True(t, ok) + require.Equal(t, "known", device.ID) + }) + + t.Run("skipped device does not trigger refresh", func(t *testing.T) { + t.Parallel() + + c := newTestCollector(map[string]gpuDevice{"software": {skip: true}}, expired, noDiscovery(t)) + + _, ok := c.getGPUDevice("software") + require.False(t, ok) + require.Equal(t, expired, c.gpuDeviceCacheLastRefresh) + }) + + t.Run("unknown device within refresh interval", func(t *testing.T) { + t.Parallel() + + lastRefresh := time.Now() + c := newTestCollector(map[string]gpuDevice{"stale": {ID: "stale"}}, lastRefresh, noDiscovery(t)) + + _, ok := c.getGPUDevice("unknown") + require.False(t, ok) + require.Equal(t, lastRefresh, c.gpuDeviceCacheLastRefresh) + require.Contains(t, c.gpuDeviceCache, "stale") + }) + + t.Run("empty LUID", func(t *testing.T) { + t.Parallel() + + c := newTestCollector(nil, expired, noDiscovery(t)) + + _, ok := c.getGPUDevice("") + require.False(t, ok) + require.Equal(t, expired, c.gpuDeviceCacheLastRefresh) + }) + + t.Run("unknown device after refresh interval", func(t *testing.T) { + t.Parallel() + + c := newTestCollector(map[string]gpuDevice{"stale": {ID: "stale"}}, expired, func() ([]gdi32.GPUDevice, error) { + return []gdi32.GPUDevice{testHardwareDevice, testSoftwareDevice, testBasicRenderDevice}, nil + }) + + device, ok := c.getGPUDevice("0x00000000_0x00000020") + require.True(t, ok) + require.Equal(t, testHardwareDeviceID, device.ID) + require.Equal(t, "Test GPU", device.gdi32.AdapterString) + require.True(t, c.gpuDeviceCacheLastRefresh.After(expired)) + require.NotContains(t, c.gpuDeviceCache, "stale") + + // Software devices are cached, but not exposed. + for _, luid := range []string{"0x00000000_0x00000010", "0x00000000_0x00000030"} { + require.Contains(t, c.gpuDeviceCache, luid) + + _, ok = c.getGPUDevice(luid) + require.False(t, ok, luid) + } + }) + + t.Run("partial discovery failure", func(t *testing.T) { + t.Parallel() + + c := newTestCollector(map[string]gpuDevice{"stale": {ID: "stale"}}, expired, func() ([]gdi32.GPUDevice, error) { + return []gdi32.GPUDevice{testHardwareDevice}, errors.New("failed to get GPU device for adapter") + }) + + _, ok := c.getGPUDevice("0x00000000_0x00000020") + require.True(t, ok) + require.NotContains(t, c.gpuDeviceCache, "stale") + }) + + t.Run("complete discovery failure keeps the previous cache", func(t *testing.T) { + t.Parallel() + + c := newTestCollector(map[string]gpuDevice{"stale": {ID: "stale"}}, expired, func() ([]gdi32.GPUDevice, error) { + return nil, gdi32.ErrNoGPUDevices + }) + + _, ok := c.getGPUDevice("unknown") + require.False(t, ok) + require.True(t, c.gpuDeviceCacheLastRefresh.After(expired)) + require.Contains(t, c.gpuDeviceCache, "stale") + }) +} diff --git a/internal/collector/gpu/sensors.go b/internal/collector/gpu/sensors.go index f40cc81f..9eb37634 100644 --- a/internal/collector/gpu/sensors.go +++ b/internal/collector/gpu/sensors.go @@ -148,11 +148,12 @@ func newSensorMetrics() sensorMetrics { } } -// isSoftwareAdapter reports whether the adapter is a Microsoft software or virtual adapter, -// like the Microsoft Basic Render Driver (PCI\VEN_1414&DEV_008C). These adapters have no -// sensors and reject the perf data queries with STATUS_INVALID_PARAMETER. +// isSoftwareAdapter reports whether the adapter is a software adapter (D3DKMT_ADAPTERTYPE +// SoftwareDevice flag) or a Microsoft virtual adapter, like the Microsoft Basic Render Driver +// (PCI\VEN_1414&DEV_008C). These adapters have no sensors and reject the perf data queries +// with STATUS_INVALID_PARAMETER. func isSoftwareAdapter(device gdi32.GPUDevice) bool { - return device.AdapterString == "" || strings.HasPrefix(device.DeviceID, `PCI\VEN_1414&`) + return device.AdapterString == "" || device.IsSoftwareDevice() || strings.HasPrefix(device.DeviceID, `PCI\VEN_1414&`) } // discoverSensors reads the static adapter data and probes which sensors the driver supports. @@ -264,6 +265,10 @@ func discoverPhysicalAdapterSensors(logger *slog.Logger, hAdapter gdi32.D3DKMT_H func (c *Collector) collectGpuSensorMetrics(ch chan<- prometheus.Metric) { for luid, device := range c.gpuDeviceCache { + if device.skip { + continue + } + // The caps were read during discovery and need no adapter handle. for _, physicalAdapter := range device.sensors.physicalAdapters { c.collectPhysicalAdapterCaps(ch, []string{luid, device.ID, physicalAdapter.phys}, physicalAdapter.caps) diff --git a/internal/collector/gpu/types.go b/internal/collector/gpu/types.go index affd96a9..3814c847 100644 --- a/internal/collector/gpu/types.go +++ b/internal/collector/gpu/types.go @@ -20,8 +20,7 @@ package gpu type gpuEnginePerfDataCounterValues struct { Name string - RunningTime float64 `perfdata:"Running Time"` - UtilizationPercentage float64 `perfdata:"Utilization Percentage"` + RunningTime float64 `perfdata:"Running Time"` } type gpuAdapterMemoryPerfDataCounterValues struct { diff --git a/internal/collector/gpu/utils.go b/internal/collector/gpu/utils.go index 902c8a0a..7c0ede7c 100644 --- a/internal/collector/gpu/utils.go +++ b/internal/collector/gpu/utils.go @@ -20,38 +20,32 @@ package gpu import ( "fmt" "strings" + + "github.com/prometheus-community/windows_exporter/internal/headers/gdi32" ) +// isBasicRenderDriver reports whether the adapter is the Microsoft Basic Render Driver. +// https://devicehunt.com/view/type/pci/vendor/1414/device/008C +// The D3DKMT_ADAPTERTYPE SoftwareDevice flag is not set for it on every system, +// e.g. Windows Server 2022, so the device ID is checked as well. +func isBasicRenderDriver(device gdi32.GPUDevice) bool { + return strings.HasPrefix(device.DeviceID, `PCI\VEN_1414&DEV_008C&`) +} + type Instance struct { - Pid string - Luid string - DeviceID string - Phys string - Eng string - Engtype string - Part string -} - -type PidPhys struct { - Pid string - Luid string - DeviceID string - Phys string -} - -type PidPhysEngEngType struct { - Pid string - Luid string - DeviceID string - Phys string - Eng string - Engtype string + Pid string + Luid string + Phys string + Eng string + Engtype string + Part string } func parseGPUCounterInstanceString(s string) Instance { // Example: "pid_1234_luid_0x00000000_0x00005678_phys_0_eng_0_engtype_3D" // Example: "luid_0x00000000_0x00005678_phys_0" // Example: "luid_0x00000000_0x00005678_phys_0_part_0" + // Example: "pid_1234_luid_0x00000000_0x00005678_phys_0_eng_1_engtype_Compute_0" parts := strings.Split(s, "_") var instance Instance @@ -75,9 +69,12 @@ func parseGPUCounterInstanceString(s string) Instance { instance.Eng = parts[i+1] } case "engtype": + // The engine type is always the last element and may contain underscores, e.g. "Compute_0". if i+1 < len(parts) { - instance.Engtype = parts[i+1] + instance.Engtype = strings.Join(parts[i+1:], "_") } + + return instance case "part": if i+1 < len(parts) { instance.Part = parts[i+1] diff --git a/internal/collector/gpu/utils_internal_test.go b/internal/collector/gpu/utils_internal_test.go new file mode 100644 index 00000000..59e85e7d --- /dev/null +++ b/internal/collector/gpu/utils_internal_test.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package gpu + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseGPUCounterInstanceString(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + input string + want Instance + }{ + { + name: "engine with pid", + input: "pid_1234_luid_0x00000000_0x0001136A_phys_0_eng_0_engtype_3D", + want: Instance{Pid: "1234", Luid: "0x00000000_0x0001136A", Phys: "0", Eng: "0", Engtype: "3D"}, + }, + { + name: "engine type with underscore", + input: "pid_42_luid_0x00000000_0x00012394_phys_1_eng_12_engtype_Compute_0", + want: Instance{Pid: "42", Luid: "0x00000000_0x00012394", Phys: "1", Eng: "12", Engtype: "Compute_0"}, + }, + { + name: "process memory", + input: "pid_10232_luid_0x00000000_0x0001136A_phys_0", + want: Instance{Pid: "10232", Luid: "0x00000000_0x0001136A", Phys: "0"}, + }, + { + name: "adapter memory", + input: "luid_0x00000000_0x0001136A_phys_0", + want: Instance{Luid: "0x00000000_0x0001136A", Phys: "0"}, + }, + { + name: "adapter memory with part", + input: "luid_0x00000000_0x0001136A_phys_0_part_0", + want: Instance{Luid: "0x00000000_0x0001136A", Phys: "0", Part: "0"}, + }, + { + name: "truncated luid", + input: "luid_0x00000000", + want: Instance{}, + }, + { + name: "empty", + input: "", + want: Instance{}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.want, parseGPUCounterInstanceString(tc.input)) + }) + } +} diff --git a/internal/headers/cfgmgr32/cfgmgr32.go b/internal/headers/cfgmgr32/cfgmgr32.go index e0b2e180..5a442e30 100644 --- a/internal/headers/cfgmgr32/cfgmgr32.go +++ b/internal/headers/cfgmgr32/cfgmgr32.go @@ -16,6 +16,7 @@ package cfgmgr32 import ( + "errors" "fmt" "unsafe" @@ -46,47 +47,64 @@ func GetDevicesInstanceIDs(deviceID string) ([]Device, error) { deviceInstanceIDs := win32.ParseMultiSz(listBuffer) devices := make([]Device, 0, len(deviceInstanceIDs)) + var errs []error + + // Devices whose properties can't be read are skipped. The remaining devices + // are returned together with an error describing the skipped ones. for _, deviceInstanceID := range deviceInstanceIDs { - var devNode *windows.Handle - - err = CMLocateDevNode(&devNode, deviceInstanceID) + device, err := getDevice(deviceInstanceID) if err != nil { - return nil, err + errs = append(errs, fmt.Errorf("device %s: %w", windows.UTF16ToString(deviceInstanceID), err)) + + continue } - var ( - busNumber uint32 - deviceAddress uint32 - propType uint32 - ) - - propLen := uint32(4) - - err = CMGetDevNodeProperty(devNode, DEVPKEYDeviceBusNumber, &propType, unsafe.Pointer(&busNumber), &propLen) - if err != nil { - return nil, err - } - - if propType != DEVPROP_TYPE_UINT32 { - return nil, fmt.Errorf("unexpected property type: 0x%08X", propType) - } - - err = CMGetDevNodeProperty(devNode, DEVPKEYDeviceAddress, &propType, unsafe.Pointer(&deviceAddress), &propLen) - if err != nil { - return nil, err - } - - if propType != DEVPROP_TYPE_UINT32 { - return nil, fmt.Errorf("unexpected property type: 0x%08X", propType) - } - - devices = append(devices, Device{ - InstanceID: windows.UTF16ToString(deviceInstanceID), - BusNumber: win32.UINT(busNumber), - DeviceNumber: win32.UINT(deviceAddress >> 16), - FunctionNumber: win32.UINT(deviceAddress & 0xFFFF), - }) + devices = append(devices, device) } - return devices, nil + return devices, errors.Join(errs...) +} + +func getDevice(deviceInstanceID []uint16) (Device, error) { + var devInst uint32 + + if err := CMLocateDevNode(&devInst, deviceInstanceID); err != nil { + return Device{}, err + } + + busNumber, err := getDevNodePropertyUint32(devInst, DEVPKEYDeviceBusNumber) + if err != nil { + return Device{}, fmt.Errorf("bus number: %w", err) + } + + deviceAddress, err := getDevNodePropertyUint32(devInst, DEVPKEYDeviceAddress) + if err != nil { + return Device{}, fmt.Errorf("device address: %w", err) + } + + return Device{ + InstanceID: windows.UTF16ToString(deviceInstanceID), + BusNumber: win32.UINT(busNumber), + DeviceNumber: win32.UINT(deviceAddress >> 16), + FunctionNumber: win32.UINT(deviceAddress & 0xFFFF), + }, nil +} + +func getDevNodePropertyUint32(devInst uint32, propKey *DEVPROPKEY) (uint32, error) { + var ( + value uint32 + propType uint32 + ) + + propLen := uint32(unsafe.Sizeof(value)) + + if err := CMGetDevNodeProperty(devInst, propKey, &propType, unsafe.Pointer(&value), &propLen); err != nil { + return 0, err + } + + if propType != DEVPROP_TYPE_UINT32 { + return 0, fmt.Errorf("unexpected property type: 0x%08X", propType) + } + + return value, nil } diff --git a/internal/headers/cfgmgr32/cfgmgr32_test.go b/internal/headers/cfgmgr32/cfgmgr32_test.go new file mode 100644 index 00000000..77bc597b --- /dev/null +++ b/internal/headers/cfgmgr32/cfgmgr32_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package cfgmgr32_test + +import ( + "strings" + "testing" + + "github.com/prometheus-community/windows_exporter/internal/headers/cfgmgr32" + "github.com/prometheus-community/windows_exporter/internal/headers/gdi32" + "github.com/stretchr/testify/require" +) + +func TestGetDevicesInstanceIDs(t *testing.T) { + t.Parallel() + + gpus, err := gdi32.GetGPUDevices() + require.NoError(t, err) + + var tested bool + + for _, gpu := range gpus { + // Only test physical render adapters, which are enumerated on the PCI bus. + // Microsoft adapters (VEN_1414), like the Microsoft Basic Render Driver, are virtual + // and not always flagged as software device. + if gpu.IsSoftwareDevice() || + strings.HasPrefix(gpu.DeviceID, `PCI\VEN_1414&`) || + gpu.AdapterType&gdi32.D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED == 0 || + gpu.AdapterType&gdi32.D3DKMT_ADAPTERTYPE_PARAVIRTUALIZED != 0 { + continue + } + + devices, err := cfgmgr32.GetDevicesInstanceIDs(gpu.DeviceID) + require.NoError(t, err) + require.NotEmpty(t, devices) + + var found bool + + for _, device := range devices { + require.NotEmpty(t, device.InstanceID) + + if device.BusNumber == gpu.BusNumber && device.DeviceNumber == gpu.DeviceNumber && device.FunctionNumber == gpu.FunctionNumber { + found = true + } + } + + require.True(t, found, "no device instance matches the PCI address of %s", gpu.DeviceID) + + tested = true + } + + if !tested { + t.Skip("no hardware GPU found") + } +} diff --git a/internal/headers/cfgmgr32/syscall.go b/internal/headers/cfgmgr32/syscall.go index 837c964d..0ecfd1bc 100644 --- a/internal/headers/cfgmgr32/syscall.go +++ b/internal/headers/cfgmgr32/syscall.go @@ -62,7 +62,8 @@ func CMGetDeviceIDList(filter *win32.LPWSTR, buf []uint16) error { return nil } -func CMLocateDevNode(devInst **windows.Handle, deviceID []uint16) error { +// CMLocateDevNode stores the DEVINST (a DWORD) of the device node in devInst. +func CMLocateDevNode(devInst *uint32, deviceID []uint16) error { ret, _, _ := procCMLocateDevNodeW.Call( uintptr(unsafe.Pointer(devInst)), uintptr(unsafe.Pointer(&deviceID[0])), @@ -76,9 +77,9 @@ func CMLocateDevNode(devInst **windows.Handle, deviceID []uint16) error { return nil } -func CMGetDevNodeProperty(devInst *windows.Handle, propKey *DEVPROPKEY, propType *uint32, buf unsafe.Pointer, bufLen *uint32) error { +func CMGetDevNodeProperty(devInst uint32, propKey *DEVPROPKEY, propType *uint32, buf unsafe.Pointer, bufLen *uint32) error { ret, _, _ := procCMGetDevNodePropertyW.Call( - uintptr(unsafe.Pointer(devInst)), + uintptr(devInst), uintptr(unsafe.Pointer(propKey)), uintptr(unsafe.Pointer(propType)), uintptr(buf), diff --git a/internal/headers/gdi32/gdi32.go b/internal/headers/gdi32/gdi32.go index aae72f2d..1274f5c5 100644 --- a/internal/headers/gdi32/gdi32.go +++ b/internal/headers/gdi32/gdi32.go @@ -34,6 +34,8 @@ const ( KMTQAITYPE_ADAPTERADDRESS = 6 // KMTQAITYPE_ADAPTERREGISTRYINFO pPrivateDriverData points to a D3DKMT_ADAPTERREGISTRYINFO structure that contains registry information about the graphics adapter. KMTQAITYPE_ADAPTERREGISTRYINFO = 8 + // KMTQAITYPE_ADAPTERTYPE pPrivateDriverData points to a D3DKMT_ADAPTERTYPE structure that specifies the type of graphics adapter. Supported starting with Windows 8 (WDDM 1.2). + KMTQAITYPE_ADAPTERTYPE = 15 // KMTQAITYPE_PHYSICALADAPTERDEVICEIDS pPrivateDriverData points to a D3DKMT_QUERY_DEVICE_IDS structure that specifies the device ID(s) of the physical adapters. Supported starting with Windows 10 (WDDM 2.0). KMTQAITYPE_PHYSICALADAPTERDEVICEIDS = 31 // KMTQAITYPE_DRIVERVERSION pPrivateDriverData points to a D3DKMT_DRIVERVERSION value that indicates the WDDM version of the display miniport driver. @@ -103,6 +105,18 @@ func GetGPUDevice(hAdapter D3DKMT_HANDLE) (GPUDevice, error) { gpuDevice.AdapterString = windows.UTF16ToString(info.AdapterString[:]) + var adapterType D3DKMT_ADAPTERTYPE + + query.queryType = KMTQAITYPE_ADAPTERTYPE + query.pPrivateDriverData = unsafe.Pointer(&adapterType) + query.privateDriverDataSize = uint32(unsafe.Sizeof(adapterType)) + + if err := D3DKMTQueryAdapterInfo(&query); err != nil { + return gpuDevice, fmt.Errorf("D3DKMTQueryAdapterInfo (adapter type) failed: %w", err) + } + + gpuDevice.AdapterType = adapterType + var deviceIDs D3DKMT_QUERY_DEVICE_IDS query.queryType = KMTQAITYPE_PHYSICALADAPTERDEVICEIDS @@ -118,6 +132,9 @@ func GetGPUDevice(hAdapter D3DKMT_HANDLE) (GPUDevice, error) { return gpuDevice, nil } +// GetGPUDevices returns all GPU adapters known to the graphics kernel. +// If a single adapter can't be queried, the remaining adapters are still +// returned, together with an error describing the failed adapters. func GetGPUDevices() ([]GPUDevice, error) { gpuDevices := make([]GPUDevice, 0, 2) @@ -177,15 +194,11 @@ func GetGPUDevices() ([]GPUDevice, error) { }() } - if len(errs) > 0 { - return gpuDevices, errors.Join(errs...) - } - if len(gpuDevices) == 0 { - return gpuDevices, ErrNoGPUDevices + errs = append(errs, ErrNoGPUDevices) } - return gpuDevices, nil + return gpuDevices, errors.Join(errs...) } func formatPNPDeviceID(deviceIDs D3DKMT_QUERY_DEVICE_IDS) string { diff --git a/internal/headers/gdi32/gdi32_test.go b/internal/headers/gdi32/gdi32_test.go index b11c7ddb..45d26abe 100644 --- a/internal/headers/gdi32/gdi32_test.go +++ b/internal/headers/gdi32/gdi32_test.go @@ -27,4 +27,11 @@ func TestGetGPUDevices(t *testing.T) { require.NoError(t, err, "Failed to get GPU devices") require.NotNil(t, devices) + + // The adapter type flags differ between systems, e.g. the Microsoft Basic Render Driver + // is not flagged as software device on Windows Server 2022. Log them for diagnostics. + for _, device := range devices { + t.Logf("adapter %q: device ID %s, adapter type 0b%b, software device %t", + device.AdapterString, device.DeviceID, uint32(device.AdapterType), device.IsSoftwareDevice()) + } } diff --git a/internal/headers/gdi32/types.go b/internal/headers/gdi32/types.go index c8546726..aaea4e71 100644 --- a/internal/headers/gdi32/types.go +++ b/internal/headers/gdi32/types.go @@ -95,6 +95,7 @@ type GPUDevice struct { BusNumber win32.UINT DeviceNumber win32.UINT FunctionNumber win32.UINT + AdapterType D3DKMT_ADAPTERTYPE } // D3DKMT_NODE_PERFDATA is the output of KMTQAITYPE_NODEPERFDATA. @@ -163,3 +164,30 @@ type D3DKMT_GPUVERSION struct { BiosVersion [32]uint16 GpuArchitecture [32]uint16 } + +// IsSoftwareDevice reports whether the adapter is a software device, +// e.g. the Microsoft Basic Render Driver. +func (d GPUDevice) IsSoftwareDevice() bool { + return d.AdapterType&D3DKMT_ADAPTERTYPE_SOFTWARE_DEVICE != 0 +} + +// D3DKMT_ADAPTERTYPE is the bitfield returned by KMTQAITYPE_ADAPTERTYPE. +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/d3dkmthk/ns-d3dkmthk-_d3dkmt_adaptertype +type D3DKMT_ADAPTERTYPE uint32 + +const ( + D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED D3DKMT_ADAPTERTYPE = 1 << 0 + D3DKMT_ADAPTERTYPE_DISPLAY_SUPPORTED D3DKMT_ADAPTERTYPE = 1 << 1 + D3DKMT_ADAPTERTYPE_SOFTWARE_DEVICE D3DKMT_ADAPTERTYPE = 1 << 2 + D3DKMT_ADAPTERTYPE_POST_DEVICE D3DKMT_ADAPTERTYPE = 1 << 3 + D3DKMT_ADAPTERTYPE_HYBRID_DISCRETE D3DKMT_ADAPTERTYPE = 1 << 4 + D3DKMT_ADAPTERTYPE_HYBRID_INTEGRATED D3DKMT_ADAPTERTYPE = 1 << 5 + D3DKMT_ADAPTERTYPE_INDIRECT_DISPLAY_DEVICE D3DKMT_ADAPTERTYPE = 1 << 6 + D3DKMT_ADAPTERTYPE_PARAVIRTUALIZED D3DKMT_ADAPTERTYPE = 1 << 7 + D3DKMT_ADAPTERTYPE_ACG_SUPPORTED D3DKMT_ADAPTERTYPE = 1 << 8 + D3DKMT_ADAPTERTYPE_SET_TIMINGS_FROM_VIDPN D3DKMT_ADAPTERTYPE = 1 << 9 + D3DKMT_ADAPTERTYPE_DETACHABLE D3DKMT_ADAPTERTYPE = 1 << 10 + D3DKMT_ADAPTERTYPE_COMPUTE_ONLY D3DKMT_ADAPTERTYPE = 1 << 11 + D3DKMT_ADAPTERTYPE_PROTOTYPE D3DKMT_ADAPTERTYPE = 1 << 12 + D3DKMT_ADAPTERTYPE_RUNTIME_POWER_MANAGEMENT D3DKMT_ADAPTERTYPE = 1 << 13 +)