gpu: make device discovery non-fatal and refresh it on unknown LUIDs (#2573)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>

internal/collector/gpu/gpu.go

@@ -39,11 +39,23 @@ type Config struct{}
 //nolint:gochecknoglobals
 var ConfigDefaults = Config{}
 
+// deviceCacheRefreshInterval limits how often the GPU device cache is
+// rebuilt when a performance counter instance references an unknown LUID.
+const deviceCacheRefreshInterval = 30 * time.Second
+
 type Collector struct {
 	config Config
 	logger *slog.Logger
 
-	gpuDeviceCache map[string]gpuDevice
+	// gpuDeviceCache maps the LUID ("0x%08X_0x%08X") of each adapter to its device information.
+	// LUIDs change if a driver is updated or a device is restarted, so the cache is
+	// refreshed whenever a performance counter instance references an unknown LUID.
+	gpuDeviceCache            map[string]gpuDevice
+	gpuDeviceCacheLastRefresh time.Time
+
+	// discoverGPUDevices enumerates the GPU adapters. If nil, gdi32.GetGPUDevices is used.
+	// It is replaced in tests to make the discovery deterministic.
+	discoverGPUDevices func() ([]gdi32.GPUDevice, error)
 
 	// GPU Engine
 	gpuEnginePerfDataCollector *pdh.Collector[gpuEnginePerfDataCounterValues]
@@ -95,6 +107,11 @@ type gpuDevice struct {
 	cfgmgr32 cfgmgr32.Device
 	sensors  gpuSensors
 	ID       string
+
+	// skip marks adapters that are known, but not exposed, e.g. software devices
+	// like the Microsoft Basic Render Driver. Keeping them in the cache prevents
+	// their performance counter instances from triggering a cache refresh.
+	skip bool
 }
 
 func New(config *Config) *Collector {
@@ -134,8 +151,8 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error {
 
 	c.gpuInfo = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "info"),
-		"A metric with a constant '1' value labeled with gpu device information.",
-		[]string{"luid", "device_id", "name", "bus_number", "phys", "function_number", "driver_version", "wddm_version", "architecture"},
+		"A metric with a constant '1' value labeled with GPU device information.",
+		[]string{"luid", "device_id", "name", "bus_number", "device_number", "function_number", "driver_version", "wddm_version", "architecture"},
 		nil,
 	)
 
@@ -160,7 +177,7 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error {
 
 	c.gpuEngineRunningTime = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "engine_time_seconds"),
-		"Total running time of the GPU in seconds.",
+		"Total running time of the GPU engine in seconds.",
 		[]string{"process_id", "luid", "device_id", "phys", "eng", "engtype"},
 		nil,
 	)
@@ -258,79 +275,131 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error {
 		errs = append(errs, fmt.Errorf("failed to create GPU Process Memory perf data collector: %w", err))
 	}
 
-	gpus, err := gdi32.GetGPUDevices()
-	if err != nil {
-		errs = append(errs, fmt.Errorf("failed to get GPU devices: %w", err))
+	// GPU discovery problems are not fatal. The collector keeps working with the
+	// adapters that could be discovered and retries on unknown LUIDs during Collect.
+	c.refreshGPUDevices()
+
+	return errors.Join(errs...)
+}
+
+// refreshGPUDevices rebuilds the GPU device cache.
+// Problems with single adapters are logged and do not stop the discovery of the remaining adapters.
+func (c *Collector) refreshGPUDevices() {
+	c.gpuDeviceCacheLastRefresh = time.Now()
+
+	discoverGPUDevices := c.discoverGPUDevices
+	if discoverGPUDevices == nil {
+		discoverGPUDevices = gdi32.GetGPUDevices
 	}
 
+	gpus, err := discoverGPUDevices()
+	if err != nil {
+		c.logger.Warn("failed to discover some GPU devices",
+			slog.Any("err", err),
+		)
+
+		// Keep the previous cache, if the discovery failed completely.
+		if len(gpus) == 0 {
+			return
+		}
+	}
+
+	gpuDeviceCache := make(map[string]gpuDevice, len(gpus))
+
 	for _, gpu := range gpus {
-		if gpu.AdapterString == "" {
-			continue
-		}
-
-		// Skip Microsoft Basic Render Driver
-		// https://devicehunt.com/view/type/pci/vendor/1414/device/008C
-		if gpu.DeviceID == `PCI\VEN_1414&DEV_008C&SUBSYS_00000000&REV_00` {
-			continue
-		}
-
-		if c.gpuDeviceCache == nil {
-			c.gpuDeviceCache = make(map[string]gpuDevice)
-		}
-
 		luidKey := fmt.Sprintf("0x%08X_0x%08X", gpu.LUID.HighPart, gpu.LUID.LowPart)
 
-		deviceID := gpu.DeviceID
+		// Skip software devices like the Microsoft Basic Render Driver.
+		if gpu.AdapterString == "" || gpu.IsSoftwareDevice() || isBasicRenderDriver(gpu) {
+			gpuDeviceCache[luidKey] = gpuDevice{gdi32: gpu, ID: gpu.DeviceID, skip: true}
 
-		cfgmgr32Devs, err := cfgmgr32.GetDevicesInstanceIDs(gpu.DeviceID)
-		if err != nil {
-			errs = append(errs, fmt.Errorf("failed to get device instance IDs for device ID %s: %w", gpu.DeviceID, err))
+			c.logger.Debug("Skipping GPU device",
+				slog.String("name", gpu.AdapterString),
+				slog.String("luid", luidKey),
+				slog.String("device_id", gpu.DeviceID),
+				slog.Uint64("adapter_type", uint64(gpu.AdapterType)),
+			)
+
+			continue
 		}
 
-		var cfgmgr32Dev cfgmgr32.Device
+		device := c.newGPUDevice(gpu, luidKey)
+		device.sensors = discoverSensors(c.logger.With(slog.String("luid", luidKey)), gpu)
+		gpuDeviceCache[luidKey] = device
 
-		for _, dev := range cfgmgr32Devs {
-			if dev.BusNumber == gpu.BusNumber && dev.DeviceNumber == gpu.DeviceNumber && dev.FunctionNumber == gpu.FunctionNumber {
-				cfgmgr32Dev = dev
-
-				break
-			}
-		}
-
-		if cfgmgr32Dev.InstanceID == "" {
-			errs = append(errs, fmt.Errorf("failed to find matching device for device ID %s", gpu.DeviceID))
-		} else {
-			deviceID = cfgmgr32Dev.InstanceID
-		}
-
-		c.gpuDeviceCache[luidKey] = gpuDevice{
-			gdi32:    gpu,
-			cfgmgr32: cfgmgr32Dev,
-			sensors:  discoverSensors(c.logger.With(slog.String("luid", luidKey)), gpu),
-			ID:       deviceID,
-		}
-
-		logger.Debug("Found GPU device",
-			slog.String("collector", Name),
+		c.logger.Debug("Found GPU device",
 			slog.String("name", gpu.AdapterString),
 			slog.String("luid", luidKey),
-			slog.String("device_id", deviceID),
-			slog.String("name", gpu.AdapterString),
+			slog.String("device_id", device.ID),
 			slog.Uint64("bus_number", uint64(gpu.BusNumber)),
 			slog.Uint64("device_number", uint64(gpu.DeviceNumber)),
 			slog.Uint64("function_number", uint64(gpu.FunctionNumber)),
 		)
 	}
 
-	return errors.Join(errs...)
+	c.gpuDeviceCache = gpuDeviceCache
+}
+
+// newGPUDevice resolves the PnP device instance ID of the adapter.
+// If the device instance can't be found, the PnP device ID reported by gdi32 is used as ID.
+func (c *Collector) newGPUDevice(gpu gdi32.GPUDevice, luidKey string) gpuDevice {
+	device := gpuDevice{
+		gdi32: gpu,
+		ID:    gpu.DeviceID,
+	}
+
+	cfgmgr32Devs, err := cfgmgr32.GetDevicesInstanceIDs(gpu.DeviceID)
+
+	for _, dev := range cfgmgr32Devs {
+		if dev.BusNumber == gpu.BusNumber && dev.DeviceNumber == gpu.DeviceNumber && dev.FunctionNumber == gpu.FunctionNumber {
+			device.cfgmgr32 = dev
+			device.ID = dev.InstanceID
+
+			return device
+		}
+	}
+
+	attrs := []any{
+		slog.String("name", gpu.AdapterString),
+		slog.String("luid", luidKey),
+		slog.String("device_id", gpu.DeviceID),
+	}
+
+	if err != nil {
+		attrs = append(attrs, slog.Any("err", err))
+	}
+
+	c.logger.Warn("failed to find matching PnP device instance for GPU device, using the PnP device ID as device_id", attrs...)
+
+	return device
+}
+
+// getGPUDevice returns the cached device for the given LUID.
+// If the LUID is unknown, e.g. because a driver was updated, a device was restarted
+// or a device was hot-plugged, the cache is refreshed at most once per deviceCacheRefreshInterval.
+// Devices marked as skip are not returned.
+func (c *Collector) getGPUDevice(luid string) (gpuDevice, bool) {
+	if luid == "" {
+		return gpuDevice{}, false
+	}
+
+	device, ok := c.gpuDeviceCache[luid]
+	if !ok && time.Since(c.gpuDeviceCacheLastRefresh) >= deviceCacheRefreshInterval {
+		c.logger.Debug("unknown GPU LUID, refreshing GPU device cache",
+			slog.String("luid", luid),
+		)
+
+		c.refreshGPUDevices()
+
+		device, ok = c.gpuDeviceCache[luid]
+	}
+
+	return device, ok && !device.skip
 }
 
 func (c *Collector) Collect(ch chan<- prometheus.Metric, _ time.Duration) error {
 	errs := make([]error, 0)
 
-	c.collectGpuInfo(ch)
-	c.collectGpuSensorMetrics(ch)
-
 	if err := c.collectGpuEngineMetrics(ch); err != nil {
 		errs = append(errs, err)
 	}
@@ -351,11 +420,19 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric, _ time.Duration) error
 		errs = append(errs, err)
 	}
 
+	// Collected last, since the metrics above may refresh the GPU device cache.
+	c.collectGpuInfo(ch)
+	c.collectGpuSensorMetrics(ch)
+
 	return errors.Join(errs...)
 }
 
 func (c *Collector) collectGpuInfo(ch chan<- prometheus.Metric) {
 	for luid, gpu := range c.gpuDeviceCache {
+		if gpu.skip {
+			continue
+		}
+
 		ch <- prometheus.MustNewConstMetric(
 			c.gpuInfo,
 			prometheus.GaugeValue,
@@ -404,7 +481,7 @@ func (c *Collector) collectGpuEngineMetrics(ch chan<- prometheus.Metric) error {
 	for _, data := range c.gpuEnginePerfDataObject {
 		instance := parseGPUCounterInstanceString(data.Name)
 
-		device, ok := c.gpuDeviceCache[instance.Luid]
+		device, ok := c.getGPUDevice(instance.Luid)
 		if !ok {
 			continue
 		}
@@ -429,7 +506,7 @@ func (c *Collector) collectGpuAdapterMemoryMetrics(ch chan<- prometheus.Metric)
 	for _, data := range c.gpuAdapterMemoryPerfDataObject {
 		instance := parseGPUCounterInstanceString(data.Name)
 
-		device, ok := c.gpuDeviceCache[instance.Luid]
+		device, ok := c.getGPUDevice(instance.Luid)
 		if !ok {
 			continue
 		}
@@ -468,7 +545,7 @@ func (c *Collector) collectGpuLocalAdapterMemoryMetrics(ch chan<- prometheus.Met
 	for _, data := range c.gpuLocalAdapterMemoryPerfDataObject {
 		instance := parseGPUCounterInstanceString(data.Name)
 
-		device, ok := c.gpuDeviceCache[instance.Luid]
+		device, ok := c.getGPUDevice(instance.Luid)
 		if !ok {
 			continue
 		}
@@ -493,7 +570,7 @@ func (c *Collector) collectGpuNonLocalAdapterMemoryMetrics(ch chan<- prometheus.
 	for _, data := range c.gpuNonLocalAdapterMemoryPerfDataObject {
 		instance := parseGPUCounterInstanceString(data.Name)
 
-		device, ok := c.gpuDeviceCache[instance.Luid]
+		device, ok := c.getGPUDevice(instance.Luid)
 		if !ok {
 			continue
 		}
@@ -518,7 +595,7 @@ func (c *Collector) collectGpuProcessMemoryMetrics(ch chan<- prometheus.Metric)
 	for _, data := range c.gpuProcessMemoryPerfDataObject {
 		instance := parseGPUCounterInstanceString(data.Name)
 
-		device, ok := c.gpuDeviceCache[instance.Luid]
+		device, ok := c.getGPUDevice(instance.Luid)
 		if !ok {
 			continue
 		}

internal/collector/gpu/gpu_internal_test.go

@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package gpu
+
+import (
+	"errors"
+	"log/slog"
+	"testing"
+	"time"
+
+	"github.com/prometheus-community/windows_exporter/internal/headers/gdi32"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/windows"
+)
+
+// Device IDs of the test adapters. No real device instance matches them,
+// so the hardware adapter keeps its PnP device ID as ID.
+const (
+	testHardwareDeviceID    = `PCI\VEN_FFFF&DEV_FFFF&SUBSYS_00000000&REV_00`
+	testBasicRenderDeviceID = `PCI\VEN_1414&DEV_008C&SUBSYS_00000000&REV_00`
+)
+
+//nolint:gochecknoglobals
+var (
+	testHardwareDevice = gdi32.GPUDevice{
+		AdapterString: "Test GPU",
+		LUID:          windows.LUID{LowPart: 0x20},
+		DeviceID:      testHardwareDeviceID,
+		AdapterType:   gdi32.D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED | gdi32.D3DKMT_ADAPTERTYPE_DISPLAY_SUPPORTED,
+	}
+	testSoftwareDevice = gdi32.GPUDevice{
+		AdapterString: "Test Software Adapter",
+		LUID:          windows.LUID{LowPart: 0x10},
+		DeviceID:      `PCI\VEN_FFFF&DEV_0001&SUBSYS_00000000&REV_00`,
+		AdapterType:   gdi32.D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED | gdi32.D3DKMT_ADAPTERTYPE_SOFTWARE_DEVICE,
+	}
+	// The Microsoft Basic Render Driver is not flagged as software device on every system,
+	// e.g. Windows Server 2022.
+	testBasicRenderDevice = gdi32.GPUDevice{
+		AdapterString: "Microsoft Basic Render Driver",
+		LUID:          windows.LUID{LowPart: 0x30},
+		DeviceID:      testBasicRenderDeviceID,
+		AdapterType:   gdi32.D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED,
+	}
+)
+
+func newTestCollector(cache map[string]gpuDevice, lastRefresh time.Time, discover func() ([]gdi32.GPUDevice, error)) *Collector {
+	return &Collector{
+		logger:                    slog.New(slog.DiscardHandler),
+		gpuDeviceCache:            cache,
+		gpuDeviceCacheLastRefresh: lastRefresh,
+		discoverGPUDevices:        discover,
+	}
+}
+
+func noDiscovery(t *testing.T) func() ([]gdi32.GPUDevice, error) {
+	t.Helper()
+
+	return func() ([]gdi32.GPUDevice, error) {
+		t.Error("unexpected GPU device discovery")
+
+		return nil, nil
+	}
+}
+
+func TestGetGPUDevice(t *testing.T) {
+	t.Parallel()
+
+	expired := time.Now().Add(-2 * deviceCacheRefreshInterval)
+
+	t.Run("known device", func(t *testing.T) {
+		t.Parallel()
+
+		c := newTestCollector(map[string]gpuDevice{"known": {ID: "known"}}, time.Now(), noDiscovery(t))
+
+		device, ok := c.getGPUDevice("known")
+		require.True(t, ok)
+		require.Equal(t, "known", device.ID)
+	})
+
+	t.Run("skipped device does not trigger refresh", func(t *testing.T) {
+		t.Parallel()
+
+		c := newTestCollector(map[string]gpuDevice{"software": {skip: true}}, expired, noDiscovery(t))
+
+		_, ok := c.getGPUDevice("software")
+		require.False(t, ok)
+		require.Equal(t, expired, c.gpuDeviceCacheLastRefresh)
+	})
+
+	t.Run("unknown device within refresh interval", func(t *testing.T) {
+		t.Parallel()
+
+		lastRefresh := time.Now()
+		c := newTestCollector(map[string]gpuDevice{"stale": {ID: "stale"}}, lastRefresh, noDiscovery(t))
+
+		_, ok := c.getGPUDevice("unknown")
+		require.False(t, ok)
+		require.Equal(t, lastRefresh, c.gpuDeviceCacheLastRefresh)
+		require.Contains(t, c.gpuDeviceCache, "stale")
+	})
+
+	t.Run("empty LUID", func(t *testing.T) {
+		t.Parallel()
+
+		c := newTestCollector(nil, expired, noDiscovery(t))
+
+		_, ok := c.getGPUDevice("")
+		require.False(t, ok)
+		require.Equal(t, expired, c.gpuDeviceCacheLastRefresh)
+	})
+
+	t.Run("unknown device after refresh interval", func(t *testing.T) {
+		t.Parallel()
+
+		c := newTestCollector(map[string]gpuDevice{"stale": {ID: "stale"}}, expired, func() ([]gdi32.GPUDevice, error) {
+			return []gdi32.GPUDevice{testHardwareDevice, testSoftwareDevice, testBasicRenderDevice}, nil
+		})
+
+		device, ok := c.getGPUDevice("0x00000000_0x00000020")
+		require.True(t, ok)
+		require.Equal(t, testHardwareDeviceID, device.ID)
+		require.Equal(t, "Test GPU", device.gdi32.AdapterString)
+		require.True(t, c.gpuDeviceCacheLastRefresh.After(expired))
+		require.NotContains(t, c.gpuDeviceCache, "stale")
+
+		// Software devices are cached, but not exposed.
+		for _, luid := range []string{"0x00000000_0x00000010", "0x00000000_0x00000030"} {
+			require.Contains(t, c.gpuDeviceCache, luid)
+
+			_, ok = c.getGPUDevice(luid)
+			require.False(t, ok, luid)
+		}
+	})
+
+	t.Run("partial discovery failure", func(t *testing.T) {
+		t.Parallel()
+
+		c := newTestCollector(map[string]gpuDevice{"stale": {ID: "stale"}}, expired, func() ([]gdi32.GPUDevice, error) {
+			return []gdi32.GPUDevice{testHardwareDevice}, errors.New("failed to get GPU device for adapter")
+		})
+
+		_, ok := c.getGPUDevice("0x00000000_0x00000020")
+		require.True(t, ok)
+		require.NotContains(t, c.gpuDeviceCache, "stale")
+	})
+
+	t.Run("complete discovery failure keeps the previous cache", func(t *testing.T) {
+		t.Parallel()
+
+		c := newTestCollector(map[string]gpuDevice{"stale": {ID: "stale"}}, expired, func() ([]gdi32.GPUDevice, error) {
+			return nil, gdi32.ErrNoGPUDevices
+		})
+
+		_, ok := c.getGPUDevice("unknown")
+		require.False(t, ok)
+		require.True(t, c.gpuDeviceCacheLastRefresh.After(expired))
+		require.Contains(t, c.gpuDeviceCache, "stale")
+	})
+}

internal/collector/gpu/sensors.go

@@ -148,11 +148,12 @@ func newSensorMetrics() sensorMetrics {
 	}
 }
 
-// isSoftwareAdapter reports whether the adapter is a Microsoft software or virtual adapter,
-// like the Microsoft Basic Render Driver (PCI\VEN_1414&DEV_008C). These adapters have no
-// sensors and reject the perf data queries with STATUS_INVALID_PARAMETER.
+// isSoftwareAdapter reports whether the adapter is a software adapter (D3DKMT_ADAPTERTYPE
+// SoftwareDevice flag) or a Microsoft virtual adapter, like the Microsoft Basic Render Driver
+// (PCI\VEN_1414&DEV_008C). These adapters have no sensors and reject the perf data queries
+// with STATUS_INVALID_PARAMETER.
 func isSoftwareAdapter(device gdi32.GPUDevice) bool {
-	return device.AdapterString == "" || strings.HasPrefix(device.DeviceID, `PCI\VEN_1414&`)
+	return device.AdapterString == "" || device.IsSoftwareDevice() || strings.HasPrefix(device.DeviceID, `PCI\VEN_1414&`)
 }
 
 // discoverSensors reads the static adapter data and probes which sensors the driver supports.
@@ -264,6 +265,10 @@ func discoverPhysicalAdapterSensors(logger *slog.Logger, hAdapter gdi32.D3DKMT_H
 
 func (c *Collector) collectGpuSensorMetrics(ch chan<- prometheus.Metric) {
 	for luid, device := range c.gpuDeviceCache {
+		if device.skip {
+			continue
+		}
+
 		// The caps were read during discovery and need no adapter handle.
 		for _, physicalAdapter := range device.sensors.physicalAdapters {
 			c.collectPhysicalAdapterCaps(ch, []string{luid, device.ID, physicalAdapter.phys}, physicalAdapter.caps)

internal/collector/gpu/types.go

@@ -20,8 +20,7 @@ package gpu
 type gpuEnginePerfDataCounterValues struct {
 	Name string
 
-	RunningTime           float64 `perfdata:"Running Time"`
-	UtilizationPercentage float64 `perfdata:"Utilization Percentage"`
+	RunningTime float64 `perfdata:"Running Time"`
 }
 
 type gpuAdapterMemoryPerfDataCounterValues struct {

internal/collector/gpu/utils.go

@@ -20,38 +20,32 @@ package gpu
 import (
 	"fmt"
 	"strings"
+
+	"github.com/prometheus-community/windows_exporter/internal/headers/gdi32"
 )
 
+// isBasicRenderDriver reports whether the adapter is the Microsoft Basic Render Driver.
+// https://devicehunt.com/view/type/pci/vendor/1414/device/008C
+// The D3DKMT_ADAPTERTYPE SoftwareDevice flag is not set for it on every system,
+// e.g. Windows Server 2022, so the device ID is checked as well.
+func isBasicRenderDriver(device gdi32.GPUDevice) bool {
+	return strings.HasPrefix(device.DeviceID, `PCI\VEN_1414&DEV_008C&`)
+}
+
 type Instance struct {
-	Pid      string
-	Luid     string
-	DeviceID string
-	Phys     string
-	Eng      string
-	Engtype  string
-	Part     string
-}
-
-type PidPhys struct {
-	Pid      string
-	Luid     string
-	DeviceID string
-	Phys     string
-}
-
-type PidPhysEngEngType struct {
-	Pid      string
-	Luid     string
-	DeviceID string
-	Phys     string
-	Eng      string
-	Engtype  string
+	Pid     string
+	Luid    string
+	Phys    string
+	Eng     string
+	Engtype string
+	Part    string
 }
 
 func parseGPUCounterInstanceString(s string) Instance {
 	// Example: "pid_1234_luid_0x00000000_0x00005678_phys_0_eng_0_engtype_3D"
 	// Example: "luid_0x00000000_0x00005678_phys_0"
 	// Example: "luid_0x00000000_0x00005678_phys_0_part_0"
+	// Example: "pid_1234_luid_0x00000000_0x00005678_phys_0_eng_1_engtype_Compute_0"
 	parts := strings.Split(s, "_")
 
 	var instance Instance
@@ -75,9 +69,12 @@ func parseGPUCounterInstanceString(s string) Instance {
 				instance.Eng = parts[i+1]
 			}
 		case "engtype":
+			// The engine type is always the last element and may contain underscores, e.g. "Compute_0".
 			if i+1 < len(parts) {
-				instance.Engtype = parts[i+1]
+				instance.Engtype = strings.Join(parts[i+1:], "_")
 			}
+
+			return instance
 		case "part":
 			if i+1 < len(parts) {
 				instance.Part = parts[i+1]

internal/collector/gpu/utils_internal_test.go

@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package gpu
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseGPUCounterInstanceString(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name  string
+		input string
+		want  Instance
+	}{
+		{
+			name:  "engine with pid",
+			input: "pid_1234_luid_0x00000000_0x0001136A_phys_0_eng_0_engtype_3D",
+			want:  Instance{Pid: "1234", Luid: "0x00000000_0x0001136A", Phys: "0", Eng: "0", Engtype: "3D"},
+		},
+		{
+			name:  "engine type with underscore",
+			input: "pid_42_luid_0x00000000_0x00012394_phys_1_eng_12_engtype_Compute_0",
+			want:  Instance{Pid: "42", Luid: "0x00000000_0x00012394", Phys: "1", Eng: "12", Engtype: "Compute_0"},
+		},
+		{
+			name:  "process memory",
+			input: "pid_10232_luid_0x00000000_0x0001136A_phys_0",
+			want:  Instance{Pid: "10232", Luid: "0x00000000_0x0001136A", Phys: "0"},
+		},
+		{
+			name:  "adapter memory",
+			input: "luid_0x00000000_0x0001136A_phys_0",
+			want:  Instance{Luid: "0x00000000_0x0001136A", Phys: "0"},
+		},
+		{
+			name:  "adapter memory with part",
+			input: "luid_0x00000000_0x0001136A_phys_0_part_0",
+			want:  Instance{Luid: "0x00000000_0x0001136A", Phys: "0", Part: "0"},
+		},
+		{
+			name:  "truncated luid",
+			input: "luid_0x00000000",
+			want:  Instance{},
+		},
+		{
+			name:  "empty",
+			input: "",
+			want:  Instance{},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			require.Equal(t, tc.want, parseGPUCounterInstanceString(tc.input))
+		})
+	}
+}

internal/headers/cfgmgr32/cfgmgr32.go

@@ -16,6 +16,7 @@
 package cfgmgr32
 
 import (
+	"errors"
 	"fmt"
 	"unsafe"
 
@@ -46,47 +47,64 @@ func GetDevicesInstanceIDs(deviceID string) ([]Device, error) {
 	deviceInstanceIDs := win32.ParseMultiSz(listBuffer)
 	devices := make([]Device, 0, len(deviceInstanceIDs))
 
+	var errs []error
+
+	// Devices whose properties can't be read are skipped. The remaining devices
+	// are returned together with an error describing the skipped ones.
 	for _, deviceInstanceID := range deviceInstanceIDs {
-		var devNode *windows.Handle
-
-		err = CMLocateDevNode(&devNode, deviceInstanceID)
+		device, err := getDevice(deviceInstanceID)
 		if err != nil {
-			return nil, err
+			errs = append(errs, fmt.Errorf("device %s: %w", windows.UTF16ToString(deviceInstanceID), err))
+
+			continue
 		}
 
-		var (
-			busNumber     uint32
-			deviceAddress uint32
-			propType      uint32
-		)
-
-		propLen := uint32(4)
-
-		err = CMGetDevNodeProperty(devNode, DEVPKEYDeviceBusNumber, &propType, unsafe.Pointer(&busNumber), &propLen)
-		if err != nil {
-			return nil, err
-		}
-
-		if propType != DEVPROP_TYPE_UINT32 {
-			return nil, fmt.Errorf("unexpected property type: 0x%08X", propType)
-		}
-
-		err = CMGetDevNodeProperty(devNode, DEVPKEYDeviceAddress, &propType, unsafe.Pointer(&deviceAddress), &propLen)
-		if err != nil {
-			return nil, err
-		}
-
-		if propType != DEVPROP_TYPE_UINT32 {
-			return nil, fmt.Errorf("unexpected property type: 0x%08X", propType)
-		}
-
-		devices = append(devices, Device{
-			InstanceID:     windows.UTF16ToString(deviceInstanceID),
-			BusNumber:      win32.UINT(busNumber),
-			DeviceNumber:   win32.UINT(deviceAddress >> 16),
-			FunctionNumber: win32.UINT(deviceAddress & 0xFFFF),
-		})
+		devices = append(devices, device)
 	}
 
-	return devices, nil
+	return devices, errors.Join(errs...)
+}
+
+func getDevice(deviceInstanceID []uint16) (Device, error) {
+	var devInst uint32
+
+	if err := CMLocateDevNode(&devInst, deviceInstanceID); err != nil {
+		return Device{}, err
+	}
+
+	busNumber, err := getDevNodePropertyUint32(devInst, DEVPKEYDeviceBusNumber)
+	if err != nil {
+		return Device{}, fmt.Errorf("bus number: %w", err)
+	}
+
+	deviceAddress, err := getDevNodePropertyUint32(devInst, DEVPKEYDeviceAddress)
+	if err != nil {
+		return Device{}, fmt.Errorf("device address: %w", err)
+	}
+
+	return Device{
+		InstanceID:     windows.UTF16ToString(deviceInstanceID),
+		BusNumber:      win32.UINT(busNumber),
+		DeviceNumber:   win32.UINT(deviceAddress >> 16),
+		FunctionNumber: win32.UINT(deviceAddress & 0xFFFF),
+	}, nil
+}
+
+func getDevNodePropertyUint32(devInst uint32, propKey *DEVPROPKEY) (uint32, error) {
+	var (
+		value    uint32
+		propType uint32
+	)
+
+	propLen := uint32(unsafe.Sizeof(value))
+
+	if err := CMGetDevNodeProperty(devInst, propKey, &propType, unsafe.Pointer(&value), &propLen); err != nil {
+		return 0, err
+	}
+
+	if propType != DEVPROP_TYPE_UINT32 {
+		return 0, fmt.Errorf("unexpected property type: 0x%08X", propType)
+	}
+
+	return value, nil
 }

internal/headers/cfgmgr32/cfgmgr32_test.go

@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package cfgmgr32_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/prometheus-community/windows_exporter/internal/headers/cfgmgr32"
+	"github.com/prometheus-community/windows_exporter/internal/headers/gdi32"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetDevicesInstanceIDs(t *testing.T) {
+	t.Parallel()
+
+	gpus, err := gdi32.GetGPUDevices()
+	require.NoError(t, err)
+
+	var tested bool
+
+	for _, gpu := range gpus {
+		// Only test physical render adapters, which are enumerated on the PCI bus.
+		// Microsoft adapters (VEN_1414), like the Microsoft Basic Render Driver, are virtual
+		// and not always flagged as software device.
+		if gpu.IsSoftwareDevice() ||
+			strings.HasPrefix(gpu.DeviceID, `PCI\VEN_1414&`) ||
+			gpu.AdapterType&gdi32.D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED == 0 ||
+			gpu.AdapterType&gdi32.D3DKMT_ADAPTERTYPE_PARAVIRTUALIZED != 0 {
+			continue
+		}
+
+		devices, err := cfgmgr32.GetDevicesInstanceIDs(gpu.DeviceID)
+		require.NoError(t, err)
+		require.NotEmpty(t, devices)
+
+		var found bool
+
+		for _, device := range devices {
+			require.NotEmpty(t, device.InstanceID)
+
+			if device.BusNumber == gpu.BusNumber && device.DeviceNumber == gpu.DeviceNumber && device.FunctionNumber == gpu.FunctionNumber {
+				found = true
+			}
+		}
+
+		require.True(t, found, "no device instance matches the PCI address of %s", gpu.DeviceID)
+
+		tested = true
+	}
+
+	if !tested {
+		t.Skip("no hardware GPU found")
+	}
+}

internal/headers/cfgmgr32/syscall.go

@@ -62,7 +62,8 @@ func CMGetDeviceIDList(filter *win32.LPWSTR, buf []uint16) error {
 	return nil
 }
 
-func CMLocateDevNode(devInst **windows.Handle, deviceID []uint16) error {
+// CMLocateDevNode stores the DEVINST (a DWORD) of the device node in devInst.
+func CMLocateDevNode(devInst *uint32, deviceID []uint16) error {
 	ret, _, _ := procCMLocateDevNodeW.Call(
 		uintptr(unsafe.Pointer(devInst)),
 		uintptr(unsafe.Pointer(&deviceID[0])),
@@ -76,9 +77,9 @@ func CMLocateDevNode(devInst **windows.Handle, deviceID []uint16) error {
 	return nil
 }
 
-func CMGetDevNodeProperty(devInst *windows.Handle, propKey *DEVPROPKEY, propType *uint32, buf unsafe.Pointer, bufLen *uint32) error {
+func CMGetDevNodeProperty(devInst uint32, propKey *DEVPROPKEY, propType *uint32, buf unsafe.Pointer, bufLen *uint32) error {
 	ret, _, _ := procCMGetDevNodePropertyW.Call(
-		uintptr(unsafe.Pointer(devInst)),
+		uintptr(devInst),
 		uintptr(unsafe.Pointer(propKey)),
 		uintptr(unsafe.Pointer(propType)),
 		uintptr(buf),

internal/headers/gdi32/gdi32.go

@@ -34,6 +34,8 @@ const (
 	KMTQAITYPE_ADAPTERADDRESS = 6
 	// KMTQAITYPE_ADAPTERREGISTRYINFO pPrivateDriverData points to a D3DKMT_ADAPTERREGISTRYINFO structure that contains registry information about the graphics adapter.
 	KMTQAITYPE_ADAPTERREGISTRYINFO = 8
+	// KMTQAITYPE_ADAPTERTYPE pPrivateDriverData points to a D3DKMT_ADAPTERTYPE structure that specifies the type of graphics adapter. Supported starting with Windows 8 (WDDM 1.2).
+	KMTQAITYPE_ADAPTERTYPE = 15
 	// KMTQAITYPE_PHYSICALADAPTERDEVICEIDS pPrivateDriverData points to a D3DKMT_QUERY_DEVICE_IDS structure that specifies the device ID(s) of the physical adapters. Supported starting with Windows 10 (WDDM 2.0).
 	KMTQAITYPE_PHYSICALADAPTERDEVICEIDS = 31
 	// KMTQAITYPE_DRIVERVERSION pPrivateDriverData points to a D3DKMT_DRIVERVERSION value that indicates the WDDM version of the display miniport driver.
@@ -103,6 +105,18 @@ func GetGPUDevice(hAdapter D3DKMT_HANDLE) (GPUDevice, error) {
 
 	gpuDevice.AdapterString = windows.UTF16ToString(info.AdapterString[:])
 
+	var adapterType D3DKMT_ADAPTERTYPE
+
+	query.queryType = KMTQAITYPE_ADAPTERTYPE
+	query.pPrivateDriverData = unsafe.Pointer(&adapterType)
+	query.privateDriverDataSize = uint32(unsafe.Sizeof(adapterType))
+
+	if err := D3DKMTQueryAdapterInfo(&query); err != nil {
+		return gpuDevice, fmt.Errorf("D3DKMTQueryAdapterInfo (adapter type) failed: %w", err)
+	}
+
+	gpuDevice.AdapterType = adapterType
+
 	var deviceIDs D3DKMT_QUERY_DEVICE_IDS
 
 	query.queryType = KMTQAITYPE_PHYSICALADAPTERDEVICEIDS
@@ -118,6 +132,9 @@ func GetGPUDevice(hAdapter D3DKMT_HANDLE) (GPUDevice, error) {
 	return gpuDevice, nil
 }
 
+// GetGPUDevices returns all GPU adapters known to the graphics kernel.
+// If a single adapter can't be queried, the remaining adapters are still
+// returned, together with an error describing the failed adapters.
 func GetGPUDevices() ([]GPUDevice, error) {
 	gpuDevices := make([]GPUDevice, 0, 2)
 
@@ -177,15 +194,11 @@ func GetGPUDevices() ([]GPUDevice, error) {
 		}()
 	}
 
-	if len(errs) > 0 {
-		return gpuDevices, errors.Join(errs...)
-	}
-
 	if len(gpuDevices) == 0 {
-		return gpuDevices, ErrNoGPUDevices
+		errs = append(errs, ErrNoGPUDevices)
 	}
 
-	return gpuDevices, nil
+	return gpuDevices, errors.Join(errs...)
 }
 
 func formatPNPDeviceID(deviceIDs D3DKMT_QUERY_DEVICE_IDS) string {

internal/headers/gdi32/gdi32_test.go

@@ -27,4 +27,11 @@ func TestGetGPUDevices(t *testing.T) {
 	require.NoError(t, err, "Failed to get GPU devices")
 
 	require.NotNil(t, devices)
+
+	// The adapter type flags differ between systems, e.g. the Microsoft Basic Render Driver
+	// is not flagged as software device on Windows Server 2022. Log them for diagnostics.
+	for _, device := range devices {
+		t.Logf("adapter %q: device ID %s, adapter type 0b%b, software device %t",
+			device.AdapterString, device.DeviceID, uint32(device.AdapterType), device.IsSoftwareDevice())
+	}
 }

internal/headers/gdi32/types.go

@@ -95,6 +95,7 @@ type GPUDevice struct {
 	BusNumber                 win32.UINT
 	DeviceNumber              win32.UINT
 	FunctionNumber            win32.UINT
+	AdapterType               D3DKMT_ADAPTERTYPE
 }
 
 // D3DKMT_NODE_PERFDATA is the output of KMTQAITYPE_NODEPERFDATA.
@@ -163,3 +164,30 @@ type D3DKMT_GPUVERSION struct {
 	BiosVersion          [32]uint16
 	GpuArchitecture      [32]uint16
 }
+
+// IsSoftwareDevice reports whether the adapter is a software device,
+// e.g. the Microsoft Basic Render Driver.
+func (d GPUDevice) IsSoftwareDevice() bool {
+	return d.AdapterType&D3DKMT_ADAPTERTYPE_SOFTWARE_DEVICE != 0
+}
+
+// D3DKMT_ADAPTERTYPE is the bitfield returned by KMTQAITYPE_ADAPTERTYPE.
+// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/d3dkmthk/ns-d3dkmthk-_d3dkmt_adaptertype
+type D3DKMT_ADAPTERTYPE uint32
+
+const (
+	D3DKMT_ADAPTERTYPE_RENDER_SUPPORTED         D3DKMT_ADAPTERTYPE = 1 << 0
+	D3DKMT_ADAPTERTYPE_DISPLAY_SUPPORTED        D3DKMT_ADAPTERTYPE = 1 << 1
+	D3DKMT_ADAPTERTYPE_SOFTWARE_DEVICE          D3DKMT_ADAPTERTYPE = 1 << 2
+	D3DKMT_ADAPTERTYPE_POST_DEVICE              D3DKMT_ADAPTERTYPE = 1 << 3
+	D3DKMT_ADAPTERTYPE_HYBRID_DISCRETE          D3DKMT_ADAPTERTYPE = 1 << 4
+	D3DKMT_ADAPTERTYPE_HYBRID_INTEGRATED        D3DKMT_ADAPTERTYPE = 1 << 5
+	D3DKMT_ADAPTERTYPE_INDIRECT_DISPLAY_DEVICE  D3DKMT_ADAPTERTYPE = 1 << 6
+	D3DKMT_ADAPTERTYPE_PARAVIRTUALIZED          D3DKMT_ADAPTERTYPE = 1 << 7
+	D3DKMT_ADAPTERTYPE_ACG_SUPPORTED            D3DKMT_ADAPTERTYPE = 1 << 8
+	D3DKMT_ADAPTERTYPE_SET_TIMINGS_FROM_VIDPN   D3DKMT_ADAPTERTYPE = 1 << 9
+	D3DKMT_ADAPTERTYPE_DETACHABLE               D3DKMT_ADAPTERTYPE = 1 << 10
+	D3DKMT_ADAPTERTYPE_COMPUTE_ONLY             D3DKMT_ADAPTERTYPE = 1 << 11
+	D3DKMT_ADAPTERTYPE_PROTOTYPE                D3DKMT_ADAPTERTYPE = 1 << 12
+	D3DKMT_ADAPTERTYPE_RUNTIME_POWER_MANAGEMENT D3DKMT_ADAPTERTYPE = 1 << 13
+)