// SPDX-License-Identifier: Apache-2.0 // // Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package gpu import ( "errors" "fmt" "log/slog" "time" "github.com/alecthomas/kingpin/v2" "github.com/prometheus-community/windows_exporter/internal/headers/cfgmgr32" "github.com/prometheus-community/windows_exporter/internal/headers/gdi32" "github.com/prometheus-community/windows_exporter/internal/labelcache" "github.com/prometheus-community/windows_exporter/internal/mi" "github.com/prometheus-community/windows_exporter/internal/pdh" "github.com/prometheus-community/windows_exporter/internal/types" "github.com/prometheus/client_golang/prometheus" ) const Name = "gpu" type Config struct{} //nolint:gochecknoglobals var ConfigDefaults = Config{} // deviceCacheRefreshInterval limits how often the GPU device cache is // rebuilt when a performance counter instance references an unknown LUID. const deviceCacheRefreshInterval = 30 * time.Second type Collector struct { config Config logger *slog.Logger // gpuDeviceCache maps the LUID ("0x%08X_0x%08X") of each adapter to its device information. // LUIDs change if a driver is updated or a device is restarted, so the cache is // refreshed whenever a performance counter instance references an unknown LUID. gpuDeviceCache map[string]gpuDevice gpuDeviceCacheLastRefresh time.Time // discoverGPUDevices enumerates the GPU adapters. If nil, gdi32.GetGPUDevices is used. // It is replaced in tests to make the discovery deterministic. discoverGPUDevices func() ([]gdi32.GPUDevice, error) // labels keeps the label pairs of the series of the last scrape, and // instances the parsed counter instance names, so neither is built again // while the GPU process or engine stays. labels labelcache.Cache instances instanceCache // GPU Engine gpuEnginePerfDataCollector *pdh.Collector[gpuEnginePerfDataCounterValues] gpuEnginePerfDataObject []gpuEnginePerfDataCounterValues gpuInfo *prometheus.Desc gpuEngineRunningTime *prometheus.Desc gpuSharedSystemMemorySize *prometheus.Desc gpuDedicatedSystemMemorySize *prometheus.Desc gpuDedicatedVideoMemorySize *prometheus.Desc // GPU Adapter Memory gpuAdapterMemoryPerfDataCollector *pdh.Collector[gpuAdapterMemoryPerfDataCounterValues] gpuAdapterMemoryPerfDataObject []gpuAdapterMemoryPerfDataCounterValues gpuAdapterMemoryDedicatedUsage *prometheus.Desc gpuAdapterMemorySharedUsage *prometheus.Desc gpuAdapterMemoryTotalCommitted *prometheus.Desc // GPU Local Adapter Memory gpuLocalAdapterMemoryPerfDataCollector *pdh.Collector[gpuLocalAdapterMemoryPerfDataCounterValues] gpuLocalAdapterMemoryPerfDataObject []gpuLocalAdapterMemoryPerfDataCounterValues gpuLocalAdapterMemoryUsage *prometheus.Desc // GPU Non Local Adapter Memory gpuNonLocalAdapterMemoryPerfDataCollector *pdh.Collector[gpuNonLocalAdapterMemoryPerfDataCounterValues] gpuNonLocalAdapterMemoryPerfDataObject []gpuNonLocalAdapterMemoryPerfDataCounterValues gpuNonLocalAdapterMemoryUsage *prometheus.Desc // GPU Process Memory gpuProcessMemoryPerfDataCollector *pdh.Collector[gpuProcessMemoryPerfDataCounterValues] gpuProcessMemoryPerfDataObject []gpuProcessMemoryPerfDataCounterValues gpuProcessMemoryDedicatedUsage *prometheus.Desc gpuProcessMemoryLocalUsage *prometheus.Desc gpuProcessMemoryNonLocalUsage *prometheus.Desc gpuProcessMemorySharedUsage *prometheus.Desc gpuProcessMemoryTotalCommitted *prometheus.Desc // GPU sensors sensorMetrics sensorMetrics } type gpuDevice struct { gdi32 gdi32.GPUDevice cfgmgr32 cfgmgr32.Device sensors gpuSensors ID string // skip marks adapters that are known, but not exposed, e.g. software devices // like the Microsoft Basic Render Driver. Keeping them in the cache prevents // their performance counter instances from triggering a cache refresh. skip bool } func New(config *Config) *Collector { if config == nil { config = &ConfigDefaults } c := &Collector{ config: *config, } return c } func NewWithFlags(_ *kingpin.Application) *Collector { return &Collector{} } func (c *Collector) GetName() string { return Name } func (c *Collector) Close() error { c.gpuEnginePerfDataCollector.Close() c.gpuAdapterMemoryPerfDataCollector.Close() c.gpuLocalAdapterMemoryPerfDataCollector.Close() c.gpuNonLocalAdapterMemoryPerfDataCollector.Close() c.gpuProcessMemoryPerfDataCollector.Close() c.labels.Reset() c.instances.reset() return nil } func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error { var err error c.logger = logger.With(slog.String("collector", Name)) c.gpuInfo = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "info"), "A metric with a constant '1' value labeled with GPU device information.", []string{"luid", "device_id", "name", "bus_number", "device_number", "function_number", "driver_version", "wddm_version", "architecture"}, nil, ) c.gpuSharedSystemMemorySize = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "shared_system_memory_size_bytes"), "The size, in bytes, of memory from system memory that can be shared by many users.", []string{"luid", "device_id"}, nil, ) c.gpuDedicatedSystemMemorySize = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "dedicated_system_memory_size_bytes"), "The size, in bytes, of memory that is dedicated from system memory.", []string{"luid", "device_id"}, nil, ) c.gpuDedicatedVideoMemorySize = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "dedicated_video_memory_size_bytes"), "The size, in bytes, of memory that is dedicated from video memory.", []string{"luid", "device_id"}, nil, ) c.gpuEngineRunningTime = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "engine_time_seconds"), "Total running time of the GPU engine in seconds.", []string{"process_id", "luid", "device_id", "phys", "eng", "engtype"}, nil, ) c.gpuAdapterMemoryDedicatedUsage = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "adapter_memory_dedicated_bytes"), "Dedicated GPU memory usage in bytes.", []string{"luid", "device_id", "phys"}, nil, ) c.gpuAdapterMemorySharedUsage = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "adapter_memory_shared_bytes"), "Shared GPU memory usage in bytes.", []string{"luid", "device_id", "phys"}, nil, ) c.gpuAdapterMemoryTotalCommitted = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "adapter_memory_committed_bytes"), "Total committed GPU memory in bytes.", []string{"luid", "device_id", "phys"}, nil, ) c.gpuLocalAdapterMemoryUsage = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "local_adapter_memory_bytes"), "Local adapter memory usage in bytes.", []string{"luid", "device_id", "phys", "part"}, nil, ) c.gpuNonLocalAdapterMemoryUsage = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "non_local_adapter_memory_bytes"), "Non-local adapter memory usage in bytes.", []string{"luid", "device_id", "phys", "part"}, nil, ) c.gpuProcessMemoryDedicatedUsage = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "process_memory_dedicated_bytes"), "Dedicated process memory usage in bytes.", []string{"process_id", "luid", "device_id", "phys"}, nil, ) c.gpuProcessMemoryLocalUsage = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "process_memory_local_bytes"), "Local process memory usage in bytes.", []string{"process_id", "luid", "device_id", "phys"}, nil, ) c.gpuProcessMemoryNonLocalUsage = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "process_memory_non_local_bytes"), "Non-local process memory usage in bytes.", []string{"process_id", "luid", "device_id", "phys"}, nil, ) c.gpuProcessMemorySharedUsage = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "process_memory_shared_bytes"), "Shared process memory usage in bytes.", []string{"process_id", "luid", "device_id", "phys"}, nil, ) c.gpuProcessMemoryTotalCommitted = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "process_memory_committed_bytes"), "Total committed process memory in bytes.", []string{"process_id", "luid", "device_id", "phys"}, nil, ) c.sensorMetrics = newSensorMetrics() errs := make([]error, 0) c.gpuEnginePerfDataCollector, err = pdh.NewCollector[gpuEnginePerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Engine", pdh.InstancesAll) if err != nil { errs = append(errs, fmt.Errorf("failed to create GPU Engine perf data collector: %w", err)) } c.gpuAdapterMemoryPerfDataCollector, err = pdh.NewCollector[gpuAdapterMemoryPerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Adapter Memory", pdh.InstancesAll) if err != nil { errs = append(errs, fmt.Errorf("failed to create GPU Adapter Memory perf data collector: %w", err)) } c.gpuLocalAdapterMemoryPerfDataCollector, err = pdh.NewCollector[gpuLocalAdapterMemoryPerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Local Adapter Memory", pdh.InstancesAll) if err != nil { errs = append(errs, fmt.Errorf("failed to create GPU Local Adapter Memory perf data collector: %w", err)) } c.gpuNonLocalAdapterMemoryPerfDataCollector, err = pdh.NewCollector[gpuNonLocalAdapterMemoryPerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Non Local Adapter Memory", pdh.InstancesAll) if err != nil { errs = append(errs, fmt.Errorf("failed to create GPU Non Local Adapter Memory perf data collector: %w", err)) } c.gpuProcessMemoryPerfDataCollector, err = pdh.NewCollector[gpuProcessMemoryPerfDataCounterValues](logger.With(slog.String("collector", Name)), pdh.CounterTypeRaw, "GPU Process Memory", pdh.InstancesAll) if err != nil { errs = append(errs, fmt.Errorf("failed to create GPU Process Memory perf data collector: %w", err)) } // GPU discovery problems are not fatal. The collector keeps working with the // adapters that could be discovered and retries on unknown LUIDs during Collect. c.refreshGPUDevices() return errors.Join(errs...) } // refreshGPUDevices rebuilds the GPU device cache. // Problems with single adapters are logged and do not stop the discovery of the remaining adapters. func (c *Collector) refreshGPUDevices() { c.gpuDeviceCacheLastRefresh = time.Now() discoverGPUDevices := c.discoverGPUDevices if discoverGPUDevices == nil { discoverGPUDevices = gdi32.GetGPUDevices } gpus, err := discoverGPUDevices() if err != nil { c.logger.Warn("failed to discover some GPU devices", slog.Any("err", err), ) // Keep the previous cache, if the discovery failed completely. if len(gpus) == 0 { return } } gpuDeviceCache := make(map[string]gpuDevice, len(gpus)) for _, gpu := range gpus { luidKey := fmt.Sprintf("0x%08X_0x%08X", gpu.LUID.HighPart, gpu.LUID.LowPart) // Skip software devices like the Microsoft Basic Render Driver. if gpu.AdapterString == "" || gpu.IsSoftwareDevice() || isBasicRenderDriver(gpu) { gpuDeviceCache[luidKey] = gpuDevice{gdi32: gpu, ID: gpu.DeviceID, skip: true} c.logger.Debug("Skipping GPU device", slog.String("name", gpu.AdapterString), slog.String("luid", luidKey), slog.String("device_id", gpu.DeviceID), slog.Uint64("adapter_type", uint64(gpu.AdapterType)), ) continue } device := c.newGPUDevice(gpu, luidKey) device.sensors = discoverSensors(c.logger.With(slog.String("luid", luidKey)), gpu) gpuDeviceCache[luidKey] = device c.logger.Debug("Found GPU device", slog.String("name", gpu.AdapterString), slog.String("luid", luidKey), slog.String("device_id", device.ID), slog.Uint64("bus_number", uint64(gpu.BusNumber)), slog.Uint64("device_number", uint64(gpu.DeviceNumber)), slog.Uint64("function_number", uint64(gpu.FunctionNumber)), ) } c.gpuDeviceCache = gpuDeviceCache } // newGPUDevice resolves the PnP device instance ID of the adapter. // If the device instance can't be found, the PnP device ID reported by gdi32 is used as ID. func (c *Collector) newGPUDevice(gpu gdi32.GPUDevice, luidKey string) gpuDevice { device := gpuDevice{ gdi32: gpu, ID: gpu.DeviceID, } cfgmgr32Devs, err := cfgmgr32.GetDevicesInstanceIDs(gpu.DeviceID) for _, dev := range cfgmgr32Devs { if dev.BusNumber == gpu.BusNumber && dev.DeviceNumber == gpu.DeviceNumber && dev.FunctionNumber == gpu.FunctionNumber { device.cfgmgr32 = dev device.ID = dev.InstanceID return device } } attrs := []any{ slog.String("name", gpu.AdapterString), slog.String("luid", luidKey), slog.String("device_id", gpu.DeviceID), } if err != nil { attrs = append(attrs, slog.Any("err", err)) } c.logger.Warn("failed to find matching PnP device instance for GPU device, using the PnP device ID as device_id", attrs...) return device } // getGPUDevice returns the cached device for the given LUID. // If the LUID is unknown, e.g. because a driver was updated, a device was restarted // or a device was hot-plugged, the cache is refreshed at most once per deviceCacheRefreshInterval. // Devices marked as skip are not returned. func (c *Collector) getGPUDevice(luid string) (gpuDevice, bool) { if luid == "" { return gpuDevice{}, false } device, ok := c.gpuDeviceCache[luid] if !ok && time.Since(c.gpuDeviceCacheLastRefresh) >= deviceCacheRefreshInterval { c.logger.Debug("unknown GPU LUID, refreshing GPU device cache", slog.String("luid", luid), ) c.refreshGPUDevices() device, ok = c.gpuDeviceCache[luid] } return device, ok && !device.skip } func (c *Collector) Collect(ch chan<- prometheus.Metric, _ time.Duration) error { errs := make([]error, 0) scrape := c.labels.Begin() defer scrape.Commit() instances := c.instances.begin() defer instances.commit() if err := c.collectGpuEngineMetrics(ch, scrape, instances); err != nil { errs = append(errs, err) } if err := c.collectGpuAdapterMemoryMetrics(ch, scrape, instances); err != nil { errs = append(errs, err) } if err := c.collectGpuLocalAdapterMemoryMetrics(ch, scrape, instances); err != nil { errs = append(errs, err) } if err := c.collectGpuNonLocalAdapterMemoryMetrics(ch, scrape, instances); err != nil { errs = append(errs, err) } if err := c.collectGpuProcessMemoryMetrics(ch, scrape, instances); err != nil { errs = append(errs, err) } // Collected last, since the metrics above may refresh the GPU device cache. c.collectGpuInfo(ch) c.collectGpuSensorMetrics(ch) return errors.Join(errs...) } func (c *Collector) collectGpuInfo(ch chan<- prometheus.Metric) { for luid, gpu := range c.gpuDeviceCache { if gpu.skip { continue } ch <- prometheus.MustNewConstMetric( c.gpuInfo, prometheus.GaugeValue, 1.0, luid, gpu.ID, gpu.gdi32.AdapterString, gpu.gdi32.BusNumber.String(), gpu.gdi32.DeviceNumber.String(), gpu.gdi32.FunctionNumber.String(), gpu.sensors.driverVersion, gpu.sensors.wddmVersion, gpu.sensors.architecture, ) ch <- prometheus.MustNewConstMetric( c.gpuSharedSystemMemorySize, prometheus.GaugeValue, float64(gpu.gdi32.SharedSystemMemorySize), luid, gpu.ID, ) ch <- prometheus.MustNewConstMetric( c.gpuDedicatedSystemMemorySize, prometheus.GaugeValue, float64(gpu.gdi32.DedicatedSystemMemorySize), luid, gpu.ID, ) ch <- prometheus.MustNewConstMetric( c.gpuDedicatedVideoMemorySize, prometheus.GaugeValue, float64(gpu.gdi32.DedicatedVideoMemorySize), luid, gpu.ID, ) } } func (c *Collector) collectGpuEngineMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error { // Collect the GPU Engine perf data. if err := c.gpuEnginePerfDataCollector.Collect(&c.gpuEnginePerfDataObject); err != nil { return fmt.Errorf("failed to collect GPU Engine perf data: %w", err) } // Iterate over the GPU Engine perf data and aggregate the values. for _, data := range c.gpuEnginePerfDataObject { instance := instances.parse(data.Name) device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } ch <- scrape.MustNewConstMetric( c.gpuEngineRunningTime, prometheus.CounterValue, data.RunningTime/10_000_000, instance.Pid, instance.Luid, device.ID, instance.Phys, instance.Eng, instance.Engtype, ) } return nil } func (c *Collector) collectGpuAdapterMemoryMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error { // Collect the GPU Adapter Memory perf data. if err := c.gpuAdapterMemoryPerfDataCollector.Collect(&c.gpuAdapterMemoryPerfDataObject); err != nil { return fmt.Errorf("failed to collect GPU Adapter Memory perf data: %w", err) } for _, data := range c.gpuAdapterMemoryPerfDataObject { instance := instances.parse(data.Name) device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } ch <- scrape.MustNewConstMetric( c.gpuAdapterMemoryDedicatedUsage, prometheus.GaugeValue, data.DedicatedUsage, instance.Luid, device.ID, instance.Phys, ) ch <- scrape.MustNewConstMetric( c.gpuAdapterMemorySharedUsage, prometheus.GaugeValue, data.SharedUsage, instance.Luid, device.ID, instance.Phys, ) ch <- scrape.MustNewConstMetric( c.gpuAdapterMemoryTotalCommitted, prometheus.GaugeValue, data.TotalCommitted, instance.Luid, device.ID, instance.Phys, ) } return nil } func (c *Collector) collectGpuLocalAdapterMemoryMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error { // Collect the GPU Local Adapter Memory perf data. if err := c.gpuLocalAdapterMemoryPerfDataCollector.Collect(&c.gpuLocalAdapterMemoryPerfDataObject); err != nil { return fmt.Errorf("failed to collect GPU Local Adapter Memory perf data: %w", err) } for _, data := range c.gpuLocalAdapterMemoryPerfDataObject { instance := instances.parse(data.Name) device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } ch <- scrape.MustNewConstMetric( c.gpuLocalAdapterMemoryUsage, prometheus.GaugeValue, data.LocalUsage, instance.Luid, device.ID, instance.Phys, instance.Part, ) } return nil } func (c *Collector) collectGpuNonLocalAdapterMemoryMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error { // Collect the GPU Non Local Adapter Memory perf data. if err := c.gpuNonLocalAdapterMemoryPerfDataCollector.Collect(&c.gpuNonLocalAdapterMemoryPerfDataObject); err != nil { return fmt.Errorf("failed to collect GPU Non Local Adapter Memory perf data: %w", err) } for _, data := range c.gpuNonLocalAdapterMemoryPerfDataObject { instance := instances.parse(data.Name) device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } ch <- scrape.MustNewConstMetric( c.gpuNonLocalAdapterMemoryUsage, prometheus.GaugeValue, data.NonLocalUsage, instance.Luid, device.ID, instance.Phys, instance.Part, ) } return nil } func (c *Collector) collectGpuProcessMemoryMetrics(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, instances *instanceScrape) error { // Collect the GPU Process Memory perf data. if err := c.gpuProcessMemoryPerfDataCollector.Collect(&c.gpuProcessMemoryPerfDataObject); err != nil { return fmt.Errorf("failed to collect GPU Process Memory perf data: %w", err) } for _, data := range c.gpuProcessMemoryPerfDataObject { instance := instances.parse(data.Name) device, ok := c.getGPUDevice(instance.Luid) if !ok { continue } ch <- scrape.MustNewConstMetric( c.gpuProcessMemoryDedicatedUsage, prometheus.GaugeValue, data.DedicatedUsage, instance.Pid, instance.Luid, device.ID, instance.Phys, ) ch <- scrape.MustNewConstMetric( c.gpuProcessMemoryLocalUsage, prometheus.GaugeValue, data.LocalUsage, instance.Pid, instance.Luid, device.ID, instance.Phys, ) ch <- scrape.MustNewConstMetric( c.gpuProcessMemoryNonLocalUsage, prometheus.GaugeValue, data.NonLocalUsage, instance.Pid, instance.Luid, device.ID, instance.Phys, ) ch <- scrape.MustNewConstMetric( c.gpuProcessMemorySharedUsage, prometheus.GaugeValue, data.SharedUsage, instance.Pid, instance.Luid, device.ID, instance.Phys, ) ch <- scrape.MustNewConstMetric( c.gpuProcessMemoryTotalCommitted, prometheus.GaugeValue, data.TotalCommitted, instance.Pid, instance.Luid, device.ID, instance.Phys, ) } return nil }