hyperv: add WMI health metric (#2494)

Co-authored-by: EisenbergD <dominik.eisenberg@beiersdorf.com>
Co-authored-by: Jan-Otto Kröpke <mail@jkroepke.de>
Signed-off-by: EisenbergD <dominik.eisenberg@beiersdorf.com>
Signed-off-by: Dominik Eisenberg <64131471+Dominik-esb@users.noreply.github.com>

docs/collector.hyperv.md

@@ -12,8 +12,7 @@ The hyperv collector exposes metrics about the Hyper-V hypervisor
 
 ### `--collectors.hyperv.enabled`
 Comma-separated list of collectors to use, for example:
-`--collectors.hyperv.enabled=dynamic_memory_balancer,dynamic_memory_vm,host,hypervisor_logical_processor,hypervisor_root_partition,hypervisor_root_virtual_processor,hypervisor_virtual_processor,legacy_network_adapter,virtual_machine_health_summary,virtual_machine_vid_partition,virtual_network_adapter,virtual_storage_device,virtual_switch`.
-Matching is case-sensitive.
+`--collectors.hyperv.enabled=dynamic_memory_balancer,dynamic_memory_vm,host,hypervisor_logical_processor,hypervisor_root_partition,hypervisor_root_virtual_processor,hypervisor_virtual_processor,legacy_network_adapter,virtual_machine_health_summary,virtual_machine_vid_partition,virtual_network_adapter,virtual_storage_device,virtual_switch,wmi_health`
 
 ## Metrics
 
@@ -274,6 +273,33 @@ Multiply by the number of threads per core to get the ratio per **physical** cor
 | `windows_hyperv_vid_remote_physical_pages`     | The number of physical pages not allocated from the preferred NUMA node | gauge | `vm`   |
 
 
+### Hyper-V WMI Health
+
+Sub-collector `wmi_health`, enabled by default. It runs a minimal WQL query
+(`SELECT Name FROM Msvm_ComputerSystem WHERE Caption = 'Hosting Computer System'`) against the
+`root/virtualization/v2` namespace on every scrape, so remove it from `--collectors.hyperv.enabled` if you don't need it.
+
+| Name                       | Description                                                                                                  | Type  | Labels |
+|----------------------------|--------------------------------------------------------------------------------------------------------------|-------|--------|
+| `windows_hyperv_wmi_health` | Hyper-V WMI health status. 1 if the Hyper-V WMI namespace is responding, 0 if it is broken or not responding | gauge | None   |
+
+WMI can stop responding while the host and the exporter otherwise keep running. On a Hyper-V
+cluster node, Failover Cluster Manager generally cannot connect to the host in that state, and
+recovery often requires a reboot. The query times out after 5 seconds and is then reported as `0`.
+
+Example alert:
+
+```yaml
+- alert: HyperVWMIUnhealthy
+  expr: windows_hyperv_wmi_health == 0
+  for: 5m
+  labels:
+    severity: critical
+  annotations:
+    summary: Hyper-V WMI is not responding on {{ $labels.instance }}
+```
+
+
 ### Hyper-V Virtual Machine Health Summary
 
 | Name                                                 | Description                                           | Type  | Labels |

internal/collector/hyperv/hyperv.go

@@ -51,6 +51,7 @@ const (
 	subCollectorVirtualSMB                       = "virtual_smb"
 	subCollectorVirtualStorageDevice             = "virtual_storage_device"
 	subCollectorVirtualSwitch                    = "virtual_switch"
+	subCollectorWMIHealth                        = "wmi_health"
 )
 
 type Config struct {
@@ -76,6 +77,7 @@ var ConfigDefaults = Config{
 		subCollectorVirtualSMB,
 		subCollectorVirtualStorageDevice,
 		subCollectorVirtualSwitch,
+		subCollectorWMIHealth,
 	},
 }
 
@@ -97,10 +99,13 @@ type Collector struct {
 	collectorVirtualSMB
 	collectorVirtualStorageDevice
 	collectorVirtualSwitch
+	collectorWMIHealth
 
 	config Config
 	logger *slog.Logger
 
+	miSession *mi.Session
+
 	collectorFns []func(ch chan<- prometheus.Metric) error
 	closeFns     []func()
 }
@@ -155,7 +160,8 @@ func (c *Collector) Close() error {
 	return nil
 }
 
-func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error {
+func (c *Collector) Build(logger *slog.Logger, miSession *mi.Session) error {
+	c.miSession = miSession
 	c.logger = logger.With(slog.String("collector", Name))
 	c.collectorFns = make([]func(ch chan<- prometheus.Metric) error, 0, len(c.config.CollectorsEnabled))
 	c.closeFns = make([]func(), 0, len(c.config.CollectorsEnabled))
@@ -252,6 +258,11 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error {
 			collect: c.collectVirtualSwitch,
 			close:   c.perfDataCollectorVirtualSwitch.Close,
 		},
+		subCollectorWMIHealth: {
+			build:   c.buildWMIHealth,
+			collect: c.collectWMIHealth,
+			close:   func() {},
+		},
 	}
 
 	buildNumber := osversion.Build()

internal/collector/hyperv/hyperv_wmi_health.go

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package hyperv
+
+import (
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/prometheus-community/windows_exporter/internal/mi"
+	"github.com/prometheus-community/windows_exporter/internal/types"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// wmiHealthQueryTimeout bounds the WMI health query, so a hung WMI service
+// is reported as unhealthy instead of blocking the scrape.
+const wmiHealthQueryTimeout = 5 * time.Second
+
+// collectorWMIHealth Hyper-V WMI health metrics
+type collectorWMIHealth struct {
+	miQuery mi.Query
+
+	wmiHealth *prometheus.Desc
+}
+
+// msvmComputerSystem is the Msvm_ComputerSystem WMI class.
+// Only Name is needed to verify the Hyper-V WMI provider is working.
+type msvmComputerSystem struct {
+	Name string `mi:"Name"`
+}
+
+func (c *Collector) buildWMIHealth() error {
+	if c.miSession == nil {
+		return mi.ErrNotInitialized
+	}
+
+	miQuery, err := mi.NewQuery("SELECT Name FROM Msvm_ComputerSystem WHERE Caption = 'Hosting Computer System'")
+	if err != nil {
+		return fmt.Errorf("failed to create WMI query: %w", err)
+	}
+
+	c.miQuery = miQuery
+
+	c.wmiHealth = prometheus.NewDesc(
+		prometheus.BuildFQName(types.Namespace, Name, "wmi_health"),
+		"Hyper-V WMI health status. 1 if the Hyper-V WMI namespace is responding, 0 if it is broken or not responding",
+		nil,
+		nil,
+	)
+
+	return nil
+}
+
+// collectWMIHealth reports 1 if a minimal query against the Hyper-V WMI
+// namespace succeeds and 0 otherwise.
+func (c *Collector) collectWMIHealth(ch chan<- prometheus.Metric) error {
+	var dst []msvmComputerSystem
+
+	healthValue := 1.0
+
+	if err := c.miSession.Query(&dst, mi.NamespaceRootVirtualizationV2, c.miQuery, wmiHealthQueryTimeout); err != nil {
+		c.logger.Debug("Hyper-V WMI health query failed", slog.Any("err", err))
+
+		healthValue = 0.0
+	}
+
+	ch <- prometheus.MustNewConstMetric(
+		c.wmiHealth,
+		prometheus.GaugeValue,
+		healthValue,
+	)
+
+	return nil
+}

internal/mi/types.go

@@ -55,6 +55,7 @@ var (
 	NamespaceRootMSCluster         = utils.Must(NewNamespace("root/MSCluster"))
 	NamespaceRootMicrosoftDNS      = utils.Must(NewNamespace("root/MicrosoftDNS"))
 	NamespaceRootStorage           = utils.Must(NewNamespace("root/Microsoft/Windows/Storage"))
+	NamespaceRootVirtualizationV2  = utils.Must(NewNamespace("root/virtualization/v2"))
 )
 
 type Query *uint16