hyperv: add the vm_state sub-collector for VMs that are not running (#2765)

Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>

docs/collector.hyperv.md

@@ -12,7 +12,7 @@ The hyperv collector exposes metrics about the Hyper-V hypervisor
 
 ### `--collector.hyperv.enabled`
 Comma-separated list of collectors to use, for example:
-`--collector.hyperv.enabled=datastore,dynamic_memory_balancer,dynamic_memory_vm,host,hypervisor_logical_processor,hypervisor_root_partition,hypervisor_root_virtual_processor,hypervisor_virtual_processor,legacy_network_adapter,replica_vm,virtual_hard_disk,virtual_machine_health_summary,virtual_machine_vid_partition,virtual_network_adapter,virtual_network_adapter_drop_reasons,virtual_smb,virtual_storage_device,virtual_switch,wmi_health`
+`--collector.hyperv.enabled=datastore,dynamic_memory_balancer,dynamic_memory_vm,host,hypervisor_logical_processor,hypervisor_root_partition,hypervisor_root_virtual_processor,hypervisor_virtual_processor,legacy_network_adapter,replica_vm,virtual_hard_disk,virtual_machine_health_summary,virtual_machine_vid_partition,virtual_network_adapter,virtual_network_adapter_drop_reasons,virtual_smb,virtual_storage_device,virtual_switch,vm_state,wmi_health`
 
 ### `--collector.hyperv.virtual-hard-disk.refresh-interval`
 Interval at which the `virtual_hard_disk` sub-collector reads the disks in the background, for example `10m`.
@@ -390,6 +390,54 @@ rate(windows_hyperv_virtual_storage_device_io_latency_seconds_total[5m])
 | `windows_hyperv_vid_remote_physical_pages`     | The number of physical pages not allocated from the preferred NUMA node | gauge | `vm`   |
 
 
+### Hyper-V VM State
+
+Sub-collector `vm_state`, enabled by default. The performance counter objects only have instances for running
+virtual machines, so a VM that is off, saved or paused is missing from all other metrics. This sub-collector lists
+every virtual machine registered on the host with one WQL query per scrape
+(`SELECT Name, ElementName, EnabledState FROM Msvm_ComputerSystem`) against the `root/virtualization/v2` namespace.
+The query times out after 5 seconds; a failed or timed-out query fails the `hyperv` collector
+(`windows_exporter_collector_success{collector="hyperv"}` is `0`), like the `replica_vm` sub-collector.
+On a failover cluster, each node reports the virtual machines it currently owns.
+
+| Name                      | Description                                                           | Type  | Labels                 |
+|---------------------------|-----------------------------------------------------------------------|-------|------------------------|
+| `windows_hyperv_vm_state` | The state of the virtual machine, 1 if the current state, 0 otherwise | gauge | `vm`, `vm_id`, `state` |
+
+Every virtual machine has one series per `state`: `running`, `off`, `starting`, `stopping`, `saving`, `saved`,
+`pausing`, `paused`, `resuming`, `reset` and `unknown`. The value is 1 for the current state and 0 for the others.
+`state` is read from `Msvm_ComputerSystem.EnabledState`. A fast-saved VM is reported as `saved`, a VM in a
+critical state (for example `RunningCritical`) as its base state, and a value without a mapping as `unknown`.
+
+Replica VMs on a Hyper-V Replica server are turned off and show as `state="off"`; use the
+`windows_hyperv_replica_vm_*` metrics to tell them apart.
+
+`vm` is the name of the virtual machine and `vm_id` its GUID, the same keys as the `windows_hyperv_replica_vm_*`
+and `windows_hyperv_virtual_hard_disk_*` metrics. VM names don't have to be unique, `vm_id` keeps the series apart.
+
+Virtual machines that are turned off:
+
+```promql
+windows_hyperv_vm_state{state="off"} == 1
+```
+
+Number of virtual machines per state and host:
+
+```promql
+sum by (instance, state) (windows_hyperv_vm_state)
+```
+
+Example alert for a virtual machine that is expected to run:
+
+```yaml
+- alert: HyperVVMNotRunning
+  expr: windows_hyperv_vm_state{vm="web01", state="running"} == 0
+  for: 10m
+  annotations:
+    summary: Hyper-V VM {{ $labels.vm }} is not running on {{ $labels.instance }}
+```
+
+
 ### Hyper-V WMI Health
 
 Sub-collector `wmi_health`, enabled by default. It runs a minimal WQL query

docs/upgrade/0.32.md

@@ -312,10 +312,10 @@ See [the logical_disk documentation](../collector.logical_disk.md#bitlocker-stat
 
 ### New default sub-collectors
 
-| Collector   | New sub-collectors enabled by default                   |
-|-------------|---------------------------------------------------------|
-| `hyperv`    | `host`, `replica_vm`, `virtual_hard_disk`, `wmi_health` |
-| `mscluster` | `shared_volumes`                                        |
+| Collector   | New sub-collectors enabled by default                               |
+|-------------|---------------------------------------------------------------------|
+| `hyperv`    | `host`, `replica_vm`, `virtual_hard_disk`, `vm_state`, `wmi_health` |
+| `mscluster` | `shared_volumes`                                                    |
 
 These add new metrics. To keep the 0.31 set, list the sub-collectors explicitly with `--collector.hyperv.enabled` or `--collector.mscluster.enabled`.
 

internal/collector/hyperv/AGENTS.md

@@ -88,8 +88,30 @@ memory) on a host with 12 logical processors.
   "the current number of Tree Connects".
 - `Hyper-V Virtual Machine Health Summary` counts VMs: `Health Ok` was 1 for
   the one VM.
-- The `wmi_health`, `virtual_hard_disk` and host VM processor metrics do not
-  come from PDH, so the table below does not cover them.
+- The `wmi_health`, `virtual_hard_disk`, `vm_state` and host VM processor
+  metrics do not come from PDH, so the table below does not cover them.
+- PDH instances exist only for running VMs. To list every VM whatever its
+  state, query `Msvm_ComputerSystem` (as `vm_state` and `virtual_hard_disk`
+  do). The host itself is a row of that class too: its `Name` is the host's
+  NetBIOS name, while a VM's `Name` is its GUID (the `vm_id` label) and
+  `ElementName` its display name (the `vm` label). `Caption` and
+  `Description` also tell them apart but are localized, so do not filter on
+  them; `vm_state` keeps the rows whose `Name` is a GUID.
+- `vm_state` maps `EnabledState` to a state set. Learn documents only the
+  CIM values 0 to 10 for `EnabledState` (2 running, 3 off, 4 stopping,
+  6 saved, 9 paused, 10 starting). 11 (Reset) is a `RequestedState` value,
+  mapped defensively. 32768 (Paused), 32769 (Suspended), 32770 (Starting)
+  and 32774 (Stopping) are version 1 `EnabledState` values that the
+  version 2 provider still reports in some cases (32768 for a VM paused on
+  low disk space). 32773 saving, 32776 pausing, 32777 resuming, 32779 fast
+  saved and 32780 fast saving are the Hyper-V states of
+  `RequestStateChange`; 32781 to 32792 are their critical variants
+  (RunningCritical, OffCritical, StoppingCritical, SavedCritical,
+  PausedCritical, StartingCritical, ResetCritical, SavingCritical,
+  PausingCritical, ResumingCritical, FastSavedCritical, FastSavingCritical)
+  and map to the base state. Other values map to `unknown`. The CI workflow
+  creates a running VM (`GitHubActions`) and one that is never started
+  (`CIVirtualHardDisk`).
 - To test emitted metrics without Hyper-V, build the descriptors with the
   `build...Descriptors` helpers, fill the `perfDataObject...` slice (or a
   `pdhtypes.Collector` fixture) and gather through a registry, as in

internal/collector/hyperv/hyperv.go

@@ -56,6 +56,7 @@ const (
 	subCollectorVirtualSMB                       = "virtual_smb"
 	subCollectorVirtualStorageDevice             = "virtual_storage_device"
 	subCollectorVirtualSwitch                    = "virtual_switch"
+	subCollectorVMState                          = "vm_state"
 	subCollectorWMIHealth                        = "wmi_health"
 )
 
@@ -91,6 +92,7 @@ var ConfigDefaults = Config{
 		subCollectorVirtualSMB,
 		subCollectorVirtualStorageDevice,
 		subCollectorVirtualSwitch,
+		subCollectorVMState,
 		subCollectorWMIHealth,
 	},
 	VirtualHardDisk: VirtualHardDiskConfig{
@@ -118,6 +120,7 @@ type Collector struct {
 	collectorVirtualSMB
 	collectorVirtualStorageDevice
 	collectorVirtualSwitch
+	collectorVMState
 	collectorWMIHealth
 
 	config Config
@@ -315,6 +318,11 @@ func (c *Collector) Build(logger *slog.Logger, miSession *mi.Session) error {
 			collect: c.collectVirtualSwitch,
 			close:   func() { c.perfDataCollectorVirtualSwitch.Close() },
 		},
+		subCollectorVMState: {
+			build:   c.buildVMState,
+			collect: c.collectVMState,
+			close:   func() {},
+		},
 		subCollectorWMIHealth: {
 			build:   c.buildWMIHealth,
 			collect: c.collectWMIHealth,

internal/collector/hyperv/hyperv_test.go

@@ -52,6 +52,16 @@ func TestCollector(t *testing.T) {
 		require.Regexp(t, `^[0-9A-Fa-f-]{36}$`, labelValue(metric, "vm_id"))
 	}
 
+	// The PDH objects only list running VMs; vm_state also lists the CIVirtualHardDisk VM, which is never started.
+	if metric := testutils.RequireFixtureMetric(t, metrics, hyperv.Name, "windows_hyperv_vm_state", prometheus.Labels{"vm": "GitHubActions", "state": "running"}); metric != nil {
+		require.InDelta(t, 1, metric.GetGauge().GetValue(), 0, "expected the GitHubActions VM to be running")
+		require.Regexp(t, `^[0-9A-Fa-f-]{36}$`, labelValue(metric, "vm_id"))
+	}
+
+	if metric := testutils.RequireFixtureMetric(t, metrics, hyperv.Name, "windows_hyperv_vm_state", prometheus.Labels{"vm": "CIVirtualHardDisk", "state": "off"}); metric != nil {
+		require.InDelta(t, 1, metric.GetGauge().GetValue(), 0, "expected the CIVirtualHardDisk VM to be off")
+	}
+
 	// The health query reports failures as 0 instead of an error, so check the value.
 	if metric := testutils.RequireFixtureMetric(t, metrics, hyperv.Name, "windows_hyperv_wmi_health", nil); metric != nil {
 		require.InDelta(t, 1, metric.GetGauge().GetValue(), 0, "Hyper-V WMI health query failed")

internal/collector/hyperv/hyperv_vm_state.go

@@ -0,0 +1,209 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package hyperv
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/prometheus-community/windows_exporter/internal/labelcache"
+	"github.com/prometheus-community/windows_exporter/internal/mi"
+	"github.com/prometheus-community/windows_exporter/internal/types"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// vmStateQueryTimeout bounds the virtual machine query, so a hung WMI service
+// does not block the scrape.
+const vmStateQueryTimeout = 5 * time.Second
+
+// vmStateUnknown is the state of an EnabledState value without a mapping.
+const vmStateUnknown = "unknown"
+
+// vmStates are the values of the state label of windows_hyperv_vm_state.
+// Every VM publishes one series per state.
+//
+//nolint:gochecknoglobals
+var vmStates = []string{
+	"running",
+	"off",
+	"starting",
+	"stopping",
+	"saving",
+	"saved",
+	"pausing",
+	"paused",
+	"resuming",
+	"reset",
+	vmStateUnknown,
+}
+
+// collectorVMState Hyper-V virtual machine state metrics.
+//
+// The PDH objects only have instances for running VMs. Msvm_ComputerSystem
+// lists every VM registered on the host, whatever its state.
+type collectorVMState struct {
+	miQueryVMState mi.Query
+
+	vmState *prometheus.Desc // Msvm_ComputerSystem.EnabledState
+}
+
+// msvmComputerSystemState is the part of Msvm_ComputerSystem needed for the
+// state of a VM.
+// https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/msvm-computersystem
+type msvmComputerSystemState struct {
+	Name         string `mi:"Name"`
+	ElementName  string `mi:"ElementName"`
+	EnabledState uint16 `mi:"EnabledState"`
+}
+
+func (c *Collector) buildVMState() error {
+	if c.miSession == nil {
+		return mi.ErrNotInitialized
+	}
+
+	var err error
+
+	c.miQueryVMState, err = mi.NewQuery("SELECT Name, ElementName, EnabledState FROM Msvm_ComputerSystem")
+	if err != nil {
+		return fmt.Errorf("failed to create WMI query: %w", err)
+	}
+
+	c.buildVMStateDescriptors()
+
+	return nil
+}
+
+func (c *Collector) buildVMStateDescriptors() {
+	c.vmState = prometheus.NewDesc(
+		prometheus.BuildFQName(types.Namespace, Name, "vm_state"),
+		"The state of the virtual machine, 1 if the current state, 0 otherwise",
+		[]string{"vm", "vm_id", "state"},
+		nil,
+	)
+}
+
+func (c *Collector) collectVMState(ch chan<- prometheus.Metric, scrape *labelcache.Scrape) error {
+	var dst []msvmComputerSystemState
+
+	if err := c.miSession.Query(&dst, mi.NamespaceRootVirtualizationV2, c.miQueryVMState, vmStateQueryTimeout); err != nil {
+		return fmt.Errorf("WMI query failed: %w", err)
+	}
+
+	c.publishVMState(ch, scrape, dst)
+
+	return nil
+}
+
+func (c *Collector) publishVMState(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, systems []msvmComputerSystemState) {
+	for _, system := range systems {
+		// The host itself is an Msvm_ComputerSystem too. Its Name is the
+		// NetBIOS name of the host (at most 15 characters), the Name of a VM
+		// is its GUID. Caption tells them apart as well, but it is localized.
+		if !isVMID(system.Name) {
+			continue
+		}
+
+		current := vmStateName(system.EnabledState)
+
+		for _, state := range vmStates {
+			value := 0.0
+			if state == current {
+				value = 1.0
+			}
+
+			ch <- scrape.MustNewConstMetric(
+				c.vmState,
+				prometheus.GaugeValue,
+				value,
+				system.ElementName,
+				system.Name,
+				state,
+			)
+		}
+	}
+}
+
+// vmStateName maps Msvm_ComputerSystem.EnabledState to the state label.
+//
+// The EnabledState documentation lists the CIM values 0 to 10. 11 (Reset) is
+// a RequestedState value, mapped in case a VM reports it. 32768 (Paused),
+// 32769 (Suspended), 32770 (Starting) and 32774 (Stopping) are the
+// EnabledState values of the version 1 provider; the version 2 provider still
+// reports some of them, for example 32768 for a VM paused on low disk space.
+// 32773 (Saving), 32776 (Pausing), 32777 (Resuming), 32779 (FastSaved) and
+// 32780 (FastSaving) are the Hyper-V specific states of RequestStateChange,
+// and 32781 to 32792 their critical variants, in the order RunningCritical,
+// OffCritical, StoppingCritical, SavedCritical, PausedCritical,
+// StartingCritical, ResetCritical, SavingCritical, PausingCritical,
+// ResumingCritical, FastSavedCritical, FastSavingCritical. A critical state
+// is published as its base state; HealthState carries the critical part.
+// https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/msvm-computersystem
+// https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/requeststatechange-msvm-computersystem
+func vmStateName(enabledState uint16) string {
+	switch enabledState {
+	case 2, 32781: // Enabled, RunningCritical
+		return "running"
+	case 3, 32782: // Disabled, OffCritical
+		return "off"
+	case 4, 32774, 32783: // Shutting Down, Stopping, StoppingCritical
+		return "stopping"
+	case 6, 32769, 32779, 32784, 32791: // Enabled but Offline, Suspended, FastSaved, SavedCritical, FastSavedCritical
+		return "saved"
+	case 9, 32768, 32785: // Quiesce, Paused, PausedCritical
+		return "paused"
+	case 10, 32770, 32786: // Starting, Starting (version 1), StartingCritical
+		return "starting"
+	case 11, 32787: // Reset, ResetCritical
+		return "reset"
+	case 32773, 32780, 32788, 32792: // Saving, FastSaving, SavingCritical, FastSavingCritical
+		return "saving"
+	case 32776, 32789: // Pausing, PausingCritical
+		return "pausing"
+	case 32777, 32790: // Resuming, ResumingCritical
+		return "resuming"
+	default:
+		return vmStateUnknown
+	}
+}
+
+// isVMID reports whether name has the GUID format
+// XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX that Hyper-V uses for VM IDs.
+func isVMID(name string) bool {
+	if len(name) != 36 {
+		return false
+	}
+
+	for i := range len(name) {
+		switch i {
+		case 8, 13, 18, 23:
+			if name[i] != '-' {
+				return false
+			}
+		default:
+			if !isHexDigit(name[i]) {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
+func isHexDigit(b byte) bool {
+	return ('0' <= b && b <= '9') || ('a' <= b && b <= 'f') || ('A' <= b && b <= 'F')
+}

internal/collector/hyperv/hyperv_vm_state_internal_test.go

@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package hyperv
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVMStateName(t *testing.T) {
+	t.Parallel()
+
+	for enabledState, want := range map[uint16]string{
+		0:     "unknown",
+		1:     "unknown",
+		2:     "running",
+		3:     "off",
+		4:     "stopping",
+		5:     "unknown",
+		6:     "saved",
+		9:     "paused",
+		10:    "starting",
+		11:    "reset",
+		32768: "paused",
+		32769: "saved",
+		32770: "starting",
+		32773: "saving",
+		32774: "stopping",
+		32776: "pausing",
+		32777: "resuming",
+		32779: "saved",
+		32780: "saving",
+		32781: "running",
+		32782: "off",
+		32783: "stopping",
+		32784: "saved",
+		32785: "paused",
+		32786: "starting",
+		32787: "reset",
+		32788: "saving",
+		32789: "pausing",
+		32790: "resuming",
+		32791: "saved",
+		32792: "saving",
+		32793: "unknown",
+		65535: "unknown",
+	} {
+		got := vmStateName(enabledState)
+		require.Equalf(t, want, got, "EnabledState %d", enabledState)
+		require.Containsf(t, vmStates, got, "EnabledState %d", enabledState)
+	}
+}
+
+func TestIsVMID(t *testing.T) {
+	t.Parallel()
+
+	require.True(t, isVMID("8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5F"))
+	require.True(t, isVMID("8e3a2c14-5b7d-4f11-9c2e-0a1b2c3d4e5f"))
+	require.False(t, isVMID("HYPERV01"))
+	require.False(t, isVMID(""))
+	require.False(t, isVMID("{8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5F}"))
+	require.False(t, isVMID("8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5G"))
+	require.False(t, isVMID("8E3A2C145-B7D-4F11-9C2E-0A1B2C3D4E5F"))
+}
+
+// TestPublishVMState checks that every VM, whatever its state, publishes one
+// series per state with 1 for the current state, that VMs with the same name
+// are kept apart by vm_id and that the host itself is dropped.
+func TestPublishVMState(t *testing.T) {
+	t.Parallel()
+
+	const (
+		runningID = "8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5F"
+		offID     = "11111111-2222-3333-4444-555555555555"
+		savedID   = "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE"
+	)
+
+	c := New(nil)
+	c.buildVMStateDescriptors()
+
+	systems := []msvmComputerSystemState{
+		{Name: "HYPERV01", ElementName: "HYPERV01", EnabledState: 2}, // the host itself
+		{Name: runningID, ElementName: "web", EnabledState: 2},
+		{Name: offID, ElementName: "web", EnabledState: 3},
+		{Name: savedID, ElementName: "db", EnabledState: 6},
+	}
+
+	registry := prometheus.NewPedanticRegistry()
+	require.NoError(t, registry.Register(publishCollector(func(ch chan<- prometheus.Metric) error {
+		c.publishVMState(ch, nil, systems)
+
+		return nil
+	})))
+
+	families, err := registry.Gather()
+	require.NoError(t, err)
+	require.Len(t, families, 1)
+
+	family := families[0]
+	require.Equal(t, "windows_hyperv_vm_state", family.GetName())
+	require.Equal(t, dto.MetricType_GAUGE, family.GetType())
+	require.Len(t, family.GetMetric(), 3*len(vmStates))
+
+	current := map[string]string{}
+	statesPerVM := map[string][]string{}
+
+	for _, metric := range family.GetMetric() {
+		labels := map[string]string{}
+		for _, label := range metric.GetLabel() {
+			labels[label.GetName()] = label.GetValue()
+		}
+
+		require.NotEqual(t, "HYPERV01", labels["vm_id"], "the host must not be published")
+
+		key := labels["vm"] + "/" + labels["vm_id"]
+		statesPerVM[key] = append(statesPerVM[key], labels["state"])
+
+		switch value := metric.GetGauge().GetValue(); value {
+		case 1:
+			require.NotContains(t, current, key, "more than one current state")
+
+			current[key] = labels["state"]
+		case 0:
+		default:
+			t.Fatalf("unexpected value %v for %s", value, key)
+		}
+	}
+
+	require.Equal(t, map[string]string{
+		"web/" + runningID: "running",
+		"web/" + offID:     "off",
+		"db/" + savedID:    "saved",
+	}, current)
+
+	for key, states := range statesPerVM {
+		require.ElementsMatchf(t, vmStates, states, "states of %s", key)
+	}
+}