hyperv: add the vm_state sub-collector for VMs that are not running (#2765)

Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
2026-10-17 10:09:05 +02:00 · 2026-10-16 21:21:21 +02:00
parent ea6126fb67
commit 4f3152d84d
7 changed files with 460 additions and 7 deletions
@@ -88,8 +88,30 @@ memory) on a host with 12 logical processors.
  "the current number of Tree Connects".
 - `Hyper-V Virtual Machine Health Summary` counts VMs: `Health Ok` was 1 for
  the one VM.
- The `wmi_health`, `virtual_hard_disk` and host VM processor metrics do not
-  come from PDH, so the table below does not cover them.
+- The `wmi_health`, `virtual_hard_disk`, `vm_state` and host VM processor
+  metrics do not come from PDH, so the table below does not cover them.
+- PDH instances exist only for running VMs. To list every VM whatever its
+  state, query `Msvm_ComputerSystem` (as `vm_state` and `virtual_hard_disk`
+  do). The host itself is a row of that class too: its `Name` is the host's
+  NetBIOS name, while a VM's `Name` is its GUID (the `vm_id` label) and
+  `ElementName` its display name (the `vm` label). `Caption` and
+  `Description` also tell them apart but are localized, so do not filter on
+  them; `vm_state` keeps the rows whose `Name` is a GUID.
+- `vm_state` maps `EnabledState` to a state set. Learn documents only the
+  CIM values 0 to 10 for `EnabledState` (2 running, 3 off, 4 stopping,
+  6 saved, 9 paused, 10 starting). 11 (Reset) is a `RequestedState` value,
+  mapped defensively. 32768 (Paused), 32769 (Suspended), 32770 (Starting)
+  and 32774 (Stopping) are version 1 `EnabledState` values that the
+  version 2 provider still reports in some cases (32768 for a VM paused on
+  low disk space). 32773 saving, 32776 pausing, 32777 resuming, 32779 fast
+  saved and 32780 fast saving are the Hyper-V states of
+  `RequestStateChange`; 32781 to 32792 are their critical variants
+  (RunningCritical, OffCritical, StoppingCritical, SavedCritical,
+  PausedCritical, StartingCritical, ResetCritical, SavingCritical,
+  PausingCritical, ResumingCritical, FastSavedCritical, FastSavingCritical)
+  and map to the base state. Other values map to `unknown`. The CI workflow
+  creates a running VM (`GitHubActions`) and one that is never started
+  (`CIVirtualHardDisk`).
 - To test emitted metrics without Hyper-V, build the descriptors with the
  `build...Descriptors` helpers, fill the `perfDataObject...` slice (or a
  `pdhtypes.Collector` fixture) and gather through a registry, as in
@@ -56,6 +56,7 @@ const (
 	subCollectorVirtualSMB                       = "virtual_smb"
 	subCollectorVirtualStorageDevice             = "virtual_storage_device"
 	subCollectorVirtualSwitch                    = "virtual_switch"
+	subCollectorVMState                          = "vm_state"
 	subCollectorWMIHealth                        = "wmi_health"
 )

@@ -91,6 +92,7 @@ var ConfigDefaults = Config{
 		subCollectorVirtualSMB,
 		subCollectorVirtualStorageDevice,
 		subCollectorVirtualSwitch,
+		subCollectorVMState,
 		subCollectorWMIHealth,
 	},
 	VirtualHardDisk: VirtualHardDiskConfig{
@@ -118,6 +120,7 @@ type Collector struct {
 	collectorVirtualSMB
 	collectorVirtualStorageDevice
 	collectorVirtualSwitch
+	collectorVMState
 	collectorWMIHealth

 	config Config
@@ -315,6 +318,11 @@ func (c *Collector) Build(logger *slog.Logger, miSession *mi.Session) error {
 			collect: c.collectVirtualSwitch,
 			close:   func() { c.perfDataCollectorVirtualSwitch.Close() },
 		},
+		subCollectorVMState: {
+			build:   c.buildVMState,
+			collect: c.collectVMState,
+			close:   func() {},
+		},
 		subCollectorWMIHealth: {
 			build:   c.buildWMIHealth,
 			collect: c.collectWMIHealth,
@@ -52,6 +52,16 @@ func TestCollector(t *testing.T) {
 		require.Regexp(t, `^[0-9A-Fa-f-]{36}$`, labelValue(metric, "vm_id"))
 	}

+	// The PDH objects only list running VMs; vm_state also lists the CIVirtualHardDisk VM, which is never started.
+	if metric := testutils.RequireFixtureMetric(t, metrics, hyperv.Name, "windows_hyperv_vm_state", prometheus.Labels{"vm": "GitHubActions", "state": "running"}); metric != nil {
+		require.InDelta(t, 1, metric.GetGauge().GetValue(), 0, "expected the GitHubActions VM to be running")
+		require.Regexp(t, `^[0-9A-Fa-f-]{36}$`, labelValue(metric, "vm_id"))
+	}
+
+	if metric := testutils.RequireFixtureMetric(t, metrics, hyperv.Name, "windows_hyperv_vm_state", prometheus.Labels{"vm": "CIVirtualHardDisk", "state": "off"}); metric != nil {
+		require.InDelta(t, 1, metric.GetGauge().GetValue(), 0, "expected the CIVirtualHardDisk VM to be off")
+	}
+
 	// The health query reports failures as 0 instead of an error, so check the value.
 	if metric := testutils.RequireFixtureMetric(t, metrics, hyperv.Name, "windows_hyperv_wmi_health", nil); metric != nil {
 		require.InDelta(t, 1, metric.GetGauge().GetValue(), 0, "Hyper-V WMI health query failed")
@@ -0,0 +1,209 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package hyperv
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/prometheus-community/windows_exporter/internal/labelcache"
+	"github.com/prometheus-community/windows_exporter/internal/mi"
+	"github.com/prometheus-community/windows_exporter/internal/types"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// vmStateQueryTimeout bounds the virtual machine query, so a hung WMI service
+// does not block the scrape.
+const vmStateQueryTimeout = 5 * time.Second
+
+// vmStateUnknown is the state of an EnabledState value without a mapping.
+const vmStateUnknown = "unknown"
+
+// vmStates are the values of the state label of windows_hyperv_vm_state.
+// Every VM publishes one series per state.
+//
+//nolint:gochecknoglobals
+var vmStates = []string{
+	"running",
+	"off",
+	"starting",
+	"stopping",
+	"saving",
+	"saved",
+	"pausing",
+	"paused",
+	"resuming",
+	"reset",
+	vmStateUnknown,
+}
+
+// collectorVMState Hyper-V virtual machine state metrics.
+//
+// The PDH objects only have instances for running VMs. Msvm_ComputerSystem
+// lists every VM registered on the host, whatever its state.
+type collectorVMState struct {
+	miQueryVMState mi.Query
+
+	vmState *prometheus.Desc // Msvm_ComputerSystem.EnabledState
+}
+
+// msvmComputerSystemState is the part of Msvm_ComputerSystem needed for the
+// state of a VM.
+// https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/msvm-computersystem
+type msvmComputerSystemState struct {
+	Name         string `mi:"Name"`
+	ElementName  string `mi:"ElementName"`
+	EnabledState uint16 `mi:"EnabledState"`
+}
+
+func (c *Collector) buildVMState() error {
+	if c.miSession == nil {
+		return mi.ErrNotInitialized
+	}
+
+	var err error
+
+	c.miQueryVMState, err = mi.NewQuery("SELECT Name, ElementName, EnabledState FROM Msvm_ComputerSystem")
+	if err != nil {
+		return fmt.Errorf("failed to create WMI query: %w", err)
+	}
+
+	c.buildVMStateDescriptors()
+
+	return nil
+}
+
+func (c *Collector) buildVMStateDescriptors() {
+	c.vmState = prometheus.NewDesc(
+		prometheus.BuildFQName(types.Namespace, Name, "vm_state"),
+		"The state of the virtual machine, 1 if the current state, 0 otherwise",
+		[]string{"vm", "vm_id", "state"},
+		nil,
+	)
+}
+
+func (c *Collector) collectVMState(ch chan<- prometheus.Metric, scrape *labelcache.Scrape) error {
+	var dst []msvmComputerSystemState
+
+	if err := c.miSession.Query(&dst, mi.NamespaceRootVirtualizationV2, c.miQueryVMState, vmStateQueryTimeout); err != nil {
+		return fmt.Errorf("WMI query failed: %w", err)
+	}
+
+	c.publishVMState(ch, scrape, dst)
+
+	return nil
+}
+
+func (c *Collector) publishVMState(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, systems []msvmComputerSystemState) {
+	for _, system := range systems {
+		// The host itself is an Msvm_ComputerSystem too. Its Name is the
+		// NetBIOS name of the host (at most 15 characters), the Name of a VM
+		// is its GUID. Caption tells them apart as well, but it is localized.
+		if !isVMID(system.Name) {
+			continue
+		}
+
+		current := vmStateName(system.EnabledState)
+
+		for _, state := range vmStates {
+			value := 0.0
+			if state == current {
+				value = 1.0
+			}
+
+			ch <- scrape.MustNewConstMetric(
+				c.vmState,
+				prometheus.GaugeValue,
+				value,
+				system.ElementName,
+				system.Name,
+				state,
+			)
+		}
+	}
+}
+
+// vmStateName maps Msvm_ComputerSystem.EnabledState to the state label.
+//
+// The EnabledState documentation lists the CIM values 0 to 10. 11 (Reset) is
+// a RequestedState value, mapped in case a VM reports it. 32768 (Paused),
+// 32769 (Suspended), 32770 (Starting) and 32774 (Stopping) are the
+// EnabledState values of the version 1 provider; the version 2 provider still
+// reports some of them, for example 32768 for a VM paused on low disk space.
+// 32773 (Saving), 32776 (Pausing), 32777 (Resuming), 32779 (FastSaved) and
+// 32780 (FastSaving) are the Hyper-V specific states of RequestStateChange,
+// and 32781 to 32792 their critical variants, in the order RunningCritical,
+// OffCritical, StoppingCritical, SavedCritical, PausedCritical,
+// StartingCritical, ResetCritical, SavingCritical, PausingCritical,
+// ResumingCritical, FastSavedCritical, FastSavingCritical. A critical state
+// is published as its base state; HealthState carries the critical part.
+// https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/msvm-computersystem
+// https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/requeststatechange-msvm-computersystem
+func vmStateName(enabledState uint16) string {
+	switch enabledState {
+	case 2, 32781: // Enabled, RunningCritical
+		return "running"
+	case 3, 32782: // Disabled, OffCritical
+		return "off"
+	case 4, 32774, 32783: // Shutting Down, Stopping, StoppingCritical
+		return "stopping"
+	case 6, 32769, 32779, 32784, 32791: // Enabled but Offline, Suspended, FastSaved, SavedCritical, FastSavedCritical
+		return "saved"
+	case 9, 32768, 32785: // Quiesce, Paused, PausedCritical
+		return "paused"
+	case 10, 32770, 32786: // Starting, Starting (version 1), StartingCritical
+		return "starting"
+	case 11, 32787: // Reset, ResetCritical
+		return "reset"
+	case 32773, 32780, 32788, 32792: // Saving, FastSaving, SavingCritical, FastSavingCritical
+		return "saving"
+	case 32776, 32789: // Pausing, PausingCritical
+		return "pausing"
+	case 32777, 32790: // Resuming, ResumingCritical
+		return "resuming"
+	default:
+		return vmStateUnknown
+	}
+}
+
+// isVMID reports whether name has the GUID format
+// XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX that Hyper-V uses for VM IDs.
+func isVMID(name string) bool {
+	if len(name) != 36 {
+		return false
+	}
+
+	for i := range len(name) {
+		switch i {
+		case 8, 13, 18, 23:
+			if name[i] != '-' {
+				return false
+			}
+		default:
+			if !isHexDigit(name[i]) {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
+func isHexDigit(b byte) bool {
+	return ('0' <= b && b <= '9') || ('a' <= b && b <= 'f') || ('A' <= b && b <= 'F')
+}
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package hyperv
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVMStateName(t *testing.T) {
+	t.Parallel()
+
+	for enabledState, want := range map[uint16]string{
+		0:     "unknown",
+		1:     "unknown",
+		2:     "running",
+		3:     "off",
+		4:     "stopping",
+		5:     "unknown",
+		6:     "saved",
+		9:     "paused",
+		10:    "starting",
+		11:    "reset",
+		32768: "paused",
+		32769: "saved",
+		32770: "starting",
+		32773: "saving",
+		32774: "stopping",
+		32776: "pausing",
+		32777: "resuming",
+		32779: "saved",
+		32780: "saving",
+		32781: "running",
+		32782: "off",
+		32783: "stopping",
+		32784: "saved",
+		32785: "paused",
+		32786: "starting",
+		32787: "reset",
+		32788: "saving",
+		32789: "pausing",
+		32790: "resuming",
+		32791: "saved",
+		32792: "saving",
+		32793: "unknown",
+		65535: "unknown",
+	} {
+		got := vmStateName(enabledState)
+		require.Equalf(t, want, got, "EnabledState %d", enabledState)
+		require.Containsf(t, vmStates, got, "EnabledState %d", enabledState)
+	}
+}
+
+func TestIsVMID(t *testing.T) {
+	t.Parallel()
+
+	require.True(t, isVMID("8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5F"))
+	require.True(t, isVMID("8e3a2c14-5b7d-4f11-9c2e-0a1b2c3d4e5f"))
+	require.False(t, isVMID("HYPERV01"))
+	require.False(t, isVMID(""))
+	require.False(t, isVMID("{8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5F}"))
+	require.False(t, isVMID("8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5G"))
+	require.False(t, isVMID("8E3A2C145-B7D-4F11-9C2E-0A1B2C3D4E5F"))
+}
+
+// TestPublishVMState checks that every VM, whatever its state, publishes one
+// series per state with 1 for the current state, that VMs with the same name
+// are kept apart by vm_id and that the host itself is dropped.
+func TestPublishVMState(t *testing.T) {
+	t.Parallel()
+
+	const (
+		runningID = "8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5F"
+		offID     = "11111111-2222-3333-4444-555555555555"
+		savedID   = "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE"
+	)
+
+	c := New(nil)
+	c.buildVMStateDescriptors()
+
+	systems := []msvmComputerSystemState{
+		{Name: "HYPERV01", ElementName: "HYPERV01", EnabledState: 2}, // the host itself
+		{Name: runningID, ElementName: "web", EnabledState: 2},
+		{Name: offID, ElementName: "web", EnabledState: 3},
+		{Name: savedID, ElementName: "db", EnabledState: 6},
+	}
+
+	registry := prometheus.NewPedanticRegistry()
+	require.NoError(t, registry.Register(publishCollector(func(ch chan<- prometheus.Metric) error {
+		c.publishVMState(ch, nil, systems)
+
+		return nil
+	})))
+
+	families, err := registry.Gather()
+	require.NoError(t, err)
+	require.Len(t, families, 1)
+
+	family := families[0]
+	require.Equal(t, "windows_hyperv_vm_state", family.GetName())
+	require.Equal(t, dto.MetricType_GAUGE, family.GetType())
+	require.Len(t, family.GetMetric(), 3*len(vmStates))
+
+	current := map[string]string{}
+	statesPerVM := map[string][]string{}
+
+	for _, metric := range family.GetMetric() {
+		labels := map[string]string{}
+		for _, label := range metric.GetLabel() {
+			labels[label.GetName()] = label.GetValue()
+		}
+
+		require.NotEqual(t, "HYPERV01", labels["vm_id"], "the host must not be published")
+
+		key := labels["vm"] + "/" + labels["vm_id"]
+		statesPerVM[key] = append(statesPerVM[key], labels["state"])
+
+		switch value := metric.GetGauge().GetValue(); value {
+		case 1:
+			require.NotContains(t, current, key, "more than one current state")
+
+			current[key] = labels["state"]
+		case 0:
+		default:
+			t.Fatalf("unexpected value %v for %s", value, key)
+		}
+	}
+
+	require.Equal(t, map[string]string{
+		"web/" + runningID: "running",
+		"web/" + offID:     "off",
+		"db/" + savedID:    "saved",
+	}, current)
+
+	for key, states := range statesPerVM {
+		require.ElementsMatchf(t, vmStates, states, "states of %s", key)
+	}
+}