diff --git a/docs/collector.hyperv.md b/docs/collector.hyperv.md index 12089866..e1bb14ce 100644 --- a/docs/collector.hyperv.md +++ b/docs/collector.hyperv.md @@ -12,7 +12,7 @@ The hyperv collector exposes metrics about the Hyper-V hypervisor ### `--collector.hyperv.enabled` Comma-separated list of collectors to use, for example: -`--collector.hyperv.enabled=datastore,dynamic_memory_balancer,dynamic_memory_vm,host,hypervisor_logical_processor,hypervisor_root_partition,hypervisor_root_virtual_processor,hypervisor_virtual_processor,legacy_network_adapter,replica_vm,virtual_hard_disk,virtual_machine_health_summary,virtual_machine_vid_partition,virtual_network_adapter,virtual_network_adapter_drop_reasons,virtual_smb,virtual_storage_device,virtual_switch,wmi_health` +`--collector.hyperv.enabled=datastore,dynamic_memory_balancer,dynamic_memory_vm,host,hypervisor_logical_processor,hypervisor_root_partition,hypervisor_root_virtual_processor,hypervisor_virtual_processor,legacy_network_adapter,replica_vm,virtual_hard_disk,virtual_machine_health_summary,virtual_machine_vid_partition,virtual_network_adapter,virtual_network_adapter_drop_reasons,virtual_smb,virtual_storage_device,virtual_switch,vm_state,wmi_health` ### `--collector.hyperv.virtual-hard-disk.refresh-interval` Interval at which the `virtual_hard_disk` sub-collector reads the disks in the background, for example `10m`. @@ -390,6 +390,54 @@ rate(windows_hyperv_virtual_storage_device_io_latency_seconds_total[5m]) | `windows_hyperv_vid_remote_physical_pages` | The number of physical pages not allocated from the preferred NUMA node | gauge | `vm` | +### Hyper-V VM State + +Sub-collector `vm_state`, enabled by default. The performance counter objects only have instances for running +virtual machines, so a VM that is off, saved or paused is missing from all other metrics. This sub-collector lists +every virtual machine registered on the host with one WQL query per scrape +(`SELECT Name, ElementName, EnabledState FROM Msvm_ComputerSystem`) against the `root/virtualization/v2` namespace. +The query times out after 5 seconds; a failed or timed-out query fails the `hyperv` collector +(`windows_exporter_collector_success{collector="hyperv"}` is `0`), like the `replica_vm` sub-collector. +On a failover cluster, each node reports the virtual machines it currently owns. + +| Name | Description | Type | Labels | +|---------------------------|-----------------------------------------------------------------------|-------|------------------------| +| `windows_hyperv_vm_state` | The state of the virtual machine, 1 if the current state, 0 otherwise | gauge | `vm`, `vm_id`, `state` | + +Every virtual machine has one series per `state`: `running`, `off`, `starting`, `stopping`, `saving`, `saved`, +`pausing`, `paused`, `resuming`, `reset` and `unknown`. The value is 1 for the current state and 0 for the others. +`state` is read from `Msvm_ComputerSystem.EnabledState`. A fast-saved VM is reported as `saved`, a VM in a +critical state (for example `RunningCritical`) as its base state, and a value without a mapping as `unknown`. + +Replica VMs on a Hyper-V Replica server are turned off and show as `state="off"`; use the +`windows_hyperv_replica_vm_*` metrics to tell them apart. + +`vm` is the name of the virtual machine and `vm_id` its GUID, the same keys as the `windows_hyperv_replica_vm_*` +and `windows_hyperv_virtual_hard_disk_*` metrics. VM names don't have to be unique, `vm_id` keeps the series apart. + +Virtual machines that are turned off: + +```promql +windows_hyperv_vm_state{state="off"} == 1 +``` + +Number of virtual machines per state and host: + +```promql +sum by (instance, state) (windows_hyperv_vm_state) +``` + +Example alert for a virtual machine that is expected to run: + +```yaml +- alert: HyperVVMNotRunning + expr: windows_hyperv_vm_state{vm="web01", state="running"} == 0 + for: 10m + annotations: + summary: Hyper-V VM {{ $labels.vm }} is not running on {{ $labels.instance }} +``` + + ### Hyper-V WMI Health Sub-collector `wmi_health`, enabled by default. It runs a minimal WQL query diff --git a/docs/upgrade/0.32.md b/docs/upgrade/0.32.md index bee55b1a..3cc734d2 100644 --- a/docs/upgrade/0.32.md +++ b/docs/upgrade/0.32.md @@ -312,10 +312,10 @@ See [the logical_disk documentation](../collector.logical_disk.md#bitlocker-stat ### New default sub-collectors -| Collector | New sub-collectors enabled by default | -|-------------|---------------------------------------------------------| -| `hyperv` | `host`, `replica_vm`, `virtual_hard_disk`, `wmi_health` | -| `mscluster` | `shared_volumes` | +| Collector | New sub-collectors enabled by default | +|-------------|---------------------------------------------------------------------| +| `hyperv` | `host`, `replica_vm`, `virtual_hard_disk`, `vm_state`, `wmi_health` | +| `mscluster` | `shared_volumes` | These add new metrics. To keep the 0.31 set, list the sub-collectors explicitly with `--collector.hyperv.enabled` or `--collector.mscluster.enabled`. diff --git a/internal/collector/hyperv/AGENTS.md b/internal/collector/hyperv/AGENTS.md index 1dd76bcc..280226d1 100644 --- a/internal/collector/hyperv/AGENTS.md +++ b/internal/collector/hyperv/AGENTS.md @@ -88,8 +88,30 @@ memory) on a host with 12 logical processors. "the current number of Tree Connects". - `Hyper-V Virtual Machine Health Summary` counts VMs: `Health Ok` was 1 for the one VM. -- The `wmi_health`, `virtual_hard_disk` and host VM processor metrics do not - come from PDH, so the table below does not cover them. +- The `wmi_health`, `virtual_hard_disk`, `vm_state` and host VM processor + metrics do not come from PDH, so the table below does not cover them. +- PDH instances exist only for running VMs. To list every VM whatever its + state, query `Msvm_ComputerSystem` (as `vm_state` and `virtual_hard_disk` + do). The host itself is a row of that class too: its `Name` is the host's + NetBIOS name, while a VM's `Name` is its GUID (the `vm_id` label) and + `ElementName` its display name (the `vm` label). `Caption` and + `Description` also tell them apart but are localized, so do not filter on + them; `vm_state` keeps the rows whose `Name` is a GUID. +- `vm_state` maps `EnabledState` to a state set. Learn documents only the + CIM values 0 to 10 for `EnabledState` (2 running, 3 off, 4 stopping, + 6 saved, 9 paused, 10 starting). 11 (Reset) is a `RequestedState` value, + mapped defensively. 32768 (Paused), 32769 (Suspended), 32770 (Starting) + and 32774 (Stopping) are version 1 `EnabledState` values that the + version 2 provider still reports in some cases (32768 for a VM paused on + low disk space). 32773 saving, 32776 pausing, 32777 resuming, 32779 fast + saved and 32780 fast saving are the Hyper-V states of + `RequestStateChange`; 32781 to 32792 are their critical variants + (RunningCritical, OffCritical, StoppingCritical, SavedCritical, + PausedCritical, StartingCritical, ResetCritical, SavingCritical, + PausingCritical, ResumingCritical, FastSavedCritical, FastSavingCritical) + and map to the base state. Other values map to `unknown`. The CI workflow + creates a running VM (`GitHubActions`) and one that is never started + (`CIVirtualHardDisk`). - To test emitted metrics without Hyper-V, build the descriptors with the `build...Descriptors` helpers, fill the `perfDataObject...` slice (or a `pdhtypes.Collector` fixture) and gather through a registry, as in diff --git a/internal/collector/hyperv/hyperv.go b/internal/collector/hyperv/hyperv.go index b68420da..a27ae738 100644 --- a/internal/collector/hyperv/hyperv.go +++ b/internal/collector/hyperv/hyperv.go @@ -56,6 +56,7 @@ const ( subCollectorVirtualSMB = "virtual_smb" subCollectorVirtualStorageDevice = "virtual_storage_device" subCollectorVirtualSwitch = "virtual_switch" + subCollectorVMState = "vm_state" subCollectorWMIHealth = "wmi_health" ) @@ -91,6 +92,7 @@ var ConfigDefaults = Config{ subCollectorVirtualSMB, subCollectorVirtualStorageDevice, subCollectorVirtualSwitch, + subCollectorVMState, subCollectorWMIHealth, }, VirtualHardDisk: VirtualHardDiskConfig{ @@ -118,6 +120,7 @@ type Collector struct { collectorVirtualSMB collectorVirtualStorageDevice collectorVirtualSwitch + collectorVMState collectorWMIHealth config Config @@ -315,6 +318,11 @@ func (c *Collector) Build(logger *slog.Logger, miSession *mi.Session) error { collect: c.collectVirtualSwitch, close: func() { c.perfDataCollectorVirtualSwitch.Close() }, }, + subCollectorVMState: { + build: c.buildVMState, + collect: c.collectVMState, + close: func() {}, + }, subCollectorWMIHealth: { build: c.buildWMIHealth, collect: c.collectWMIHealth, diff --git a/internal/collector/hyperv/hyperv_test.go b/internal/collector/hyperv/hyperv_test.go index 2565f841..49263946 100644 --- a/internal/collector/hyperv/hyperv_test.go +++ b/internal/collector/hyperv/hyperv_test.go @@ -52,6 +52,16 @@ func TestCollector(t *testing.T) { require.Regexp(t, `^[0-9A-Fa-f-]{36}$`, labelValue(metric, "vm_id")) } + // The PDH objects only list running VMs; vm_state also lists the CIVirtualHardDisk VM, which is never started. + if metric := testutils.RequireFixtureMetric(t, metrics, hyperv.Name, "windows_hyperv_vm_state", prometheus.Labels{"vm": "GitHubActions", "state": "running"}); metric != nil { + require.InDelta(t, 1, metric.GetGauge().GetValue(), 0, "expected the GitHubActions VM to be running") + require.Regexp(t, `^[0-9A-Fa-f-]{36}$`, labelValue(metric, "vm_id")) + } + + if metric := testutils.RequireFixtureMetric(t, metrics, hyperv.Name, "windows_hyperv_vm_state", prometheus.Labels{"vm": "CIVirtualHardDisk", "state": "off"}); metric != nil { + require.InDelta(t, 1, metric.GetGauge().GetValue(), 0, "expected the CIVirtualHardDisk VM to be off") + } + // The health query reports failures as 0 instead of an error, so check the value. if metric := testutils.RequireFixtureMetric(t, metrics, hyperv.Name, "windows_hyperv_wmi_health", nil); metric != nil { require.InDelta(t, 1, metric.GetGauge().GetValue(), 0, "Hyper-V WMI health query failed") diff --git a/internal/collector/hyperv/hyperv_vm_state.go b/internal/collector/hyperv/hyperv_vm_state.go new file mode 100644 index 00000000..3ad28727 --- /dev/null +++ b/internal/collector/hyperv/hyperv_vm_state.go @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package hyperv + +import ( + "fmt" + "time" + + "github.com/prometheus-community/windows_exporter/internal/labelcache" + "github.com/prometheus-community/windows_exporter/internal/mi" + "github.com/prometheus-community/windows_exporter/internal/types" + "github.com/prometheus/client_golang/prometheus" +) + +// vmStateQueryTimeout bounds the virtual machine query, so a hung WMI service +// does not block the scrape. +const vmStateQueryTimeout = 5 * time.Second + +// vmStateUnknown is the state of an EnabledState value without a mapping. +const vmStateUnknown = "unknown" + +// vmStates are the values of the state label of windows_hyperv_vm_state. +// Every VM publishes one series per state. +// +//nolint:gochecknoglobals +var vmStates = []string{ + "running", + "off", + "starting", + "stopping", + "saving", + "saved", + "pausing", + "paused", + "resuming", + "reset", + vmStateUnknown, +} + +// collectorVMState Hyper-V virtual machine state metrics. +// +// The PDH objects only have instances for running VMs. Msvm_ComputerSystem +// lists every VM registered on the host, whatever its state. +type collectorVMState struct { + miQueryVMState mi.Query + + vmState *prometheus.Desc // Msvm_ComputerSystem.EnabledState +} + +// msvmComputerSystemState is the part of Msvm_ComputerSystem needed for the +// state of a VM. +// https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/msvm-computersystem +type msvmComputerSystemState struct { + Name string `mi:"Name"` + ElementName string `mi:"ElementName"` + EnabledState uint16 `mi:"EnabledState"` +} + +func (c *Collector) buildVMState() error { + if c.miSession == nil { + return mi.ErrNotInitialized + } + + var err error + + c.miQueryVMState, err = mi.NewQuery("SELECT Name, ElementName, EnabledState FROM Msvm_ComputerSystem") + if err != nil { + return fmt.Errorf("failed to create WMI query: %w", err) + } + + c.buildVMStateDescriptors() + + return nil +} + +func (c *Collector) buildVMStateDescriptors() { + c.vmState = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "vm_state"), + "The state of the virtual machine, 1 if the current state, 0 otherwise", + []string{"vm", "vm_id", "state"}, + nil, + ) +} + +func (c *Collector) collectVMState(ch chan<- prometheus.Metric, scrape *labelcache.Scrape) error { + var dst []msvmComputerSystemState + + if err := c.miSession.Query(&dst, mi.NamespaceRootVirtualizationV2, c.miQueryVMState, vmStateQueryTimeout); err != nil { + return fmt.Errorf("WMI query failed: %w", err) + } + + c.publishVMState(ch, scrape, dst) + + return nil +} + +func (c *Collector) publishVMState(ch chan<- prometheus.Metric, scrape *labelcache.Scrape, systems []msvmComputerSystemState) { + for _, system := range systems { + // The host itself is an Msvm_ComputerSystem too. Its Name is the + // NetBIOS name of the host (at most 15 characters), the Name of a VM + // is its GUID. Caption tells them apart as well, but it is localized. + if !isVMID(system.Name) { + continue + } + + current := vmStateName(system.EnabledState) + + for _, state := range vmStates { + value := 0.0 + if state == current { + value = 1.0 + } + + ch <- scrape.MustNewConstMetric( + c.vmState, + prometheus.GaugeValue, + value, + system.ElementName, + system.Name, + state, + ) + } + } +} + +// vmStateName maps Msvm_ComputerSystem.EnabledState to the state label. +// +// The EnabledState documentation lists the CIM values 0 to 10. 11 (Reset) is +// a RequestedState value, mapped in case a VM reports it. 32768 (Paused), +// 32769 (Suspended), 32770 (Starting) and 32774 (Stopping) are the +// EnabledState values of the version 1 provider; the version 2 provider still +// reports some of them, for example 32768 for a VM paused on low disk space. +// 32773 (Saving), 32776 (Pausing), 32777 (Resuming), 32779 (FastSaved) and +// 32780 (FastSaving) are the Hyper-V specific states of RequestStateChange, +// and 32781 to 32792 their critical variants, in the order RunningCritical, +// OffCritical, StoppingCritical, SavedCritical, PausedCritical, +// StartingCritical, ResetCritical, SavingCritical, PausingCritical, +// ResumingCritical, FastSavedCritical, FastSavingCritical. A critical state +// is published as its base state; HealthState carries the critical part. +// https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/msvm-computersystem +// https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/requeststatechange-msvm-computersystem +func vmStateName(enabledState uint16) string { + switch enabledState { + case 2, 32781: // Enabled, RunningCritical + return "running" + case 3, 32782: // Disabled, OffCritical + return "off" + case 4, 32774, 32783: // Shutting Down, Stopping, StoppingCritical + return "stopping" + case 6, 32769, 32779, 32784, 32791: // Enabled but Offline, Suspended, FastSaved, SavedCritical, FastSavedCritical + return "saved" + case 9, 32768, 32785: // Quiesce, Paused, PausedCritical + return "paused" + case 10, 32770, 32786: // Starting, Starting (version 1), StartingCritical + return "starting" + case 11, 32787: // Reset, ResetCritical + return "reset" + case 32773, 32780, 32788, 32792: // Saving, FastSaving, SavingCritical, FastSavingCritical + return "saving" + case 32776, 32789: // Pausing, PausingCritical + return "pausing" + case 32777, 32790: // Resuming, ResumingCritical + return "resuming" + default: + return vmStateUnknown + } +} + +// isVMID reports whether name has the GUID format +// XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX that Hyper-V uses for VM IDs. +func isVMID(name string) bool { + if len(name) != 36 { + return false + } + + for i := range len(name) { + switch i { + case 8, 13, 18, 23: + if name[i] != '-' { + return false + } + default: + if !isHexDigit(name[i]) { + return false + } + } + } + + return true +} + +func isHexDigit(b byte) bool { + return ('0' <= b && b <= '9') || ('a' <= b && b <= 'f') || ('A' <= b && b <= 'F') +} diff --git a/internal/collector/hyperv/hyperv_vm_state_internal_test.go b/internal/collector/hyperv/hyperv_vm_state_internal_test.go new file mode 100644 index 00000000..347fda60 --- /dev/null +++ b/internal/collector/hyperv/hyperv_vm_state_internal_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package hyperv + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +func TestVMStateName(t *testing.T) { + t.Parallel() + + for enabledState, want := range map[uint16]string{ + 0: "unknown", + 1: "unknown", + 2: "running", + 3: "off", + 4: "stopping", + 5: "unknown", + 6: "saved", + 9: "paused", + 10: "starting", + 11: "reset", + 32768: "paused", + 32769: "saved", + 32770: "starting", + 32773: "saving", + 32774: "stopping", + 32776: "pausing", + 32777: "resuming", + 32779: "saved", + 32780: "saving", + 32781: "running", + 32782: "off", + 32783: "stopping", + 32784: "saved", + 32785: "paused", + 32786: "starting", + 32787: "reset", + 32788: "saving", + 32789: "pausing", + 32790: "resuming", + 32791: "saved", + 32792: "saving", + 32793: "unknown", + 65535: "unknown", + } { + got := vmStateName(enabledState) + require.Equalf(t, want, got, "EnabledState %d", enabledState) + require.Containsf(t, vmStates, got, "EnabledState %d", enabledState) + } +} + +func TestIsVMID(t *testing.T) { + t.Parallel() + + require.True(t, isVMID("8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5F")) + require.True(t, isVMID("8e3a2c14-5b7d-4f11-9c2e-0a1b2c3d4e5f")) + require.False(t, isVMID("HYPERV01")) + require.False(t, isVMID("")) + require.False(t, isVMID("{8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5F}")) + require.False(t, isVMID("8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5G")) + require.False(t, isVMID("8E3A2C145-B7D-4F11-9C2E-0A1B2C3D4E5F")) +} + +// TestPublishVMState checks that every VM, whatever its state, publishes one +// series per state with 1 for the current state, that VMs with the same name +// are kept apart by vm_id and that the host itself is dropped. +func TestPublishVMState(t *testing.T) { + t.Parallel() + + const ( + runningID = "8E3A2C14-5B7D-4F11-9C2E-0A1B2C3D4E5F" + offID = "11111111-2222-3333-4444-555555555555" + savedID = "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE" + ) + + c := New(nil) + c.buildVMStateDescriptors() + + systems := []msvmComputerSystemState{ + {Name: "HYPERV01", ElementName: "HYPERV01", EnabledState: 2}, // the host itself + {Name: runningID, ElementName: "web", EnabledState: 2}, + {Name: offID, ElementName: "web", EnabledState: 3}, + {Name: savedID, ElementName: "db", EnabledState: 6}, + } + + registry := prometheus.NewPedanticRegistry() + require.NoError(t, registry.Register(publishCollector(func(ch chan<- prometheus.Metric) error { + c.publishVMState(ch, nil, systems) + + return nil + }))) + + families, err := registry.Gather() + require.NoError(t, err) + require.Len(t, families, 1) + + family := families[0] + require.Equal(t, "windows_hyperv_vm_state", family.GetName()) + require.Equal(t, dto.MetricType_GAUGE, family.GetType()) + require.Len(t, family.GetMetric(), 3*len(vmStates)) + + current := map[string]string{} + statesPerVM := map[string][]string{} + + for _, metric := range family.GetMetric() { + labels := map[string]string{} + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + + require.NotEqual(t, "HYPERV01", labels["vm_id"], "the host must not be published") + + key := labels["vm"] + "/" + labels["vm_id"] + statesPerVM[key] = append(statesPerVM[key], labels["state"]) + + switch value := metric.GetGauge().GetValue(); value { + case 1: + require.NotContains(t, current, key, "more than one current state") + + current[key] = labels["state"] + case 0: + default: + t.Fatalf("unexpected value %v for %s", value, key) + } + } + + require.Equal(t, map[string]string{ + "web/" + runningID: "running", + "web/" + offID: "off", + "db/" + savedID: "saved", + }, current) + + for key, states := range statesPerVM { + require.ElementsMatchf(t, vmStates, states, "states of %s", key) + } +}