mixin: add a Hyper-V Cluster dashboard with tabs (#2780)

Co-authored-by: EisenbergD <dominik.eisenberg@beiersdorf.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: EisenbergD <dominik.eisenberg@beiersdorf.com>

contrib/mixin/README.md

@@ -265,7 +265,9 @@ queries, IDs, variables, annotations, transformations, and tab/row layout.
 `git add --intent-to-add` and runs `git diff --exit-code`, displaying any drift. CI runs that check before the mixin tests. The dashboard uses its own
 variables for selectors; from `_config` it reads only `collectors`,
 `enableFleetTab`, `enableOverviewTab`, `enableTabDashboards`,
-`separateDashboardTabs` and the dashboard metadata below.
+`separateDashboardTabs`, `hypervDiskLatencyThresholdSeconds` (average latency
+per I/O above which the Hyper-V tab counts a virtual disk as slow, default
+`0.01`) and the dashboard metadata below.
 
 ### Metadata
 
@@ -276,6 +278,7 @@ Every dashboard has a readable UID (`metadata.name`) built from
 | --- | --- | --- |
 | Full, all tabs | `windows-exporter-full` | Windows Exporter / Full |
 | One tab | `windows-exporter-<tab>`, for example `windows-exporter-cpu` or `windows-exporter-scheduled-tasks` | Windows Exporter / `<tab>` |
+| Hyper-V cluster, with the `hyperv` and `mscluster` collectors | `windows-exporter-hyper-v-cluster` | Windows Exporter / Hyper-V Cluster |
 
 `dashboardUID`, `dashboardTitle` and `dashboardDescription` override the UID,
 title and description of the full dashboard. `dashboardTags` sets the tags of

contrib/mixin/config.libsonnet

@@ -45,6 +45,7 @@
       hyperv: false,
       logical_disk: true,
       memory: true,
+      mscluster: false,
       mssql: false,
       net: true,
       os: true,
@@ -68,6 +69,8 @@
     memoryHighUsageThreshold: 90,
     memoryCommitHighUsageThreshold: 90,
     diskLatencyThresholdSeconds: 0.05,
+    // Average latency per I/O above which a Hyper-V virtual disk counts as slow.
+    hypervDiskLatencyThresholdSeconds: 0.01,
     networkErrorRateThreshold: 1,
     networkDiscardRateThreshold: 1,
     clockOffsetThresholdSeconds: 0.05,

contrib/mixin/dashboards/builders.libsonnet

@@ -79,6 +79,15 @@ local merge(base, patch) = base + {
     )
     + $.panel.withOptions({ graphMode: 'area' }),
 
+  // Current value of a label instead of a number, for example the host a VM
+  // runs on. Shows the label of every returned series.
+  labelStat(id, title, description, expr, label):
+    $.panel.new(id, title, 'stat')
+    + $.panel.withDescription(description)
+    + $.panel.withQueries([$.query.new(expr) + $.query.withInstant() + $.query.withLegendFormat('{{%s}}' % label)])
+    + $.panel.withDefaults({ unit: 'short', color: { fixedColor: 'text', mode: 'fixed' } })
+    + $.panel.withOptions({ graphMode: 'none', textMode: 'name' }),
+
   // Time series of [expr, legend] queries. Series whose name matches below are
   // drawn below the axis, for example reads against writes.
   graph(id, title, description, unit, queries, below=null):

contrib/mixin/dashboards/cluster_shared_volumes.libsonnet

@@ -0,0 +1,41 @@
+local b = import 'builders.libsonnet';
+
+// Cluster Shared Volume panels, shared by the Failover Cluster tab and the
+// Storage tab of the Hyper-V cluster dashboard. Every node reports all CSVs of
+// its cluster, so the queries keep one series per volume.
+{
+  local volume(name, selector) = 'max by (shared_volume) (windows_mscluster_shared_volumes_%s{%s})' % [name, selector],
+  local used(selector) = '1 - %s / %s' % [volume('free_bytes', selector), volume('total_bytes', selector)],
+
+  table(id, selector):
+    b.joinedTable(id, 'Cluster Shared Volumes', 'Size, free space and used share of each Cluster Shared Volume. Requires the mscluster collector.', 'shared_volume', [
+      volume('total_bytes', selector),
+      volume('free_bytes', selector),
+      used(selector),
+    ], [
+      ['shared_volume', 'Volume'],
+      ['Value #A', 'Size'],
+      ['Value #B', 'Free'],
+      ['Value #C', 'Used'],
+    ], [
+      { matcher: { id: 'byRegexp', options: '^(Size|Free)$' }, properties: [{ id: 'unit', value: 'bytes' }] },
+      {
+        matcher: { id: 'byName', options: 'Used' },
+        properties: [
+          { id: 'unit', value: 'percentunit' },
+          { id: 'min', value: 0 },
+          { id: 'max', value: 1 },
+          { id: 'decimals', value: 1 },
+          { id: 'thresholds', value: { mode: 'absolute', steps: b.levels(0.8, 0.9) } },
+          { id: 'color', value: { mode: 'thresholds' } },
+          { id: 'custom.cellOptions', value: { mode: 'basic', type: 'gauge', valueDisplayMode: 'text' } },
+        ],
+      },
+    ]),
+
+  usage(id, selector):
+    b.graph(id, 'Cluster Shared Volume usage', 'Used share of each Cluster Shared Volume. Thin provisioned virtual hard disks keep growing into the free space.', 'percentunit', [
+      [used(selector), '{{shared_volume}}'],
+    ])
+    + b.panel.withDefaults({ max: 1, custom: { thresholdsStyle: { mode: 'dashed' } }, thresholds: { steps: b.levels(0.8, 0.9) } }),
+}

contrib/mixin/dashboards/hyper_v.libsonnet

@@ -1,14 +1,39 @@
+local h = import '../lib/hyperv.libsonnet';
 local b = import 'builders.libsonnet';
 
-function(on)
+function(on, config)
+  local host = 'job=~"$job", instance="$instance"';
+  local metric(name) = '%s{%s}' % [name, host];
+  local threshold = std.toString(config.hypervDiskLatencyThresholdSeconds);
+  // Latency per I/O of each virtual disk with at least one I/O per second.
+  local deviceLatency = '%s / (%s > 1)' % [h.deviceRate('io_latency_seconds_total', host), h.deviceRate('throughput_total', host)];
+  local byVm(expr) = 'sum by (vm) (%s)' % h.withVm(expr, host);
+  local size = 'max by (instance, path) (%s)' % metric('windows_hyperv_virtual_hard_disk_size_bytes');
+  local file = 'max by (instance, path) (%s)' % metric('windows_hyperv_virtual_hard_disk_file_size_bytes');
+  // Only dynamic disks grow; a differencing disk's parent already holds most of its size.
+  local dynamic(name) = 'max by (instance, path) (windows_hyperv_virtual_hard_disk_%s{%s, type="dynamic"})' % [name, host];
+  // Range-wide values of the slowest disks: time above the threshold in
+  // 5-minute steps ranks permanently slow disks above short spikes.
+  local slowest = 'topk(10, count_over_time((%s > %s)[$__range:5m]) * 300)' % [deviceLatency, threshold];
+  // Tables show every label as a column, so keep only the joined ones.
+  local slowTable(expr) = 'max by (device, disk, vm) (%s)' % h.diskName(h.withVm(expr, host));
   b.tab(
     'Hyper-V',
     b.rows([
-      if on('hyperv') then b.summary([63, 64, 65, 66, 67, 68]),
-      if on('hyperv') then b.row('CPU', b.flow([[69, 12, 8], [70, 12, 8]])),
+      if on('hyperv') then b.summary([63, 64, 65, 66, 67, 68, if on('memory') then 290, 291]),
+      if on('hyperv') then b.row('Virtual machines', b.flow([[292, 24, 10]])),
+      if on('hyperv') then b.row('CPU', b.flow([[69, 12, 8], [70, 12, 8], [293, 24, 8]])),
       if on('hyperv') then b.row('Memory', b.flow([[72, 8, 8], [73, 8, 8], [74, 8, 8]])),
-      if on('hyperv') then b.row('Storage', b.flow([[76, 8, 8], [77, 8, 8], [78, 8, 8]])),
-      if on('hyperv') then b.row('Network', b.flow([[80, 8, 8], [81, 8, 8], [82, 8, 8]])),
+      if on('hyperv') then b.row('Storage', b.flow([
+        [76, 8, 8],
+        [77, 8, 8],
+        [78, 8, 8],
+        [294, 24, 9],
+        [295, 8, 4],
+        [296, 8, 4],
+        [297, 8, 4],
+      ])),
+      if on('hyperv') then b.row('Network', b.flow([[80, 8, 8], [81, 8, 8], [82, 8, 8], [298, 24, 8]])),
     ]),
     [
       b.stat(63, 'Virtual machines', 'Needs the hyperv collector, which is not enabled by default.', 'sum(windows_hyperv_virtual_machines{job=~"$job", instance="$instance"})', 'short'),
@@ -122,7 +147,7 @@ function(on)
       }),
 
       b.panel.new(73, 'VM memory pressure', 'timeseries')
-      + b.panel.withDescription('Memory a VM needs as a share of the memory it has. Above 100% the VM needs more memory than it is assigned.')
+      + b.panel.withDescription("Hyper-V memory demand of each VM as a share of its assigned memory. The demand follows the guest's commit charge, not the RAM in use, and is reported for static and dynamic memory. Above 100% the guest has committed more than its RAM: the rest is only reserved or paged out. Check paging inside the guest before adding RAM.")
       + b.panel.withQueries([
         b.query.new('windows_hyperv_dynamic_memory_vm_pressure_current_ratio{job=~"$job", instance="$instance"}', 'A')
         + b.query.withLegendFormat('{{vm}}'),
@@ -319,5 +344,70 @@ function(on)
           ],
         },
       ]),
+
+      b.stat(290, 'VM memory / host RAM', "Memory assigned to all VMs as a share of the host's physical memory. Needs the memory collector.", 'sum(%s) / max(%s)' % [metric('windows_hyperv_dynamic_memory_vm_physical_bytes'), metric('windows_memory_physical_total_bytes')], 'percentunit', b.levels(0.85, 0.95)),
+
+      b.stat(291, 'Slow virtual disks', 'Virtual disks with an average latency per I/O above %s s and at least one I/O per second.' % threshold, 'count((%s) > %s) or vector(0)' % [deviceLatency, threshold], 'short', b.levels(1, 5)),
+
+      b.joinedTable(292, 'Virtual machines', "VMs running on this host. CPU usage is busy time on the host's physical processors relative to the VM's own virtual processors, not the utilization Task Manager shows inside the VM; committed vs. RAM is the Hyper-V memory demand (guest commit charge) as a share of the assigned memory.", 'vm', [
+        'max by (vm) (%s)' % metric('windows_hyperv_vm_processors'),
+        'max by (vm) (%s)' % metric('windows_hyperv_dynamic_memory_vm_physical_bytes'),
+        '1 - max by (vm) (%s) / max by (vm) (%s)' % [metric('windows_hyperv_dynamic_memory_vm_guest_available_bytes'), metric('windows_hyperv_dynamic_memory_vm_guest_visible_physical_memory_bytes')],
+        'max by (vm) (%s)' % metric('windows_hyperv_dynamic_memory_vm_pressure_current_ratio'),
+        'sum by (vm) (rate(%s[$__rate_interval])) / max by (vm) (%s)' % [metric('windows_hyperv_hypervisor_virtual_processor_run_time_seconds_total'), metric('windows_hyperv_vm_processors')],
+        byVm('(%s + %s)' % [h.deviceRate('operations_read_total', host), h.deviceRate('operations_written_total', host)]),
+        h.latency(byVm(h.deviceRate('io_latency_seconds_total', host)), byVm(h.deviceRate('throughput_total', host))),
+      ], [
+        ['vm', 'VM'],
+        ['Value #A', 'vCPUs'],
+        ['Value #B', 'Assigned memory'],
+        ['Value #C', 'RAM in use'],
+        ['Value #D', 'Committed vs. RAM'],
+        ['Value #E', 'CPU'],
+        ['Value #F', 'Disk IOPS'],
+        ['Value #G', 'Disk latency'],
+      ], [
+        { matcher: { id: 'byName', options: 'Assigned memory' }, properties: [{ id: 'unit', value: 'bytes' }] },
+        { matcher: { id: 'byRegexp', options: '^(RAM in use|CPU)$' }, properties: [{ id: 'unit', value: 'percentunit' }, { id: 'custom.cellOptions', value: { type: 'color-text' } }, { id: 'thresholds', value: { mode: 'absolute', steps: b.levels(0.85, 0.95) } }] },
+        { matcher: { id: 'byName', options: 'Committed vs. RAM' }, properties: [{ id: 'unit', value: 'percentunit' }, { id: 'custom.cellOptions', value: { type: 'color-text' } }, { id: 'thresholds', value: { mode: 'absolute', steps: b.levels(1, 1.5) } }] },
+        { matcher: { id: 'byName', options: 'Disk IOPS' }, properties: [{ id: 'unit', value: 'iops' }] },
+        { matcher: { id: 'byName', options: 'Disk latency' }, properties: [{ id: 'unit', value: 's' }, { id: 'custom.cellOptions', value: { type: 'color-text' } }, { id: 'thresholds', value: { mode: 'absolute', steps: [{ color: 'green', value: null }, { color: 'orange', value: config.hypervDiskLatencyThresholdSeconds }] } }] },
+      ]),
+
+      b.graph(293, 'Busiest VMs', "The ten VMs with the highest CPU usage at each point in time, so the graph can show more than ten VMs over the time range relative to their own virtual processors (run time per second; 100% means every virtual processor ran all the time). Near 100% a VM runs at its limit and needs more virtual processors, even when the host has CPU left. This is busy time on the host's physical processors as Hyper-V sees it, not the utilization Task Manager shows inside the VM: when a virtual processor waits for a physical processor, the guest counts that time as busy, the host doesn't.", 'percentunit', [
+        ['topk(10, sum by (vm) (rate(%s[$__rate_interval])) / max by (vm) (%s))' % [metric('windows_hyperv_hypervisor_virtual_processor_run_time_seconds_total'), metric('windows_hyperv_vm_processors')], '{{vm}}'],
+      ])
+      + b.panel.withDefaults({ max: 1 }),
+
+      b.joinedTable(294, 'Slowest virtual disks', 'Virtual disks that were above %s s average latency per I/O in the time range, worst first. Time above threshold is counted in 5-minute steps, so permanently slow disks rank above short spikes. Disks with less than one I/O per second are ignored.' % threshold, 'device', [
+        slowTable(slowest),
+        slowTable('avg_over_time((%s)[$__range:5m]) and on (instance, device) %s' % [deviceLatency, slowest]),
+        slowTable('max_over_time((%s)[$__range:5m]) and on (instance, device) %s' % [deviceLatency, slowest]),
+        slowTable('avg_over_time((%s + %s)[$__range:5m]) and on (instance, device) %s' % [h.deviceRate('operations_read_total', host), h.deviceRate('operations_written_total', host), slowest]),
+      ], [
+        ['disk', 'Disk'],
+        ['vm', 'VM'],
+        ['Value #A', 'Time above threshold'],
+        ['Value #B', 'Average latency'],
+        ['Value #C', 'Maximum latency'],
+        ['Value #D', 'IOPS'],
+      ], [
+        { matcher: { id: 'byName', options: 'device' }, properties: [{ id: 'custom.hidden', value: true }] },
+        { matcher: { id: 'byName', options: 'Time above threshold' }, properties: [{ id: 'unit', value: 's' }, { id: 'custom.cellOptions', value: { type: 'color-background', mode: 'basic' } }, { id: 'thresholds', value: { mode: 'absolute', steps: b.levels(900, 3600) } }] },
+        { matcher: { id: 'byRegexp', options: '^(Average|Maximum) latency$' }, properties: [{ id: 'unit', value: 's' }] },
+        { matcher: { id: 'byName', options: 'IOPS' }, properties: [{ id: 'unit', value: 'iops' }] },
+      ])
+      + b.panel.withOptions({ sortBy: [{ desc: true, displayName: 'Time above threshold' }] }),
+
+      b.stat(295, 'VHD size', 'Maximum size of all virtual hard disks of the VMs on this host.', 'sum(%s)' % size, 'bytes'),
+
+      b.stat(296, 'VHD file size', 'Space the virtual hard disk files of the VMs on this host take on storage. For a VM with checkpoints, only the current .avhdx counts, not its parent disks.', 'sum(%s)' % file, 'bytes'),
+
+      b.stat(297, 'VHD growth if thick', 'Space the dynamic (thin provisioned) disks of the VMs on this host would still take if every one grew to its full size.', 'sum(clamp_min(%s - %s, 0))' % [dynamic('size_bytes'), dynamic('file_size_bytes')], 'bytes'),
+
+      b.graph(298, 'Dropped packets per VM', 'Packets dropped per second by the network adapters of each VM; incoming drops are drawn below the axis. Adapters are matched to VMs by the VM GUID in their name.', 'pps', [
+        ['sum by (vm) (%s)' % h.vnicVm('rate(%s[$__rate_interval])' % metric('windows_hyperv_virtual_network_adapter_outgoing_dropped_packets_total'), host), '{{vm}} outgoing'],
+        ['sum by (vm) (%s)' % h.vnicVm('rate(%s[$__rate_interval])' % metric('windows_hyperv_virtual_network_adapter_incoming_dropped_packets_total'), host), '{{vm}} incoming'],
+      ], below='.* incoming$'),
     ]
   )

contrib/mixin/dashboards/hyper_v_cluster_overview.libsonnet

@@ -0,0 +1,97 @@
+local h = import '../lib/hyperv.libsonnet';
+local b = import 'builders.libsonnet';
+
+// Capacity and health of all Hyper-V hosts of a failover cluster.
+function(on, config)
+  local hosts = 'job=~"$job", instance=~"$cluster_instance"';
+  local metric(name) = '%s{%s}' % [name, hosts];
+  local threshold = std.toString(config.hypervDiskLatencyThresholdSeconds);
+  local deviceLatency = '%s / (%s > 1)' % [h.deviceRate('io_latency_seconds_total', hosts), h.deviceRate('throughput_total', hosts)];
+  local nodeState = 'max by (node) (%s)' % metric('windows_mscluster_node_state');
+  local vmMemory = 'sum by (instance) (%s)' % metric('windows_hyperv_dynamic_memory_vm_physical_bytes');
+  local hostMemory = 'max by (instance) (%s)' % metric('windows_memory_physical_total_bytes');
+  local hostBusy = 'sum by (instance) (rate(windows_hyperv_hypervisor_logical_processor_time_seconds_total{%s, state!="idle"}[$__rate_interval])) / count by (instance) (windows_hyperv_hypervisor_logical_processor_time_seconds_total{%s, state="idle"})' % [hosts, hosts];
+  local runningVms = 'count by (instance) (max by (instance, vm) (%s))' % metric('windows_hyperv_vm_processors');
+  b.tab(
+    'Overview',
+    b.rows([
+      if on('hyperv') then b.summary([
+        if on('mscluster') then 360,
+        361,
+        362,
+        363,
+        if on('memory') then 364,
+        if on('memory') then 365,
+        366,
+        367,
+      ]),
+      if on('hyperv') then b.row('Hosts', b.flow([[368, 24, 8]])),
+      if on('hyperv') then b.row('Load', b.flow([
+        [369, 12, 8],
+        [370, 12, 8],
+        if on('memory') then [371, 12, 8],
+        [372, 12, 8],
+      ])),
+    ]),
+    [
+      b.stat(360, 'Nodes up', 'Cluster nodes in state Up, of all nodes. Requires the mscluster collector.', 'count(%s == 0) or (0 * count(%s))' % [nodeState, nodeState]),
+
+      b.stat(361, 'Running VMs', 'Virtual machines running on the hosts of the cluster. Needs the hyperv collector, which is not enabled by default.', 'sum(%s)' % runningVms),
+
+      b.stat(362, 'VMs in critical health', 'Virtual machines reporting critical health on any host of the cluster.', 'sum(windows_hyperv_virtual_machines{%s, state="critical"})' % hosts, steps=[{ color: 'green', value: null }, { color: 'red', value: 1 }]),
+
+      b.stat(363, 'vCPU per logical CPU', 'Virtual processors assigned to VMs per logical processor, over all hosts of the cluster.', 'sum(%s) / sum(%s)' % [metric('windows_hyperv_assigned_vm_processors'), metric('windows_hyperv_host_logical_processors')])
+      + b.panel.withDefaults({ decimals: 2 }),
+
+      b.stat(364, 'VM memory / host RAM', 'Memory assigned to all VMs as a share of the physical memory of all hosts. Needs the memory collector.', 'sum(%s) / sum(%s)' % [vmMemory, hostMemory], 'percentunit', b.levels(0.85, 0.95)),
+
+      b.stat(365, 'VM memory without largest host', 'Memory assigned to all VMs as a share of the physical memory left when the host with the most memory fails. Above 100%, not all VMs can restart after a host failure.', 'sum(%s) / (sum(%s) - max(%s))' % [vmMemory, hostMemory, hostMemory], 'percentunit', b.levels(0.9, 1)),
+
+      b.stat(366, 'Host CPU usage', 'Share of all logical processors of the cluster running guest code or the hypervisor.', 'sum(rate(windows_hyperv_hypervisor_logical_processor_time_seconds_total{%s, state!="idle"}[$__rate_interval])) / count(windows_hyperv_hypervisor_logical_processor_time_seconds_total{%s, state="idle"})' % [hosts, hosts], 'percentunit', b.levels(0.7, 0.9)),
+
+      b.stat(367, 'Slow virtual disks', 'Virtual disks of the cluster with an average latency per I/O above %s s and at least one I/O per second.' % threshold, 'count((%s) > %s) or vector(0)' % [deviceLatency, threshold], 'short', b.levels(1, 5)),
+
+      b.joinedTable(368, 'Hosts', "Running VMs, virtual processors per logical processor, CPU usage and the share of the host's memory assigned to VMs, per host of the cluster.", 'instance', [
+        runningVms,
+        'max by (instance) (%s) / max by (instance) (%s)' % [metric('windows_hyperv_assigned_vm_processors'), metric('windows_hyperv_host_logical_processors')],
+        hostBusy,
+      ] + (if on('memory') then ['%s / %s' % [vmMemory, hostMemory]] else []), [
+        ['instance', 'Host'],
+        ['Value #A', 'Running VMs'],
+        ['Value #B', 'vCPU per logical CPU'],
+        ['Value #C', 'CPU usage'],
+      ] + (if on('memory') then [['Value #D', 'VM memory / RAM']] else []), [
+        { matcher: { id: 'byName', options: 'vCPU per logical CPU' }, properties: [{ id: 'decimals', value: 2 }] },
+        {
+          matcher: { id: 'byRegexp', options: '^(CPU usage|VM memory / RAM)$' },
+          properties: [
+            { id: 'unit', value: 'percentunit' },
+            { id: 'min', value: 0 },
+            { id: 'max', value: 1 },
+            { id: 'thresholds', value: { mode: 'absolute', steps: b.levels(0.85, 0.95) } },
+            { id: 'color', value: { mode: 'thresholds' } },
+            { id: 'custom.cellOptions', value: { mode: 'basic', type: 'gauge', valueDisplayMode: 'text' } },
+          ],
+        },
+      ]),
+
+      b.graph(369, 'Host CPU usage', 'Share of the logical processors of each host running guest code or the hypervisor.', 'percentunit', [
+        [hostBusy, '{{instance}}'],
+      ])
+      + b.panel.withDefaults({ max: 1 }),
+
+      b.graph(370, 'Running VMs per host', 'Virtual machines running on each host. Steps show live migrations and failovers.', 'short', [
+        [runningVms, '{{instance}}'],
+      ]),
+
+      b.graph(371, 'VM memory per host', "Memory assigned to VMs as a share of each host's physical memory.", 'percentunit', [
+        ['%s / %s' % [vmMemory, hostMemory], '{{instance}}'],
+      ])
+      + b.panel.withDefaults({ max: 1, custom: { thresholdsStyle: { mode: 'dashed' } }, thresholds: { steps: b.levels(0.85, 0.95) } }),
+
+      b.graph(372, 'Virtual disk IOPS', 'Read and write operations per second of all virtual disks on each host; reads are drawn below the axis.', 'iops', [
+        ['sum by (instance) (%s)' % h.deviceRate('operations_written_total', hosts), '{{instance}} write'],
+        ['sum by (instance) (%s)' % h.deviceRate('operations_read_total', hosts), '{{instance}} read'],
+      ], below='.* read$'),
+    ]
+  )

contrib/mixin/dashboards/hyper_v_storage.libsonnet

@@ -0,0 +1,132 @@
+local b = import 'builders.libsonnet';
+local csv = import 'cluster_shared_volumes.libsonnet';
+
+// Cluster Shared Volumes, the virtual hard disks on them and the Storage
+// Spaces Direct pools of a failover cluster. Every node reports the volumes
+// and pools of the whole cluster, so the queries keep one series per object.
+function(on)
+  local hosts = 'job=~"$job", instance=~"$cluster_instance"';
+  local volume(name) = 'max by (shared_volume) (windows_mscluster_shared_volumes_%s{%s})' % [name, hosts];
+  // A literal backslash in a PromQL regex inside a Jsonnet string.
+  local bs = '\\\\\\\\';
+  // Adds folder, the folder below C:\ClusterStorage that holds the disk.
+  local onCsv = 'path=~"(?i).:%sClusterStorage%s.*"' % [bs, bs];
+  local inFolder(name) = 'label_replace(max by (path) (windows_hyperv_virtual_hard_disk_%s{%s, %s}), "folder", "$1", "path", "(?i).:%sClusterStorage%s([^%s]+)%s.*")' % [name, hosts, onCsv, bs, bs, bs, bs];
+  local byFolder(expr) = 'sum by (folder) (%s)' % expr;
+  local size = inFolder('size_bytes');
+  local file = inFolder('file_size_bytes');
+  // Only dynamic disks grow; a differencing disk's parent already holds most of its size.
+  local dynamicSize = 'label_replace(max by (path) (windows_hyperv_virtual_hard_disk_size_bytes{%s, %s, type="dynamic"}), "folder", "$1", "path", "(?i).:%sClusterStorage%s([^%s]+)%s.*")' % [hosts, onCsv, bs, bs, bs, bs];
+  local spaces(name, labels, matchers='') = 'max by (%s) (windows_storage_spaces_%s{%s%s})' % [labels, name, hosts, matchers];
+  local concretePools = spaces('pool_info', 'pool', ', primordial="false"');
+  // The primordial pool of unpooled disks has no capacity of its own.
+  local pool(name) = '(%s * on (pool) group_left () %s)' % [spaces('pool_' + name, 'pool'), concretePools];
+  local health = {
+    matcher: { id: 'byName', options: 'Health' },
+    properties: [
+      { id: 'mappings', value: [{ type: 'value', options: {
+        '0': { color: 'green', index: 0, text: 'Healthy' },
+        '1': { color: 'orange', index: 1, text: 'Warning' },
+        '2': { color: 'red', index: 2, text: 'Unhealthy' },
+        '5': { color: 'text', index: 3, text: 'Unknown' },
+      } }] },
+      { id: 'custom.cellOptions', value: { type: 'color-text' } },
+    ],
+  };
+  local bytes(pattern) = { matcher: { id: 'byRegexp', options: pattern }, properties: [{ id: 'unit', value: 'bytes' }] };
+  local growth = 'sum(clamp_min(%s - %s, 0))' % [
+    'max by (path) (windows_hyperv_virtual_hard_disk_size_bytes{%s, %s, type="dynamic"})' % [hosts, onCsv],
+    'max by (path) (windows_hyperv_virtual_hard_disk_file_size_bytes{%s, %s})' % [hosts, onCsv],
+  ];
+  b.tab(
+    'Storage',
+    b.rows([
+      if on('mscluster') then b.summary([400, 401, if on('hyperv') then 402, if on('storage_spaces') then 403, if on('storage_spaces') then 404]),
+      if on('mscluster') then b.row('Cluster Shared Volumes', b.flow([[405, 12, 8], [406, 12, 8], if on('hyperv') then [407, 24, 8]])),
+      if on('storage_spaces') then b.row('Storage Spaces Direct', b.flow([[408, 24, 5], [409, 24, 8], [410, 12, 8], [411, 12, 8]])),
+    ]),
+    [
+      b.stat(400, 'CSV size', 'Size of all Cluster Shared Volumes. Requires the mscluster collector.', 'sum(%s)' % volume('total_bytes'), 'bytes'),
+
+      b.stat(401, 'CSV free', 'Free space on all Cluster Shared Volumes. On thin provisioned Storage Spaces virtual disks, the volumes can be larger than their pool; then the pool usage is the limit.', 'sum(%s)' % volume('free_bytes'), 'bytes'),
+
+      b.stat(402, 'CSV free if thick', 'Free space left on all Cluster Shared Volumes if every dynamic (thin provisioned) virtual hard disk on them grew to its full size. Below zero, the volumes are overcommitted by thin provisioned disks.', 'sum(%s) - %s' % [volume('free_bytes'), growth], 'bytes', [{ color: 'red', value: null }, { color: 'green', value: 0 }]),
+
+      b.stat(403, 'Pools not OK', 'Storage pools whose health status is not Healthy, excluding the primordial pool. Requires the storage_spaces collector.', 'count(%s != 0) or (0 * count(%s))' % [pool('health_status'), concretePools], steps=[{ color: 'green', value: null }, { color: 'red', value: 1 }]),
+
+      b.stat(404, 'Virtual disks not OK', 'Storage Spaces virtual disks whose health status is not Healthy.', 'count(%s != 0) or (0 * count(%s))' % [spaces('virtual_disk_health_status', 'virtual_disk'), spaces('virtual_disk_info', 'virtual_disk')], steps=[{ color: 'green', value: null }, { color: 'red', value: 1 }]),
+
+      csv.table(405, hosts),
+      csv.usage(406, hosts),
+
+      b.joinedTable(407, 'Virtual hard disks per CSV folder', 'Virtual hard disks by the folder below C:\\ClusterStorage that holds them, usually the mount point of one Cluster Shared Volume. Growth if thick is the space the dynamic disks would still take if they grew to their full size.', 'folder', [
+        'count by (folder) (%s)' % size,
+        byFolder(size),
+        byFolder(file),
+        byFolder('clamp_min(%s - on (path, folder) %s, 0)' % [dynamicSize, file]),
+      ], [
+        ['folder', 'Folder'],
+        ['Value #A', 'Disks'],
+        ['Value #B', 'Size'],
+        ['Value #C', 'File size'],
+        ['Value #D', 'Growth if thick'],
+      ], [bytes('^(Size|File size|Growth if thick)$')]),
+
+      b.joinedTable(408, 'Storage pools', 'Health, size, allocated capacity and used share of each storage pool, excluding the primordial pool of unpooled disks.', 'pool', [
+        pool('health_status'),
+        pool('size_bytes'),
+        pool('allocated_size_bytes'),
+        '%s / %s' % [pool('allocated_size_bytes'), pool('size_bytes')],
+      ], [
+        ['pool', 'Pool'],
+        ['Value #A', 'Health'],
+        ['Value #B', 'Size'],
+        ['Value #C', 'Allocated'],
+        ['Value #D', 'Used'],
+      ], [
+        health,
+        bytes('^(Size|Allocated)$'),
+        {
+          matcher: { id: 'byName', options: 'Used' },
+          properties: [
+            { id: 'unit', value: 'percentunit' },
+            { id: 'min', value: 0 },
+            { id: 'max', value: 1 },
+            { id: 'decimals', value: 1 },
+            { id: 'thresholds', value: { mode: 'absolute', steps: b.levels(0.8, 0.9) } },
+            { id: 'color', value: { mode: 'thresholds' } },
+            { id: 'custom.cellOptions', value: { mode: 'basic', type: 'gauge', valueDisplayMode: 'text' } },
+          ],
+        },
+      ]),
+
+      b.joinedTable(409, 'Storage Spaces virtual disks', 'Health, size, allocated capacity, pool footprint and storage efficiency (allocated size per pool footprint) of each virtual disk. Mirrors and parity lower the efficiency.', 'virtual_disk', [
+        spaces('virtual_disk_health_status', 'virtual_disk'),
+        spaces('virtual_disk_size_bytes', 'virtual_disk'),
+        spaces('virtual_disk_allocated_size_bytes', 'virtual_disk'),
+        spaces('virtual_disk_footprint_on_pool_bytes', 'virtual_disk'),
+        spaces('virtual_disk_storage_efficiency_ratio', 'virtual_disk'),
+      ], [
+        ['virtual_disk', 'Virtual disk'],
+        ['Value #A', 'Health'],
+        ['Value #B', 'Size'],
+        ['Value #C', 'Allocated'],
+        ['Value #D', 'Footprint'],
+        ['Value #E', 'Efficiency'],
+      ], [
+        health,
+        bytes('^(Size|Allocated|Footprint)$'),
+        { matcher: { id: 'byName', options: 'Efficiency' }, properties: [{ id: 'unit', value: 'percentunit' }, { id: 'decimals', value: 1 }] },
+      ]),
+
+      b.graph(410, 'Pool usage', 'Allocated share of each storage pool, excluding the primordial pool. A full pool cannot extend thin provisioned virtual disks, which then fail writes.', 'percentunit', [
+        ['%s / %s' % [pool('allocated_size_bytes'), pool('size_bytes')], '{{pool}}'],
+      ])
+      + b.panel.withDefaults({ max: 1, custom: { thresholdsStyle: { mode: 'dashed' } }, thresholds: { steps: b.levels(0.8, 0.9) } }),
+
+      b.graph(411, 'Virtual disk footprint', 'Pool capacity each Storage Spaces virtual disk consumes, including resiliency copies.', 'bytes', [
+        [spaces('virtual_disk_footprint_on_pool_bytes', 'virtual_disk'), '{{virtual_disk}}'],
+      ])
+      + b.panel.withDefaults({ custom: { fillOpacity: 40, stacking: { mode: 'normal' } } }),
+    ]
+  )

contrib/mixin/dashboards/hyper_v_vm.libsonnet

@@ -0,0 +1,127 @@
+local h = import '../lib/hyperv.libsonnet';
+local b = import 'builders.libsonnet';
+
+// One virtual machine on whichever host of the cluster runs it. The tab
+// filters by the hosts of the cluster, not by one host, so it follows a VM
+// through live migrations.
+function(on, config)
+  local vm = 'job=~"$job", instance=~"$cluster_instance", vm="$vm"';
+  local metric(name) = '%s{%s}' % [name, vm];
+  local vcpus = 'max(%s)' % metric('windows_hyperv_vm_processors');
+  local job = 'job=~"$job", instance=~"$cluster_instance"';
+  local size = 'max by (path) (%s)' % metric('windows_hyperv_virtual_hard_disk_size_bytes');
+  local file = 'max by (path) (%s)' % metric('windows_hyperv_virtual_hard_disk_file_size_bytes');
+  // Only dynamic disks grow; a differencing disk's parent already holds most of its size.
+  local growth = 'clamp_min(max by (path) (windows_hyperv_virtual_hard_disk_size_bytes{%s, type="dynamic"}) - %s, 0)' % [vm, file];
+  // I/O of the VM's disks, by disk file.
+  local perDisk(metric_) = h.diskName('sum by (path) (%s)' % h.withVm(h.deviceRate(metric_, job), vm));
+  local vnic(metric_) = h.vnicVm('rate(%s{%s}[$__rate_interval])' % [metric_, job], vm);
+  local nicName(expr) = 'label_replace(%s, "nic", "$1", "adapter", "[^_]*_(.*)_[0-9A-Fa-f]{8}-.*")' % expr;
+  b.tab(
+    'VM',
+    b.rows([
+      if on('hyperv') then b.summary([300, 301, 302, 303, 304, 305]),
+      if on('hyperv') then b.row('CPU', b.flow([[306, 12, 8], [307, 12, 8]])),
+      if on('hyperv') then b.row('Memory', b.flow([[308, 8, 8], [309, 8, 8], [310, 8, 8]])),
+      if on('hyperv') then b.row('Disks', b.flow([
+        [311, 6, 4],
+        [312, 6, 4],
+        [313, 6, 4],
+        [314, 6, 4],
+        [315, 24, 7],
+        [316, 12, 8],
+        [317, 12, 8],
+        [318, 24, 8],
+      ])),
+      if on('hyperv') then b.row('Network', b.flow([[319, 12, 8], [320, 12, 8]])),
+    ]),
+    [
+      b.labelStat(300, 'Host', 'Host that runs the VM now. During a live migration, both hosts are shown.', 'max by (instance) (%s)' % metric('windows_hyperv_vm_processors'), 'instance'),
+
+      b.stat(301, 'Virtual processors', 'Virtual processors assigned to the VM.', vcpus),
+
+      b.stat(302, 'Assigned memory', 'Physical memory the host currently assigns to the VM, for static and dynamic memory.', 'max(%s)' % metric('windows_hyperv_dynamic_memory_vm_physical_bytes'), 'bytes'),
+
+      b.stat(303, 'RAM in use', "Memory the guest uses: visible memory minus the memory the guest reports as available. Empty when the guest doesn't report available memory (no integration services).", '1 - max(%s) / max(%s)' % [metric('windows_hyperv_dynamic_memory_vm_guest_available_bytes'), metric('windows_hyperv_dynamic_memory_vm_guest_visible_physical_memory_bytes')], 'percentunit', b.levels(0.85, 0.95)),
+
+      b.stat(304, 'Committed vs. RAM', "Hyper-V memory demand as a share of the assigned memory. The demand follows the guest's commit charge, not the RAM in use. Above 100% the guest has committed more memory than it has RAM: the rest is only reserved or paged out. Check paging inside the guest before adding RAM. Applies to static and dynamic memory.", 'max(%s)' % metric('windows_hyperv_dynamic_memory_vm_pressure_current_ratio'), 'percentunit', b.levels(1, 1.5)),
+
+      b.stat(305, 'CPU usage', "Run time per second of all virtual processors, as a share of the VM's virtual processors: 100% means every virtual processor ran all the time. Near 100% the VM runs at its own limit. This is busy time on the host's physical processors as Hyper-V sees it, not the utilization Task Manager shows inside the VM: when a virtual processor waits for a physical processor, the guest counts that time as busy, the host doesn't.", 'sum(rate(%s[$__rate_interval])) / %s' % [metric('windows_hyperv_hypervisor_virtual_processor_run_time_seconds_total'), vcpus], 'percentunit', b.levels(0.8, 0.95)),
+
+      b.graph(306, 'CPU usage by state', "Run time per second of the VM's virtual processors, split into running guest code, the hypervisor working for the VM, and remote (on another NUMA node); 100% means every virtual processor ran all the time. This is busy time on the host's physical processors as Hyper-V sees it, not the utilization Task Manager shows inside the VM: when a virtual processor waits for a physical processor, the guest counts that time as busy, the host doesn't.", 'percentunit', [
+        ['sum by (state) (rate(%s[$__rate_interval])) / scalar(%s)' % [metric('windows_hyperv_hypervisor_virtual_processor_mode_time_seconds_total'), vcpus], '{{state}}'],
+      ])
+      + b.panel.withDefaults({ custom: { fillOpacity: 40, stacking: { mode: 'normal' } } }),
+
+      b.graph(307, 'CPU usage per virtual processor', "Share of time each virtual processor spent running: run time per second, so 100% means it ran all the time. One busy virtual processor while the others idle points to a single-threaded workload. This is busy time on the host's physical processors as Hyper-V sees it, not the utilization Task Manager shows inside the VM: when a virtual processor waits for a physical processor, the guest counts that time as busy, the host doesn't.", 'percentunit', [
+        ['sum by (core) (rate(%s[$__rate_interval]))' % metric('windows_hyperv_hypervisor_virtual_processor_run_time_seconds_total'), 'vCPU {{core}}'],
+      ])
+      + b.panel.withDefaults({ max: 1 }),
+
+      b.graph(308, 'RAM in use and assigned', 'In use: memory the guest occupies. Assigned: physical memory from the host. Visible: memory the guest sees (with dynamic memory up to its maximum).', 'bytes', [
+        ['max(%s) - max(%s)' % [metric('windows_hyperv_dynamic_memory_vm_guest_visible_physical_memory_bytes'), metric('windows_hyperv_dynamic_memory_vm_guest_available_bytes')], 'in use'],
+        ['max(%s)' % metric('windows_hyperv_dynamic_memory_vm_physical_bytes'), 'assigned'],
+        ['max(%s)' % metric('windows_hyperv_dynamic_memory_vm_guest_visible_physical_memory_bytes'), 'visible'],
+      ]),
+
+      b.graph(309, 'Committed vs. RAM', 'Hyper-V memory demand (commit charge of the guest) as a share of the assigned memory. Above 100% the guest has committed more than its RAM; that is only a problem when it pages a lot. With dynamic memory, Hyper-V adds memory to keep this near its buffer.', 'percentunit', [
+        ['max(%s)' % metric('windows_hyperv_dynamic_memory_vm_pressure_current_ratio'), 'committed / RAM'],
+      ])
+      + b.panel.withDefaults({ custom: { thresholdsStyle: { mode: 'dashed' } }, thresholds: { steps: b.levels(1, 1.5) } }),
+
+      b.graph(310, 'Dynamic memory changes', 'Memory Hyper-V added to (up) and removed from (down) the VM. Always zero for static memory.', 'Bps', [
+        ['sum(rate(%s[$__rate_interval]))' % metric('windows_hyperv_dynamic_memory_vm_added_total'), 'added'],
+        ['sum(rate(%s[$__rate_interval]))' % metric('windows_hyperv_dynamic_memory_vm_removed_bytes_total'), 'removed'],
+      ], below='removed'),
+
+      b.stat(311, 'Disk size', 'Maximum size of all virtual hard disks of the VM, as the guest sees them. For a VM with checkpoints, the current checkpoint disk counts.', 'sum(%s)' % size, 'bytes'),
+
+      b.stat(312, 'Disk file size', 'Space the virtual hard disk files take on storage. Thin provisioned (dynamic) disks grow up to their size.', 'sum(%s)' % file, 'bytes'),
+
+      b.stat(313, 'Growth if thick', 'Space the dynamic (thin provisioned) disks would still take if every one grew to its full size.', 'sum(%s)' % growth, 'bytes'),
+
+      b.stat(314, 'Disk fill', 'File size of all disks as a share of their size.', 'sum(%s) / sum(%s)' % [file, size], 'percentunit'),
+
+      b.joinedTable(315, 'Virtual hard disks', 'Every virtual hard disk of the VM. Type is fixed (thick), dynamic (thin), differencing (checkpoint or child disk) or unknown. For a checkpoint, the file size only counts the current .avhdx, not its parent disks.', 'path', [
+        'max by (path, type) (%s)' % metric('windows_hyperv_virtual_hard_disk_size_bytes'),
+        file,
+        '%s / %s' % [file, size],
+        growth,
+      ], [
+        ['path', 'Path'],
+        ['type', 'Type'],
+        ['Value #A', 'Size'],
+        ['Value #B', 'File size'],
+        ['Value #C', 'Fill'],
+        ['Value #D', 'Growth if thick'],
+      ], [
+        { matcher: { id: 'byRegexp', options: '^(Size|File size|Growth if thick)$' }, properties: [{ id: 'unit', value: 'bytes' }] },
+        { matcher: { id: 'byName', options: 'Fill' }, properties: [{ id: 'unit', value: 'percentunit' }] },
+      ]),
+
+      b.graph(316, 'Disk IOPS', "Read and write operations per second of each disk; reads are drawn below the axis. Disks of a VM that isn't running have no I/O.", 'iops', [
+        [perDisk('operations_written_total'), '{{disk}} write'],
+        [perDisk('operations_read_total'), '{{disk}} read'],
+      ], below='.* read$'),
+
+      b.graph(317, 'Disk throughput', 'Bytes written and read per second of each disk; reads are drawn below the axis.', 'Bps', [
+        [perDisk('written_bytes_total'), '{{disk}} write'],
+        [perDisk('read_bytes_total'), '{{disk}} read'],
+      ], below='.* read$'),
+
+      b.graph(318, 'Disk latency', 'Average latency per I/O of each disk: total I/O latency divided by completed I/O transfers. The dashed line is the slow-disk threshold.', 's', [
+        [h.diskName(h.latency('sum by (path) (%s)' % h.withVm(h.deviceRate('io_latency_seconds_total', job), vm), 'sum by (path) (%s)' % h.withVm(h.deviceRate('throughput_total', job), vm))), '{{disk}}'],
+      ])
+      + b.panel.withDefaults({ custom: { thresholdsStyle: { mode: 'dashed' } }, thresholds: { steps: [{ color: 'green', value: null }, { color: 'orange', value: config.hypervDiskLatencyThresholdSeconds }] } }),
+
+      b.graph(319, 'Network throughput', 'Traffic of each network adapter of the VM; received traffic is drawn below the axis. Adapters are matched by the VM GUID in their name.', 'bps', [
+        [nicName('sum by (adapter) (%s) * 8' % vnic('windows_hyperv_virtual_network_adapter_sent_bytes_total')), '{{nic}} sent'],
+        [nicName('sum by (adapter) (%s) * 8' % vnic('windows_hyperv_virtual_network_adapter_received_bytes_total')), '{{nic}} received'],
+      ], below='.* received$'),
+
+      b.graph(320, 'Network dropped packets', 'Packets dropped per second by each network adapter of the VM; incoming drops are drawn below the axis.', 'pps', [
+        [nicName('sum by (adapter) (%s)' % vnic('windows_hyperv_virtual_network_adapter_outgoing_dropped_packets_total')), '{{nic}} outgoing'],
+        [nicName('sum by (adapter) (%s)' % vnic('windows_hyperv_virtual_network_adapter_incoming_dropped_packets_total')), '{{nic}} incoming'],
+      ], below='.* incoming$'),
+    ]
+  )

contrib/mixin/dashboards/hyper_v_vm_disks.libsonnet

@@ -0,0 +1,107 @@
+local h = import '../lib/hyperv.libsonnet';
+local b = import 'builders.libsonnet';
+
+// Virtual hard disks of all VMs on the hosts of a failover cluster.
+function(on, config)
+  local hosts = 'job=~"$job", instance=~"$cluster_instance"';
+  local threshold = std.toString(config.hypervDiskLatencyThresholdSeconds);
+  local disk(name, labels='path') = 'max by (%s) (windows_hyperv_virtual_hard_disk_%s{%s})' % [labels, name, hosts];
+  local size = disk('size_bytes');
+  local file = disk('file_size_bytes');
+  // Only dynamic disks grow; a differencing disk's parent already holds most of its size.
+  local growth = 'clamp_min(max by (path) (windows_hyperv_virtual_hard_disk_size_bytes{%s, type="dynamic"}) - %s, 0)' % [hosts, file];
+  local byType(name) = 'sum by (type) (%s)' % disk(name, 'path, type');
+  local deviceLatency = '%s / (%s > 1)' % [h.deviceRate('io_latency_seconds_total', hosts), h.deviceRate('throughput_total', hosts)];
+  local iops = '(%s + %s)' % [h.deviceRate('operations_read_total', hosts), h.deviceRate('operations_written_total', hosts)];
+  local slowest = 'topk(10, count_over_time((%s > %s)[$__range:5m]) * 300)' % [deviceLatency, threshold];
+  // device is only unique per host, so the rows are joined by host and device.
+  local slowRows(expr) = 'label_join(max by (device, disk, vm, instance) (%s), "key", "/", "instance", "device")' % h.diskName(h.withVm(expr, hosts));
+  local slowValues(expr) = 'max by (key) (%s)' % slowRows(expr);
+  local bytes(pattern) = { matcher: { id: 'byRegexp', options: pattern }, properties: [{ id: 'unit', value: 'bytes' }] };
+  b.tab(
+    'VM disks',
+    b.rows([
+      if on('hyperv') then b.summary([380, 381, 382, 383, 384, 385]),
+      if on('hyperv') then b.row('Capacity', b.flow([[386, 12, 7], [387, 12, 7]])),
+      if on('hyperv') then b.row('Virtual hard disks', b.flow([[388, 24, 10]])),
+      if on('hyperv') then b.row('Performance', b.flow([[389, 24, 9], [390, 12, 8], [391, 12, 8]])),
+    ]),
+    [
+      b.stat(380, 'Virtual hard disks', 'Virtual hard disks attached to the VMs of the cluster. Needs the hyperv collector with its virtual_hard_disk sub-collector.', 'count(%s)' % size),
+
+      b.stat(381, 'Size', 'Maximum size of all virtual hard disks, as the guests see them.', 'sum(%s)' % size, 'bytes'),
+
+      b.stat(382, 'File size', 'Space the virtual hard disk files take on storage.', 'sum(%s)' % file, 'bytes'),
+
+      b.stat(383, 'Growth if thick', 'Space the dynamic (thin provisioned) disks would still take if every one grew to its full size. Keep this much free on the Cluster Shared Volumes, or watch it.', 'sum(%s)' % growth, 'bytes'),
+
+      b.stat(384, 'Differencing disks', 'Disks of checkpoints and child disks. Old checkpoints grow and slow down I/O; merge them when they are no longer needed.', 'count(max by (path) (windows_hyperv_virtual_hard_disk_size_bytes{%s, type="differencing"})) or (0 * count(%s))' % [hosts, size]),
+
+      b.stat(385, 'Slow virtual disks', 'Virtual disks with an average latency per I/O above %s s and at least one I/O per second.' % threshold, 'count((%s) > %s) or vector(0)' % [deviceLatency, threshold], 'short', b.levels(1, 5)),
+
+      b.joinedTable(386, 'Capacity by type', 'Disks, size and file size per provisioning type: fixed (thick), dynamic (thin) and differencing (checkpoint or child disk).', 'type', [
+        'count by (type) (%s)' % disk('size_bytes', 'path, type'),
+        byType('size_bytes'),
+        byType('file_size_bytes'),
+      ], [
+        ['type', 'Type'],
+        ['Value #A', 'Disks'],
+        ['Value #B', 'Size'],
+        ['Value #C', 'File size'],
+      ], [bytes('^(Size|File size)$')]),
+
+      b.graph(387, 'File size by type', 'Space the virtual hard disk files take on storage, per provisioning type. Dynamic and differencing disks grow with the data the guests write.', 'bytes', [
+        [byType('file_size_bytes'), '{{type}}'],
+      ])
+      + b.panel.withDefaults({ custom: { fillOpacity: 40, stacking: { mode: 'normal' } } }),
+
+      b.joinedTable(388, 'Virtual hard disks', 'Every virtual hard disk of the cluster with its VM, host and type. For a checkpoint, the file size only counts the current .avhdx, not its parent disks.', 'path', [
+        disk('size_bytes', 'path, vm, type, instance'),
+        file,
+        '%s / %s' % [file, size],
+        growth,
+      ], [
+        ['vm', 'VM'],
+        ['path', 'Path'],
+        ['type', 'Type'],
+        ['instance', 'Host'],
+        ['Value #A', 'Size'],
+        ['Value #B', 'File size'],
+        ['Value #C', 'Fill'],
+        ['Value #D', 'Growth if thick'],
+      ], [
+        bytes('^(Size|File size|Growth if thick)$'),
+        { matcher: { id: 'byName', options: 'Fill' }, properties: [{ id: 'unit', value: 'percentunit' }] },
+      ]),
+
+      b.joinedTable(389, 'Slowest virtual disks', 'Virtual disks of the cluster that were above %s s average latency per I/O in the time range, worst first. Time above threshold is counted in 5-minute steps, so permanently slow disks rank above short spikes. Disks with less than one I/O per second are ignored.' % threshold, 'key', [
+        slowRows(slowest),
+        slowValues('avg_over_time((%s)[$__range:5m]) and on (instance, device) %s' % [deviceLatency, slowest]),
+        slowValues('max_over_time((%s)[$__range:5m]) and on (instance, device) %s' % [deviceLatency, slowest]),
+        slowValues('avg_over_time((%s)[$__range:5m]) and on (instance, device) %s' % [iops, slowest]),
+      ], [
+        ['disk', 'Disk'],
+        ['vm', 'VM'],
+        ['instance', 'Host'],
+        ['Value #A', 'Time above threshold'],
+        ['Value #B', 'Average latency'],
+        ['Value #C', 'Maximum latency'],
+        ['Value #D', 'IOPS'],
+      ], [
+        { matcher: { id: 'byRegexp', options: '^(key|device)$' }, properties: [{ id: 'custom.hidden', value: true }] },
+        { matcher: { id: 'byName', options: 'Time above threshold' }, properties: [{ id: 'unit', value: 's' }, { id: 'custom.cellOptions', value: { type: 'color-background', mode: 'basic' } }, { id: 'thresholds', value: { mode: 'absolute', steps: b.levels(900, 3600) } }] },
+        { matcher: { id: 'byRegexp', options: '^(Average|Maximum) latency$' }, properties: [{ id: 'unit', value: 's' }] },
+        { matcher: { id: 'byName', options: 'IOPS' }, properties: [{ id: 'unit', value: 'iops' }] },
+      ])
+      + b.panel.withOptions({ sortBy: [{ desc: true, displayName: 'Time above threshold' }] }),
+
+      b.graph(390, 'Busiest virtual disks', 'The ten virtual disks with the most read and write operations per second.', 'iops', [
+        [h.diskName('topk(10, sum by (vm, path) (%s))' % h.withVm(iops, hosts)), '{{vm}} {{disk}}'],
+      ]),
+
+      b.graph(391, 'Highest disk latency', 'The ten virtual disks with the highest average latency per I/O and at least one I/O per second. The dashed line is the slow-disk threshold.', 's', [
+        [h.diskName('topk(10, max by (vm, path) (%s))' % h.withVm(deviceLatency, hosts)), '{{vm}} {{disk}}'],
+      ])
+      + b.panel.withDefaults({ custom: { thresholdsStyle: { mode: 'dashed' } }, thresholds: { steps: [{ color: 'green', value: null }, { color: 'orange', value: config.hypervDiskLatencyThresholdSeconds }] } }),
+    ]
+  )

contrib/mixin/dashboards/mscluster.libsonnet

@@ -0,0 +1,131 @@
+local b = import 'builders.libsonnet';
+local csv = import 'cluster_shared_volumes.libsonnet';
+
+// Failover cluster state as the nodes see it. Every node reports the whole
+// cluster, so selector can pick one node or all nodes of a cluster; the queries
+// keep one series per node, group, resource and network.
+function(on, selector='job=~"$job", instance="$instance"', title='Failover Cluster', withVolumes=true)
+  local metric(name) = 'windows_mscluster_%s{%s}' % [name, selector];
+  local by(labels, name) = 'max by (%s) (%s)' % [labels, metric(name)];
+  local nodeState = by('node', 'node_state');
+  local groupState = by('resource_group', 'resourcegroup_state');
+  local resourceState = by('resource, type, resource_group', 'resource_state');
+  local networkState = by('network', 'network_state');
+  // Count of the series of expr, 0 while the collector reports anything.
+  local count(expr, all) = 'count(%s) or (0 * count(%s))' % [expr, all];
+  local mapping(states) = [{
+    type: 'value',
+    options: {
+      [states[i][0]]: { index: i, text: states[i][1], color: states[i][2] }
+      for i in std.range(0, std.length(states) - 1)
+    },
+  }];
+  local nodeStates = mapping([['-1', 'Unknown', 'text'], ['0', 'Up', 'green'], ['1', 'Down', 'red'], ['2', 'Paused', 'orange'], ['3', 'Joining', 'blue']]);
+  local groupStates = mapping([['-1', 'Unknown', 'text'], ['0', 'Online', 'green'], ['1', 'Offline', 'text'], ['2', 'Failed', 'red'], ['3', 'Partial online', 'orange'], ['4', 'Pending', 'blue']]);
+  local resourceStates = mapping([['-1', 'Unknown', 'text'], ['0', 'Inherited', 'text'], ['1', 'Initializing', 'blue'], ['2', 'Online', 'green'], ['3', 'Offline', 'text'], ['4', 'Failed', 'red'], ['128', 'Pending', 'blue'], ['129', 'Online pending', 'blue'], ['130', 'Offline pending', 'blue']]);
+  local networkStates = mapping([['-1', 'Unknown', 'text'], ['0', 'Unavailable', 'red'], ['1', 'Down', 'red'], ['2', 'Partitioned', 'orange'], ['3', 'Up', 'green']]);
+  local drainStates = mapping([['0', 'Not initiated', 'text'], ['1', 'In progress', 'blue'], ['2', 'Completed', 'text'], ['3', 'Failed', 'red']]);
+  local roles = mapping([['0', 'None', 'text'], ['1', 'Cluster', 'text'], ['2', 'Client', 'text'], ['3', 'Cluster and client', 'text']]);
+  local column(name, mappings) = {
+    matcher: { id: 'byName', options: name },
+    properties: [
+      { id: 'mappings', value: mappings },
+      { id: 'custom.cellOptions', value: { type: 'color-text' } },
+    ],
+  };
+  // The node that owns each resource group or resource.
+  local owner(labels, name) = 'max by (%s, node) (%s == 1)' % [labels, metric(name)];
+  b.tab(
+    title,
+    b.rows([
+      if on('mscluster') then b.summary([330, 331, 332, 333, 334, 335, 336]),
+      if on('mscluster') then b.row('Nodes', b.flow([[337, 24, 7], [338, 12, 8], [339, 12, 8]])),
+      if on('mscluster') then b.row('Resource groups and resources', b.flow([[340, 12, 9], [341, 12, 9]])),
+      if on('mscluster') then b.row('Networks', b.flow([[342, 24, 6]])),
+      if on('mscluster') && withVolumes then b.row('Cluster Shared Volumes', b.flow([[343, 12, 8], [344, 12, 8]])),
+    ]),
+    [
+      b.stat(330, 'Nodes up', 'Cluster nodes in state Up. Requires the mscluster collector, which is not enabled by default.', count('%s == 0' % nodeState, nodeState)),
+
+      b.stat(331, 'Nodes not up', 'Cluster nodes that are down, paused or joining. A paused node runs no new roles, for example during maintenance.', count('%s != 0' % nodeState, nodeState), steps=b.levels(1, 2)),
+
+      b.stat(332, 'Groups failed', 'Resource groups (cluster roles, for example VMs) that failed or are only partially online.', count('%s >= 2 < 4' % groupState, groupState), steps=[{ color: 'green', value: null }, { color: 'red', value: 1 }]),
+
+      b.stat(333, 'Groups offline', 'Resource groups that are offline, for example stopped VMs. Offline is often intended.', count('%s == 1' % groupState, groupState)),
+
+      b.stat(334, 'Resources failed', 'Cluster resources in state Failed.', count('%s == 4' % resourceState, resourceState), steps=[{ color: 'green', value: null }, { color: 'red', value: 1 }]),
+
+      b.stat(335, 'Networks not up', 'Cluster networks that are unavailable, down or partitioned. A partitioned network has lost connectivity between some nodes.', count('%s != 3' % networkState, networkState), steps=[{ color: 'green', value: null }, { color: 'red', value: 1 }]),
+
+      b.stat(336, 'Quorum', 'Quorum type of the cluster: node majority, file share witness or disk (storage) witness.', 'max(%s)' % metric('cluster_quorum_type_value'))
+      + b.panel.withDefaults({ mappings: mapping([['-1', 'Unknown', 'text'], ['1', 'Node majority', 'text'], ['2', 'File share witness', 'text'], ['3', 'Disk witness', 'text'], ['4', 'None', 'text']]) }),
+
+      b.joinedTable(337, 'Nodes', 'State, drain status, current vote, owned resource groups and build of each cluster node. Dynamic quorum removes the vote of some nodes to keep quorum when nodes fail.', 'node', [
+        nodeState,
+        by('node', 'node_node_drain_status'),
+        by('node', 'node_dynamic_weight'),
+        'sum by (node) (%s)' % by('resource_group, node', 'resourcegroup_owner_node'),
+        by('node', 'node_build_number'),
+      ], [
+        ['node', 'Node'],
+        ['Value #A', 'State'],
+        ['Value #B', 'Drain'],
+        ['Value #C', 'Vote'],
+        ['Value #D', 'Resource groups'],
+        ['Value #E', 'Build'],
+      ], [
+        column('State', nodeStates),
+        column('Drain', drainStates),
+      ]),
+
+      b.panel.new(338, 'Node state', 'state-timeline')
+      + b.panel.withDescription('State of each cluster node over time.')
+      + b.panel.withQueries([b.query.new(nodeState) + b.query.withLegendFormat('{{node}}')])
+      + b.panel.withDefaults({ mappings: nodeStates, color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'text', value: null }] } }),
+
+      b.graph(339, 'Resource groups per node', 'Resource groups each node owns. Steps show failovers and live migrations; a node without groups runs no roles.', 'short', [
+        ['sum by (node) (%s)' % by('resource_group, node', 'resourcegroup_owner_node'), '{{node}}'],
+      ]),
+
+      b.joinedTable(340, 'Resource groups not online', 'Resource groups that are not online and the node that owns them. Offline groups of stopped VMs are listed too.', 'resource_group', [
+        '%s != 0' % groupState,
+        '%s and on (resource_group) (%s != 0)' % [owner('resource_group', 'resourcegroup_owner_node'), groupState],
+      ], [
+        ['resource_group', 'Resource group'],
+        ['Value #A', 'State'],
+        ['node', 'Node'],
+      ], [
+        column('State', groupStates),
+      ]),
+
+      b.joinedTable(341, 'Resources not online', 'Cluster resources that are not online, with their type, resource group and owner node.', 'resource', [
+        '%s != 2' % resourceState,
+        '%s and on (resource) (%s != 2)' % [owner('resource', 'resource_owner_node'), resourceState],
+      ], [
+        ['resource', 'Resource'],
+        ['type', 'Type'],
+        ['resource_group', 'Resource group'],
+        ['Value #A', 'State'],
+        ['node', 'Node'],
+      ], [
+        column('State', resourceStates),
+      ]),
+
+      b.joinedTable(342, 'Networks', 'State, role and metric of each cluster network. The cluster prefers networks with a lower metric for cluster traffic.', 'network', [
+        networkState,
+        by('network', 'network_role'),
+        by('network', 'network_metric'),
+      ], [
+        ['network', 'Network'],
+        ['Value #A', 'State'],
+        ['Value #B', 'Role'],
+        ['Value #C', 'Metric'],
+      ], [
+        column('State', networkStates),
+        column('Role', roles),
+      ]),
+
+      csv.table(343, selector),
+      csv.usage(344, selector),
+    ]
+  )

contrib/mixin/dashboards/variables.libsonnet

@@ -22,8 +22,38 @@ local new(name, definition) = common
                                 refId: 'PrometheusVariableQueryEditor-VariableQuery',
                               }));
 
-// Variables of disabled collectors are left out.
-function(on) [variable for variable in [
+local all = query.spec.withCurrent({
+              text: [
+                'All',
+              ],
+              value: [
+                '$__all',
+              ],
+            })
+            + query.spec.withIncludeAll(true)
+            + query.spec.withMulti(true);
+local job = new('job', 'label_values(windows_os_hostname, job)')
+            + all
+            + query.spec.withLabel('Job');
+
+// The Hyper-V cluster dashboard selects a failover cluster by its name, which
+// the mscluster collector reports, and the hosts that are nodes of it.
+local cluster = [
+  new('cluster', 'label_values(windows_mscluster_cluster_quorum_type_value{job=~"$job"}, cluster)')
+  + query.spec.withLabel('Cluster'),
+  new('cluster_instance', 'label_values(windows_mscluster_cluster_quorum_type_value{job=~"$job", cluster="$cluster"}, instance)')
+  + all
+  + query.spec.withHide('hideVariable')
+  + query.spec.withLabel('Cluster hosts'),
+  new('instance', 'label_values(windows_mscluster_cluster_quorum_type_value{job=~"$job", cluster="$cluster"}, instance)')
+  + query.spec.withLabel('Host'),
+  new('vm', 'label_values(windows_hyperv_vm_processors{job=~"$job", instance=~"$cluster_instance"}, vm)')
+  + query.spec.withLabel('VM'),
+];
+
+// Variables of disabled collectors are left out. With hypervCluster, the
+// variables of the Hyper-V cluster dashboard.
+function(on, hypervCluster=false) [variable for variable in [
   datasource.withKind()
   + datasource.spec.withAllowCustomValue(true)
   + datasource.spec.withCurrent({
@@ -41,36 +71,52 @@ function(on) [variable for variable in [
   + datasource.spec.withRegex('')
   + datasource.spec.withSkipUrlSync(false)
   ,
-  new('job', 'label_values(windows_os_hostname, job)')
-  + query.spec.withCurrent({
-    text: [
-      'All',
-    ],
-    value: [
-      '$__all',
-    ],
-  })
-  + query.spec.withIncludeAll(true)
-  + query.spec.withLabel('Job')
-  + query.spec.withMulti(true)
-  ,
-  new('hostname', 'label_values(windows_os_hostname{job=~"$job"}, hostname)')
-  + query.spec.withCurrent({
-    text: [
-      'All',
-    ],
-    value: [
-      '$__all',
-    ],
-  })
-  + query.spec.withIncludeAll(true)
-  + query.spec.withLabel('Hostname')
-  + query.spec.withMulti(true)
-  ,
-  new('instance', 'label_values(windows_os_hostname{job=~"$job", hostname=~"$hostname"}, instance)')
-  + query.spec.withLabel('Instance')
-  ,
-  if on('logical_disk') then new('volume', 'label_values(windows_logical_disk_size_bytes{job=~"$job", instance="$instance"}, volume)')
+  job,
+] + (if hypervCluster then cluster else [
+       new('hostname', 'label_values(windows_os_hostname{job=~"$job"}, hostname)')
+       + query.spec.withCurrent({
+         text: [
+           'All',
+         ],
+         value: [
+           '$__all',
+         ],
+       })
+       + query.spec.withIncludeAll(true)
+       + query.spec.withLabel('Hostname')
+       + query.spec.withMulti(true)
+       ,
+       new('instance', 'label_values(windows_os_hostname{job=~"$job", hostname=~"$hostname"}, instance)')
+       + query.spec.withLabel('Instance')
+       ,
+       if on('logical_disk') then new('volume', 'label_values(windows_logical_disk_size_bytes{job=~"$job", instance="$instance"}, volume)')
+                                  + query.spec.withCurrent({
+                                    text: [
+                                      'All',
+                                    ],
+                                    value: [
+                                      '$__all',
+                                    ],
+                                  })
+                                  + query.spec.withIncludeAll(true)
+                                  + query.spec.withLabel('Volume')
+                                  + query.spec.withMulti(true)
+                                  + query.spec.withRegex('/^(?!HarddiskVolume).+$/')
+       ,
+       if on('net') then new('nic', 'label_values(windows_net_bytes_total{job=~"$job", instance="$instance"}, nic)')
+                         + query.spec.withCurrent({
+                           text: [
+                             'All',
+                           ],
+                           value: [
+                             '$__all',
+                           ],
+                         })
+                         + query.spec.withIncludeAll(true)
+                         + query.spec.withLabel('Network interface')
+                         + query.spec.withMulti(true)
+                         + query.spec.withRegex('/^(?!isatap|Teredo|6to4).+$/'),
+       if on('process') then new('process', 'label_values(windows_process_info{job=~"$job", instance="$instance", process_id!="0"}, process)')
                              + query.spec.withCurrent({
                                text: [
                                  'All',
@@ -80,61 +126,34 @@ function(on) [variable for variable in [
                                ],
                              })
                              + query.spec.withIncludeAll(true)
-                             + query.spec.withLabel('Volume')
-                             + query.spec.withMulti(true)
-                             + query.spec.withRegex('/^(?!HarddiskVolume).+$/')
-  ,
-  if on('net') then new('nic', 'label_values(windows_net_bytes_total{job=~"$job", instance="$instance"}, nic)')
-                    + query.spec.withCurrent({
-                      text: [
-                        'All',
-                      ],
-                      value: [
-                        '$__all',
-                      ],
-                    })
-                    + query.spec.withIncludeAll(true)
-                    + query.spec.withLabel('Network interface')
-                    + query.spec.withMulti(true)
-                    + query.spec.withRegex('/^(?!isatap|Teredo|6to4).+$/'),
-  if on('process') then new('process', 'label_values(windows_process_info{job=~"$job", instance="$instance", process_id!="0"}, process)')
-                        + query.spec.withCurrent({
-                          text: [
-                            'All',
-                          ],
-                          value: [
-                            '$__all',
-                          ],
-                        })
-                        + query.spec.withIncludeAll(true)
-                        + query.spec.withLabel('Process')
-                        + query.spec.withMulti(true),
-  if on('process') then variables.CustomVariableKind.withKind()
-                        + variables.CustomVariableKind.withSpec({
-                          name: 'process_top',
-                          label: 'Top N processes',
-                          description: 'Maximum number of processes per graph. Summary counters include all selected processes.',
-                          query: '5,10,20',
-                          current: { text: '10', value: '10' },
-                          options: [{ text: value, value: value, selected: value == '10' } for value in ['5', '10', '20']],
-                          multi: false,
-                          includeAll: false,
-                          allowCustomValue: false,
-                          hide: 'dontHide',
-                          skipUrlSync: false,
-                        }),
-  if on('mssql') then new('mssql_instance', 'label_values(windows_mssql_collector_success{job=~"$job", instance="$instance"}, mssql_instance)')
-                      + query.spec.withLabel('SQL Server instance'),
-  if on('mssql') then new('database', 'label_values(windows_mssql_databases_data_files_size_bytes{job=~"$job", instance="$instance", mssql_instance="$mssql_instance"}, database)')
-                      + query.spec.withCurrent({
-                        text: [
-                          'All',
-                        ],
-                        value: [
-                          '$__all',
-                        ],
-                      })
-                      + query.spec.withIncludeAll(true)
-                      + query.spec.withLabel('Database')
-                      + query.spec.withMulti(true),
-] if variable != null]
+                             + query.spec.withLabel('Process')
+                             + query.spec.withMulti(true),
+       if on('process') then variables.CustomVariableKind.withKind()
+                             + variables.CustomVariableKind.withSpec({
+                               name: 'process_top',
+                               label: 'Top N processes',
+                               description: 'Maximum number of processes per graph. Summary counters include all selected processes.',
+                               query: '5,10,20',
+                               current: { text: '10', value: '10' },
+                               options: [{ text: value, value: value, selected: value == '10' } for value in ['5', '10', '20']],
+                               multi: false,
+                               includeAll: false,
+                               allowCustomValue: false,
+                               hide: 'dontHide',
+                               skipUrlSync: false,
+                             }),
+       if on('mssql') then new('mssql_instance', 'label_values(windows_mssql_collector_success{job=~"$job", instance="$instance"}, mssql_instance)')
+                           + query.spec.withLabel('SQL Server instance'),
+       if on('mssql') then new('database', 'label_values(windows_mssql_databases_data_files_size_bytes{job=~"$job", instance="$instance", mssql_instance="$mssql_instance"}, database)')
+                           + query.spec.withCurrent({
+                             text: [
+                               'All',
+                             ],
+                             value: [
+                               '$__all',
+                             ],
+                           })
+                           + query.spec.withIncludeAll(true)
+                           + query.spec.withLabel('Database')
+                           + query.spec.withMulti(true),
+     ]) if variable != null]

contrib/mixin/dashboards/windows-exporter.libsonnet

@@ -16,6 +16,7 @@ local descriptions = {
   Updates: 'Pending Windows updates and the last update check of one Windows host.',
   GPU: 'GPU utilization, memory, sensors and top processes of one Windows host.',
   'Hyper-V': 'Hyper-V virtual machines, virtual disks, switches and network adapters of one Windows host.',
+  'Failover Cluster': 'Nodes, resource groups, resources, networks and Cluster Shared Volumes of the failover cluster of one Windows host.',
   Time: 'Clock synchronization, NTP sources and clock offset of one Windows host.',
   MSSQL: 'Activity, waits, locks, memory, databases and availability replicas of one SQL Server instance on a Windows host.',
   Exporter: 'Scrape status, collector health and resource usage of windows_exporter on one host.',
@@ -75,7 +76,8 @@ function(config)
       (import 'scheduled_tasks.libsonnet')(on),
       (import 'updates.libsonnet')(on),
       (import 'gpu.libsonnet')(on),
-      (import 'hyper_v.libsonnet')(on),
+      (import 'hyper_v.libsonnet')(on, config),
+      (import 'mscluster.libsonnet')(on),
       (import 'time.libsonnet')(on),
       (import 'mssql.libsonnet')(on),
       (import 'exporter.libsonnet')(on),
@@ -88,11 +90,29 @@ function(config)
   assert std.length(panelNames) == std.length(std.set(panelNames)) : 'Panel IDs must be unique across tabs';
 
   local variables = (import 'variables.libsonnet')(on);
+  local hypervClusterVariables = (import 'variables.libsonnet')(on, hypervCluster=true);
+
+  // The Hyper-V cluster dashboard: the hosts of one failover cluster, its
+  // VMs and their storage. Its panel IDs are unique within the dashboard.
+  local hypervClusterTabs = [
+    tab
+    for tab in [
+      (import 'hyper_v_cluster_overview.libsonnet')(on, config),
+      (import 'mscluster.libsonnet')(on, 'job=~"$job", instance=~"$cluster_instance"', 'Cluster', withVolumes=false),
+      (import 'hyper_v.libsonnet')(on, config) + { layout+: { spec+: { title: 'Host' } } },
+      (import 'hyper_v_vm.libsonnet')(on, config),
+      (import 'hyper_v_vm_disks.libsonnet')(on, config),
+      (import 'hyper_v_storage.libsonnet')(on),
+    ]
+    if !tab.empty
+  ];
+  local hypervClusterPanels = std.flattenArrays([std.objectFields(tab.elements) for tab in hypervClusterTabs]);
+  assert std.length(hypervClusterPanels) == std.length(std.set(hypervClusterPanels)) : 'Panel IDs must be unique across the Hyper-V cluster tabs';
   local annotations = (import 'annotations.libsonnet')(on);
 
   // Tab dashboards link to each other through a dropdown of the dashboards
   // with the same tags; the full dashboard has the tabs instead.
-  local new(uid, title, description, elements, layout, dropdown=true) =
+  local new(uid, title, description, elements, layout, dropdown=true, dashboardVariables=variables) =
     // Grafana rejects longer dashboard UIDs.
     assert std.length(uid) <= 40 : 'Dashboard UID ' + uid + ' is longer than 40 characters; shorten dashboardUIDPrefix or dashboardUID';
     // Keeps the variables that the panels reference, directly or through
@@ -103,10 +123,10 @@ function(config)
         if std.setMember(variable.spec.name, used)
         then std.setUnion(used, references(variable))
         else used,
-      std.reverse(variables),
+      std.reverse(dashboardVariables),
       references(elements),
     );
-    local dropped = std.set([variable.spec.name for variable in variables if !std.setMember(variable.spec.name, kept)]);
+    local dropped = std.set([variable.spec.name for variable in dashboardVariables if !std.setMember(variable.spec.name, kept)]);
     dashboard.new(uid, title)
     + dashboard.spec.withCursorSync('Crosshair')
     + dashboard.spec.withDescription(description)
@@ -168,7 +188,7 @@ function(config)
       for annotation in annotations
       if std.length(std.setInter(references(annotation), dropped)) == 0
     ])
-    + dashboard.spec.withVariables([variable for variable in variables if std.setMember(variable.spec.name, kept)])
+    + dashboard.spec.withVariables([variable for variable in dashboardVariables if std.setMember(variable.spec.name, kept)])
     + dashboard.spec.withElements(elements)
     + dashboard.spec.withLayout(layout);
 
@@ -191,5 +211,16 @@ function(config)
       )
       for tab in allTabs(if config.enableOverviewTab then uid('Overview'))
       if config.enableTabDashboards || separate(tab)
-    },
+    } + (
+      if on('hyperv') && on('mscluster') then {
+        'windows-exporter-hyper-v-cluster.json': new(
+          uid('Hyper-V Cluster'),
+          config.dashboardTitlePrefix + ' / Hyper-V Cluster',
+          'Hosts, virtual machines, virtual hard disks and storage of a Hyper-V failover cluster.',
+          std.foldl(function(elements, tab) elements + tab.elements, hypervClusterTabs, {}),
+          b.tabs([tab.layout for tab in hypervClusterTabs]),
+          dashboardVariables=hypervClusterVariables,
+        ),
+      } else {}
+    ),
   }

contrib/mixin/lib/hyperv.libsonnet

@@ -0,0 +1,35 @@
+// PromQL building blocks shared by the Hyper-V tabs. Selectors are label
+// matchers without braces; every function returns a PromQL string.
+{
+  // Right-hand side of a join that adds vm and path to series of
+  // windows_hyperv_virtual_storage_device_*. The virtual_hard_disk metrics carry
+  // the counter instance name of each running disk in device. Several disks can
+  // share one device (a VHD-format checkpoint uses the name of its base disk), so
+  // one is kept per device. Turned into 1, so the join keeps the left values.
+  vhdVm(selector):
+    '(0 * topk by (instance, device) (1, max by (instance, device, vm, path) (windows_hyperv_virtual_hard_disk_size_bytes{%s, device!=""})) + 1)' % selector,
+
+  // expr, a series of windows_hyperv_virtual_storage_device_*, with the vm and
+  // path of its disk. Disks of other VMs drop out when vmSelector selects a VM.
+  withVm(expr, vmSelector):
+    '%s * on (instance, device) group_left (vm, path) %s' % [expr, self.vhdVm(vmSelector)],
+
+  // Per-second rate of a windows_hyperv_virtual_storage_device_* counter.
+  deviceRate(metric, selector):
+    'rate(windows_hyperv_virtual_storage_device_%s{%s}[$__rate_interval])' % [metric, selector],
+
+  // numerator per unit of denominator; idle devices drop out instead of
+  // dividing by zero. Latency per I/O is
+  // latency(rate(io_latency_seconds_total), rate(throughput_total)).
+  latency(numerator, denominator): '%s / (%s > 0)' % [numerator, denominator],
+
+  // Adds disk, the file name at the end of path, for legends and tables.
+  diskName(expr):
+    'label_replace(%s, "disk", "$1", "path", ".*[\\\\\\\\/]([^\\\\\\\\/]+)$")' % expr,
+
+  // Hyper-V names the vNIC of a VM <vm>_<adapter>_<vm_id>--<adapter GUID>. Adds
+  // vm_id from the name and vm from windows_hyperv_vm_processors, so vNICs
+  // match their VM by GUID and not by a name prefix.
+  vnicVm(expr, vmSelector):
+    'label_replace(%s, "vm_id", "$1", "adapter", ".*_([0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12})--.*") * on (instance, vm_id) group_left (vm) (0 * max by (instance, vm_id, vm) (windows_hyperv_vm_processors{%s}) + 1)' % [expr, vmSelector],
+}

contrib/mixin/mise.toml

@@ -102,6 +102,7 @@ description = "Run all mixin lint, runbook, and rule tests"
 depends = ["lint", "check-runbooks", "vendor"]
 run = [
   "jsonnet tests/runbooks.test.jsonnet",
+  "jsonnet tests/hyperv.test.jsonnet > /dev/null",
   "jsonnet -J vendor tests/dashboard.test.jsonnet > /dev/null",
   "promtool test rules tests/*.test.yaml",
 ]

contrib/mixin/tests/dashboard.test.jsonnet

@@ -35,6 +35,8 @@ local results = {
   diskWithoutDiskdrive: summary(render(only({ logical_disk: true, physical_disk: true }))).Disk,
   diskWithDiskdrive: summary(render(only({ logical_disk: true, physical_disk: true, diskdrive: true }))).Disk,
   storageSpacesOnly: summary(render(only({ storage_spaces: true }))).Disk,
+  hypervOnly: summary(render(only({ hyperv: true }))),
+  msclusterOnly: summary(render(only({ mscluster: true }))),
   // A collectors object given without +: enables only the listed collectors.
   replaced: summary(render(function(_) { os: true, cpu: true })),
   // Only collector tabs, for a compact dashboard.
@@ -109,7 +111,7 @@ assert custom['windows-exporter-cpu.json'].spec.title == 'Windows / CPU' : 'cust
 assert custom['windows-exporter.json'].metadata.name == 'win-full' : 'custom prefix: got full UID ' + custom['windows-exporter.json'].metadata.name;
 assert std.objectFields(dashboards(function(collectors) collectors, { enableTabDashboards: false })) == ['windows-exporter.json'] : 'enableTabDashboards: false renders tab dashboards';
 
-assert std.length(std.objectFields(results.all)) == 15 : 'all: got ' + std.join(', ', std.objectFields(results.all));
+assert std.length(std.objectFields(results.all)) == 16 : 'all: got ' + std.join(', ', std.objectFields(results.all));
 // MSSQL renders only as a dashboard of its own unless separateDashboardTabs drops it.
 local withMSSQL = dashboards(only({ mssql: true }));
 assert !std.objectHas(results.all, 'MSSQL') : 'all: the full dashboard has the MSSQL tab';
@@ -145,4 +147,40 @@ assert expect('collectorTabsOnly', {
   Exporter: exporter,
 });
 
+assert expect('hypervOnly', {
+  Exporter: exporter,
+  Fleet: ['Summary', 'Hosts'],
+  'Hyper-V': ['Summary', 'Virtual machines', 'CPU', 'Memory', 'Storage', 'Network'],
+  Overview: ['Summary'],
+});
+
+local hypervFull = render(only({ hyperv: true }));
+local hypervMemory = render(only({ hyperv: true, memory: true }));
+assert !std.objectHas(hypervFull.spec.elements, 'panel-290') : 'hyperv without memory: VM memory / host RAM stat rendered';
+assert std.objectHas(hypervMemory.spec.elements, 'panel-290') : 'hyperv with memory: VM memory / host RAM stat missing';
+
+assert std.all([std.objectHas(hypervFull.spec.elements, 'panel-' + id) for id in [293, 294, 295, 296, 297, 298]]) : 'hyperv: Hyper-V tab additions missing';
+
+assert expect('msclusterOnly', {
+  Exporter: exporter,
+  'Failover Cluster': ['Summary', 'Nodes', 'Resource groups and resources', 'Networks', 'Cluster Shared Volumes'],
+  Fleet: ['Summary', 'Hosts'],
+  Overview: ['Summary'],
+});
+
+// The Hyper-V cluster dashboard needs the hyperv and mscluster collectors.
+assert !std.objectHas(dashboards(only({ hyperv: true })), 'windows-exporter-hyper-v-cluster.json') : 'hyperv without mscluster: got a Hyper-V cluster dashboard';
+local hypervCluster = dashboards(only({ hyperv: true, mscluster: true, memory: true, storage_spaces: true }))['windows-exporter-hyper-v-cluster.json'];
+assert hypervCluster.metadata.name == 'windows-exporter-hyper-v-cluster' : 'hyper-v cluster: got UID ' + hypervCluster.metadata.name;
+assert hypervCluster.spec.title == 'Windows Exporter / Hyper-V Cluster' : 'hyper-v cluster: got title ' + hypervCluster.spec.title;
+assert summary(hypervCluster) == {
+  Overview: ['Summary', 'Hosts', 'Load'],
+  Cluster: ['Summary', 'Nodes', 'Resource groups and resources', 'Networks'],
+  Host: ['Summary', 'Virtual machines', 'CPU', 'Memory', 'Storage', 'Network'],
+  VM: ['Summary', 'CPU', 'Memory', 'Disks', 'Network'],
+  'VM disks': ['Summary', 'Capacity', 'Virtual hard disks', 'Performance'],
+  Storage: ['Summary', 'Cluster Shared Volumes', 'Storage Spaces Direct'],
+} : 'hyper-v cluster: got ' + std.manifestJson(summary(hypervCluster));
+assert variableNames(hypervCluster) == ['datasource', 'job', 'cluster', 'cluster_instance', 'instance', 'vm'] : 'hyper-v cluster: got variables ' + std.join(', ', variableNames(hypervCluster));
+
 results

contrib/mixin/tests/hyperv.test.jsonnet

@@ -0,0 +1,37 @@
+// Checks the PromQL the Hyper-V helpers build. Run with jsonnet tests/hyperv.test.jsonnet.
+local h = import '../lib/hyperv.libsonnet';
+local config = (import '../config.libsonnet')._config;
+
+local eq(name, got, want) =
+  assert got == want : name + ':\n got  ' + got + '\n want ' + want;
+  true;
+
+std.all([
+  eq(
+    'vhdVm',
+    h.vhdVm('job=~"$job", vm="$vm"'),
+    '(0 * topk by (instance, device) (1, max by (instance, device, vm, path) (windows_hyperv_virtual_hard_disk_size_bytes{job=~"$job", vm="$vm", device!=""})) + 1)'
+  ),
+  eq(
+    'withVm',
+    h.withVm('rate(x[1m])', 'vm="a"'),
+    'rate(x[1m]) * on (instance, device) group_left (vm, path) (0 * topk by (instance, device) (1, max by (instance, device, vm, path) (windows_hyperv_virtual_hard_disk_size_bytes{vm="a", device!=""})) + 1)'
+  ),
+  eq(
+    'deviceRate',
+    h.deviceRate('throughput_total', 'job=~"$job"'),
+    'rate(windows_hyperv_virtual_storage_device_throughput_total{job=~"$job"}[$__rate_interval])'
+  ),
+  eq('latency', h.latency('a', 'b'), 'a / (b > 0)'),
+  eq(
+    'diskName',
+    h.diskName('x'),
+    'label_replace(x, "disk", "$1", "path", ".*[\\\\\\\\/]([^\\\\\\\\/]+)$")'
+  ),
+  eq(
+    'vnicVm',
+    h.vnicVm('rate(y[1m])', 'vm="a"'),
+    'label_replace(rate(y[1m]), "vm_id", "$1", "adapter", ".*_([0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12})--.*") * on (instance, vm_id) group_left (vm) (0 * max by (instance, vm_id, vm) (windows_hyperv_vm_processors{vm="a"}) + 1)'
+  ),
+  eq('threshold', std.toString(config.hypervDiskLatencyThresholdSeconds), '0.01'),
+])

dashboard/README.md

@@ -9,7 +9,7 @@ and takes several ideas from Grafana's [windows-observ-lib](https://github.com/g
 
 - Grafana 13 or later with a Prometheus data source. The file uses the v2 dashboard schema, which Grafana needs for tabs.
 - windows_exporter with the default collectors (`cpu`, `logical_disk`, `memory`, `net`, `os`, `physical_disk`, `service`, `system`).
-  The SMB, Processes, Scheduled tasks, Updates, GPU, Hyper-V and Time tabs and the MSSQL dashboard, the TCP and UDP rows of the Network tab and the Drives and Storage Spaces rows of the Disk tab need the `smb` or `smbclient`, `process`, `scheduled_task`, `update`, `gpu`, `hyperv`, `time`, `mssql`, `tcp`, `udp`, `diskdrive` and `storage_spaces` collectors, which are not enabled by default.
+  The SMB, Processes, Scheduled tasks, Updates, GPU, Hyper-V, Failover Cluster and Time tabs, the MSSQL and Hyper-V Cluster dashboards, the TCP and UDP rows of the Network tab and the Drives and Storage Spaces rows of the Disk tab need the `smb` or `smbclient`, `process`, `scheduled_task`, `update`, `gpu`, `hyperv`, `mscluster`, `time`, `mssql`, `tcp`, `udp`, `diskdrive` and `storage_spaces` collectors, which are not enabled by default.
   The sample shows every tab; build the dashboard from the [mixin](../contrib/mixin#collectors) with your collectors to hide the others.
   The process names in the GPU tab need the `process` collector.
   GPU temperature, fan, power and clock panels require WDDM 2.4 or newer and driver support; unsupported sensors are absent.
@@ -21,8 +21,10 @@ Import the JSON files through **Dashboards > New > Import** and pick the Prometh
 | Windows Exporter / Full | [windows-exporter-dashboard.json](windows-exporter-dashboard.json) | `windows-exporter-full` |
 | Windows Exporter / `<tab>` | `windows-exporter-<tab>.json`, for example [windows-exporter-cpu.json](windows-exporter-cpu.json) | `windows-exporter-<tab>`, for example `windows-exporter-cpu` |
 | Windows Exporter / MSSQL | [windows-exporter-mssql.json](windows-exporter-mssql.json) | `windows-exporter-mssql` |
+| Windows Exporter / Hyper-V Cluster | [windows-exporter-hyper-v-cluster.json](windows-exporter-hyper-v-cluster.json) | `windows-exporter-hyper-v-cluster` |
 
 The MSSQL dashboard is not a tab of the full dashboard. The [mixin](../contrib/mixin#metadata) adds it as a tab with `separateDashboardTabs: []`.
+The Hyper-V Cluster dashboard has tabs of its own, see [Hyper-V Cluster dashboard](#hyper-v-cluster-dashboard).
 
 The tab dashboards show only the variables their panels use.
 The **Windows Exporter** dropdown in the links of the tab dashboards opens the other imported dashboards and keeps the time range and variables.
@@ -61,13 +63,31 @@ The **Reboots** annotation marks each reboot of the selected host on all graphs.
 | Scheduled tasks | Task count, running, disabled and never-run tasks, enabled tasks with an unknown result code or missed runs, tasks that ran in the time range, and all tasks with state, last result and missed runs. |
 | Updates  | Pending updates by severity and category, the oldest pending update, the time since the last Windows Update check, a list of pending updates and the query duration. |
 | GPU      | Utilization by engine type, memory capacity, commitment and locality, temperature, fans, power, clocks, and top processes by GPU utilization and memory.                                                                   |
-| Hyper-V  | VM count and health, Hyper-V WMI health, host CPU time, CPU and memory per VM, virtual disks, virtual switches and VM network adapters.                             |
+| Hyper-V  | VM count and health, Hyper-V WMI health, VM memory share, slow virtual disks, a table of the host's VMs, host CPU time, CPU usage per VM and the busiest VMs, memory per VM, virtual disks and the slowest of them, VHD capacity, virtual switches, VM network adapters and dropped packets per VM. |
+| Failover Cluster | Nodes up and not up, failed and offline resource groups, failed resources, cluster networks not up and the quorum type; state, drain status, vote, owned resource groups and build of each node, node state over time and resource groups per node; resource groups and resources that are not online with their owner node; state, role and metric of each cluster network; size, free space and usage of the Cluster Shared Volumes. Every node reports the whole cluster, so any node of a cluster shows it. |
 | Time     | Time zone, clock sync source, NTP time sources, clock offset and NTP round-trip delay.                                                                              |
 | MSSQL    | A separate dashboard for one SQL Server instance: failed mssql sub-collectors, connections, batch requests, compilations, logins, access methods and errors; wait time by wait class, lock waits, average lock wait, deadlocks and lock timeouts; server memory against its target, memory by consumer, page life expectancy, buffer pool I/O and memory grants; size, log usage, transactions, log flushes and log flush write time per database; tempdb free space, the version store and active transactions; send and redo queues and traffic of Always On availability replicas. |
 | Exporter | Scrape status and duration, exporter version and uptime, collector status, duration, failures and timeouts, and the exporter's CPU, memory, handles and Go runtime. |
 
 Click a hostname in the Fleet tab to open that host in the Overview tab. In the Fleet dashboard, it opens the Overview dashboard.
 
+### Hyper-V Cluster dashboard
+
+The Hyper-V Cluster dashboard shows the Hyper-V hosts of one failover cluster. It needs the `hyperv` and `mscluster` collectors on all nodes,
+the `memory` collector for the memory shares and the `storage_spaces` collector for the Storage Spaces Direct row.
+The **Cluster** variable lists the clusters by the `cluster` label of the mscluster metrics, and the dashboard shows the instances that report that cluster.
+If your scrape configuration sets its own `cluster` target label, Prometheus renames the exporter's label to `exported_cluster` unless `honor_labels` is set; the variable then lists your label values instead.
+**Host** selects the host of the Host tab and **VM** the virtual machine of the VM tab, on whichever node runs it.
+
+| Tab      | Content |
+|----------|---------|
+| Overview | Nodes up, running VMs, VMs in critical health, virtual processors per logical processor, VM memory as a share of all host memory and of the memory left when the largest host fails, host CPU usage and slow virtual disks; running VMs, vCPU ratio, CPU usage and VM memory share per host; CPU usage, running VMs, VM memory and virtual disk IOPS per host over time. |
+| Cluster  | The Failover Cluster tab for the selected cluster, without the Cluster Shared Volumes, which are in the Storage tab. |
+| Host     | The Hyper-V tab for the selected host. |
+| VM       | The host that runs the VM, virtual processors, assigned memory, RAM in use and committed memory, CPU usage by state and per virtual processor, dynamic memory changes, size, file size and fill of its virtual hard disks, disk IOPS, throughput and latency, and network throughput and dropped packets per adapter. |
+| VM disks | Count, size, file size and growth if thick of all virtual hard disks, differencing disks and slow disks; capacity by provisioning type; every disk with its VM, host and type; the slowest, busiest and highest-latency disks. |
+| Storage  | Size and free space of the Cluster Shared Volumes, also if every dynamic virtual hard disk on them grew to its full size; usage per volume; virtual hard disks per folder below `C:\ClusterStorage`; health, size and usage of the Storage Spaces pools and health, size, footprint and efficiency of their virtual disks. |
+
 Each per-host tab starts with a row of current values, each with a sparkline
 of the time range behind it. Green, orange and red show health against fixed
 thresholds; values without a meaningful threshold, like throughput or counts,