dashboard: add process metrics with bounded graphs and summaries (#2631)

Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>

contrib/mixin/.dashboard-lint.yaml

@@ -1,5 +1,24 @@
 # Keep exceptions scoped to the existing sample dashboard and panels.
 exclusions:
+  target-promql-rule:
+    reason: dashboard-linter 0.3.0's v2 adapter omits custom variable current values and options, so the numeric process_top value becomes a vector identifier. These queries are validated against live Prometheus with limits 5, 10 and 20.
+    entries:
+      - dashboard: Windows Exporter
+        panel: Process CPU usage
+      - dashboard: Windows Exporter
+        panel: Process working set
+      - dashboard: Windows Exporter
+        panel: Process private bytes
+      - dashboard: Windows Exporter
+        panel: Process I/O throughput
+      - dashboard: Windows Exporter
+        panel: Process threads
+      - dashboard: Windows Exporter
+        panel: Process handles
+      - dashboard: Windows Exporter
+        panel: Process I/O operations
+      - dashboard: Windows Exporter
+        panel: Process page faults
   target-instance-rule:
     reason: Fleet panels intentionally span instances; per-host panels use an exact matcher for the single-select instance variable.
     entries:

contrib/mixin/dashboards/processes.libsonnet

@@ -0,0 +1,57 @@
+local b = import 'builders.libsonnet';
+// PID 0 is the synthetic Idle process, not CPU consumed by an application.
+local selector = '{job=~"$job", instance="$instance", process=~"$process", process_id!="0"}';
+local gauge(metric, at='') = 'windows_process_' + metric + selector + at;
+local rate(metric, at='') = 'rate(' + gauge(metric) + '[$__rate_interval]' + at + ')';
+local sumRates(metric, at='') = 'sum by (process, process_id) (' + rate(metric, at) + ')';
+local stat(id, title, description, expr, unit) =
+  b.panel.new(id, title, 'stat')
+  + b.panel.withDescription(description + ' Includes all selected processes, regardless of Top N. Requires the process collector.')
+  + b.panel.withQueries([b.query.new(expr) + b.query.withInstant()])
+  + b.panel.withDefaults({ min: 0, unit: unit });
+local graph(id, title, description, expr, ranking, unit) =
+  b.panel.new(id, title, 'timeseries')
+  + b.panel.withDescription(description + ' Shows the Top N selected processes ranked at the end of the time range; that set stays fixed throughout the graph. Requires the process collector.')
+  + b.panel.withQueries([
+    b.query.new(expr + ' and on (process, process_id) topk($process_top, ' + ranking + ')')
+    + b.query.withLegendFormat('{{process}} ({{process_id}})'),
+  ])
+  + b.panel.withQueryOptions({ maxDataPoints: 300 })
+  + b.panel.withDefaults({ min: 0, unit: unit });
+
+b.tab(
+  'Processes',
+  b.grid([
+    b.place(150, 0, 0, 4, 4),
+    b.place(151, 4, 0, 4, 4),
+    b.place(152, 8, 0, 4, 4),
+    b.place(153, 12, 0, 4, 4),
+    b.place(154, 16, 0, 4, 4),
+    b.place(155, 20, 0, 4, 4),
+    b.place(112, 0, 4, 12, 8),
+    b.place(113, 12, 4, 12, 8),
+    b.place(114, 0, 12, 12, 8),
+    b.place(115, 12, 12, 12, 8),
+    b.place(116, 0, 20, 12, 8),
+    b.place(117, 12, 20, 12, 8),
+    b.place(118, 0, 28, 12, 8),
+    b.place(119, 12, 28, 12, 8),
+  ]),
+  [
+    stat(150, 'Processes', 'Number of distinct process IDs currently collected, excluding Idle.', 'count(count by (process_id) (' + gauge('info') + '))', 'short'),
+    stat(151, 'Threads', 'Total active threads in the selected processes.', 'sum(' + gauge('threads') + ')', 'short'),
+    stat(152, 'Handles', 'Total open handles in the selected processes.', 'sum(' + gauge('handles') + ')', 'short'),
+    stat(153, 'Process CPU busy', 'CPU time used by the selected processes as a share of all logical processors of the host, excluding Idle.', 'sum(' + rate('cpu_time_total') + ') / scalar(max(windows_cpu_logical_processor{job=~"$job", instance="$instance"}))', 'percentunit'),
+    stat(154, 'Private working set', 'Physical memory private to the selected processes; shared pages are excluded.', 'sum(' + gauge('working_set_private_bytes') + ')', 'bytes'),
+    stat(155, 'Private bytes', 'Total memory allocated by the selected processes that cannot be shared; it may include nonresident memory.', 'sum(' + gauge('private_bytes') + ')', 'bytes'),
+
+    graph(112, 'Process CPU usage', 'User and privileged CPU time per process as a share of one logical processor. Values above 100% mean the process uses more than one core.', sumRates('cpu_time_total'), sumRates('cpu_time_total', ' @ end()'), 'percentunit'),
+    graph(113, 'Process working set', 'Physical memory in the working set of each process, including shared pages.', gauge('working_set_bytes'), gauge('working_set_bytes', ' @ end()'), 'bytes'),
+    graph(114, 'Process private bytes', 'Memory allocated by each process that cannot be shared with other processes.', gauge('private_bytes'), gauge('private_bytes', ' @ end()'), 'bytes'),
+    graph(115, 'Process I/O throughput', 'Total bytes issued for read, write and other I/O per process, including file, network and device operations.', sumRates('io_bytes_total'), sumRates('io_bytes_total', ' @ end()'), 'Bps'),
+    graph(116, 'Process threads', 'Active threads in each process.', gauge('threads'), gauge('threads', ' @ end()'), 'short'),
+    graph(117, 'Process handles', 'Open handles in each process. Steady growth can indicate a handle leak.', gauge('handles'), gauge('handles', ' @ end()'), 'short'),
+    graph(118, 'Process I/O operations', 'Total read, write and other I/O operations per second per process. Includes file, network and device I/O.', sumRates('io_operations_total'), sumRates('io_operations_total', ' @ end()'), 'iops'),
+    graph(119, 'Process page faults', 'Page faults per second by each process, including soft faults that do not require reading from disk.', rate('page_faults_total'), rate('page_faults_total', ' @ end()'), 'ops'),
+  ]
+)

contrib/mixin/dashboards/variables.libsonnet

@@ -99,4 +99,30 @@ local new(name, definition) = common
   + query.spec.withLabel('Network interface')
   + query.spec.withMulti(true)
   + query.spec.withRegex('/^(?!isatap|Teredo|6to4).+$/'),
+  new('process', 'label_values(windows_process_info{job=~"$job", instance="$instance", process_id!="0"}, process)')
+  + query.spec.withCurrent({
+    text: [
+      'All',
+    ],
+    value: [
+      '$__all',
+    ],
+  })
+  + query.spec.withIncludeAll(true)
+  + query.spec.withLabel('Process')
+  + query.spec.withMulti(true),
+  variables.CustomVariableKind.withKind()
+  + variables.CustomVariableKind.withSpec({
+    name: 'process_top',
+    label: 'Top N processes',
+    description: 'Maximum number of processes per graph. Summary counters include all selected processes.',
+    query: '5,10,20',
+    current: { text: '10', value: '10' },
+    options: [{ text: value, value: value, selected: value == '10' } for value in ['5', '10', '20']],
+    multi: false,
+    includeAll: false,
+    allowCustomValue: false,
+    hide: 'dontHide',
+    skipUrlSync: false,
+  }),
 ]

contrib/mixin/dashboards/windows-exporter.libsonnet

@@ -7,6 +7,7 @@ local tabs = [
   import 'memory.libsonnet',
   import 'disk.libsonnet',
   import 'network.libsonnet',
+  import 'processes.libsonnet',
   import 'services.libsonnet',
   import 'gpu.libsonnet',
   import 'hyper_v.libsonnet',

dashboard/README.md

@@ -8,7 +8,7 @@ and takes several ideas from Grafana's [windows-observ-lib](https://github.com/g
 
 - Grafana 13 or later with a Prometheus data source. The file uses the v2 dashboard schema, which Grafana needs for tabs.
 - windows_exporter with the default collectors (`cpu`, `logical_disk`, `memory`, `net`, `os`, `physical_disk`, `service`, `system`).
-  The GPU, Hyper-V and Time tabs and the TCP and UDP rows of the Network tab need the `gpu`, `hyperv`, `time`, `tcp` and `udp` collectors, which are not enabled by default.
+  The Processes, GPU, Hyper-V and Time tabs and the TCP and UDP rows of the Network tab need the `process`, `gpu`, `hyperv`, `time`, `tcp` and `udp` collectors, which are not enabled by default.
   The process names in the GPU tab need the `process` collector.
   GPU temperature, fan, power and clock panels require WDDM 2.4 or newer and driver support; unsupported sensors are absent.
 
@@ -24,6 +24,8 @@ Import the JSON file through **Dashboards > New > Import** and pick the Promethe
 | Instance          | The host shown in all tabs except Fleet.                                    |
 | Volume            | Volumes shown in the Overview and Disk tabs. `HarddiskVolume*` are hidden.  |
 | Network interface | Interfaces shown in the Overview and Network tabs.                          |
+| Process           | One or more process names shown in the Processes tab, or All.               |
+| Top N processes   | Maximum processes per graph: 5, 10 (default), or 20.                         |
 
 The **Reboots** annotation marks each reboot of the selected host on all graphs.
 
@@ -38,6 +40,7 @@ The **Reboots** annotation marks each reboot of the selected host on all graphs.
 | Disk     | Volume size and free space, usage, busy time, throughput, IOPS, latency and queue length.                                                                           |
 | Network  | Throughput, utilization, packets, errors and discards per interface; TCP connection states and rates, segments and retransmissions; UDP datagrams and errors.       |
 | Services | Running, pending and disabled services, automatic services that are not running, and services that started or stopped in the time range.                            |
+| Processes | CPU usage, working set, private bytes, I/O throughput and operations, threads, handles and page faults per process, labelled with name and PID.                  |
 | GPU      | Utilization by engine type, memory capacity, commitment and locality, temperature, fans, power, clocks, and top processes by GPU utilization and memory.                                                                   |
 | Hyper-V  | VM count and health, Hyper-V WMI health, host CPU time, CPU and memory per VM, virtual disks, virtual switches and VM network adapters.                             |
 | Time     | Time zone, clock sync source, NTP time sources, clock offset and NTP round-trip delay.                                                                              |
@@ -45,6 +48,13 @@ The **Reboots** annotation marks each reboot of the selected host on all graphs.
 
 Click a hostname in the Fleet tab to open that host in the Overview tab.
 
+The Processes tab starts with totals for all selected processes: process count,
+threads, handles, CPU usage across the host, private working set and private bytes.
+The synthetic Idle process (PID 0) is excluded. Each graph shows at most **Top N**
+processes, ranked separately at the end of the time range, with a fixed set of
+process IDs across the graph. Processes that exited before that point may not
+appear. I/O graphs combine read, write and other operations per process.
+
 ### Changing the dashboard
 
 Don't edit the JSON file by hand. It is generated from Jsonnet with [grafonnet](https://github.com/grafana/grafonnet) as part of the [windows_exporter mixin](../contrib/mixin):