mirror of
https://github.com/prometheus-community/windows_exporter.git
synced 2026-10-18 18:49:05 +02:00
Jan-Otto Kröpke
ae5d0cca2a
The alerts of the mixin now come in one rule group per collector area (windows-exporter, windows-cpu, windows-memory, windows-disk, windows-network, windows-service, windows-system, windows-active-directory, windows-time). Every alert names the collectors it reads and is generated only when _config.collectors enables them, the same switch the dashboard uses; enableActiveDirectory and enableTime set collectors.ad and collectors.time. _config.alerts switches single alerts off by name, and an unknown name fails the build. New alerts for the default collectors: WindowsCollectorTimeout, WindowsScrapeQueueTimeout, WindowsCPUSaturation (processor queue per core), WindowsMemoryPagingHigh, WindowsNetworkUtilizationHigh, WindowsNetworkInterfaceFlapping, WindowsServiceNotRunning (automatic services without triggers), WindowsServiceCrashLooping (processes per service in 15 minutes) and WindowsHostRebooted (info, fires once per reboot). WindowsNetworkErrors and WindowsNetworkDiscards compare the error and discard rates with the packet rate (networkErrorRatioThreshold and networkDiscardRatioThreshold, in percent) instead of an absolute packet rate. Threshold alerts set keep_firing_for from keepFiringFor (5m); the new infoSeverity and serviceSelector knobs follow the existing ones. New recordings: windows:cpu_queue_per_core:ratio, windows:net_received_packets:rate and windows:net_sent_packets:rate. Each new alert has a runbook, a README row and a unit test; tests/alerts.test.jsonnet checks the collector gating, the alert switches and the group layout. The runbook check renders the alerts of every collector. contrib/mixin/AGENTS.md documents the checklist for new alerts. Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
272 lines
16 KiB
YAML
272 lines
16 KiB
YAML
rule_files:
|
||
- custom_rules.yaml
|
||
- custom_alerts.yaml
|
||
evaluation_interval: 1m
|
||
group_eval_order:
|
||
- windows-exporter-recording
|
||
- windows-exporter
|
||
- windows-cpu
|
||
- windows-memory
|
||
- windows-disk
|
||
- windows-network
|
||
- windows-service
|
||
- windows-system
|
||
- windows-active-directory
|
||
- windows-time
|
||
|
||
tests:
|
||
- name: Custom commit, latency and network thresholds preserve selectors and device filters
|
||
interval: 1m
|
||
input_series:
|
||
- series: 'windows_memory_committed_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '85x20'
|
||
- series: 'windows_memory_commit_limit_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '100x20'
|
||
- series: 'windows_memory_committed_bytes{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||
values: '99x20'
|
||
- series: 'windows_memory_commit_limit_bytes{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||
values: '100x20'
|
||
- series: 'windows_logical_disk_read_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||
values: '0+3.6x20'
|
||
- series: 'windows_logical_disk_reads_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||
values: '0+120x20'
|
||
- series: 'windows_logical_disk_write_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||
values: '0+3.6x20'
|
||
- series: 'windows_logical_disk_writes_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||
values: '0+120x20'
|
||
- series: 'windows_logical_disk_read_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
|
||
values: '0+12x20'
|
||
- series: 'windows_logical_disk_reads_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
|
||
values: '0+120x20'
|
||
- series: 'windows_logical_disk_write_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
|
||
values: '0+12x20'
|
||
- series: 'windows_logical_disk_writes_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
|
||
values: '0+120x20'
|
||
- series: 'windows_net_packets_received_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||
values: '0+36x20'
|
||
- series: 'windows_net_packets_outbound_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||
values: '0x20'
|
||
- series: 'windows_net_packets_received_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||
values: '0x20'
|
||
- series: 'windows_net_packets_outbound_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||
values: '0+36x20'
|
||
- series: 'windows_net_packets_received_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
|
||
values: '0+120x20'
|
||
- series: 'windows_net_packets_outbound_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
|
||
values: '0+120x20'
|
||
- series: 'windows_net_packets_received_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
|
||
values: '0+120x20'
|
||
- series: 'windows_net_packets_outbound_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
|
||
values: '0+120x20'
|
||
- series: 'windows_net_packets_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||
values: '0+3000x20'
|
||
- series: 'windows_net_packets_sent_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||
values: '0+3000x20'
|
||
- series: 'windows_net_packets_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
|
||
values: '0+120x20'
|
||
- series: 'windows_net_packets_sent_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
|
||
values: '0+120x20'
|
||
alert_rule_test:
|
||
- alertname: WindowsMemoryCommitHighUsage
|
||
eval_time: 20m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsmemorycommithighusage/
|
||
summary: Windows memory commit usage is high
|
||
description: 'Committed memory on host:9182 has exceeded 80% of the commit limit for 15 minutes.'
|
||
- alertname: WindowsDiskReadLatencyHigh
|
||
eval_time: 20m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsdiskreadlatencyhigh/
|
||
summary: Windows disk read latency is high
|
||
description: 'Volume C: on host:9182 has averaged more than 0.02 seconds per read operation for 15 minutes.'
|
||
- alertname: WindowsDiskWriteLatencyHigh
|
||
eval_time: 20m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsdiskwritelatencyhigh/
|
||
summary: Windows disk write latency is high
|
||
description: 'Volume C: on host:9182 has averaged more than 0.02 seconds per write operation for 15 minutes.'
|
||
- alertname: WindowsNetworkErrors
|
||
eval_time: 20m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, nic: Ethernet, severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsnetworkerrors/
|
||
summary: Windows network packet errors are high
|
||
description: 'Packet errors on interface Ethernet on host:9182 have exceeded 0.5% of all packets for 15 minutes.'
|
||
- alertname: WindowsNetworkDiscards
|
||
eval_time: 20m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, nic: Ethernet, severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsnetworkdiscards/
|
||
summary: Windows network packet discards are high
|
||
description: 'Packet discards on interface Ethernet on host:9182 have exceeded 0.5% of all packets for 15 minutes.'
|
||
|
||
- name: Clock alerts opt in, use absolute offset, custom URLs and recover
|
||
interval: 1m
|
||
input_series:
|
||
- series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '0x10 1x10'
|
||
- series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '-0.2x10 0x10'
|
||
- series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="healthy:9182", cluster="test"}'
|
||
values: '1x21'
|
||
- series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="healthy:9182", cluster="test"}'
|
||
values: '0.1x21'
|
||
- series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||
values: '0x21'
|
||
- series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||
values: '-1x21'
|
||
alert_rule_test:
|
||
- {alertname: WindowsClockNotSynchronising, eval_time: 9m, exp_alerts: []}
|
||
- alertname: WindowsClockNotSynchronising
|
||
eval_time: 10m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsclocknotsynchronising/
|
||
summary: Windows clock has no NTP time source
|
||
description: 'Windows Time on host:9182 has had no active NTP time source for 10 minutes.'
|
||
- {alertname: WindowsClockNotSynchronising, eval_time: 11m, exp_alerts: []}
|
||
- {alertname: WindowsClockSkewDetected, eval_time: 9m, exp_alerts: []}
|
||
- alertname: WindowsClockSkewDetected
|
||
eval_time: 10m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsclockskewdetected/
|
||
summary: Windows clock offset is high
|
||
description: 'Windows Time on host:9182 reports a clock offset above 0.1 seconds for 10 minutes.'
|
||
- {alertname: WindowsClockSkewDetected, eval_time: 16m, exp_alerts: []}
|
||
- name: Custom selectors, thresholds and critical severity propagate to alerts
|
||
interval: 1m
|
||
input_series:
|
||
- series: 'up{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '0x20'
|
||
- series: 'up{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||
values: '0x20'
|
||
- series: 'windows_cpu_time_seconds_total{job="windows-custom", instance="host:9182", cluster="test", core="0", mode="idle"}'
|
||
values: '0+9x20'
|
||
- series: 'windows_memory_available_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '12x20'
|
||
- series: 'windows_memory_physical_total_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '100x20'
|
||
- series: 'windows_logical_disk_free_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||
values: '8x20'
|
||
- series: 'windows_logical_disk_size_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||
values: '100x20'
|
||
- series: 'windows_logical_disk_free_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
|
||
values: '1x20'
|
||
- series: 'windows_logical_disk_size_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
|
||
values: '100x20'
|
||
- series: 'windows_net_bytes_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||
values: '0+600x20'
|
||
- series: 'windows_net_bytes_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
|
||
values: '0+600x20'
|
||
promql_expr_test:
|
||
- expr: windows:net_received_bytes:rate
|
||
eval_time: 20m
|
||
exp_samples:
|
||
- labels: 'windows:net_received_bytes:rate{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||
value: 10
|
||
- expr: windows:logical_disk_free:ratio
|
||
eval_time: 20m
|
||
exp_samples:
|
||
- labels: 'windows:logical_disk_free:ratio{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||
value: 0.08
|
||
alert_rule_test:
|
||
- alertname: WindowsExporterDown
|
||
eval_time: 5m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: page}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsexporterdown/
|
||
summary: Windows exporter is unreachable
|
||
description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows-custom).'
|
||
- alertname: WindowsCPUHighUsage
|
||
eval_time: 16m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowscpuhighusage/
|
||
summary: Windows CPU usage is high
|
||
description: 'CPU usage on host:9182 has exceeded 80% for 15 minutes.'
|
||
- alertname: WindowsMemoryHighUsage
|
||
eval_time: 15m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsmemoryhighusage/
|
||
summary: Windows memory usage is high
|
||
description: 'Physical memory usage on host:9182 has exceeded 85% for 15 minutes.'
|
||
- alertname: WindowsDiskAlmostFull
|
||
eval_time: 15m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsdiskalmostfull/
|
||
summary: Windows disk space is low
|
||
description: 'Volume C: on host:9182 has less than 20% free space. Windows may delay space metrics by 10–15 minutes.'
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: page}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsdiskalmostfull/
|
||
summary: Windows disk space is low
|
||
description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.'
|
||
|
||
- name: Active Directory opt-in honors thresholds, selectors and recovery
|
||
interval: 1m
|
||
input_series:
|
||
- series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '20x20 0x5'
|
||
- series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="healthy:9182", cluster="test"}'
|
||
values: '10x26'
|
||
- series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||
values: '20x26'
|
||
- series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '0+120x26'
|
||
- series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="host:9182", cluster="test"}'
|
||
values: '0+60x20 1320+120x5'
|
||
- series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="healthy:9182", cluster="test"}'
|
||
values: '0+120x26'
|
||
- series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="healthy:9182", cluster="test"}'
|
||
values: '0+120x26'
|
||
- series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||
values: '0+120x26'
|
||
- series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||
values: '0+60x26'
|
||
alert_rule_test:
|
||
- alertname: WindowsADPendingReplication
|
||
eval_time: 14m
|
||
exp_alerts: []
|
||
- alertname: WindowsADPendingReplication
|
||
eval_time: 15m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsadpendingreplication/
|
||
summary: Active Directory replication queue is high
|
||
description: 'Pending replication operations on host:9182 have exceeded 10 for 15 minutes.'
|
||
- alertname: WindowsADPendingReplication
|
||
eval_time: 26m
|
||
exp_alerts: []
|
||
- alertname: WindowsADReplicationFailures
|
||
eval_time: 16m
|
||
exp_alerts: []
|
||
- alertname: WindowsADReplicationFailures
|
||
eval_time: 17m
|
||
exp_alerts:
|
||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||
exp_annotations:
|
||
runbook_url: https://runbooks.example.com/windows/windowsadreplicationfailures/
|
||
summary: Active Directory replication requests are failing
|
||
description: 'Replication synchronization requests on host:9182 have been failing for 15 minutes.'
|
||
- alertname: WindowsADReplicationFailures
|
||
eval_time: 28m
|
||
exp_alerts: []
|