sendnrw/windows_exporter
2
0
Fork 0
mirror of https://github.com/prometheus-community/windows_exporter.git synced 2026-10-21 03:59:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

contrib: expand Windows mixin rules and publish runbooks (#2575)

Browse Source 
Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
This commit is contained in:
Jan-Otto Kröpke
2026-10-09 09:10:59 +02:00
committed by GitHub
parent c7fa244d0e
commit d06f168926
43 changed files with 1629 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/mixin/.pint.hcl
+6
View File
@@ -21,4 +21,10 @@ rule {
value = ".+"
severity = "bug"
}
annotation "runbook_url" {
required = true
value = "https?://.+"
severity = "bug"
}
}
contrib/mixin/Makefile
+13 -6
View File
@@ -2,24 +2,27 @@ JSONNET ?= jsonnet
JSONNETFMT ?= jsonnetfmt
PROMTOOL ?= promtool
PINT ?= pint
PYTHON ?= python3
# renovate: go=github.com/google/go-jsonnet
JSONNET_VERSION := v0.22.0
# renovate: go=github.com/cloudflare/pint
PINT_VERSION := v0.89.2
SOURCES := $(wildcard *.libsonnet *.jsonnet lib/*.libsonnet rules/*.libsonnet alerts/*.libsonnet tests/*.libsonnet)
SOURCES := $(wildcard *.libsonnet *.jsonnet lib/*.libsonnet rules/*.libsonnet alerts/*.libsonnet tests/*.libsonnet tests/*.jsonnet)
FMT_FLAGS := -n 2 --max-blank-lines 2 --string-style s --comment-style s
.DEFAULT_GOAL := generate
.DELETE_ON_ERROR:
.PHONY: generate tools fmt lint test clean
.PHONY: generate tools jsonnet-tools fmt lint test check-runbooks clean
generate: windows_rules.yaml windows_alerts.yaml
tools:
tools: jsonnet-tools
go install github.com/cloudflare/pint/cmd/pint@$(PINT_VERSION)
jsonnet-tools:
go install github.com/google/go-jsonnet/cmd/jsonnet@$(JSONNET_VERSION)
go install github.com/google/go-jsonnet/cmd/jsonnetfmt@$(JSONNET_VERSION)
go install github.com/cloudflare/pint/cmd/pint@$(PINT_VERSION)
windows_rules.yaml: $(SOURCES)
$(JSONNET) -S rules.jsonnet > $@
@@ -44,8 +47,12 @@ lint: generate tests/custom_rules.yaml tests/custom_alerts.yaml
$(PINT) --offline lint windows_rules.yaml windows_alerts.yaml
$(PINT) --offline lint tests/custom_rules.yaml tests/custom_alerts.yaml
test: lint
$(PROMTOOL) test rules tests/rules.test.yaml tests/alerts.test.yaml tests/custom.test.yaml
check-runbooks:
$(JSONNET) -e '(import "mixin.libsonnet") { _config+:: { enableActiveDirectory: true, enableTime: true } }.prometheusAlerts' | $(PYTHON) tests/check_runbooks.py ../../docs/runbooks
test: lint check-runbooks
$(JSONNET) tests/runbooks.test.jsonnet
$(PROMTOOL) test rules tests/*.test.yaml
clean:
rm -f windows_rules.yaml windows_alerts.yaml tests/custom_rules.yaml tests/custom_alerts.yaml
contrib/mixin/README.md
+90 -11
View File
@@ -24,7 +24,8 @@ to generate rules. Generated files are ignored by Git:
The default selector is `job="windows_exporter"`. Change it to match your scrape
job before generating rules. Resource rules use the default `cpu`, `memory`,
`logical_disk`, and `net` collectors. Enable `ad` on domain controllers when
opting into Active Directory alerts.
opting into Active Directory alerts. Clock alerts require the `time` collector
with its `ntp` subcollector and are intended for hosts using NTP synchronization.
Copy both generated files into your Prometheus rule directory and reference
them in `prometheus.yml`:
@@ -53,6 +54,8 @@ modifying its source. For example, create `custom.libsonnet` in this directory:
rateInterval: '5m',
cpuHighUsageThreshold: 95,
enableActiveDirectory: true,
enableTime: true,
runbookURLPattern: 'https://runbooks.example.com/windows/%s/',
},
}
```
@@ -68,7 +71,8 @@ Selectors are comma-separated Prometheus label matchers without braces. They
apply to both recording rules and alerts. All target labels, including `job`,
`instance`, `cluster`, and custom selector labels, are preserved. CPU usage
averages idle rates across cores and removes only `core` and `mode`. Disk and
network recordings also retain `volume` and `nic` respectively.
network recordings also retain `volume` and `nic` respectively. The CPU mode
recording retains `mode` and removes only `core`.
`rateInterval` is the fixed window for recording rules and Active Directory
failure rates; keep it at least four times your scrape interval. Grafana's
@@ -80,18 +84,47 @@ rules.
| Metric | Meaning | Unit |
| --- | --- | --- |
| `windows:cpu_usage:ratio` | Non-idle CPU time averaged across cores | Ratio, 0–1 |
| `windows:cpu_cores:count` | Logical processors observed in idle CPU metrics | Count |
| `windows:cpu_time:ratio_rate` | CPU time averaged across cores, retaining `mode` | Seconds/second |
| `windows:memory_usage:ratio` | Physical memory in use, based on available bytes | Ratio, 0–1 |
| `windows:memory_used:bytes` | Total physical memory minus available memory | Bytes |
| `windows:memory_cached:bytes` | Cache, modified pages, and three standby lists | Bytes |
| `windows:memory_committed:ratio` | Committed bytes divided by the system commit limit | Ratio |
| `windows:memory_swap_page_operations:rate` | Paging operations | Operations/second |
| `windows:logical_disk_free:ratio` | Free disk space per volume | Ratio, 0–1 |
| `windows:logical_disk_used:bytes` | Used disk space per volume | Bytes |
| `windows:logical_disk_busy:ratio` | Non-idle time per volume, clamped to 0–1 | Ratio, 0–1 |
| `windows:logical_disk_read_bytes:rate` | Disk read throughput per volume | Bytes/second |
| `windows:logical_disk_write_bytes:rate` | Disk write throughput per volume | Bytes/second |
| `windows:logical_disk_read_operations:rate` | Disk reads per volume | Operations/second |
| `windows:logical_disk_write_operations:rate` | Disk writes per volume | Operations/second |
| `windows:logical_disk_read_latency:seconds` | Read-time rate divided by read-operation rate | Seconds/operation |
| `windows:logical_disk_write_latency:seconds` | Write-time rate divided by write-operation rate | Seconds/operation |
| `windows:net_received_bytes:rate` | Received network traffic per interface | Bytes/second |
| `windows:net_sent_bytes:rate` | Sent network traffic per interface | Bytes/second |
| `windows:net_received_utilization:ratio` | Received throughput divided by interface bandwidth | Ratio |
| `windows:net_sent_utilization:ratio` | Sent throughput divided by interface bandwidth | Ratio |
| `windows:net_received_errors:rate` | Inbound packet errors | Packets/second |
| `windows:net_outbound_errors:rate` | Outbound packet errors | Packets/second |
| `windows:net_received_discarded:rate` | Inbound discarded packets | Packets/second |
| `windows:net_outbound_discarded:rate` | Outbound discarded packets | Packets/second |
The names remain stable when `rateInterval` changes. Dashboard authors can use
these recordings or query raw exporter metrics directly. CPU usage measures
non-idle time; it is not the frequency-adjusted processor utility shown in
Windows Task Manager.
CPU DPC and interrupt modes overlap privileged time; do not sum all modes.
Disk busy time uses the idle counter rather than summing read and write time,
which can exceed wall time for concurrent I/O. Latency is an average and is
omitted when there are no operations. Memory, disk space, commit, and network
utilization ratios are omitted when their capacity is zero or missing. Network
bandwidth is already in bytes/second; receive and send utilization are separate
for full-duplex links.
The cache recording includes modified pages and is not a measure of immediately
reclaimable memory. Paging operations can include file-backed pages as well as
pagefile traffic.
## Alerts
Threshold configuration values below are percentages except where noted.
@@ -102,11 +135,18 @@ Threshold configuration values below are percentages except where noted.
| `WindowsCollectorFailed` | `windows_exporter_collector_success == 0` | 5m | Warning |
| `WindowsCPUHighUsage` | CPU usage above `cpuHighUsageThreshold` (90%) | 15m | Warning |
| `WindowsMemoryHighUsage` | Memory usage above `memoryHighUsageThreshold` (90%) | 15m | Warning |
| `WindowsMemoryCommitHighUsage` | Commit usage above `memoryCommitHighUsageThreshold` (90%) | 15m | Warning |
| `WindowsDiskReadLatencyHigh` | Average read latency above `diskLatencyThresholdSeconds` (0.05 seconds) | 15m | Warning |
| `WindowsDiskWriteLatencyHigh` | Average write latency above `diskLatencyThresholdSeconds` (0.05 seconds) | 15m | Warning |
| `WindowsNetworkErrors` | Combined receive/outbound errors above `networkErrorRateThreshold` (1 packet/second) | 15m | Warning |
| `WindowsNetworkDiscards` | Combined receive/outbound discards above `networkDiscardRateThreshold` (1 packet/second) | 15m | Warning |
| `WindowsDiskAlmostFull` | Free space below `diskFreeWarningThreshold` (10%) | 15m | Warning |
| `WindowsDiskAlmostFull` | Free space below `diskFreeCriticalThreshold` (5%) | 15m | Critical |
| `WindowsDiskFillingUp` | Below the warning threshold and predicted to fill within `diskPredictionHours` (24h), using `diskPredictionWindow` (6h) | 1h | Warning |
| `WindowsADPendingReplication` | Pending operations above `adPendingReplicationThreshold` (50) | 15m | Warning |
| `WindowsADReplicationFailures` | Failed synchronization requests above `adReplicationFailureRateThreshold` (0 requests/second) | 15m | Warning |
| `WindowsClockNotSynchronising` | No active NTP time sources | 10m | Warning |
| `WindowsClockSkewDetected` | Absolute reported time offset above `clockOffsetThresholdSeconds` (0.05 seconds) | 10m | Warning |
Active Directory alerts are generated only when `enableActiveDirectory` is
`true`. `criticalSeverity` defaults to `critical` and can be overridden to match
@@ -114,6 +154,12 @@ your routing labels. Below the critical disk threshold, both disk severity
levels fire; configure Alertmanager inhibition if only the critical
notification should be sent.
Clock alerts are generated only when `enableTime` is `true`. A zero NTP source
count may be expected on hosts using another time provider. The offset compares
the host to its chosen source and does not independently verify that source.
Disk latency and network error/discard thresholds should be tuned for the
storage tier, traffic volume, and application requirements.
Windows disk size/free-space counters may lag by 10–15 minutes. Predictions
depend on a sustained trend and are intentionally delayed for an hour. A target
removed from service discovery has no `up` series and will not trigger
@@ -121,9 +167,35 @@ removed from service discovery has no `up` series and will not trigger
scrape and collector-failure alerts cover explicit failures, not disabled
collectors.
## Runbooks and GitHub Pages
Every alert receives a `runbook_url` annotation pointing to a dedicated page in
[docs/runbooks](../../docs/runbooks). Each page describes the meaning, impact,
diagnosis, and mitigation, including Windows commands and related metrics.
The default URL is
`https://prometheus-community.github.io/windows_exporter/runbooks/<lowercase-alert-name>/`.
Override `runbookURLPattern` for a fork, custom domain, or internal runbook site;
it takes one `%s` placeholder for the lowercase alert name. Explicit
`runbook_url` annotations supplied by downstream alerts are preserved.
The theme-free Hugo site uses `docs/hugo.toml`. With Hugo 0.167.0 installed,
preview it from the repository root:
```sh
hugo server --source docs
```
Build it with `hugo --source docs --minify`. The `Runbooks` workflow builds site
changes on pull requests, checks alert coverage and internal links, and publishes
default-branch changes with `configure-pages`, `upload-pages-artifact`, and
`deploy-pages`. This uses the
repository's GitHub Pages **Source: GitHub Actions** setting. The workflow reads
the configured Pages URL, including custom domains and repository subpaths.
Fork users should override `runbookURLPattern` to match their published URL.
## Validate
Install `promtool` from a [Prometheus release](https://github.com/prometheus/prometheus/releases),
Install Python 3 and `promtool` from a [Prometheus release](https://github.com/prometheus/prometheus/releases),
then run:
```sh
@@ -132,10 +204,12 @@ make test
This checks Jsonnet formatting, validates generated rules with `promtool`, runs
`pint --offline lint`, and evaluates the rules against synthetic time series.
Tests cover all seven recordings, alert timing and recovery, threshold
boundaries, counter resets, disk forecasts, target-label preservation, and
custom selectors/thresholds with Active Directory enabled. CI uses Prometheus
3.15.0 and the same pinned tool versions as `make tools`.
Tests cover all 25 recordings, alert timing and recovery, threshold
boundaries, counter resets, zero denominators, disk forecasts, target-label
preservation, and custom selectors/thresholds with Active Directory and clock
alerts enabled. Runbook checks include opt-in alerts and verify that their pages
and URLs exist. CI uses Prometheus 3.15.0 and the same pinned tool versions as
`make tools`.
If you prefer to run `promtool` through Docker:
@@ -143,10 +217,15 @@ If you prefer to run `promtool` through Docker:
make test PROMTOOL="docker run --rm --entrypoint promtool -v \"$PWD:/mixin:ro\" -w /mixin prom/prometheus:v3.15.0"
```
`.pint.hcl` requires a non-empty severity label and summary/description
annotations on alerts, in addition to pint's built-in checks. CI uses offline
mode without a Prometheus endpoint. To validate metric availability and labels
against your own server, copy `.pint.hcl` to a local configuration, add:
The dedicated `Mixin` workflow runs on changes to `contrib/mixin/**` or its own
workflow file, and can also be started manually. It caches pinned tool binaries
and runs the same `make test` checks used locally.
`.pint.hcl` requires a non-empty severity label, summary/description annotations,
and an HTTP(S) runbook URL on alerts, in addition to pint's built-in checks.
CI uses offline mode without a Prometheus endpoint. To validate metric
availability and labels against your own server, copy `.pint.hcl` to a local
configuration, add:
```hcl
prometheus "local" {
contrib/mixin/alerts/alerts.libsonnet
+69 -12
View File
@@ -45,6 +45,41 @@
'Windows memory usage is high',
'Physical memory usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.memoryHighUsageThreshold
),
alert(
'WindowsMemoryCommitHighUsage',
q.metric('windows:memory_committed:ratio') + ' * 100 > %s' % c.memoryCommitHighUsageThreshold,
'15m',
'warning',
'Windows memory commit usage is high',
'Committed memory on {{ $labels.instance }} has exceeded %s%% of the commit limit for 15 minutes.' % c.memoryCommitHighUsageThreshold
),
] + [
alert(
'WindowsDisk%sLatencyHigh' % direction[0],
q.metric('windows:logical_disk_%s_latency:seconds' % direction[1], [c.volumeSelector]) + ' > %g' % c.diskLatencyThresholdSeconds,
'15m',
'warning',
'Windows disk %s latency is high' % direction[1],
'Volume {{ $labels.volume }} on {{ $labels.instance }} has averaged more than %g seconds per %s operation for 15 minutes.' % [c.diskLatencyThresholdSeconds, direction[1]]
)
for direction in [['Read', 'read'], ['Write', 'write']]
] + [
alert(
'WindowsNetwork%s' % outcome[0],
'(%s + %s) > %g' % [
q.metric('windows:net_received_%s:rate' % outcome[1], [c.nicSelector]),
q.metric('windows:net_outbound_%s:rate' % outcome[1], [c.nicSelector]),
outcome[2],
],
'15m',
'warning',
'Windows network packet %s are high' % outcome[3],
'Packet %s on interface {{ $labels.nic }} on {{ $labels.instance }} have exceeded %g per second for 15 minutes.' % [outcome[3], outcome[2]]
)
for outcome in [
['Errors', 'errors', c.networkErrorRateThreshold, 'errors'],
['Discards', 'discarded', c.networkDiscardRateThreshold, 'discards'],
]
] + [
alert(
'WindowsDiskAlmostFull',
@@ -74,25 +109,47 @@
),
],
},
] + if c.enableActiveDirectory then [
] + (if c.enableActiveDirectory then [
{
name: 'windows-active-directory',
rules: [
alert(
'WindowsADPendingReplication',
q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold,
'15m',
'warning',
'Active Directory replication queue is high',
'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold
),
alert(
'WindowsADReplicationFailures',
'(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold,
'15m',
'warning',
'Active Directory replication requests are failing',
'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.'
),
],
},
] else []) + if c.enableTime then [
{
name: 'windows-active-directory',
name: 'windows-time',
rules: [
alert(
'WindowsADPendingReplication',
q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold,
'15m',
'WindowsClockNotSynchronising',
q.metric('windows_time_ntp_client_time_sources') + ' == 0',
'10m',
'warning',
'Active Directory replication queue is high',
'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold
'Windows clock has no NTP time source',
'Windows Time on {{ $labels.instance }} has had no active NTP time source for 10 minutes.'
),
alert(
'WindowsADReplicationFailures',
'(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold,
'15m',
'WindowsClockSkewDetected',
'abs(%s) > %g' % [q.metric('windows_time_computed_time_offset_seconds'), c.clockOffsetThresholdSeconds],
'10m',
'warning',
'Active Directory replication requests are failing',
'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.'
'Windows clock offset is high',
'Windows Time on {{ $labels.instance }} reports a clock offset above %g seconds for 10 minutes.' % c.clockOffsetThresholdSeconds
),
],
},
contrib/mixin/config.libsonnet
+9
View File
@@ -8,9 +8,18 @@
// Opt in only for targets with the ad collector enabled.
enableActiveDirectory: false,
// Requires the time collector with its ntp subcollector enabled.
enableTime: false,
runbookURLPattern: 'https://prometheus-community.github.io/windows_exporter/runbooks/%s/',
cpuHighUsageThreshold: 90,
memoryHighUsageThreshold: 90,
memoryCommitHighUsageThreshold: 90,
diskLatencyThresholdSeconds: 0.05,
networkErrorRateThreshold: 1,
networkDiscardRateThreshold: 1,
clockOffsetThresholdSeconds: 0.05,
diskFreeWarningThreshold: 10,
diskFreeCriticalThreshold: 5,
diskPredictionWindow: '6h',
contrib/mixin/lib/add-runbook-links.libsonnet
+19
View File
@@ -0,0 +1,19 @@
{
prometheusAlerts+:: {
groups: [
group {
rules: [
if std.objectHas(rule, 'alert') && !(std.objectHas(rule, 'annotations') && std.objectHas(rule.annotations, 'runbook_url')) then
rule {
annotations+: {
runbook_url: $._config.runbookURLPattern % std.asciiLower(rule.alert),
},
}
else rule
for rule in group.rules
],
}
for group in super.groups
],
},
}
contrib/mixin/lib/queries.libsonnet
+20 -6
View File
@@ -3,16 +3,30 @@ function(c)
selector(extra=[]): std.join(', ', [s for s in [c.windowsExporterSelector] + extra if s != '']),
metric(name, extra=[]): '%s{%s}' % [name, self.selector(extra)],
rate(name, extra=[]): 'rate(%s[%s])' % [self.metric(name, extra), c.rateInterval],
// Drop undefined ratios instead of recording NaN or +Inf for idle devices.
ratio(numerator, denominator): '(%s / (%s > 0))' % [numerator, denominator],
cpuUsage:
'1 - avg without (core, mode) (%s)' % self.rate('windows_cpu_time_total', ['mode="idle"']),
memoryUsage:
'1 - %s / %s' % [
'1 - %s' % self.ratio(
self.metric('windows_memory_available_bytes'),
self.metric('windows_memory_physical_total_bytes'),
],
self.metric('windows_memory_physical_total_bytes')
),
diskFree:
'%s / %s' % [
self.ratio(
self.metric('windows_logical_disk_free_bytes', [c.volumeSelector]),
self.metric('windows_logical_disk_size_bytes', [c.volumeSelector]),
],
self.metric('windows_logical_disk_size_bytes', [c.volumeSelector])
),
memoryCommitted:
self.ratio(self.metric('windows_memory_committed_bytes'), self.metric('windows_memory_commit_limit')),
diskLatency(direction):
self.ratio(
self.rate('windows_logical_disk_%s_seconds_total' % direction, [c.volumeSelector]),
self.rate('windows_logical_disk_%ss_total' % direction, [c.volumeSelector])
),
networkUtilization(direction):
self.ratio(
self.rate('windows_net_bytes_%s_total' % direction, [c.nicSelector]),
self.metric('windows_net_current_bandwidth_bytes', [c.nicSelector])
),
}
contrib/mixin/mixin.libsonnet
+2 -1
View File
@@ -1,3 +1,4 @@
(import 'config.libsonnet') +
(import 'rules/rules.libsonnet') +
(import 'alerts/alerts.libsonnet')
(import 'alerts/alerts.libsonnet') +
(import 'lib/add-runbook-links.libsonnet')
contrib/mixin/rules/rules.libsonnet
+45
View File
@@ -7,20 +7,65 @@
name: 'windows-exporter-recording',
rules: [
{ record: 'windows:cpu_usage:ratio', expr: q.cpuUsage },
{ record: 'windows:cpu_cores:count', expr: 'count without (core, mode) (%s)' % q.metric('windows_cpu_time_total', ['mode="idle"']) },
{ record: 'windows:cpu_time:ratio_rate', expr: 'avg without (core) (%s)' % q.rate('windows_cpu_time_total') },
{ record: 'windows:memory_usage:ratio', expr: q.memoryUsage },
{
record: 'windows:memory_used:bytes',
expr: '%s - %s' % [q.metric('windows_memory_physical_total_bytes'), q.metric('windows_memory_available_bytes')],
},
{
record: 'windows:memory_cached:bytes',
expr: std.join(' + ', [
q.metric('windows_memory_%s_bytes' % component)
for component in ['cache', 'modified_page_list', 'standby_cache_core', 'standby_cache_normal_priority', 'standby_cache_reserve']
]),
},
{ record: 'windows:memory_committed:ratio', expr: q.memoryCommitted },
{ record: 'windows:memory_swap_page_operations:rate', expr: q.rate('windows_memory_swap_page_operations_total') },
{ record: 'windows:logical_disk_free:ratio', expr: q.diskFree },
{
record: 'windows:logical_disk_used:bytes',
expr: '%s - %s' % [
q.metric('windows_logical_disk_size_bytes', [c.volumeSelector]),
q.metric('windows_logical_disk_free_bytes', [c.volumeSelector]),
],
},
{
record: 'windows:logical_disk_busy:ratio',
expr: 'clamp(1 - %s, 0, 1)' % q.rate('windows_logical_disk_idle_seconds_total', [c.volumeSelector]),
},
] + [
{
record: 'windows:logical_disk_%s_bytes:rate' % direction,
expr: q.rate('windows_logical_disk_%s_bytes_total' % direction, [c.volumeSelector]),
}
for direction in ['read', 'write']
] + [
{
record: 'windows:logical_disk_%s_operations:rate' % direction,
expr: q.rate('windows_logical_disk_%ss_total' % direction, [c.volumeSelector]),
}
for direction in ['read', 'write']
] + [
{ record: 'windows:logical_disk_%s_latency:seconds' % direction, expr: q.diskLatency(direction) }
for direction in ['read', 'write']
] + [
{
record: 'windows:net_%s_bytes:rate' % direction,
expr: q.rate('windows_net_bytes_%s_total' % direction, [c.nicSelector]),
}
for direction in ['received', 'sent']
] + [
{ record: 'windows:net_%s_utilization:ratio' % direction, expr: q.networkUtilization(direction) }
for direction in ['received', 'sent']
] + [
{
record: 'windows:net_%s_%s:rate' % [direction, outcome],
expr: q.rate('windows_net_packets_%s_%s_total' % [direction, outcome], [c.nicSelector]),
}
for direction in ['received', 'outbound']
for outcome in ['errors', 'discarded']
],
},
],
contrib/mixin/tests/alerts.test.yaml
+118
View File
@@ -7,6 +7,116 @@ group_eval_order:
- windows-exporter-alerts
tests:
- name: Commit pressure, disk latency and network failures wait and recover
interval: 1m
input_series:
- series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '95x20 50x10'
- series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '100x31'
- series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+12x20 252x10'
- series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+120x31'
- series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+24x20 504x10'
- series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+240x31'
- series: 'windows_net_packets_received_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+120x20 2520x10'
- series: 'windows_net_packets_outbound_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0x31'
- series: 'windows_net_packets_received_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0x31'
- series: 'windows_net_packets_outbound_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+120x20 2520x10'
alert_rule_test:
- {alertname: WindowsMemoryCommitHighUsage, eval_time: 14m, exp_alerts: []}
- alertname: WindowsMemoryCommitHighUsage
eval_time: 15m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsmemorycommithighusage/
summary: Windows memory commit usage is high
description: 'Committed memory on host:9182 has exceeded 90% of the commit limit for 15 minutes.'
- {alertname: WindowsMemoryCommitHighUsage, eval_time: 21m, exp_alerts: []}
- {alertname: WindowsDiskReadLatencyHigh, eval_time: 15m, exp_alerts: []}
- alertname: WindowsDiskReadLatencyHigh
eval_time: 16m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskreadlatencyhigh/
summary: Windows disk read latency is high
description: 'Volume C: on host:9182 has averaged more than 0.05 seconds per read operation for 15 minutes.'
- {alertname: WindowsDiskReadLatencyHigh, eval_time: 25m, exp_alerts: []}
- {alertname: WindowsDiskWriteLatencyHigh, eval_time: 15m, exp_alerts: []}
- alertname: WindowsDiskWriteLatencyHigh
eval_time: 16m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskwritelatencyhigh/
summary: Windows disk write latency is high
description: 'Volume C: on host:9182 has averaged more than 0.05 seconds per write operation for 15 minutes.'
- {alertname: WindowsDiskWriteLatencyHigh, eval_time: 25m, exp_alerts: []}
- {alertname: WindowsNetworkErrors, eval_time: 17m, exp_alerts: []}
- alertname: WindowsNetworkErrors
eval_time: 18m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, nic: Ethernet, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsnetworkerrors/
summary: Windows network packet errors are high
description: 'Packet errors on interface Ethernet on host:9182 have exceeded 1 per second for 15 minutes.'
- {alertname: WindowsNetworkErrors, eval_time: 25m, exp_alerts: []}
- {alertname: WindowsNetworkDiscards, eval_time: 17m, exp_alerts: []}
- alertname: WindowsNetworkDiscards
eval_time: 18m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, nic: Ethernet, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsnetworkdiscards/
summary: Windows network packet discards are high
description: 'Packet discards on interface Ethernet on host:9182 have exceeded 1 per second for 15 minutes.'
- {alertname: WindowsNetworkDiscards, eval_time: 25m, exp_alerts: []}
- name: New resource alerts stay quiet at exact thresholds and clock alerts are disabled
interval: 1m
input_series:
- series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182"}'
values: '90x20'
- series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182"}'
values: '100x20'
- series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+6x20'
- series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+120x20'
- series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+6x20'
- series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+120x20'
- series: 'windows_net_packets_received_errors_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+30x20'
- series: 'windows_net_packets_outbound_errors_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+30x20'
- series: 'windows_net_packets_received_discarded_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+30x20'
- series: 'windows_net_packets_outbound_discarded_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+30x20'
- series: 'windows_time_ntp_client_time_sources{job="windows_exporter", instance="host:9182"}'
values: '0x20'
- series: 'windows_time_computed_time_offset_seconds{job="windows_exporter", instance="host:9182"}'
values: '1x20'
alert_rule_test:
- {alertname: WindowsMemoryCommitHighUsage, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsDiskReadLatencyHigh, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsDiskWriteLatencyHigh, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsNetworkErrors, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsNetworkDiscards, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsClockNotSynchronising, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsClockSkewDetected, eval_time: 20m, exp_alerts: []}
- name: Exporter and collector failures wait five minutes and recover
interval: 1m
input_series:
@@ -25,6 +135,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', severity: critical}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsexporterdown/
summary: Windows exporter is unreachable
description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows_exporter).'
- alertname: WindowsExporterDown
@@ -38,6 +149,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', collector: cpu, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowscollectorfailed/
summary: Windows collector failed
description: 'Collector cpu on host:9182 has failed for at least 5 minutes.'
- alertname: WindowsCollectorFailed
@@ -62,6 +174,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowscpuhighusage/
summary: Windows CPU usage is high
description: 'CPU usage on host:9182 has exceeded 90% for 15 minutes.'
- alertname: WindowsCPUHighUsage
@@ -75,6 +188,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsmemoryhighusage/
summary: Windows memory usage is high
description: 'Physical memory usage on host:9182 has exceeded 90% for 15 minutes.'
- alertname: WindowsMemoryHighUsage
@@ -97,10 +211,12 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskalmostfull/
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.'
- exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: critical}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskalmostfull/
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 5% free space. Windows may delay space metrics by 10–15 minutes.'
- alertname: WindowsDiskAlmostFull
@@ -153,6 +269,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskfillingup/
summary: Windows disk is predicted to fill up
description: 'Volume C: on host:9182 is predicted to fill within 24 hours.'
- alertname: WindowsDiskFillingUp
@@ -160,6 +277,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskfillingup/
summary: Windows disk is predicted to fill up
description: 'Volume C: on host:9182 is predicted to fill within 24 hours.'
contrib/mixin/tests/check_runbooks.py
+36
View File
@@ -0,0 +1,36 @@
"""Check every mixin alert has a complete runbook at its default URL."""
import json
from pathlib import Path
import sys
from urllib.parse import urlparse
runbooks = Path(sys.argv[1])
alerts = json.load(sys.stdin)
errors = []
names = set()
for group in alerts["groups"]:
for rule in group["rules"]:
if "alert" not in rule:
continue
name = rule["alert"]
names.add(name)
slug = name.lower()
expected_path = f"/windows_exporter/runbooks/{slug}/"
if urlparse(rule["annotations"].get("runbook_url", "")).path != expected_path:
errors.append(f"{name}: runbook URL must end in {expected_path}")
path = runbooks / f"{slug}.md"
if not path.is_file():
errors.append(f"{name}: missing {path}")
continue
content = path.read_text(encoding="utf-8")
for section in ("Meaning", "Impact", "Diagnosis", "Mitigation"):
if f"## {section}\n" not in content:
errors.append(f"{name}: missing {section} section in {path}")
if "TODO" in content:
errors.append(f"{name}: unfinished runbook {path}")
if errors:
sys.exit("\n".join(errors))
print(f"Checked runbooks and URLs for {len(names)} alerts")
contrib/mixin/tests/config.libsonnet
+7
View File
@@ -10,6 +10,13 @@
diskFreeCriticalThreshold: 10,
criticalSeverity: 'page',
enableActiveDirectory: true,
enableTime: true,
runbookURLPattern: 'https://runbooks.example.com/windows/%s/',
memoryCommitHighUsageThreshold: 80,
diskLatencyThresholdSeconds: 0.02,
networkErrorRateThreshold: 0.5,
networkDiscardRateThreshold: 0.5,
clockOffsetThresholdSeconds: 0.1,
adPendingReplicationThreshold: 10,
adReplicationFailureRateThreshold: 0.5,
},
contrib/mixin/tests/custom.test.yaml
+129
View File
@@ -6,8 +6,130 @@ group_eval_order:
- windows-exporter-recording
- windows-exporter-alerts
- windows-active-directory
- windows-time
tests:
- name: Custom commit, latency and network thresholds preserve selectors and device filters
interval: 1m
input_series:
- series: 'windows_memory_committed_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
values: '85x20'
- series: 'windows_memory_commit_limit{job="windows-custom", instance="host:9182", cluster="test"}'
values: '100x20'
- series: 'windows_memory_committed_bytes{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '99x20'
- series: 'windows_memory_commit_limit{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '100x20'
- series: 'windows_logical_disk_read_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '0+3.6x20'
- series: 'windows_logical_disk_reads_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '0+120x20'
- series: 'windows_logical_disk_write_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '0+3.6x20'
- series: 'windows_logical_disk_writes_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '0+120x20'
- series: 'windows_logical_disk_read_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '0+12x20'
- series: 'windows_logical_disk_reads_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '0+120x20'
- series: 'windows_logical_disk_write_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '0+12x20'
- series: 'windows_logical_disk_writes_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '0+120x20'
- series: 'windows_net_packets_received_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
values: '0+36x20'
- series: 'windows_net_packets_outbound_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
values: '0x20'
- series: 'windows_net_packets_received_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
values: '0x20'
- series: 'windows_net_packets_outbound_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
values: '0+36x20'
- series: 'windows_net_packets_received_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
values: '0+120x20'
- series: 'windows_net_packets_outbound_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
values: '0+120x20'
- series: 'windows_net_packets_received_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
values: '0+120x20'
- series: 'windows_net_packets_outbound_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
values: '0+120x20'
alert_rule_test:
- alertname: WindowsMemoryCommitHighUsage
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsmemorycommithighusage/
summary: Windows memory commit usage is high
description: 'Committed memory on host:9182 has exceeded 80% of the commit limit for 15 minutes.'
- alertname: WindowsDiskReadLatencyHigh
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsdiskreadlatencyhigh/
summary: Windows disk read latency is high
description: 'Volume C: on host:9182 has averaged more than 0.02 seconds per read operation for 15 minutes.'
- alertname: WindowsDiskWriteLatencyHigh
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsdiskwritelatencyhigh/
summary: Windows disk write latency is high
description: 'Volume C: on host:9182 has averaged more than 0.02 seconds per write operation for 15 minutes.'
- alertname: WindowsNetworkErrors
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, nic: Ethernet, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsnetworkerrors/
summary: Windows network packet errors are high
description: 'Packet errors on interface Ethernet on host:9182 have exceeded 0.5 per second for 15 minutes.'
- alertname: WindowsNetworkDiscards
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, nic: Ethernet, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsnetworkdiscards/
summary: Windows network packet discards are high
description: 'Packet discards on interface Ethernet on host:9182 have exceeded 0.5 per second for 15 minutes.'
- name: Clock alerts opt in, use absolute offset, custom URLs and recover
interval: 1m
input_series:
- series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="host:9182", cluster="test"}'
values: '0x10 1x10'
- series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="host:9182", cluster="test"}'
values: '-0.2x10 0x10'
- series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="healthy:9182", cluster="test"}'
values: '1x21'
- series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="healthy:9182", cluster="test"}'
values: '0.1x21'
- series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '0x21'
- series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '-1x21'
alert_rule_test:
- {alertname: WindowsClockNotSynchronising, eval_time: 9m, exp_alerts: []}
- alertname: WindowsClockNotSynchronising
eval_time: 10m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsclocknotsynchronising/
summary: Windows clock has no NTP time source
description: 'Windows Time on host:9182 has had no active NTP time source for 10 minutes.'
- {alertname: WindowsClockNotSynchronising, eval_time: 11m, exp_alerts: []}
- {alertname: WindowsClockSkewDetected, eval_time: 9m, exp_alerts: []}
- alertname: WindowsClockSkewDetected
eval_time: 10m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsclockskewdetected/
summary: Windows clock offset is high
description: 'Windows Time on host:9182 reports a clock offset above 0.1 seconds for 10 minutes.'
- {alertname: WindowsClockSkewDetected, eval_time: 11m, exp_alerts: []}
- name: Custom selectors, thresholds and critical severity propagate to alerts
interval: 1m
input_series:
@@ -50,6 +172,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: page}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsexporterdown/
summary: Windows exporter is unreachable
description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows-custom).'
- alertname: WindowsCPUHighUsage
@@ -57,6 +180,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowscpuhighusage/
summary: Windows CPU usage is high
description: 'CPU usage on host:9182 has exceeded 80% for 15 minutes.'
- alertname: WindowsMemoryHighUsage
@@ -64,6 +188,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsmemoryhighusage/
summary: Windows memory usage is high
description: 'Physical memory usage on host:9182 has exceeded 85% for 15 minutes.'
- alertname: WindowsDiskAlmostFull
@@ -71,10 +196,12 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsdiskalmostfull/
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 20% free space. Windows may delay space metrics by 10–15 minutes.'
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: page}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsdiskalmostfull/
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.'
@@ -108,6 +235,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsadpendingreplication/
summary: Active Directory replication queue is high
description: 'Pending replication operations on host:9182 have exceeded 10 for 15 minutes.'
- alertname: WindowsADPendingReplication
@@ -121,6 +249,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsadreplicationfailures/
summary: Active Directory replication requests are failing
description: 'Replication synchronization requests on host:9182 have been failing for 15 minutes.'
- alertname: WindowsADReplicationFailures
contrib/mixin/tests/resources.test.yaml
+183
View File
@@ -0,0 +1,183 @@
rule_files:
- ../windows_rules.yaml
- ../windows_alerts.yaml
evaluation_interval: 1m
group_eval_order:
- windows-exporter-recording
- windows-exporter-alerts
tests:
- name: Memory cache includes all lists, commit is a ratio and paging is a rate
interval: 1m
input_series:
- series: 'windows_memory_cache_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '10x10'
- series: 'windows_memory_modified_page_list_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '20x10'
- series: 'windows_memory_standby_cache_core_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '30x10'
- series: 'windows_memory_standby_cache_normal_priority_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '40x10'
- series: 'windows_memory_standby_cache_reserve_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '50x10'
- series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '80x10'
- series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '100x10'
- series: 'windows_memory_swap_page_operations_total{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '0+120x10'
- series: 'windows_memory_committed_bytes{job="other", instance="host:9182"}'
values: '95x10'
- series: 'windows_memory_commit_limit{job="other", instance="host:9182"}'
values: '100x10'
promql_expr_test:
- expr: windows:memory_cached:bytes
eval_time: 10m
exp_samples:
- labels: 'windows:memory_cached:bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 150
- expr: windows:memory_committed:ratio
eval_time: 10m
exp_samples:
- labels: 'windows:memory_committed:ratio{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 0.8
- expr: windows:memory_swap_page_operations:rate
eval_time: 10m
exp_samples:
- labels: 'windows:memory_swap_page_operations:rate{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 2
- name: Disk latency uses operations, handles resets and busy time stays bounded
interval: 1m
input_series:
- series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+12x5 12+12x4'
- series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+120x5 120+120x4'
- series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+24x10'
- series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+240x10'
- series: 'windows_logical_disk_idle_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+30x10'
- series: 'windows_logical_disk_idle_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}'
values: '0+90x10'
promql_expr_test:
- expr: windows:logical_disk_read_operations:rate
eval_time: 8m
exp_samples:
- labels: 'windows:logical_disk_read_operations:rate{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 2
- expr: windows:logical_disk_write_operations:rate
eval_time: 10m
exp_samples:
- labels: 'windows:logical_disk_write_operations:rate{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 4
- expr: windows:logical_disk_read_latency:seconds
eval_time: 8m
exp_samples:
- labels: 'windows:logical_disk_read_latency:seconds{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 0.1
- expr: windows:logical_disk_write_latency:seconds
eval_time: 10m
exp_samples:
- labels: 'windows:logical_disk_write_latency:seconds{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 0.1
- expr: windows:logical_disk_busy:ratio
eval_time: 10m
exp_samples:
- labels: 'windows:logical_disk_busy:ratio{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 0.5
- labels: 'windows:logical_disk_busy:ratio{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}'
value: 0
- name: Full duplex network utilization uses bytes and keeps directions separate
interval: 1m
input_series:
- series: 'windows_net_bytes_received_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+600x10'
- series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+1200x10'
- series: 'windows_net_current_bandwidth_bytes{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '100x10'
- series: 'windows_net_packets_received_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+60x10'
- series: 'windows_net_packets_outbound_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+120x10'
- series: 'windows_net_packets_received_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+180x10'
- series: 'windows_net_packets_outbound_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+240x10'
promql_expr_test:
- expr: windows:net_received_utilization:ratio
eval_time: 10m
exp_samples:
- labels: 'windows:net_received_utilization:ratio{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 0.1
- expr: windows:net_sent_utilization:ratio
eval_time: 10m
exp_samples:
- labels: 'windows:net_sent_utilization:ratio{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 0.2
- expr: windows:net_received_errors:rate
eval_time: 10m
exp_samples:
- labels: 'windows:net_received_errors:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 1
- expr: windows:net_outbound_errors:rate
eval_time: 10m
exp_samples:
- labels: 'windows:net_outbound_errors:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 2
- expr: windows:net_received_discarded:rate
eval_time: 10m
exp_samples:
- labels: 'windows:net_received_discarded:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 3
- expr: windows:net_outbound_discarded:rate
eval_time: 10m
exp_samples:
- labels: 'windows:net_outbound_discarded:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 4
- name: Zero capacity and idle devices produce no undefined ratios or latency alerts
interval: 1m
input_series:
- series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182"}'
values: '1x20'
- series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182"}'
values: '0x20'
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '1x20'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182"}'
values: '1x20'
- series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182"}'
values: '0x20'
- series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_net_bytes_received_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+600x20'
- series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+600x20'
- series: 'windows_net_current_bandwidth_bytes{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0x20'
promql_expr_test:
- {expr: 'windows:memory_usage:ratio', eval_time: 20m, exp_samples: []}
- {expr: 'windows:logical_disk_free:ratio', eval_time: 20m, exp_samples: []}
- {expr: 'windows:memory_committed:ratio', eval_time: 20m, exp_samples: []}
- {expr: 'windows:logical_disk_read_latency:seconds', eval_time: 20m, exp_samples: []}
- {expr: 'windows:logical_disk_write_latency:seconds', eval_time: 20m, exp_samples: []}
- {expr: 'windows:net_received_utilization:ratio', eval_time: 20m, exp_samples: []}
- {expr: 'windows:net_sent_utilization:ratio', eval_time: 20m, exp_samples: []}
alert_rule_test:
- {alertname: WindowsMemoryCommitHighUsage, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsDiskReadLatencyHigh, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsDiskWriteLatencyHigh, eval_time: 20m, exp_alerts: []}
contrib/mixin/tests/rules.test.yaml
+24
View File
@@ -18,6 +18,20 @@ tests:
- series: 'windows_cpu_time_total{job="other", instance="host:9182", core="0", mode="idle"}'
values: '0+6x10'
promql_expr_test:
- expr: windows:cpu_cores:count
eval_time: 10m
exp_samples:
- labels: 'windows:cpu_cores:count{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 2
- labels: 'windows:cpu_cores:count{job="windows_exporter", instance="host:9182", cluster="b"}'
value: 1
- expr: 'windows:cpu_time:ratio_rate{mode="idle"}'
eval_time: 10m
exp_samples:
- labels: 'windows:cpu_time:ratio_rate{job="windows_exporter", instance="host:9182", cluster="a", mode="idle"}'
value: 0.4
- labels: 'windows:cpu_time:ratio_rate{job="windows_exporter", instance="host:9182", cluster="b", mode="idle"}'
value: 0.5
- expr: windows:cpu_usage:ratio
eval_time: 10m
exp_samples:
@@ -42,6 +56,16 @@ tests:
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume=""}'
values: '100x5'
promql_expr_test:
- expr: windows:memory_used:bytes
eval_time: 5m
exp_samples:
- labels: 'windows:memory_used:bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 75
- expr: windows:logical_disk_used:bytes
eval_time: 5m
exp_samples:
- labels: 'windows:logical_disk_used:bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 80
- expr: windows:memory_usage:ratio
eval_time: 5m
exp_samples:
contrib/mixin/tests/runbooks.test.jsonnet
+23
View File
@@ -0,0 +1,23 @@
local linked = {
_config:: { runbookURLPattern: 'https://example.com/runbooks/%s/' },
prometheusAlerts:: {
groups: [
{
name: 'custom',
rules: [
{ alert: 'WindowsCustomAlert', expr: 'up == 0', annotations: { summary: 'Existing summary' } },
{ alert: 'WindowsOverride', expr: 'up == 0', annotations: { runbook_url: 'https://example.com/override' } },
{ record: 'windows:custom:sum', expr: 'sum(up)' },
{ alert: 'WindowsNoAnnotations', expr: 'up == 0' },
],
},
],
},
} + (import '../lib/add-runbook-links.libsonnet');
local rules = linked.prometheusAlerts.groups[0].rules;
assert rules[0].annotations == { summary: 'Existing summary', runbook_url: 'https://example.com/runbooks/windowscustomalert/' };
assert rules[1].annotations.runbook_url == 'https://example.com/override';
assert !std.objectHas(rules[2], 'annotations');
assert rules[3].annotations.runbook_url == 'https://example.com/runbooks/windowsnoannotations/';
assert rules[0].expr == 'up == 0' && linked.prometheusAlerts.groups[0].name == 'custom';
{ result: 'Runbook link composition passed' }