contrib: add Windows exporter monitoring mixin (#2558)

Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
This commit is contained in:
Jan-Otto Kröpke
2026-10-08 23:29:23 +02:00
committed by GitHub
parent 64559e8f76
commit ad1db73b26
19 changed files with 904 additions and 19 deletions
+22 -19
View File
@@ -1,19 +1,22 @@
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
tab_width = 4
[*.{json,wxs,xml}]
indent_style = space
indent_size = 4
[renovate.json]
indent_size = 2
[*.{yml,yaml}]
indent_style = space
indent_size = 2
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
tab_width = 4
[*.{json,wxs,xml}]
indent_style = space
indent_size = 4
[renovate.json]
indent_size = 2
[contrib/mixin/{jsonnetfile.json,jsonnetfile.lock.json}]
indent_size = 2
[*.{yml,yaml}]
indent_style = space
indent_size = 2
+44
View File
@@ -0,0 +1,44 @@
name: Mixin
on:
pull_request:
paths:
- contrib/mixin/**
- .github/workflows/mixin.yml
push:
branches:
- master
- next
- main
paths:
- contrib/mixin/**
- .github/workflows/mixin.yml
workflow_dispatch:
permissions: {}
jobs:
rules:
name: Validate Prometheus rules
runs-on: ubuntu-24.04
timeout-minutes: 15
permissions:
contents: read
defaults:
run:
working-directory: contrib/mixin
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: 'false'
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version: '1.27.1'
- name: Install pinned tools
run: make tools
- name: Lint and test rules
run: |
make test PROMTOOL="docker run --rm --entrypoint promtool -v \"$PWD:/mixin:ro\" -w /mixin prom/prometheus:v3.15.0"
+6
View File
@@ -190,6 +190,12 @@ The Docker image is tagged with the version of the exporter. The `latest` tag is
See detailed steps to install on Windows Kubernetes [here](./kubernetes/kubernetes.md).
## Prometheus rules
The [Windows exporter mixin](contrib/mixin/README.md) provides configurable
recording rules and alerts for Prometheus, with Jsonnet generation, pint linting,
and promtool tests.
## Supported versions
`windows_exporter` supports Windows Server versions 2016 and later, and desktop Windows version 10 and 11 (21H2 or later).
+5
View File
@@ -0,0 +1,5 @@
/vendor/
/windows_alerts.yaml
/windows_rules.yaml
/tests/*_alerts.yaml
/tests/*_rules.yaml
+24
View File
@@ -0,0 +1,24 @@
// CI validates generated rules without contacting a Prometheus server.
rule {
match {
kind = "alerting"
}
label "severity" {
required = true
value = ".+"
severity = "bug"
}
annotation "summary" {
required = true
value = ".+"
severity = "bug"
}
annotation "description" {
required = true
value = ".+"
severity = "bug"
}
}
+51
View File
@@ -0,0 +1,51 @@
JSONNET ?= jsonnet
JSONNETFMT ?= jsonnetfmt
PROMTOOL ?= promtool
PINT ?= pint
# renovate: go=github.com/google/go-jsonnet
JSONNET_VERSION := v0.22.0
# renovate: go=github.com/cloudflare/pint
PINT_VERSION := v0.89.2
SOURCES := $(wildcard *.libsonnet *.jsonnet lib/*.libsonnet rules/*.libsonnet alerts/*.libsonnet tests/*.libsonnet)
FMT_FLAGS := -n 2 --max-blank-lines 2 --string-style s --comment-style s
.DEFAULT_GOAL := generate
.DELETE_ON_ERROR:
.PHONY: generate tools fmt lint test clean
generate: windows_rules.yaml windows_alerts.yaml
tools:
go install github.com/google/go-jsonnet/cmd/jsonnet@$(JSONNET_VERSION)
go install github.com/google/go-jsonnet/cmd/jsonnetfmt@$(JSONNET_VERSION)
go install github.com/cloudflare/pint/cmd/pint@$(PINT_VERSION)
windows_rules.yaml: $(SOURCES)
$(JSONNET) -S rules.jsonnet > $@
windows_alerts.yaml: $(SOURCES)
$(JSONNET) -S alerts.jsonnet > $@
tests/custom_rules.yaml: $(SOURCES)
$(JSONNET) -S -e 'std.manifestYamlDoc((import "tests/config.libsonnet").prometheusRules)' > $@
tests/custom_alerts.yaml: $(SOURCES)
$(JSONNET) -S -e 'std.manifestYamlDoc((import "tests/config.libsonnet").prometheusAlerts)' > $@
fmt:
$(JSONNETFMT) $(FMT_FLAGS) -i $(SOURCES)
lint: generate tests/custom_rules.yaml tests/custom_alerts.yaml
@set -e; for file in $(SOURCES); do \
$(JSONNETFMT) $(FMT_FLAGS) "$$file" | diff -u "$$file" -; \
done
$(PROMTOOL) check rules windows_rules.yaml windows_alerts.yaml tests/custom_rules.yaml tests/custom_alerts.yaml
$(PINT) --offline lint windows_rules.yaml windows_alerts.yaml
$(PINT) --offline lint tests/custom_rules.yaml tests/custom_alerts.yaml
test: lint
$(PROMTOOL) test rules tests/rules.test.yaml tests/alerts.test.yaml tests/custom.test.yaml
clean:
rm -f windows_rules.yaml windows_alerts.yaml tests/custom_rules.yaml tests/custom_alerts.yaml
+165
View File
@@ -0,0 +1,165 @@
# Windows exporter mixin
Configurable Prometheus recording rules and alerts for windows_exporter. The
source uses Jsonnet and exposes the standard `prometheusRules` and
`prometheusAlerts` mixin fields. Dashboard source is maintained separately.
## Generate rules
Install the pinned Jsonnet and [pint](https://github.com/cloudflare/pint) tools:
```sh
cd contrib/mixin
make tools
export PATH="$(go env GOPATH)/bin:$PATH"
make generate
```
The mixin has no Jsonnet library dependencies, so `jb install` is not required
to generate rules. Generated files are ignored by Git:
- `windows_rules.yaml`: recording rules.
- `windows_alerts.yaml`: alerting rules.
The default selector is `job="windows_exporter"`. Change it to match your scrape
job before generating rules. Resource rules use the default `cpu`, `memory`,
`logical_disk`, and `net` collectors. Enable `ad` on domain controllers when
opting into Active Directory alerts.
Copy both generated files into your Prometheus rule directory and reference
them in `prometheus.yml`:
```yaml
rule_files:
- /etc/prometheus/rules/windows_rules.yaml
- /etc/prometheus/rules/windows_alerts.yaml
```
Resource alerts depend on the recording rules; deploy both files. Separate rule
groups can add up to one evaluation interval of delay before alerts observe a
new recording-rule value.
## Customize or compose
Edit `_config` in `config.libsonnet`, or import and extend the mixin without
modifying its source. For example, create `custom.libsonnet` in this directory:
```jsonnet
(import 'mixin.libsonnet') {
_config+:: {
windowsExporterSelector: 'job=~"windows.*", environment="production"',
volumeSelector: 'volume!~"HarddiskVolume.*"',
nicSelector: 'nic!="Loopback"',
rateInterval: '5m',
cpuHighUsageThreshold: 95,
enableActiveDirectory: true,
},
}
```
Render this configuration with:
```sh
jsonnet -S -e 'std.manifestYamlDoc((import "custom.libsonnet").prometheusRules)' > custom_rules.yaml
jsonnet -S -e 'std.manifestYamlDoc((import "custom.libsonnet").prometheusAlerts)' > custom_alerts.yaml
```
Selectors are comma-separated Prometheus label matchers without braces. They
apply to both recording rules and alerts. All target labels, including `job`,
`instance`, `cluster`, and custom selector labels, are preserved. CPU usage
averages idle rates across cores and removes only `core` and `mode`. Disk and
network recordings also retain `volume` and `nic` respectively.
`rateInterval` is the fixed window for recording rules and Active Directory
failure rates; keep it at least four times your scrape interval. Grafana's
`$__rate_interval` is a dashboard variable and must not be used in Prometheus
rules.
## Recording rules
| Metric | Meaning | Unit |
| --- | --- | --- |
| `windows:cpu_usage:ratio` | Non-idle CPU time averaged across cores | Ratio, 0–1 |
| `windows:memory_usage:ratio` | Physical memory in use, based on available bytes | Ratio, 0–1 |
| `windows:logical_disk_free:ratio` | Free disk space per volume | Ratio, 0–1 |
| `windows:logical_disk_read_bytes:rate` | Disk read throughput per volume | Bytes/second |
| `windows:logical_disk_write_bytes:rate` | Disk write throughput per volume | Bytes/second |
| `windows:net_received_bytes:rate` | Received network traffic per interface | Bytes/second |
| `windows:net_sent_bytes:rate` | Sent network traffic per interface | Bytes/second |
The names remain stable when `rateInterval` changes. Dashboard authors can use
these recordings or query raw exporter metrics directly. CPU usage measures
non-idle time; it is not the frequency-adjusted processor utility shown in
Windows Task Manager.
## Alerts
Threshold configuration values below are percentages except where noted.
| Alert | Default condition | Duration | Severity |
| --- | --- | --- | --- |
| `WindowsExporterDown` | Scrape target `up == 0` | 5m | Critical |
| `WindowsCollectorFailed` | `windows_exporter_collector_success == 0` | 5m | Warning |
| `WindowsCPUHighUsage` | CPU usage above `cpuHighUsageThreshold` (90%) | 15m | Warning |
| `WindowsMemoryHighUsage` | Memory usage above `memoryHighUsageThreshold` (90%) | 15m | Warning |
| `WindowsDiskAlmostFull` | Free space below `diskFreeWarningThreshold` (10%) | 15m | Warning |
| `WindowsDiskAlmostFull` | Free space below `diskFreeCriticalThreshold` (5%) | 15m | Critical |
| `WindowsDiskFillingUp` | Below the warning threshold and predicted to fill within `diskPredictionHours` (24h), using `diskPredictionWindow` (6h) | 1h | Warning |
| `WindowsADPendingReplication` | Pending operations above `adPendingReplicationThreshold` (50) | 15m | Warning |
| `WindowsADReplicationFailures` | Failed synchronization requests above `adReplicationFailureRateThreshold` (0 requests/second) | 15m | Warning |
Active Directory alerts are generated only when `enableActiveDirectory` is
`true`. `criticalSeverity` defaults to `critical` and can be overridden to match
your routing labels. Below the critical disk threshold, both disk severity
levels fire; configure Alertmanager inhibition if only the critical
notification should be sent.
Windows disk size/free-space counters may lag by 10–15 minutes. Predictions
depend on a sustained trend and are intentionally delayed for an hour. A target
removed from service discovery has no `up` series and will not trigger
`WindowsExporterDown`. Missing resource metrics produce no resource alerts;
scrape and collector-failure alerts cover explicit failures, not disabled
collectors.
## Validate
Install `promtool` from a [Prometheus release](https://github.com/prometheus/prometheus/releases),
then run:
```sh
make test
```
This checks Jsonnet formatting, validates generated rules with `promtool`, runs
`pint --offline lint`, and evaluates the rules against synthetic time series.
Tests cover all seven recordings, alert timing and recovery, threshold
boundaries, counter resets, disk forecasts, target-label preservation, and
custom selectors/thresholds with Active Directory enabled. CI uses Prometheus
3.15.0 and the same pinned tool versions as `make tools`.
If you prefer to run `promtool` through Docker:
```sh
make test PROMTOOL="docker run --rm --entrypoint promtool -v \"$PWD:/mixin:ro\" -w /mixin prom/prometheus:v3.15.0"
```
`.pint.hcl` requires a non-empty severity label and summary/description
annotations on alerts, in addition to pint's built-in checks. CI uses offline
mode without a Prometheus endpoint. To validate metric availability and labels
against your own server, copy `.pint.hcl` to a local configuration, add:
```hcl
prometheus "local" {
uri = "http://localhost:9090"
}
```
Then run `pint --config /path/to/local.pint.hcl lint windows_rules.yaml windows_alerts.yaml`
without `--offline`. See the [pint documentation](https://cloudflare.github.io/pint/)
for available checks.
Run `make fmt` after source changes and `make clean` to remove generated rule
files. For downstream composition with Jsonnet Bundler, import
`github.com/prometheus-community/windows_exporter/contrib/mixin/mixin.libsonnet`
and extend `_config`. See the [monitoring mixin documentation](https://github.com/monitoring-mixins/docs)
for the standard composition workflow.
+1
View File
@@ -0,0 +1 @@
std.manifestYamlDoc((import 'mixin.libsonnet').prometheusAlerts)
+101
View File
@@ -0,0 +1,101 @@
{
local c = self._config,
local q = (import '../lib/queries.libsonnet')(c),
local alert(name, expr, duration, severity, summary, description) = {
alert: name,
expr: expr,
'for': duration,
labels: { severity: severity },
annotations: { summary: summary, description: description },
},
prometheusAlerts+:: {
groups: [
{
name: 'windows-exporter-alerts',
rules: [
alert(
'WindowsExporterDown',
q.metric('up') + ' == 0',
'5m',
c.criticalSeverity,
'Windows exporter is unreachable',
'Prometheus cannot scrape Windows exporter on {{ $labels.instance }} (job {{ $labels.job }}).'
),
alert(
'WindowsCollectorFailed',
q.metric('windows_exporter_collector_success') + ' == 0',
'5m',
'warning',
'Windows collector failed',
'Collector {{ $labels.collector }} on {{ $labels.instance }} has failed for at least 5 minutes.'
),
alert(
'WindowsCPUHighUsage',
q.metric('windows:cpu_usage:ratio') + ' * 100 > %s' % c.cpuHighUsageThreshold,
'15m',
'warning',
'Windows CPU usage is high',
'CPU usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.cpuHighUsageThreshold
),
alert(
'WindowsMemoryHighUsage',
q.metric('windows:memory_usage:ratio') + ' * 100 > %s' % c.memoryHighUsageThreshold,
'15m',
'warning',
'Windows memory usage is high',
'Physical memory usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.memoryHighUsageThreshold
),
] + [
alert(
'WindowsDiskAlmostFull',
q.metric('windows:logical_disk_free:ratio', [c.volumeSelector]) + ' * 100 < %s' % entry[1],
'15m',
entry[0],
'Windows disk space is low',
'Volume {{ $labels.volume }} on {{ $labels.instance }} has less than %s%% free space. Windows may delay space metrics by 10–15 minutes.' % entry[1]
)
for entry in [
['warning', c.diskFreeWarningThreshold],
[c.criticalSeverity, c.diskFreeCriticalThreshold],
]
] + [
alert(
'WindowsDiskFillingUp',
q.metric('windows:logical_disk_free:ratio', [c.volumeSelector]) + ' * 100 < %s and predict_linear(%s[%s], %s) < 0' % [
c.diskFreeWarningThreshold,
q.metric('windows_logical_disk_free_bytes', [c.volumeSelector]),
c.diskPredictionWindow,
c.diskPredictionHours * 3600,
],
'1h',
'warning',
'Windows disk is predicted to fill up',
'Volume {{ $labels.volume }} on {{ $labels.instance }} is predicted to fill within %s hours.' % c.diskPredictionHours
),
],
},
] + if c.enableActiveDirectory then [
{
name: 'windows-active-directory',
rules: [
alert(
'WindowsADPendingReplication',
q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold,
'15m',
'warning',
'Active Directory replication queue is high',
'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold
),
alert(
'WindowsADReplicationFailures',
'(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold,
'15m',
'warning',
'Active Directory replication requests are failing',
'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.'
),
],
},
] else [],
},
}
+22
View File
@@ -0,0 +1,22 @@
{
_config+:: {
// Prometheus label matchers, without enclosing braces.
windowsExporterSelector: 'job="windows_exporter"',
volumeSelector: 'volume!=""',
nicSelector: 'nic!=""',
rateInterval: '5m',
// Opt in only for targets with the ad collector enabled.
enableActiveDirectory: false,
cpuHighUsageThreshold: 90,
memoryHighUsageThreshold: 90,
diskFreeWarningThreshold: 10,
diskFreeCriticalThreshold: 5,
diskPredictionWindow: '6h',
diskPredictionHours: 24,
criticalSeverity: 'critical',
adPendingReplicationThreshold: 50,
adReplicationFailureRateThreshold: 0,
},
}
+5
View File
@@ -0,0 +1,5 @@
{
"version": 1,
"dependencies": [],
"legacyImports": false
}
+18
View File
@@ -0,0 +1,18 @@
function(c)
{
selector(extra=[]): std.join(', ', [s for s in [c.windowsExporterSelector] + extra if s != '']),
metric(name, extra=[]): '%s{%s}' % [name, self.selector(extra)],
rate(name, extra=[]): 'rate(%s[%s])' % [self.metric(name, extra), c.rateInterval],
cpuUsage:
'1 - avg without (core, mode) (%s)' % self.rate('windows_cpu_time_total', ['mode="idle"']),
memoryUsage:
'1 - %s / %s' % [
self.metric('windows_memory_available_bytes'),
self.metric('windows_memory_physical_total_bytes'),
],
diskFree:
'%s / %s' % [
self.metric('windows_logical_disk_free_bytes', [c.volumeSelector]),
self.metric('windows_logical_disk_size_bytes', [c.volumeSelector]),
],
}
+3
View File
@@ -0,0 +1,3 @@
(import 'config.libsonnet') +
(import 'rules/rules.libsonnet') +
(import 'alerts/alerts.libsonnet')
+1
View File
@@ -0,0 +1 @@
std.manifestYamlDoc((import 'mixin.libsonnet').prometheusRules)
+28
View File
@@ -0,0 +1,28 @@
{
local c = self._config,
local q = (import '../lib/queries.libsonnet')(c),
prometheusRules+:: {
groups: [
{
name: 'windows-exporter-recording',
rules: [
{ record: 'windows:cpu_usage:ratio', expr: q.cpuUsage },
{ record: 'windows:memory_usage:ratio', expr: q.memoryUsage },
{ record: 'windows:logical_disk_free:ratio', expr: q.diskFree },
] + [
{
record: 'windows:logical_disk_%s_bytes:rate' % direction,
expr: q.rate('windows_logical_disk_%s_bytes_total' % direction, [c.volumeSelector]),
}
for direction in ['read', 'write']
] + [
{
record: 'windows:net_%s_bytes:rate' % direction,
expr: q.rate('windows_net_bytes_%s_total' % direction, [c.nicSelector]),
}
for direction in ['received', 'sent']
],
},
],
},
}
+175
View File
@@ -0,0 +1,175 @@
rule_files:
- ../windows_rules.yaml
- ../windows_alerts.yaml
evaluation_interval: 1m
group_eval_order:
- windows-exporter-recording
- windows-exporter-alerts
tests:
- name: Exporter and collector failures wait five minutes and recover
interval: 1m
input_series:
- series: 'up{job="windows_exporter", instance="host:9182"}'
values: '0x5 1x5'
- series: 'up{job="other", instance="other:9182"}'
values: '0x10'
- series: 'windows_exporter_collector_success{job="windows_exporter", instance="host:9182", collector="cpu"}'
values: '0x5 1x5'
alert_rule_test:
- alertname: WindowsExporterDown
eval_time: 4m
exp_alerts: []
- alertname: WindowsExporterDown
eval_time: 5m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', severity: critical}
exp_annotations:
summary: Windows exporter is unreachable
description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows_exporter).'
- alertname: WindowsExporterDown
eval_time: 6m
exp_alerts: []
- alertname: WindowsCollectorFailed
eval_time: 4m
exp_alerts: []
- alertname: WindowsCollectorFailed
eval_time: 5m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', collector: cpu, severity: warning}
exp_annotations:
summary: Windows collector failed
description: 'Collector cpu on host:9182 has failed for at least 5 minutes.'
- alertname: WindowsCollectorFailed
eval_time: 6m
exp_alerts: []
- name: Resource alerts use recording rules and wait fifteen minutes
interval: 1m
input_series:
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="0", mode="idle"}'
values: '0+3x20 120+60x10'
- series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '5x20 50x10'
- series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '100x31'
alert_rule_test:
- alertname: WindowsCPUHighUsage
eval_time: 15m
exp_alerts: []
- alertname: WindowsCPUHighUsage
eval_time: 16m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
exp_annotations:
summary: Windows CPU usage is high
description: 'CPU usage on host:9182 has exceeded 90% for 15 minutes.'
- alertname: WindowsCPUHighUsage
eval_time: 25m
exp_alerts: []
- alertname: WindowsMemoryHighUsage
eval_time: 14m
exp_alerts: []
- alertname: WindowsMemoryHighUsage
eval_time: 15m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
exp_annotations:
summary: Windows memory usage is high
description: 'Physical memory usage on host:9182 has exceeded 90% for 15 minutes.'
- alertname: WindowsMemoryHighUsage
eval_time: 21m
exp_alerts: []
- name: Disk alerts retain volume labels and fire at both severity levels
interval: 1m
input_series:
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '3x15 30x5'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '100x21'
alert_rule_test:
- alertname: WindowsDiskAlmostFull
eval_time: 14m
exp_alerts: []
- alertname: WindowsDiskAlmostFull
eval_time: 15m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: warning}
exp_annotations:
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.'
- exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: critical}
exp_annotations:
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 5% free space. Windows may delay space metrics by 10–15 minutes.'
- alertname: WindowsDiskAlmostFull
eval_time: 16m
exp_alerts: []
- name: Healthy resources at the exact thresholds do not fire
interval: 1m
input_series:
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", core="0", mode="idle"}'
values: '0+6x20'
- series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182"}'
values: '10x20'
- series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182"}'
values: '100x20'
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '10x20'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '100x20'
alert_rule_test:
- {alertname: WindowsCPUHighUsage, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsMemoryHighUsage, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsDiskAlmostFull, eval_time: 20m, exp_alerts: []}
- name: Prediction fires only for low and declining disk space
interval: 1m
input_series:
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '1000-1x420'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '20000x420'
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}'
values: '1000x420'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}'
values: '20000x420'
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="E:"}'
values: '500+1x420'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="E:"}'
values: '20000x420'
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="b", volume="C:"}'
values: '1000x420'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="b", volume="C:"}'
values: '20000x420'
alert_rule_test:
- alertname: WindowsDiskFillingUp
eval_time: 60m
exp_alerts: []
- alertname: WindowsDiskFillingUp
eval_time: 61m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
summary: Windows disk is predicted to fill up
description: 'Volume C: on host:9182 is predicted to fill within 24 hours.'
- alertname: WindowsDiskFillingUp
eval_time: 7h
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
summary: Windows disk is predicted to fill up
description: 'Volume C: on host:9182 is predicted to fill within 24 hours.'
- name: Missing collector metrics do not produce resource alerts
interval: 1m
input_series:
- series: 'up{job="windows_exporter", instance="host:9182"}'
values: '1x70'
alert_rule_test:
- {alertname: WindowsCPUHighUsage, eval_time: 70m, exp_alerts: []}
- {alertname: WindowsMemoryHighUsage, eval_time: 70m, exp_alerts: []}
- {alertname: WindowsDiskAlmostFull, eval_time: 70m, exp_alerts: []}
- {alertname: WindowsDiskFillingUp, eval_time: 70m, exp_alerts: []}
+16
View File
@@ -0,0 +1,16 @@
(import '../mixin.libsonnet') {
_config+:: {
windowsExporterSelector: 'job=~"windows.*", cluster="test"',
volumeSelector: 'volume!~"HarddiskVolume.*"',
nicSelector: 'nic!="Loopback"',
rateInterval: '2m',
cpuHighUsageThreshold: 80,
memoryHighUsageThreshold: 85,
diskFreeWarningThreshold: 20,
diskFreeCriticalThreshold: 10,
criticalSeverity: 'page',
enableActiveDirectory: true,
adPendingReplicationThreshold: 10,
adReplicationFailureRateThreshold: 0.5,
},
}
+128
View File
@@ -0,0 +1,128 @@
rule_files:
- custom_rules.yaml
- custom_alerts.yaml
evaluation_interval: 1m
group_eval_order:
- windows-exporter-recording
- windows-exporter-alerts
- windows-active-directory
tests:
- name: Custom selectors, thresholds and critical severity propagate to alerts
interval: 1m
input_series:
- series: 'up{job="windows-custom", instance="host:9182", cluster="test"}'
values: '0x20'
- series: 'up{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '0x20'
- series: 'windows_cpu_time_total{job="windows-custom", instance="host:9182", cluster="test", core="0", mode="idle"}'
values: '0+9x20'
- series: 'windows_memory_available_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
values: '12x20'
- series: 'windows_memory_physical_total_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
values: '100x20'
- series: 'windows_logical_disk_free_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '8x20'
- series: 'windows_logical_disk_size_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '100x20'
- series: 'windows_logical_disk_free_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '1x20'
- series: 'windows_logical_disk_size_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '100x20'
- series: 'windows_net_bytes_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
values: '0+600x20'
- series: 'windows_net_bytes_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
values: '0+600x20'
promql_expr_test:
- expr: windows:net_received_bytes:rate
eval_time: 20m
exp_samples:
- labels: 'windows:net_received_bytes:rate{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
value: 10
- expr: windows:logical_disk_free:ratio
eval_time: 20m
exp_samples:
- labels: 'windows:logical_disk_free:ratio{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
value: 0.08
alert_rule_test:
- alertname: WindowsExporterDown
eval_time: 5m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: page}
exp_annotations:
summary: Windows exporter is unreachable
description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows-custom).'
- alertname: WindowsCPUHighUsage
eval_time: 16m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
summary: Windows CPU usage is high
description: 'CPU usage on host:9182 has exceeded 80% for 15 minutes.'
- alertname: WindowsMemoryHighUsage
eval_time: 15m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
summary: Windows memory usage is high
description: 'Physical memory usage on host:9182 has exceeded 85% for 15 minutes.'
- alertname: WindowsDiskAlmostFull
eval_time: 15m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
exp_annotations:
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 20% free space. Windows may delay space metrics by 10–15 minutes.'
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: page}
exp_annotations:
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.'
- name: Active Directory opt-in honors thresholds, selectors and recovery
interval: 1m
input_series:
- series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="host:9182", cluster="test"}'
values: '20x20 0x5'
- series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="healthy:9182", cluster="test"}'
values: '10x26'
- series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '20x26'
- series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="host:9182", cluster="test"}'
values: '0+120x26'
- series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="host:9182", cluster="test"}'
values: '0+60x20 1320+120x5'
- series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="healthy:9182", cluster="test"}'
values: '0+120x26'
- series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="healthy:9182", cluster="test"}'
values: '0+120x26'
- series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '0+120x26'
- series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '0+60x26'
alert_rule_test:
- alertname: WindowsADPendingReplication
eval_time: 14m
exp_alerts: []
- alertname: WindowsADPendingReplication
eval_time: 15m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
summary: Active Directory replication queue is high
description: 'Pending replication operations on host:9182 have exceeded 10 for 15 minutes.'
- alertname: WindowsADPendingReplication
eval_time: 21m
exp_alerts: []
- alertname: WindowsADReplicationFailures
eval_time: 16m
exp_alerts: []
- alertname: WindowsADReplicationFailures
eval_time: 17m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
summary: Active Directory replication requests are failing
description: 'Replication synchronization requests on host:9182 have been failing for 15 minutes.'
- alertname: WindowsADReplicationFailures
eval_time: 23m
exp_alerts: []
+89
View File
@@ -0,0 +1,89 @@
rule_files:
- ../windows_rules.yaml
evaluation_interval: 1m
fuzzy_compare: true
tests:
- name: CPU is averaged across cores and preserves target labels
interval: 1m
input_series:
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="0", mode="idle"}'
values: '0+12x10'
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="1", mode="idle"}'
values: '0+36x10'
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="0", mode="user"}'
values: '0+48x10'
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="b", core="0", mode="idle"}'
values: '0+30x10'
- series: 'windows_cpu_time_total{job="other", instance="host:9182", core="0", mode="idle"}'
values: '0+6x10'
promql_expr_test:
- expr: windows:cpu_usage:ratio
eval_time: 10m
exp_samples:
- labels: 'windows:cpu_usage:ratio{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 0.6
- labels: 'windows:cpu_usage:ratio{job="windows_exporter", instance="host:9182", cluster="b"}'
value: 0.5
- name: Memory availability and disk free space are ratios, not percentages
interval: 1m
input_series:
- series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '25x5'
- series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '100x5'
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '20x5'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '100x5'
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume=""}'
values: '1x5'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume=""}'
values: '100x5'
promql_expr_test:
- expr: windows:memory_usage:ratio
eval_time: 5m
exp_samples:
- labels: 'windows:memory_usage:ratio{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 0.75
- expr: windows:logical_disk_free:ratio
eval_time: 5m
exp_samples:
- labels: 'windows:logical_disk_free:ratio{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 0.2
- name: Disk and network rates retain device labels and handle counter resets
interval: 1m
input_series:
- series: 'windows_logical_disk_read_bytes_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+600x10'
- series: 'windows_logical_disk_write_bytes_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+1200x10'
- series: 'windows_net_bytes_received_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+600x5 600+600x4'
- series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+1800x10'
- series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", nic=""}'
values: '0+1800x10'
promql_expr_test:
- expr: windows:logical_disk_read_bytes:rate
eval_time: 10m
exp_samples:
- labels: 'windows:logical_disk_read_bytes:rate{job="windows_exporter", instance="host:9182", volume="C:"}'
value: 10
- expr: windows:logical_disk_write_bytes:rate
eval_time: 10m
exp_samples:
- labels: 'windows:logical_disk_write_bytes:rate{job="windows_exporter", instance="host:9182", volume="C:"}'
value: 20
- expr: windows:net_received_bytes:rate
eval_time: 8m
exp_samples:
- labels: 'windows:net_received_bytes:rate{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
value: 10
- expr: windows:net_sent_bytes:rate
eval_time: 10m
exp_samples:
- labels: 'windows:net_sent_bytes:rate{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
value: 30