mirror of
https://github.com/prometheus-community/windows_exporter.git
synced 2026-10-17 10:09:05 +02:00
contrib: add Windows exporter monitoring mixin (#2558)
Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
This commit is contained in:
Jan-Otto Kröpke
@@ -0,0 +1,5 @@
|
||||
/vendor/
|
||||
/windows_alerts.yaml
|
||||
/windows_rules.yaml
|
||||
/tests/*_alerts.yaml
|
||||
/tests/*_rules.yaml
|
||||
@@ -0,0 +1,24 @@
|
||||
// CI validates generated rules without contacting a Prometheus server.
|
||||
rule {
|
||||
match {
|
||||
kind = "alerting"
|
||||
}
|
||||
|
||||
label "severity" {
|
||||
required = true
|
||||
value = ".+"
|
||||
severity = "bug"
|
||||
}
|
||||
|
||||
annotation "summary" {
|
||||
required = true
|
||||
value = ".+"
|
||||
severity = "bug"
|
||||
}
|
||||
|
||||
annotation "description" {
|
||||
required = true
|
||||
value = ".+"
|
||||
severity = "bug"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
JSONNET ?= jsonnet
|
||||
JSONNETFMT ?= jsonnetfmt
|
||||
PROMTOOL ?= promtool
|
||||
PINT ?= pint
|
||||
|
||||
# renovate: go=github.com/google/go-jsonnet
|
||||
JSONNET_VERSION := v0.22.0
|
||||
# renovate: go=github.com/cloudflare/pint
|
||||
PINT_VERSION := v0.89.2
|
||||
SOURCES := $(wildcard *.libsonnet *.jsonnet lib/*.libsonnet rules/*.libsonnet alerts/*.libsonnet tests/*.libsonnet)
|
||||
FMT_FLAGS := -n 2 --max-blank-lines 2 --string-style s --comment-style s
|
||||
|
||||
.DEFAULT_GOAL := generate
|
||||
.DELETE_ON_ERROR:
|
||||
.PHONY: generate tools fmt lint test clean
|
||||
|
||||
generate: windows_rules.yaml windows_alerts.yaml
|
||||
|
||||
tools:
|
||||
go install github.com/google/go-jsonnet/cmd/jsonnet@$(JSONNET_VERSION)
|
||||
go install github.com/google/go-jsonnet/cmd/jsonnetfmt@$(JSONNET_VERSION)
|
||||
go install github.com/cloudflare/pint/cmd/pint@$(PINT_VERSION)
|
||||
|
||||
windows_rules.yaml: $(SOURCES)
|
||||
$(JSONNET) -S rules.jsonnet > $@
|
||||
|
||||
windows_alerts.yaml: $(SOURCES)
|
||||
$(JSONNET) -S alerts.jsonnet > $@
|
||||
|
||||
tests/custom_rules.yaml: $(SOURCES)
|
||||
$(JSONNET) -S -e 'std.manifestYamlDoc((import "tests/config.libsonnet").prometheusRules)' > $@
|
||||
|
||||
tests/custom_alerts.yaml: $(SOURCES)
|
||||
$(JSONNET) -S -e 'std.manifestYamlDoc((import "tests/config.libsonnet").prometheusAlerts)' > $@
|
||||
|
||||
fmt:
|
||||
$(JSONNETFMT) $(FMT_FLAGS) -i $(SOURCES)
|
||||
|
||||
lint: generate tests/custom_rules.yaml tests/custom_alerts.yaml
|
||||
@set -e; for file in $(SOURCES); do \
|
||||
$(JSONNETFMT) $(FMT_FLAGS) "$$file" | diff -u "$$file" -; \
|
||||
done
|
||||
$(PROMTOOL) check rules windows_rules.yaml windows_alerts.yaml tests/custom_rules.yaml tests/custom_alerts.yaml
|
||||
$(PINT) --offline lint windows_rules.yaml windows_alerts.yaml
|
||||
$(PINT) --offline lint tests/custom_rules.yaml tests/custom_alerts.yaml
|
||||
|
||||
test: lint
|
||||
$(PROMTOOL) test rules tests/rules.test.yaml tests/alerts.test.yaml tests/custom.test.yaml
|
||||
|
||||
clean:
|
||||
rm -f windows_rules.yaml windows_alerts.yaml tests/custom_rules.yaml tests/custom_alerts.yaml
|
||||
@@ -0,0 +1,165 @@
|
||||
# Windows exporter mixin
|
||||
|
||||
Configurable Prometheus recording rules and alerts for windows_exporter. The
|
||||
source uses Jsonnet and exposes the standard `prometheusRules` and
|
||||
`prometheusAlerts` mixin fields. Dashboard source is maintained separately.
|
||||
|
||||
## Generate rules
|
||||
|
||||
Install the pinned Jsonnet and [pint](https://github.com/cloudflare/pint) tools:
|
||||
|
||||
```sh
|
||||
cd contrib/mixin
|
||||
make tools
|
||||
export PATH="$(go env GOPATH)/bin:$PATH"
|
||||
make generate
|
||||
```
|
||||
|
||||
The mixin has no Jsonnet library dependencies, so `jb install` is not required
|
||||
to generate rules. Generated files are ignored by Git:
|
||||
|
||||
- `windows_rules.yaml`: recording rules.
|
||||
- `windows_alerts.yaml`: alerting rules.
|
||||
|
||||
The default selector is `job="windows_exporter"`. Change it to match your scrape
|
||||
job before generating rules. Resource rules use the default `cpu`, `memory`,
|
||||
`logical_disk`, and `net` collectors. Enable `ad` on domain controllers when
|
||||
opting into Active Directory alerts.
|
||||
|
||||
Copy both generated files into your Prometheus rule directory and reference
|
||||
them in `prometheus.yml`:
|
||||
|
||||
```yaml
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/windows_rules.yaml
|
||||
- /etc/prometheus/rules/windows_alerts.yaml
|
||||
```
|
||||
|
||||
Resource alerts depend on the recording rules; deploy both files. Separate rule
|
||||
groups can add up to one evaluation interval of delay before alerts observe a
|
||||
new recording-rule value.
|
||||
|
||||
## Customize or compose
|
||||
|
||||
Edit `_config` in `config.libsonnet`, or import and extend the mixin without
|
||||
modifying its source. For example, create `custom.libsonnet` in this directory:
|
||||
|
||||
```jsonnet
|
||||
(import 'mixin.libsonnet') {
|
||||
_config+:: {
|
||||
windowsExporterSelector: 'job=~"windows.*", environment="production"',
|
||||
volumeSelector: 'volume!~"HarddiskVolume.*"',
|
||||
nicSelector: 'nic!="Loopback"',
|
||||
rateInterval: '5m',
|
||||
cpuHighUsageThreshold: 95,
|
||||
enableActiveDirectory: true,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
Render this configuration with:
|
||||
|
||||
```sh
|
||||
jsonnet -S -e 'std.manifestYamlDoc((import "custom.libsonnet").prometheusRules)' > custom_rules.yaml
|
||||
jsonnet -S -e 'std.manifestYamlDoc((import "custom.libsonnet").prometheusAlerts)' > custom_alerts.yaml
|
||||
```
|
||||
|
||||
Selectors are comma-separated Prometheus label matchers without braces. They
|
||||
apply to both recording rules and alerts. All target labels, including `job`,
|
||||
`instance`, `cluster`, and custom selector labels, are preserved. CPU usage
|
||||
averages idle rates across cores and removes only `core` and `mode`. Disk and
|
||||
network recordings also retain `volume` and `nic` respectively.
|
||||
|
||||
`rateInterval` is the fixed window for recording rules and Active Directory
|
||||
failure rates; keep it at least four times your scrape interval. Grafana's
|
||||
`$__rate_interval` is a dashboard variable and must not be used in Prometheus
|
||||
rules.
|
||||
|
||||
## Recording rules
|
||||
|
||||
| Metric | Meaning | Unit |
|
||||
| --- | --- | --- |
|
||||
| `windows:cpu_usage:ratio` | Non-idle CPU time averaged across cores | Ratio, 0–1 |
|
||||
| `windows:memory_usage:ratio` | Physical memory in use, based on available bytes | Ratio, 0–1 |
|
||||
| `windows:logical_disk_free:ratio` | Free disk space per volume | Ratio, 0–1 |
|
||||
| `windows:logical_disk_read_bytes:rate` | Disk read throughput per volume | Bytes/second |
|
||||
| `windows:logical_disk_write_bytes:rate` | Disk write throughput per volume | Bytes/second |
|
||||
| `windows:net_received_bytes:rate` | Received network traffic per interface | Bytes/second |
|
||||
| `windows:net_sent_bytes:rate` | Sent network traffic per interface | Bytes/second |
|
||||
|
||||
The names remain stable when `rateInterval` changes. Dashboard authors can use
|
||||
these recordings or query raw exporter metrics directly. CPU usage measures
|
||||
non-idle time; it is not the frequency-adjusted processor utility shown in
|
||||
Windows Task Manager.
|
||||
|
||||
## Alerts
|
||||
|
||||
Threshold configuration values below are percentages except where noted.
|
||||
|
||||
| Alert | Default condition | Duration | Severity |
|
||||
| --- | --- | --- | --- |
|
||||
| `WindowsExporterDown` | Scrape target `up == 0` | 5m | Critical |
|
||||
| `WindowsCollectorFailed` | `windows_exporter_collector_success == 0` | 5m | Warning |
|
||||
| `WindowsCPUHighUsage` | CPU usage above `cpuHighUsageThreshold` (90%) | 15m | Warning |
|
||||
| `WindowsMemoryHighUsage` | Memory usage above `memoryHighUsageThreshold` (90%) | 15m | Warning |
|
||||
| `WindowsDiskAlmostFull` | Free space below `diskFreeWarningThreshold` (10%) | 15m | Warning |
|
||||
| `WindowsDiskAlmostFull` | Free space below `diskFreeCriticalThreshold` (5%) | 15m | Critical |
|
||||
| `WindowsDiskFillingUp` | Below the warning threshold and predicted to fill within `diskPredictionHours` (24h), using `diskPredictionWindow` (6h) | 1h | Warning |
|
||||
| `WindowsADPendingReplication` | Pending operations above `adPendingReplicationThreshold` (50) | 15m | Warning |
|
||||
| `WindowsADReplicationFailures` | Failed synchronization requests above `adReplicationFailureRateThreshold` (0 requests/second) | 15m | Warning |
|
||||
|
||||
Active Directory alerts are generated only when `enableActiveDirectory` is
|
||||
`true`. `criticalSeverity` defaults to `critical` and can be overridden to match
|
||||
your routing labels. Below the critical disk threshold, both disk severity
|
||||
levels fire; configure Alertmanager inhibition if only the critical
|
||||
notification should be sent.
|
||||
|
||||
Windows disk size/free-space counters may lag by 10–15 minutes. Predictions
|
||||
depend on a sustained trend and are intentionally delayed for an hour. A target
|
||||
removed from service discovery has no `up` series and will not trigger
|
||||
`WindowsExporterDown`. Missing resource metrics produce no resource alerts;
|
||||
scrape and collector-failure alerts cover explicit failures, not disabled
|
||||
collectors.
|
||||
|
||||
## Validate
|
||||
|
||||
Install `promtool` from a [Prometheus release](https://github.com/prometheus/prometheus/releases),
|
||||
then run:
|
||||
|
||||
```sh
|
||||
make test
|
||||
```
|
||||
|
||||
This checks Jsonnet formatting, validates generated rules with `promtool`, runs
|
||||
`pint --offline lint`, and evaluates the rules against synthetic time series.
|
||||
Tests cover all seven recordings, alert timing and recovery, threshold
|
||||
boundaries, counter resets, disk forecasts, target-label preservation, and
|
||||
custom selectors/thresholds with Active Directory enabled. CI uses Prometheus
|
||||
3.15.0 and the same pinned tool versions as `make tools`.
|
||||
|
||||
If you prefer to run `promtool` through Docker:
|
||||
|
||||
```sh
|
||||
make test PROMTOOL="docker run --rm --entrypoint promtool -v \"$PWD:/mixin:ro\" -w /mixin prom/prometheus:v3.15.0"
|
||||
```
|
||||
|
||||
`.pint.hcl` requires a non-empty severity label and summary/description
|
||||
annotations on alerts, in addition to pint's built-in checks. CI uses offline
|
||||
mode without a Prometheus endpoint. To validate metric availability and labels
|
||||
against your own server, copy `.pint.hcl` to a local configuration, add:
|
||||
|
||||
```hcl
|
||||
prometheus "local" {
|
||||
uri = "http://localhost:9090"
|
||||
}
|
||||
```
|
||||
|
||||
Then run `pint --config /path/to/local.pint.hcl lint windows_rules.yaml windows_alerts.yaml`
|
||||
without `--offline`. See the [pint documentation](https://cloudflare.github.io/pint/)
|
||||
for available checks.
|
||||
|
||||
Run `make fmt` after source changes and `make clean` to remove generated rule
|
||||
files. For downstream composition with Jsonnet Bundler, import
|
||||
`github.com/prometheus-community/windows_exporter/contrib/mixin/mixin.libsonnet`
|
||||
and extend `_config`. See the [monitoring mixin documentation](https://github.com/monitoring-mixins/docs)
|
||||
for the standard composition workflow.
|
||||
@@ -0,0 +1 @@
|
||||
std.manifestYamlDoc((import 'mixin.libsonnet').prometheusAlerts)
|
||||
@@ -0,0 +1,101 @@
|
||||
{
|
||||
local c = self._config,
|
||||
local q = (import '../lib/queries.libsonnet')(c),
|
||||
local alert(name, expr, duration, severity, summary, description) = {
|
||||
alert: name,
|
||||
expr: expr,
|
||||
'for': duration,
|
||||
labels: { severity: severity },
|
||||
annotations: { summary: summary, description: description },
|
||||
},
|
||||
prometheusAlerts+:: {
|
||||
groups: [
|
||||
{
|
||||
name: 'windows-exporter-alerts',
|
||||
rules: [
|
||||
alert(
|
||||
'WindowsExporterDown',
|
||||
q.metric('up') + ' == 0',
|
||||
'5m',
|
||||
c.criticalSeverity,
|
||||
'Windows exporter is unreachable',
|
||||
'Prometheus cannot scrape Windows exporter on {{ $labels.instance }} (job {{ $labels.job }}).'
|
||||
),
|
||||
alert(
|
||||
'WindowsCollectorFailed',
|
||||
q.metric('windows_exporter_collector_success') + ' == 0',
|
||||
'5m',
|
||||
'warning',
|
||||
'Windows collector failed',
|
||||
'Collector {{ $labels.collector }} on {{ $labels.instance }} has failed for at least 5 minutes.'
|
||||
),
|
||||
alert(
|
||||
'WindowsCPUHighUsage',
|
||||
q.metric('windows:cpu_usage:ratio') + ' * 100 > %s' % c.cpuHighUsageThreshold,
|
||||
'15m',
|
||||
'warning',
|
||||
'Windows CPU usage is high',
|
||||
'CPU usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.cpuHighUsageThreshold
|
||||
),
|
||||
alert(
|
||||
'WindowsMemoryHighUsage',
|
||||
q.metric('windows:memory_usage:ratio') + ' * 100 > %s' % c.memoryHighUsageThreshold,
|
||||
'15m',
|
||||
'warning',
|
||||
'Windows memory usage is high',
|
||||
'Physical memory usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.memoryHighUsageThreshold
|
||||
),
|
||||
] + [
|
||||
alert(
|
||||
'WindowsDiskAlmostFull',
|
||||
q.metric('windows:logical_disk_free:ratio', [c.volumeSelector]) + ' * 100 < %s' % entry[1],
|
||||
'15m',
|
||||
entry[0],
|
||||
'Windows disk space is low',
|
||||
'Volume {{ $labels.volume }} on {{ $labels.instance }} has less than %s%% free space. Windows may delay space metrics by 10–15 minutes.' % entry[1]
|
||||
)
|
||||
for entry in [
|
||||
['warning', c.diskFreeWarningThreshold],
|
||||
[c.criticalSeverity, c.diskFreeCriticalThreshold],
|
||||
]
|
||||
] + [
|
||||
alert(
|
||||
'WindowsDiskFillingUp',
|
||||
q.metric('windows:logical_disk_free:ratio', [c.volumeSelector]) + ' * 100 < %s and predict_linear(%s[%s], %s) < 0' % [
|
||||
c.diskFreeWarningThreshold,
|
||||
q.metric('windows_logical_disk_free_bytes', [c.volumeSelector]),
|
||||
c.diskPredictionWindow,
|
||||
c.diskPredictionHours * 3600,
|
||||
],
|
||||
'1h',
|
||||
'warning',
|
||||
'Windows disk is predicted to fill up',
|
||||
'Volume {{ $labels.volume }} on {{ $labels.instance }} is predicted to fill within %s hours.' % c.diskPredictionHours
|
||||
),
|
||||
],
|
||||
},
|
||||
] + if c.enableActiveDirectory then [
|
||||
{
|
||||
name: 'windows-active-directory',
|
||||
rules: [
|
||||
alert(
|
||||
'WindowsADPendingReplication',
|
||||
q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold,
|
||||
'15m',
|
||||
'warning',
|
||||
'Active Directory replication queue is high',
|
||||
'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold
|
||||
),
|
||||
alert(
|
||||
'WindowsADReplicationFailures',
|
||||
'(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold,
|
||||
'15m',
|
||||
'warning',
|
||||
'Active Directory replication requests are failing',
|
||||
'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.'
|
||||
),
|
||||
],
|
||||
},
|
||||
] else [],
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
_config+:: {
|
||||
// Prometheus label matchers, without enclosing braces.
|
||||
windowsExporterSelector: 'job="windows_exporter"',
|
||||
volumeSelector: 'volume!=""',
|
||||
nicSelector: 'nic!=""',
|
||||
rateInterval: '5m',
|
||||
|
||||
// Opt in only for targets with the ad collector enabled.
|
||||
enableActiveDirectory: false,
|
||||
|
||||
cpuHighUsageThreshold: 90,
|
||||
memoryHighUsageThreshold: 90,
|
||||
diskFreeWarningThreshold: 10,
|
||||
diskFreeCriticalThreshold: 5,
|
||||
diskPredictionWindow: '6h',
|
||||
diskPredictionHours: 24,
|
||||
criticalSeverity: 'critical',
|
||||
adPendingReplicationThreshold: 50,
|
||||
adReplicationFailureRateThreshold: 0,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"version": 1,
|
||||
"dependencies": [],
|
||||
"legacyImports": false
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
function(c)
|
||||
{
|
||||
selector(extra=[]): std.join(', ', [s for s in [c.windowsExporterSelector] + extra if s != '']),
|
||||
metric(name, extra=[]): '%s{%s}' % [name, self.selector(extra)],
|
||||
rate(name, extra=[]): 'rate(%s[%s])' % [self.metric(name, extra), c.rateInterval],
|
||||
cpuUsage:
|
||||
'1 - avg without (core, mode) (%s)' % self.rate('windows_cpu_time_total', ['mode="idle"']),
|
||||
memoryUsage:
|
||||
'1 - %s / %s' % [
|
||||
self.metric('windows_memory_available_bytes'),
|
||||
self.metric('windows_memory_physical_total_bytes'),
|
||||
],
|
||||
diskFree:
|
||||
'%s / %s' % [
|
||||
self.metric('windows_logical_disk_free_bytes', [c.volumeSelector]),
|
||||
self.metric('windows_logical_disk_size_bytes', [c.volumeSelector]),
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
(import 'config.libsonnet') +
|
||||
(import 'rules/rules.libsonnet') +
|
||||
(import 'alerts/alerts.libsonnet')
|
||||
@@ -0,0 +1 @@
|
||||
std.manifestYamlDoc((import 'mixin.libsonnet').prometheusRules)
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
local c = self._config,
|
||||
local q = (import '../lib/queries.libsonnet')(c),
|
||||
prometheusRules+:: {
|
||||
groups: [
|
||||
{
|
||||
name: 'windows-exporter-recording',
|
||||
rules: [
|
||||
{ record: 'windows:cpu_usage:ratio', expr: q.cpuUsage },
|
||||
{ record: 'windows:memory_usage:ratio', expr: q.memoryUsage },
|
||||
{ record: 'windows:logical_disk_free:ratio', expr: q.diskFree },
|
||||
] + [
|
||||
{
|
||||
record: 'windows:logical_disk_%s_bytes:rate' % direction,
|
||||
expr: q.rate('windows_logical_disk_%s_bytes_total' % direction, [c.volumeSelector]),
|
||||
}
|
||||
for direction in ['read', 'write']
|
||||
] + [
|
||||
{
|
||||
record: 'windows:net_%s_bytes:rate' % direction,
|
||||
expr: q.rate('windows_net_bytes_%s_total' % direction, [c.nicSelector]),
|
||||
}
|
||||
for direction in ['received', 'sent']
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
rule_files:
|
||||
- ../windows_rules.yaml
|
||||
- ../windows_alerts.yaml
|
||||
evaluation_interval: 1m
|
||||
group_eval_order:
|
||||
- windows-exporter-recording
|
||||
- windows-exporter-alerts
|
||||
|
||||
tests:
|
||||
- name: Exporter and collector failures wait five minutes and recover
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'up{job="windows_exporter", instance="host:9182"}'
|
||||
values: '0x5 1x5'
|
||||
- series: 'up{job="other", instance="other:9182"}'
|
||||
values: '0x10'
|
||||
- series: 'windows_exporter_collector_success{job="windows_exporter", instance="host:9182", collector="cpu"}'
|
||||
values: '0x5 1x5'
|
||||
alert_rule_test:
|
||||
- alertname: WindowsExporterDown
|
||||
eval_time: 4m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsExporterDown
|
||||
eval_time: 5m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows_exporter, instance: 'host:9182', severity: critical}
|
||||
exp_annotations:
|
||||
summary: Windows exporter is unreachable
|
||||
description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows_exporter).'
|
||||
- alertname: WindowsExporterDown
|
||||
eval_time: 6m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsCollectorFailed
|
||||
eval_time: 4m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsCollectorFailed
|
||||
eval_time: 5m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows_exporter, instance: 'host:9182', collector: cpu, severity: warning}
|
||||
exp_annotations:
|
||||
summary: Windows collector failed
|
||||
description: 'Collector cpu on host:9182 has failed for at least 5 minutes.'
|
||||
- alertname: WindowsCollectorFailed
|
||||
eval_time: 6m
|
||||
exp_alerts: []
|
||||
|
||||
- name: Resource alerts use recording rules and wait fifteen minutes
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="0", mode="idle"}'
|
||||
values: '0+3x20 120+60x10'
|
||||
- series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
|
||||
values: '5x20 50x10'
|
||||
- series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
|
||||
values: '100x31'
|
||||
alert_rule_test:
|
||||
- alertname: WindowsCPUHighUsage
|
||||
eval_time: 15m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsCPUHighUsage
|
||||
eval_time: 16m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
|
||||
exp_annotations:
|
||||
summary: Windows CPU usage is high
|
||||
description: 'CPU usage on host:9182 has exceeded 90% for 15 minutes.'
|
||||
- alertname: WindowsCPUHighUsage
|
||||
eval_time: 25m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsMemoryHighUsage
|
||||
eval_time: 14m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsMemoryHighUsage
|
||||
eval_time: 15m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
|
||||
exp_annotations:
|
||||
summary: Windows memory usage is high
|
||||
description: 'Physical memory usage on host:9182 has exceeded 90% for 15 minutes.'
|
||||
- alertname: WindowsMemoryHighUsage
|
||||
eval_time: 21m
|
||||
exp_alerts: []
|
||||
|
||||
- name: Disk alerts retain volume labels and fire at both severity levels
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
|
||||
values: '3x15 30x5'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
|
||||
values: '100x21'
|
||||
alert_rule_test:
|
||||
- alertname: WindowsDiskAlmostFull
|
||||
eval_time: 14m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsDiskAlmostFull
|
||||
eval_time: 15m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: warning}
|
||||
exp_annotations:
|
||||
summary: Windows disk space is low
|
||||
description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.'
|
||||
- exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: critical}
|
||||
exp_annotations:
|
||||
summary: Windows disk space is low
|
||||
description: 'Volume C: on host:9182 has less than 5% free space. Windows may delay space metrics by 10–15 minutes.'
|
||||
- alertname: WindowsDiskAlmostFull
|
||||
eval_time: 16m
|
||||
exp_alerts: []
|
||||
|
||||
- name: Healthy resources at the exact thresholds do not fire
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", core="0", mode="idle"}'
|
||||
values: '0+6x20'
|
||||
- series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182"}'
|
||||
values: '10x20'
|
||||
- series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182"}'
|
||||
values: '100x20'
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
|
||||
values: '10x20'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
|
||||
values: '100x20'
|
||||
alert_rule_test:
|
||||
- {alertname: WindowsCPUHighUsage, eval_time: 20m, exp_alerts: []}
|
||||
- {alertname: WindowsMemoryHighUsage, eval_time: 20m, exp_alerts: []}
|
||||
- {alertname: WindowsDiskAlmostFull, eval_time: 20m, exp_alerts: []}
|
||||
|
||||
- name: Prediction fires only for low and declining disk space
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
|
||||
values: '1000-1x420'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
|
||||
values: '20000x420'
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}'
|
||||
values: '1000x420'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}'
|
||||
values: '20000x420'
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="E:"}'
|
||||
values: '500+1x420'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="E:"}'
|
||||
values: '20000x420'
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="b", volume="C:"}'
|
||||
values: '1000x420'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="b", volume="C:"}'
|
||||
values: '20000x420'
|
||||
alert_rule_test:
|
||||
- alertname: WindowsDiskFillingUp
|
||||
eval_time: 60m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsDiskFillingUp
|
||||
eval_time: 61m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
|
||||
exp_annotations:
|
||||
summary: Windows disk is predicted to fill up
|
||||
description: 'Volume C: on host:9182 is predicted to fill within 24 hours.'
|
||||
- alertname: WindowsDiskFillingUp
|
||||
eval_time: 7h
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
|
||||
exp_annotations:
|
||||
summary: Windows disk is predicted to fill up
|
||||
description: 'Volume C: on host:9182 is predicted to fill within 24 hours.'
|
||||
|
||||
- name: Missing collector metrics do not produce resource alerts
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'up{job="windows_exporter", instance="host:9182"}'
|
||||
values: '1x70'
|
||||
alert_rule_test:
|
||||
- {alertname: WindowsCPUHighUsage, eval_time: 70m, exp_alerts: []}
|
||||
- {alertname: WindowsMemoryHighUsage, eval_time: 70m, exp_alerts: []}
|
||||
- {alertname: WindowsDiskAlmostFull, eval_time: 70m, exp_alerts: []}
|
||||
- {alertname: WindowsDiskFillingUp, eval_time: 70m, exp_alerts: []}
|
||||
@@ -0,0 +1,16 @@
|
||||
(import '../mixin.libsonnet') {
|
||||
_config+:: {
|
||||
windowsExporterSelector: 'job=~"windows.*", cluster="test"',
|
||||
volumeSelector: 'volume!~"HarddiskVolume.*"',
|
||||
nicSelector: 'nic!="Loopback"',
|
||||
rateInterval: '2m',
|
||||
cpuHighUsageThreshold: 80,
|
||||
memoryHighUsageThreshold: 85,
|
||||
diskFreeWarningThreshold: 20,
|
||||
diskFreeCriticalThreshold: 10,
|
||||
criticalSeverity: 'page',
|
||||
enableActiveDirectory: true,
|
||||
adPendingReplicationThreshold: 10,
|
||||
adReplicationFailureRateThreshold: 0.5,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
rule_files:
|
||||
- custom_rules.yaml
|
||||
- custom_alerts.yaml
|
||||
evaluation_interval: 1m
|
||||
group_eval_order:
|
||||
- windows-exporter-recording
|
||||
- windows-exporter-alerts
|
||||
- windows-active-directory
|
||||
|
||||
tests:
|
||||
- name: Custom selectors, thresholds and critical severity propagate to alerts
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'up{job="windows-custom", instance="host:9182", cluster="test"}'
|
||||
values: '0x20'
|
||||
- series: 'up{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||||
values: '0x20'
|
||||
- series: 'windows_cpu_time_total{job="windows-custom", instance="host:9182", cluster="test", core="0", mode="idle"}'
|
||||
values: '0+9x20'
|
||||
- series: 'windows_memory_available_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
|
||||
values: '12x20'
|
||||
- series: 'windows_memory_physical_total_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
|
||||
values: '100x20'
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||||
values: '8x20'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||||
values: '100x20'
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
|
||||
values: '1x20'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
|
||||
values: '100x20'
|
||||
- series: 'windows_net_bytes_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||||
values: '0+600x20'
|
||||
- series: 'windows_net_bytes_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
|
||||
values: '0+600x20'
|
||||
promql_expr_test:
|
||||
- expr: windows:net_received_bytes:rate
|
||||
eval_time: 20m
|
||||
exp_samples:
|
||||
- labels: 'windows:net_received_bytes:rate{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
|
||||
value: 10
|
||||
- expr: windows:logical_disk_free:ratio
|
||||
eval_time: 20m
|
||||
exp_samples:
|
||||
- labels: 'windows:logical_disk_free:ratio{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
|
||||
value: 0.08
|
||||
alert_rule_test:
|
||||
- alertname: WindowsExporterDown
|
||||
eval_time: 5m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: page}
|
||||
exp_annotations:
|
||||
summary: Windows exporter is unreachable
|
||||
description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows-custom).'
|
||||
- alertname: WindowsCPUHighUsage
|
||||
eval_time: 16m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||||
exp_annotations:
|
||||
summary: Windows CPU usage is high
|
||||
description: 'CPU usage on host:9182 has exceeded 80% for 15 minutes.'
|
||||
- alertname: WindowsMemoryHighUsage
|
||||
eval_time: 15m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||||
exp_annotations:
|
||||
summary: Windows memory usage is high
|
||||
description: 'Physical memory usage on host:9182 has exceeded 85% for 15 minutes.'
|
||||
- alertname: WindowsDiskAlmostFull
|
||||
eval_time: 15m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
|
||||
exp_annotations:
|
||||
summary: Windows disk space is low
|
||||
description: 'Volume C: on host:9182 has less than 20% free space. Windows may delay space metrics by 10–15 minutes.'
|
||||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: page}
|
||||
exp_annotations:
|
||||
summary: Windows disk space is low
|
||||
description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.'
|
||||
|
||||
- name: Active Directory opt-in honors thresholds, selectors and recovery
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="host:9182", cluster="test"}'
|
||||
values: '20x20 0x5'
|
||||
- series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="healthy:9182", cluster="test"}'
|
||||
values: '10x26'
|
||||
- series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||||
values: '20x26'
|
||||
- series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="host:9182", cluster="test"}'
|
||||
values: '0+120x26'
|
||||
- series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="host:9182", cluster="test"}'
|
||||
values: '0+60x20 1320+120x5'
|
||||
- series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="healthy:9182", cluster="test"}'
|
||||
values: '0+120x26'
|
||||
- series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="healthy:9182", cluster="test"}'
|
||||
values: '0+120x26'
|
||||
- series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||||
values: '0+120x26'
|
||||
- series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="host:9182", cluster="prod"}'
|
||||
values: '0+60x26'
|
||||
alert_rule_test:
|
||||
- alertname: WindowsADPendingReplication
|
||||
eval_time: 14m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsADPendingReplication
|
||||
eval_time: 15m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||||
exp_annotations:
|
||||
summary: Active Directory replication queue is high
|
||||
description: 'Pending replication operations on host:9182 have exceeded 10 for 15 minutes.'
|
||||
- alertname: WindowsADPendingReplication
|
||||
eval_time: 21m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsADReplicationFailures
|
||||
eval_time: 16m
|
||||
exp_alerts: []
|
||||
- alertname: WindowsADReplicationFailures
|
||||
eval_time: 17m
|
||||
exp_alerts:
|
||||
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
|
||||
exp_annotations:
|
||||
summary: Active Directory replication requests are failing
|
||||
description: 'Replication synchronization requests on host:9182 have been failing for 15 minutes.'
|
||||
- alertname: WindowsADReplicationFailures
|
||||
eval_time: 23m
|
||||
exp_alerts: []
|
||||
@@ -0,0 +1,89 @@
|
||||
rule_files:
|
||||
- ../windows_rules.yaml
|
||||
evaluation_interval: 1m
|
||||
fuzzy_compare: true
|
||||
|
||||
tests:
|
||||
- name: CPU is averaged across cores and preserves target labels
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="0", mode="idle"}'
|
||||
values: '0+12x10'
|
||||
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="1", mode="idle"}'
|
||||
values: '0+36x10'
|
||||
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="0", mode="user"}'
|
||||
values: '0+48x10'
|
||||
- series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="b", core="0", mode="idle"}'
|
||||
values: '0+30x10'
|
||||
- series: 'windows_cpu_time_total{job="other", instance="host:9182", core="0", mode="idle"}'
|
||||
values: '0+6x10'
|
||||
promql_expr_test:
|
||||
- expr: windows:cpu_usage:ratio
|
||||
eval_time: 10m
|
||||
exp_samples:
|
||||
- labels: 'windows:cpu_usage:ratio{job="windows_exporter", instance="host:9182", cluster="a"}'
|
||||
value: 0.6
|
||||
- labels: 'windows:cpu_usage:ratio{job="windows_exporter", instance="host:9182", cluster="b"}'
|
||||
value: 0.5
|
||||
|
||||
- name: Memory availability and disk free space are ratios, not percentages
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
|
||||
values: '25x5'
|
||||
- series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
|
||||
values: '100x5'
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
|
||||
values: '20x5'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
|
||||
values: '100x5'
|
||||
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume=""}'
|
||||
values: '1x5'
|
||||
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume=""}'
|
||||
values: '100x5'
|
||||
promql_expr_test:
|
||||
- expr: windows:memory_usage:ratio
|
||||
eval_time: 5m
|
||||
exp_samples:
|
||||
- labels: 'windows:memory_usage:ratio{job="windows_exporter", instance="host:9182", cluster="a"}'
|
||||
value: 0.75
|
||||
- expr: windows:logical_disk_free:ratio
|
||||
eval_time: 5m
|
||||
exp_samples:
|
||||
- labels: 'windows:logical_disk_free:ratio{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
|
||||
value: 0.2
|
||||
|
||||
- name: Disk and network rates retain device labels and handle counter resets
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'windows_logical_disk_read_bytes_total{job="windows_exporter", instance="host:9182", volume="C:"}'
|
||||
values: '0+600x10'
|
||||
- series: 'windows_logical_disk_write_bytes_total{job="windows_exporter", instance="host:9182", volume="C:"}'
|
||||
values: '0+1200x10'
|
||||
- series: 'windows_net_bytes_received_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
|
||||
values: '0+600x5 600+600x4'
|
||||
- series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
|
||||
values: '0+1800x10'
|
||||
- series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", nic=""}'
|
||||
values: '0+1800x10'
|
||||
promql_expr_test:
|
||||
- expr: windows:logical_disk_read_bytes:rate
|
||||
eval_time: 10m
|
||||
exp_samples:
|
||||
- labels: 'windows:logical_disk_read_bytes:rate{job="windows_exporter", instance="host:9182", volume="C:"}'
|
||||
value: 10
|
||||
- expr: windows:logical_disk_write_bytes:rate
|
||||
eval_time: 10m
|
||||
exp_samples:
|
||||
- labels: 'windows:logical_disk_write_bytes:rate{job="windows_exporter", instance="host:9182", volume="C:"}'
|
||||
value: 20
|
||||
- expr: windows:net_received_bytes:rate
|
||||
eval_time: 8m
|
||||
exp_samples:
|
||||
- labels: 'windows:net_received_bytes:rate{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
|
||||
value: 10
|
||||
- expr: windows:net_sent_bytes:rate
|
||||
eval_time: 10m
|
||||
exp_samples:
|
||||
- labels: 'windows:net_sent_bytes:rate{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
|
||||
value: 30
|
||||
Reference in New Issue
Block a user