sendnrw/windows_exporter
2
0
Fork 0
mirror of https://github.com/prometheus-community/windows_exporter.git synced 2026-10-17 01:59:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

contrib: expand Windows mixin rules and publish runbooks (#2575)

Browse Source 
Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
This commit is contained in:
Jan-Otto Kröpke
2026-10-09 09:10:59 +02:00
committed by GitHub
parent c7fa244d0e
commit d06f168926
43 changed files with 1629 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/mixin.yml
+13 -1
View File
@@ -17,6 +17,10 @@ on:
permissions: {}
concurrency:
group: mixin-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
rules:
name: Validate Prometheus rules
@@ -34,9 +38,17 @@ jobs:
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version: '1.27.1'
go-version-file: go.mod
- name: Cache mixin tools
id: tools-cache
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/go/bin
key: ${{ runner.os }}-${{ runner.arch }}-mixin-tools-${{ hashFiles('contrib/mixin/Makefile', 'go.mod') }}
- name: Install pinned tools
if: steps.tools-cache.outputs.cache-hit != 'true'
run: make tools
- name: Lint and test rules
.github/workflows/runbooks.yml
+112
View File
@@ -0,0 +1,112 @@
name: Runbooks
on:
pull_request:
paths:
- docs/runbooks/**
- docs/layouts/**
- docs/static/**
- docs/hugo.toml
- docs/check_links.py
- contrib/mixin/**
- .github/workflows/runbooks.yml
push:
branches:
- master
- main
- next
paths:
- docs/runbooks/**
- docs/layouts/**
- docs/static/**
- docs/hugo.toml
- docs/check_links.py
- contrib/mixin/**
- .github/workflows/runbooks.yml
workflow_dispatch:
permissions: {}
concurrency:
group: runbooks-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
# renovate: github=gohugoio/hugo
HUGO_VERSION: '0.167.0'
jobs:
build:
name: Build runbooks
runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
contents: read
pages: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: 'false'
- name: Install Hugo
run: |
curl --fail --location --silent --show-error \
"https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_linux-amd64.tar.gz" \
-o "$RUNNER_TEMP/hugo.tar.gz"
tar -xzf "$RUNNER_TEMP/hugo.tar.gz" -C "$RUNNER_TEMP" hugo
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
- name: Cache Jsonnet tools
id: jsonnet-cache
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/go/bin/jsonnet
~/go/bin/jsonnetfmt
key: ${{ runner.os }}-${{ runner.arch }}-runbooks-jsonnet-${{ hashFiles('contrib/mixin/Makefile', 'go.mod') }}
- name: Install Jsonnet tools
if: steps.jsonnet-cache.outputs.cache-hit != 'true'
run: make -C contrib/mixin jsonnet-tools
- name: Check alert runbook coverage
run: make -C contrib/mixin check-runbooks
- name: Set up Pages
id: pages
if: github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
- name: Build site
env:
PAGES_BASE_URL: ${{ steps.pages.outputs.base_url || 'https://prometheus-community.github.io/windows_exporter' }}
run: |
hugo --source docs --minify --baseURL "${PAGES_BASE_URL%/}/" --panicOnWarning
python3 docs/check_links.py docs/public "${PAGES_BASE_URL%/}/"
- name: Upload Pages artifact
if: github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
with:
path: docs/public
deploy:
name: Publish runbooks
if: github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
needs: build
runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
pages: write
id-token: write
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1
README.md
+2 -1
View File
@@ -194,7 +194,8 @@ See detailed steps to install on Windows Kubernetes [here](./kubernetes/kubernet
The [Windows exporter mixin](contrib/mixin/README.md) provides configurable
recording rules and alerts for Prometheus, with Jsonnet generation, pint linting,
and promtool tests.
and promtool tests. Dedicated [alert runbooks](docs/runbooks) provide Windows
diagnosis and mitigation steps and are published through GitHub Pages.
## Supported versions
contrib/mixin/.pint.hcl
+6
View File
@@ -21,4 +21,10 @@ rule {
value = ".+"
severity = "bug"
}
annotation "runbook_url" {
required = true
value = "https?://.+"
severity = "bug"
}
}
contrib/mixin/Makefile
+13 -6
View File
@@ -2,24 +2,27 @@ JSONNET ?= jsonnet
JSONNETFMT ?= jsonnetfmt
PROMTOOL ?= promtool
PINT ?= pint
PYTHON ?= python3
# renovate: go=github.com/google/go-jsonnet
JSONNET_VERSION := v0.22.0
# renovate: go=github.com/cloudflare/pint
PINT_VERSION := v0.89.2
SOURCES := $(wildcard *.libsonnet *.jsonnet lib/*.libsonnet rules/*.libsonnet alerts/*.libsonnet tests/*.libsonnet)
SOURCES := $(wildcard *.libsonnet *.jsonnet lib/*.libsonnet rules/*.libsonnet alerts/*.libsonnet tests/*.libsonnet tests/*.jsonnet)
FMT_FLAGS := -n 2 --max-blank-lines 2 --string-style s --comment-style s
.DEFAULT_GOAL := generate
.DELETE_ON_ERROR:
.PHONY: generate tools fmt lint test clean
.PHONY: generate tools jsonnet-tools fmt lint test check-runbooks clean
generate: windows_rules.yaml windows_alerts.yaml
tools:
tools: jsonnet-tools
go install github.com/cloudflare/pint/cmd/pint@$(PINT_VERSION)
jsonnet-tools:
go install github.com/google/go-jsonnet/cmd/jsonnet@$(JSONNET_VERSION)
go install github.com/google/go-jsonnet/cmd/jsonnetfmt@$(JSONNET_VERSION)
go install github.com/cloudflare/pint/cmd/pint@$(PINT_VERSION)
windows_rules.yaml: $(SOURCES)
$(JSONNET) -S rules.jsonnet > $@
@@ -44,8 +47,12 @@ lint: generate tests/custom_rules.yaml tests/custom_alerts.yaml
$(PINT) --offline lint windows_rules.yaml windows_alerts.yaml
$(PINT) --offline lint tests/custom_rules.yaml tests/custom_alerts.yaml
test: lint
$(PROMTOOL) test rules tests/rules.test.yaml tests/alerts.test.yaml tests/custom.test.yaml
check-runbooks:
$(JSONNET) -e '(import "mixin.libsonnet") { _config+:: { enableActiveDirectory: true, enableTime: true } }.prometheusAlerts' | $(PYTHON) tests/check_runbooks.py ../../docs/runbooks
test: lint check-runbooks
$(JSONNET) tests/runbooks.test.jsonnet
$(PROMTOOL) test rules tests/*.test.yaml
clean:
rm -f windows_rules.yaml windows_alerts.yaml tests/custom_rules.yaml tests/custom_alerts.yaml
contrib/mixin/README.md
+90 -11
View File
@@ -24,7 +24,8 @@ to generate rules. Generated files are ignored by Git:
The default selector is `job="windows_exporter"`. Change it to match your scrape
job before generating rules. Resource rules use the default `cpu`, `memory`,
`logical_disk`, and `net` collectors. Enable `ad` on domain controllers when
opting into Active Directory alerts.
opting into Active Directory alerts. Clock alerts require the `time` collector
with its `ntp` subcollector and are intended for hosts using NTP synchronization.
Copy both generated files into your Prometheus rule directory and reference
them in `prometheus.yml`:
@@ -53,6 +54,8 @@ modifying its source. For example, create `custom.libsonnet` in this directory:
rateInterval: '5m',
cpuHighUsageThreshold: 95,
enableActiveDirectory: true,
enableTime: true,
runbookURLPattern: 'https://runbooks.example.com/windows/%s/',
},
}
```
@@ -68,7 +71,8 @@ Selectors are comma-separated Prometheus label matchers without braces. They
apply to both recording rules and alerts. All target labels, including `job`,
`instance`, `cluster`, and custom selector labels, are preserved. CPU usage
averages idle rates across cores and removes only `core` and `mode`. Disk and
network recordings also retain `volume` and `nic` respectively.
network recordings also retain `volume` and `nic` respectively. The CPU mode
recording retains `mode` and removes only `core`.
`rateInterval` is the fixed window for recording rules and Active Directory
failure rates; keep it at least four times your scrape interval. Grafana's
@@ -80,18 +84,47 @@ rules.
| Metric | Meaning | Unit |
| --- | --- | --- |
| `windows:cpu_usage:ratio` | Non-idle CPU time averaged across cores | Ratio, 0–1 |
| `windows:cpu_cores:count` | Logical processors observed in idle CPU metrics | Count |
| `windows:cpu_time:ratio_rate` | CPU time averaged across cores, retaining `mode` | Seconds/second |
| `windows:memory_usage:ratio` | Physical memory in use, based on available bytes | Ratio, 0–1 |
| `windows:memory_used:bytes` | Total physical memory minus available memory | Bytes |
| `windows:memory_cached:bytes` | Cache, modified pages, and three standby lists | Bytes |
| `windows:memory_committed:ratio` | Committed bytes divided by the system commit limit | Ratio |
| `windows:memory_swap_page_operations:rate` | Paging operations | Operations/second |
| `windows:logical_disk_free:ratio` | Free disk space per volume | Ratio, 0–1 |
| `windows:logical_disk_used:bytes` | Used disk space per volume | Bytes |
| `windows:logical_disk_busy:ratio` | Non-idle time per volume, clamped to 0–1 | Ratio, 0–1 |
| `windows:logical_disk_read_bytes:rate` | Disk read throughput per volume | Bytes/second |
| `windows:logical_disk_write_bytes:rate` | Disk write throughput per volume | Bytes/second |
| `windows:logical_disk_read_operations:rate` | Disk reads per volume | Operations/second |
| `windows:logical_disk_write_operations:rate` | Disk writes per volume | Operations/second |
| `windows:logical_disk_read_latency:seconds` | Read-time rate divided by read-operation rate | Seconds/operation |
| `windows:logical_disk_write_latency:seconds` | Write-time rate divided by write-operation rate | Seconds/operation |
| `windows:net_received_bytes:rate` | Received network traffic per interface | Bytes/second |
| `windows:net_sent_bytes:rate` | Sent network traffic per interface | Bytes/second |
| `windows:net_received_utilization:ratio` | Received throughput divided by interface bandwidth | Ratio |
| `windows:net_sent_utilization:ratio` | Sent throughput divided by interface bandwidth | Ratio |
| `windows:net_received_errors:rate` | Inbound packet errors | Packets/second |
| `windows:net_outbound_errors:rate` | Outbound packet errors | Packets/second |
| `windows:net_received_discarded:rate` | Inbound discarded packets | Packets/second |
| `windows:net_outbound_discarded:rate` | Outbound discarded packets | Packets/second |
The names remain stable when `rateInterval` changes. Dashboard authors can use
these recordings or query raw exporter metrics directly. CPU usage measures
non-idle time; it is not the frequency-adjusted processor utility shown in
Windows Task Manager.
CPU DPC and interrupt modes overlap privileged time; do not sum all modes.
Disk busy time uses the idle counter rather than summing read and write time,
which can exceed wall time for concurrent I/O. Latency is an average and is
omitted when there are no operations. Memory, disk space, commit, and network
utilization ratios are omitted when their capacity is zero or missing. Network
bandwidth is already in bytes/second; receive and send utilization are separate
for full-duplex links.
The cache recording includes modified pages and is not a measure of immediately
reclaimable memory. Paging operations can include file-backed pages as well as
pagefile traffic.
## Alerts
Threshold configuration values below are percentages except where noted.
@@ -102,11 +135,18 @@ Threshold configuration values below are percentages except where noted.
| `WindowsCollectorFailed` | `windows_exporter_collector_success == 0` | 5m | Warning |
| `WindowsCPUHighUsage` | CPU usage above `cpuHighUsageThreshold` (90%) | 15m | Warning |
| `WindowsMemoryHighUsage` | Memory usage above `memoryHighUsageThreshold` (90%) | 15m | Warning |
| `WindowsMemoryCommitHighUsage` | Commit usage above `memoryCommitHighUsageThreshold` (90%) | 15m | Warning |
| `WindowsDiskReadLatencyHigh` | Average read latency above `diskLatencyThresholdSeconds` (0.05 seconds) | 15m | Warning |
| `WindowsDiskWriteLatencyHigh` | Average write latency above `diskLatencyThresholdSeconds` (0.05 seconds) | 15m | Warning |
| `WindowsNetworkErrors` | Combined receive/outbound errors above `networkErrorRateThreshold` (1 packet/second) | 15m | Warning |
| `WindowsNetworkDiscards` | Combined receive/outbound discards above `networkDiscardRateThreshold` (1 packet/second) | 15m | Warning |
| `WindowsDiskAlmostFull` | Free space below `diskFreeWarningThreshold` (10%) | 15m | Warning |
| `WindowsDiskAlmostFull` | Free space below `diskFreeCriticalThreshold` (5%) | 15m | Critical |
| `WindowsDiskFillingUp` | Below the warning threshold and predicted to fill within `diskPredictionHours` (24h), using `diskPredictionWindow` (6h) | 1h | Warning |
| `WindowsADPendingReplication` | Pending operations above `adPendingReplicationThreshold` (50) | 15m | Warning |
| `WindowsADReplicationFailures` | Failed synchronization requests above `adReplicationFailureRateThreshold` (0 requests/second) | 15m | Warning |
| `WindowsClockNotSynchronising` | No active NTP time sources | 10m | Warning |
| `WindowsClockSkewDetected` | Absolute reported time offset above `clockOffsetThresholdSeconds` (0.05 seconds) | 10m | Warning |
Active Directory alerts are generated only when `enableActiveDirectory` is
`true`. `criticalSeverity` defaults to `critical` and can be overridden to match
@@ -114,6 +154,12 @@ your routing labels. Below the critical disk threshold, both disk severity
levels fire; configure Alertmanager inhibition if only the critical
notification should be sent.
Clock alerts are generated only when `enableTime` is `true`. A zero NTP source
count may be expected on hosts using another time provider. The offset compares
the host to its chosen source and does not independently verify that source.
Disk latency and network error/discard thresholds should be tuned for the
storage tier, traffic volume, and application requirements.
Windows disk size/free-space counters may lag by 10–15 minutes. Predictions
depend on a sustained trend and are intentionally delayed for an hour. A target
removed from service discovery has no `up` series and will not trigger
@@ -121,9 +167,35 @@ removed from service discovery has no `up` series and will not trigger
scrape and collector-failure alerts cover explicit failures, not disabled
collectors.
## Runbooks and GitHub Pages
Every alert receives a `runbook_url` annotation pointing to a dedicated page in
[docs/runbooks](../../docs/runbooks). Each page describes the meaning, impact,
diagnosis, and mitigation, including Windows commands and related metrics.
The default URL is
`https://prometheus-community.github.io/windows_exporter/runbooks/<lowercase-alert-name>/`.
Override `runbookURLPattern` for a fork, custom domain, or internal runbook site;
it takes one `%s` placeholder for the lowercase alert name. Explicit
`runbook_url` annotations supplied by downstream alerts are preserved.
The theme-free Hugo site uses `docs/hugo.toml`. With Hugo 0.167.0 installed,
preview it from the repository root:
```sh
hugo server --source docs
```
Build it with `hugo --source docs --minify`. The `Runbooks` workflow builds site
changes on pull requests, checks alert coverage and internal links, and publishes
default-branch changes with `configure-pages`, `upload-pages-artifact`, and
`deploy-pages`. This uses the
repository's GitHub Pages **Source: GitHub Actions** setting. The workflow reads
the configured Pages URL, including custom domains and repository subpaths.
Fork users should override `runbookURLPattern` to match their published URL.
## Validate
Install `promtool` from a [Prometheus release](https://github.com/prometheus/prometheus/releases),
Install Python 3 and `promtool` from a [Prometheus release](https://github.com/prometheus/prometheus/releases),
then run:
```sh
@@ -132,10 +204,12 @@ make test
This checks Jsonnet formatting, validates generated rules with `promtool`, runs
`pint --offline lint`, and evaluates the rules against synthetic time series.
Tests cover all seven recordings, alert timing and recovery, threshold
boundaries, counter resets, disk forecasts, target-label preservation, and
custom selectors/thresholds with Active Directory enabled. CI uses Prometheus
3.15.0 and the same pinned tool versions as `make tools`.
Tests cover all 25 recordings, alert timing and recovery, threshold
boundaries, counter resets, zero denominators, disk forecasts, target-label
preservation, and custom selectors/thresholds with Active Directory and clock
alerts enabled. Runbook checks include opt-in alerts and verify that their pages
and URLs exist. CI uses Prometheus 3.15.0 and the same pinned tool versions as
`make tools`.
If you prefer to run `promtool` through Docker:
@@ -143,10 +217,15 @@ If you prefer to run `promtool` through Docker:
make test PROMTOOL="docker run --rm --entrypoint promtool -v \"$PWD:/mixin:ro\" -w /mixin prom/prometheus:v3.15.0"
```
`.pint.hcl` requires a non-empty severity label and summary/description
annotations on alerts, in addition to pint's built-in checks. CI uses offline
mode without a Prometheus endpoint. To validate metric availability and labels
against your own server, copy `.pint.hcl` to a local configuration, add:
The dedicated `Mixin` workflow runs on changes to `contrib/mixin/**` or its own
workflow file, and can also be started manually. It caches pinned tool binaries
and runs the same `make test` checks used locally.
`.pint.hcl` requires a non-empty severity label, summary/description annotations,
and an HTTP(S) runbook URL on alerts, in addition to pint's built-in checks.
CI uses offline mode without a Prometheus endpoint. To validate metric
availability and labels against your own server, copy `.pint.hcl` to a local
configuration, add:
```hcl
prometheus "local" {
contrib/mixin/alerts/alerts.libsonnet
+69 -12
View File
@@ -45,6 +45,41 @@
'Windows memory usage is high',
'Physical memory usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.memoryHighUsageThreshold
),
alert(
'WindowsMemoryCommitHighUsage',
q.metric('windows:memory_committed:ratio') + ' * 100 > %s' % c.memoryCommitHighUsageThreshold,
'15m',
'warning',
'Windows memory commit usage is high',
'Committed memory on {{ $labels.instance }} has exceeded %s%% of the commit limit for 15 minutes.' % c.memoryCommitHighUsageThreshold
),
] + [
alert(
'WindowsDisk%sLatencyHigh' % direction[0],
q.metric('windows:logical_disk_%s_latency:seconds' % direction[1], [c.volumeSelector]) + ' > %g' % c.diskLatencyThresholdSeconds,
'15m',
'warning',
'Windows disk %s latency is high' % direction[1],
'Volume {{ $labels.volume }} on {{ $labels.instance }} has averaged more than %g seconds per %s operation for 15 minutes.' % [c.diskLatencyThresholdSeconds, direction[1]]
)
for direction in [['Read', 'read'], ['Write', 'write']]
] + [
alert(
'WindowsNetwork%s' % outcome[0],
'(%s + %s) > %g' % [
q.metric('windows:net_received_%s:rate' % outcome[1], [c.nicSelector]),
q.metric('windows:net_outbound_%s:rate' % outcome[1], [c.nicSelector]),
outcome[2],
],
'15m',
'warning',
'Windows network packet %s are high' % outcome[3],
'Packet %s on interface {{ $labels.nic }} on {{ $labels.instance }} have exceeded %g per second for 15 minutes.' % [outcome[3], outcome[2]]
)
for outcome in [
['Errors', 'errors', c.networkErrorRateThreshold, 'errors'],
['Discards', 'discarded', c.networkDiscardRateThreshold, 'discards'],
]
] + [
alert(
'WindowsDiskAlmostFull',
@@ -74,25 +109,47 @@
),
],
},
] + if c.enableActiveDirectory then [
] + (if c.enableActiveDirectory then [
{
name: 'windows-active-directory',
rules: [
alert(
'WindowsADPendingReplication',
q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold,
'15m',
'warning',
'Active Directory replication queue is high',
'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold
),
alert(
'WindowsADReplicationFailures',
'(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold,
'15m',
'warning',
'Active Directory replication requests are failing',
'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.'
),
],
},
] else []) + if c.enableTime then [
{
name: 'windows-active-directory',
name: 'windows-time',
rules: [
alert(
'WindowsADPendingReplication',
q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold,
'15m',
'WindowsClockNotSynchronising',
q.metric('windows_time_ntp_client_time_sources') + ' == 0',
'10m',
'warning',
'Active Directory replication queue is high',
'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold
'Windows clock has no NTP time source',
'Windows Time on {{ $labels.instance }} has had no active NTP time source for 10 minutes.'
),
alert(
'WindowsADReplicationFailures',
'(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold,
'15m',
'WindowsClockSkewDetected',
'abs(%s) > %g' % [q.metric('windows_time_computed_time_offset_seconds'), c.clockOffsetThresholdSeconds],
'10m',
'warning',
'Active Directory replication requests are failing',
'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.'
'Windows clock offset is high',
'Windows Time on {{ $labels.instance }} reports a clock offset above %g seconds for 10 minutes.' % c.clockOffsetThresholdSeconds
),
],
},
contrib/mixin/config.libsonnet
+9
View File
@@ -8,9 +8,18 @@
// Opt in only for targets with the ad collector enabled.
enableActiveDirectory: false,
// Requires the time collector with its ntp subcollector enabled.
enableTime: false,
runbookURLPattern: 'https://prometheus-community.github.io/windows_exporter/runbooks/%s/',
cpuHighUsageThreshold: 90,
memoryHighUsageThreshold: 90,
memoryCommitHighUsageThreshold: 90,
diskLatencyThresholdSeconds: 0.05,
networkErrorRateThreshold: 1,
networkDiscardRateThreshold: 1,
clockOffsetThresholdSeconds: 0.05,
diskFreeWarningThreshold: 10,
diskFreeCriticalThreshold: 5,
diskPredictionWindow: '6h',
contrib/mixin/lib/add-runbook-links.libsonnet
+19
View File
@@ -0,0 +1,19 @@
{
prometheusAlerts+:: {
groups: [
group {
rules: [
if std.objectHas(rule, 'alert') && !(std.objectHas(rule, 'annotations') && std.objectHas(rule.annotations, 'runbook_url')) then
rule {
annotations+: {
runbook_url: $._config.runbookURLPattern % std.asciiLower(rule.alert),
},
}
else rule
for rule in group.rules
],
}
for group in super.groups
],
},
}
contrib/mixin/lib/queries.libsonnet
+20 -6
View File
@@ -3,16 +3,30 @@ function(c)
selector(extra=[]): std.join(', ', [s for s in [c.windowsExporterSelector] + extra if s != '']),
metric(name, extra=[]): '%s{%s}' % [name, self.selector(extra)],
rate(name, extra=[]): 'rate(%s[%s])' % [self.metric(name, extra), c.rateInterval],
// Drop undefined ratios instead of recording NaN or +Inf for idle devices.
ratio(numerator, denominator): '(%s / (%s > 0))' % [numerator, denominator],
cpuUsage:
'1 - avg without (core, mode) (%s)' % self.rate('windows_cpu_time_total', ['mode="idle"']),
memoryUsage:
'1 - %s / %s' % [
'1 - %s' % self.ratio(
self.metric('windows_memory_available_bytes'),
self.metric('windows_memory_physical_total_bytes'),
],
self.metric('windows_memory_physical_total_bytes')
),
diskFree:
'%s / %s' % [
self.ratio(
self.metric('windows_logical_disk_free_bytes', [c.volumeSelector]),
self.metric('windows_logical_disk_size_bytes', [c.volumeSelector]),
],
self.metric('windows_logical_disk_size_bytes', [c.volumeSelector])
),
memoryCommitted:
self.ratio(self.metric('windows_memory_committed_bytes'), self.metric('windows_memory_commit_limit')),
diskLatency(direction):
self.ratio(
self.rate('windows_logical_disk_%s_seconds_total' % direction, [c.volumeSelector]),
self.rate('windows_logical_disk_%ss_total' % direction, [c.volumeSelector])
),
networkUtilization(direction):
self.ratio(
self.rate('windows_net_bytes_%s_total' % direction, [c.nicSelector]),
self.metric('windows_net_current_bandwidth_bytes', [c.nicSelector])
),
}
contrib/mixin/mixin.libsonnet
+2 -1
View File
@@ -1,3 +1,4 @@
(import 'config.libsonnet') +
(import 'rules/rules.libsonnet') +
(import 'alerts/alerts.libsonnet')
(import 'alerts/alerts.libsonnet') +
(import 'lib/add-runbook-links.libsonnet')
contrib/mixin/rules/rules.libsonnet
+45
View File
@@ -7,20 +7,65 @@
name: 'windows-exporter-recording',
rules: [
{ record: 'windows:cpu_usage:ratio', expr: q.cpuUsage },
{ record: 'windows:cpu_cores:count', expr: 'count without (core, mode) (%s)' % q.metric('windows_cpu_time_total', ['mode="idle"']) },
{ record: 'windows:cpu_time:ratio_rate', expr: 'avg without (core) (%s)' % q.rate('windows_cpu_time_total') },
{ record: 'windows:memory_usage:ratio', expr: q.memoryUsage },
{
record: 'windows:memory_used:bytes',
expr: '%s - %s' % [q.metric('windows_memory_physical_total_bytes'), q.metric('windows_memory_available_bytes')],
},
{
record: 'windows:memory_cached:bytes',
expr: std.join(' + ', [
q.metric('windows_memory_%s_bytes' % component)
for component in ['cache', 'modified_page_list', 'standby_cache_core', 'standby_cache_normal_priority', 'standby_cache_reserve']
]),
},
{ record: 'windows:memory_committed:ratio', expr: q.memoryCommitted },
{ record: 'windows:memory_swap_page_operations:rate', expr: q.rate('windows_memory_swap_page_operations_total') },
{ record: 'windows:logical_disk_free:ratio', expr: q.diskFree },
{
record: 'windows:logical_disk_used:bytes',
expr: '%s - %s' % [
q.metric('windows_logical_disk_size_bytes', [c.volumeSelector]),
q.metric('windows_logical_disk_free_bytes', [c.volumeSelector]),
],
},
{
record: 'windows:logical_disk_busy:ratio',
expr: 'clamp(1 - %s, 0, 1)' % q.rate('windows_logical_disk_idle_seconds_total', [c.volumeSelector]),
},
] + [
{
record: 'windows:logical_disk_%s_bytes:rate' % direction,
expr: q.rate('windows_logical_disk_%s_bytes_total' % direction, [c.volumeSelector]),
}
for direction in ['read', 'write']
] + [
{
record: 'windows:logical_disk_%s_operations:rate' % direction,
expr: q.rate('windows_logical_disk_%ss_total' % direction, [c.volumeSelector]),
}
for direction in ['read', 'write']
] + [
{ record: 'windows:logical_disk_%s_latency:seconds' % direction, expr: q.diskLatency(direction) }
for direction in ['read', 'write']
] + [
{
record: 'windows:net_%s_bytes:rate' % direction,
expr: q.rate('windows_net_bytes_%s_total' % direction, [c.nicSelector]),
}
for direction in ['received', 'sent']
] + [
{ record: 'windows:net_%s_utilization:ratio' % direction, expr: q.networkUtilization(direction) }
for direction in ['received', 'sent']
] + [
{
record: 'windows:net_%s_%s:rate' % [direction, outcome],
expr: q.rate('windows_net_packets_%s_%s_total' % [direction, outcome], [c.nicSelector]),
}
for direction in ['received', 'outbound']
for outcome in ['errors', 'discarded']
],
},
],
contrib/mixin/tests/alerts.test.yaml
+118
View File
@@ -7,6 +7,116 @@ group_eval_order:
- windows-exporter-alerts
tests:
- name: Commit pressure, disk latency and network failures wait and recover
interval: 1m
input_series:
- series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '95x20 50x10'
- series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '100x31'
- series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+12x20 252x10'
- series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+120x31'
- series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+24x20 504x10'
- series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+240x31'
- series: 'windows_net_packets_received_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+120x20 2520x10'
- series: 'windows_net_packets_outbound_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0x31'
- series: 'windows_net_packets_received_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0x31'
- series: 'windows_net_packets_outbound_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+120x20 2520x10'
alert_rule_test:
- {alertname: WindowsMemoryCommitHighUsage, eval_time: 14m, exp_alerts: []}
- alertname: WindowsMemoryCommitHighUsage
eval_time: 15m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsmemorycommithighusage/
summary: Windows memory commit usage is high
description: 'Committed memory on host:9182 has exceeded 90% of the commit limit for 15 minutes.'
- {alertname: WindowsMemoryCommitHighUsage, eval_time: 21m, exp_alerts: []}
- {alertname: WindowsDiskReadLatencyHigh, eval_time: 15m, exp_alerts: []}
- alertname: WindowsDiskReadLatencyHigh
eval_time: 16m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskreadlatencyhigh/
summary: Windows disk read latency is high
description: 'Volume C: on host:9182 has averaged more than 0.05 seconds per read operation for 15 minutes.'
- {alertname: WindowsDiskReadLatencyHigh, eval_time: 25m, exp_alerts: []}
- {alertname: WindowsDiskWriteLatencyHigh, eval_time: 15m, exp_alerts: []}
- alertname: WindowsDiskWriteLatencyHigh
eval_time: 16m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskwritelatencyhigh/
summary: Windows disk write latency is high
description: 'Volume C: on host:9182 has averaged more than 0.05 seconds per write operation for 15 minutes.'
- {alertname: WindowsDiskWriteLatencyHigh, eval_time: 25m, exp_alerts: []}
- {alertname: WindowsNetworkErrors, eval_time: 17m, exp_alerts: []}
- alertname: WindowsNetworkErrors
eval_time: 18m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, nic: Ethernet, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsnetworkerrors/
summary: Windows network packet errors are high
description: 'Packet errors on interface Ethernet on host:9182 have exceeded 1 per second for 15 minutes.'
- {alertname: WindowsNetworkErrors, eval_time: 25m, exp_alerts: []}
- {alertname: WindowsNetworkDiscards, eval_time: 17m, exp_alerts: []}
- alertname: WindowsNetworkDiscards
eval_time: 18m
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, nic: Ethernet, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsnetworkdiscards/
summary: Windows network packet discards are high
description: 'Packet discards on interface Ethernet on host:9182 have exceeded 1 per second for 15 minutes.'
- {alertname: WindowsNetworkDiscards, eval_time: 25m, exp_alerts: []}
- name: New resource alerts stay quiet at exact thresholds and clock alerts are disabled
interval: 1m
input_series:
- series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182"}'
values: '90x20'
- series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182"}'
values: '100x20'
- series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+6x20'
- series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+120x20'
- series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+6x20'
- series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0+120x20'
- series: 'windows_net_packets_received_errors_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+30x20'
- series: 'windows_net_packets_outbound_errors_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+30x20'
- series: 'windows_net_packets_received_discarded_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+30x20'
- series: 'windows_net_packets_outbound_discarded_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+30x20'
- series: 'windows_time_ntp_client_time_sources{job="windows_exporter", instance="host:9182"}'
values: '0x20'
- series: 'windows_time_computed_time_offset_seconds{job="windows_exporter", instance="host:9182"}'
values: '1x20'
alert_rule_test:
- {alertname: WindowsMemoryCommitHighUsage, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsDiskReadLatencyHigh, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsDiskWriteLatencyHigh, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsNetworkErrors, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsNetworkDiscards, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsClockNotSynchronising, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsClockSkewDetected, eval_time: 20m, exp_alerts: []}
- name: Exporter and collector failures wait five minutes and recover
interval: 1m
input_series:
@@ -25,6 +135,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', severity: critical}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsexporterdown/
summary: Windows exporter is unreachable
description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows_exporter).'
- alertname: WindowsExporterDown
@@ -38,6 +149,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', collector: cpu, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowscollectorfailed/
summary: Windows collector failed
description: 'Collector cpu on host:9182 has failed for at least 5 minutes.'
- alertname: WindowsCollectorFailed
@@ -62,6 +174,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowscpuhighusage/
summary: Windows CPU usage is high
description: 'CPU usage on host:9182 has exceeded 90% for 15 minutes.'
- alertname: WindowsCPUHighUsage
@@ -75,6 +188,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsmemoryhighusage/
summary: Windows memory usage is high
description: 'Physical memory usage on host:9182 has exceeded 90% for 15 minutes.'
- alertname: WindowsMemoryHighUsage
@@ -97,10 +211,12 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskalmostfull/
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.'
- exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: critical}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskalmostfull/
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 5% free space. Windows may delay space metrics by 10–15 minutes.'
- alertname: WindowsDiskAlmostFull
@@ -153,6 +269,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskfillingup/
summary: Windows disk is predicted to fill up
description: 'Volume C: on host:9182 is predicted to fill within 24 hours.'
- alertname: WindowsDiskFillingUp
@@ -160,6 +277,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskfillingup/
summary: Windows disk is predicted to fill up
description: 'Volume C: on host:9182 is predicted to fill within 24 hours.'
contrib/mixin/tests/check_runbooks.py
+36
View File
@@ -0,0 +1,36 @@
"""Check every mixin alert has a complete runbook at its default URL."""
import json
from pathlib import Path
import sys
from urllib.parse import urlparse
runbooks = Path(sys.argv[1])
alerts = json.load(sys.stdin)
errors = []
names = set()
for group in alerts["groups"]:
for rule in group["rules"]:
if "alert" not in rule:
continue
name = rule["alert"]
names.add(name)
slug = name.lower()
expected_path = f"/windows_exporter/runbooks/{slug}/"
if urlparse(rule["annotations"].get("runbook_url", "")).path != expected_path:
errors.append(f"{name}: runbook URL must end in {expected_path}")
path = runbooks / f"{slug}.md"
if not path.is_file():
errors.append(f"{name}: missing {path}")
continue
content = path.read_text(encoding="utf-8")
for section in ("Meaning", "Impact", "Diagnosis", "Mitigation"):
if f"## {section}\n" not in content:
errors.append(f"{name}: missing {section} section in {path}")
if "TODO" in content:
errors.append(f"{name}: unfinished runbook {path}")
if errors:
sys.exit("\n".join(errors))
print(f"Checked runbooks and URLs for {len(names)} alerts")
contrib/mixin/tests/config.libsonnet
+7
View File
@@ -10,6 +10,13 @@
diskFreeCriticalThreshold: 10,
criticalSeverity: 'page',
enableActiveDirectory: true,
enableTime: true,
runbookURLPattern: 'https://runbooks.example.com/windows/%s/',
memoryCommitHighUsageThreshold: 80,
diskLatencyThresholdSeconds: 0.02,
networkErrorRateThreshold: 0.5,
networkDiscardRateThreshold: 0.5,
clockOffsetThresholdSeconds: 0.1,
adPendingReplicationThreshold: 10,
adReplicationFailureRateThreshold: 0.5,
},
contrib/mixin/tests/custom.test.yaml
+129
View File
@@ -6,8 +6,130 @@ group_eval_order:
- windows-exporter-recording
- windows-exporter-alerts
- windows-active-directory
- windows-time
tests:
- name: Custom commit, latency and network thresholds preserve selectors and device filters
interval: 1m
input_series:
- series: 'windows_memory_committed_bytes{job="windows-custom", instance="host:9182", cluster="test"}'
values: '85x20'
- series: 'windows_memory_commit_limit{job="windows-custom", instance="host:9182", cluster="test"}'
values: '100x20'
- series: 'windows_memory_committed_bytes{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '99x20'
- series: 'windows_memory_commit_limit{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '100x20'
- series: 'windows_logical_disk_read_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '0+3.6x20'
- series: 'windows_logical_disk_reads_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '0+120x20'
- series: 'windows_logical_disk_write_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '0+3.6x20'
- series: 'windows_logical_disk_writes_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}'
values: '0+120x20'
- series: 'windows_logical_disk_read_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '0+12x20'
- series: 'windows_logical_disk_reads_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '0+120x20'
- series: 'windows_logical_disk_write_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '0+12x20'
- series: 'windows_logical_disk_writes_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}'
values: '0+120x20'
- series: 'windows_net_packets_received_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
values: '0+36x20'
- series: 'windows_net_packets_outbound_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
values: '0x20'
- series: 'windows_net_packets_received_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
values: '0x20'
- series: 'windows_net_packets_outbound_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}'
values: '0+36x20'
- series: 'windows_net_packets_received_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
values: '0+120x20'
- series: 'windows_net_packets_outbound_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
values: '0+120x20'
- series: 'windows_net_packets_received_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
values: '0+120x20'
- series: 'windows_net_packets_outbound_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}'
values: '0+120x20'
alert_rule_test:
- alertname: WindowsMemoryCommitHighUsage
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsmemorycommithighusage/
summary: Windows memory commit usage is high
description: 'Committed memory on host:9182 has exceeded 80% of the commit limit for 15 minutes.'
- alertname: WindowsDiskReadLatencyHigh
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsdiskreadlatencyhigh/
summary: Windows disk read latency is high
description: 'Volume C: on host:9182 has averaged more than 0.02 seconds per read operation for 15 minutes.'
- alertname: WindowsDiskWriteLatencyHigh
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsdiskwritelatencyhigh/
summary: Windows disk write latency is high
description: 'Volume C: on host:9182 has averaged more than 0.02 seconds per write operation for 15 minutes.'
- alertname: WindowsNetworkErrors
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, nic: Ethernet, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsnetworkerrors/
summary: Windows network packet errors are high
description: 'Packet errors on interface Ethernet on host:9182 have exceeded 0.5 per second for 15 minutes.'
- alertname: WindowsNetworkDiscards
eval_time: 20m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, nic: Ethernet, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsnetworkdiscards/
summary: Windows network packet discards are high
description: 'Packet discards on interface Ethernet on host:9182 have exceeded 0.5 per second for 15 minutes.'
- name: Clock alerts opt in, use absolute offset, custom URLs and recover
interval: 1m
input_series:
- series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="host:9182", cluster="test"}'
values: '0x10 1x10'
- series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="host:9182", cluster="test"}'
values: '-0.2x10 0x10'
- series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="healthy:9182", cluster="test"}'
values: '1x21'
- series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="healthy:9182", cluster="test"}'
values: '0.1x21'
- series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '0x21'
- series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="host:9182", cluster="prod"}'
values: '-1x21'
alert_rule_test:
- {alertname: WindowsClockNotSynchronising, eval_time: 9m, exp_alerts: []}
- alertname: WindowsClockNotSynchronising
eval_time: 10m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsclocknotsynchronising/
summary: Windows clock has no NTP time source
description: 'Windows Time on host:9182 has had no active NTP time source for 10 minutes.'
- {alertname: WindowsClockNotSynchronising, eval_time: 11m, exp_alerts: []}
- {alertname: WindowsClockSkewDetected, eval_time: 9m, exp_alerts: []}
- alertname: WindowsClockSkewDetected
eval_time: 10m
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsclockskewdetected/
summary: Windows clock offset is high
description: 'Windows Time on host:9182 reports a clock offset above 0.1 seconds for 10 minutes.'
- {alertname: WindowsClockSkewDetected, eval_time: 11m, exp_alerts: []}
- name: Custom selectors, thresholds and critical severity propagate to alerts
interval: 1m
input_series:
@@ -50,6 +172,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: page}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsexporterdown/
summary: Windows exporter is unreachable
description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows-custom).'
- alertname: WindowsCPUHighUsage
@@ -57,6 +180,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowscpuhighusage/
summary: Windows CPU usage is high
description: 'CPU usage on host:9182 has exceeded 80% for 15 minutes.'
- alertname: WindowsMemoryHighUsage
@@ -64,6 +188,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsmemoryhighusage/
summary: Windows memory usage is high
description: 'Physical memory usage on host:9182 has exceeded 85% for 15 minutes.'
- alertname: WindowsDiskAlmostFull
@@ -71,10 +196,12 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsdiskalmostfull/
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 20% free space. Windows may delay space metrics by 10–15 minutes.'
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: page}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsdiskalmostfull/
summary: Windows disk space is low
description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.'
@@ -108,6 +235,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsadpendingreplication/
summary: Active Directory replication queue is high
description: 'Pending replication operations on host:9182 have exceeded 10 for 15 minutes.'
- alertname: WindowsADPendingReplication
@@ -121,6 +249,7 @@ tests:
exp_alerts:
- exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning}
exp_annotations:
runbook_url: https://runbooks.example.com/windows/windowsadreplicationfailures/
summary: Active Directory replication requests are failing
description: 'Replication synchronization requests on host:9182 have been failing for 15 minutes.'
- alertname: WindowsADReplicationFailures
contrib/mixin/tests/resources.test.yaml
+183
View File
@@ -0,0 +1,183 @@
rule_files:
- ../windows_rules.yaml
- ../windows_alerts.yaml
evaluation_interval: 1m
group_eval_order:
- windows-exporter-recording
- windows-exporter-alerts
tests:
- name: Memory cache includes all lists, commit is a ratio and paging is a rate
interval: 1m
input_series:
- series: 'windows_memory_cache_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '10x10'
- series: 'windows_memory_modified_page_list_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '20x10'
- series: 'windows_memory_standby_cache_core_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '30x10'
- series: 'windows_memory_standby_cache_normal_priority_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '40x10'
- series: 'windows_memory_standby_cache_reserve_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '50x10'
- series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '80x10'
- series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '100x10'
- series: 'windows_memory_swap_page_operations_total{job="windows_exporter", instance="host:9182", cluster="a"}'
values: '0+120x10'
- series: 'windows_memory_committed_bytes{job="other", instance="host:9182"}'
values: '95x10'
- series: 'windows_memory_commit_limit{job="other", instance="host:9182"}'
values: '100x10'
promql_expr_test:
- expr: windows:memory_cached:bytes
eval_time: 10m
exp_samples:
- labels: 'windows:memory_cached:bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 150
- expr: windows:memory_committed:ratio
eval_time: 10m
exp_samples:
- labels: 'windows:memory_committed:ratio{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 0.8
- expr: windows:memory_swap_page_operations:rate
eval_time: 10m
exp_samples:
- labels: 'windows:memory_swap_page_operations:rate{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 2
- name: Disk latency uses operations, handles resets and busy time stays bounded
interval: 1m
input_series:
- series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+12x5 12+12x4'
- series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+120x5 120+120x4'
- series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+24x10'
- series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+240x10'
- series: 'windows_logical_disk_idle_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
values: '0+30x10'
- series: 'windows_logical_disk_idle_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}'
values: '0+90x10'
promql_expr_test:
- expr: windows:logical_disk_read_operations:rate
eval_time: 8m
exp_samples:
- labels: 'windows:logical_disk_read_operations:rate{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 2
- expr: windows:logical_disk_write_operations:rate
eval_time: 10m
exp_samples:
- labels: 'windows:logical_disk_write_operations:rate{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 4
- expr: windows:logical_disk_read_latency:seconds
eval_time: 8m
exp_samples:
- labels: 'windows:logical_disk_read_latency:seconds{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 0.1
- expr: windows:logical_disk_write_latency:seconds
eval_time: 10m
exp_samples:
- labels: 'windows:logical_disk_write_latency:seconds{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 0.1
- expr: windows:logical_disk_busy:ratio
eval_time: 10m
exp_samples:
- labels: 'windows:logical_disk_busy:ratio{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 0.5
- labels: 'windows:logical_disk_busy:ratio{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}'
value: 0
- name: Full duplex network utilization uses bytes and keeps directions separate
interval: 1m
input_series:
- series: 'windows_net_bytes_received_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+600x10'
- series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+1200x10'
- series: 'windows_net_current_bandwidth_bytes{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '100x10'
- series: 'windows_net_packets_received_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+60x10'
- series: 'windows_net_packets_outbound_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+120x10'
- series: 'windows_net_packets_received_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+180x10'
- series: 'windows_net_packets_outbound_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
values: '0+240x10'
promql_expr_test:
- expr: windows:net_received_utilization:ratio
eval_time: 10m
exp_samples:
- labels: 'windows:net_received_utilization:ratio{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 0.1
- expr: windows:net_sent_utilization:ratio
eval_time: 10m
exp_samples:
- labels: 'windows:net_sent_utilization:ratio{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 0.2
- expr: windows:net_received_errors:rate
eval_time: 10m
exp_samples:
- labels: 'windows:net_received_errors:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 1
- expr: windows:net_outbound_errors:rate
eval_time: 10m
exp_samples:
- labels: 'windows:net_outbound_errors:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 2
- expr: windows:net_received_discarded:rate
eval_time: 10m
exp_samples:
- labels: 'windows:net_received_discarded:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 3
- expr: windows:net_outbound_discarded:rate
eval_time: 10m
exp_samples:
- labels: 'windows:net_outbound_discarded:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}'
value: 4
- name: Zero capacity and idle devices produce no undefined ratios or latency alerts
interval: 1m
input_series:
- series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182"}'
values: '1x20'
- series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182"}'
values: '0x20'
- series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '1x20'
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182"}'
values: '1x20'
- series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182"}'
values: '0x20'
- series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", volume="C:"}'
values: '0x20'
- series: 'windows_net_bytes_received_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+600x20'
- series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0+600x20'
- series: 'windows_net_current_bandwidth_bytes{job="windows_exporter", instance="host:9182", nic="Ethernet"}'
values: '0x20'
promql_expr_test:
- {expr: 'windows:memory_usage:ratio', eval_time: 20m, exp_samples: []}
- {expr: 'windows:logical_disk_free:ratio', eval_time: 20m, exp_samples: []}
- {expr: 'windows:memory_committed:ratio', eval_time: 20m, exp_samples: []}
- {expr: 'windows:logical_disk_read_latency:seconds', eval_time: 20m, exp_samples: []}
- {expr: 'windows:logical_disk_write_latency:seconds', eval_time: 20m, exp_samples: []}
- {expr: 'windows:net_received_utilization:ratio', eval_time: 20m, exp_samples: []}
- {expr: 'windows:net_sent_utilization:ratio', eval_time: 20m, exp_samples: []}
alert_rule_test:
- {alertname: WindowsMemoryCommitHighUsage, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsDiskReadLatencyHigh, eval_time: 20m, exp_alerts: []}
- {alertname: WindowsDiskWriteLatencyHigh, eval_time: 20m, exp_alerts: []}
contrib/mixin/tests/rules.test.yaml
+24
View File
@@ -18,6 +18,20 @@ tests:
- series: 'windows_cpu_time_total{job="other", instance="host:9182", core="0", mode="idle"}'
values: '0+6x10'
promql_expr_test:
- expr: windows:cpu_cores:count
eval_time: 10m
exp_samples:
- labels: 'windows:cpu_cores:count{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 2
- labels: 'windows:cpu_cores:count{job="windows_exporter", instance="host:9182", cluster="b"}'
value: 1
- expr: 'windows:cpu_time:ratio_rate{mode="idle"}'
eval_time: 10m
exp_samples:
- labels: 'windows:cpu_time:ratio_rate{job="windows_exporter", instance="host:9182", cluster="a", mode="idle"}'
value: 0.4
- labels: 'windows:cpu_time:ratio_rate{job="windows_exporter", instance="host:9182", cluster="b", mode="idle"}'
value: 0.5
- expr: windows:cpu_usage:ratio
eval_time: 10m
exp_samples:
@@ -42,6 +56,16 @@ tests:
- series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume=""}'
values: '100x5'
promql_expr_test:
- expr: windows:memory_used:bytes
eval_time: 5m
exp_samples:
- labels: 'windows:memory_used:bytes{job="windows_exporter", instance="host:9182", cluster="a"}'
value: 75
- expr: windows:logical_disk_used:bytes
eval_time: 5m
exp_samples:
- labels: 'windows:logical_disk_used:bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}'
value: 80
- expr: windows:memory_usage:ratio
eval_time: 5m
exp_samples:
contrib/mixin/tests/runbooks.test.jsonnet
+23
View File
@@ -0,0 +1,23 @@
local linked = {
_config:: { runbookURLPattern: 'https://example.com/runbooks/%s/' },
prometheusAlerts:: {
groups: [
{
name: 'custom',
rules: [
{ alert: 'WindowsCustomAlert', expr: 'up == 0', annotations: { summary: 'Existing summary' } },
{ alert: 'WindowsOverride', expr: 'up == 0', annotations: { runbook_url: 'https://example.com/override' } },
{ record: 'windows:custom:sum', expr: 'sum(up)' },
{ alert: 'WindowsNoAnnotations', expr: 'up == 0' },
],
},
],
},
} + (import '../lib/add-runbook-links.libsonnet');
local rules = linked.prometheusAlerts.groups[0].rules;
assert rules[0].annotations == { summary: 'Existing summary', runbook_url: 'https://example.com/runbooks/windowscustomalert/' };
assert rules[1].annotations.runbook_url == 'https://example.com/override';
assert !std.objectHas(rules[2], 'annotations');
assert rules[3].annotations.runbook_url == 'https://example.com/runbooks/windowsnoannotations/';
assert rules[0].expr == 'up == 0' && linked.prometheusAlerts.groups[0].name == 'custom';
{ result: 'Runbook link composition passed' }
docs/.gitignore
+3
View File
@@ -0,0 +1,3 @@
/public/
/resources/
/.hugo_build.lock
docs/README.md
+5
View File
@@ -1,6 +1,11 @@
# Documentation
This directory contains documentation of the collectors in the windows_exporter, with information such as what metrics are exported, any flags for additional configuration, and some example usage in alerts and queries.
The [alert runbooks](runbooks) describe how to diagnose and mitigate alerts from
the [Windows exporter mixin](../contrib/mixin/README.md). Preview the runbook site
from the repository root with `hugo server --source docs`; its dedicated GitHub
Actions workflow publishes default-branch changes to GitHub Pages.
# Collectors
- [`ad`](collector.ad.md)
- [`adcs`](collector.adcs.md)
docs/check_links.py
+48
View File
@@ -0,0 +1,48 @@
"""Check generated site links while respecting the configured Pages base path."""
from html.parser import HTMLParser
from pathlib import Path
import sys
from urllib.parse import unquote, urlsplit
class Links(HTMLParser):
def __init__(self):
super().__init__()
self.links = []
def handle_starttag(self, tag, attrs):
for name, value in attrs:
if name in ("href", "src") and value:
self.links.append(value)
root = Path(sys.argv[1])
prefix = urlsplit(sys.argv[2]).path.rstrip("/") + "/"
errors = []
checked = 0
pages = list(root.rglob("*.html"))
if not pages:
sys.exit(f"No generated HTML pages in {root}")
for page in pages:
parser = Links()
parser.feed(page.read_text(encoding="utf-8"))
for link in parser.links:
url = urlsplit(link)
if url.scheme or url.netloc or not url.path:
continue
if url.path.startswith("/"):
if not url.path.startswith(prefix):
errors.append(f"{page}: link {link} is outside the site base path {prefix}")
continue
target = root / unquote(url.path.removeprefix(prefix))
else:
target = page.parent / unquote(url.path)
if target.is_dir():
target /= "index.html"
if not target.is_file():
errors.append(f"{page}: broken link {link}")
checked += 1
if errors:
sys.exit("\n".join(errors))
print(f"Checked {checked} internal links across {len(pages)} HTML pages")
docs/hugo.toml
+15
View File
@@ -0,0 +1,15 @@
baseURL = "https://prometheus-community.github.io/windows_exporter/"
title = "Windows exporter runbooks"
locale = "en"
disableKinds = ["taxonomy", "term", "RSS", "sitemap"]
[module]
[[module.mounts]]
source = "runbooks"
target = "content/runbooks"
[[module.mounts]]
source = "layouts"
target = "layouts"
[[module.mounts]]
source = "static"
target = "static"
docs/layouts/baseof.html
+16
View File
@@ -0,0 +1,16 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{{ if not .IsHome }}{{ .Title }} | {{ end }}{{ site.Title }}</title>
<link rel="stylesheet" href="{{ site.Home.RelPermalink }}runbooks.css">
</head>
<body>
<header>
<a href="{{ site.Home.RelPermalink }}">{{ site.Title }}</a>
<a href="https://github.com/prometheus-community/windows_exporter">GitHub</a>
</header>
<main>{{ block "main" . }}{{ end }}</main>
</body>
</html>
docs/layouts/list.html
+13
View File
@@ -0,0 +1,13 @@
{{/* The base template declares the doctype. <!-- htmlhint-disable doctype-first --> */}}
{{ define "main" }}
<h1>{{ .Title }}</h1>
{{ .Content }}
{{ if .IsHome }}
<p>Diagnosis and mitigation for alerts from the Windows exporter mixin.</p>
{{ end }}
<ul class="runbooks">
{{ range (site.GetPage "/runbooks").RegularPages.ByTitle }}
<li><a href="{{ .RelPermalink }}">{{ .Title }}</a></li>
{{ end }}
</ul>
{{ end }}
docs/layouts/single.html
+8
View File
@@ -0,0 +1,8 @@
{{/* The base template declares the doctype. <!-- htmlhint-disable doctype-first --> */}}
{{ define "main" }}
<nav aria-label="Breadcrumb"><a href="{{ site.Home.RelPermalink }}runbooks/">All runbooks</a></nav>
<article>
<h1>{{ .Title }}</h1>
{{ .Content }}
</article>
{{ end }}
docs/runbooks/_index.md
+12
View File
@@ -0,0 +1,12 @@
---
title: Windows exporter runbooks
---
These runbooks describe alerts from the [Windows exporter mixin](https://github.com/prometheus-community/windows_exporter/tree/master/contrib/mixin).
Use the alert's `job`, `instance`, and any `cluster` or environment labels to find
the affected host. Disk alerts also identify a `volume`; network alerts identify
a `nic`. Adjust example PromQL selectors to match your scrape configuration.
The default thresholds are starting points. Check the generated alert rules for
your configured thresholds and durations. Active Directory and NTP clock alerts
are opt-in and require their collectors to be enabled.
docs/runbooks/windowsadpendingreplication.md
+41
View File
@@ -0,0 +1,41 @@
---
title: WindowsADPendingReplication
---
## Meaning
Pending Active Directory replication operations exceed the configured threshold
for 15 minutes. The default is 50 operations. This alert is generated only with
`enableActiveDirectory: true` and requires the `ad` collector on domain controllers.
## Impact
Directory changes may be delayed between domain controllers, causing inconsistent
authentication, group membership, DNS, or policy behavior.
## Diagnosis
1. Inspect `windows_ad_replication_pending_operations` and determine whether the
queue is growing or draining.
2. On a domain controller, inspect replication health:
```powershell
repadmin /replsummary
repadmin /showrepl
repadmin /queue
dcdiag /test:Replications
```
3. Check the Directory Service event log, partner reachability, DNS resolution,
site links and schedules, and CPU/disk/network pressure.
4. Correlate the queue with [WindowsADReplicationFailures](../windowsadreplicationfailures/)
and planned large directory changes.
## Mitigation
Restore connectivity and DNS between replication partners. Correct replication
errors and review site-link capacity or schedules with the directory owner.
Relieve resource pressure on domain controllers. Allow expected bulk changes to
drain while monitoring progress. Avoid forcing widespread replication or removing
partners before understanding the failure. Confirm the queue decreases and
replication health checks succeed.
docs/runbooks/windowsadreplicationfailures.md
+40
View File
@@ -0,0 +1,40 @@
---
title: WindowsADReplicationFailures
---
## Meaning
The rate of replication synchronization requests minus successful requests
exceeds the configured threshold for 15 minutes. The default is any sustained
positive failure rate. This alert is opt-in and requires the `ad` collector.
## Impact
Failed replication may leave domain controllers with stale directory data,
affecting authentication, DNS, policies, and directory changes.
## Diagnosis
1. Inspect rates of `windows_ad_replication_sync_requests_total` and
`windows_ad_replication_sync_requests_success_total` on the affected controller.
Counter resets around a restart may temporarily distort their difference.
2. Use the Active Directory diagnostic tools to identify partners and error codes:
```powershell
repadmin /replsummary
repadmin /showrepl
dcdiag /test:Replications /test:DNS
```
3. Check Directory Service events, DNS resolution, RPC connectivity, replication
permissions, and clock synchronization on both partners.
4. Check for an offline or decommissioned controller and review recent topology,
firewall, or credential changes.
## Mitigation
Resolve the specific error reported by replication diagnostics. Restore partner
connectivity, DNS, time synchronization, or permissions as appropriate. Handle
retired domain controllers through the documented Active Directory removal
process. Coordinate topology and recovery changes with the directory owner.
Confirm replication succeeds and the pending queue drains.
docs/runbooks/windowsclocknotsynchronising.md
+43
View File
@@ -0,0 +1,43 @@
---
title: WindowsClockNotSynchronising
---
## Meaning
Windows Time reports no active NTP time source for ten minutes. This alert is
generated only with `enableTime: true` and requires the `time` collector and its
`ntp` subcollector. Enable it on hosts expected to synchronize through NTP;
hosts intentionally using another clock source may legitimately report zero.
## Impact
The clock can drift without a working time source. Clock errors can affect
Kerberos authentication, TLS validation, scheduled work, and event timestamps.
## Diagnosis
1. Inspect `windows_time_ntp_client_time_sources` and the exporter collector
success metric. Missing metrics indicate a different problem than a zero value.
2. On the affected host, inspect Windows Time:
```powershell
Get-Service W32Time
w32tm /query /status
w32tm /query /source
w32tm /query /peers
w32tm /query /configuration
```
3. Check the Windows Time-Service event log, source DNS resolution, and network
access to the source over UDP port 123.
4. For domain members, inspect the domain time hierarchy and the PDC emulator's
upstream source. For virtual machines, check hypervisor time integration.
## Mitigation
Restore the expected Windows Time service, configured source, and UDP reachability.
Respect domain policy; configure external peers on the appropriate authoritative
server rather than overriding every domain member. After correcting the cause,
request rediscovery with `w32tm /resync /rediscover` and verify a valid source and
successful synchronization. Follow [WindowsClockSkewDetected](../windowsclockskewdetected/)
if a significant offset remains.
docs/runbooks/windowsclockskewdetected.md
+43
View File
@@ -0,0 +1,43 @@
---
title: WindowsClockSkewDetected
---
## Meaning
The absolute clock offset reported by Windows Time exceeds the configured
threshold for ten minutes. The default is 0.05 seconds (50 ms). This opt-in alert
requires the `time` collector with its `ntp` subcollector. The offset is relative
to the selected source and does not independently validate that source's accuracy.
## Impact
Clock skew can affect authentication, certificate validity, scheduled work, and
the ordering of events across hosts. The default threshold is an early warning;
application tolerances vary.
## Diagnosis
1. Inspect `windows_time_computed_time_offset_seconds`, source count, and
`windows_time_ntp_round_trip_delay_seconds` for the affected host.
2. Check synchronization state and the selected source:
```powershell
w32tm /query /status
w32tm /query /source
w32tm /query /configuration
```
3. Compare with a trusted source using
`w32tm /stripchart /computer:<trusted-time-server> /samples:5 /dataonly`.
Check source health, network delay, Windows Time-Service events, and domain
or hypervisor time configuration.
4. Consult [WindowsClockNotSynchronising](../windowsclocknotsynchronising/) when
no NTP source is available.
## Mitigation
Restore a healthy source and correct time configuration before requesting
`w32tm /resync /rediscover`. Coordinate large clock corrections with application
owners because abrupt changes can disrupt running workloads. Avoid setting the
clock manually as a recurring fix. Confirm the reported offset decreases and
remains within the application's required tolerance.
docs/runbooks/windowscollectorfailed.md
+40
View File
@@ -0,0 +1,40 @@
---
title: WindowsCollectorFailed
---
## Meaning
The collector identified by the `collector` label has reported
`windows_exporter_collector_success == 0` for five minutes.
## Impact
Metrics from this collector are incomplete or missing. Alerts relying on those
metrics may stop evaluating even though the exporter remains reachable.
## Diagnosis
1. Find the failing collector and compare the affected hosts:
```promql
windows_exporter_collector_success{job="windows_exporter"} == 0
```
2. Review exporter logs for that collector's error and duration. Check whether
the failure started after an exporter upgrade, Windows update, or account
permission change.
3. Check the collector's documentation for supported Windows roles, required
privileges, and performance counters. Confirm the required service or role
exists and is running on the host.
4. Compare `windows_exporter_collector_duration_seconds` with the scrape timeout.
Test the relevant performance counters locally with Windows Performance
Monitor.
## Mitigation
Restore the required service, privileges, or performance counter availability.
Limit expensive collector queries when collection exceeds the timeout, and
adjust timeouts only after measuring normal collection time. Disable a collector
on hosts where its role is absent. If the problem began with an upgrade, check
release notes and apply a fix or roll back through the normal change process.
Confirm the collector succeeds and its dependent metrics resume.
docs/runbooks/windowscpuhighusage.md
+37
View File
@@ -0,0 +1,37 @@
---
title: WindowsCPUHighUsage
---
## Meaning
Average non-idle CPU usage across logical processors exceeds the configured
threshold for 15 minutes. The default is 90%, measured over a five-minute rate
window. This measures CPU time; Task Manager's frequency-adjusted utility can
show a different value.
## Impact
The host has little CPU capacity for additional work. Applications may show
increased latency, timeouts, or reduced throughput.
## Diagnosis
1. Inspect `windows:cpu_usage:ratio` for the affected host and compare it with
its workload and normal baseline.
2. Inspect `windows:cpu_time:ratio_rate` by `mode`. High user time points toward
application work; high privileged, interrupt, or DPC time can indicate kernel
or driver activity. DPC and interrupt modes overlap privileged time; do not
sum all modes to estimate usage.
3. Use Task Manager, Resource Monitor, or Performance Monitor to identify the
busy processes. Compare individual core usage to detect a single-threaded
bottleneck.
4. Check scheduled tasks, antivirus scans, recent deployments, and hypervisor
CPU contention.
## Mitigation
Reduce or redistribute the workload, reschedule expensive background tasks, or
add CPU capacity. Investigate a process or driver consuming unexpected CPU
before restarting it. Confirm CPU usage and application latency return to normal.
For expected sustained workloads, tune the threshold against service objectives
and available capacity.
docs/runbooks/windowsdiskalmostfull.md
+38
View File
@@ -0,0 +1,38 @@
---
title: WindowsDiskAlmostFull
---
## Meaning
The volume's free space remains below a configured threshold for 15 minutes.
Defaults are 10% for warning and 5% for critical. Both severity levels fire below
the critical threshold unless Alertmanager inhibition suppresses the warning.
Windows can delay the underlying space counters by 10–15 minutes.
## Impact
Applications may fail to write data or logs. Low free space can prevent updates,
pagefile growth, database operations, and normal operating system behavior.
## Diagnosis
1. Use the `volume` label to identify the affected filesystem and inspect
`windows:logical_disk_free:ratio` and `windows_logical_disk_free_bytes`.
2. Verify current capacity locally; the exporter counters may lag:
```powershell
Get-Volume | Select-Object DriveLetter, FileSystemLabel, SizeRemaining, Size
```
3. Identify growing logs, temporary files, backups, database files, and application
data. Check retention policies and recent jobs.
4. Consult [WindowsDiskFillingUp](../windowsdiskfillingup/) when free space is
declining rapidly.
## Mitigation
Expand the volume or remove files through the owning application's supported
retention and cleanup process. Preserve required data and backups; avoid manually
deleting database files or Windows system files. Reduce incoming writes if the
volume may fill before capacity is restored. Verify space locally and allow the
Windows counters to refresh before expecting the alert to resolve.
docs/runbooks/windowsdiskfillingup.md
+37
View File
@@ -0,0 +1,37 @@
---
title: WindowsDiskFillingUp
---
## Meaning
Free space is below the warning threshold and a linear forecast predicts the
volume will fill within the configured horizon. Defaults use six hours of
history, a 24-hour prediction horizon, and a one-hour alert duration.
## Impact
Continued growth can exhaust the volume before routine capacity maintenance,
causing write failures or an application outage.
## Diagnosis
1. Inspect the free-space trend for the volume and verify local free space with
`Get-Volume`. Windows disk space counters can lag by 10–15 minutes.
2. Inspect the forecast using the configured selector and window, for example:
```promql
predict_linear(windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}[6h], 24 * 3600)
```
3. Identify the application, logs, backups, or temporary data causing growth.
Check whether a recent bulk write distorted the trend or growth is sustained.
4. Compare absolute remaining bytes with the application's write rate. A linear
forecast is an estimate and does not model scheduled cleanup or sudden growth.
## Mitigation
Restore retention and rotation, reduce write growth, or expand the volume before
it fills. Coordinate data cleanup with the owner. If the forecast reflects a
known temporary job, verify its end time and remaining capacity before silencing
the alert. Follow [WindowsDiskAlmostFull](../windowsdiskalmostfull/) when free
space is already critically low.
docs/runbooks/windowsdiskreadlatencyhigh.md
+34
View File
@@ -0,0 +1,34 @@
---
title: WindowsDiskReadLatencyHigh
---
## Meaning
Average disk read service time exceeds the configured threshold for 15 minutes.
The default is 0.05 seconds (50 ms) per operation, computed from read-time and
read-operation counter rates. Volumes with no reads produce no latency sample.
## Impact
Slow storage reads may delay application requests, database queries, and paging.
The recording is an average, so it does not expose tail latency.
## Diagnosis
1. Inspect `windows:logical_disk_read_latency:seconds` for the affected `volume`
alongside read IOPS, read throughput, and `windows:logical_disk_busy:ratio`.
2. Inspect `windows_logical_disk_requests_queued` and use Resource Monitor to
identify processes generating disk reads. Look for backup, antivirus, and
indexing jobs.
3. Check System event logs and storage-controller logs for disk, controller,
multipath, or network-storage errors.
4. For virtual machines, check datastore latency and competing workloads on the
hypervisor or storage array. Low IOPS can make averages noisy.
## Mitigation
Reduce competing reads or reschedule background jobs. Address storage-path errors
and provision enough IOPS and throughput for the workload. Investigate excessive
paging if memory pressure is contributing. Tune the latency threshold to the
storage tier and application's service objectives, then confirm both service
latency and disk read latency improve.
docs/runbooks/windowsdiskwritelatencyhigh.md
+33
View File
@@ -0,0 +1,33 @@
---
title: WindowsDiskWriteLatencyHigh
---
## Meaning
Average disk write service time exceeds the configured threshold for 15 minutes.
The default is 0.05 seconds (50 ms) per operation, computed from write-time and
write-operation counter rates. Volumes with no writes produce no latency sample.
## Impact
Slow writes may delay transactions, log flushing, file uploads, or paging. An
average can conceal short periods of very high latency.
## Diagnosis
1. Inspect `windows:logical_disk_write_latency:seconds`, write IOPS, write
throughput, `windows:logical_disk_busy:ratio`, and queued requests for the
affected `volume`.
2. Use Resource Monitor to find processes generating writes. Review backup jobs,
database checkpoints, log growth, and scheduled maintenance.
3. Check free space and storage-controller health. Inspect System event logs for
disk, controller, multipath, or network-storage errors.
4. For virtual machines or remote storage, inspect the backing datastore and
array for throughput limits and contention.
## Mitigation
Reduce competing writes or move maintenance jobs. Repair storage-path problems
and expand IOPS or throughput capacity. Coordinate database or logging changes
with the application owner. Avoid disabling durability guarantees to silence an
alert. Confirm application write latency recovers and the disk metrics improve.
docs/runbooks/windowsexporterdown.md
+39
View File
@@ -0,0 +1,39 @@
---
title: WindowsExporterDown
---
## Meaning
Prometheus has failed to scrape the target for five minutes (`up == 0`).
The alert does not detect hosts removed from service discovery.
## Impact
Monitoring data and resource alerts for the host are unavailable. The host,
exporter service, or the network path may be down.
## Diagnosis
1. Open the target in Prometheus's **Targets** page and read the scrape error.
Check the target address, TLS settings, authentication, and scrape timeout.
2. From the Prometheus network, check the configured metrics endpoint, normally
`http://<host>:9182/metrics`. Check DNS resolution and routing.
3. On the affected host, inspect the service and listening socket:
```powershell
Get-Service windows_exporter
Get-NetTCPConnection -LocalPort 9182 -State Listen
Test-NetConnection -ComputerName localhost -Port 9182
```
4. Review exporter logs and the Windows Application/System event logs for
startup errors, permission problems, or collector timeouts.
## Mitigation
Restore the network path or correct the scrape configuration. If the exporter
service is stopped, resolve its startup error and start it with
`Start-Service windows_exporter`. If local requests succeed but remote requests
fail, check the host firewall and network ACLs for the configured listener.
Confirm the target becomes healthy and fresh metrics appear before closing the
incident. Update service discovery when a host was intentionally decommissioned.
docs/runbooks/windowsmemorycommithighusage.md
+40
View File
@@ -0,0 +1,40 @@
---
title: WindowsMemoryCommitHighUsage
---
## Meaning
Committed memory exceeds the configured percentage of the system commit limit
for 15 minutes. The default threshold is 90%. The commit limit depends on
physical memory and pagefile capacity.
## Impact
The system may reject new memory commitments, causing application failures even
when some physical memory remains available.
## Diagnosis
1. Inspect `windows:memory_committed:ratio`, `windows_memory_committed_bytes`,
and `windows_memory_commit_limit`. Determine whether committed bytes increased
or the limit decreased.
2. In Task Manager or Resource Monitor, inspect per-process commit size and look
for persistent growth. Check recent workload and deployment changes.
3. Inspect pagefile configuration and the free space on its volumes:
```powershell
Get-CimInstance Win32_PageFileUsage
Get-CimInstance Win32_PageFileSetting
Get-Volume
```
4. Check physical memory pressure and paging alongside this alert. The recording
rule omits invalid zero commit limits rather than reporting infinity.
## Mitigation
Reduce memory commitments or address the leaking process. Restore pagefile
capacity and disk free space when the commit limit is unexpectedly low. Review
pagefile sizing with the workload owner; some changes require a reboot. Add
physical memory for sustained demand. Confirm committed bytes remain safely
below the commit limit and applications can allocate memory normally.
docs/runbooks/windowsmemoryhighusage.md
+34
View File
@@ -0,0 +1,34 @@
---
title: WindowsMemoryHighUsage
---
## Meaning
Physical memory usage exceeds the configured threshold for 15 minutes. The
default is 90%. Usage is total physical memory minus available memory; available
memory includes reclaimable pages and differs from strictly free memory.
## Impact
Memory pressure may cause paging, application latency, and allocation failures.
A high ratio alone does not prove the system is paging heavily.
## Diagnosis
1. Inspect `windows:memory_usage:ratio`, `windows:memory_used:bytes`, and
`windows_memory_available_bytes` on the affected host.
2. Check `windows:memory_swap_page_operations:rate`, disk latency, and
`windows:memory_committed:ratio` for related pressure.
3. In Resource Monitor, inspect processes' working sets, commit usage, and hard
faults. Look for growth after a deployment or an unexpected workload increase.
4. Inspect `windows_memory_pool_nonpaged_bytes` and
`windows_memory_pool_paged_bytes` if application working sets do not explain
the usage. Driver or kernel pools may be growing.
## Mitigation
Reduce concurrent work, move workloads, or add physical memory. Investigate leaks
in the responsible application or driver. Restart a service only after assessing
its availability impact and preserving useful diagnostics. Pagefile changes
affect the commit limit and do not add physical memory. Confirm available memory
and service latency recover.
docs/runbooks/windowsnetworkdiscards.md
+34
View File
@@ -0,0 +1,34 @@
---
title: WindowsNetworkDiscards
---
## Meaning
The sum of inbound and outbound discarded packet rates exceeds the configured
threshold for 15 minutes. The default is one discard per second per interface.
Discards count packets dropped without a packet error, such as buffer exhaustion.
## Impact
Dropped packets can cause retransmissions, reduced throughput, and application
timeouts. Some drops may be expected under a deliberate traffic policy.
## Diagnosis
1. Inspect `windows:net_received_discarded:rate` and
`windows:net_outbound_discarded:rate` to identify the direction and affected
`nic`.
2. Compare traffic, directional utilization ratios, and
`windows_net_output_queue_length_packets` with the normal baseline.
3. Inspect `Get-NetAdapterStatistics`, switch-port drops, adapter buffers,
virtual switch counters, and network QoS policies.
4. Correlate drops with CPU pressure, traffic bursts, hypervisor contention, and
application failures. An unknown or zero bandwidth estimate produces no
utilization sample and should not be interpreted as zero traffic.
## Mitigation
Relieve congestion, distribute traffic, or add bandwidth. Correct buffer, driver,
or virtual switch problems when evidence identifies them. Review intentional
drop policies with the network owner before changing thresholds. Confirm discard
rates and application retransmissions return to the expected baseline.
docs/runbooks/windowsnetworkerrors.md
+39
View File
@@ -0,0 +1,39 @@
---
title: WindowsNetworkErrors
---
## Meaning
The sum of inbound and outbound packet error rates exceeds the configured
threshold for 15 minutes. The default is one error per second per interface.
The alert identifies the interface using its `nic` label.
## Impact
Packet errors may cause retransmissions, lower throughput, and intermittent
connection failures.
## Diagnosis
1. Inspect `windows:net_received_errors:rate` and
`windows:net_outbound_errors:rate` separately to identify the direction.
2. Compare traffic and `windows:net_received_utilization:ratio` and
`windows:net_sent_utilization:ratio` with the normal baseline. These ratios
use bandwidth in bytes per second and keep each direction separate.
3. Inspect local adapter state and counters:
```powershell
Get-NetAdapter
Get-NetAdapterStatistics
```
4. Inspect switch-port counters, cabling, link negotiation, NIC drivers, and
System event logs. For virtual NICs, inspect the host and virtual switch.
## Mitigation
Repair faulty links or switch ports and correct driver or adapter configuration
problems. Apply driver and firmware fixes through the normal change process.
Relieve congestion if it contributes. Verify errors stop increasing and affected
application connections recover; cumulative counters do not need to return to
zero for the rate alert to resolve.
docs/static/runbooks.css
Vendored
+17
View File
@@ -0,0 +1,17 @@
:root {
color-scheme: light dark;
font-family: system-ui, sans-serif;
line-height: 1.6;
color: light-dark(#20252b, #e5e9ef);
background: light-dark(#fff, #15191f);
}
body { max-width: 64rem; margin: auto; padding: 1.5rem; }
header { display: flex; justify-content: space-between; gap: 1rem; border-bottom: 1px solid #8993a0; padding-bottom: 1rem; }
main { max-width: 54rem; margin: 2rem auto; }
a { color: light-dark(#0059b3, #80bfff); overflow-wrap: anywhere; }
h1, h2 { line-height: 1.3; overflow-wrap: anywhere; }
h2 { margin-top: 2rem; }
pre { overflow-x: auto; padding: 1rem; border: 1px solid #8993a0; border-radius: .3rem; }
code { font-size: .95em; }
li { margin: .4rem 0; }
.runbooks { padding-left: 1.5rem; }