From d06f16892619d8c9d58fcf1ff641ab0fdb7f4ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Fri, 9 Oct 2026 09:10:59 +0200 Subject: [PATCH] contrib: expand Windows mixin rules and publish runbooks (#2575) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Otto Kröpke --- .github/workflows/mixin.yml | 14 +- .github/workflows/runbooks.yml | 112 +++++++++++ README.md | 3 +- contrib/mixin/.pint.hcl | 6 + contrib/mixin/Makefile | 19 +- contrib/mixin/README.md | 101 ++++++++-- contrib/mixin/alerts/alerts.libsonnet | 81 ++++++-- contrib/mixin/config.libsonnet | 9 + contrib/mixin/lib/add-runbook-links.libsonnet | 19 ++ contrib/mixin/lib/queries.libsonnet | 26 ++- contrib/mixin/mixin.libsonnet | 3 +- contrib/mixin/rules/rules.libsonnet | 45 +++++ contrib/mixin/tests/alerts.test.yaml | 118 +++++++++++ contrib/mixin/tests/check_runbooks.py | 36 ++++ contrib/mixin/tests/config.libsonnet | 7 + contrib/mixin/tests/custom.test.yaml | 129 ++++++++++++ contrib/mixin/tests/resources.test.yaml | 183 ++++++++++++++++++ contrib/mixin/tests/rules.test.yaml | 24 +++ contrib/mixin/tests/runbooks.test.jsonnet | 23 +++ docs/.gitignore | 3 + docs/README.md | 5 + docs/check_links.py | 48 +++++ docs/hugo.toml | 15 ++ docs/layouts/baseof.html | 16 ++ docs/layouts/list.html | 13 ++ docs/layouts/single.html | 8 + docs/runbooks/_index.md | 12 ++ docs/runbooks/windowsadpendingreplication.md | 41 ++++ docs/runbooks/windowsadreplicationfailures.md | 40 ++++ docs/runbooks/windowsclocknotsynchronising.md | 43 ++++ docs/runbooks/windowsclockskewdetected.md | 43 ++++ docs/runbooks/windowscollectorfailed.md | 40 ++++ docs/runbooks/windowscpuhighusage.md | 37 ++++ docs/runbooks/windowsdiskalmostfull.md | 38 ++++ docs/runbooks/windowsdiskfillingup.md | 37 ++++ docs/runbooks/windowsdiskreadlatencyhigh.md | 34 ++++ docs/runbooks/windowsdiskwritelatencyhigh.md | 33 ++++ docs/runbooks/windowsexporterdown.md | 39 ++++ docs/runbooks/windowsmemorycommithighusage.md | 40 ++++ docs/runbooks/windowsmemoryhighusage.md | 34 ++++ docs/runbooks/windowsnetworkdiscards.md | 34 ++++ docs/runbooks/windowsnetworkerrors.md | 39 ++++ docs/static/runbooks.css | 17 ++ 43 files changed, 1629 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/runbooks.yml create mode 100644 contrib/mixin/lib/add-runbook-links.libsonnet create mode 100644 contrib/mixin/tests/check_runbooks.py create mode 100644 contrib/mixin/tests/resources.test.yaml create mode 100644 contrib/mixin/tests/runbooks.test.jsonnet create mode 100644 docs/.gitignore create mode 100644 docs/check_links.py create mode 100644 docs/hugo.toml create mode 100644 docs/layouts/baseof.html create mode 100644 docs/layouts/list.html create mode 100644 docs/layouts/single.html create mode 100644 docs/runbooks/_index.md create mode 100644 docs/runbooks/windowsadpendingreplication.md create mode 100644 docs/runbooks/windowsadreplicationfailures.md create mode 100644 docs/runbooks/windowsclocknotsynchronising.md create mode 100644 docs/runbooks/windowsclockskewdetected.md create mode 100644 docs/runbooks/windowscollectorfailed.md create mode 100644 docs/runbooks/windowscpuhighusage.md create mode 100644 docs/runbooks/windowsdiskalmostfull.md create mode 100644 docs/runbooks/windowsdiskfillingup.md create mode 100644 docs/runbooks/windowsdiskreadlatencyhigh.md create mode 100644 docs/runbooks/windowsdiskwritelatencyhigh.md create mode 100644 docs/runbooks/windowsexporterdown.md create mode 100644 docs/runbooks/windowsmemorycommithighusage.md create mode 100644 docs/runbooks/windowsmemoryhighusage.md create mode 100644 docs/runbooks/windowsnetworkdiscards.md create mode 100644 docs/runbooks/windowsnetworkerrors.md create mode 100644 docs/static/runbooks.css diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml index c7724d50..d97d4552 100644 --- a/.github/workflows/mixin.yml +++ b/.github/workflows/mixin.yml @@ -17,6 +17,10 @@ on: permissions: {} +concurrency: + group: mixin-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: rules: name: Validate Prometheus rules @@ -34,9 +38,17 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.27.1' + go-version-file: go.mod + + - name: Cache mixin tools + id: tools-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/bin + key: ${{ runner.os }}-${{ runner.arch }}-mixin-tools-${{ hashFiles('contrib/mixin/Makefile', 'go.mod') }} - name: Install pinned tools + if: steps.tools-cache.outputs.cache-hit != 'true' run: make tools - name: Lint and test rules diff --git a/.github/workflows/runbooks.yml b/.github/workflows/runbooks.yml new file mode 100644 index 00000000..197b3dd7 --- /dev/null +++ b/.github/workflows/runbooks.yml @@ -0,0 +1,112 @@ +name: Runbooks + +on: + pull_request: + paths: + - docs/runbooks/** + - docs/layouts/** + - docs/static/** + - docs/hugo.toml + - docs/check_links.py + - contrib/mixin/** + - .github/workflows/runbooks.yml + push: + branches: + - master + - main + - next + paths: + - docs/runbooks/** + - docs/layouts/** + - docs/static/** + - docs/hugo.toml + - docs/check_links.py + - contrib/mixin/** + - .github/workflows/runbooks.yml + workflow_dispatch: + +permissions: {} + +concurrency: + group: runbooks-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + # renovate: github=gohugoio/hugo + HUGO_VERSION: '0.167.0' + +jobs: + build: + name: Build runbooks + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + pages: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: 'false' + + - name: Install Hugo + run: | + curl --fail --location --silent --show-error \ + "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_linux-amd64.tar.gz" \ + -o "$RUNNER_TEMP/hugo.tar.gz" + tar -xzf "$RUNNER_TEMP/hugo.tar.gz" -C "$RUNNER_TEMP" hugo + echo "$RUNNER_TEMP" >> "$GITHUB_PATH" + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + + - name: Cache Jsonnet tools + id: jsonnet-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/go/bin/jsonnet + ~/go/bin/jsonnetfmt + key: ${{ runner.os }}-${{ runner.arch }}-runbooks-jsonnet-${{ hashFiles('contrib/mixin/Makefile', 'go.mod') }} + + - name: Install Jsonnet tools + if: steps.jsonnet-cache.outputs.cache-hit != 'true' + run: make -C contrib/mixin jsonnet-tools + + - name: Check alert runbook coverage + run: make -C contrib/mixin check-runbooks + + - name: Set up Pages + id: pages + if: github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + + - name: Build site + env: + PAGES_BASE_URL: ${{ steps.pages.outputs.base_url || 'https://prometheus-community.github.io/windows_exporter' }} + run: | + hugo --source docs --minify --baseURL "${PAGES_BASE_URL%/}/" --panicOnWarning + python3 docs/check_links.py docs/public "${PAGES_BASE_URL%/}/" + + - name: Upload Pages artifact + if: github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: docs/public + + deploy: + name: Publish runbooks + if: github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/README.md b/README.md index ebffe315..2dc5e880 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,8 @@ See detailed steps to install on Windows Kubernetes [here](./kubernetes/kubernet The [Windows exporter mixin](contrib/mixin/README.md) provides configurable recording rules and alerts for Prometheus, with Jsonnet generation, pint linting, -and promtool tests. +and promtool tests. Dedicated [alert runbooks](docs/runbooks) provide Windows +diagnosis and mitigation steps and are published through GitHub Pages. ## Supported versions diff --git a/contrib/mixin/.pint.hcl b/contrib/mixin/.pint.hcl index 17005e0b..08839fea 100644 --- a/contrib/mixin/.pint.hcl +++ b/contrib/mixin/.pint.hcl @@ -21,4 +21,10 @@ rule { value = ".+" severity = "bug" } + + annotation "runbook_url" { + required = true + value = "https?://.+" + severity = "bug" + } } diff --git a/contrib/mixin/Makefile b/contrib/mixin/Makefile index 1f3db391..fbafd9d1 100644 --- a/contrib/mixin/Makefile +++ b/contrib/mixin/Makefile @@ -2,24 +2,27 @@ JSONNET ?= jsonnet JSONNETFMT ?= jsonnetfmt PROMTOOL ?= promtool PINT ?= pint +PYTHON ?= python3 # renovate: go=github.com/google/go-jsonnet JSONNET_VERSION := v0.22.0 # renovate: go=github.com/cloudflare/pint PINT_VERSION := v0.89.2 -SOURCES := $(wildcard *.libsonnet *.jsonnet lib/*.libsonnet rules/*.libsonnet alerts/*.libsonnet tests/*.libsonnet) +SOURCES := $(wildcard *.libsonnet *.jsonnet lib/*.libsonnet rules/*.libsonnet alerts/*.libsonnet tests/*.libsonnet tests/*.jsonnet) FMT_FLAGS := -n 2 --max-blank-lines 2 --string-style s --comment-style s .DEFAULT_GOAL := generate .DELETE_ON_ERROR: -.PHONY: generate tools fmt lint test clean +.PHONY: generate tools jsonnet-tools fmt lint test check-runbooks clean generate: windows_rules.yaml windows_alerts.yaml -tools: +tools: jsonnet-tools + go install github.com/cloudflare/pint/cmd/pint@$(PINT_VERSION) + +jsonnet-tools: go install github.com/google/go-jsonnet/cmd/jsonnet@$(JSONNET_VERSION) go install github.com/google/go-jsonnet/cmd/jsonnetfmt@$(JSONNET_VERSION) - go install github.com/cloudflare/pint/cmd/pint@$(PINT_VERSION) windows_rules.yaml: $(SOURCES) $(JSONNET) -S rules.jsonnet > $@ @@ -44,8 +47,12 @@ lint: generate tests/custom_rules.yaml tests/custom_alerts.yaml $(PINT) --offline lint windows_rules.yaml windows_alerts.yaml $(PINT) --offline lint tests/custom_rules.yaml tests/custom_alerts.yaml -test: lint - $(PROMTOOL) test rules tests/rules.test.yaml tests/alerts.test.yaml tests/custom.test.yaml +check-runbooks: + $(JSONNET) -e '(import "mixin.libsonnet") { _config+:: { enableActiveDirectory: true, enableTime: true } }.prometheusAlerts' | $(PYTHON) tests/check_runbooks.py ../../docs/runbooks + +test: lint check-runbooks + $(JSONNET) tests/runbooks.test.jsonnet + $(PROMTOOL) test rules tests/*.test.yaml clean: rm -f windows_rules.yaml windows_alerts.yaml tests/custom_rules.yaml tests/custom_alerts.yaml diff --git a/contrib/mixin/README.md b/contrib/mixin/README.md index a42157e3..02834b9d 100644 --- a/contrib/mixin/README.md +++ b/contrib/mixin/README.md @@ -24,7 +24,8 @@ to generate rules. Generated files are ignored by Git: The default selector is `job="windows_exporter"`. Change it to match your scrape job before generating rules. Resource rules use the default `cpu`, `memory`, `logical_disk`, and `net` collectors. Enable `ad` on domain controllers when -opting into Active Directory alerts. +opting into Active Directory alerts. Clock alerts require the `time` collector +with its `ntp` subcollector and are intended for hosts using NTP synchronization. Copy both generated files into your Prometheus rule directory and reference them in `prometheus.yml`: @@ -53,6 +54,8 @@ modifying its source. For example, create `custom.libsonnet` in this directory: rateInterval: '5m', cpuHighUsageThreshold: 95, enableActiveDirectory: true, + enableTime: true, + runbookURLPattern: 'https://runbooks.example.com/windows/%s/', }, } ``` @@ -68,7 +71,8 @@ Selectors are comma-separated Prometheus label matchers without braces. They apply to both recording rules and alerts. All target labels, including `job`, `instance`, `cluster`, and custom selector labels, are preserved. CPU usage averages idle rates across cores and removes only `core` and `mode`. Disk and -network recordings also retain `volume` and `nic` respectively. +network recordings also retain `volume` and `nic` respectively. The CPU mode +recording retains `mode` and removes only `core`. `rateInterval` is the fixed window for recording rules and Active Directory failure rates; keep it at least four times your scrape interval. Grafana's @@ -80,18 +84,47 @@ rules. | Metric | Meaning | Unit | | --- | --- | --- | | `windows:cpu_usage:ratio` | Non-idle CPU time averaged across cores | Ratio, 0–1 | +| `windows:cpu_cores:count` | Logical processors observed in idle CPU metrics | Count | +| `windows:cpu_time:ratio_rate` | CPU time averaged across cores, retaining `mode` | Seconds/second | | `windows:memory_usage:ratio` | Physical memory in use, based on available bytes | Ratio, 0–1 | +| `windows:memory_used:bytes` | Total physical memory minus available memory | Bytes | +| `windows:memory_cached:bytes` | Cache, modified pages, and three standby lists | Bytes | +| `windows:memory_committed:ratio` | Committed bytes divided by the system commit limit | Ratio | +| `windows:memory_swap_page_operations:rate` | Paging operations | Operations/second | | `windows:logical_disk_free:ratio` | Free disk space per volume | Ratio, 0–1 | +| `windows:logical_disk_used:bytes` | Used disk space per volume | Bytes | +| `windows:logical_disk_busy:ratio` | Non-idle time per volume, clamped to 0–1 | Ratio, 0–1 | | `windows:logical_disk_read_bytes:rate` | Disk read throughput per volume | Bytes/second | | `windows:logical_disk_write_bytes:rate` | Disk write throughput per volume | Bytes/second | +| `windows:logical_disk_read_operations:rate` | Disk reads per volume | Operations/second | +| `windows:logical_disk_write_operations:rate` | Disk writes per volume | Operations/second | +| `windows:logical_disk_read_latency:seconds` | Read-time rate divided by read-operation rate | Seconds/operation | +| `windows:logical_disk_write_latency:seconds` | Write-time rate divided by write-operation rate | Seconds/operation | | `windows:net_received_bytes:rate` | Received network traffic per interface | Bytes/second | | `windows:net_sent_bytes:rate` | Sent network traffic per interface | Bytes/second | +| `windows:net_received_utilization:ratio` | Received throughput divided by interface bandwidth | Ratio | +| `windows:net_sent_utilization:ratio` | Sent throughput divided by interface bandwidth | Ratio | +| `windows:net_received_errors:rate` | Inbound packet errors | Packets/second | +| `windows:net_outbound_errors:rate` | Outbound packet errors | Packets/second | +| `windows:net_received_discarded:rate` | Inbound discarded packets | Packets/second | +| `windows:net_outbound_discarded:rate` | Outbound discarded packets | Packets/second | The names remain stable when `rateInterval` changes. Dashboard authors can use these recordings or query raw exporter metrics directly. CPU usage measures non-idle time; it is not the frequency-adjusted processor utility shown in Windows Task Manager. +CPU DPC and interrupt modes overlap privileged time; do not sum all modes. +Disk busy time uses the idle counter rather than summing read and write time, +which can exceed wall time for concurrent I/O. Latency is an average and is +omitted when there are no operations. Memory, disk space, commit, and network +utilization ratios are omitted when their capacity is zero or missing. Network +bandwidth is already in bytes/second; receive and send utilization are separate +for full-duplex links. +The cache recording includes modified pages and is not a measure of immediately +reclaimable memory. Paging operations can include file-backed pages as well as +pagefile traffic. + ## Alerts Threshold configuration values below are percentages except where noted. @@ -102,11 +135,18 @@ Threshold configuration values below are percentages except where noted. | `WindowsCollectorFailed` | `windows_exporter_collector_success == 0` | 5m | Warning | | `WindowsCPUHighUsage` | CPU usage above `cpuHighUsageThreshold` (90%) | 15m | Warning | | `WindowsMemoryHighUsage` | Memory usage above `memoryHighUsageThreshold` (90%) | 15m | Warning | +| `WindowsMemoryCommitHighUsage` | Commit usage above `memoryCommitHighUsageThreshold` (90%) | 15m | Warning | +| `WindowsDiskReadLatencyHigh` | Average read latency above `diskLatencyThresholdSeconds` (0.05 seconds) | 15m | Warning | +| `WindowsDiskWriteLatencyHigh` | Average write latency above `diskLatencyThresholdSeconds` (0.05 seconds) | 15m | Warning | +| `WindowsNetworkErrors` | Combined receive/outbound errors above `networkErrorRateThreshold` (1 packet/second) | 15m | Warning | +| `WindowsNetworkDiscards` | Combined receive/outbound discards above `networkDiscardRateThreshold` (1 packet/second) | 15m | Warning | | `WindowsDiskAlmostFull` | Free space below `diskFreeWarningThreshold` (10%) | 15m | Warning | | `WindowsDiskAlmostFull` | Free space below `diskFreeCriticalThreshold` (5%) | 15m | Critical | | `WindowsDiskFillingUp` | Below the warning threshold and predicted to fill within `diskPredictionHours` (24h), using `diskPredictionWindow` (6h) | 1h | Warning | | `WindowsADPendingReplication` | Pending operations above `adPendingReplicationThreshold` (50) | 15m | Warning | | `WindowsADReplicationFailures` | Failed synchronization requests above `adReplicationFailureRateThreshold` (0 requests/second) | 15m | Warning | +| `WindowsClockNotSynchronising` | No active NTP time sources | 10m | Warning | +| `WindowsClockSkewDetected` | Absolute reported time offset above `clockOffsetThresholdSeconds` (0.05 seconds) | 10m | Warning | Active Directory alerts are generated only when `enableActiveDirectory` is `true`. `criticalSeverity` defaults to `critical` and can be overridden to match @@ -114,6 +154,12 @@ your routing labels. Below the critical disk threshold, both disk severity levels fire; configure Alertmanager inhibition if only the critical notification should be sent. +Clock alerts are generated only when `enableTime` is `true`. A zero NTP source +count may be expected on hosts using another time provider. The offset compares +the host to its chosen source and does not independently verify that source. +Disk latency and network error/discard thresholds should be tuned for the +storage tier, traffic volume, and application requirements. + Windows disk size/free-space counters may lag by 10–15 minutes. Predictions depend on a sustained trend and are intentionally delayed for an hour. A target removed from service discovery has no `up` series and will not trigger @@ -121,9 +167,35 @@ removed from service discovery has no `up` series and will not trigger scrape and collector-failure alerts cover explicit failures, not disabled collectors. +## Runbooks and GitHub Pages + +Every alert receives a `runbook_url` annotation pointing to a dedicated page in +[docs/runbooks](../../docs/runbooks). Each page describes the meaning, impact, +diagnosis, and mitigation, including Windows commands and related metrics. +The default URL is +`https://prometheus-community.github.io/windows_exporter/runbooks//`. +Override `runbookURLPattern` for a fork, custom domain, or internal runbook site; +it takes one `%s` placeholder for the lowercase alert name. Explicit +`runbook_url` annotations supplied by downstream alerts are preserved. + +The theme-free Hugo site uses `docs/hugo.toml`. With Hugo 0.167.0 installed, +preview it from the repository root: + +```sh +hugo server --source docs +``` + +Build it with `hugo --source docs --minify`. The `Runbooks` workflow builds site +changes on pull requests, checks alert coverage and internal links, and publishes +default-branch changes with `configure-pages`, `upload-pages-artifact`, and +`deploy-pages`. This uses the +repository's GitHub Pages **Source: GitHub Actions** setting. The workflow reads +the configured Pages URL, including custom domains and repository subpaths. +Fork users should override `runbookURLPattern` to match their published URL. + ## Validate -Install `promtool` from a [Prometheus release](https://github.com/prometheus/prometheus/releases), +Install Python 3 and `promtool` from a [Prometheus release](https://github.com/prometheus/prometheus/releases), then run: ```sh @@ -132,10 +204,12 @@ make test This checks Jsonnet formatting, validates generated rules with `promtool`, runs `pint --offline lint`, and evaluates the rules against synthetic time series. -Tests cover all seven recordings, alert timing and recovery, threshold -boundaries, counter resets, disk forecasts, target-label preservation, and -custom selectors/thresholds with Active Directory enabled. CI uses Prometheus -3.15.0 and the same pinned tool versions as `make tools`. +Tests cover all 25 recordings, alert timing and recovery, threshold +boundaries, counter resets, zero denominators, disk forecasts, target-label +preservation, and custom selectors/thresholds with Active Directory and clock +alerts enabled. Runbook checks include opt-in alerts and verify that their pages +and URLs exist. CI uses Prometheus 3.15.0 and the same pinned tool versions as +`make tools`. If you prefer to run `promtool` through Docker: @@ -143,10 +217,15 @@ If you prefer to run `promtool` through Docker: make test PROMTOOL="docker run --rm --entrypoint promtool -v \"$PWD:/mixin:ro\" -w /mixin prom/prometheus:v3.15.0" ``` -`.pint.hcl` requires a non-empty severity label and summary/description -annotations on alerts, in addition to pint's built-in checks. CI uses offline -mode without a Prometheus endpoint. To validate metric availability and labels -against your own server, copy `.pint.hcl` to a local configuration, add: +The dedicated `Mixin` workflow runs on changes to `contrib/mixin/**` or its own +workflow file, and can also be started manually. It caches pinned tool binaries +and runs the same `make test` checks used locally. + +`.pint.hcl` requires a non-empty severity label, summary/description annotations, +and an HTTP(S) runbook URL on alerts, in addition to pint's built-in checks. +CI uses offline mode without a Prometheus endpoint. To validate metric +availability and labels against your own server, copy `.pint.hcl` to a local +configuration, add: ```hcl prometheus "local" { diff --git a/contrib/mixin/alerts/alerts.libsonnet b/contrib/mixin/alerts/alerts.libsonnet index 288a46d6..0da169d2 100644 --- a/contrib/mixin/alerts/alerts.libsonnet +++ b/contrib/mixin/alerts/alerts.libsonnet @@ -45,6 +45,41 @@ 'Windows memory usage is high', 'Physical memory usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.memoryHighUsageThreshold ), + alert( + 'WindowsMemoryCommitHighUsage', + q.metric('windows:memory_committed:ratio') + ' * 100 > %s' % c.memoryCommitHighUsageThreshold, + '15m', + 'warning', + 'Windows memory commit usage is high', + 'Committed memory on {{ $labels.instance }} has exceeded %s%% of the commit limit for 15 minutes.' % c.memoryCommitHighUsageThreshold + ), + ] + [ + alert( + 'WindowsDisk%sLatencyHigh' % direction[0], + q.metric('windows:logical_disk_%s_latency:seconds' % direction[1], [c.volumeSelector]) + ' > %g' % c.diskLatencyThresholdSeconds, + '15m', + 'warning', + 'Windows disk %s latency is high' % direction[1], + 'Volume {{ $labels.volume }} on {{ $labels.instance }} has averaged more than %g seconds per %s operation for 15 minutes.' % [c.diskLatencyThresholdSeconds, direction[1]] + ) + for direction in [['Read', 'read'], ['Write', 'write']] + ] + [ + alert( + 'WindowsNetwork%s' % outcome[0], + '(%s + %s) > %g' % [ + q.metric('windows:net_received_%s:rate' % outcome[1], [c.nicSelector]), + q.metric('windows:net_outbound_%s:rate' % outcome[1], [c.nicSelector]), + outcome[2], + ], + '15m', + 'warning', + 'Windows network packet %s are high' % outcome[3], + 'Packet %s on interface {{ $labels.nic }} on {{ $labels.instance }} have exceeded %g per second for 15 minutes.' % [outcome[3], outcome[2]] + ) + for outcome in [ + ['Errors', 'errors', c.networkErrorRateThreshold, 'errors'], + ['Discards', 'discarded', c.networkDiscardRateThreshold, 'discards'], + ] ] + [ alert( 'WindowsDiskAlmostFull', @@ -74,25 +109,47 @@ ), ], }, - ] + if c.enableActiveDirectory then [ + ] + (if c.enableActiveDirectory then [ + { + name: 'windows-active-directory', + rules: [ + alert( + 'WindowsADPendingReplication', + q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold, + '15m', + 'warning', + 'Active Directory replication queue is high', + 'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold + ), + alert( + 'WindowsADReplicationFailures', + '(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold, + '15m', + 'warning', + 'Active Directory replication requests are failing', + 'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.' + ), + ], + }, + ] else []) + if c.enableTime then [ { - name: 'windows-active-directory', + name: 'windows-time', rules: [ alert( - 'WindowsADPendingReplication', - q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold, - '15m', + 'WindowsClockNotSynchronising', + q.metric('windows_time_ntp_client_time_sources') + ' == 0', + '10m', 'warning', - 'Active Directory replication queue is high', - 'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold + 'Windows clock has no NTP time source', + 'Windows Time on {{ $labels.instance }} has had no active NTP time source for 10 minutes.' ), alert( - 'WindowsADReplicationFailures', - '(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold, - '15m', + 'WindowsClockSkewDetected', + 'abs(%s) > %g' % [q.metric('windows_time_computed_time_offset_seconds'), c.clockOffsetThresholdSeconds], + '10m', 'warning', - 'Active Directory replication requests are failing', - 'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.' + 'Windows clock offset is high', + 'Windows Time on {{ $labels.instance }} reports a clock offset above %g seconds for 10 minutes.' % c.clockOffsetThresholdSeconds ), ], }, diff --git a/contrib/mixin/config.libsonnet b/contrib/mixin/config.libsonnet index 21addf80..ee45f633 100644 --- a/contrib/mixin/config.libsonnet +++ b/contrib/mixin/config.libsonnet @@ -8,9 +8,18 @@ // Opt in only for targets with the ad collector enabled. enableActiveDirectory: false, + // Requires the time collector with its ntp subcollector enabled. + enableTime: false, + + runbookURLPattern: 'https://prometheus-community.github.io/windows_exporter/runbooks/%s/', cpuHighUsageThreshold: 90, memoryHighUsageThreshold: 90, + memoryCommitHighUsageThreshold: 90, + diskLatencyThresholdSeconds: 0.05, + networkErrorRateThreshold: 1, + networkDiscardRateThreshold: 1, + clockOffsetThresholdSeconds: 0.05, diskFreeWarningThreshold: 10, diskFreeCriticalThreshold: 5, diskPredictionWindow: '6h', diff --git a/contrib/mixin/lib/add-runbook-links.libsonnet b/contrib/mixin/lib/add-runbook-links.libsonnet new file mode 100644 index 00000000..29c42937 --- /dev/null +++ b/contrib/mixin/lib/add-runbook-links.libsonnet @@ -0,0 +1,19 @@ +{ + prometheusAlerts+:: { + groups: [ + group { + rules: [ + if std.objectHas(rule, 'alert') && !(std.objectHas(rule, 'annotations') && std.objectHas(rule.annotations, 'runbook_url')) then + rule { + annotations+: { + runbook_url: $._config.runbookURLPattern % std.asciiLower(rule.alert), + }, + } + else rule + for rule in group.rules + ], + } + for group in super.groups + ], + }, +} diff --git a/contrib/mixin/lib/queries.libsonnet b/contrib/mixin/lib/queries.libsonnet index 30d18942..15bc0418 100644 --- a/contrib/mixin/lib/queries.libsonnet +++ b/contrib/mixin/lib/queries.libsonnet @@ -3,16 +3,30 @@ function(c) selector(extra=[]): std.join(', ', [s for s in [c.windowsExporterSelector] + extra if s != '']), metric(name, extra=[]): '%s{%s}' % [name, self.selector(extra)], rate(name, extra=[]): 'rate(%s[%s])' % [self.metric(name, extra), c.rateInterval], + // Drop undefined ratios instead of recording NaN or +Inf for idle devices. + ratio(numerator, denominator): '(%s / (%s > 0))' % [numerator, denominator], cpuUsage: '1 - avg without (core, mode) (%s)' % self.rate('windows_cpu_time_total', ['mode="idle"']), memoryUsage: - '1 - %s / %s' % [ + '1 - %s' % self.ratio( self.metric('windows_memory_available_bytes'), - self.metric('windows_memory_physical_total_bytes'), - ], + self.metric('windows_memory_physical_total_bytes') + ), diskFree: - '%s / %s' % [ + self.ratio( self.metric('windows_logical_disk_free_bytes', [c.volumeSelector]), - self.metric('windows_logical_disk_size_bytes', [c.volumeSelector]), - ], + self.metric('windows_logical_disk_size_bytes', [c.volumeSelector]) + ), + memoryCommitted: + self.ratio(self.metric('windows_memory_committed_bytes'), self.metric('windows_memory_commit_limit')), + diskLatency(direction): + self.ratio( + self.rate('windows_logical_disk_%s_seconds_total' % direction, [c.volumeSelector]), + self.rate('windows_logical_disk_%ss_total' % direction, [c.volumeSelector]) + ), + networkUtilization(direction): + self.ratio( + self.rate('windows_net_bytes_%s_total' % direction, [c.nicSelector]), + self.metric('windows_net_current_bandwidth_bytes', [c.nicSelector]) + ), } diff --git a/contrib/mixin/mixin.libsonnet b/contrib/mixin/mixin.libsonnet index 9a95d4b6..e92e6a38 100644 --- a/contrib/mixin/mixin.libsonnet +++ b/contrib/mixin/mixin.libsonnet @@ -1,3 +1,4 @@ (import 'config.libsonnet') + (import 'rules/rules.libsonnet') + -(import 'alerts/alerts.libsonnet') +(import 'alerts/alerts.libsonnet') + +(import 'lib/add-runbook-links.libsonnet') diff --git a/contrib/mixin/rules/rules.libsonnet b/contrib/mixin/rules/rules.libsonnet index b364b067..eb53befa 100644 --- a/contrib/mixin/rules/rules.libsonnet +++ b/contrib/mixin/rules/rules.libsonnet @@ -7,20 +7,65 @@ name: 'windows-exporter-recording', rules: [ { record: 'windows:cpu_usage:ratio', expr: q.cpuUsage }, + { record: 'windows:cpu_cores:count', expr: 'count without (core, mode) (%s)' % q.metric('windows_cpu_time_total', ['mode="idle"']) }, + { record: 'windows:cpu_time:ratio_rate', expr: 'avg without (core) (%s)' % q.rate('windows_cpu_time_total') }, { record: 'windows:memory_usage:ratio', expr: q.memoryUsage }, + { + record: 'windows:memory_used:bytes', + expr: '%s - %s' % [q.metric('windows_memory_physical_total_bytes'), q.metric('windows_memory_available_bytes')], + }, + { + record: 'windows:memory_cached:bytes', + expr: std.join(' + ', [ + q.metric('windows_memory_%s_bytes' % component) + for component in ['cache', 'modified_page_list', 'standby_cache_core', 'standby_cache_normal_priority', 'standby_cache_reserve'] + ]), + }, + { record: 'windows:memory_committed:ratio', expr: q.memoryCommitted }, + { record: 'windows:memory_swap_page_operations:rate', expr: q.rate('windows_memory_swap_page_operations_total') }, { record: 'windows:logical_disk_free:ratio', expr: q.diskFree }, + { + record: 'windows:logical_disk_used:bytes', + expr: '%s - %s' % [ + q.metric('windows_logical_disk_size_bytes', [c.volumeSelector]), + q.metric('windows_logical_disk_free_bytes', [c.volumeSelector]), + ], + }, + { + record: 'windows:logical_disk_busy:ratio', + expr: 'clamp(1 - %s, 0, 1)' % q.rate('windows_logical_disk_idle_seconds_total', [c.volumeSelector]), + }, ] + [ { record: 'windows:logical_disk_%s_bytes:rate' % direction, expr: q.rate('windows_logical_disk_%s_bytes_total' % direction, [c.volumeSelector]), } for direction in ['read', 'write'] + ] + [ + { + record: 'windows:logical_disk_%s_operations:rate' % direction, + expr: q.rate('windows_logical_disk_%ss_total' % direction, [c.volumeSelector]), + } + for direction in ['read', 'write'] + ] + [ + { record: 'windows:logical_disk_%s_latency:seconds' % direction, expr: q.diskLatency(direction) } + for direction in ['read', 'write'] ] + [ { record: 'windows:net_%s_bytes:rate' % direction, expr: q.rate('windows_net_bytes_%s_total' % direction, [c.nicSelector]), } for direction in ['received', 'sent'] + ] + [ + { record: 'windows:net_%s_utilization:ratio' % direction, expr: q.networkUtilization(direction) } + for direction in ['received', 'sent'] + ] + [ + { + record: 'windows:net_%s_%s:rate' % [direction, outcome], + expr: q.rate('windows_net_packets_%s_%s_total' % [direction, outcome], [c.nicSelector]), + } + for direction in ['received', 'outbound'] + for outcome in ['errors', 'discarded'] ], }, ], diff --git a/contrib/mixin/tests/alerts.test.yaml b/contrib/mixin/tests/alerts.test.yaml index 7e775bed..9e356867 100644 --- a/contrib/mixin/tests/alerts.test.yaml +++ b/contrib/mixin/tests/alerts.test.yaml @@ -7,6 +7,116 @@ group_eval_order: - windows-exporter-alerts tests: + - name: Commit pressure, disk latency and network failures wait and recover + interval: 1m + input_series: + - series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '95x20 50x10' + - series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '100x31' + - series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '0+12x20 252x10' + - series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '0+120x31' + - series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '0+24x20 504x10' + - series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '0+240x31' + - series: 'windows_net_packets_received_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0+120x20 2520x10' + - series: 'windows_net_packets_outbound_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0x31' + - series: 'windows_net_packets_received_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0x31' + - series: 'windows_net_packets_outbound_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0+120x20 2520x10' + alert_rule_test: + - {alertname: WindowsMemoryCommitHighUsage, eval_time: 14m, exp_alerts: []} + - alertname: WindowsMemoryCommitHighUsage + eval_time: 15m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning} + exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsmemorycommithighusage/ + summary: Windows memory commit usage is high + description: 'Committed memory on host:9182 has exceeded 90% of the commit limit for 15 minutes.' + - {alertname: WindowsMemoryCommitHighUsage, eval_time: 21m, exp_alerts: []} + - {alertname: WindowsDiskReadLatencyHigh, eval_time: 15m, exp_alerts: []} + - alertname: WindowsDiskReadLatencyHigh + eval_time: 16m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning} + exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskreadlatencyhigh/ + summary: Windows disk read latency is high + description: 'Volume C: on host:9182 has averaged more than 0.05 seconds per read operation for 15 minutes.' + - {alertname: WindowsDiskReadLatencyHigh, eval_time: 25m, exp_alerts: []} + - {alertname: WindowsDiskWriteLatencyHigh, eval_time: 15m, exp_alerts: []} + - alertname: WindowsDiskWriteLatencyHigh + eval_time: 16m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning} + exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskwritelatencyhigh/ + summary: Windows disk write latency is high + description: 'Volume C: on host:9182 has averaged more than 0.05 seconds per write operation for 15 minutes.' + - {alertname: WindowsDiskWriteLatencyHigh, eval_time: 25m, exp_alerts: []} + - {alertname: WindowsNetworkErrors, eval_time: 17m, exp_alerts: []} + - alertname: WindowsNetworkErrors + eval_time: 18m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, nic: Ethernet, severity: warning} + exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsnetworkerrors/ + summary: Windows network packet errors are high + description: 'Packet errors on interface Ethernet on host:9182 have exceeded 1 per second for 15 minutes.' + - {alertname: WindowsNetworkErrors, eval_time: 25m, exp_alerts: []} + - {alertname: WindowsNetworkDiscards, eval_time: 17m, exp_alerts: []} + - alertname: WindowsNetworkDiscards + eval_time: 18m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, nic: Ethernet, severity: warning} + exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsnetworkdiscards/ + summary: Windows network packet discards are high + description: 'Packet discards on interface Ethernet on host:9182 have exceeded 1 per second for 15 minutes.' + - {alertname: WindowsNetworkDiscards, eval_time: 25m, exp_alerts: []} + + - name: New resource alerts stay quiet at exact thresholds and clock alerts are disabled + interval: 1m + input_series: + - series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182"}' + values: '90x20' + - series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182"}' + values: '100x20' + - series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0+6x20' + - series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0+120x20' + - series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0+6x20' + - series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0+120x20' + - series: 'windows_net_packets_received_errors_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + values: '0+30x20' + - series: 'windows_net_packets_outbound_errors_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + values: '0+30x20' + - series: 'windows_net_packets_received_discarded_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + values: '0+30x20' + - series: 'windows_net_packets_outbound_discarded_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + values: '0+30x20' + - series: 'windows_time_ntp_client_time_sources{job="windows_exporter", instance="host:9182"}' + values: '0x20' + - series: 'windows_time_computed_time_offset_seconds{job="windows_exporter", instance="host:9182"}' + values: '1x20' + alert_rule_test: + - {alertname: WindowsMemoryCommitHighUsage, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsDiskReadLatencyHigh, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsDiskWriteLatencyHigh, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsNetworkErrors, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsNetworkDiscards, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsClockNotSynchronising, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsClockSkewDetected, eval_time: 20m, exp_alerts: []} - name: Exporter and collector failures wait five minutes and recover interval: 1m input_series: @@ -25,6 +135,7 @@ tests: exp_alerts: - exp_labels: {job: windows_exporter, instance: 'host:9182', severity: critical} exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsexporterdown/ summary: Windows exporter is unreachable description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows_exporter).' - alertname: WindowsExporterDown @@ -38,6 +149,7 @@ tests: exp_alerts: - exp_labels: {job: windows_exporter, instance: 'host:9182', collector: cpu, severity: warning} exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowscollectorfailed/ summary: Windows collector failed description: 'Collector cpu on host:9182 has failed for at least 5 minutes.' - alertname: WindowsCollectorFailed @@ -62,6 +174,7 @@ tests: exp_alerts: - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning} exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowscpuhighusage/ summary: Windows CPU usage is high description: 'CPU usage on host:9182 has exceeded 90% for 15 minutes.' - alertname: WindowsCPUHighUsage @@ -75,6 +188,7 @@ tests: exp_alerts: - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning} exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsmemoryhighusage/ summary: Windows memory usage is high description: 'Physical memory usage on host:9182 has exceeded 90% for 15 minutes.' - alertname: WindowsMemoryHighUsage @@ -97,10 +211,12 @@ tests: exp_alerts: - exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: warning} exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskalmostfull/ summary: Windows disk space is low description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.' - exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: critical} exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskalmostfull/ summary: Windows disk space is low description: 'Volume C: on host:9182 has less than 5% free space. Windows may delay space metrics by 10–15 minutes.' - alertname: WindowsDiskAlmostFull @@ -153,6 +269,7 @@ tests: exp_alerts: - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning} exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskfillingup/ summary: Windows disk is predicted to fill up description: 'Volume C: on host:9182 is predicted to fill within 24 hours.' - alertname: WindowsDiskFillingUp @@ -160,6 +277,7 @@ tests: exp_alerts: - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning} exp_annotations: + runbook_url: https://prometheus-community.github.io/windows_exporter/runbooks/windowsdiskfillingup/ summary: Windows disk is predicted to fill up description: 'Volume C: on host:9182 is predicted to fill within 24 hours.' diff --git a/contrib/mixin/tests/check_runbooks.py b/contrib/mixin/tests/check_runbooks.py new file mode 100644 index 00000000..4a63a13a --- /dev/null +++ b/contrib/mixin/tests/check_runbooks.py @@ -0,0 +1,36 @@ +"""Check every mixin alert has a complete runbook at its default URL.""" + +import json +from pathlib import Path +import sys +from urllib.parse import urlparse + + +runbooks = Path(sys.argv[1]) +alerts = json.load(sys.stdin) +errors = [] +names = set() +for group in alerts["groups"]: + for rule in group["rules"]: + if "alert" not in rule: + continue + name = rule["alert"] + names.add(name) + slug = name.lower() + expected_path = f"/windows_exporter/runbooks/{slug}/" + if urlparse(rule["annotations"].get("runbook_url", "")).path != expected_path: + errors.append(f"{name}: runbook URL must end in {expected_path}") + path = runbooks / f"{slug}.md" + if not path.is_file(): + errors.append(f"{name}: missing {path}") + continue + content = path.read_text(encoding="utf-8") + for section in ("Meaning", "Impact", "Diagnosis", "Mitigation"): + if f"## {section}\n" not in content: + errors.append(f"{name}: missing {section} section in {path}") + if "TODO" in content: + errors.append(f"{name}: unfinished runbook {path}") + +if errors: + sys.exit("\n".join(errors)) +print(f"Checked runbooks and URLs for {len(names)} alerts") diff --git a/contrib/mixin/tests/config.libsonnet b/contrib/mixin/tests/config.libsonnet index b67781f2..fc9837e3 100644 --- a/contrib/mixin/tests/config.libsonnet +++ b/contrib/mixin/tests/config.libsonnet @@ -10,6 +10,13 @@ diskFreeCriticalThreshold: 10, criticalSeverity: 'page', enableActiveDirectory: true, + enableTime: true, + runbookURLPattern: 'https://runbooks.example.com/windows/%s/', + memoryCommitHighUsageThreshold: 80, + diskLatencyThresholdSeconds: 0.02, + networkErrorRateThreshold: 0.5, + networkDiscardRateThreshold: 0.5, + clockOffsetThresholdSeconds: 0.1, adPendingReplicationThreshold: 10, adReplicationFailureRateThreshold: 0.5, }, diff --git a/contrib/mixin/tests/custom.test.yaml b/contrib/mixin/tests/custom.test.yaml index 5e4dddba..1b16b5e6 100644 --- a/contrib/mixin/tests/custom.test.yaml +++ b/contrib/mixin/tests/custom.test.yaml @@ -6,8 +6,130 @@ group_eval_order: - windows-exporter-recording - windows-exporter-alerts - windows-active-directory + - windows-time tests: + - name: Custom commit, latency and network thresholds preserve selectors and device filters + interval: 1m + input_series: + - series: 'windows_memory_committed_bytes{job="windows-custom", instance="host:9182", cluster="test"}' + values: '85x20' + - series: 'windows_memory_commit_limit{job="windows-custom", instance="host:9182", cluster="test"}' + values: '100x20' + - series: 'windows_memory_committed_bytes{job="windows-custom", instance="host:9182", cluster="prod"}' + values: '99x20' + - series: 'windows_memory_commit_limit{job="windows-custom", instance="host:9182", cluster="prod"}' + values: '100x20' + - series: 'windows_logical_disk_read_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}' + values: '0+3.6x20' + - series: 'windows_logical_disk_reads_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}' + values: '0+120x20' + - series: 'windows_logical_disk_write_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}' + values: '0+3.6x20' + - series: 'windows_logical_disk_writes_total{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}' + values: '0+120x20' + - series: 'windows_logical_disk_read_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}' + values: '0+12x20' + - series: 'windows_logical_disk_reads_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}' + values: '0+120x20' + - series: 'windows_logical_disk_write_seconds_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}' + values: '0+12x20' + - series: 'windows_logical_disk_writes_total{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}' + values: '0+120x20' + - series: 'windows_net_packets_received_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}' + values: '0+36x20' + - series: 'windows_net_packets_outbound_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}' + values: '0x20' + - series: 'windows_net_packets_received_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}' + values: '0x20' + - series: 'windows_net_packets_outbound_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}' + values: '0+36x20' + - series: 'windows_net_packets_received_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}' + values: '0+120x20' + - series: 'windows_net_packets_outbound_errors_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}' + values: '0+120x20' + - series: 'windows_net_packets_received_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}' + values: '0+120x20' + - series: 'windows_net_packets_outbound_discarded_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}' + values: '0+120x20' + alert_rule_test: + - alertname: WindowsMemoryCommitHighUsage + eval_time: 20m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} + exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsmemorycommithighusage/ + summary: Windows memory commit usage is high + description: 'Committed memory on host:9182 has exceeded 80% of the commit limit for 15 minutes.' + - alertname: WindowsDiskReadLatencyHigh + eval_time: 20m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning} + exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsdiskreadlatencyhigh/ + summary: Windows disk read latency is high + description: 'Volume C: on host:9182 has averaged more than 0.02 seconds per read operation for 15 minutes.' + - alertname: WindowsDiskWriteLatencyHigh + eval_time: 20m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning} + exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsdiskwritelatencyhigh/ + summary: Windows disk write latency is high + description: 'Volume C: on host:9182 has averaged more than 0.02 seconds per write operation for 15 minutes.' + - alertname: WindowsNetworkErrors + eval_time: 20m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, nic: Ethernet, severity: warning} + exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsnetworkerrors/ + summary: Windows network packet errors are high + description: 'Packet errors on interface Ethernet on host:9182 have exceeded 0.5 per second for 15 minutes.' + - alertname: WindowsNetworkDiscards + eval_time: 20m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, nic: Ethernet, severity: warning} + exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsnetworkdiscards/ + summary: Windows network packet discards are high + description: 'Packet discards on interface Ethernet on host:9182 have exceeded 0.5 per second for 15 minutes.' + + - name: Clock alerts opt in, use absolute offset, custom URLs and recover + interval: 1m + input_series: + - series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="host:9182", cluster="test"}' + values: '0x10 1x10' + - series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="host:9182", cluster="test"}' + values: '-0.2x10 0x10' + - series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="healthy:9182", cluster="test"}' + values: '1x21' + - series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="healthy:9182", cluster="test"}' + values: '0.1x21' + - series: 'windows_time_ntp_client_time_sources{job="windows-custom", instance="host:9182", cluster="prod"}' + values: '0x21' + - series: 'windows_time_computed_time_offset_seconds{job="windows-custom", instance="host:9182", cluster="prod"}' + values: '-1x21' + alert_rule_test: + - {alertname: WindowsClockNotSynchronising, eval_time: 9m, exp_alerts: []} + - alertname: WindowsClockNotSynchronising + eval_time: 10m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} + exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsclocknotsynchronising/ + summary: Windows clock has no NTP time source + description: 'Windows Time on host:9182 has had no active NTP time source for 10 minutes.' + - {alertname: WindowsClockNotSynchronising, eval_time: 11m, exp_alerts: []} + - {alertname: WindowsClockSkewDetected, eval_time: 9m, exp_alerts: []} + - alertname: WindowsClockSkewDetected + eval_time: 10m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} + exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsclockskewdetected/ + summary: Windows clock offset is high + description: 'Windows Time on host:9182 reports a clock offset above 0.1 seconds for 10 minutes.' + - {alertname: WindowsClockSkewDetected, eval_time: 11m, exp_alerts: []} - name: Custom selectors, thresholds and critical severity propagate to alerts interval: 1m input_series: @@ -50,6 +172,7 @@ tests: exp_alerts: - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: page} exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsexporterdown/ summary: Windows exporter is unreachable description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows-custom).' - alertname: WindowsCPUHighUsage @@ -57,6 +180,7 @@ tests: exp_alerts: - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowscpuhighusage/ summary: Windows CPU usage is high description: 'CPU usage on host:9182 has exceeded 80% for 15 minutes.' - alertname: WindowsMemoryHighUsage @@ -64,6 +188,7 @@ tests: exp_alerts: - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsmemoryhighusage/ summary: Windows memory usage is high description: 'Physical memory usage on host:9182 has exceeded 85% for 15 minutes.' - alertname: WindowsDiskAlmostFull @@ -71,10 +196,12 @@ tests: exp_alerts: - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning} exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsdiskalmostfull/ summary: Windows disk space is low description: 'Volume C: on host:9182 has less than 20% free space. Windows may delay space metrics by 10–15 minutes.' - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: page} exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsdiskalmostfull/ summary: Windows disk space is low description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.' @@ -108,6 +235,7 @@ tests: exp_alerts: - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsadpendingreplication/ summary: Active Directory replication queue is high description: 'Pending replication operations on host:9182 have exceeded 10 for 15 minutes.' - alertname: WindowsADPendingReplication @@ -121,6 +249,7 @@ tests: exp_alerts: - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} exp_annotations: + runbook_url: https://runbooks.example.com/windows/windowsadreplicationfailures/ summary: Active Directory replication requests are failing description: 'Replication synchronization requests on host:9182 have been failing for 15 minutes.' - alertname: WindowsADReplicationFailures diff --git a/contrib/mixin/tests/resources.test.yaml b/contrib/mixin/tests/resources.test.yaml new file mode 100644 index 00000000..53472952 --- /dev/null +++ b/contrib/mixin/tests/resources.test.yaml @@ -0,0 +1,183 @@ +rule_files: + - ../windows_rules.yaml + - ../windows_alerts.yaml +evaluation_interval: 1m +group_eval_order: + - windows-exporter-recording + - windows-exporter-alerts + +tests: + - name: Memory cache includes all lists, commit is a ratio and paging is a rate + interval: 1m + input_series: + - series: 'windows_memory_cache_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '10x10' + - series: 'windows_memory_modified_page_list_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '20x10' + - series: 'windows_memory_standby_cache_core_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '30x10' + - series: 'windows_memory_standby_cache_normal_priority_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '40x10' + - series: 'windows_memory_standby_cache_reserve_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '50x10' + - series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '80x10' + - series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '100x10' + - series: 'windows_memory_swap_page_operations_total{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '0+120x10' + - series: 'windows_memory_committed_bytes{job="other", instance="host:9182"}' + values: '95x10' + - series: 'windows_memory_commit_limit{job="other", instance="host:9182"}' + values: '100x10' + promql_expr_test: + - expr: windows:memory_cached:bytes + eval_time: 10m + exp_samples: + - labels: 'windows:memory_cached:bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + value: 150 + - expr: windows:memory_committed:ratio + eval_time: 10m + exp_samples: + - labels: 'windows:memory_committed:ratio{job="windows_exporter", instance="host:9182", cluster="a"}' + value: 0.8 + - expr: windows:memory_swap_page_operations:rate + eval_time: 10m + exp_samples: + - labels: 'windows:memory_swap_page_operations:rate{job="windows_exporter", instance="host:9182", cluster="a"}' + value: 2 + + - name: Disk latency uses operations, handles resets and busy time stays bounded + interval: 1m + input_series: + - series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '0+12x5 12+12x4' + - series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '0+120x5 120+120x4' + - series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '0+24x10' + - series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '0+240x10' + - series: 'windows_logical_disk_idle_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '0+30x10' + - series: 'windows_logical_disk_idle_seconds_total{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}' + values: '0+90x10' + promql_expr_test: + - expr: windows:logical_disk_read_operations:rate + eval_time: 8m + exp_samples: + - labels: 'windows:logical_disk_read_operations:rate{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + value: 2 + - expr: windows:logical_disk_write_operations:rate + eval_time: 10m + exp_samples: + - labels: 'windows:logical_disk_write_operations:rate{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + value: 4 + - expr: windows:logical_disk_read_latency:seconds + eval_time: 8m + exp_samples: + - labels: 'windows:logical_disk_read_latency:seconds{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + value: 0.1 + - expr: windows:logical_disk_write_latency:seconds + eval_time: 10m + exp_samples: + - labels: 'windows:logical_disk_write_latency:seconds{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + value: 0.1 + - expr: windows:logical_disk_busy:ratio + eval_time: 10m + exp_samples: + - labels: 'windows:logical_disk_busy:ratio{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + value: 0.5 + - labels: 'windows:logical_disk_busy:ratio{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}' + value: 0 + + - name: Full duplex network utilization uses bytes and keeps directions separate + interval: 1m + input_series: + - series: 'windows_net_bytes_received_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0+600x10' + - series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0+1200x10' + - series: 'windows_net_current_bandwidth_bytes{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '100x10' + - series: 'windows_net_packets_received_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0+60x10' + - series: 'windows_net_packets_outbound_errors_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0+120x10' + - series: 'windows_net_packets_received_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0+180x10' + - series: 'windows_net_packets_outbound_discarded_total{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + values: '0+240x10' + promql_expr_test: + - expr: windows:net_received_utilization:ratio + eval_time: 10m + exp_samples: + - labels: 'windows:net_received_utilization:ratio{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + value: 0.1 + - expr: windows:net_sent_utilization:ratio + eval_time: 10m + exp_samples: + - labels: 'windows:net_sent_utilization:ratio{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + value: 0.2 + - expr: windows:net_received_errors:rate + eval_time: 10m + exp_samples: + - labels: 'windows:net_received_errors:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + value: 1 + - expr: windows:net_outbound_errors:rate + eval_time: 10m + exp_samples: + - labels: 'windows:net_outbound_errors:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + value: 2 + - expr: windows:net_received_discarded:rate + eval_time: 10m + exp_samples: + - labels: 'windows:net_received_discarded:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + value: 3 + - expr: windows:net_outbound_discarded:rate + eval_time: 10m + exp_samples: + - labels: 'windows:net_outbound_discarded:rate{job="windows_exporter", instance="host:9182", cluster="a", nic="Ethernet"}' + value: 4 + + - name: Zero capacity and idle devices produce no undefined ratios or latency alerts + interval: 1m + input_series: + - series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182"}' + values: '1x20' + - series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182"}' + values: '0x20' + - series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '1x20' + - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0x20' + - series: 'windows_memory_committed_bytes{job="windows_exporter", instance="host:9182"}' + values: '1x20' + - series: 'windows_memory_commit_limit{job="windows_exporter", instance="host:9182"}' + values: '0x20' + - series: 'windows_logical_disk_read_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0x20' + - series: 'windows_logical_disk_reads_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0x20' + - series: 'windows_logical_disk_write_seconds_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0x20' + - series: 'windows_logical_disk_writes_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0x20' + - series: 'windows_net_bytes_received_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + values: '0+600x20' + - series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + values: '0+600x20' + - series: 'windows_net_current_bandwidth_bytes{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + values: '0x20' + promql_expr_test: + - {expr: 'windows:memory_usage:ratio', eval_time: 20m, exp_samples: []} + - {expr: 'windows:logical_disk_free:ratio', eval_time: 20m, exp_samples: []} + - {expr: 'windows:memory_committed:ratio', eval_time: 20m, exp_samples: []} + - {expr: 'windows:logical_disk_read_latency:seconds', eval_time: 20m, exp_samples: []} + - {expr: 'windows:logical_disk_write_latency:seconds', eval_time: 20m, exp_samples: []} + - {expr: 'windows:net_received_utilization:ratio', eval_time: 20m, exp_samples: []} + - {expr: 'windows:net_sent_utilization:ratio', eval_time: 20m, exp_samples: []} + alert_rule_test: + - {alertname: WindowsMemoryCommitHighUsage, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsDiskReadLatencyHigh, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsDiskWriteLatencyHigh, eval_time: 20m, exp_alerts: []} diff --git a/contrib/mixin/tests/rules.test.yaml b/contrib/mixin/tests/rules.test.yaml index 81b5ecd9..e5a35831 100644 --- a/contrib/mixin/tests/rules.test.yaml +++ b/contrib/mixin/tests/rules.test.yaml @@ -18,6 +18,20 @@ tests: - series: 'windows_cpu_time_total{job="other", instance="host:9182", core="0", mode="idle"}' values: '0+6x10' promql_expr_test: + - expr: windows:cpu_cores:count + eval_time: 10m + exp_samples: + - labels: 'windows:cpu_cores:count{job="windows_exporter", instance="host:9182", cluster="a"}' + value: 2 + - labels: 'windows:cpu_cores:count{job="windows_exporter", instance="host:9182", cluster="b"}' + value: 1 + - expr: 'windows:cpu_time:ratio_rate{mode="idle"}' + eval_time: 10m + exp_samples: + - labels: 'windows:cpu_time:ratio_rate{job="windows_exporter", instance="host:9182", cluster="a", mode="idle"}' + value: 0.4 + - labels: 'windows:cpu_time:ratio_rate{job="windows_exporter", instance="host:9182", cluster="b", mode="idle"}' + value: 0.5 - expr: windows:cpu_usage:ratio eval_time: 10m exp_samples: @@ -42,6 +56,16 @@ tests: - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume=""}' values: '100x5' promql_expr_test: + - expr: windows:memory_used:bytes + eval_time: 5m + exp_samples: + - labels: 'windows:memory_used:bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + value: 75 + - expr: windows:logical_disk_used:bytes + eval_time: 5m + exp_samples: + - labels: 'windows:logical_disk_used:bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + value: 80 - expr: windows:memory_usage:ratio eval_time: 5m exp_samples: diff --git a/contrib/mixin/tests/runbooks.test.jsonnet b/contrib/mixin/tests/runbooks.test.jsonnet new file mode 100644 index 00000000..388b2623 --- /dev/null +++ b/contrib/mixin/tests/runbooks.test.jsonnet @@ -0,0 +1,23 @@ +local linked = { + _config:: { runbookURLPattern: 'https://example.com/runbooks/%s/' }, + prometheusAlerts:: { + groups: [ + { + name: 'custom', + rules: [ + { alert: 'WindowsCustomAlert', expr: 'up == 0', annotations: { summary: 'Existing summary' } }, + { alert: 'WindowsOverride', expr: 'up == 0', annotations: { runbook_url: 'https://example.com/override' } }, + { record: 'windows:custom:sum', expr: 'sum(up)' }, + { alert: 'WindowsNoAnnotations', expr: 'up == 0' }, + ], + }, + ], + }, +} + (import '../lib/add-runbook-links.libsonnet'); +local rules = linked.prometheusAlerts.groups[0].rules; +assert rules[0].annotations == { summary: 'Existing summary', runbook_url: 'https://example.com/runbooks/windowscustomalert/' }; +assert rules[1].annotations.runbook_url == 'https://example.com/override'; +assert !std.objectHas(rules[2], 'annotations'); +assert rules[3].annotations.runbook_url == 'https://example.com/runbooks/windowsnoannotations/'; +assert rules[0].expr == 'up == 0' && linked.prometheusAlerts.groups[0].name == 'custom'; +{ result: 'Runbook link composition passed' } diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..eab55162 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +/public/ +/resources/ +/.hugo_build.lock diff --git a/docs/README.md b/docs/README.md index 4723fe8f..eb57e6f0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,11 @@ # Documentation This directory contains documentation of the collectors in the windows_exporter, with information such as what metrics are exported, any flags for additional configuration, and some example usage in alerts and queries. +The [alert runbooks](runbooks) describe how to diagnose and mitigate alerts from +the [Windows exporter mixin](../contrib/mixin/README.md). Preview the runbook site +from the repository root with `hugo server --source docs`; its dedicated GitHub +Actions workflow publishes default-branch changes to GitHub Pages. + # Collectors - [`ad`](collector.ad.md) - [`adcs`](collector.adcs.md) diff --git a/docs/check_links.py b/docs/check_links.py new file mode 100644 index 00000000..ccd1ed8f --- /dev/null +++ b/docs/check_links.py @@ -0,0 +1,48 @@ +"""Check generated site links while respecting the configured Pages base path.""" + +from html.parser import HTMLParser +from pathlib import Path +import sys +from urllib.parse import unquote, urlsplit + + +class Links(HTMLParser): + def __init__(self): + super().__init__() + self.links = [] + + def handle_starttag(self, tag, attrs): + for name, value in attrs: + if name in ("href", "src") and value: + self.links.append(value) + + +root = Path(sys.argv[1]) +prefix = urlsplit(sys.argv[2]).path.rstrip("/") + "/" +errors = [] +checked = 0 +pages = list(root.rglob("*.html")) +if not pages: + sys.exit(f"No generated HTML pages in {root}") +for page in pages: + parser = Links() + parser.feed(page.read_text(encoding="utf-8")) + for link in parser.links: + url = urlsplit(link) + if url.scheme or url.netloc or not url.path: + continue + if url.path.startswith("/"): + if not url.path.startswith(prefix): + errors.append(f"{page}: link {link} is outside the site base path {prefix}") + continue + target = root / unquote(url.path.removeprefix(prefix)) + else: + target = page.parent / unquote(url.path) + if target.is_dir(): + target /= "index.html" + if not target.is_file(): + errors.append(f"{page}: broken link {link}") + checked += 1 +if errors: + sys.exit("\n".join(errors)) +print(f"Checked {checked} internal links across {len(pages)} HTML pages") diff --git a/docs/hugo.toml b/docs/hugo.toml new file mode 100644 index 00000000..e4df3aeb --- /dev/null +++ b/docs/hugo.toml @@ -0,0 +1,15 @@ +baseURL = "https://prometheus-community.github.io/windows_exporter/" +title = "Windows exporter runbooks" +locale = "en" +disableKinds = ["taxonomy", "term", "RSS", "sitemap"] + +[module] + [[module.mounts]] + source = "runbooks" + target = "content/runbooks" + [[module.mounts]] + source = "layouts" + target = "layouts" + [[module.mounts]] + source = "static" + target = "static" diff --git a/docs/layouts/baseof.html b/docs/layouts/baseof.html new file mode 100644 index 00000000..a02394af --- /dev/null +++ b/docs/layouts/baseof.html @@ -0,0 +1,16 @@ + + + + + + {{ if not .IsHome }}{{ .Title }} | {{ end }}{{ site.Title }} + + + +
+ {{ site.Title }} + GitHub +
+
{{ block "main" . }}{{ end }}
+ + diff --git a/docs/layouts/list.html b/docs/layouts/list.html new file mode 100644 index 00000000..15a1deed --- /dev/null +++ b/docs/layouts/list.html @@ -0,0 +1,13 @@ +{{/* The base template declares the doctype. */}} +{{ define "main" }} +

{{ .Title }}

+{{ .Content }} +{{ if .IsHome }} +

Diagnosis and mitigation for alerts from the Windows exporter mixin.

+{{ end }} +
    + {{ range (site.GetPage "/runbooks").RegularPages.ByTitle }} +
  • {{ .Title }}
    • + {{ end }} +
+{{ end }} diff --git a/docs/layouts/single.html b/docs/layouts/single.html new file mode 100644 index 00000000..731e66ae --- /dev/null +++ b/docs/layouts/single.html @@ -0,0 +1,8 @@ +{{/* The base template declares the doctype. */}} +{{ define "main" }} + +
+

{{ .Title }}

+ {{ .Content }} +
+{{ end }} diff --git a/docs/runbooks/_index.md b/docs/runbooks/_index.md new file mode 100644 index 00000000..200967bc --- /dev/null +++ b/docs/runbooks/_index.md @@ -0,0 +1,12 @@ +--- +title: Windows exporter runbooks +--- + +These runbooks describe alerts from the [Windows exporter mixin](https://github.com/prometheus-community/windows_exporter/tree/master/contrib/mixin). +Use the alert's `job`, `instance`, and any `cluster` or environment labels to find +the affected host. Disk alerts also identify a `volume`; network alerts identify +a `nic`. Adjust example PromQL selectors to match your scrape configuration. + +The default thresholds are starting points. Check the generated alert rules for +your configured thresholds and durations. Active Directory and NTP clock alerts +are opt-in and require their collectors to be enabled. diff --git a/docs/runbooks/windowsadpendingreplication.md b/docs/runbooks/windowsadpendingreplication.md new file mode 100644 index 00000000..856de2f1 --- /dev/null +++ b/docs/runbooks/windowsadpendingreplication.md @@ -0,0 +1,41 @@ +--- +title: WindowsADPendingReplication +--- + +## Meaning + +Pending Active Directory replication operations exceed the configured threshold +for 15 minutes. The default is 50 operations. This alert is generated only with +`enableActiveDirectory: true` and requires the `ad` collector on domain controllers. + +## Impact + +Directory changes may be delayed between domain controllers, causing inconsistent +authentication, group membership, DNS, or policy behavior. + +## Diagnosis + +1. Inspect `windows_ad_replication_pending_operations` and determine whether the + queue is growing or draining. +2. On a domain controller, inspect replication health: + + ```powershell + repadmin /replsummary + repadmin /showrepl + repadmin /queue + dcdiag /test:Replications + ``` + +3. Check the Directory Service event log, partner reachability, DNS resolution, + site links and schedules, and CPU/disk/network pressure. +4. Correlate the queue with [WindowsADReplicationFailures](../windowsadreplicationfailures/) + and planned large directory changes. + +## Mitigation + +Restore connectivity and DNS between replication partners. Correct replication +errors and review site-link capacity or schedules with the directory owner. +Relieve resource pressure on domain controllers. Allow expected bulk changes to +drain while monitoring progress. Avoid forcing widespread replication or removing +partners before understanding the failure. Confirm the queue decreases and +replication health checks succeed. diff --git a/docs/runbooks/windowsadreplicationfailures.md b/docs/runbooks/windowsadreplicationfailures.md new file mode 100644 index 00000000..e265a5ca --- /dev/null +++ b/docs/runbooks/windowsadreplicationfailures.md @@ -0,0 +1,40 @@ +--- +title: WindowsADReplicationFailures +--- + +## Meaning + +The rate of replication synchronization requests minus successful requests +exceeds the configured threshold for 15 minutes. The default is any sustained +positive failure rate. This alert is opt-in and requires the `ad` collector. + +## Impact + +Failed replication may leave domain controllers with stale directory data, +affecting authentication, DNS, policies, and directory changes. + +## Diagnosis + +1. Inspect rates of `windows_ad_replication_sync_requests_total` and + `windows_ad_replication_sync_requests_success_total` on the affected controller. + Counter resets around a restart may temporarily distort their difference. +2. Use the Active Directory diagnostic tools to identify partners and error codes: + + ```powershell + repadmin /replsummary + repadmin /showrepl + dcdiag /test:Replications /test:DNS + ``` + +3. Check Directory Service events, DNS resolution, RPC connectivity, replication + permissions, and clock synchronization on both partners. +4. Check for an offline or decommissioned controller and review recent topology, + firewall, or credential changes. + +## Mitigation + +Resolve the specific error reported by replication diagnostics. Restore partner +connectivity, DNS, time synchronization, or permissions as appropriate. Handle +retired domain controllers through the documented Active Directory removal +process. Coordinate topology and recovery changes with the directory owner. +Confirm replication succeeds and the pending queue drains. diff --git a/docs/runbooks/windowsclocknotsynchronising.md b/docs/runbooks/windowsclocknotsynchronising.md new file mode 100644 index 00000000..b7b42064 --- /dev/null +++ b/docs/runbooks/windowsclocknotsynchronising.md @@ -0,0 +1,43 @@ +--- +title: WindowsClockNotSynchronising +--- + +## Meaning + +Windows Time reports no active NTP time source for ten minutes. This alert is +generated only with `enableTime: true` and requires the `time` collector and its +`ntp` subcollector. Enable it on hosts expected to synchronize through NTP; +hosts intentionally using another clock source may legitimately report zero. + +## Impact + +The clock can drift without a working time source. Clock errors can affect +Kerberos authentication, TLS validation, scheduled work, and event timestamps. + +## Diagnosis + +1. Inspect `windows_time_ntp_client_time_sources` and the exporter collector + success metric. Missing metrics indicate a different problem than a zero value. +2. On the affected host, inspect Windows Time: + + ```powershell + Get-Service W32Time + w32tm /query /status + w32tm /query /source + w32tm /query /peers + w32tm /query /configuration + ``` + +3. Check the Windows Time-Service event log, source DNS resolution, and network + access to the source over UDP port 123. +4. For domain members, inspect the domain time hierarchy and the PDC emulator's + upstream source. For virtual machines, check hypervisor time integration. + +## Mitigation + +Restore the expected Windows Time service, configured source, and UDP reachability. +Respect domain policy; configure external peers on the appropriate authoritative +server rather than overriding every domain member. After correcting the cause, +request rediscovery with `w32tm /resync /rediscover` and verify a valid source and +successful synchronization. Follow [WindowsClockSkewDetected](../windowsclockskewdetected/) +if a significant offset remains. diff --git a/docs/runbooks/windowsclockskewdetected.md b/docs/runbooks/windowsclockskewdetected.md new file mode 100644 index 00000000..f475a801 --- /dev/null +++ b/docs/runbooks/windowsclockskewdetected.md @@ -0,0 +1,43 @@ +--- +title: WindowsClockSkewDetected +--- + +## Meaning + +The absolute clock offset reported by Windows Time exceeds the configured +threshold for ten minutes. The default is 0.05 seconds (50 ms). This opt-in alert +requires the `time` collector with its `ntp` subcollector. The offset is relative +to the selected source and does not independently validate that source's accuracy. + +## Impact + +Clock skew can affect authentication, certificate validity, scheduled work, and +the ordering of events across hosts. The default threshold is an early warning; +application tolerances vary. + +## Diagnosis + +1. Inspect `windows_time_computed_time_offset_seconds`, source count, and + `windows_time_ntp_round_trip_delay_seconds` for the affected host. +2. Check synchronization state and the selected source: + + ```powershell + w32tm /query /status + w32tm /query /source + w32tm /query /configuration + ``` + +3. Compare with a trusted source using + `w32tm /stripchart /computer: /samples:5 /dataonly`. + Check source health, network delay, Windows Time-Service events, and domain + or hypervisor time configuration. +4. Consult [WindowsClockNotSynchronising](../windowsclocknotsynchronising/) when + no NTP source is available. + +## Mitigation + +Restore a healthy source and correct time configuration before requesting +`w32tm /resync /rediscover`. Coordinate large clock corrections with application +owners because abrupt changes can disrupt running workloads. Avoid setting the +clock manually as a recurring fix. Confirm the reported offset decreases and +remains within the application's required tolerance. diff --git a/docs/runbooks/windowscollectorfailed.md b/docs/runbooks/windowscollectorfailed.md new file mode 100644 index 00000000..ae59bd56 --- /dev/null +++ b/docs/runbooks/windowscollectorfailed.md @@ -0,0 +1,40 @@ +--- +title: WindowsCollectorFailed +--- + +## Meaning + +The collector identified by the `collector` label has reported +`windows_exporter_collector_success == 0` for five minutes. + +## Impact + +Metrics from this collector are incomplete or missing. Alerts relying on those +metrics may stop evaluating even though the exporter remains reachable. + +## Diagnosis + +1. Find the failing collector and compare the affected hosts: + + ```promql + windows_exporter_collector_success{job="windows_exporter"} == 0 + ``` + +2. Review exporter logs for that collector's error and duration. Check whether + the failure started after an exporter upgrade, Windows update, or account + permission change. +3. Check the collector's documentation for supported Windows roles, required + privileges, and performance counters. Confirm the required service or role + exists and is running on the host. +4. Compare `windows_exporter_collector_duration_seconds` with the scrape timeout. + Test the relevant performance counters locally with Windows Performance + Monitor. + +## Mitigation + +Restore the required service, privileges, or performance counter availability. +Limit expensive collector queries when collection exceeds the timeout, and +adjust timeouts only after measuring normal collection time. Disable a collector +on hosts where its role is absent. If the problem began with an upgrade, check +release notes and apply a fix or roll back through the normal change process. +Confirm the collector succeeds and its dependent metrics resume. diff --git a/docs/runbooks/windowscpuhighusage.md b/docs/runbooks/windowscpuhighusage.md new file mode 100644 index 00000000..7403bdbe --- /dev/null +++ b/docs/runbooks/windowscpuhighusage.md @@ -0,0 +1,37 @@ +--- +title: WindowsCPUHighUsage +--- + +## Meaning + +Average non-idle CPU usage across logical processors exceeds the configured +threshold for 15 minutes. The default is 90%, measured over a five-minute rate +window. This measures CPU time; Task Manager's frequency-adjusted utility can +show a different value. + +## Impact + +The host has little CPU capacity for additional work. Applications may show +increased latency, timeouts, or reduced throughput. + +## Diagnosis + +1. Inspect `windows:cpu_usage:ratio` for the affected host and compare it with + its workload and normal baseline. +2. Inspect `windows:cpu_time:ratio_rate` by `mode`. High user time points toward + application work; high privileged, interrupt, or DPC time can indicate kernel + or driver activity. DPC and interrupt modes overlap privileged time; do not + sum all modes to estimate usage. +3. Use Task Manager, Resource Monitor, or Performance Monitor to identify the + busy processes. Compare individual core usage to detect a single-threaded + bottleneck. +4. Check scheduled tasks, antivirus scans, recent deployments, and hypervisor + CPU contention. + +## Mitigation + +Reduce or redistribute the workload, reschedule expensive background tasks, or +add CPU capacity. Investigate a process or driver consuming unexpected CPU +before restarting it. Confirm CPU usage and application latency return to normal. +For expected sustained workloads, tune the threshold against service objectives +and available capacity. diff --git a/docs/runbooks/windowsdiskalmostfull.md b/docs/runbooks/windowsdiskalmostfull.md new file mode 100644 index 00000000..5c277f84 --- /dev/null +++ b/docs/runbooks/windowsdiskalmostfull.md @@ -0,0 +1,38 @@ +--- +title: WindowsDiskAlmostFull +--- + +## Meaning + +The volume's free space remains below a configured threshold for 15 minutes. +Defaults are 10% for warning and 5% for critical. Both severity levels fire below +the critical threshold unless Alertmanager inhibition suppresses the warning. +Windows can delay the underlying space counters by 10–15 minutes. + +## Impact + +Applications may fail to write data or logs. Low free space can prevent updates, +pagefile growth, database operations, and normal operating system behavior. + +## Diagnosis + +1. Use the `volume` label to identify the affected filesystem and inspect + `windows:logical_disk_free:ratio` and `windows_logical_disk_free_bytes`. +2. Verify current capacity locally; the exporter counters may lag: + + ```powershell + Get-Volume | Select-Object DriveLetter, FileSystemLabel, SizeRemaining, Size + ``` + +3. Identify growing logs, temporary files, backups, database files, and application + data. Check retention policies and recent jobs. +4. Consult [WindowsDiskFillingUp](../windowsdiskfillingup/) when free space is + declining rapidly. + +## Mitigation + +Expand the volume or remove files through the owning application's supported +retention and cleanup process. Preserve required data and backups; avoid manually +deleting database files or Windows system files. Reduce incoming writes if the +volume may fill before capacity is restored. Verify space locally and allow the +Windows counters to refresh before expecting the alert to resolve. diff --git a/docs/runbooks/windowsdiskfillingup.md b/docs/runbooks/windowsdiskfillingup.md new file mode 100644 index 00000000..9760312b --- /dev/null +++ b/docs/runbooks/windowsdiskfillingup.md @@ -0,0 +1,37 @@ +--- +title: WindowsDiskFillingUp +--- + +## Meaning + +Free space is below the warning threshold and a linear forecast predicts the +volume will fill within the configured horizon. Defaults use six hours of +history, a 24-hour prediction horizon, and a one-hour alert duration. + +## Impact + +Continued growth can exhaust the volume before routine capacity maintenance, +causing write failures or an application outage. + +## Diagnosis + +1. Inspect the free-space trend for the volume and verify local free space with + `Get-Volume`. Windows disk space counters can lag by 10–15 minutes. +2. Inspect the forecast using the configured selector and window, for example: + + ```promql + predict_linear(windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}[6h], 24 * 3600) + ``` + +3. Identify the application, logs, backups, or temporary data causing growth. + Check whether a recent bulk write distorted the trend or growth is sustained. +4. Compare absolute remaining bytes with the application's write rate. A linear + forecast is an estimate and does not model scheduled cleanup or sudden growth. + +## Mitigation + +Restore retention and rotation, reduce write growth, or expand the volume before +it fills. Coordinate data cleanup with the owner. If the forecast reflects a +known temporary job, verify its end time and remaining capacity before silencing +the alert. Follow [WindowsDiskAlmostFull](../windowsdiskalmostfull/) when free +space is already critically low. diff --git a/docs/runbooks/windowsdiskreadlatencyhigh.md b/docs/runbooks/windowsdiskreadlatencyhigh.md new file mode 100644 index 00000000..6929d7ea --- /dev/null +++ b/docs/runbooks/windowsdiskreadlatencyhigh.md @@ -0,0 +1,34 @@ +--- +title: WindowsDiskReadLatencyHigh +--- + +## Meaning + +Average disk read service time exceeds the configured threshold for 15 minutes. +The default is 0.05 seconds (50 ms) per operation, computed from read-time and +read-operation counter rates. Volumes with no reads produce no latency sample. + +## Impact + +Slow storage reads may delay application requests, database queries, and paging. +The recording is an average, so it does not expose tail latency. + +## Diagnosis + +1. Inspect `windows:logical_disk_read_latency:seconds` for the affected `volume` + alongside read IOPS, read throughput, and `windows:logical_disk_busy:ratio`. +2. Inspect `windows_logical_disk_requests_queued` and use Resource Monitor to + identify processes generating disk reads. Look for backup, antivirus, and + indexing jobs. +3. Check System event logs and storage-controller logs for disk, controller, + multipath, or network-storage errors. +4. For virtual machines, check datastore latency and competing workloads on the + hypervisor or storage array. Low IOPS can make averages noisy. + +## Mitigation + +Reduce competing reads or reschedule background jobs. Address storage-path errors +and provision enough IOPS and throughput for the workload. Investigate excessive +paging if memory pressure is contributing. Tune the latency threshold to the +storage tier and application's service objectives, then confirm both service +latency and disk read latency improve. diff --git a/docs/runbooks/windowsdiskwritelatencyhigh.md b/docs/runbooks/windowsdiskwritelatencyhigh.md new file mode 100644 index 00000000..ae2b3cfc --- /dev/null +++ b/docs/runbooks/windowsdiskwritelatencyhigh.md @@ -0,0 +1,33 @@ +--- +title: WindowsDiskWriteLatencyHigh +--- + +## Meaning + +Average disk write service time exceeds the configured threshold for 15 minutes. +The default is 0.05 seconds (50 ms) per operation, computed from write-time and +write-operation counter rates. Volumes with no writes produce no latency sample. + +## Impact + +Slow writes may delay transactions, log flushing, file uploads, or paging. An +average can conceal short periods of very high latency. + +## Diagnosis + +1. Inspect `windows:logical_disk_write_latency:seconds`, write IOPS, write + throughput, `windows:logical_disk_busy:ratio`, and queued requests for the + affected `volume`. +2. Use Resource Monitor to find processes generating writes. Review backup jobs, + database checkpoints, log growth, and scheduled maintenance. +3. Check free space and storage-controller health. Inspect System event logs for + disk, controller, multipath, or network-storage errors. +4. For virtual machines or remote storage, inspect the backing datastore and + array for throughput limits and contention. + +## Mitigation + +Reduce competing writes or move maintenance jobs. Repair storage-path problems +and expand IOPS or throughput capacity. Coordinate database or logging changes +with the application owner. Avoid disabling durability guarantees to silence an +alert. Confirm application write latency recovers and the disk metrics improve. diff --git a/docs/runbooks/windowsexporterdown.md b/docs/runbooks/windowsexporterdown.md new file mode 100644 index 00000000..d2ea9d14 --- /dev/null +++ b/docs/runbooks/windowsexporterdown.md @@ -0,0 +1,39 @@ +--- +title: WindowsExporterDown +--- + +## Meaning + +Prometheus has failed to scrape the target for five minutes (`up == 0`). +The alert does not detect hosts removed from service discovery. + +## Impact + +Monitoring data and resource alerts for the host are unavailable. The host, +exporter service, or the network path may be down. + +## Diagnosis + +1. Open the target in Prometheus's **Targets** page and read the scrape error. + Check the target address, TLS settings, authentication, and scrape timeout. +2. From the Prometheus network, check the configured metrics endpoint, normally + `http://:9182/metrics`. Check DNS resolution and routing. +3. On the affected host, inspect the service and listening socket: + + ```powershell + Get-Service windows_exporter + Get-NetTCPConnection -LocalPort 9182 -State Listen + Test-NetConnection -ComputerName localhost -Port 9182 + ``` + +4. Review exporter logs and the Windows Application/System event logs for + startup errors, permission problems, or collector timeouts. + +## Mitigation + +Restore the network path or correct the scrape configuration. If the exporter +service is stopped, resolve its startup error and start it with +`Start-Service windows_exporter`. If local requests succeed but remote requests +fail, check the host firewall and network ACLs for the configured listener. +Confirm the target becomes healthy and fresh metrics appear before closing the +incident. Update service discovery when a host was intentionally decommissioned. diff --git a/docs/runbooks/windowsmemorycommithighusage.md b/docs/runbooks/windowsmemorycommithighusage.md new file mode 100644 index 00000000..72dda84d --- /dev/null +++ b/docs/runbooks/windowsmemorycommithighusage.md @@ -0,0 +1,40 @@ +--- +title: WindowsMemoryCommitHighUsage +--- + +## Meaning + +Committed memory exceeds the configured percentage of the system commit limit +for 15 minutes. The default threshold is 90%. The commit limit depends on +physical memory and pagefile capacity. + +## Impact + +The system may reject new memory commitments, causing application failures even +when some physical memory remains available. + +## Diagnosis + +1. Inspect `windows:memory_committed:ratio`, `windows_memory_committed_bytes`, + and `windows_memory_commit_limit`. Determine whether committed bytes increased + or the limit decreased. +2. In Task Manager or Resource Monitor, inspect per-process commit size and look + for persistent growth. Check recent workload and deployment changes. +3. Inspect pagefile configuration and the free space on its volumes: + + ```powershell + Get-CimInstance Win32_PageFileUsage + Get-CimInstance Win32_PageFileSetting + Get-Volume + ``` + +4. Check physical memory pressure and paging alongside this alert. The recording + rule omits invalid zero commit limits rather than reporting infinity. + +## Mitigation + +Reduce memory commitments or address the leaking process. Restore pagefile +capacity and disk free space when the commit limit is unexpectedly low. Review +pagefile sizing with the workload owner; some changes require a reboot. Add +physical memory for sustained demand. Confirm committed bytes remain safely +below the commit limit and applications can allocate memory normally. diff --git a/docs/runbooks/windowsmemoryhighusage.md b/docs/runbooks/windowsmemoryhighusage.md new file mode 100644 index 00000000..a0e81ae3 --- /dev/null +++ b/docs/runbooks/windowsmemoryhighusage.md @@ -0,0 +1,34 @@ +--- +title: WindowsMemoryHighUsage +--- + +## Meaning + +Physical memory usage exceeds the configured threshold for 15 minutes. The +default is 90%. Usage is total physical memory minus available memory; available +memory includes reclaimable pages and differs from strictly free memory. + +## Impact + +Memory pressure may cause paging, application latency, and allocation failures. +A high ratio alone does not prove the system is paging heavily. + +## Diagnosis + +1. Inspect `windows:memory_usage:ratio`, `windows:memory_used:bytes`, and + `windows_memory_available_bytes` on the affected host. +2. Check `windows:memory_swap_page_operations:rate`, disk latency, and + `windows:memory_committed:ratio` for related pressure. +3. In Resource Monitor, inspect processes' working sets, commit usage, and hard + faults. Look for growth after a deployment or an unexpected workload increase. +4. Inspect `windows_memory_pool_nonpaged_bytes` and + `windows_memory_pool_paged_bytes` if application working sets do not explain + the usage. Driver or kernel pools may be growing. + +## Mitigation + +Reduce concurrent work, move workloads, or add physical memory. Investigate leaks +in the responsible application or driver. Restart a service only after assessing +its availability impact and preserving useful diagnostics. Pagefile changes +affect the commit limit and do not add physical memory. Confirm available memory +and service latency recover. diff --git a/docs/runbooks/windowsnetworkdiscards.md b/docs/runbooks/windowsnetworkdiscards.md new file mode 100644 index 00000000..d96cdaca --- /dev/null +++ b/docs/runbooks/windowsnetworkdiscards.md @@ -0,0 +1,34 @@ +--- +title: WindowsNetworkDiscards +--- + +## Meaning + +The sum of inbound and outbound discarded packet rates exceeds the configured +threshold for 15 minutes. The default is one discard per second per interface. +Discards count packets dropped without a packet error, such as buffer exhaustion. + +## Impact + +Dropped packets can cause retransmissions, reduced throughput, and application +timeouts. Some drops may be expected under a deliberate traffic policy. + +## Diagnosis + +1. Inspect `windows:net_received_discarded:rate` and + `windows:net_outbound_discarded:rate` to identify the direction and affected + `nic`. +2. Compare traffic, directional utilization ratios, and + `windows_net_output_queue_length_packets` with the normal baseline. +3. Inspect `Get-NetAdapterStatistics`, switch-port drops, adapter buffers, + virtual switch counters, and network QoS policies. +4. Correlate drops with CPU pressure, traffic bursts, hypervisor contention, and + application failures. An unknown or zero bandwidth estimate produces no + utilization sample and should not be interpreted as zero traffic. + +## Mitigation + +Relieve congestion, distribute traffic, or add bandwidth. Correct buffer, driver, +or virtual switch problems when evidence identifies them. Review intentional +drop policies with the network owner before changing thresholds. Confirm discard +rates and application retransmissions return to the expected baseline. diff --git a/docs/runbooks/windowsnetworkerrors.md b/docs/runbooks/windowsnetworkerrors.md new file mode 100644 index 00000000..882cb728 --- /dev/null +++ b/docs/runbooks/windowsnetworkerrors.md @@ -0,0 +1,39 @@ +--- +title: WindowsNetworkErrors +--- + +## Meaning + +The sum of inbound and outbound packet error rates exceeds the configured +threshold for 15 minutes. The default is one error per second per interface. +The alert identifies the interface using its `nic` label. + +## Impact + +Packet errors may cause retransmissions, lower throughput, and intermittent +connection failures. + +## Diagnosis + +1. Inspect `windows:net_received_errors:rate` and + `windows:net_outbound_errors:rate` separately to identify the direction. +2. Compare traffic and `windows:net_received_utilization:ratio` and + `windows:net_sent_utilization:ratio` with the normal baseline. These ratios + use bandwidth in bytes per second and keep each direction separate. +3. Inspect local adapter state and counters: + + ```powershell + Get-NetAdapter + Get-NetAdapterStatistics + ``` + +4. Inspect switch-port counters, cabling, link negotiation, NIC drivers, and + System event logs. For virtual NICs, inspect the host and virtual switch. + +## Mitigation + +Repair faulty links or switch ports and correct driver or adapter configuration +problems. Apply driver and firmware fixes through the normal change process. +Relieve congestion if it contributes. Verify errors stop increasing and affected +application connections recover; cumulative counters do not need to return to +zero for the rate alert to resolve. diff --git a/docs/static/runbooks.css b/docs/static/runbooks.css new file mode 100644 index 00000000..fa479e10 --- /dev/null +++ b/docs/static/runbooks.css @@ -0,0 +1,17 @@ +:root { + color-scheme: light dark; + font-family: system-ui, sans-serif; + line-height: 1.6; + color: light-dark(#20252b, #e5e9ef); + background: light-dark(#fff, #15191f); +} +body { max-width: 64rem; margin: auto; padding: 1.5rem; } +header { display: flex; justify-content: space-between; gap: 1rem; border-bottom: 1px solid #8993a0; padding-bottom: 1rem; } +main { max-width: 54rem; margin: 2rem auto; } +a { color: light-dark(#0059b3, #80bfff); overflow-wrap: anywhere; } +h1, h2 { line-height: 1.3; overflow-wrap: anywhere; } +h2 { margin-top: 2rem; } +pre { overflow-x: auto; padding: 1rem; border: 1px solid #8993a0; border-radius: .3rem; } +code { font-size: .95em; } +li { margin: .4rem 0; } +.runbooks { padding-left: 1.5rem; }