From ad1db73b26a3e286dde8bfc61379b863056d49cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Thu, 8 Oct 2026 23:29:23 +0200 Subject: [PATCH] contrib: add Windows exporter monitoring mixin (#2558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Otto Kröpke --- .editorconfig | 41 +++--- .github/workflows/mixin.yml | 44 +++++++ README.md | 6 + contrib/mixin/.gitignore | 5 + contrib/mixin/.pint.hcl | 24 ++++ contrib/mixin/Makefile | 51 ++++++++ contrib/mixin/README.md | 165 ++++++++++++++++++++++++ contrib/mixin/alerts.jsonnet | 1 + contrib/mixin/alerts/alerts.libsonnet | 101 +++++++++++++++ contrib/mixin/config.libsonnet | 22 ++++ contrib/mixin/jsonnetfile.json | 5 + contrib/mixin/lib/queries.libsonnet | 18 +++ contrib/mixin/mixin.libsonnet | 3 + contrib/mixin/rules.jsonnet | 1 + contrib/mixin/rules/rules.libsonnet | 28 +++++ contrib/mixin/tests/alerts.test.yaml | 175 ++++++++++++++++++++++++++ contrib/mixin/tests/config.libsonnet | 16 +++ contrib/mixin/tests/custom.test.yaml | 128 +++++++++++++++++++ contrib/mixin/tests/rules.test.yaml | 89 +++++++++++++ 19 files changed, 904 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/mixin.yml create mode 100644 contrib/mixin/.gitignore create mode 100644 contrib/mixin/.pint.hcl create mode 100644 contrib/mixin/Makefile create mode 100644 contrib/mixin/README.md create mode 100644 contrib/mixin/alerts.jsonnet create mode 100644 contrib/mixin/alerts/alerts.libsonnet create mode 100644 contrib/mixin/config.libsonnet create mode 100644 contrib/mixin/jsonnetfile.json create mode 100644 contrib/mixin/lib/queries.libsonnet create mode 100644 contrib/mixin/mixin.libsonnet create mode 100644 contrib/mixin/rules.jsonnet create mode 100644 contrib/mixin/rules/rules.libsonnet create mode 100644 contrib/mixin/tests/alerts.test.yaml create mode 100644 contrib/mixin/tests/config.libsonnet create mode 100644 contrib/mixin/tests/custom.test.yaml create mode 100644 contrib/mixin/tests/rules.test.yaml diff --git a/.editorconfig b/.editorconfig index ac1d1b1a..6c325659 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,19 +1,22 @@ -root = true - -[*] -charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true -tab_width = 4 - -[*.{json,wxs,xml}] -indent_style = space -indent_size = 4 - -[renovate.json] -indent_size = 2 - -[*.{yml,yaml}] -indent_style = space -indent_size = 2 +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +tab_width = 4 + +[*.{json,wxs,xml}] +indent_style = space +indent_size = 4 + +[renovate.json] +indent_size = 2 + +[contrib/mixin/{jsonnetfile.json,jsonnetfile.lock.json}] +indent_size = 2 + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml new file mode 100644 index 00000000..c7724d50 --- /dev/null +++ b/.github/workflows/mixin.yml @@ -0,0 +1,44 @@ +name: Mixin + +on: + pull_request: + paths: + - contrib/mixin/** + - .github/workflows/mixin.yml + push: + branches: + - master + - next + - main + paths: + - contrib/mixin/** + - .github/workflows/mixin.yml + workflow_dispatch: + +permissions: {} + +jobs: + rules: + name: Validate Prometheus rules + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + defaults: + run: + working-directory: contrib/mixin + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: 'false' + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.27.1' + + - name: Install pinned tools + run: make tools + + - name: Lint and test rules + run: | + make test PROMTOOL="docker run --rm --entrypoint promtool -v \"$PWD:/mixin:ro\" -w /mixin prom/prometheus:v3.15.0" diff --git a/README.md b/README.md index 5858f84c..6d6ccb5f 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,12 @@ The Docker image is tagged with the version of the exporter. The `latest` tag is See detailed steps to install on Windows Kubernetes [here](./kubernetes/kubernetes.md). +## Prometheus rules + +The [Windows exporter mixin](contrib/mixin/README.md) provides configurable +recording rules and alerts for Prometheus, with Jsonnet generation, pint linting, +and promtool tests. + ## Supported versions `windows_exporter` supports Windows Server versions 2016 and later, and desktop Windows version 10 and 11 (21H2 or later). diff --git a/contrib/mixin/.gitignore b/contrib/mixin/.gitignore new file mode 100644 index 00000000..bc8811b7 --- /dev/null +++ b/contrib/mixin/.gitignore @@ -0,0 +1,5 @@ +/vendor/ +/windows_alerts.yaml +/windows_rules.yaml +/tests/*_alerts.yaml +/tests/*_rules.yaml diff --git a/contrib/mixin/.pint.hcl b/contrib/mixin/.pint.hcl new file mode 100644 index 00000000..17005e0b --- /dev/null +++ b/contrib/mixin/.pint.hcl @@ -0,0 +1,24 @@ +// CI validates generated rules without contacting a Prometheus server. +rule { + match { + kind = "alerting" + } + + label "severity" { + required = true + value = ".+" + severity = "bug" + } + + annotation "summary" { + required = true + value = ".+" + severity = "bug" + } + + annotation "description" { + required = true + value = ".+" + severity = "bug" + } +} diff --git a/contrib/mixin/Makefile b/contrib/mixin/Makefile new file mode 100644 index 00000000..1f3db391 --- /dev/null +++ b/contrib/mixin/Makefile @@ -0,0 +1,51 @@ +JSONNET ?= jsonnet +JSONNETFMT ?= jsonnetfmt +PROMTOOL ?= promtool +PINT ?= pint + +# renovate: go=github.com/google/go-jsonnet +JSONNET_VERSION := v0.22.0 +# renovate: go=github.com/cloudflare/pint +PINT_VERSION := v0.89.2 +SOURCES := $(wildcard *.libsonnet *.jsonnet lib/*.libsonnet rules/*.libsonnet alerts/*.libsonnet tests/*.libsonnet) +FMT_FLAGS := -n 2 --max-blank-lines 2 --string-style s --comment-style s + +.DEFAULT_GOAL := generate +.DELETE_ON_ERROR: +.PHONY: generate tools fmt lint test clean + +generate: windows_rules.yaml windows_alerts.yaml + +tools: + go install github.com/google/go-jsonnet/cmd/jsonnet@$(JSONNET_VERSION) + go install github.com/google/go-jsonnet/cmd/jsonnetfmt@$(JSONNET_VERSION) + go install github.com/cloudflare/pint/cmd/pint@$(PINT_VERSION) + +windows_rules.yaml: $(SOURCES) + $(JSONNET) -S rules.jsonnet > $@ + +windows_alerts.yaml: $(SOURCES) + $(JSONNET) -S alerts.jsonnet > $@ + +tests/custom_rules.yaml: $(SOURCES) + $(JSONNET) -S -e 'std.manifestYamlDoc((import "tests/config.libsonnet").prometheusRules)' > $@ + +tests/custom_alerts.yaml: $(SOURCES) + $(JSONNET) -S -e 'std.manifestYamlDoc((import "tests/config.libsonnet").prometheusAlerts)' > $@ + +fmt: + $(JSONNETFMT) $(FMT_FLAGS) -i $(SOURCES) + +lint: generate tests/custom_rules.yaml tests/custom_alerts.yaml + @set -e; for file in $(SOURCES); do \ + $(JSONNETFMT) $(FMT_FLAGS) "$$file" | diff -u "$$file" -; \ + done + $(PROMTOOL) check rules windows_rules.yaml windows_alerts.yaml tests/custom_rules.yaml tests/custom_alerts.yaml + $(PINT) --offline lint windows_rules.yaml windows_alerts.yaml + $(PINT) --offline lint tests/custom_rules.yaml tests/custom_alerts.yaml + +test: lint + $(PROMTOOL) test rules tests/rules.test.yaml tests/alerts.test.yaml tests/custom.test.yaml + +clean: + rm -f windows_rules.yaml windows_alerts.yaml tests/custom_rules.yaml tests/custom_alerts.yaml diff --git a/contrib/mixin/README.md b/contrib/mixin/README.md new file mode 100644 index 00000000..a42157e3 --- /dev/null +++ b/contrib/mixin/README.md @@ -0,0 +1,165 @@ +# Windows exporter mixin + +Configurable Prometheus recording rules and alerts for windows_exporter. The +source uses Jsonnet and exposes the standard `prometheusRules` and +`prometheusAlerts` mixin fields. Dashboard source is maintained separately. + +## Generate rules + +Install the pinned Jsonnet and [pint](https://github.com/cloudflare/pint) tools: + +```sh +cd contrib/mixin +make tools +export PATH="$(go env GOPATH)/bin:$PATH" +make generate +``` + +The mixin has no Jsonnet library dependencies, so `jb install` is not required +to generate rules. Generated files are ignored by Git: + +- `windows_rules.yaml`: recording rules. +- `windows_alerts.yaml`: alerting rules. + +The default selector is `job="windows_exporter"`. Change it to match your scrape +job before generating rules. Resource rules use the default `cpu`, `memory`, +`logical_disk`, and `net` collectors. Enable `ad` on domain controllers when +opting into Active Directory alerts. + +Copy both generated files into your Prometheus rule directory and reference +them in `prometheus.yml`: + +```yaml +rule_files: + - /etc/prometheus/rules/windows_rules.yaml + - /etc/prometheus/rules/windows_alerts.yaml +``` + +Resource alerts depend on the recording rules; deploy both files. Separate rule +groups can add up to one evaluation interval of delay before alerts observe a +new recording-rule value. + +## Customize or compose + +Edit `_config` in `config.libsonnet`, or import and extend the mixin without +modifying its source. For example, create `custom.libsonnet` in this directory: + +```jsonnet +(import 'mixin.libsonnet') { + _config+:: { + windowsExporterSelector: 'job=~"windows.*", environment="production"', + volumeSelector: 'volume!~"HarddiskVolume.*"', + nicSelector: 'nic!="Loopback"', + rateInterval: '5m', + cpuHighUsageThreshold: 95, + enableActiveDirectory: true, + }, +} +``` + +Render this configuration with: + +```sh +jsonnet -S -e 'std.manifestYamlDoc((import "custom.libsonnet").prometheusRules)' > custom_rules.yaml +jsonnet -S -e 'std.manifestYamlDoc((import "custom.libsonnet").prometheusAlerts)' > custom_alerts.yaml +``` + +Selectors are comma-separated Prometheus label matchers without braces. They +apply to both recording rules and alerts. All target labels, including `job`, +`instance`, `cluster`, and custom selector labels, are preserved. CPU usage +averages idle rates across cores and removes only `core` and `mode`. Disk and +network recordings also retain `volume` and `nic` respectively. + +`rateInterval` is the fixed window for recording rules and Active Directory +failure rates; keep it at least four times your scrape interval. Grafana's +`$__rate_interval` is a dashboard variable and must not be used in Prometheus +rules. + +## Recording rules + +| Metric | Meaning | Unit | +| --- | --- | --- | +| `windows:cpu_usage:ratio` | Non-idle CPU time averaged across cores | Ratio, 0–1 | +| `windows:memory_usage:ratio` | Physical memory in use, based on available bytes | Ratio, 0–1 | +| `windows:logical_disk_free:ratio` | Free disk space per volume | Ratio, 0–1 | +| `windows:logical_disk_read_bytes:rate` | Disk read throughput per volume | Bytes/second | +| `windows:logical_disk_write_bytes:rate` | Disk write throughput per volume | Bytes/second | +| `windows:net_received_bytes:rate` | Received network traffic per interface | Bytes/second | +| `windows:net_sent_bytes:rate` | Sent network traffic per interface | Bytes/second | + +The names remain stable when `rateInterval` changes. Dashboard authors can use +these recordings or query raw exporter metrics directly. CPU usage measures +non-idle time; it is not the frequency-adjusted processor utility shown in +Windows Task Manager. + +## Alerts + +Threshold configuration values below are percentages except where noted. + +| Alert | Default condition | Duration | Severity | +| --- | --- | --- | --- | +| `WindowsExporterDown` | Scrape target `up == 0` | 5m | Critical | +| `WindowsCollectorFailed` | `windows_exporter_collector_success == 0` | 5m | Warning | +| `WindowsCPUHighUsage` | CPU usage above `cpuHighUsageThreshold` (90%) | 15m | Warning | +| `WindowsMemoryHighUsage` | Memory usage above `memoryHighUsageThreshold` (90%) | 15m | Warning | +| `WindowsDiskAlmostFull` | Free space below `diskFreeWarningThreshold` (10%) | 15m | Warning | +| `WindowsDiskAlmostFull` | Free space below `diskFreeCriticalThreshold` (5%) | 15m | Critical | +| `WindowsDiskFillingUp` | Below the warning threshold and predicted to fill within `diskPredictionHours` (24h), using `diskPredictionWindow` (6h) | 1h | Warning | +| `WindowsADPendingReplication` | Pending operations above `adPendingReplicationThreshold` (50) | 15m | Warning | +| `WindowsADReplicationFailures` | Failed synchronization requests above `adReplicationFailureRateThreshold` (0 requests/second) | 15m | Warning | + +Active Directory alerts are generated only when `enableActiveDirectory` is +`true`. `criticalSeverity` defaults to `critical` and can be overridden to match +your routing labels. Below the critical disk threshold, both disk severity +levels fire; configure Alertmanager inhibition if only the critical +notification should be sent. + +Windows disk size/free-space counters may lag by 10–15 minutes. Predictions +depend on a sustained trend and are intentionally delayed for an hour. A target +removed from service discovery has no `up` series and will not trigger +`WindowsExporterDown`. Missing resource metrics produce no resource alerts; +scrape and collector-failure alerts cover explicit failures, not disabled +collectors. + +## Validate + +Install `promtool` from a [Prometheus release](https://github.com/prometheus/prometheus/releases), +then run: + +```sh +make test +``` + +This checks Jsonnet formatting, validates generated rules with `promtool`, runs +`pint --offline lint`, and evaluates the rules against synthetic time series. +Tests cover all seven recordings, alert timing and recovery, threshold +boundaries, counter resets, disk forecasts, target-label preservation, and +custom selectors/thresholds with Active Directory enabled. CI uses Prometheus +3.15.0 and the same pinned tool versions as `make tools`. + +If you prefer to run `promtool` through Docker: + +```sh +make test PROMTOOL="docker run --rm --entrypoint promtool -v \"$PWD:/mixin:ro\" -w /mixin prom/prometheus:v3.15.0" +``` + +`.pint.hcl` requires a non-empty severity label and summary/description +annotations on alerts, in addition to pint's built-in checks. CI uses offline +mode without a Prometheus endpoint. To validate metric availability and labels +against your own server, copy `.pint.hcl` to a local configuration, add: + +```hcl +prometheus "local" { + uri = "http://localhost:9090" +} +``` + +Then run `pint --config /path/to/local.pint.hcl lint windows_rules.yaml windows_alerts.yaml` +without `--offline`. See the [pint documentation](https://cloudflare.github.io/pint/) +for available checks. + +Run `make fmt` after source changes and `make clean` to remove generated rule +files. For downstream composition with Jsonnet Bundler, import +`github.com/prometheus-community/windows_exporter/contrib/mixin/mixin.libsonnet` +and extend `_config`. See the [monitoring mixin documentation](https://github.com/monitoring-mixins/docs) +for the standard composition workflow. diff --git a/contrib/mixin/alerts.jsonnet b/contrib/mixin/alerts.jsonnet new file mode 100644 index 00000000..75e7c1b2 --- /dev/null +++ b/contrib/mixin/alerts.jsonnet @@ -0,0 +1 @@ +std.manifestYamlDoc((import 'mixin.libsonnet').prometheusAlerts) diff --git a/contrib/mixin/alerts/alerts.libsonnet b/contrib/mixin/alerts/alerts.libsonnet new file mode 100644 index 00000000..288a46d6 --- /dev/null +++ b/contrib/mixin/alerts/alerts.libsonnet @@ -0,0 +1,101 @@ +{ + local c = self._config, + local q = (import '../lib/queries.libsonnet')(c), + local alert(name, expr, duration, severity, summary, description) = { + alert: name, + expr: expr, + 'for': duration, + labels: { severity: severity }, + annotations: { summary: summary, description: description }, + }, + prometheusAlerts+:: { + groups: [ + { + name: 'windows-exporter-alerts', + rules: [ + alert( + 'WindowsExporterDown', + q.metric('up') + ' == 0', + '5m', + c.criticalSeverity, + 'Windows exporter is unreachable', + 'Prometheus cannot scrape Windows exporter on {{ $labels.instance }} (job {{ $labels.job }}).' + ), + alert( + 'WindowsCollectorFailed', + q.metric('windows_exporter_collector_success') + ' == 0', + '5m', + 'warning', + 'Windows collector failed', + 'Collector {{ $labels.collector }} on {{ $labels.instance }} has failed for at least 5 minutes.' + ), + alert( + 'WindowsCPUHighUsage', + q.metric('windows:cpu_usage:ratio') + ' * 100 > %s' % c.cpuHighUsageThreshold, + '15m', + 'warning', + 'Windows CPU usage is high', + 'CPU usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.cpuHighUsageThreshold + ), + alert( + 'WindowsMemoryHighUsage', + q.metric('windows:memory_usage:ratio') + ' * 100 > %s' % c.memoryHighUsageThreshold, + '15m', + 'warning', + 'Windows memory usage is high', + 'Physical memory usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.memoryHighUsageThreshold + ), + ] + [ + alert( + 'WindowsDiskAlmostFull', + q.metric('windows:logical_disk_free:ratio', [c.volumeSelector]) + ' * 100 < %s' % entry[1], + '15m', + entry[0], + 'Windows disk space is low', + 'Volume {{ $labels.volume }} on {{ $labels.instance }} has less than %s%% free space. Windows may delay space metrics by 10–15 minutes.' % entry[1] + ) + for entry in [ + ['warning', c.diskFreeWarningThreshold], + [c.criticalSeverity, c.diskFreeCriticalThreshold], + ] + ] + [ + alert( + 'WindowsDiskFillingUp', + q.metric('windows:logical_disk_free:ratio', [c.volumeSelector]) + ' * 100 < %s and predict_linear(%s[%s], %s) < 0' % [ + c.diskFreeWarningThreshold, + q.metric('windows_logical_disk_free_bytes', [c.volumeSelector]), + c.diskPredictionWindow, + c.diskPredictionHours * 3600, + ], + '1h', + 'warning', + 'Windows disk is predicted to fill up', + 'Volume {{ $labels.volume }} on {{ $labels.instance }} is predicted to fill within %s hours.' % c.diskPredictionHours + ), + ], + }, + ] + if c.enableActiveDirectory then [ + { + name: 'windows-active-directory', + rules: [ + alert( + 'WindowsADPendingReplication', + q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold, + '15m', + 'warning', + 'Active Directory replication queue is high', + 'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold + ), + alert( + 'WindowsADReplicationFailures', + '(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold, + '15m', + 'warning', + 'Active Directory replication requests are failing', + 'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.' + ), + ], + }, + ] else [], + }, +} diff --git a/contrib/mixin/config.libsonnet b/contrib/mixin/config.libsonnet new file mode 100644 index 00000000..21addf80 --- /dev/null +++ b/contrib/mixin/config.libsonnet @@ -0,0 +1,22 @@ +{ + _config+:: { + // Prometheus label matchers, without enclosing braces. + windowsExporterSelector: 'job="windows_exporter"', + volumeSelector: 'volume!=""', + nicSelector: 'nic!=""', + rateInterval: '5m', + + // Opt in only for targets with the ad collector enabled. + enableActiveDirectory: false, + + cpuHighUsageThreshold: 90, + memoryHighUsageThreshold: 90, + diskFreeWarningThreshold: 10, + diskFreeCriticalThreshold: 5, + diskPredictionWindow: '6h', + diskPredictionHours: 24, + criticalSeverity: 'critical', + adPendingReplicationThreshold: 50, + adReplicationFailureRateThreshold: 0, + }, +} diff --git a/contrib/mixin/jsonnetfile.json b/contrib/mixin/jsonnetfile.json new file mode 100644 index 00000000..746f0752 --- /dev/null +++ b/contrib/mixin/jsonnetfile.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "dependencies": [], + "legacyImports": false +} diff --git a/contrib/mixin/lib/queries.libsonnet b/contrib/mixin/lib/queries.libsonnet new file mode 100644 index 00000000..30d18942 --- /dev/null +++ b/contrib/mixin/lib/queries.libsonnet @@ -0,0 +1,18 @@ +function(c) + { + selector(extra=[]): std.join(', ', [s for s in [c.windowsExporterSelector] + extra if s != '']), + metric(name, extra=[]): '%s{%s}' % [name, self.selector(extra)], + rate(name, extra=[]): 'rate(%s[%s])' % [self.metric(name, extra), c.rateInterval], + cpuUsage: + '1 - avg without (core, mode) (%s)' % self.rate('windows_cpu_time_total', ['mode="idle"']), + memoryUsage: + '1 - %s / %s' % [ + self.metric('windows_memory_available_bytes'), + self.metric('windows_memory_physical_total_bytes'), + ], + diskFree: + '%s / %s' % [ + self.metric('windows_logical_disk_free_bytes', [c.volumeSelector]), + self.metric('windows_logical_disk_size_bytes', [c.volumeSelector]), + ], + } diff --git a/contrib/mixin/mixin.libsonnet b/contrib/mixin/mixin.libsonnet new file mode 100644 index 00000000..9a95d4b6 --- /dev/null +++ b/contrib/mixin/mixin.libsonnet @@ -0,0 +1,3 @@ +(import 'config.libsonnet') + +(import 'rules/rules.libsonnet') + +(import 'alerts/alerts.libsonnet') diff --git a/contrib/mixin/rules.jsonnet b/contrib/mixin/rules.jsonnet new file mode 100644 index 00000000..dbe13f41 --- /dev/null +++ b/contrib/mixin/rules.jsonnet @@ -0,0 +1 @@ +std.manifestYamlDoc((import 'mixin.libsonnet').prometheusRules) diff --git a/contrib/mixin/rules/rules.libsonnet b/contrib/mixin/rules/rules.libsonnet new file mode 100644 index 00000000..b364b067 --- /dev/null +++ b/contrib/mixin/rules/rules.libsonnet @@ -0,0 +1,28 @@ +{ + local c = self._config, + local q = (import '../lib/queries.libsonnet')(c), + prometheusRules+:: { + groups: [ + { + name: 'windows-exporter-recording', + rules: [ + { record: 'windows:cpu_usage:ratio', expr: q.cpuUsage }, + { record: 'windows:memory_usage:ratio', expr: q.memoryUsage }, + { record: 'windows:logical_disk_free:ratio', expr: q.diskFree }, + ] + [ + { + record: 'windows:logical_disk_%s_bytes:rate' % direction, + expr: q.rate('windows_logical_disk_%s_bytes_total' % direction, [c.volumeSelector]), + } + for direction in ['read', 'write'] + ] + [ + { + record: 'windows:net_%s_bytes:rate' % direction, + expr: q.rate('windows_net_bytes_%s_total' % direction, [c.nicSelector]), + } + for direction in ['received', 'sent'] + ], + }, + ], + }, +} diff --git a/contrib/mixin/tests/alerts.test.yaml b/contrib/mixin/tests/alerts.test.yaml new file mode 100644 index 00000000..7e775bed --- /dev/null +++ b/contrib/mixin/tests/alerts.test.yaml @@ -0,0 +1,175 @@ +rule_files: + - ../windows_rules.yaml + - ../windows_alerts.yaml +evaluation_interval: 1m +group_eval_order: + - windows-exporter-recording + - windows-exporter-alerts + +tests: + - name: Exporter and collector failures wait five minutes and recover + interval: 1m + input_series: + - series: 'up{job="windows_exporter", instance="host:9182"}' + values: '0x5 1x5' + - series: 'up{job="other", instance="other:9182"}' + values: '0x10' + - series: 'windows_exporter_collector_success{job="windows_exporter", instance="host:9182", collector="cpu"}' + values: '0x5 1x5' + alert_rule_test: + - alertname: WindowsExporterDown + eval_time: 4m + exp_alerts: [] + - alertname: WindowsExporterDown + eval_time: 5m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', severity: critical} + exp_annotations: + summary: Windows exporter is unreachable + description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows_exporter).' + - alertname: WindowsExporterDown + eval_time: 6m + exp_alerts: [] + - alertname: WindowsCollectorFailed + eval_time: 4m + exp_alerts: [] + - alertname: WindowsCollectorFailed + eval_time: 5m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', collector: cpu, severity: warning} + exp_annotations: + summary: Windows collector failed + description: 'Collector cpu on host:9182 has failed for at least 5 minutes.' + - alertname: WindowsCollectorFailed + eval_time: 6m + exp_alerts: [] + + - name: Resource alerts use recording rules and wait fifteen minutes + interval: 1m + input_series: + - series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="0", mode="idle"}' + values: '0+3x20 120+60x10' + - series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '5x20 50x10' + - series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '100x31' + alert_rule_test: + - alertname: WindowsCPUHighUsage + eval_time: 15m + exp_alerts: [] + - alertname: WindowsCPUHighUsage + eval_time: 16m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning} + exp_annotations: + summary: Windows CPU usage is high + description: 'CPU usage on host:9182 has exceeded 90% for 15 minutes.' + - alertname: WindowsCPUHighUsage + eval_time: 25m + exp_alerts: [] + - alertname: WindowsMemoryHighUsage + eval_time: 14m + exp_alerts: [] + - alertname: WindowsMemoryHighUsage + eval_time: 15m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, severity: warning} + exp_annotations: + summary: Windows memory usage is high + description: 'Physical memory usage on host:9182 has exceeded 90% for 15 minutes.' + - alertname: WindowsMemoryHighUsage + eval_time: 21m + exp_alerts: [] + + - name: Disk alerts retain volume labels and fire at both severity levels + interval: 1m + input_series: + - series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '3x15 30x5' + - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '100x21' + alert_rule_test: + - alertname: WindowsDiskAlmostFull + eval_time: 14m + exp_alerts: [] + - alertname: WindowsDiskAlmostFull + eval_time: 15m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: warning} + exp_annotations: + summary: Windows disk space is low + description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.' + - exp_labels: {job: windows_exporter, instance: 'host:9182', volume: 'C:', severity: critical} + exp_annotations: + summary: Windows disk space is low + description: 'Volume C: on host:9182 has less than 5% free space. Windows may delay space metrics by 10–15 minutes.' + - alertname: WindowsDiskAlmostFull + eval_time: 16m + exp_alerts: [] + + - name: Healthy resources at the exact thresholds do not fire + interval: 1m + input_series: + - series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", core="0", mode="idle"}' + values: '0+6x20' + - series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182"}' + values: '10x20' + - series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182"}' + values: '100x20' + - series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '10x20' + - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '100x20' + alert_rule_test: + - {alertname: WindowsCPUHighUsage, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsMemoryHighUsage, eval_time: 20m, exp_alerts: []} + - {alertname: WindowsDiskAlmostFull, eval_time: 20m, exp_alerts: []} + + - name: Prediction fires only for low and declining disk space + interval: 1m + input_series: + - series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '1000-1x420' + - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '20000x420' + - series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}' + values: '1000x420' + - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="D:"}' + values: '20000x420' + - series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="E:"}' + values: '500+1x420' + - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="E:"}' + values: '20000x420' + - series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="b", volume="C:"}' + values: '1000x420' + - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="b", volume="C:"}' + values: '20000x420' + alert_rule_test: + - alertname: WindowsDiskFillingUp + eval_time: 60m + exp_alerts: [] + - alertname: WindowsDiskFillingUp + eval_time: 61m + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning} + exp_annotations: + summary: Windows disk is predicted to fill up + description: 'Volume C: on host:9182 is predicted to fill within 24 hours.' + - alertname: WindowsDiskFillingUp + eval_time: 7h + exp_alerts: + - exp_labels: {job: windows_exporter, instance: 'host:9182', cluster: a, volume: 'C:', severity: warning} + exp_annotations: + summary: Windows disk is predicted to fill up + description: 'Volume C: on host:9182 is predicted to fill within 24 hours.' + + - name: Missing collector metrics do not produce resource alerts + interval: 1m + input_series: + - series: 'up{job="windows_exporter", instance="host:9182"}' + values: '1x70' + alert_rule_test: + - {alertname: WindowsCPUHighUsage, eval_time: 70m, exp_alerts: []} + - {alertname: WindowsMemoryHighUsage, eval_time: 70m, exp_alerts: []} + - {alertname: WindowsDiskAlmostFull, eval_time: 70m, exp_alerts: []} + - {alertname: WindowsDiskFillingUp, eval_time: 70m, exp_alerts: []} diff --git a/contrib/mixin/tests/config.libsonnet b/contrib/mixin/tests/config.libsonnet new file mode 100644 index 00000000..b67781f2 --- /dev/null +++ b/contrib/mixin/tests/config.libsonnet @@ -0,0 +1,16 @@ +(import '../mixin.libsonnet') { + _config+:: { + windowsExporterSelector: 'job=~"windows.*", cluster="test"', + volumeSelector: 'volume!~"HarddiskVolume.*"', + nicSelector: 'nic!="Loopback"', + rateInterval: '2m', + cpuHighUsageThreshold: 80, + memoryHighUsageThreshold: 85, + diskFreeWarningThreshold: 20, + diskFreeCriticalThreshold: 10, + criticalSeverity: 'page', + enableActiveDirectory: true, + adPendingReplicationThreshold: 10, + adReplicationFailureRateThreshold: 0.5, + }, +} diff --git a/contrib/mixin/tests/custom.test.yaml b/contrib/mixin/tests/custom.test.yaml new file mode 100644 index 00000000..5e4dddba --- /dev/null +++ b/contrib/mixin/tests/custom.test.yaml @@ -0,0 +1,128 @@ +rule_files: + - custom_rules.yaml + - custom_alerts.yaml +evaluation_interval: 1m +group_eval_order: + - windows-exporter-recording + - windows-exporter-alerts + - windows-active-directory + +tests: + - name: Custom selectors, thresholds and critical severity propagate to alerts + interval: 1m + input_series: + - series: 'up{job="windows-custom", instance="host:9182", cluster="test"}' + values: '0x20' + - series: 'up{job="windows-custom", instance="host:9182", cluster="prod"}' + values: '0x20' + - series: 'windows_cpu_time_total{job="windows-custom", instance="host:9182", cluster="test", core="0", mode="idle"}' + values: '0+9x20' + - series: 'windows_memory_available_bytes{job="windows-custom", instance="host:9182", cluster="test"}' + values: '12x20' + - series: 'windows_memory_physical_total_bytes{job="windows-custom", instance="host:9182", cluster="test"}' + values: '100x20' + - series: 'windows_logical_disk_free_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}' + values: '8x20' + - series: 'windows_logical_disk_size_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}' + values: '100x20' + - series: 'windows_logical_disk_free_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}' + values: '1x20' + - series: 'windows_logical_disk_size_bytes{job="windows-custom", instance="host:9182", cluster="test", volume="HarddiskVolume1"}' + values: '100x20' + - series: 'windows_net_bytes_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}' + values: '0+600x20' + - series: 'windows_net_bytes_received_total{job="windows-custom", instance="host:9182", cluster="test", nic="Loopback"}' + values: '0+600x20' + promql_expr_test: + - expr: windows:net_received_bytes:rate + eval_time: 20m + exp_samples: + - labels: 'windows:net_received_bytes:rate{job="windows-custom", instance="host:9182", cluster="test", nic="Ethernet"}' + value: 10 + - expr: windows:logical_disk_free:ratio + eval_time: 20m + exp_samples: + - labels: 'windows:logical_disk_free:ratio{job="windows-custom", instance="host:9182", cluster="test", volume="C:"}' + value: 0.08 + alert_rule_test: + - alertname: WindowsExporterDown + eval_time: 5m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: page} + exp_annotations: + summary: Windows exporter is unreachable + description: 'Prometheus cannot scrape Windows exporter on host:9182 (job windows-custom).' + - alertname: WindowsCPUHighUsage + eval_time: 16m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} + exp_annotations: + summary: Windows CPU usage is high + description: 'CPU usage on host:9182 has exceeded 80% for 15 minutes.' + - alertname: WindowsMemoryHighUsage + eval_time: 15m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} + exp_annotations: + summary: Windows memory usage is high + description: 'Physical memory usage on host:9182 has exceeded 85% for 15 minutes.' + - alertname: WindowsDiskAlmostFull + eval_time: 15m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: warning} + exp_annotations: + summary: Windows disk space is low + description: 'Volume C: on host:9182 has less than 20% free space. Windows may delay space metrics by 10–15 minutes.' + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, volume: 'C:', severity: page} + exp_annotations: + summary: Windows disk space is low + description: 'Volume C: on host:9182 has less than 10% free space. Windows may delay space metrics by 10–15 minutes.' + + - name: Active Directory opt-in honors thresholds, selectors and recovery + interval: 1m + input_series: + - series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="host:9182", cluster="test"}' + values: '20x20 0x5' + - series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="healthy:9182", cluster="test"}' + values: '10x26' + - series: 'windows_ad_replication_pending_operations{job="windows-custom", instance="host:9182", cluster="prod"}' + values: '20x26' + - series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="host:9182", cluster="test"}' + values: '0+120x26' + - series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="host:9182", cluster="test"}' + values: '0+60x20 1320+120x5' + - series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="healthy:9182", cluster="test"}' + values: '0+120x26' + - series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="healthy:9182", cluster="test"}' + values: '0+120x26' + - series: 'windows_ad_replication_sync_requests_total{job="windows-custom", instance="host:9182", cluster="prod"}' + values: '0+120x26' + - series: 'windows_ad_replication_sync_requests_success_total{job="windows-custom", instance="host:9182", cluster="prod"}' + values: '0+60x26' + alert_rule_test: + - alertname: WindowsADPendingReplication + eval_time: 14m + exp_alerts: [] + - alertname: WindowsADPendingReplication + eval_time: 15m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} + exp_annotations: + summary: Active Directory replication queue is high + description: 'Pending replication operations on host:9182 have exceeded 10 for 15 minutes.' + - alertname: WindowsADPendingReplication + eval_time: 21m + exp_alerts: [] + - alertname: WindowsADReplicationFailures + eval_time: 16m + exp_alerts: [] + - alertname: WindowsADReplicationFailures + eval_time: 17m + exp_alerts: + - exp_labels: {job: windows-custom, instance: 'host:9182', cluster: test, severity: warning} + exp_annotations: + summary: Active Directory replication requests are failing + description: 'Replication synchronization requests on host:9182 have been failing for 15 minutes.' + - alertname: WindowsADReplicationFailures + eval_time: 23m + exp_alerts: [] diff --git a/contrib/mixin/tests/rules.test.yaml b/contrib/mixin/tests/rules.test.yaml new file mode 100644 index 00000000..81b5ecd9 --- /dev/null +++ b/contrib/mixin/tests/rules.test.yaml @@ -0,0 +1,89 @@ +rule_files: + - ../windows_rules.yaml +evaluation_interval: 1m +fuzzy_compare: true + +tests: + - name: CPU is averaged across cores and preserves target labels + interval: 1m + input_series: + - series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="0", mode="idle"}' + values: '0+12x10' + - series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="1", mode="idle"}' + values: '0+36x10' + - series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="a", core="0", mode="user"}' + values: '0+48x10' + - series: 'windows_cpu_time_total{job="windows_exporter", instance="host:9182", cluster="b", core="0", mode="idle"}' + values: '0+30x10' + - series: 'windows_cpu_time_total{job="other", instance="host:9182", core="0", mode="idle"}' + values: '0+6x10' + promql_expr_test: + - expr: windows:cpu_usage:ratio + eval_time: 10m + exp_samples: + - labels: 'windows:cpu_usage:ratio{job="windows_exporter", instance="host:9182", cluster="a"}' + value: 0.6 + - labels: 'windows:cpu_usage:ratio{job="windows_exporter", instance="host:9182", cluster="b"}' + value: 0.5 + + - name: Memory availability and disk free space are ratios, not percentages + interval: 1m + input_series: + - series: 'windows_memory_available_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '25x5' + - series: 'windows_memory_physical_total_bytes{job="windows_exporter", instance="host:9182", cluster="a"}' + values: '100x5' + - series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '20x5' + - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + values: '100x5' + - series: 'windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume=""}' + values: '1x5' + - series: 'windows_logical_disk_size_bytes{job="windows_exporter", instance="host:9182", volume=""}' + values: '100x5' + promql_expr_test: + - expr: windows:memory_usage:ratio + eval_time: 5m + exp_samples: + - labels: 'windows:memory_usage:ratio{job="windows_exporter", instance="host:9182", cluster="a"}' + value: 0.75 + - expr: windows:logical_disk_free:ratio + eval_time: 5m + exp_samples: + - labels: 'windows:logical_disk_free:ratio{job="windows_exporter", instance="host:9182", cluster="a", volume="C:"}' + value: 0.2 + + - name: Disk and network rates retain device labels and handle counter resets + interval: 1m + input_series: + - series: 'windows_logical_disk_read_bytes_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0+600x10' + - series: 'windows_logical_disk_write_bytes_total{job="windows_exporter", instance="host:9182", volume="C:"}' + values: '0+1200x10' + - series: 'windows_net_bytes_received_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + values: '0+600x5 600+600x4' + - series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + values: '0+1800x10' + - series: 'windows_net_bytes_sent_total{job="windows_exporter", instance="host:9182", nic=""}' + values: '0+1800x10' + promql_expr_test: + - expr: windows:logical_disk_read_bytes:rate + eval_time: 10m + exp_samples: + - labels: 'windows:logical_disk_read_bytes:rate{job="windows_exporter", instance="host:9182", volume="C:"}' + value: 10 + - expr: windows:logical_disk_write_bytes:rate + eval_time: 10m + exp_samples: + - labels: 'windows:logical_disk_write_bytes:rate{job="windows_exporter", instance="host:9182", volume="C:"}' + value: 20 + - expr: windows:net_received_bytes:rate + eval_time: 8m + exp_samples: + - labels: 'windows:net_received_bytes:rate{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + value: 10 + - expr: windows:net_sent_bytes:rate + eval_time: 10m + exp_samples: + - labels: 'windows:net_sent_bytes:rate{job="windows_exporter", instance="host:9182", nic="Ethernet"}' + value: 30