mirror of
https://github.com/prometheus-community/windows_exporter.git
synced 2026-10-17 10:09:05 +02:00
contrib: expand Windows mixin rules and publish runbooks (#2575)
Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
This commit is contained in:
Jan-Otto Kröpke
@@ -0,0 +1,3 @@
|
||||
/public/
|
||||
/resources/
|
||||
/.hugo_build.lock
|
||||
@@ -1,6 +1,11 @@
|
||||
# Documentation
|
||||
This directory contains documentation of the collectors in the windows_exporter, with information such as what metrics are exported, any flags for additional configuration, and some example usage in alerts and queries.
|
||||
|
||||
The [alert runbooks](runbooks) describe how to diagnose and mitigate alerts from
|
||||
the [Windows exporter mixin](../contrib/mixin/README.md). Preview the runbook site
|
||||
from the repository root with `hugo server --source docs`; its dedicated GitHub
|
||||
Actions workflow publishes default-branch changes to GitHub Pages.
|
||||
|
||||
# Collectors
|
||||
- [`ad`](collector.ad.md)
|
||||
- [`adcs`](collector.adcs.md)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Check generated site links while respecting the configured Pages base path."""
|
||||
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
|
||||
class Links(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.links = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
for name, value in attrs:
|
||||
if name in ("href", "src") and value:
|
||||
self.links.append(value)
|
||||
|
||||
|
||||
root = Path(sys.argv[1])
|
||||
prefix = urlsplit(sys.argv[2]).path.rstrip("/") + "/"
|
||||
errors = []
|
||||
checked = 0
|
||||
pages = list(root.rglob("*.html"))
|
||||
if not pages:
|
||||
sys.exit(f"No generated HTML pages in {root}")
|
||||
for page in pages:
|
||||
parser = Links()
|
||||
parser.feed(page.read_text(encoding="utf-8"))
|
||||
for link in parser.links:
|
||||
url = urlsplit(link)
|
||||
if url.scheme or url.netloc or not url.path:
|
||||
continue
|
||||
if url.path.startswith("/"):
|
||||
if not url.path.startswith(prefix):
|
||||
errors.append(f"{page}: link {link} is outside the site base path {prefix}")
|
||||
continue
|
||||
target = root / unquote(url.path.removeprefix(prefix))
|
||||
else:
|
||||
target = page.parent / unquote(url.path)
|
||||
if target.is_dir():
|
||||
target /= "index.html"
|
||||
if not target.is_file():
|
||||
errors.append(f"{page}: broken link {link}")
|
||||
checked += 1
|
||||
if errors:
|
||||
sys.exit("\n".join(errors))
|
||||
print(f"Checked {checked} internal links across {len(pages)} HTML pages")
|
||||
@@ -0,0 +1,15 @@
|
||||
baseURL = "https://prometheus-community.github.io/windows_exporter/"
|
||||
title = "Windows exporter runbooks"
|
||||
locale = "en"
|
||||
disableKinds = ["taxonomy", "term", "RSS", "sitemap"]
|
||||
|
||||
[module]
|
||||
[[module.mounts]]
|
||||
source = "runbooks"
|
||||
target = "content/runbooks"
|
||||
[[module.mounts]]
|
||||
source = "layouts"
|
||||
target = "layouts"
|
||||
[[module.mounts]]
|
||||
source = "static"
|
||||
target = "static"
|
||||
@@ -0,0 +1,16 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{{ if not .IsHome }}{{ .Title }} | {{ end }}{{ site.Title }}</title>
|
||||
<link rel="stylesheet" href="{{ site.Home.RelPermalink }}runbooks.css">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<a href="{{ site.Home.RelPermalink }}">{{ site.Title }}</a>
|
||||
<a href="https://github.com/prometheus-community/windows_exporter">GitHub</a>
|
||||
</header>
|
||||
<main>{{ block "main" . }}{{ end }}</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,13 @@
|
||||
{{/* The base template declares the doctype. <!-- htmlhint-disable doctype-first --> */}}
|
||||
{{ define "main" }}
|
||||
<h1>{{ .Title }}</h1>
|
||||
{{ .Content }}
|
||||
{{ if .IsHome }}
|
||||
<p>Diagnosis and mitigation for alerts from the Windows exporter mixin.</p>
|
||||
{{ end }}
|
||||
<ul class="runbooks">
|
||||
{{ range (site.GetPage "/runbooks").RegularPages.ByTitle }}
|
||||
<li><a href="{{ .RelPermalink }}">{{ .Title }}</a></li>
|
||||
{{ end }}
|
||||
</ul>
|
||||
{{ end }}
|
||||
@@ -0,0 +1,8 @@
|
||||
{{/* The base template declares the doctype. <!-- htmlhint-disable doctype-first --> */}}
|
||||
{{ define "main" }}
|
||||
<nav aria-label="Breadcrumb"><a href="{{ site.Home.RelPermalink }}runbooks/">All runbooks</a></nav>
|
||||
<article>
|
||||
<h1>{{ .Title }}</h1>
|
||||
{{ .Content }}
|
||||
</article>
|
||||
{{ end }}
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
title: Windows exporter runbooks
|
||||
---
|
||||
|
||||
These runbooks describe alerts from the [Windows exporter mixin](https://github.com/prometheus-community/windows_exporter/tree/master/contrib/mixin).
|
||||
Use the alert's `job`, `instance`, and any `cluster` or environment labels to find
|
||||
the affected host. Disk alerts also identify a `volume`; network alerts identify
|
||||
a `nic`. Adjust example PromQL selectors to match your scrape configuration.
|
||||
|
||||
The default thresholds are starting points. Check the generated alert rules for
|
||||
your configured thresholds and durations. Active Directory and NTP clock alerts
|
||||
are opt-in and require their collectors to be enabled.
|
||||
@@ -0,0 +1,41 @@
|
||||
---
|
||||
title: WindowsADPendingReplication
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
Pending Active Directory replication operations exceed the configured threshold
|
||||
for 15 minutes. The default is 50 operations. This alert is generated only with
|
||||
`enableActiveDirectory: true` and requires the `ad` collector on domain controllers.
|
||||
|
||||
## Impact
|
||||
|
||||
Directory changes may be delayed between domain controllers, causing inconsistent
|
||||
authentication, group membership, DNS, or policy behavior.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows_ad_replication_pending_operations` and determine whether the
|
||||
queue is growing or draining.
|
||||
2. On a domain controller, inspect replication health:
|
||||
|
||||
```powershell
|
||||
repadmin /replsummary
|
||||
repadmin /showrepl
|
||||
repadmin /queue
|
||||
dcdiag /test:Replications
|
||||
```
|
||||
|
||||
3. Check the Directory Service event log, partner reachability, DNS resolution,
|
||||
site links and schedules, and CPU/disk/network pressure.
|
||||
4. Correlate the queue with [WindowsADReplicationFailures](../windowsadreplicationfailures/)
|
||||
and planned large directory changes.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Restore connectivity and DNS between replication partners. Correct replication
|
||||
errors and review site-link capacity or schedules with the directory owner.
|
||||
Relieve resource pressure on domain controllers. Allow expected bulk changes to
|
||||
drain while monitoring progress. Avoid forcing widespread replication or removing
|
||||
partners before understanding the failure. Confirm the queue decreases and
|
||||
replication health checks succeed.
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
title: WindowsADReplicationFailures
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
The rate of replication synchronization requests minus successful requests
|
||||
exceeds the configured threshold for 15 minutes. The default is any sustained
|
||||
positive failure rate. This alert is opt-in and requires the `ad` collector.
|
||||
|
||||
## Impact
|
||||
|
||||
Failed replication may leave domain controllers with stale directory data,
|
||||
affecting authentication, DNS, policies, and directory changes.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect rates of `windows_ad_replication_sync_requests_total` and
|
||||
`windows_ad_replication_sync_requests_success_total` on the affected controller.
|
||||
Counter resets around a restart may temporarily distort their difference.
|
||||
2. Use the Active Directory diagnostic tools to identify partners and error codes:
|
||||
|
||||
```powershell
|
||||
repadmin /replsummary
|
||||
repadmin /showrepl
|
||||
dcdiag /test:Replications /test:DNS
|
||||
```
|
||||
|
||||
3. Check Directory Service events, DNS resolution, RPC connectivity, replication
|
||||
permissions, and clock synchronization on both partners.
|
||||
4. Check for an offline or decommissioned controller and review recent topology,
|
||||
firewall, or credential changes.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Resolve the specific error reported by replication diagnostics. Restore partner
|
||||
connectivity, DNS, time synchronization, or permissions as appropriate. Handle
|
||||
retired domain controllers through the documented Active Directory removal
|
||||
process. Coordinate topology and recovery changes with the directory owner.
|
||||
Confirm replication succeeds and the pending queue drains.
|
||||
@@ -0,0 +1,43 @@
|
||||
---
|
||||
title: WindowsClockNotSynchronising
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
Windows Time reports no active NTP time source for ten minutes. This alert is
|
||||
generated only with `enableTime: true` and requires the `time` collector and its
|
||||
`ntp` subcollector. Enable it on hosts expected to synchronize through NTP;
|
||||
hosts intentionally using another clock source may legitimately report zero.
|
||||
|
||||
## Impact
|
||||
|
||||
The clock can drift without a working time source. Clock errors can affect
|
||||
Kerberos authentication, TLS validation, scheduled work, and event timestamps.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows_time_ntp_client_time_sources` and the exporter collector
|
||||
success metric. Missing metrics indicate a different problem than a zero value.
|
||||
2. On the affected host, inspect Windows Time:
|
||||
|
||||
```powershell
|
||||
Get-Service W32Time
|
||||
w32tm /query /status
|
||||
w32tm /query /source
|
||||
w32tm /query /peers
|
||||
w32tm /query /configuration
|
||||
```
|
||||
|
||||
3. Check the Windows Time-Service event log, source DNS resolution, and network
|
||||
access to the source over UDP port 123.
|
||||
4. For domain members, inspect the domain time hierarchy and the PDC emulator's
|
||||
upstream source. For virtual machines, check hypervisor time integration.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Restore the expected Windows Time service, configured source, and UDP reachability.
|
||||
Respect domain policy; configure external peers on the appropriate authoritative
|
||||
server rather than overriding every domain member. After correcting the cause,
|
||||
request rediscovery with `w32tm /resync /rediscover` and verify a valid source and
|
||||
successful synchronization. Follow [WindowsClockSkewDetected](../windowsclockskewdetected/)
|
||||
if a significant offset remains.
|
||||
@@ -0,0 +1,43 @@
|
||||
---
|
||||
title: WindowsClockSkewDetected
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
The absolute clock offset reported by Windows Time exceeds the configured
|
||||
threshold for ten minutes. The default is 0.05 seconds (50 ms). This opt-in alert
|
||||
requires the `time` collector with its `ntp` subcollector. The offset is relative
|
||||
to the selected source and does not independently validate that source's accuracy.
|
||||
|
||||
## Impact
|
||||
|
||||
Clock skew can affect authentication, certificate validity, scheduled work, and
|
||||
the ordering of events across hosts. The default threshold is an early warning;
|
||||
application tolerances vary.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows_time_computed_time_offset_seconds`, source count, and
|
||||
`windows_time_ntp_round_trip_delay_seconds` for the affected host.
|
||||
2. Check synchronization state and the selected source:
|
||||
|
||||
```powershell
|
||||
w32tm /query /status
|
||||
w32tm /query /source
|
||||
w32tm /query /configuration
|
||||
```
|
||||
|
||||
3. Compare with a trusted source using
|
||||
`w32tm /stripchart /computer:<trusted-time-server> /samples:5 /dataonly`.
|
||||
Check source health, network delay, Windows Time-Service events, and domain
|
||||
or hypervisor time configuration.
|
||||
4. Consult [WindowsClockNotSynchronising](../windowsclocknotsynchronising/) when
|
||||
no NTP source is available.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Restore a healthy source and correct time configuration before requesting
|
||||
`w32tm /resync /rediscover`. Coordinate large clock corrections with application
|
||||
owners because abrupt changes can disrupt running workloads. Avoid setting the
|
||||
clock manually as a recurring fix. Confirm the reported offset decreases and
|
||||
remains within the application's required tolerance.
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
title: WindowsCollectorFailed
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
The collector identified by the `collector` label has reported
|
||||
`windows_exporter_collector_success == 0` for five minutes.
|
||||
|
||||
## Impact
|
||||
|
||||
Metrics from this collector are incomplete or missing. Alerts relying on those
|
||||
metrics may stop evaluating even though the exporter remains reachable.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Find the failing collector and compare the affected hosts:
|
||||
|
||||
```promql
|
||||
windows_exporter_collector_success{job="windows_exporter"} == 0
|
||||
```
|
||||
|
||||
2. Review exporter logs for that collector's error and duration. Check whether
|
||||
the failure started after an exporter upgrade, Windows update, or account
|
||||
permission change.
|
||||
3. Check the collector's documentation for supported Windows roles, required
|
||||
privileges, and performance counters. Confirm the required service or role
|
||||
exists and is running on the host.
|
||||
4. Compare `windows_exporter_collector_duration_seconds` with the scrape timeout.
|
||||
Test the relevant performance counters locally with Windows Performance
|
||||
Monitor.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Restore the required service, privileges, or performance counter availability.
|
||||
Limit expensive collector queries when collection exceeds the timeout, and
|
||||
adjust timeouts only after measuring normal collection time. Disable a collector
|
||||
on hosts where its role is absent. If the problem began with an upgrade, check
|
||||
release notes and apply a fix or roll back through the normal change process.
|
||||
Confirm the collector succeeds and its dependent metrics resume.
|
||||
@@ -0,0 +1,37 @@
|
||||
---
|
||||
title: WindowsCPUHighUsage
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
Average non-idle CPU usage across logical processors exceeds the configured
|
||||
threshold for 15 minutes. The default is 90%, measured over a five-minute rate
|
||||
window. This measures CPU time; Task Manager's frequency-adjusted utility can
|
||||
show a different value.
|
||||
|
||||
## Impact
|
||||
|
||||
The host has little CPU capacity for additional work. Applications may show
|
||||
increased latency, timeouts, or reduced throughput.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows:cpu_usage:ratio` for the affected host and compare it with
|
||||
its workload and normal baseline.
|
||||
2. Inspect `windows:cpu_time:ratio_rate` by `mode`. High user time points toward
|
||||
application work; high privileged, interrupt, or DPC time can indicate kernel
|
||||
or driver activity. DPC and interrupt modes overlap privileged time; do not
|
||||
sum all modes to estimate usage.
|
||||
3. Use Task Manager, Resource Monitor, or Performance Monitor to identify the
|
||||
busy processes. Compare individual core usage to detect a single-threaded
|
||||
bottleneck.
|
||||
4. Check scheduled tasks, antivirus scans, recent deployments, and hypervisor
|
||||
CPU contention.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Reduce or redistribute the workload, reschedule expensive background tasks, or
|
||||
add CPU capacity. Investigate a process or driver consuming unexpected CPU
|
||||
before restarting it. Confirm CPU usage and application latency return to normal.
|
||||
For expected sustained workloads, tune the threshold against service objectives
|
||||
and available capacity.
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
title: WindowsDiskAlmostFull
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
The volume's free space remains below a configured threshold for 15 minutes.
|
||||
Defaults are 10% for warning and 5% for critical. Both severity levels fire below
|
||||
the critical threshold unless Alertmanager inhibition suppresses the warning.
|
||||
Windows can delay the underlying space counters by 10–15 minutes.
|
||||
|
||||
## Impact
|
||||
|
||||
Applications may fail to write data or logs. Low free space can prevent updates,
|
||||
pagefile growth, database operations, and normal operating system behavior.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Use the `volume` label to identify the affected filesystem and inspect
|
||||
`windows:logical_disk_free:ratio` and `windows_logical_disk_free_bytes`.
|
||||
2. Verify current capacity locally; the exporter counters may lag:
|
||||
|
||||
```powershell
|
||||
Get-Volume | Select-Object DriveLetter, FileSystemLabel, SizeRemaining, Size
|
||||
```
|
||||
|
||||
3. Identify growing logs, temporary files, backups, database files, and application
|
||||
data. Check retention policies and recent jobs.
|
||||
4. Consult [WindowsDiskFillingUp](../windowsdiskfillingup/) when free space is
|
||||
declining rapidly.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Expand the volume or remove files through the owning application's supported
|
||||
retention and cleanup process. Preserve required data and backups; avoid manually
|
||||
deleting database files or Windows system files. Reduce incoming writes if the
|
||||
volume may fill before capacity is restored. Verify space locally and allow the
|
||||
Windows counters to refresh before expecting the alert to resolve.
|
||||
@@ -0,0 +1,37 @@
|
||||
---
|
||||
title: WindowsDiskFillingUp
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
Free space is below the warning threshold and a linear forecast predicts the
|
||||
volume will fill within the configured horizon. Defaults use six hours of
|
||||
history, a 24-hour prediction horizon, and a one-hour alert duration.
|
||||
|
||||
## Impact
|
||||
|
||||
Continued growth can exhaust the volume before routine capacity maintenance,
|
||||
causing write failures or an application outage.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect the free-space trend for the volume and verify local free space with
|
||||
`Get-Volume`. Windows disk space counters can lag by 10–15 minutes.
|
||||
2. Inspect the forecast using the configured selector and window, for example:
|
||||
|
||||
```promql
|
||||
predict_linear(windows_logical_disk_free_bytes{job="windows_exporter", instance="host:9182", volume="C:"}[6h], 24 * 3600)
|
||||
```
|
||||
|
||||
3. Identify the application, logs, backups, or temporary data causing growth.
|
||||
Check whether a recent bulk write distorted the trend or growth is sustained.
|
||||
4. Compare absolute remaining bytes with the application's write rate. A linear
|
||||
forecast is an estimate and does not model scheduled cleanup or sudden growth.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Restore retention and rotation, reduce write growth, or expand the volume before
|
||||
it fills. Coordinate data cleanup with the owner. If the forecast reflects a
|
||||
known temporary job, verify its end time and remaining capacity before silencing
|
||||
the alert. Follow [WindowsDiskAlmostFull](../windowsdiskalmostfull/) when free
|
||||
space is already critically low.
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
title: WindowsDiskReadLatencyHigh
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
Average disk read service time exceeds the configured threshold for 15 minutes.
|
||||
The default is 0.05 seconds (50 ms) per operation, computed from read-time and
|
||||
read-operation counter rates. Volumes with no reads produce no latency sample.
|
||||
|
||||
## Impact
|
||||
|
||||
Slow storage reads may delay application requests, database queries, and paging.
|
||||
The recording is an average, so it does not expose tail latency.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows:logical_disk_read_latency:seconds` for the affected `volume`
|
||||
alongside read IOPS, read throughput, and `windows:logical_disk_busy:ratio`.
|
||||
2. Inspect `windows_logical_disk_requests_queued` and use Resource Monitor to
|
||||
identify processes generating disk reads. Look for backup, antivirus, and
|
||||
indexing jobs.
|
||||
3. Check System event logs and storage-controller logs for disk, controller,
|
||||
multipath, or network-storage errors.
|
||||
4. For virtual machines, check datastore latency and competing workloads on the
|
||||
hypervisor or storage array. Low IOPS can make averages noisy.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Reduce competing reads or reschedule background jobs. Address storage-path errors
|
||||
and provision enough IOPS and throughput for the workload. Investigate excessive
|
||||
paging if memory pressure is contributing. Tune the latency threshold to the
|
||||
storage tier and application's service objectives, then confirm both service
|
||||
latency and disk read latency improve.
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
title: WindowsDiskWriteLatencyHigh
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
Average disk write service time exceeds the configured threshold for 15 minutes.
|
||||
The default is 0.05 seconds (50 ms) per operation, computed from write-time and
|
||||
write-operation counter rates. Volumes with no writes produce no latency sample.
|
||||
|
||||
## Impact
|
||||
|
||||
Slow writes may delay transactions, log flushing, file uploads, or paging. An
|
||||
average can conceal short periods of very high latency.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows:logical_disk_write_latency:seconds`, write IOPS, write
|
||||
throughput, `windows:logical_disk_busy:ratio`, and queued requests for the
|
||||
affected `volume`.
|
||||
2. Use Resource Monitor to find processes generating writes. Review backup jobs,
|
||||
database checkpoints, log growth, and scheduled maintenance.
|
||||
3. Check free space and storage-controller health. Inspect System event logs for
|
||||
disk, controller, multipath, or network-storage errors.
|
||||
4. For virtual machines or remote storage, inspect the backing datastore and
|
||||
array for throughput limits and contention.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Reduce competing writes or move maintenance jobs. Repair storage-path problems
|
||||
and expand IOPS or throughput capacity. Coordinate database or logging changes
|
||||
with the application owner. Avoid disabling durability guarantees to silence an
|
||||
alert. Confirm application write latency recovers and the disk metrics improve.
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
title: WindowsExporterDown
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
Prometheus has failed to scrape the target for five minutes (`up == 0`).
|
||||
The alert does not detect hosts removed from service discovery.
|
||||
|
||||
## Impact
|
||||
|
||||
Monitoring data and resource alerts for the host are unavailable. The host,
|
||||
exporter service, or the network path may be down.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Open the target in Prometheus's **Targets** page and read the scrape error.
|
||||
Check the target address, TLS settings, authentication, and scrape timeout.
|
||||
2. From the Prometheus network, check the configured metrics endpoint, normally
|
||||
`http://<host>:9182/metrics`. Check DNS resolution and routing.
|
||||
3. On the affected host, inspect the service and listening socket:
|
||||
|
||||
```powershell
|
||||
Get-Service windows_exporter
|
||||
Get-NetTCPConnection -LocalPort 9182 -State Listen
|
||||
Test-NetConnection -ComputerName localhost -Port 9182
|
||||
```
|
||||
|
||||
4. Review exporter logs and the Windows Application/System event logs for
|
||||
startup errors, permission problems, or collector timeouts.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Restore the network path or correct the scrape configuration. If the exporter
|
||||
service is stopped, resolve its startup error and start it with
|
||||
`Start-Service windows_exporter`. If local requests succeed but remote requests
|
||||
fail, check the host firewall and network ACLs for the configured listener.
|
||||
Confirm the target becomes healthy and fresh metrics appear before closing the
|
||||
incident. Update service discovery when a host was intentionally decommissioned.
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
title: WindowsMemoryCommitHighUsage
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
Committed memory exceeds the configured percentage of the system commit limit
|
||||
for 15 minutes. The default threshold is 90%. The commit limit depends on
|
||||
physical memory and pagefile capacity.
|
||||
|
||||
## Impact
|
||||
|
||||
The system may reject new memory commitments, causing application failures even
|
||||
when some physical memory remains available.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows:memory_committed:ratio`, `windows_memory_committed_bytes`,
|
||||
and `windows_memory_commit_limit`. Determine whether committed bytes increased
|
||||
or the limit decreased.
|
||||
2. In Task Manager or Resource Monitor, inspect per-process commit size and look
|
||||
for persistent growth. Check recent workload and deployment changes.
|
||||
3. Inspect pagefile configuration and the free space on its volumes:
|
||||
|
||||
```powershell
|
||||
Get-CimInstance Win32_PageFileUsage
|
||||
Get-CimInstance Win32_PageFileSetting
|
||||
Get-Volume
|
||||
```
|
||||
|
||||
4. Check physical memory pressure and paging alongside this alert. The recording
|
||||
rule omits invalid zero commit limits rather than reporting infinity.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Reduce memory commitments or address the leaking process. Restore pagefile
|
||||
capacity and disk free space when the commit limit is unexpectedly low. Review
|
||||
pagefile sizing with the workload owner; some changes require a reboot. Add
|
||||
physical memory for sustained demand. Confirm committed bytes remain safely
|
||||
below the commit limit and applications can allocate memory normally.
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
title: WindowsMemoryHighUsage
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
Physical memory usage exceeds the configured threshold for 15 minutes. The
|
||||
default is 90%. Usage is total physical memory minus available memory; available
|
||||
memory includes reclaimable pages and differs from strictly free memory.
|
||||
|
||||
## Impact
|
||||
|
||||
Memory pressure may cause paging, application latency, and allocation failures.
|
||||
A high ratio alone does not prove the system is paging heavily.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows:memory_usage:ratio`, `windows:memory_used:bytes`, and
|
||||
`windows_memory_available_bytes` on the affected host.
|
||||
2. Check `windows:memory_swap_page_operations:rate`, disk latency, and
|
||||
`windows:memory_committed:ratio` for related pressure.
|
||||
3. In Resource Monitor, inspect processes' working sets, commit usage, and hard
|
||||
faults. Look for growth after a deployment or an unexpected workload increase.
|
||||
4. Inspect `windows_memory_pool_nonpaged_bytes` and
|
||||
`windows_memory_pool_paged_bytes` if application working sets do not explain
|
||||
the usage. Driver or kernel pools may be growing.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Reduce concurrent work, move workloads, or add physical memory. Investigate leaks
|
||||
in the responsible application or driver. Restart a service only after assessing
|
||||
its availability impact and preserving useful diagnostics. Pagefile changes
|
||||
affect the commit limit and do not add physical memory. Confirm available memory
|
||||
and service latency recover.
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
title: WindowsNetworkDiscards
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
The sum of inbound and outbound discarded packet rates exceeds the configured
|
||||
threshold for 15 minutes. The default is one discard per second per interface.
|
||||
Discards count packets dropped without a packet error, such as buffer exhaustion.
|
||||
|
||||
## Impact
|
||||
|
||||
Dropped packets can cause retransmissions, reduced throughput, and application
|
||||
timeouts. Some drops may be expected under a deliberate traffic policy.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows:net_received_discarded:rate` and
|
||||
`windows:net_outbound_discarded:rate` to identify the direction and affected
|
||||
`nic`.
|
||||
2. Compare traffic, directional utilization ratios, and
|
||||
`windows_net_output_queue_length_packets` with the normal baseline.
|
||||
3. Inspect `Get-NetAdapterStatistics`, switch-port drops, adapter buffers,
|
||||
virtual switch counters, and network QoS policies.
|
||||
4. Correlate drops with CPU pressure, traffic bursts, hypervisor contention, and
|
||||
application failures. An unknown or zero bandwidth estimate produces no
|
||||
utilization sample and should not be interpreted as zero traffic.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Relieve congestion, distribute traffic, or add bandwidth. Correct buffer, driver,
|
||||
or virtual switch problems when evidence identifies them. Review intentional
|
||||
drop policies with the network owner before changing thresholds. Confirm discard
|
||||
rates and application retransmissions return to the expected baseline.
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
title: WindowsNetworkErrors
|
||||
---
|
||||
|
||||
## Meaning
|
||||
|
||||
The sum of inbound and outbound packet error rates exceeds the configured
|
||||
threshold for 15 minutes. The default is one error per second per interface.
|
||||
The alert identifies the interface using its `nic` label.
|
||||
|
||||
## Impact
|
||||
|
||||
Packet errors may cause retransmissions, lower throughput, and intermittent
|
||||
connection failures.
|
||||
|
||||
## Diagnosis
|
||||
|
||||
1. Inspect `windows:net_received_errors:rate` and
|
||||
`windows:net_outbound_errors:rate` separately to identify the direction.
|
||||
2. Compare traffic and `windows:net_received_utilization:ratio` and
|
||||
`windows:net_sent_utilization:ratio` with the normal baseline. These ratios
|
||||
use bandwidth in bytes per second and keep each direction separate.
|
||||
3. Inspect local adapter state and counters:
|
||||
|
||||
```powershell
|
||||
Get-NetAdapter
|
||||
Get-NetAdapterStatistics
|
||||
```
|
||||
|
||||
4. Inspect switch-port counters, cabling, link negotiation, NIC drivers, and
|
||||
System event logs. For virtual NICs, inspect the host and virtual switch.
|
||||
|
||||
## Mitigation
|
||||
|
||||
Repair faulty links or switch ports and correct driver or adapter configuration
|
||||
problems. Apply driver and firmware fixes through the normal change process.
|
||||
Relieve congestion if it contributes. Verify errors stop increasing and affected
|
||||
application connections recover; cumulative counters do not need to return to
|
||||
zero for the rate alert to resolve.
|
||||
Vendored
+17
@@ -0,0 +1,17 @@
|
||||
:root {
|
||||
color-scheme: light dark;
|
||||
font-family: system-ui, sans-serif;
|
||||
line-height: 1.6;
|
||||
color: light-dark(#20252b, #e5e9ef);
|
||||
background: light-dark(#fff, #15191f);
|
||||
}
|
||||
body { max-width: 64rem; margin: auto; padding: 1.5rem; }
|
||||
header { display: flex; justify-content: space-between; gap: 1rem; border-bottom: 1px solid #8993a0; padding-bottom: 1rem; }
|
||||
main { max-width: 54rem; margin: 2rem auto; }
|
||||
a { color: light-dark(#0059b3, #80bfff); overflow-wrap: anywhere; }
|
||||
h1, h2 { line-height: 1.3; overflow-wrap: anywhere; }
|
||||
h2 { margin-top: 2rem; }
|
||||
pre { overflow-x: auto; padding: 1rem; border: 1px solid #8993a0; border-radius: .3rem; }
|
||||
code { font-size: .95em; }
|
||||
li { margin: .4rem 0; }
|
||||
.runbooks { padding-left: 1.5rem; }
|
||||
Reference in New Issue
Block a user