{ local c = self._config, local q = (import '../lib/queries.libsonnet')(c), local alert(name, expr, duration, severity, summary, description) = { alert: name, expr: expr, 'for': duration, labels: { severity: severity }, annotations: { summary: summary, description: description }, }, prometheusAlerts+:: { groups: [ { name: 'windows-exporter-alerts', rules: [ alert( 'WindowsExporterDown', q.metric('up') + ' == 0', '5m', c.criticalSeverity, 'Windows exporter is unreachable', 'Prometheus cannot scrape Windows exporter on {{ $labels.instance }} (job {{ $labels.job }}).' ), alert( 'WindowsCollectorFailed', q.metric('windows_exporter_collector_success') + ' == 0', '5m', 'warning', 'Windows collector failed', 'Collector {{ $labels.collector }} on {{ $labels.instance }} has failed for at least 5 minutes.' ), alert( 'WindowsCPUHighUsage', q.metric('windows:cpu_usage:ratio') + ' * 100 > %s' % c.cpuHighUsageThreshold, '15m', 'warning', 'Windows CPU usage is high', 'CPU usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.cpuHighUsageThreshold ), alert( 'WindowsMemoryHighUsage', q.metric('windows:memory_usage:ratio') + ' * 100 > %s' % c.memoryHighUsageThreshold, '15m', 'warning', 'Windows memory usage is high', 'Physical memory usage on {{ $labels.instance }} has exceeded %s%% for 15 minutes.' % c.memoryHighUsageThreshold ), alert( 'WindowsMemoryCommitHighUsage', q.metric('windows:memory_committed:ratio') + ' * 100 > %s' % c.memoryCommitHighUsageThreshold, '15m', 'warning', 'Windows memory commit usage is high', 'Committed memory on {{ $labels.instance }} has exceeded %s%% of the commit limit for 15 minutes.' % c.memoryCommitHighUsageThreshold ), ] + [ alert( 'WindowsDisk%sLatencyHigh' % direction[0], q.metric('windows:logical_disk_%s_latency:seconds' % direction[1], [c.volumeSelector]) + ' > %g' % c.diskLatencyThresholdSeconds, '15m', 'warning', 'Windows disk %s latency is high' % direction[1], 'Volume {{ $labels.volume }} on {{ $labels.instance }} has averaged more than %g seconds per %s operation for 15 minutes.' % [c.diskLatencyThresholdSeconds, direction[1]] ) for direction in [['Read', 'read'], ['Write', 'write']] ] + [ alert( 'WindowsNetwork%s' % outcome[0], '(%s + %s) > %g' % [ q.metric('windows:net_received_%s:rate' % outcome[1], [c.nicSelector]), q.metric('windows:net_outbound_%s:rate' % outcome[1], [c.nicSelector]), outcome[2], ], '15m', 'warning', 'Windows network packet %s are high' % outcome[3], 'Packet %s on interface {{ $labels.nic }} on {{ $labels.instance }} have exceeded %g per second for 15 minutes.' % [outcome[3], outcome[2]] ) for outcome in [ ['Errors', 'errors', c.networkErrorRateThreshold, 'errors'], ['Discards', 'discarded', c.networkDiscardRateThreshold, 'discards'], ] ] + [ alert( 'WindowsDiskAlmostFull', q.metric('windows:logical_disk_free:ratio', [c.volumeSelector]) + ' * 100 < %s' % entry[1], '15m', entry[0], 'Windows disk space is low', 'Volume {{ $labels.volume }} on {{ $labels.instance }} has less than %s%% free space. Windows may delay space metrics by 10–15 minutes.' % entry[1] ) for entry in [ ['warning', c.diskFreeWarningThreshold], [c.criticalSeverity, c.diskFreeCriticalThreshold], ] ] + [ alert( 'WindowsDiskFillingUp', q.metric('windows:logical_disk_free:ratio', [c.volumeSelector]) + ' * 100 < %s and predict_linear(%s[%s], %s) < 0' % [ c.diskFreeWarningThreshold, q.metric('windows_logical_disk_free_bytes', [c.volumeSelector]), c.diskPredictionWindow, c.diskPredictionHours * 3600, ], '1h', 'warning', 'Windows disk is predicted to fill up', 'Volume {{ $labels.volume }} on {{ $labels.instance }} is predicted to fill within %s hours.' % c.diskPredictionHours ), ], }, ] + (if c.enableActiveDirectory then [ { name: 'windows-active-directory', rules: [ alert( 'WindowsADPendingReplication', q.metric('windows_ad_replication_pending_operations') + ' > %s' % c.adPendingReplicationThreshold, '15m', 'warning', 'Active Directory replication queue is high', 'Pending replication operations on {{ $labels.instance }} have exceeded %s for 15 minutes.' % c.adPendingReplicationThreshold ), alert( 'WindowsADReplicationFailures', '(' + q.rate('windows_ad_replication_sync_requests_total') + ' - ' + q.rate('windows_ad_replication_sync_requests_success_total') + ') > %s' % c.adReplicationFailureRateThreshold, '15m', 'warning', 'Active Directory replication requests are failing', 'Replication synchronization requests on {{ $labels.instance }} have been failing for 15 minutes.' ), ], }, ] else []) + if c.enableTime then [ { name: 'windows-time', rules: [ alert( 'WindowsClockNotSynchronising', q.metric('windows_time_ntp_client_time_sources') + ' == 0', '10m', 'warning', 'Windows clock has no NTP time source', 'Windows Time on {{ $labels.instance }} has had no active NTP time source for 10 minutes.' ), alert( 'WindowsClockSkewDetected', 'abs(%s) > %g' % [q.metric('windows_time_computed_time_offset_seconds'), c.clockOffsetThresholdSeconds], '10m', 'warning', 'Windows clock offset is high', 'Windows Time on {{ $labels.instance }} reports a clock offset above %g seconds for 10 minutes.' % c.clockOffsetThresholdSeconds ), ], }, ] else [], }, }