Anpassung von Prometheus, Grafana und Backend auf Anomalieerkennung.
All checks were successful
release-tag / release-image (push) Successful in 2m20s

This commit is contained in:
2026-04-24 21:38:25 +02:00
parent cdee259fb1
commit 3447af8d44
5 changed files with 1502 additions and 32 deletions

View File

@@ -3,74 +3,743 @@
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"panels": [
{
"type": "stat",
"title": "Active Agents",
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
"targets": [
{
"expr": "eventcollector_active_agents",
"refId": "A"
}
]
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "green", "value": 1 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto"
}
},
{
"type": "stat",
"title": "High Detections (5m)",
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
"title": "Events/s",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
"targets": [
{
"expr": "increase(eventcollector_detection_hits_total{severity=\"high\"}[5m])",
"expr": "sum(rate(eventcollector_ingest_events_total{channel=~\"$channel\",event_id=~\"$event_id\"}[5m]))",
"refId": "A"
}
]
],
"fieldConfig": {
"defaults": {
"unit": "eps",
"decimals": 2
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "stat",
"title": "High Detections 5m",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
"targets": [
{
"expr": "sum(increase(eventcollector_detection_hits_total{severity=\"high\",rule=~\"$rule\"}[5m]))",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "stat",
"title": "Baseline Max Z-Score",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
"targets": [
{
"expr": "max(eventcollector_anomaly_score{host=~\"$host\",rule=\"baseline_event_rate_anomaly\"})",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 3 },
{ "color": "red", "value": 5 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "stat",
"title": "Rule Errors 5m",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
"targets": [
{
"expr": "sum(increase(eventcollector_rule_errors_total{rule=~\"$rule\"}[5m]))",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "stat",
"title": "DB Insert Failures 5m",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
"targets": [
{
"expr": "increase(eventcollector_db_insert_failures_total[5m])",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "timeseries",
"title": "HTTP Requests",
"title": "Ingested Events / Second by Channel",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
"targets": [
{
"expr": "rate(eventcollector_http_requests_total[5m])",
"legendFormat": "{{path}} {{status}}",
"expr": "sum by (channel) (rate(eventcollector_ingest_events_total{channel=~\"$channel\",event_id=~\"$event_id\"}[5m]))",
"legendFormat": "{{channel}}",
"refId": "A"
}
]
],
"fieldConfig": {
"defaults": {
"unit": "eps",
"decimals": 2
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Detection Hits",
"title": "Detection Hits by Rule / Severity",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
"targets": [
{
"expr": "increase(eventcollector_detection_hits_total[5m])",
"legendFormat": "{{rule}} {{severity}}",
"expr": "sum by (rule,severity) (increase(eventcollector_detection_hits_total{rule=~\"$rule\",severity=~\"$severity\"}[5m]))",
"legendFormat": "{{rule}} / {{severity}}",
"refId": "A"
}
]
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Baseline: Current Count vs Average",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
"targets": [
{
"expr": "eventcollector_baseline_current_count{host=~\"$host\",channel=~\"$channel\",event_id=~\"$event_id\"}",
"legendFormat": "current {{host}} {{channel}} {{event_id}}",
"refId": "A"
},
{
"expr": "eventcollector_baseline_avg_count{host=~\"$host\",channel=~\"$channel\",event_id=~\"$event_id\"}",
"legendFormat": "avg {{host}} {{channel}} {{event_id}}",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 2
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Ingested Events",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 12 },
"title": "Baseline Z-Score",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
"targets": [
{
"expr": "rate(eventcollector_ingest_events_total[5m])",
"legendFormat": "{{channel}} {{event_id}}",
"expr": "eventcollector_anomaly_score{host=~\"$host\",rule=\"baseline_event_rate_anomaly\"}",
"legendFormat": "{{host}}",
"refId": "A"
}
]
],
"fieldConfig": {
"defaults": {
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 3 },
{ "color": "red", "value": 5 }
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "bargauge",
"title": "Top Baseline Z-Scores",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 20 },
"targets": [
{
"expr": "topk(10, eventcollector_anomaly_score{host=~\"$host\",rule=\"baseline_event_rate_anomaly\"})",
"legendFormat": "{{host}}",
"refId": "A",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 3 },
{ "color": "red", "value": 5 }
]
}
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showUnfilled": true
}
},
{
"type": "bargauge",
"title": "Top EventIDs by Ingest Rate",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 20 },
"targets": [
{
"expr": "topk(15, sum by (channel,event_id) (rate(eventcollector_ingest_events_total{channel=~\"$channel\",event_id=~\"$event_id\"}[5m])))",
"legendFormat": "{{channel}} / {{event_id}}",
"refId": "A",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "eps",
"decimals": 2
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showUnfilled": true
}
},
{
"type": "bargauge",
"title": "Top Detection Rules 1h",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 20 },
"targets": [
{
"expr": "topk(15, sum by (rule,severity) (increase(eventcollector_detection_hits_total{rule=~\"$rule\",severity=~\"$severity\"}[1h])))",
"legendFormat": "{{rule}} / {{severity}}",
"refId": "A",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showUnfilled": true
}
},
{
"type": "timeseries",
"title": "HTTP Requests by Path / Status",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
"targets": [
{
"expr": "sum by (path,status) (rate(eventcollector_http_requests_total[5m]))",
"legendFormat": "{{path}} {{status}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"decimals": 2
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "HTTP Latency p95",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le,path) (rate(eventcollector_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "{{path}} p95",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 3
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "DB Insert Transaction Latency p95",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 },
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(eventcollector_db_tx_duration_seconds_bucket[5m])))",
"legendFormat": "db tx p95",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 3
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "DB Batch Size p95",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 },
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(eventcollector_db_batch_size_bucket[5m])))",
"legendFormat": "batch size p95",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "table",
"title": "Agent Last Seen",
"datasource": "$datasource",
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 44 },
"targets": [
{
"expr": "time() - eventcollector_agent_last_seen_unixtime{host=~\"$host\"}",
"legendFormat": "{{host}}",
"refId": "A",
"instant": true,
"format": "table"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0
},
"overrides": []
},
"options": {
"showHeader": true
}
},
{
"type": "table",
"title": "Baseline Samples",
"datasource": "$datasource",
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 44 },
"targets": [
{
"expr": "eventcollector_baseline_sample_count{host=~\"$host\",channel=~\"$channel\",event_id=~\"$event_id\"}",
"legendFormat": "{{host}} {{channel}} {{event_id}}",
"refId": "A",
"instant": true,
"format": "table"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"showHeader": true
}
}
],
"refresh": "30s",
"schemaVersion": 39,
"style": "dark",
"tags": ["siem"],
"templating": { "list": [] },
"tags": ["siem", "baseline", "ad"],
"templating": {
"list": [
{
"name": "datasource",
"type": "datasource",
"query": "prometheus",
"current": {},
"hide": 0,
"label": "Datasource"
},
{
"name": "host",
"type": "query",
"datasource": "$datasource",
"query": "label_values(eventcollector_agent_last_seen_unixtime, host)",
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Host"
},
{
"name": "channel",
"type": "query",
"datasource": "$datasource",
"query": "label_values(eventcollector_ingest_events_total, channel)",
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Channel"
},
{
"name": "event_id",
"type": "query",
"datasource": "$datasource",
"query": "label_values(eventcollector_ingest_events_total, event_id)",
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Event ID"
},
{
"name": "rule",
"type": "query",
"datasource": "$datasource",
"query": "label_values(eventcollector_detection_hits_total, rule)",
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Rule"
},
{
"name": "severity",
"type": "custom",
"query": "low,medium,high",
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Severity"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"title": "SIEM Overview",
"timezone": "browser",
"title": "SIEM Overview Extended",
"uid": "siem-overview-extended",
"version": 1
}

View File

@@ -89,14 +89,14 @@ CREATE TABLE IF NOT EXISTS detections (
CREATE TABLE detection_rules (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(128) NOT NULL UNIQUE,
name VARCHAR(255) NOT NULL UNIQUE,
description TEXT,
severity VARCHAR(16) NOT NULL DEFAULT 'medium',
channel VARCHAR(64) NOT NULL DEFAULT 'Security',
channel VARCHAR(255) NOT NULL DEFAULT 'Security',
event_ids VARCHAR(255) NOT NULL,
match_field VARCHAR(64) DEFAULT '',
match_field VARCHAR(255) DEFAULT '',
match_operator VARCHAR(16) DEFAULT '',
match_value TEXT,
@@ -1312,4 +1312,42 @@ ALTER TABLE detection_rules
MODIFY description TEXT NULL,
MODIFY match_value TEXT NULL,
MODIFY match_field VARCHAR(64) NOT NULL DEFAULT '',
MODIFY match_operator VARCHAR(16) NOT NULL DEFAULT '';
MODIFY match_operator VARCHAR(16) NOT NULL DEFAULT '';
CREATE TABLE baseline_event_stats (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
hostname VARCHAR(255) NOT NULL,
channel_name VARCHAR(255) NOT NULL,
event_id INT NOT NULL,
hour_of_day TINYINT NOT NULL,
day_of_week TINYINT NOT NULL,
avg_count DOUBLE NOT NULL DEFAULT 0,
m2_count DOUBLE NOT NULL DEFAULT 0,
stddev_count DOUBLE NOT NULL DEFAULT 0,
sample_count INT NOT NULL DEFAULT 0,
last_updated TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
UNIQUE KEY uniq_baseline_event (
hostname,
channel_name,
event_id,
hour_of_day,
day_of_week
)
);
CREATE INDEX idx_baseline_event_lookup
ON baseline_event_stats (
hostname,
channel_name,
event_id,
hour_of_day,
day_of_week,
sample_count
);

View File

@@ -1,5 +1,5 @@
groups:
- name: siem-backend
- name: siem-backend-availability
rules:
- alert: SiemBackendDown
expr: up{job="siem-backend"} == 0
@@ -10,6 +10,26 @@ groups:
summary: "SIEM backend nicht erreichbar"
description: "Prometheus kann das SIEM-Backend seit mindestens 2 Minuten nicht scrapen."
- alert: SiemNoIngestEvents
expr: sum(rate(eventcollector_ingest_events_total[15m])) == 0
for: 15m
labels:
severity: warning
annotations:
summary: "Keine eingehenden SIEM Events"
description: "Seit mindestens 15 Minuten wurden keine Events mehr ingestiert."
- alert: SiemTooFewActiveAgents
expr: eventcollector_active_agents < 1
for: 5m
labels:
severity: warning
annotations:
summary: "Zu wenige aktive Agents"
description: "Es wurden weniger aktive Agents erkannt als erwartet."
- name: siem-backend-detections
rules:
- alert: SiemHighDetections
expr: increase(eventcollector_detection_hits_total{severity="high"}[5m]) > 0
for: 1m
@@ -19,6 +39,33 @@ groups:
summary: "Neue High-Severity Detection"
description: "Es wurde mindestens eine neue High-Severity-Detection in den letzten 5 Minuten erzeugt."
- alert: SiemManyMediumDetections
expr: sum(increase(eventcollector_detection_hits_total{severity="medium"}[15m])) > 10
for: 2m
labels:
severity: warning
annotations:
summary: "Viele Medium-Detections"
description: "Es wurden mehr als 10 Medium-Detections in 15 Minuten erzeugt."
- alert: SiemBaselineHighAnomaly
expr: eventcollector_anomaly_score{rule="baseline_event_rate_anomaly"} >= 5
for: 2m
labels:
severity: high
annotations:
summary: "Hohe Baseline-Anomalie"
description: "Host {{ $labels.host }} hat einen hohen Baseline-Z-Score: {{ $value }}."
- alert: SiemBaselineMediumAnomaly
expr: eventcollector_anomaly_score{rule="baseline_event_rate_anomaly"} >= 3
for: 5m
labels:
severity: warning
annotations:
summary: "Baseline-Anomalie"
description: "Host {{ $labels.host }} hat einen erhöhten Baseline-Z-Score: {{ $value }}."
- alert: SiemRuleErrors
expr: increase(eventcollector_rule_errors_total[5m]) > 0
for: 1m
@@ -28,11 +75,51 @@ groups:
summary: "Fehler in Detection-Regeln"
description: "Mindestens eine Detection-Regel hat in den letzten 5 Minuten einen Fehler erzeugt."
- alert: SiemTooFewActiveAgents
expr: eventcollector_active_agents < 1
- name: siem-backend-ingest
rules:
- alert: SiemIngestRejected
expr: sum(increase(eventcollector_ingest_rejected_total[5m])) > 0
for: 1m
labels:
severity: warning
annotations:
summary: "Ingest Requests abgelehnt"
description: "In den letzten 5 Minuten wurden Ingest Requests abgelehnt."
- alert: SiemDBInsertFailures
expr: increase(eventcollector_db_insert_failures_total[5m]) > 0
for: 1m
labels:
severity: high
annotations:
summary: "DB Insert Fehler"
description: "Das SIEM-Backend konnte Events nicht in die Datenbank schreiben."
- alert: SiemHighIngestRate
expr: sum(rate(eventcollector_ingest_events_total[5m])) > 500
for: 5m
labels:
severity: warning
annotations:
summary: "Zu wenige aktive Agents"
description: "Es wurden weniger aktive Agents erkannt als erwartet."
summary: "Sehr hohe Eventrate"
description: "Die Eventrate liegt seit 5 Minuten über 500 Events/s."
- name: siem-backend-baseline
rules:
- alert: SiemBaselineNotEnoughSamples
expr: eventcollector_baseline_sample_count > 0 and eventcollector_baseline_sample_count < 24
for: 30m
labels:
severity: info
annotations:
summary: "Baseline lernt noch"
description: "Für {{ $labels.host }} / {{ $labels.channel }} / {{ $labels.event_id }} gibt es erst {{ $value }} Samples."
- alert: SiemBaselineCurrentFarAboveAverage
expr: eventcollector_baseline_avg_count > 0 and (eventcollector_baseline_current_count / eventcollector_baseline_avg_count) > 10
for: 2m
labels:
severity: warning
annotations:
summary: "Eventrate deutlich über Baseline"
description: "{{ $labels.host }} / {{ $labels.channel }} / {{ $labels.event_id }} liegt mehr als 10x über Durchschnitt."