Update Datenbankproblem

deploy/mariadb/cardinality-diagnostics.sql

@@ -0,0 +1,35 @@
+-- Shows where rows are growing and how effective aggregation currently is.
+SET time_zone = '+00:00';
+
+SELECT table_name,
+       table_rows,
+       ROUND(data_length/1024/1024,1) AS data_mb,
+       ROUND(index_length/1024/1024,1) AS index_mb,
+       ROUND((data_length+index_length)/1024/1024,1) AS total_mb
+FROM information_schema.tables
+WHERE table_schema = DATABASE()
+ORDER BY data_length + index_length DESC;
+
+SELECT 'event_count_buckets' AS source,
+       COUNT(*) AS rows_24h,
+       COALESCE(SUM(cnt),0) AS represented_events_24h,
+       ROUND(COALESCE(SUM(cnt),0) / NULLIF(COUNT(*),0), 1) AS events_per_row
+FROM event_count_buckets
+WHERE bucket_start >= UTC_TIMESTAMP() - INTERVAL 24 HOUR
+UNION ALL
+SELECT 'event_occurrences',
+       COUNT(*),
+       COALESCE(SUM(cnt),0),
+       ROUND(COALESCE(SUM(cnt),0) / NULLIF(COUNT(*),0), 1)
+FROM event_occurrences
+WHERE bucket_start >= UTC_TIMESTAMP() - INTERVAL 24 HOUR;
+
+SELECT channel_name, event_id,
+       COUNT(*) AS context_rows_24h,
+       SUM(cnt) AS represented_events_24h,
+       ROUND(SUM(cnt)/NULLIF(COUNT(*),0),1) AS events_per_row
+FROM event_occurrences
+WHERE bucket_start >= UTC_TIMESTAMP() - INTERVAL 24 HOUR
+GROUP BY channel_name, event_id
+ORDER BY context_rows_24h DESC
+LIMIT 50;

deploy/mariadb/diagnostics.sql

@@ -0,0 +1,52 @@
+-- Read-only diagnostics for sizing and partition verification.
+SET time_zone = '+00:00';
+
+SELECT
+    TABLE_NAME,
+    ENGINE,
+    TABLE_ROWS,
+    ROUND(DATA_LENGTH / 1024 / 1024, 1) AS data_mb,
+    ROUND(INDEX_LENGTH / 1024 / 1024, 1) AS index_mb,
+    ROUND((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024, 1) AS total_mb
+FROM information_schema.TABLES
+WHERE TABLE_SCHEMA = DATABASE()
+ORDER BY (DATA_LENGTH + INDEX_LENGTH) DESC;
+
+SELECT
+    TABLE_NAME,
+    COUNT(*) AS partitions,
+    MIN(PARTITION_NAME) AS oldest_partition,
+    MAX(PARTITION_NAME) AS newest_partition,
+    ROUND(SUM(DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024, 1) AS total_mb
+FROM information_schema.PARTITIONS
+WHERE TABLE_SCHEMA = DATABASE()
+  AND PARTITION_NAME IS NOT NULL
+GROUP BY TABLE_NAME
+ORDER BY total_mb DESC;
+
+SELECT
+    channel_name,
+    event_id,
+    SUM(cnt) AS events_24h,
+    COUNT(*) AS metadata_rows_24h,
+    ROUND(SUM(cnt) / NULLIF(COUNT(*), 0), 2) AS compression_factor
+FROM event_occurrences
+WHERE bucket_start >= UTC_TIMESTAMP() - INTERVAL 24 HOUR
+GROUP BY channel_name, event_id
+ORDER BY events_24h DESC
+LIMIT 50;
+
+SELECT
+    first_event_ts,
+    last_event_ts,
+    cnt,
+    hostname,
+    target_user,
+    workstation,
+    src_ip
+FROM event_occurrences
+WHERE channel_name = 'Security'
+  AND event_id = 4740
+  AND bucket_start >= UTC_TIMESTAMP() - INTERVAL 7 DAY
+ORDER BY bucket_start DESC
+LIMIT 200;

deploy/mariadb/emergency-compact-cleanup.sql

@@ -0,0 +1,43 @@
+-- EMERGENCY CLEANUP FOR THE PREVIOUS METADATA-FIRST BUILD
+--
+-- Run ONLY AFTER the new backend is deployed with:
+--   STORE_EVENT_ROWS=false
+--   STORE_RAW_XML=false
+--
+-- This intentionally removes redundant per-event/full-context history generated
+-- by older builds. Aggregate event history remains in event_count_buckets and
+-- event_catalog. Existing detections and UEBA baselines are not deleted.
+--
+-- Take a database backup/snapshot first if old raw/event detail may still matter.
+
+SET NAMES utf8mb4;
+SET time_zone = '+00:00';
+
+SELECT 'before' AS phase,
+       table_name,
+       table_rows,
+       ROUND((data_length + index_length) / 1024 / 1024, 1) AS size_mb
+FROM information_schema.tables
+WHERE table_schema = DATABASE()
+  AND table_name IN ('event_logs','event_log_raw','event_occurrences','event_count_buckets','event_catalog','detections')
+ORDER BY size_mb DESC;
+
+-- Raw XML must be cleared before its logical parent event rows.
+TRUNCATE TABLE event_log_raw;
+TRUNCATE TABLE event_logs;
+
+-- Old versions used too many dimensions and could make this table nearly as
+-- large as event_logs. The new backend will repopulate only selected Security
+-- contexts in 5-minute buckets.
+TRUNCATE TABLE event_occurrences;
+
+ANALYZE TABLE event_count_buckets, event_catalog, detections;
+
+SELECT 'after' AS phase,
+       table_name,
+       table_rows,
+       ROUND((data_length + index_length) / 1024 / 1024, 1) AS size_mb
+FROM information_schema.tables
+WHERE table_schema = DATABASE()
+  AND table_name IN ('event_logs','event_log_raw','event_occurrences','event_count_buckets','event_catalog','detections')
+ORDER BY size_mb DESC;

deploy/mariadb/init/001-schema.sql

@@ -62,7 +62,7 @@ CREATE TABLE agents (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ---------------------------------------------------------------------
--- Event Logs, normalisierte Haupttabelle
+-- Event Logs, optionaler kurzlebiger Full-Event-Hotstore (STORE_EVENT_ROWS=true)
 -- ---------------------------------------------------------------------
 
 CREATE TABLE event_logs (
@@ -202,8 +202,9 @@ CREATE TABLE event_catalog (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 -- ---------------------------------------------------------------------
--- Langfristige, kompakte Event-Metadaten
--- Eine Zeile pro Zeit-Bucket und Dimensionskombination statt einer Zeile pro XML.
+-- Kompakte Security-Kontext-Metadaten für eine kleine Event-ID-Allowlist.
+-- Alle Events werden universell in event_count_buckets gezählt; diese Tabelle
+-- speichert nur Benutzer/IP/Workstation-Kontext für ausgewählte Security-Events.
 -- ---------------------------------------------------------------------
 
 CREATE TABLE event_occurrences (
@@ -234,7 +235,15 @@ CREATE TABLE event_occurrences (
     KEY idx_occurrences_host_event_time (hostname, channel_name, event_id, bucket_start),
     KEY idx_occurrences_target_user_time (target_user, bucket_start),
     KEY idx_occurrences_subject_user_time (subject_user, bucket_start),
-    KEY idx_occurrences_src_ip_time (src_ip, bucket_start)
+    KEY idx_occurrences_src_ip_time (src_ip, bucket_start),
+    KEY idx_occurrences_security_event_time (
+        channel_name,
+        event_id,
+        bucket_start,
+        hostname,
+        target_user,
+        src_ip
+    )
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
 PARTITION BY RANGE COLUMNS(bucket_start) (
     PARTITION pmax VALUES LESS THAN (MAXVALUE)
@@ -419,6 +428,13 @@ CREATE TABLE event_count_buckets (
     KEY idx_event_count_buckets_time (
         bucket_start,
         bucket_end
+    ),
+
+    KEY idx_event_count_buckets_event_time (
+        channel_name,
+        event_id,
+        bucket_start,
+        hostname
     )
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
 PARTITION BY RANGE COLUMNS(bucket_start) (

deploy/mariadb/migrations/002-metadata-first.sql

@@ -0,0 +1,98 @@
+-- Metadata-first migration for an existing SIEM-lite database.
+-- IMPORTANT: ALTER TABLE ... PARTITION BY can rebuild and lock large tables.
+-- Run during a maintenance window and take a backup first.
+
+SET NAMES utf8mb4;
+SET time_zone = '+00:00';
+
+CREATE TABLE IF NOT EXISTS event_catalog (
+    hostname VARCHAR(191) NOT NULL,
+    channel_name VARCHAR(128) NOT NULL,
+    event_id INT UNSIGNED NOT NULL,
+    first_seen DATETIME(6) NOT NULL,
+    last_seen DATETIME(6) NOT NULL,
+    total_count BIGINT UNSIGNED NOT NULL DEFAULT 0,
+    updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+    PRIMARY KEY (hostname, channel_name, event_id),
+    KEY idx_event_catalog_first_seen (first_seen),
+    KEY idx_event_catalog_last_seen (last_seen)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- Warm the catalog from the much smaller baseline buckets so the
+-- new_event_id rule does not treat every known event type as new after migration.
+INSERT INTO event_catalog
+(hostname, channel_name, event_id, first_seen, last_seen, total_count, updated_at)
+SELECT hostname,
+       channel_name,
+       event_id,
+       MIN(COALESCE(first_event_ts, bucket_start)),
+       MAX(COALESCE(last_event_ts, bucket_end)),
+       SUM(cnt),
+       UTC_TIMESTAMP(6)
+FROM event_count_buckets
+GROUP BY hostname, channel_name, event_id
+ON DUPLICATE KEY UPDATE
+    first_seen = LEAST(first_seen, VALUES(first_seen)),
+    last_seen = GREATEST(last_seen, VALUES(last_seen)),
+    total_count = GREATEST(total_count, VALUES(total_count)),
+    updated_at = UTC_TIMESTAMP(6);
+
+CREATE TABLE IF NOT EXISTS event_occurrences (
+    bucket_start DATETIME(6) NOT NULL,
+    bucket_end DATETIME(6) NOT NULL,
+    dimension_key BINARY(16) NOT NULL,
+    hostname VARCHAR(191) NOT NULL,
+    channel_name VARCHAR(128) NOT NULL,
+    event_id INT UNSIGNED NOT NULL,
+    provider_name VARCHAR(191) NOT NULL DEFAULT '',
+    target_user VARCHAR(191) NOT NULL DEFAULT '',
+    subject_user VARCHAR(191) NOT NULL DEFAULT '',
+    src_ip VARCHAR(64) NOT NULL DEFAULT '',
+    workstation VARCHAR(191) NOT NULL DEFAULT '',
+    logon_type VARCHAR(32) NOT NULL DEFAULT '',
+    status_text VARCHAR(128) NOT NULL DEFAULT '',
+    failure_reason VARCHAR(255) NOT NULL DEFAULT '',
+    cnt BIGINT UNSIGNED NOT NULL DEFAULT 0,
+    first_event_ts DATETIME(6) NOT NULL,
+    last_event_ts DATETIME(6) NOT NULL,
+    updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+    PRIMARY KEY (bucket_start, dimension_key),
+    KEY idx_occurrences_time_host_event (bucket_start, hostname, channel_name, event_id),
+    KEY idx_occurrences_host_event_time (hostname, channel_name, event_id, bucket_start),
+    KEY idx_occurrences_target_user_time (target_user, bucket_start),
+    KEY idx_occurrences_subject_user_time (subject_user, bucket_start),
+    KEY idx_occurrences_src_ip_time (src_ip, bucket_start)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+PARTITION BY RANGE COLUMNS(bucket_start) (
+    PARTITION pmax VALUES LESS THAN (MAXVALUE)
+);
+
+-- Partition bucket tables only when needed, so this migration can be rerun.
+DELIMITER $$
+CREATE PROCEDURE partition_bucket_table_if_needed(IN p_table VARCHAR(64))
+BEGIN
+    DECLARE v_partition_count INT DEFAULT 0;
+
+    SELECT COUNT(*) INTO v_partition_count
+    FROM information_schema.PARTITIONS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = p_table
+      AND PARTITION_NAME IS NOT NULL;
+
+    IF v_partition_count = 0 THEN
+        SET @partition_sql = CONCAT(
+            'ALTER TABLE `', p_table, '` ',
+            'PARTITION BY RANGE COLUMNS(bucket_start) ',
+            '(PARTITION pmax VALUES LESS THAN (MAXVALUE))'
+        );
+        PREPARE partition_stmt FROM @partition_sql;
+        EXECUTE partition_stmt;
+        DEALLOCATE PREPARE partition_stmt;
+    END IF;
+END$$
+DELIMITER ;
+
+CALL partition_bucket_table_if_needed('event_occurrences');
+CALL partition_bucket_table_if_needed('event_count_buckets');
+CALL partition_bucket_table_if_needed('ueba_context_buckets');
+DROP PROCEDURE partition_bucket_table_if_needed;

deploy/mariadb/migrations/003-new-event-id-classification.sql

@@ -0,0 +1,27 @@
+-- Metadata-first SIEM: realistischere Einordnung der Regel new_event_id
+-- Diese Migration verändert kein Schema. Sie bereinigt nur bereits offene
+-- WMI-Operational-Inventarmeldungen, die von der alten Regel als Medium-Alarm
+-- angelegt wurden. Vor produktiver Ausführung wie üblich ein Backup erstellen.
+
+START TRANSACTION;
+
+UPDATE detections
+SET severity = 'info',
+    status = 'legitimate',
+    is_legitimate = 1,
+    reviewed_by = 'system:migration-003',
+    reviewed_at = UTC_TIMESTAMP(6),
+    analyst_note = CASE
+        WHEN analyst_note IS NULL OR TRIM(analyst_note) = '' THEN
+            'Automatisch neu klassifiziert: neue Event-IDs im WMI-Activity/Operational-Channel sind Inventar-/Diagnoseinformationen und allein kein Incident.'
+        ELSE CONCAT(
+            analyst_note,
+            '\nAutomatisch neu klassifiziert: neue Event-IDs im WMI-Activity/Operational-Channel sind Inventar-/Diagnoseinformationen und allein kein Incident.'
+        )
+    END
+WHERE rule_name = 'new_event_id'
+  AND channel_name = 'Microsoft-Windows-WMI-Activity/Operational'
+  AND status IN ('open', 'acknowledged', 'investigating', 'plausible')
+  AND is_legitimate = 0;
+
+COMMIT;

deploy/mariadb/migrations/004-compact-storage.sql

@@ -0,0 +1,21 @@
+-- Compact-storage migration.
+-- Safe to apply without deleting event history. The destructive cleanup of old
+-- per-event rows is intentionally kept in ../emergency-compact-cleanup.sql.
+
+SET NAMES utf8mb4;
+SET time_zone = '+00:00';
+
+CREATE INDEX IF NOT EXISTS idx_event_count_buckets_event_time
+    ON event_count_buckets (channel_name, event_id, bucket_start, hostname);
+
+CREATE INDEX IF NOT EXISTS idx_occurrences_security_event_time
+    ON event_occurrences (
+        channel_name,
+        event_id,
+        bucket_start,
+        hostname,
+        target_user,
+        src_ip
+    );
+
+ANALYZE TABLE event_count_buckets, event_occurrences, event_catalog, detections;