Update Datenbankproblem

2026-07-23 11:13:05 +02:00
parent f9662780f7
commit 42db7d18f6
18 changed files with 1802 additions and 220 deletions
@@ -0,0 +1,98 @@
+-- Metadata-first migration for an existing SIEM-lite database.
+-- IMPORTANT: ALTER TABLE ... PARTITION BY can rebuild and lock large tables.
+-- Run during a maintenance window and take a backup first.
+
+SET NAMES utf8mb4;
+SET time_zone = '+00:00';
+
+CREATE TABLE IF NOT EXISTS event_catalog (
+    hostname VARCHAR(191) NOT NULL,
+    channel_name VARCHAR(128) NOT NULL,
+    event_id INT UNSIGNED NOT NULL,
+    first_seen DATETIME(6) NOT NULL,
+    last_seen DATETIME(6) NOT NULL,
+    total_count BIGINT UNSIGNED NOT NULL DEFAULT 0,
+    updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+    PRIMARY KEY (hostname, channel_name, event_id),
+    KEY idx_event_catalog_first_seen (first_seen),
+    KEY idx_event_catalog_last_seen (last_seen)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- Warm the catalog from the much smaller baseline buckets so the
+-- new_event_id rule does not treat every known event type as new after migration.
+INSERT INTO event_catalog
+(hostname, channel_name, event_id, first_seen, last_seen, total_count, updated_at)
+SELECT hostname,
+       channel_name,
+       event_id,
+       MIN(COALESCE(first_event_ts, bucket_start)),
+       MAX(COALESCE(last_event_ts, bucket_end)),
+       SUM(cnt),
+       UTC_TIMESTAMP(6)
+FROM event_count_buckets
+GROUP BY hostname, channel_name, event_id
+ON DUPLICATE KEY UPDATE
+    first_seen = LEAST(first_seen, VALUES(first_seen)),
+    last_seen = GREATEST(last_seen, VALUES(last_seen)),
+    total_count = GREATEST(total_count, VALUES(total_count)),
+    updated_at = UTC_TIMESTAMP(6);
+
+CREATE TABLE IF NOT EXISTS event_occurrences (
+    bucket_start DATETIME(6) NOT NULL,
+    bucket_end DATETIME(6) NOT NULL,
+    dimension_key BINARY(16) NOT NULL,
+    hostname VARCHAR(191) NOT NULL,
+    channel_name VARCHAR(128) NOT NULL,
+    event_id INT UNSIGNED NOT NULL,
+    provider_name VARCHAR(191) NOT NULL DEFAULT '',
+    target_user VARCHAR(191) NOT NULL DEFAULT '',
+    subject_user VARCHAR(191) NOT NULL DEFAULT '',
+    src_ip VARCHAR(64) NOT NULL DEFAULT '',
+    workstation VARCHAR(191) NOT NULL DEFAULT '',
+    logon_type VARCHAR(32) NOT NULL DEFAULT '',
+    status_text VARCHAR(128) NOT NULL DEFAULT '',
+    failure_reason VARCHAR(255) NOT NULL DEFAULT '',
+    cnt BIGINT UNSIGNED NOT NULL DEFAULT 0,
+    first_event_ts DATETIME(6) NOT NULL,
+    last_event_ts DATETIME(6) NOT NULL,
+    updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+    PRIMARY KEY (bucket_start, dimension_key),
+    KEY idx_occurrences_time_host_event (bucket_start, hostname, channel_name, event_id),
+    KEY idx_occurrences_host_event_time (hostname, channel_name, event_id, bucket_start),
+    KEY idx_occurrences_target_user_time (target_user, bucket_start),
+    KEY idx_occurrences_subject_user_time (subject_user, bucket_start),
+    KEY idx_occurrences_src_ip_time (src_ip, bucket_start)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+PARTITION BY RANGE COLUMNS(bucket_start) (
+    PARTITION pmax VALUES LESS THAN (MAXVALUE)
+);
+
+-- Partition bucket tables only when needed, so this migration can be rerun.
+DELIMITER $$
+CREATE PROCEDURE partition_bucket_table_if_needed(IN p_table VARCHAR(64))
+BEGIN
+    DECLARE v_partition_count INT DEFAULT 0;
+
+    SELECT COUNT(*) INTO v_partition_count
+    FROM information_schema.PARTITIONS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = p_table
+      AND PARTITION_NAME IS NOT NULL;
+
+    IF v_partition_count = 0 THEN
+        SET @partition_sql = CONCAT(
+            'ALTER TABLE `', p_table, '` ',
+            'PARTITION BY RANGE COLUMNS(bucket_start) ',
+            '(PARTITION pmax VALUES LESS THAN (MAXVALUE))'
+        );
+        PREPARE partition_stmt FROM @partition_sql;
+        EXECUTE partition_stmt;
+        DEALLOCATE PREPARE partition_stmt;
+    END IF;
+END$$
+DELIMITER ;
+
+CALL partition_bucket_table_if_needed('event_occurrences');
+CALL partition_bucket_table_if_needed('event_count_buckets');
+CALL partition_bucket_table_if_needed('ueba_context_buckets');
+DROP PROCEDURE partition_bucket_table_if_needed;
@@ -0,0 +1,27 @@
+-- Metadata-first SIEM: realistischere Einordnung der Regel new_event_id
+-- Diese Migration verändert kein Schema. Sie bereinigt nur bereits offene
+-- WMI-Operational-Inventarmeldungen, die von der alten Regel als Medium-Alarm
+-- angelegt wurden. Vor produktiver Ausführung wie üblich ein Backup erstellen.
+
+START TRANSACTION;
+
+UPDATE detections
+SET severity = 'info',
+    status = 'legitimate',
+    is_legitimate = 1,
+    reviewed_by = 'system:migration-003',
+    reviewed_at = UTC_TIMESTAMP(6),
+    analyst_note = CASE
+        WHEN analyst_note IS NULL OR TRIM(analyst_note) = '' THEN
+            'Automatisch neu klassifiziert: neue Event-IDs im WMI-Activity/Operational-Channel sind Inventar-/Diagnoseinformationen und allein kein Incident.'
+        ELSE CONCAT(
+            analyst_note,
+            '\nAutomatisch neu klassifiziert: neue Event-IDs im WMI-Activity/Operational-Channel sind Inventar-/Diagnoseinformationen und allein kein Incident.'
+        )
+    END
+WHERE rule_name = 'new_event_id'
+  AND channel_name = 'Microsoft-Windows-WMI-Activity/Operational'
+  AND status IN ('open', 'acknowledged', 'investigating', 'plausible')
+  AND is_legitimate = 0;
+
+COMMIT;
@@ -0,0 +1,21 @@
+-- Compact-storage migration.
+-- Safe to apply without deleting event history. The destructive cleanup of old
+-- per-event rows is intentionally kept in ../emergency-compact-cleanup.sql.
+
+SET NAMES utf8mb4;
+SET time_zone = '+00:00';
+
+CREATE INDEX IF NOT EXISTS idx_event_count_buckets_event_time
+    ON event_count_buckets (channel_name, event_id, bucket_start, hostname);
+
+CREATE INDEX IF NOT EXISTS idx_occurrences_security_event_time
+    ON event_occurrences (
+        channel_name,
+        event_id,
+        bucket_start,
+        hostname,
+        target_user,
+        src_ip
+    );
+
+ANALYZE TABLE event_count_buckets, event_occurrences, event_catalog, detections;