Vollständiges Redesign

2026-07-23 14:45:35 +02:00
parent 42db7d18f6
commit c49bfa1c34
58 changed files with 2781 additions and 18124 deletions
@@ -0,0 +1,82 @@
+CREATE DATABASE IF NOT EXISTS siem;
+
+CREATE TABLE IF NOT EXISTS siem.events
+(
+    event_uid String,
+    queue_partition Int32,
+    queue_offset Int64,
+    tenant_id LowCardinality(String),
+    event_time DateTime64(3, 'UTC'),
+    ingest_time DateTime64(3, 'UTC'),
+    event_date Date MATERIALIZED toDate(event_time),
+    agent_id String,
+    host_name LowCardinality(String),
+    source_type LowCardinality(String),
+    channel LowCardinality(String),
+    provider LowCardinality(String),
+    event_code UInt32,
+    category LowCardinality(String),
+    action LowCardinality(String),
+    outcome LowCardinality(String),
+    severity UInt8,
+    user_name String,
+    user_domain LowCardinality(String),
+    subject_user String,
+    subject_domain LowCardinality(String),
+    target_user String,
+    target_domain LowCardinality(String),
+    source_ip String,
+    source_port UInt16,
+    destination_ip String,
+    destination_port UInt16,
+    workstation String,
+    logon_type LowCardinality(String),
+    authentication_package LowCardinality(String),
+    logon_process LowCardinality(String),
+    status_code LowCardinality(String),
+    sub_status_code LowCardinality(String),
+    failure_reason LowCardinality(String),
+    process_path String,
+    parent_process_path String,
+    command_line String CODEC(ZSTD(3)),
+    message String CODEC(ZSTD(3)),
+    attributes Map(String, String) CODEC(ZSTD(3)),
+    raw_object_key String,
+    raw_index UInt32,
+    payload_hash FixedString(64),
+    schema_version UInt16,
+    parser_version UInt16,
+    ingest_delay_ms Int64,
+    INDEX idx_host host_name TYPE set(10000) GRANULARITY 4,
+    INDEX idx_event_code event_code TYPE set(256) GRANULARITY 4,
+    INDEX idx_user user_name TYPE bloom_filter(0.01) GRANULARITY 4,
+    INDEX idx_target target_user TYPE bloom_filter(0.01) GRANULARITY 4,
+    INDEX idx_source_ip source_ip TYPE bloom_filter(0.01) GRANULARITY 4,
+    INDEX idx_process process_path TYPE bloom_filter(0.01) GRANULARITY 8
+)
+ENGINE = ReplacingMergeTree(ingest_time)
+PARTITION BY toYYYYMM(event_time)
+ORDER BY (tenant_id, event_time, host_name, event_code, event_uid)
+TTL event_time + INTERVAL __EVENT_RETENTION_DAYS__ DAY DELETE
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE IF NOT EXISTS siem.events_5m
+(
+    tenant_id LowCardinality(String),
+    bucket DateTime('UTC'),
+    host_name LowCardinality(String),
+    event_code UInt32,
+    category LowCardinality(String),
+    action LowCardinality(String),
+    outcome LowCardinality(String),
+    cnt_state AggregateFunction(uniqExact, String)
+)
+ENGINE = AggregatingMergeTree
+PARTITION BY toYYYYMM(bucket)
+ORDER BY (tenant_id, bucket, host_name, event_code, category, action, outcome)
+TTL bucket + INTERVAL __ROLLUP_RETENTION_DAYS__ DAY DELETE;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS siem.events_5m_mv TO siem.events_5m AS
+SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state
+FROM siem.events
+GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome;