sendnrw/siem-backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Vollständiges Redesign
release-tag / release-image (push) Failing after 5m54s
Details

Browse Source
This commit is contained in:
groot
2026-07-23 14:45:35 +02:00
parent 42db7d18f6
commit c49bfa1c34
58 changed files with 2781 additions and 18124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/clickhouse/init.sql
+82
View File
@@ -0,0 +1,82 @@
CREATE DATABASE IF NOT EXISTS siem;
CREATE TABLE IF NOT EXISTS siem.events
(
event_uid String,
queue_partition Int32,
queue_offset Int64,
tenant_id LowCardinality(String),
event_time DateTime64(3, 'UTC'),
ingest_time DateTime64(3, 'UTC'),
event_date Date MATERIALIZED toDate(event_time),
agent_id String,
host_name LowCardinality(String),
source_type LowCardinality(String),
channel LowCardinality(String),
provider LowCardinality(String),
event_code UInt32,
category LowCardinality(String),
action LowCardinality(String),
outcome LowCardinality(String),
severity UInt8,
user_name String,
user_domain LowCardinality(String),
subject_user String,
subject_domain LowCardinality(String),
target_user String,
target_domain LowCardinality(String),
source_ip String,
source_port UInt16,
destination_ip String,
destination_port UInt16,
workstation String,
logon_type LowCardinality(String),
authentication_package LowCardinality(String),
logon_process LowCardinality(String),
status_code LowCardinality(String),
sub_status_code LowCardinality(String),
failure_reason LowCardinality(String),
process_path String,
parent_process_path String,
command_line String CODEC(ZSTD(3)),
message String CODEC(ZSTD(3)),
attributes Map(String, String) CODEC(ZSTD(3)),
raw_object_key String,
raw_index UInt32,
payload_hash FixedString(64),
schema_version UInt16,
parser_version UInt16,
ingest_delay_ms Int64,
INDEX idx_host host_name TYPE set(10000) GRANULARITY 4,
INDEX idx_event_code event_code TYPE set(256) GRANULARITY 4,
INDEX idx_user user_name TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_target target_user TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_source_ip source_ip TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_process process_path TYPE bloom_filter(0.01) GRANULARITY 8
)
ENGINE = ReplacingMergeTree(ingest_time)
PARTITION BY toYYYYMM(event_time)
ORDER BY (tenant_id, event_time, host_name, event_code, event_uid)
TTL event_time + INTERVAL 90 DAY DELETE
SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS siem.events_5m
(
tenant_id LowCardinality(String),
bucket DateTime('UTC'),
host_name LowCardinality(String),
event_code UInt32,
category LowCardinality(String),
action LowCardinality(String),
outcome LowCardinality(String),
cnt_state AggregateFunction(uniqExact, String)
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(bucket)
ORDER BY (tenant_id, bucket, host_name, event_code, category, action, outcome)
TTL bucket + INTERVAL 730 DAY DELETE;
CREATE MATERIALIZED VIEW IF NOT EXISTS siem.events_5m_mv TO siem.events_5m AS
SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state
FROM siem.events
GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome;
deploy/clickhouse/init.sql.template
+82
View File
@@ -0,0 +1,82 @@
CREATE DATABASE IF NOT EXISTS siem;
CREATE TABLE IF NOT EXISTS siem.events
(
event_uid String,
queue_partition Int32,
queue_offset Int64,
tenant_id LowCardinality(String),
event_time DateTime64(3, 'UTC'),
ingest_time DateTime64(3, 'UTC'),
event_date Date MATERIALIZED toDate(event_time),
agent_id String,
host_name LowCardinality(String),
source_type LowCardinality(String),
channel LowCardinality(String),
provider LowCardinality(String),
event_code UInt32,
category LowCardinality(String),
action LowCardinality(String),
outcome LowCardinality(String),
severity UInt8,
user_name String,
user_domain LowCardinality(String),
subject_user String,
subject_domain LowCardinality(String),
target_user String,
target_domain LowCardinality(String),
source_ip String,
source_port UInt16,
destination_ip String,
destination_port UInt16,
workstation String,
logon_type LowCardinality(String),
authentication_package LowCardinality(String),
logon_process LowCardinality(String),
status_code LowCardinality(String),
sub_status_code LowCardinality(String),
failure_reason LowCardinality(String),
process_path String,
parent_process_path String,
command_line String CODEC(ZSTD(3)),
message String CODEC(ZSTD(3)),
attributes Map(String, String) CODEC(ZSTD(3)),
raw_object_key String,
raw_index UInt32,
payload_hash FixedString(64),
schema_version UInt16,
parser_version UInt16,
ingest_delay_ms Int64,
INDEX idx_host host_name TYPE set(10000) GRANULARITY 4,
INDEX idx_event_code event_code TYPE set(256) GRANULARITY 4,
INDEX idx_user user_name TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_target target_user TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_source_ip source_ip TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_process process_path TYPE bloom_filter(0.01) GRANULARITY 8
)
ENGINE = ReplacingMergeTree(ingest_time)
PARTITION BY toYYYYMM(event_time)
ORDER BY (tenant_id, event_time, host_name, event_code, event_uid)
TTL event_time + INTERVAL __EVENT_RETENTION_DAYS__ DAY DELETE
SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS siem.events_5m
(
tenant_id LowCardinality(String),
bucket DateTime('UTC'),
host_name LowCardinality(String),
event_code UInt32,
category LowCardinality(String),
action LowCardinality(String),
outcome LowCardinality(String),
cnt_state AggregateFunction(uniqExact, String)
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(bucket)
ORDER BY (tenant_id, bucket, host_name, event_code, category, action, outcome)
TTL bucket + INTERVAL __ROLLUP_RETENTION_DAYS__ DAY DELETE;
CREATE MATERIALIZED VIEW IF NOT EXISTS siem.events_5m_mv TO siem.events_5m AS
SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state
FROM siem.events
GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome;
deploy/clickhouse/prometheus.xml
+10
View File
@@ -0,0 +1,10 @@
<clickhouse>
<prometheus>
<endpoint>/metrics</endpoint>
<port>9363</port>
<metrics>true</metrics>
<events>true</events>
<asynchronous_metrics>true</asynchronous_metrics>
<status_info>true</status_info>
</prometheus>
</clickhouse>
deploy/garage/garage.toml.template
+17
View File
@@ -0,0 +1,17 @@
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "garage:3901"
rpc_secret = "__GARAGE_RPC_SECRET__"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage.localhost"
[admin]
api_bind_addr = "[::]:3903"
admin_token = "__GARAGE_ADMIN_TOKEN__"
metrics_token = "__GARAGE_METRICS_TOKEN__"
deploy/grafana/provisioning/dashboards/dashboards.yml
-12
View File
@@ -1,12 +0,0 @@
apiVersion: 1
providers:
- name: SIEM Dashboards
orgId: 1
folder: SIEM
type: file
disableDeletion: false
editable: true
updateIntervalSeconds: 30
options:
path: /var/lib/grafana/dashboards
deploy/grafana/provisioning/dashboards/siem-overview.json
-745
View File
@@ -1,745 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"panels": [
{
"type": "stat",
"title": "Active Agents",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
"targets": [
{
"expr": "eventcollector_active_agents",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "green", "value": 1 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto"
}
},
{
"type": "stat",
"title": "Events/s",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
"targets": [
{
"expr": "sum(rate(eventcollector_ingest_events_total{channel=~\"$channel\",event_id=~\"$event_id\"}[5m]))",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "eps",
"decimals": 2
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "stat",
"title": "High Detections 5m",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
"targets": [
{
"expr": "sum(increase(eventcollector_detection_hits_total{severity=\"high\",rule=~\"$rule\"}[5m]))",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "stat",
"title": "Baseline Max Z-Score",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
"targets": [
{
"expr": "max(eventcollector_anomaly_score{host=~\"$host\",rule=\"baseline_event_rate_anomaly\"})",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 3 },
{ "color": "red", "value": 5 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "stat",
"title": "Rule Errors 5m",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
"targets": [
{
"expr": "sum(increase(eventcollector_rule_errors_total{rule=~\"$rule\"}[5m]))",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "stat",
"title": "DB Insert Failures 5m",
"datasource": "$datasource",
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
"targets": [
{
"expr": "increase(eventcollector_db_insert_failures_total[5m])",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"textMode": "auto"
}
},
{
"type": "timeseries",
"title": "Ingested Events / Second by Channel",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
"targets": [
{
"expr": "sum by (channel) (rate(eventcollector_ingest_events_total{channel=~\"$channel\",event_id=~\"$event_id\"}[5m]))",
"legendFormat": "{{channel}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "eps",
"decimals": 2
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Detection Hits by Rule / Severity",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
"targets": [
{
"expr": "sum by (rule,severity) (increase(eventcollector_detection_hits_total{rule=~\"$rule\",severity=~\"$severity\"}[5m]))",
"legendFormat": "{{rule}} / {{severity}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Baseline: Current Count vs Average",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
"targets": [
{
"expr": "eventcollector_baseline_current_count{host=~\"$host\",channel=~\"$channel\",event_id=~\"$event_id\"}",
"legendFormat": "current {{host}} {{channel}} {{event_id}}",
"refId": "A"
},
{
"expr": "eventcollector_baseline_avg_count{host=~\"$host\",channel=~\"$channel\",event_id=~\"$event_id\"}",
"legendFormat": "avg {{host}} {{channel}} {{event_id}}",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 2
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Baseline Z-Score",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
"targets": [
{
"expr": "eventcollector_anomaly_score{host=~\"$host\",rule=\"baseline_event_rate_anomaly\"}",
"legendFormat": "{{host}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 3 },
{ "color": "red", "value": 5 }
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "bargauge",
"title": "Top Baseline Z-Scores",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 20 },
"targets": [
{
"expr": "topk(10, eventcollector_anomaly_score{host=~\"$host\",rule=\"baseline_event_rate_anomaly\"})",
"legendFormat": "{{host}}",
"refId": "A",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 3 },
{ "color": "red", "value": 5 }
]
}
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showUnfilled": true
}
},
{
"type": "bargauge",
"title": "Top EventIDs by Ingest Rate",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 20 },
"targets": [
{
"expr": "topk(15, sum by (channel,event_id) (rate(eventcollector_ingest_events_total{channel=~\"$channel\",event_id=~\"$event_id\"}[5m])))",
"legendFormat": "{{channel}} / {{event_id}}",
"refId": "A",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "eps",
"decimals": 2
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showUnfilled": true
}
},
{
"type": "bargauge",
"title": "Top Detection Rules 1h",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 20 },
"targets": [
{
"expr": "topk(15, sum by (rule,severity) (increase(eventcollector_detection_hits_total{rule=~\"$rule\",severity=~\"$severity\"}[1h])))",
"legendFormat": "{{rule}} / {{severity}}",
"refId": "A",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showUnfilled": true
}
},
{
"type": "timeseries",
"title": "HTTP Requests by Path / Status",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
"targets": [
{
"expr": "sum by (path,status) (rate(eventcollector_http_requests_total[5m]))",
"legendFormat": "{{path}} {{status}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"decimals": 2
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "HTTP Latency p95",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le,path) (rate(eventcollector_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "{{path}} p95",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 3
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "DB Insert Transaction Latency p95",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 },
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(eventcollector_db_tx_duration_seconds_bucket[5m])))",
"legendFormat": "db tx p95",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 3
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "DB Batch Size p95",
"datasource": "$datasource",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 },
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(eventcollector_db_batch_size_bucket[5m])))",
"legendFormat": "batch size p95",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "table",
"title": "Agent Last Seen",
"datasource": "$datasource",
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 44 },
"targets": [
{
"expr": "time() - eventcollector_agent_last_seen_unixtime{host=~\"$host\"}",
"legendFormat": "{{host}}",
"refId": "A",
"instant": true,
"format": "table"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0
},
"overrides": []
},
"options": {
"showHeader": true
}
},
{
"type": "table",
"title": "Baseline Samples",
"datasource": "$datasource",
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 44 },
"targets": [
{
"expr": "eventcollector_baseline_sample_count{host=~\"$host\",channel=~\"$channel\",event_id=~\"$event_id\"}",
"legendFormat": "{{host}} {{channel}} {{event_id}}",
"refId": "A",
"instant": true,
"format": "table"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"showHeader": true
}
}
],
"refresh": "30s",
"schemaVersion": 39,
"style": "dark",
"tags": ["siem", "baseline", "ad"],
"templating": {
"list": [
{
"name": "datasource",
"type": "datasource",
"query": "prometheus",
"current": {},
"hide": 0,
"label": "Datasource"
},
{
"name": "host",
"type": "query",
"datasource": "$datasource",
"query": "label_values(eventcollector_agent_last_seen_unixtime, host)",
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Host"
},
{
"name": "channel",
"type": "query",
"datasource": "$datasource",
"query": "label_values(eventcollector_ingest_events_total, channel)",
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Channel"
},
{
"name": "event_id",
"type": "query",
"datasource": "$datasource",
"query": "label_values(eventcollector_ingest_events_total, event_id)",
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Event ID"
},
{
"name": "rule",
"type": "query",
"datasource": "$datasource",
"query": "label_values(eventcollector_detection_hits_total, rule)",
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Rule"
},
{
"name": "severity",
"type": "custom",
"query": "low,medium,high",
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"label": "Severity"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "browser",
"title": "SIEM Overview Extended",
"uid": "siem-overview-extended",
"version": 1
}
deploy/grafana/provisioning/dashboards/siem-soc-privileged.json
-1006
View File
File diff suppressed because it is too large Load Diff
deploy/grafana/provisioning/datasources/datasource.yml
-10
View File
@@ -1,10 +0,0 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: true
deploy/mariadb/cardinality-diagnostics.sql
-35
View File
@@ -1,35 +0,0 @@
-- Shows where rows are growing and how effective aggregation currently is.
SET time_zone = '+00:00';
SELECT table_name,
table_rows,
ROUND(data_length/1024/1024,1) AS data_mb,
ROUND(index_length/1024/1024,1) AS index_mb,
ROUND((data_length+index_length)/1024/1024,1) AS total_mb
FROM information_schema.tables
WHERE table_schema = DATABASE()
ORDER BY data_length + index_length DESC;
SELECT 'event_count_buckets' AS source,
COUNT(*) AS rows_24h,
COALESCE(SUM(cnt),0) AS represented_events_24h,
ROUND(COALESCE(SUM(cnt),0) / NULLIF(COUNT(*),0), 1) AS events_per_row
FROM event_count_buckets
WHERE bucket_start >= UTC_TIMESTAMP() - INTERVAL 24 HOUR
UNION ALL
SELECT 'event_occurrences',
COUNT(*),
COALESCE(SUM(cnt),0),
ROUND(COALESCE(SUM(cnt),0) / NULLIF(COUNT(*),0), 1)
FROM event_occurrences
WHERE bucket_start >= UTC_TIMESTAMP() - INTERVAL 24 HOUR;
SELECT channel_name, event_id,
COUNT(*) AS context_rows_24h,
SUM(cnt) AS represented_events_24h,
ROUND(SUM(cnt)/NULLIF(COUNT(*),0),1) AS events_per_row
FROM event_occurrences
WHERE bucket_start >= UTC_TIMESTAMP() - INTERVAL 24 HOUR
GROUP BY channel_name, event_id
ORDER BY context_rows_24h DESC
LIMIT 50;
deploy/mariadb/diagnostics.sql
-52
View File
@@ -1,52 +0,0 @@
-- Read-only diagnostics for sizing and partition verification.
SET time_zone = '+00:00';
SELECT
TABLE_NAME,
ENGINE,
TABLE_ROWS,
ROUND(DATA_LENGTH / 1024 / 1024, 1) AS data_mb,
ROUND(INDEX_LENGTH / 1024 / 1024, 1) AS index_mb,
ROUND((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024, 1) AS total_mb
FROM information_schema.TABLES
WHERE TABLE_SCHEMA = DATABASE()
ORDER BY (DATA_LENGTH + INDEX_LENGTH) DESC;
SELECT
TABLE_NAME,
COUNT(*) AS partitions,
MIN(PARTITION_NAME) AS oldest_partition,
MAX(PARTITION_NAME) AS newest_partition,
ROUND(SUM(DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024, 1) AS total_mb
FROM information_schema.PARTITIONS
WHERE TABLE_SCHEMA = DATABASE()
AND PARTITION_NAME IS NOT NULL
GROUP BY TABLE_NAME
ORDER BY total_mb DESC;
SELECT
channel_name,
event_id,
SUM(cnt) AS events_24h,
COUNT(*) AS metadata_rows_24h,
ROUND(SUM(cnt) / NULLIF(COUNT(*), 0), 2) AS compression_factor
FROM event_occurrences
WHERE bucket_start >= UTC_TIMESTAMP() - INTERVAL 24 HOUR
GROUP BY channel_name, event_id
ORDER BY events_24h DESC
LIMIT 50;
SELECT
first_event_ts,
last_event_ts,
cnt,
hostname,
target_user,
workstation,
src_ip
FROM event_occurrences
WHERE channel_name = 'Security'
AND event_id = 4740
AND bucket_start >= UTC_TIMESTAMP() - INTERVAL 7 DAY
ORDER BY bucket_start DESC
LIMIT 200;
deploy/mariadb/emergency-compact-cleanup.sql
-43
View File
@@ -1,43 +0,0 @@
-- EMERGENCY CLEANUP FOR THE PREVIOUS METADATA-FIRST BUILD
--
-- Run ONLY AFTER the new backend is deployed with:
-- STORE_EVENT_ROWS=false
-- STORE_RAW_XML=false
--
-- This intentionally removes redundant per-event/full-context history generated
-- by older builds. Aggregate event history remains in event_count_buckets and
-- event_catalog. Existing detections and UEBA baselines are not deleted.
--
-- Take a database backup/snapshot first if old raw/event detail may still matter.
SET NAMES utf8mb4;
SET time_zone = '+00:00';
SELECT 'before' AS phase,
table_name,
table_rows,
ROUND((data_length + index_length) / 1024 / 1024, 1) AS size_mb
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_name IN ('event_logs','event_log_raw','event_occurrences','event_count_buckets','event_catalog','detections')
ORDER BY size_mb DESC;
-- Raw XML must be cleared before its logical parent event rows.
TRUNCATE TABLE event_log_raw;
TRUNCATE TABLE event_logs;
-- Old versions used too many dimensions and could make this table nearly as
-- large as event_logs. The new backend will repopulate only selected Security
-- contexts in 5-minute buckets.
TRUNCATE TABLE event_occurrences;
ANALYZE TABLE event_count_buckets, event_catalog, detections;
SELECT 'after' AS phase,
table_name,
table_rows,
ROUND((data_length + index_length) / 1024 / 1024, 1) AS size_mb
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_name IN ('event_logs','event_log_raw','event_occurrences','event_count_buckets','event_catalog','detections')
ORDER BY size_mb DESC;
deploy/mariadb/init/001-schema.sql
-2239
View File
File diff suppressed because it is too large Load Diff
deploy/mariadb/migrations/002-metadata-first.sql
-98
View File
@@ -1,98 +0,0 @@
-- Metadata-first migration for an existing SIEM-lite database.
-- IMPORTANT: ALTER TABLE ... PARTITION BY can rebuild and lock large tables.
-- Run during a maintenance window and take a backup first.
SET NAMES utf8mb4;
SET time_zone = '+00:00';
CREATE TABLE IF NOT EXISTS event_catalog (
hostname VARCHAR(191) NOT NULL,
channel_name VARCHAR(128) NOT NULL,
event_id INT UNSIGNED NOT NULL,
first_seen DATETIME(6) NOT NULL,
last_seen DATETIME(6) NOT NULL,
total_count BIGINT UNSIGNED NOT NULL DEFAULT 0,
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
PRIMARY KEY (hostname, channel_name, event_id),
KEY idx_event_catalog_first_seen (first_seen),
KEY idx_event_catalog_last_seen (last_seen)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- Warm the catalog from the much smaller baseline buckets so the
-- new_event_id rule does not treat every known event type as new after migration.
INSERT INTO event_catalog
(hostname, channel_name, event_id, first_seen, last_seen, total_count, updated_at)
SELECT hostname,
channel_name,
event_id,
MIN(COALESCE(first_event_ts, bucket_start)),
MAX(COALESCE(last_event_ts, bucket_end)),
SUM(cnt),
UTC_TIMESTAMP(6)
FROM event_count_buckets
GROUP BY hostname, channel_name, event_id
ON DUPLICATE KEY UPDATE
first_seen = LEAST(first_seen, VALUES(first_seen)),
last_seen = GREATEST(last_seen, VALUES(last_seen)),
total_count = GREATEST(total_count, VALUES(total_count)),
updated_at = UTC_TIMESTAMP(6);
CREATE TABLE IF NOT EXISTS event_occurrences (
bucket_start DATETIME(6) NOT NULL,
bucket_end DATETIME(6) NOT NULL,
dimension_key BINARY(16) NOT NULL,
hostname VARCHAR(191) NOT NULL,
channel_name VARCHAR(128) NOT NULL,
event_id INT UNSIGNED NOT NULL,
provider_name VARCHAR(191) NOT NULL DEFAULT '',
target_user VARCHAR(191) NOT NULL DEFAULT '',
subject_user VARCHAR(191) NOT NULL DEFAULT '',
src_ip VARCHAR(64) NOT NULL DEFAULT '',
workstation VARCHAR(191) NOT NULL DEFAULT '',
logon_type VARCHAR(32) NOT NULL DEFAULT '',
status_text VARCHAR(128) NOT NULL DEFAULT '',
failure_reason VARCHAR(255) NOT NULL DEFAULT '',
cnt BIGINT UNSIGNED NOT NULL DEFAULT 0,
first_event_ts DATETIME(6) NOT NULL,
last_event_ts DATETIME(6) NOT NULL,
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
PRIMARY KEY (bucket_start, dimension_key),
KEY idx_occurrences_time_host_event (bucket_start, hostname, channel_name, event_id),
KEY idx_occurrences_host_event_time (hostname, channel_name, event_id, bucket_start),
KEY idx_occurrences_target_user_time (target_user, bucket_start),
KEY idx_occurrences_subject_user_time (subject_user, bucket_start),
KEY idx_occurrences_src_ip_time (src_ip, bucket_start)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
PARTITION BY RANGE COLUMNS(bucket_start) (
PARTITION pmax VALUES LESS THAN (MAXVALUE)
);
-- Partition bucket tables only when needed, so this migration can be rerun.
DELIMITER $$
CREATE PROCEDURE partition_bucket_table_if_needed(IN p_table VARCHAR(64))
BEGIN
DECLARE v_partition_count INT DEFAULT 0;
SELECT COUNT(*) INTO v_partition_count
FROM information_schema.PARTITIONS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = p_table
AND PARTITION_NAME IS NOT NULL;
IF v_partition_count = 0 THEN
SET @partition_sql = CONCAT(
'ALTER TABLE `', p_table, '` ',
'PARTITION BY RANGE COLUMNS(bucket_start) ',
'(PARTITION pmax VALUES LESS THAN (MAXVALUE))'
);
PREPARE partition_stmt FROM @partition_sql;
EXECUTE partition_stmt;
DEALLOCATE PREPARE partition_stmt;
END IF;
END$$
DELIMITER ;
CALL partition_bucket_table_if_needed('event_occurrences');
CALL partition_bucket_table_if_needed('event_count_buckets');
CALL partition_bucket_table_if_needed('ueba_context_buckets');
DROP PROCEDURE partition_bucket_table_if_needed;
deploy/mariadb/migrations/003-new-event-id-classification.sql
-27
View File
@@ -1,27 +0,0 @@
-- Metadata-first SIEM: realistischere Einordnung der Regel new_event_id
-- Diese Migration verändert kein Schema. Sie bereinigt nur bereits offene
-- WMI-Operational-Inventarmeldungen, die von der alten Regel als Medium-Alarm
-- angelegt wurden. Vor produktiver Ausführung wie üblich ein Backup erstellen.
START TRANSACTION;
UPDATE detections
SET severity = 'info',
status = 'legitimate',
is_legitimate = 1,
reviewed_by = 'system:migration-003',
reviewed_at = UTC_TIMESTAMP(6),
analyst_note = CASE
WHEN analyst_note IS NULL OR TRIM(analyst_note) = '' THEN
'Automatisch neu klassifiziert: neue Event-IDs im WMI-Activity/Operational-Channel sind Inventar-/Diagnoseinformationen und allein kein Incident.'
ELSE CONCAT(
analyst_note,
'\nAutomatisch neu klassifiziert: neue Event-IDs im WMI-Activity/Operational-Channel sind Inventar-/Diagnoseinformationen und allein kein Incident.'
)
END
WHERE rule_name = 'new_event_id'
AND channel_name = 'Microsoft-Windows-WMI-Activity/Operational'
AND status IN ('open', 'acknowledged', 'investigating', 'plausible')
AND is_legitimate = 0;
COMMIT;
deploy/mariadb/migrations/004-compact-storage.sql
-21
View File
@@ -1,21 +0,0 @@
-- Compact-storage migration.
-- Safe to apply without deleting event history. The destructive cleanup of old
-- per-event rows is intentionally kept in ../emergency-compact-cleanup.sql.
SET NAMES utf8mb4;
SET time_zone = '+00:00';
CREATE INDEX IF NOT EXISTS idx_event_count_buckets_event_time
ON event_count_buckets (channel_name, event_id, bucket_start, hostname);
CREATE INDEX IF NOT EXISTS idx_occurrences_security_event_time
ON event_occurrences (
channel_name,
event_id,
bucket_start,
hostname,
target_user,
src_ip
);
ANALYZE TABLE event_count_buckets, event_occurrences, event_catalog, detections;
deploy/postgres/init.sql
+38
View File
@@ -0,0 +1,38 @@
CREATE EXTENSION IF NOT EXISTS pgcrypto;
CREATE TABLE IF NOT EXISTS agents (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id text NOT NULL,
hostname text NOT NULL,
api_key_hash char(64) NOT NULL,
enabled boolean NOT NULL DEFAULT true,
last_ip text NOT NULL DEFAULT '',
first_seen timestamptz NOT NULL DEFAULT now(),
last_seen timestamptz NOT NULL DEFAULT now(),
UNIQUE (tenant_id, hostname)
);
CREATE INDEX IF NOT EXISTS agents_last_seen_idx ON agents(tenant_id,last_seen DESC);
CREATE TABLE IF NOT EXISTS detections (
id bigserial PRIMARY KEY,
tenant_id text NOT NULL,
fingerprint char(64) NOT NULL,
rule_name text NOT NULL,
severity text NOT NULL,
status text NOT NULL DEFAULT 'open',
hostname text NOT NULL DEFAULT '',
user_name text NOT NULL DEFAULT '',
source_ip text NOT NULL DEFAULT '',
workstation text NOT NULL DEFAULT '',
event_code integer NOT NULL DEFAULT 0,
score double precision NOT NULL DEFAULT 0,
window_start timestamptz NOT NULL,
window_end timestamptz NOT NULL,
summary text NOT NULL,
hit_count bigint NOT NULL DEFAULT 1,
first_seen timestamptz NOT NULL DEFAULT now(),
last_seen timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (tenant_id, fingerprint)
);
CREATE INDEX IF NOT EXISTS detections_open_idx ON detections(tenant_id,status,last_seen DESC);
CREATE INDEX IF NOT EXISTS detections_rule_idx ON detections(tenant_id,rule_name,last_seen DESC);
deploy/prometheus/prometheus.yml
+7 -14
View File
@@ -1,18 +1,11 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: siem-backend
- job_name: siem-ingress
static_configs: [{targets: ['ingress:8080']}]
- job_name: redpanda
metrics_path: /public_metrics
static_configs: [{targets: ['redpanda:9644']}]
- job_name: clickhouse
metrics_path: /metrics
static_configs:
- targets:
- siem-backend:8080
- job_name: prometheus
static_configs:
- targets:
- localhost:9090
static_configs: [{targets: ['clickhouse:9363']}]
deploy/prometheus/rules/siem-alerts.yml
-132
View File
@@ -1,132 +0,0 @@
groups:
- name: siem-backend-availability
rules:
- alert: SiemBackendDown
expr: up{job="siem-backend"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "SIEM backend nicht erreichbar"
description: "Prometheus kann das SIEM-Backend seit mindestens 2 Minuten nicht scrapen."
- alert: SiemNoIngestEvents
expr: sum(rate(eventcollector_ingest_events_total[15m])) == 0
for: 15m
labels:
severity: warning
annotations:
summary: "Keine eingehenden SIEM Events"
description: "Seit mindestens 15 Minuten wurden keine Events mehr ingestiert."
- alert: SiemTooFewActiveAgents
expr: eventcollector_active_agents < 1
for: 5m
labels:
severity: warning
annotations:
summary: "Zu wenige aktive Agents"
description: "Es wurden weniger aktive Agents erkannt als erwartet."
- alert: SiemCriticalDetections
expr: increase(eventcollector_detection_hits_total{severity="critical"}[5m]) > 0
for: 1m
labels:
severity: critical
annotations:
summary: "Neue Critical Detection"
description: "Es wurde mindestens eine Critical-Detection erzeugt."
- alert: SiemHighDetections
expr: increase(eventcollector_detection_hits_total{severity="high"}[5m]) > 0
for: 1m
labels:
severity: high
annotations:
summary: "Neue High-Severity Detection"
description: "Es wurde mindestens eine High-Severity-Detection erzeugt."
- alert: SiemManyMediumDetections
expr: sum(increase(eventcollector_detection_hits_total{severity="medium"}[15m])) > 10
for: 2m
labels:
severity: warning
annotations:
summary: "Viele Medium-Detections"
description: "Es wurden mehr als 10 Medium-Detections in 15 Minuten erzeugt."
- alert: SiemBaselineHighAnomaly
expr: eventcollector_anomaly_score{rule="baseline_event_rate_anomaly"} >= 5
for: 2m
labels:
severity: high
annotations:
summary: "Hohe Baseline-Anomalie"
description: "Host {{ $labels.host }} hat einen hohen Baseline-Z-Score: {{ $value }}."
- alert: SiemBaselineMediumAnomaly
expr: eventcollector_anomaly_score{rule="baseline_event_rate_anomaly"} >= 3
for: 5m
labels:
severity: warning
annotations:
summary: "Baseline-Anomalie"
description: "Host {{ $labels.host }} hat einen erhöhten Baseline-Z-Score: {{ $value }}."
- alert: SiemRuleErrors
expr: increase(eventcollector_rule_errors_total[5m]) > 0
for: 1m
labels:
severity: warning
annotations:
summary: "Fehler in Detection-Regeln"
description: "Mindestens eine Detection-Regel hat in den letzten 5 Minuten einen Fehler erzeugt."
- name: siem-backend-ingest
rules:
- alert: SiemIngestRejected
expr: sum(increase(eventcollector_ingest_rejected_total[5m])) > 0
for: 1m
labels:
severity: warning
annotations:
summary: "Ingest Requests abgelehnt"
description: "In den letzten 5 Minuten wurden Ingest Requests abgelehnt."
- alert: SiemDBInsertFailures
expr: increase(eventcollector_db_insert_failures_total[5m]) > 0
for: 1m
labels:
severity: high
annotations:
summary: "DB Insert Fehler"
description: "Das SIEM-Backend konnte Events nicht in die Datenbank schreiben."
- alert: SiemHighIngestRate
expr: sum(rate(eventcollector_ingest_events_total[5m])) > 500
for: 5m
labels:
severity: warning
annotations:
summary: "Sehr hohe Eventrate"
description: "Die Eventrate liegt seit 5 Minuten über 500 Events/s."
- name: siem-backend-baseline
rules:
- alert: SiemBaselineNotEnoughSamples
expr: eventcollector_baseline_sample_count > 0 and eventcollector_baseline_sample_count < 24
for: 30m
labels:
severity: info
annotations:
summary: "Baseline lernt noch"
description: "Für {{ $labels.host }} / {{ $labels.channel }} / {{ $labels.event_id }} gibt es erst {{ $value }} Samples."
- alert: SiemBaselineCurrentFarAboveAverage
expr: eventcollector_baseline_avg_count > 0 and (eventcollector_baseline_current_count / eventcollector_baseline_avg_count) > 10
for: 2m
labels:
severity: warning
annotations:
summary: "Eventrate deutlich über Baseline"
description: "{{ $labels.host }} / {{ $labels.channel }} / {{ $labels.event_id }} liegt mehr als 10x über Durchschnitt."