Bugfix 2.0
Some checks failed
release-tag / release-image (push) Failing after 1m6s

This commit is contained in:
2026-07-23 17:33:00 +02:00
parent 466ca83238
commit 9e2f48ceae
24 changed files with 796 additions and 465 deletions

4
.dockerignore Normal file
View File

@@ -0,0 +1,4 @@
.git
.env
spool
*.zip

144
.env
View File

@@ -1,77 +1,85 @@
TZ=UTC
# ============================================================
# Greenfield SIEM - complete configuration
# deploy.sh / deploy.ps1 creates .env and replaces CHANGE_ME.
# ============================================================
LISTEN_ADDR=:8080
DB_DSN=eventuser:DEINPASSWORT@tcp(mariadb:3306)/eventcollector?parseTime=true&charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&loc=UTC
# Public SIEM endpoints
INGRESS_BIND=0.0.0.0
INGRESS_PORT=8090
UI_BIND=0.0.0.0
UI_PORT=8080
TENANT_ID=default
UI_USERNAME=admin
UI_PASSWORD=CHANGE_ME
ENROLLMENT_KEY=CHANGE_ME
DB_MAX_OPEN_CONNS=50
DB_MAX_IDLE_CONNS=25
DB_CONN_MAX_LIFETIME=3m
DB_CONN_MAX_IDLE_TIME=1m
# Ingress limits
MAX_BODY_BYTES=8388608
MAX_BATCH_EVENTS=1000
MAX_BODY_BYTES=10485760
HTTP_READ_TIMEOUT=15s
HTTP_WRITE_TIMEOUT=30s
HTTP_IDLE_TIMEOUT=60s
# PostgreSQL control plane
POSTGRES_IMAGE=postgres:18.4-alpine
POSTGRES_DB=siem
POSTGRES_USER=siem
POSTGRES_PASSWORD=CHANGE_ME
POSTGRES_BIND=127.0.0.1
POSTGRES_PORT=5432
DETECTION_INTERVAL=1m
OFFLINE_AFTER=10m
OFFLINE_ALERT_MAX=120m
FAILED_LOGON_WINDOW=5m
FAILED_LOGON_THRESHOLD=25
REBOOT_WINDOW=15m
REBOOT_THRESHOLD=3
PASSWORD_SPRAY_WINDOW=5m
PASSWORD_SPRAY_MIN_USERS=5
PASSWORD_SPRAY_MIN_ATTEMPTS=15
SUCCESS_AFTER_FAILURE_WINDOW=10m
NEW_SOURCE_IP_LOOKBACK=720h
NEW_SOURCE_IP_WINDOW=10m
DETECTIONS_LIMIT=100
# ClickHouse event store
CLICKHOUSE_IMAGE=clickhouse/clickhouse-server:26.6.2.81
CLICKHOUSE_DB=siem
CLICKHOUSE_USER=siem
CLICKHOUSE_PASSWORD=CHANGE_ME
CLICKHOUSE_HTTP_BIND=127.0.0.1
CLICKHOUSE_HTTP_PORT=8123
CLICKHOUSE_NATIVE_BIND=127.0.0.1
CLICKHOUSE_NATIVE_PORT=9000
EVENT_RETENTION_DAYS=90
ROLLUP_RETENTION_DAYS=730
MARIADB_DATABASE=eventcollector
MARIADB_USER=eventuser
MARIADB_PASSWORD=DEINPASSWORT
MARIADB_ROOT_PASSWORD=ROOTPASSWORT
# Redpanda durable ingest queue
REDPANDA_IMAGE=docker.redpanda.com/redpandadata/redpanda:v26.1.13
REDPANDA_CONSOLE_IMAGE=docker.redpanda.com/redpandadata/console:v3.8.0
REDPANDA_BIND=127.0.0.1
REDPANDA_KAFKA_PORT=19092
REDPANDA_ADMIN_PORT=19644
REDPANDA_CONSOLE_BIND=127.0.0.1
REDPANDA_CONSOLE_PORT=8081
REDPANDA_SMP=2
REDPANDA_LOG_LEVEL=info
KAFKA_TOPIC=siem-events
KAFKA_GROUP=siem-processor-v1
KAFKA_PARTITIONS=6
KAFKA_RETENTION_MS=86400000
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=admin
# Garage S3-compatible raw archive
GARAGE_IMAGE=dxflrs/garage:v2.3.0
ALPINE_IMAGE=alpine:3.22
GARAGE_BIND=127.0.0.1
GARAGE_S3_PORT=3900
GARAGE_ADMIN_PORT=3903
GARAGE_BUCKET=siem-raw
GARAGE_ACCESS_KEY=CHANGE_ME
GARAGE_SECRET_KEY=CHANGE_ME
GARAGE_RPC_SECRET=CHANGE_ME
GARAGE_ADMIN_TOKEN=CHANGE_ME
GARAGE_METRICS_TOKEN=CHANGE_ME
GARAGE_LOG_LEVEL=garage=info
RAW_ARCHIVE_ENABLED=true
RAW_RETENTION=720h
ARCHIVE_UPLOAD_INTERVAL_SECONDS=30
ENROLLMENT_KEY=BITTE_SEHR_LANG_UND_ZUFAELLIG
# Detector / API
DETECTOR_INTERVAL=30s
DETECTOR_LOOKBACK=20m
UI_QUERY_LIMIT=500
#Woche 1
BASELINE_WINDOW=15m
BASELINE_MIN_SAMPLES=72
BASELINE_MIN_COUNT=30
BASELINE_MEDIUM_Z=3.5
BASELINE_HIGH_Z=6.0
BASELINE_SUPPRESS_FOR=6h
# Prometheus
PROMETHEUS_IMAGE=prom/prometheus:v3.13.1
PROMETHEUS_BIND=127.0.0.1
PROMETHEUS_PORT=9090
PROMETHEUS_RETENTION=30d
#Woche 2-4
#BASELINE_WINDOW=15m
#BASELINE_MIN_SAMPLES=48
#BASELINE_MIN_COUNT=20
#BASELINE_MEDIUM_Z=3.0
#BASELINE_HIGH_Z=5.0
#BASELINE_SUPPRESS_FOR=4h
#Dauerlauf Aggressiv
#BASELINE_WINDOW=10m
#BASELINE_MIN_SAMPLES=36
#BASELINE_MIN_COUNT=15
#BASELINE_MEDIUM_Z=2.8
#BASELINE_HIGH_Z=4.5
#BASELINE_SUPPRESS_FOR=2h
#Dauerlauf Passiv
#BASELINE_WINDOW=15m
#BASELINE_MIN_SAMPLES=48
#BASELINE_MIN_COUNT=20
#BASELINE_MEDIUM_Z=3.0
#BASELINE_HIGH_Z=5.0
#BASELINE_SUPPRESS_FOR=4h
PARTITION_MAINTENANCE_ENABLED=true
PARTITION_MAINTENANCE_INTERVAL=15m
PARTITION_INTERVAL=3h
PARTITION_AHEAD=24h
PARTITION_BEHIND=6h
# Raw uploader and application image
RCLONE_IMAGE=rclone/rclone:1.74.4
APP_IMAGE=greenfield-siem-app:local

85
.env.example Normal file
View File

@@ -0,0 +1,85 @@
# ============================================================
# Greenfield SIEM - complete configuration
# deploy.sh / deploy.ps1 creates .env and replaces CHANGE_ME.
# ============================================================
# Public SIEM endpoints
INGRESS_BIND=0.0.0.0
INGRESS_PORT=8090
UI_BIND=0.0.0.0
UI_PORT=8080
TENANT_ID=default
UI_USERNAME=admin
UI_PASSWORD=CHANGE_ME
ENROLLMENT_KEY=CHANGE_ME
# Ingress limits
MAX_BODY_BYTES=8388608
MAX_BATCH_EVENTS=1000
# PostgreSQL control plane
POSTGRES_IMAGE=postgres:18.4-alpine
POSTGRES_DB=siem
POSTGRES_USER=siem
POSTGRES_PASSWORD=CHANGE_ME
POSTGRES_BIND=127.0.0.1
POSTGRES_PORT=5432
# ClickHouse event store
CLICKHOUSE_IMAGE=clickhouse/clickhouse-server:26.6.2.81
CLICKHOUSE_DB=siem
CLICKHOUSE_USER=siem
CLICKHOUSE_PASSWORD=CHANGE_ME
CLICKHOUSE_HTTP_BIND=127.0.0.1
CLICKHOUSE_HTTP_PORT=8123
CLICKHOUSE_NATIVE_BIND=127.0.0.1
CLICKHOUSE_NATIVE_PORT=9000
EVENT_RETENTION_DAYS=90
ROLLUP_RETENTION_DAYS=730
# Redpanda durable ingest queue
REDPANDA_IMAGE=docker.redpanda.com/redpandadata/redpanda:v26.1.13
REDPANDA_CONSOLE_IMAGE=docker.redpanda.com/redpandadata/console:v3.8.0
REDPANDA_BIND=127.0.0.1
REDPANDA_KAFKA_PORT=19092
REDPANDA_ADMIN_PORT=19644
REDPANDA_CONSOLE_BIND=127.0.0.1
REDPANDA_CONSOLE_PORT=8081
REDPANDA_SMP=2
REDPANDA_LOG_LEVEL=info
KAFKA_TOPIC=siem-events
KAFKA_GROUP=siem-processor-v1
KAFKA_PARTITIONS=6
KAFKA_RETENTION_MS=86400000
# Garage S3-compatible raw archive
GARAGE_IMAGE=dxflrs/garage:v2.3.0
ALPINE_IMAGE=alpine:3.22
GARAGE_BIND=127.0.0.1
GARAGE_S3_PORT=3900
GARAGE_ADMIN_PORT=3903
GARAGE_BUCKET=siem-raw
GARAGE_ACCESS_KEY=CHANGE_ME
GARAGE_SECRET_KEY=CHANGE_ME
GARAGE_RPC_SECRET=CHANGE_ME
GARAGE_ADMIN_TOKEN=CHANGE_ME
GARAGE_METRICS_TOKEN=CHANGE_ME
GARAGE_LOG_LEVEL=garage=info
RAW_ARCHIVE_ENABLED=true
RAW_RETENTION=720h
ARCHIVE_UPLOAD_INTERVAL_SECONDS=30
# Detector / API
DETECTOR_INTERVAL=30s
DETECTOR_LOOKBACK=20m
UI_QUERY_LIMIT=500
# Prometheus
PROMETHEUS_IMAGE=prom/prometheus:v3.13.1
PROMETHEUS_BIND=127.0.0.1
PROMETHEUS_PORT=9090
PROMETHEUS_RETENTION=30d
# Raw uploader and application image
RCLONE_IMAGE=rclone/rclone:1.74.4
APP_IMAGE=greenfield-siem-app:local

4
.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
.env
deploy/garage/garage.toml
spool/
*.zip

42
DEPLOYMENT-FIXES.md Normal file
View File

@@ -0,0 +1,42 @@
# Deployment-Fixes Release 1.1
Stand: 2026-07-23
Diese Revision repariert die Startprobleme der ersten Greenfield-Ausgabe, ohne die Event-Architektur wieder zu verändern.
## Behobene Ursachen
1. **Garage-Konfiguration war nicht Bestandteil des eigentlichen Compose-Lebenszyklus.**
Die alte Ausgabe erwartete eine zuvor auf dem Host erzeugte `deploy/garage/garage.toml`. Jetzt erzeugt der einmalige Service `garage-config` die Konfiguration in einem Docker-Volume. `garage` startet erst nach erfolgreicher Erzeugung.
2. **Garage Single-Node RPC-Adresse war ungeeignet.**
`rpc_public_addr` ist jetzt `127.0.0.1:3901`; der Server bindet intern weiterhin auf `0.0.0.0:3901`.
3. **ClickHouse hatte nur HTTP 8123 exponiert.**
Zusätzlich ist jetzt der native Client-Port 9000 konfigurierbar und standardmäßig nur auf `127.0.0.1` gebunden.
4. **`.env.example` war unvollständig.**
Sie enthält jetzt sämtliche 63 von Compose verwendeten Laufzeitparameter. `deploy.sh` und `deploy.ps1` ergänzen fehlende Schlüssel in einer bestehenden `.env`, ohne bestehende Secrets zu ersetzen. Eigene zusätzliche Schlüssel bleiben erhalten.
5. **Schema-Setup hing vom ersten Containerstart ab.**
`postgres-schema` und `clickhouse-schema` laufen idempotent nach erfolgreicher Datenbank-Readiness. Damit wird ein teilweise abgebrochener Erststart nicht mehr stillschweigend als vollständig initialisiert behandelt.
6. **ClickHouse-Datenbankname war im Go-Code teilweise fest auf `siem` verdrahtet.**
API, Detector und Inserts verwenden jetzt konsistent `CLICKHOUSE_DB`.
7. **Deployment-Fehler waren zu wenig sichtbar.**
Der One-Click-Start validiert zuerst `docker compose config`, prüft nach dem Start API, Ingress, ClickHouse und Garage und gibt bei Fehlschlag automatisch die relevanten Logs aus.
## Sichere Reparatur einer vorhandenen Installation
Die Daten-Volumes werden durch `./deploy.sh` **nicht** gelöscht.
Am sichersten ist es, die bestehende `.env` in die korrigierte Projektversion zu übernehmen und danach `./deploy.sh` auszuführen. So bleiben die bereits verwendeten PostgreSQL-/ClickHouse-/Garage-Secrets erhalten.
`./reset-DANGEROUS.sh` löscht dagegen die Docker-Volumes und darf bei erhaltenswerten Daten nicht verwendet werden.
## Diagnose
- `./credentials.sh` zeigt die tatsächlich verwendeten Zugangsdaten und Ports.
- `./doctor.sh` prüft Compose, API/Ingress, Garage, ClickHouse, Redpanda, PostgreSQL und den Raw-Spool.
- `docker compose logs --tail=200 garage garage-config clickhouse clickhouse-schema` zeigt die wichtigsten Startfehler.

View File

@@ -1,5 +1,7 @@
# Greenfield SIEM
**Release 1.1 Deployment-Fix:** Garage-Bootstrap, vollständige `.env`, ClickHouse-Native-Port und idempotentes ClickHouse-Schema wurden gegenüber der ersten Greenfield-Version korrigiert.
Kompletter Neuaufbau des bisherigen Projekts. Vom Altprojekt bleibt absichtlich nur der HTTP-Ingress-Vertrag erhalten.
## Was dieses Projekt löst
@@ -45,7 +47,7 @@ chmod +x deploy.sh
./deploy.ps1
```
Das Deployment erzeugt beim ersten Start kryptographisch zufällige Secrets in `.env`, erzeugt die Garage-Konfiguration, baut die SIEM-Binaries und startet alle Dienste.
Das Deployment vervollständigt bei jedem Start die `.env`, ohne vorhandene Secrets zu überschreiben. Fehlende Secrets werden kryptographisch zufällig erzeugt. Die Garage-Konfiguration wird anschließend automatisch von einem Compose-Init-Service erzeugt; ClickHouse-Schema und TTLs werden bei jedem Start idempotent geprüft. Danach werden die SIEM-Binaries gebaut und alle Dienste gestartet.
Danach:
@@ -56,10 +58,72 @@ Danach:
Beim ersten Start werden `UI_USERNAME=admin` und ein zufälliges `UI_PASSWORD` erzeugt. Browser fragen diese Zugangsdaten beim Öffnen von `/ui` ab.
Alle konfigurierbaren Parameter stehen in `.env.example`. `deploy.sh`/`deploy.ps1` ergänzt fehlende Schlüssel automatisch in einer vorhandenen `.env`; damit werden auch unvollständige `.env`-Dateien aus der ersten Greenfield-Version repariert. Zugangsdaten lassen sich jederzeit mit `./credentials.sh` anzeigen.
UI-Benutzer/Passwort und Enrollment-Key werden am Ende des Deployments ausgegeben und stehen außerdem in der geschützten `.env`. Die UI/API ist standardmäßig per HTTP Basic Auth geschützt; für Zugriff über ein fremdes Netz gehört zusätzlich TLS davor.
> Bestehende Collector können den bisherigen Enrollment-Key weiterverwenden, ohne eine `.env` von Hand anzulegen. Linux/macOS/WSL: `ENROLLMENT_KEY="ALTER_KEY" ./deploy.sh`. PowerShell: `$env:ENROLLMENT_KEY="ALTER_KEY"; ./deploy.ps1`. Alle anderen Secrets werden trotzdem zufällig erzeugt.
## Upgrade / Reparatur der ersten Greenfield-Version
Die erste ausgelieferte Greenfield-Version hatte drei Deployment-Probleme:
1. `deploy/garage/garage.toml` wurde erst auf dem Host erzeugt und fehlte im ZIP; außerdem war `rpc_public_addr` für das Single-Node-Setup ungünstig gesetzt.
2. ClickHouse exponierte nur HTTP `8123`, nicht den nativen Client-Port `9000`.
3. `.env` enthielt nur einen Teil der tatsächlich konfigurierbaren Parameter.
Mit dieser Version reicht im bestehenden Projektverzeichnis:
```bash
./deploy.sh
./doctor.sh
```
Das Deploy-Skript behält bestehende Secrets, ergänzt fehlende `.env`-Werte und erzeugt Garage-Konfiguration sowie ClickHouse-Schema über Compose-Init-Services neu. **Die Daten-Volumes werden dabei nicht gelöscht.**
Falls `.env` gelöscht wurde, aber alte PostgreSQL-Volumes mit einem unbekannten alten Passwort weiter existieren, kann dieses Passwort nicht automatisch rekonstruiert werden. In einer reinen Testinstallation ist dann `./reset-DANGEROUS.sh` der saubere Neustart; bei produktiven Daten zuerst Backup beziehungsweise Passwort-Recovery durchführen.
## ClickHouse-Zugang
ClickHouse besitzt zwei verschiedene Schnittstellen:
- HTTP: `127.0.0.1:8123`
- nativer ClickHouse-Client: `127.0.0.1:9000`
Benutzer, Passwort und Datenbank stehen vollständig in `.env`:
```env
CLICKHOUSE_DB=siem
CLICKHOUSE_USER=siem
CLICKHOUSE_PASSWORD=...
CLICKHOUSE_HTTP_PORT=8123
CLICKHOUSE_NATIVE_PORT=9000
```
Am einfachsten:
```bash
./credentials.sh
docker compose exec clickhouse clickhouse-client \
--user "$(grep '^CLICKHOUSE_USER=' .env | cut -d= -f2-)" \
--password "$(grep '^CLICKHOUSE_PASSWORD=' .env | cut -d= -f2-)" \
--database "$(grep '^CLICKHOUSE_DB=' .env | cut -d= -f2-)"
```
Für DBeaver/DataGrip oder einen lokalen `clickhouse-client` den nativen Port `9000` verwenden. Port `8123` ist die HTTP-Schnittstelle, keine Login-Weboberfläche.
## Garage-Zugang / Diagnose
Garage wird im Single-Node-Modus automatisch konfiguriert. Die Konfiguration liegt in einem Docker-Volume und muss nicht mehr als Host-Datei erzeugt werden. Prüfung:
```bash
docker compose exec garage /garage status
docker compose logs --tail=100 garage garage-config
```
S3 läuft lokal auf `127.0.0.1:3900`. Access-Key, Secret-Key und Bucket stehen in `.env`.
## Bestehender Ingress bleibt kompatibel
```http
@@ -253,8 +317,9 @@ ClickHouse Zeilenzahl / Speicher:
```bash
docker compose exec clickhouse clickhouse-client \
--user siem --password "$(grep '^CLICKHOUSE_PASSWORD=' .env | cut -d= -f2-)" \
--query "SELECT formatReadableQuantity(sum(rows)), formatReadableSize(sum(bytes_on_disk)) FROM system.parts WHERE database='siem' AND table='events' AND active"
--user "$(grep '^CLICKHOUSE_USER=' .env | cut -d= -f2-)" \
--password "$(grep '^CLICKHOUSE_PASSWORD=' .env | cut -d= -f2-)" \
--query "SELECT formatReadableQuantity(sum(rows)), formatReadableSize(sum(bytes_on_disk)) FROM system.parts WHERE database='$(grep '^CLICKHOUSE_DB=' .env | cut -d= -f2-)' AND table='events' AND active"
```
Queue:
@@ -320,7 +385,7 @@ doctor.sh Pipeline-/Kapazitätsdiagnose
raw-event.sh Raw-Archiv abrufen
deploy/clickhouse analytisches Schema
deploy/postgres Control-Plane-Schema
deploy/garage S3-Archivkonfiguration
compose.yml erzeugt Garage-Konfiguration zur Laufzeit aus `.env`
deploy/prometheus Betriebsmetriken
```

View File

@@ -1,24 +1,39 @@
# Release checks
Stand: 2026-07-23
Stand: 2026-07-23 Deployment-Fix 1.1
Durchgeführt:
- `compose.yml` mit YAML-Parser geladen: 15 Services, erfolgreich.
- Alle 63 nicht-escaped Compose-Variablen mit `.env.example` abgeglichen: **0 fehlende, 0 überzählige**.
- Sämtliche Host-Bind-Mount-Quellen aus `compose.yml` auf Existenz geprüft: **0 fehlende Pfade**.
- Prometheus-YAML geparst: erfolgreich.
- ClickHouse-Prometheus-XML geparst: erfolgreich.
- Garage-Konfigurationskommando aus Compose isoliert ausgeführt und resultierendes TOML mit `tomllib` geparst: erfolgreich.
- ClickHouse-Schema-Rendering aus Compose isoliert ausgeführt: Datenbankname und TTL-Platzhalter vollständig ersetzt; Schema-/TTL-Kommandos wurden aufgerufen.
- `deploy.sh` mit simuliertem Docker/HTTP end-to-end ausgeführt: erfolgreich.
- frische `.env`: 63 Schlüssel, keine `CHANGE_ME`-Secrets;
- externer bestehender Enrollment-Key bleibt erhalten;
- partielle Alt-`.env` wird vollständig ergänzt;
- vorhandene UI-/PostgreSQL-/ClickHouse-/Enrollment-Secrets bleiben unverändert;
- eigener unbekannter `.env`-Schlüssel bleibt erhalten.
- POSIX-Shellskripte mit `sh -n` geprüft: erfolgreich.
- `go test ./...` gegen lokale Interface-Stubs für pgx/kafka-go mit der vorhandenen Go-1.23.2-Toolchain: erfolgreich.
- `go vet ./...` unter denselben isolierten Bedingungen: erfolgreich.
- Ingress-Tests: Batch-UID/Retry-Idempotenz und Validierung: erfolgreich.
- Normalizer-Tests: Windows-Metadaten/XML und Lockout-Workstation: erfolgreich.
- `compose.yml` und Prometheus-YAML syntaktisch geparst: erfolgreich.
- ClickHouse-Prometheus-XML geparst: erfolgreich.
- POSIX-Shellskripte mit `sh -n` geprüft: erfolgreich.
- `deploy.sh` mit simuliertem Docker/Healthcheck end-to-end ausgeführt: Secret-Erzeugung, bestehender Enrollment-Key, Template-Rendering und Compose-Aufrufe erfolgreich.
- ZIP-Integrität mit `unzip -t`: erfolgreich.
Zusätzlich gegen aktuelle Primärquellen geprüft:
- Garage v2.3 Single-Node/Default-Bucket-Start und die erwarteten Default-Credential-Umgebungsvariablen.
- ClickHouse Docker: HTTP-Port 8123, nativer Client-Port 9000 sowie `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD` und Access-Management-Initialisierung.
- Redpanda Single-Broker Docker-Beispiel und aktuelle Broker-/Console-Versionen.
Nicht in dieser Ausführungsumgebung möglich:
- echter Docker-Compose-Start der Datenbanken/Broker/Object-Storage-Komponenten;
- echter Docker-Compose-Start der Datenbanken/Broker/Object-Storage-Komponenten, da hier kein Docker-Daemon/CLI verfügbar ist;
- Integrationstest gegen reale ClickHouse-, PostgreSQL-, Redpanda- und Garage-Container;
- Ausführung von `deploy.ps1`, da PowerShell hier nicht installiert ist;
- Build mit Go 1.26.5, da die lokale Umgebung keinen Netzwerkzugriff für die Toolchain/Module besitzt. Das Dockerfile verwendet Go 1.26.5.
- regulärer Build mit den realen externen Go-Modulen; die lokale Prüfung verwendete Schnittstellen-Stubs, da die vorhandene lokale Go-Version 1.23.2 ist, während das Dockerfile Go 1.26.5 verwendet.
Der erste reale Deployment-Lauf sollte daher anschließend mit `./doctor.sh` und einem begrenzten Stress-Test geprüft werden.
Der erste reale Lauf sollte deshalb direkt mit `./doctor.sh` verifiziert werden. Bei einem Startfehler gibt `deploy.sh` bereits automatisch die relevanten Service-Logs aus.

1
VERSION Normal file
View File

@@ -0,0 +1 @@
1.1.0-deployment-fix

View File

@@ -2,50 +2,97 @@ name: greenfield-siem
services:
postgres:
image: postgres:18.4-alpine
image: ${POSTGRES_IMAGE:-postgres:18.4-alpine}
environment:
POSTGRES_DB: siem
POSTGRES_USER: siem
POSTGRES_DB: ${POSTGRES_DB:-siem}
POSTGRES_USER: ${POSTGRES_USER:-siem}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
volumes:
- postgres_data:/var/lib/postgresql
- ./deploy/postgres/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
healthcheck:
test: ["CMD-SHELL", "pg_isready -U siem -d siem"]
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
interval: 5s
timeout: 3s
retries: 30
start_period: 5s
restart: unless-stopped
ports:
- "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
- "${POSTGRES_BIND:-127.0.0.1}:${POSTGRES_PORT:-5432}:5432"
postgres-schema:
image: ${POSTGRES_IMAGE:-postgres:18.4-alpine}
depends_on:
postgres:
condition: service_healthy
environment:
PGHOST: postgres
PGPORT: "5432"
PGDATABASE: ${POSTGRES_DB:-siem}
PGUSER: ${POSTGRES_USER:-siem}
PGPASSWORD: ${POSTGRES_PASSWORD}
volumes:
- ./deploy/postgres/init.sql:/schema/init.sql:ro
entrypoint: ["/bin/sh", "-ec"]
command: ["psql -v ON_ERROR_STOP=1 -f /schema/init.sql"]
restart: "no"
clickhouse:
image: clickhouse/clickhouse-server:26.6.2.81
image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81}
environment:
CLICKHOUSE_DB: siem
CLICKHOUSE_USER: siem
CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem}
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
volumes:
- clickhouse_data:/var/lib/clickhouse
- clickhouse_logs:/var/log/clickhouse-server
- ./deploy/clickhouse/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
- ./deploy/clickhouse/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
ulimits:
nofile:
soft: 262144
hard: 262144
healthcheck:
test: ["CMD-SHELL", "clickhouse-client --user siem --password $$CLICKHOUSE_PASSWORD --query 'SELECT 1' >/dev/null"]
test: ["CMD-SHELL", "clickhouse-client --host 127.0.0.1 --user $${CLICKHOUSE_USER} --password $${CLICKHOUSE_PASSWORD} --query 'SELECT 1' >/dev/null"]
interval: 5s
timeout: 5s
retries: 30
retries: 40
start_period: 10s
restart: unless-stopped
ports:
- "127.0.0.1:${CLICKHOUSE_HTTP_PORT:-8123}:8123"
- "${CLICKHOUSE_HTTP_BIND:-127.0.0.1}:${CLICKHOUSE_HTTP_PORT:-8123}:8123"
- "${CLICKHOUSE_NATIVE_BIND:-127.0.0.1}:${CLICKHOUSE_NATIVE_PORT:-9000}:9000"
clickhouse-schema:
image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81}
depends_on:
clickhouse:
condition: service_healthy
entrypoint: ["/bin/bash", "-ec"]
environment:
CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem}
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
EVENT_RETENTION_DAYS: ${EVENT_RETENTION_DAYS:-90}
ROLLUP_RETENTION_DAYS: ${ROLLUP_RETENTION_DAYS:-730}
volumes:
- ./deploy/clickhouse/init.sql.template:/schema/init.sql.template:ro
command:
- |
sed \
-e "s/__CLICKHOUSE_DB__/$${CLICKHOUSE_DB}/g" \
-e "s/__EVENT_RETENTION_DAYS__/$${EVENT_RETENTION_DAYS}/g" \
-e "s/__ROLLUP_RETENTION_DAYS__/$${ROLLUP_RETENTION_DAYS}/g" \
/schema/init.sql.template > /tmp/init.sql
clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --multiquery < /tmp/init.sql
clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \
"ALTER TABLE $${CLICKHOUSE_DB}.events MODIFY TTL event_time + INTERVAL $${EVENT_RETENTION_DAYS} DAY DELETE"
clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \
"ALTER TABLE $${CLICKHOUSE_DB}.events_5m MODIFY TTL bucket + INTERVAL $${ROLLUP_RETENTION_DAYS} DAY DELETE"
restart: "no"
redpanda:
image: docker.redpanda.com/redpandadata/redpanda:v26.1.13
image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13}
command:
- redpanda
- start
@@ -60,14 +107,14 @@ services:
- --schema-registry-addr
- internal://0.0.0.0:8081,external://0.0.0.0:18081
- --rpc-addr
- redpanda:33145
- 0.0.0.0:33145
- --advertise-rpc-addr
- redpanda:33145
- --mode
- dev-container
- --smp
- "2"
- --default-log-level=info
- "${REDPANDA_SMP:-2}"
- --default-log-level=${REDPANDA_LOG_LEVEL:-info}
volumes:
- redpanda_data:/var/lib/redpanda/data
healthcheck:
@@ -75,32 +122,36 @@ services:
interval: 5s
timeout: 5s
retries: 40
start_period: 10s
restart: unless-stopped
ports:
- "127.0.0.1:${REDPANDA_KAFKA_PORT:-19092}:19092"
- "127.0.0.1:${REDPANDA_ADMIN_PORT:-19644}:9644"
- "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_KAFKA_PORT:-19092}:19092"
- "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_ADMIN_PORT:-19644}:9644"
redpanda-init:
image: docker.redpanda.com/redpandadata/redpanda:v26.1.13
image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13}
depends_on:
redpanda:
condition: service_healthy
entrypoint: ["/bin/sh", "-ec"]
environment:
RPK_BROKERS: redpanda:9092
command: >-
rpk topic create --if-not-exists -p ${KAFKA_PARTITIONS:-6} -r 1
-c retention.ms=${KAFKA_RETENTION_MS:-86400000}
siem-events
KAFKA_PARTITIONS: ${KAFKA_PARTITIONS:-6}
KAFKA_RETENTION_MS: ${KAFKA_RETENTION_MS:-86400000}
KAFKA_TOPIC: ${KAFKA_TOPIC:-siem-events}
command:
- |
rpk topic create --if-not-exists -p "$${KAFKA_PARTITIONS}" -r 1 \
-c retention.ms="$${KAFKA_RETENTION_MS}" "$${KAFKA_TOPIC}"
restart: "no"
redpanda-console:
image: docker.redpanda.com/redpandadata/console:v3.8.0
image: ${REDPANDA_CONSOLE_IMAGE:-docker.redpanda.com/redpandadata/console:v3.8.0}
depends_on:
redpanda:
condition: service_healthy
entrypoint: /bin/sh
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
command: -c 'printf "%s" "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
environment:
CONFIG_FILEPATH: /tmp/config.yml
CONSOLE_CONFIG_FILE: |
@@ -115,46 +166,93 @@ services:
urls: ["http://redpanda:9644"]
restart: unless-stopped
ports:
- "127.0.0.1:${REDPANDA_CONSOLE_PORT:-8081}:8080"
- "${REDPANDA_CONSOLE_BIND:-127.0.0.1}:${REDPANDA_CONSOLE_PORT:-8081}:8080"
garage-config:
image: ${ALPINE_IMAGE:-alpine:3.22}
environment:
GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET}
GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN}
GARAGE_METRICS_TOKEN: ${GARAGE_METRICS_TOKEN}
volumes:
- garage_config:/config
entrypoint: ["/bin/sh", "-ec"]
command:
- |
umask 077
cat > /config/garage.toml <<CFG
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "0.0.0.0:3901"
rpc_public_addr = "127.0.0.1:3901"
rpc_secret = "$${GARAGE_RPC_SECRET}"
[s3_api]
s3_region = "garage"
api_bind_addr = "0.0.0.0:3900"
root_domain = ".s3.garage.localhost"
[admin]
api_bind_addr = "0.0.0.0:3903"
admin_token = "$${GARAGE_ADMIN_TOKEN}"
metrics_token = "$${GARAGE_METRICS_TOKEN}"
CFG
test -s /config/garage.toml
restart: "no"
garage:
image: dxflrs/garage:v2.3.0
image: ${GARAGE_IMAGE:-dxflrs/garage:v2.3.0}
command: ["/garage", "server", "--single-node", "--default-bucket"]
environment:
GARAGE_CONFIG_FILE: /config/garage.toml
GARAGE_DEFAULT_ACCESS_KEY: ${GARAGE_ACCESS_KEY}
GARAGE_DEFAULT_SECRET_KEY: ${GARAGE_SECRET_KEY}
GARAGE_DEFAULT_BUCKET: ${GARAGE_BUCKET:-siem-raw}
RUST_LOG: ${GARAGE_LOG_LEVEL:-garage=info}
volumes:
- garage_config:/config:ro
- garage_meta:/var/lib/garage/meta
- garage_data:/var/lib/garage/data
- ./deploy/garage/garage.toml:/etc/garage.toml:ro
depends_on:
garage-config:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "/garage", "status"]
interval: 5s
timeout: 5s
retries: 40
start_period: 10s
restart: unless-stopped
ports:
- "127.0.0.1:${GARAGE_S3_PORT:-3900}:3900"
- "127.0.0.1:${GARAGE_ADMIN_PORT:-3903}:3903"
- "${GARAGE_BIND:-127.0.0.1}:${GARAGE_S3_PORT:-3900}:3900"
- "${GARAGE_BIND:-127.0.0.1}:${GARAGE_ADMIN_PORT:-3903}:3903"
ingress:
image: greenfield-siem-app:local
image: ${APP_IMAGE:-greenfield-siem-app:local}
build: .
command: ["ingress"]
environment: &app-env
SERVICE_ADDR: :8080
POSTGRES_URL: postgres://siem:${POSTGRES_PASSWORD}@postgres:5432/siem?sslmode=disable
POSTGRES_URL: postgres://${POSTGRES_USER:-siem}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-siem}?sslmode=disable
CLICKHOUSE_URL: http://clickhouse:8123
CLICKHOUSE_USER: siem
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
KAFKA_BROKERS: redpanda:9092
KAFKA_TOPIC: siem-events
KAFKA_TOPIC: ${KAFKA_TOPIC:-siem-events}
TENANT_ID: ${TENANT_ID:-default}
UI_USERNAME: ${UI_USERNAME:-admin}
UI_PASSWORD: ${UI_PASSWORD}
ENROLLMENT_KEY: ${ENROLLMENT_KEY}
MAX_BODY_BYTES: ${MAX_BODY_BYTES:-8388608}
MAX_BATCH_EVENTS: ${MAX_BATCH_EVENTS:-1000}
RAW_ARCHIVE_ENABLED: ${RAW_ARCHIVE_ENABLED:-true}
RAW_SPOOL_DIR: /var/spool/siem-raw
RAW_RETENTION: ${RAW_RETENTION:-720h}
depends_on:
postgres:
condition: service_healthy
postgres-schema:
condition: service_completed_successfully
redpanda-init:
condition: service_completed_successfully
restart: unless-stopped
@@ -162,61 +260,65 @@ services:
- "${INGRESS_BIND:-0.0.0.0}:${INGRESS_PORT:-8090}:8080"
processor:
image: greenfield-siem-app:local
image: ${APP_IMAGE:-greenfield-siem-app:local}
build: .
command: ["processor"]
user: "0:0"
environment:
<<: *app-env
KAFKA_GROUP: siem-processor-v1
KAFKA_GROUP: ${KAFKA_GROUP:-siem-processor-v1}
volumes:
- raw_spool:/var/spool/siem-raw
depends_on:
clickhouse:
condition: service_healthy
clickhouse-schema:
condition: service_completed_successfully
redpanda-init:
condition: service_completed_successfully
restart: unless-stopped
detector:
image: greenfield-siem-app:local
image: ${APP_IMAGE:-greenfield-siem-app:local}
build: .
command: ["detector"]
environment:
<<: *app-env
DETECTOR_INTERVAL: ${DETECTOR_INTERVAL:-30s}
DETECTOR_LOOKBACK: ${DETECTOR_LOOKBACK:-20m}
depends_on:
clickhouse:
condition: service_healthy
postgres:
condition: service_healthy
clickhouse-schema:
condition: service_completed_successfully
postgres-schema:
condition: service_completed_successfully
processor:
condition: service_started
restart: unless-stopped
api:
image: greenfield-siem-app:local
image: ${APP_IMAGE:-greenfield-siem-app:local}
build: .
command: ["api"]
environment:
<<: *app-env
UI_QUERY_LIMIT: ${UI_QUERY_LIMIT:-500}
depends_on:
clickhouse:
condition: service_healthy
postgres:
condition: service_healthy
clickhouse-schema:
condition: service_completed_successfully
postgres-schema:
condition: service_completed_successfully
restart: unless-stopped
ports:
- "${UI_BIND:-0.0.0.0}:${UI_PORT:-8080}:8080"
archive-uploader:
image: rclone/rclone:1.74.4
image: ${RCLONE_IMAGE:-rclone/rclone:1.74.4}
entrypoint: ["/bin/sh", "-ec"]
command: |
while true; do
rclone move /spool garage:${GARAGE_BUCKET:-siem-raw} --exclude '*.tmp' --create-empty-src-dirs --transfers 8 --checkers 8 --retries 5 || true
rclone delete garage:${GARAGE_BUCKET:-siem-raw} --min-age ${RAW_RETENTION:-720h} --retries 5 || true
sleep 30
done
command:
- |
while true; do
rclone move /spool garage:${GARAGE_BUCKET:-siem-raw} --exclude '*.tmp' --create-empty-src-dirs --transfers 8 --checkers 8 --retries 5 || true
rclone delete garage:${GARAGE_BUCKET:-siem-raw} --min-age ${RAW_RETENTION:-720h} --retries 5 || true
sleep ${ARCHIVE_UPLOAD_INTERVAL_SECONDS:-30}
done
environment:
RCLONE_CONFIG_GARAGE_TYPE: s3
RCLONE_CONFIG_GARAGE_PROVIDER: Other
@@ -228,13 +330,17 @@ services:
volumes:
- raw_spool:/spool
depends_on:
- garage
- processor
garage:
condition: service_healthy
processor:
condition: service_started
restart: unless-stopped
prometheus:
image: prom/prometheus:v3.13.1
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.retention.time=30d"]
image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.13.1}
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d}
volumes:
- ./deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
@@ -243,13 +349,14 @@ services:
- redpanda
restart: unless-stopped
ports:
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
- "${PROMETHEUS_BIND:-127.0.0.1}:${PROMETHEUS_PORT:-9090}:9090"
volumes:
postgres_data:
clickhouse_data:
clickhouse_logs:
redpanda_data:
garage_config:
garage_meta:
garage_data:
raw_spool:

30
credentials.sh Normal file
View File

@@ -0,0 +1,30 @@
#!/usr/bin/env sh
set -eu
cd "$(dirname "$0")"
[ -f .env ] || { echo ".env fehlt. Zuerst ./deploy.sh ausführen." >&2; exit 1; }
set -a; . ./.env; set +a
cat <<EOT
UI
URL: http://127.0.0.1:${UI_PORT:-8080}/ui
User: ${UI_USERNAME:-admin}
Passwort: ${UI_PASSWORD}
ClickHouse
Datenbank: ${CLICKHOUSE_DB:-siem}
User: ${CLICKHOUSE_USER:-siem}
Passwort: ${CLICKHOUSE_PASSWORD}
HTTP: http://127.0.0.1:${CLICKHOUSE_HTTP_PORT:-8123}
Native: 127.0.0.1:${CLICKHOUSE_NATIVE_PORT:-9000}
CLI im Container:
docker compose exec clickhouse clickhouse-client --user ${CLICKHOUSE_USER:-siem} --password '${CLICKHOUSE_PASSWORD}' --database ${CLICKHOUSE_DB:-siem}
PostgreSQL
Datenbank: ${POSTGRES_DB:-siem}
User: ${POSTGRES_USER:-siem}
Passwort: ${POSTGRES_PASSWORD}
Host: 127.0.0.1:${POSTGRES_PORT:-5432}
Ingress
Enrollment-Key: ${ENROLLMENT_KEY}
EOT

View File

@@ -2,69 +2,108 @@ $ErrorActionPreference = "Stop"
Set-Location $PSScriptRoot
if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { throw "Docker fehlt." }
docker compose version | Out-Null
function Hex([int]$bytes) { $b = New-Object byte[] $bytes; [Security.Cryptography.RandomNumberGenerator]::Fill($b); return ([BitConverter]::ToString($b)).Replace("-","").ToLower() }
if (-not (Test-Path .env)) {
$presetEnrollment = $env:ENROLLMENT_KEY
$enrollment = if ([string]::IsNullOrWhiteSpace($presetEnrollment)) { Hex 32 } else { $presetEnrollment }
$vals = @{
POSTGRES_PASSWORD=Hex 24; CLICKHOUSE_PASSWORD=Hex 24; ENROLLMENT_KEY=$enrollment; UI_PASSWORD=Hex 18;
GARAGE_ACCESS_KEY="GK$(Hex 16)"; GARAGE_SECRET_KEY=Hex 32; GARAGE_RPC_SECRET=Hex 32;
GARAGE_ADMIN_TOKEN=Hex 32; GARAGE_METRICS_TOKEN=Hex 32
}
@"
INGRESS_BIND=0.0.0.0
INGRESS_PORT=8090
UI_BIND=0.0.0.0
UI_PORT=8080
TENANT_ID=default
UI_USERNAME=admin
UI_PASSWORD=$($vals.UI_PASSWORD)
POSTGRES_PASSWORD=$($vals.POSTGRES_PASSWORD)
CLICKHOUSE_PASSWORD=$($vals.CLICKHOUSE_PASSWORD)
ENROLLMENT_KEY=$($vals.ENROLLMENT_KEY)
GARAGE_ACCESS_KEY=$($vals.GARAGE_ACCESS_KEY)
GARAGE_SECRET_KEY=$($vals.GARAGE_SECRET_KEY)
GARAGE_RPC_SECRET=$($vals.GARAGE_RPC_SECRET)
GARAGE_ADMIN_TOKEN=$($vals.GARAGE_ADMIN_TOKEN)
GARAGE_METRICS_TOKEN=$($vals.GARAGE_METRICS_TOKEN)
GARAGE_BUCKET=siem-raw
EVENT_RETENTION_DAYS=90
ROLLUP_RETENTION_DAYS=730
RAW_RETENTION=720h
RAW_ARCHIVE_ENABLED=true
KAFKA_RETENTION_MS=86400000
KAFKA_PARTITIONS=6
DETECTOR_INTERVAL=30s
DETECTOR_LOOKBACK=20m
UI_QUERY_LIMIT=500
"@ | Set-Content .env
function Hex([int]$bytes) {
$b = New-Object byte[] $bytes
[Security.Cryptography.RandomNumberGenerator]::Fill($b)
return ([BitConverter]::ToString($b)).Replace("-","").ToLower()
}
$envMap=@{}; Get-Content .env | ForEach-Object { if ($_ -match '^([^#=]+)=(.*)$') {$envMap[$matches[1]]=$matches[2]} }
if (-not $envMap.ContainsKey('UI_USERNAME')) { $envMap['UI_USERNAME']='admin'; Add-Content .env "UI_USERNAME=admin" }
if (-not $envMap.ContainsKey('UI_PASSWORD') -or [string]::IsNullOrWhiteSpace($envMap['UI_PASSWORD'])) { $envMap['UI_PASSWORD']=Hex 18; Add-Content .env "UI_PASSWORD=$($envMap.UI_PASSWORD)" }
(Get-Content deploy/garage/garage.toml.template -Raw).Replace('__GARAGE_RPC_SECRET__',$envMap.GARAGE_RPC_SECRET).Replace('__GARAGE_ADMIN_TOKEN__',$envMap.GARAGE_ADMIN_TOKEN).Replace('__GARAGE_METRICS_TOKEN__',$envMap.GARAGE_METRICS_TOKEN) | Set-Content deploy/garage/garage.toml
(Get-Content deploy/clickhouse/init.sql.template -Raw).Replace('__EVENT_RETENTION_DAYS__',$envMap.EVENT_RETENTION_DAYS).Replace('__ROLLUP_RETENTION_DAYS__',$envMap.ROLLUP_RETENTION_DAYS) | Set-Content deploy/clickhouse/init.sql
function Read-EnvFile([string]$path) {
$map = [ordered]@{}
if (Test-Path $path) {
foreach ($line in [IO.File]::ReadAllLines($path)) {
if ($line -match '^([^#=][^=]*)=(.*)$') { $map[$matches[1]] = $matches[2] }
}
}
return $map
}
$template = Read-EnvFile ".env.example"
$current = Read-EnvFile ".env"
foreach ($k in $template.Keys) {
if (-not $current.Contains($k)) { $current[$k] = $template[$k] }
}
$presetEnrollment = $env:ENROLLMENT_KEY
$secretLengths = @{
POSTGRES_PASSWORD=24; CLICKHOUSE_PASSWORD=24; UI_PASSWORD=18;
GARAGE_SECRET_KEY=32; GARAGE_RPC_SECRET=32; GARAGE_ADMIN_TOKEN=32; GARAGE_METRICS_TOKEN=32
}
foreach ($k in $secretLengths.Keys) {
if ([string]::IsNullOrWhiteSpace($current[$k]) -or $current[$k] -eq 'CHANGE_ME') {
$current[$k] = Hex $secretLengths[$k]
}
}
if ([string]::IsNullOrWhiteSpace($current['GARAGE_ACCESS_KEY']) -or $current['GARAGE_ACCESS_KEY'] -eq 'CHANGE_ME') {
$current['GARAGE_ACCESS_KEY'] = "GK$(Hex 16)"
}
if (-not [string]::IsNullOrWhiteSpace($presetEnrollment)) {
$current['ENROLLMENT_KEY'] = $presetEnrollment
} elseif ([string]::IsNullOrWhiteSpace($current['ENROLLMENT_KEY']) -or $current['ENROLLMENT_KEY'] -eq 'CHANGE_ME') {
$current['ENROLLMENT_KEY'] = Hex 32
}
# Write a complete UTF-8 .env. This fixes partial .env files from earlier releases
# and preserves custom keys that are not part of the current template.
$lines = New-Object System.Collections.Generic.List[string]
$templateKeys = New-Object 'System.Collections.Generic.HashSet[string]'
foreach ($line in [IO.File]::ReadAllLines(".env.example")) {
if ($line -match '^([^#=][^=]*)=(.*)$') {
$key = $matches[1]
[void]$templateKeys.Add($key)
$lines.Add("$key=$($current[$key])")
} else {
$lines.Add($line)
}
}
foreach ($key in $current.Keys) {
if (-not $templateKeys.Contains([string]$key)) {
$lines.Add("$key=$($current[$key])")
}
}
$utf8NoBom = New-Object System.Text.UTF8Encoding($false)
[IO.File]::WriteAllLines((Join-Path $PSScriptRoot '.env'), $lines, $utf8NoBom)
foreach ($key in @('POSTGRES_PASSWORD','CLICKHOUSE_PASSWORD','UI_PASSWORD','ENROLLMENT_KEY','GARAGE_ACCESS_KEY','GARAGE_SECRET_KEY','GARAGE_RPC_SECRET','GARAGE_ADMIN_TOKEN','GARAGE_METRICS_TOKEN')) {
if ([string]::IsNullOrWhiteSpace($current[$key]) -or $current[$key] -eq 'CHANGE_ME') {
throw "Ungueltige .env: $key fehlt oder ist noch CHANGE_ME."
}
}
Write-Host "Pruefe Compose-Konfiguration ..."
docker compose config | Out-Null
Write-Host "Baue und starte Greenfield SIEM ..."
docker compose build --pull
docker compose up -d --remove-orphans
Write-Host "Pruefe Dienste ..."
Write-Host "Pruefe Readiness ..."
$healthy = $false
for ($i = 0; $i -lt 60; $i++) {
try {
Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($envMap.UI_PORT)/readyz" | Out-Null
Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($envMap.INGRESS_PORT)/readyz" | Out-Null
$healthy = $true
break
} catch {
Start-Sleep -Seconds 2
}
for ($i = 0; $i -lt 90; $i++) {
$apiOk = $false; $ingressOk = $false; $chOk = $false; $garageOk = $false
try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.UI_PORT)/readyz" | Out-Null; $apiOk = $true } catch {}
try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.INGRESS_PORT)/readyz" | Out-Null; $ingressOk = $true } catch {}
docker compose exec -T clickhouse clickhouse-client --user $current.CLICKHOUSE_USER --password $current.CLICKHOUSE_PASSWORD --query "SELECT 1" *> $null
if ($LASTEXITCODE -eq 0) { $chOk = $true }
docker compose exec -T garage /garage status *> $null
if ($LASTEXITCODE -eq 0) { $garageOk = $true }
if ($apiOk -and $ingressOk -and $chOk -and $garageOk) { $healthy = $true; break }
Start-Sleep -Seconds 2
}
if (-not $healthy) { throw "Deployment gestartet, Healthcheck noch nicht gruen. Pruefe: docker compose ps; docker compose logs" }
if (-not $healthy) {
Write-Host "Mindestens ein Dienst wurde nicht bereit. Diagnose:" -ForegroundColor Red
docker compose ps
docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor api
throw "Deployment nicht vollstaendig bereit."
}
Write-Host ""
Write-Host "Greenfield SIEM laeuft."
Write-Host "UI: http://127.0.0.1:$($envMap.UI_PORT)/ui"
Write-Host "Ingress: http://127.0.0.1:$($envMap.INGRESS_PORT)/ingest"
Write-Host "UI-Login: $($envMap.UI_USERNAME) / $($envMap.UI_PASSWORD)"
Write-Host "Enrollment-Key: $($envMap.ENROLLMENT_KEY)"
Write-Host "Secrets stehen in .env (Datei schuetzen und sichern)."
Write-Host "UI: http://127.0.0.1:$($current.UI_PORT)/ui"
Write-Host "Ingress: http://127.0.0.1:$($current.INGRESS_PORT)/ingest"
Write-Host "UI-Login: $($current.UI_USERNAME) / $($current.UI_PASSWORD)"
Write-Host "ClickHouse: user=$($current.CLICKHOUSE_USER) db=$($current.CLICKHOUSE_DB) http=127.0.0.1:$($current.CLICKHOUSE_HTTP_PORT) native=127.0.0.1:$($current.CLICKHOUSE_NATIVE_PORT)"
Write-Host "Enrollment-Key: $($current.ENROLLMENT_KEY)"
Write-Host "Alle Parameter stehen vollstaendig in .env."

190
deploy.sh
View File

@@ -1,88 +1,140 @@
#!/usr/bin/env sh
set -eu
cd "$(dirname "$0")"
command -v docker >/dev/null 2>&1 || { echo "Docker fehlt." >&2; exit 1; }
docker compose version >/dev/null 2>&1 || { echo "Docker Compose Plugin fehlt." >&2; exit 1; }
PRESET_ENROLLMENT_KEY="${ENROLLMENT_KEY:-}"
randhex(){ n="$1"; if command -v openssl >/dev/null 2>&1; then openssl rand -hex "$n"; else dd if=/dev/urandom bs="$n" count=1 2>/dev/null | od -An -tx1 | tr -d ' \n'; fi; }
http_ok(){
url="$1"
if command -v curl >/dev/null 2>&1; then curl -fsS "$url" >/dev/null 2>&1; return $?; fi
if command -v wget >/dev/null 2>&1; then wget -q -O /dev/null "$url" >/dev/null 2>&1; return $?; fi
return 2
randhex() {
n="$1"
if command -v openssl >/dev/null 2>&1; then
openssl rand -hex "$n"
else
dd if=/dev/urandom bs="$n" count=1 2>/dev/null | od -An -tx1 | tr -d ' \n'
fi
}
if [ ! -f .env ]; then
POSTGRES_PASSWORD="$(randhex 24)"
CLICKHOUSE_PASSWORD="$(randhex 24)"
ENROLLMENT_KEY="${PRESET_ENROLLMENT_KEY:-$(randhex 32)}"
UI_PASSWORD="$(randhex 18)"
GARAGE_ACCESS_KEY="GK$(randhex 16)"
GARAGE_SECRET_KEY="$(randhex 32)"
GARAGE_RPC_SECRET="$(randhex 32)"
GARAGE_ADMIN_TOKEN="$(randhex 32)"
GARAGE_METRICS_TOKEN="$(randhex 32)"
cat > .env <<EOT
INGRESS_BIND=0.0.0.0
INGRESS_PORT=8090
UI_BIND=0.0.0.0
UI_PORT=8080
TENANT_ID=default
UI_USERNAME=admin
UI_PASSWORD=$UI_PASSWORD
POSTGRES_PASSWORD=$POSTGRES_PASSWORD
CLICKHOUSE_PASSWORD=$CLICKHOUSE_PASSWORD
ENROLLMENT_KEY=$ENROLLMENT_KEY
GARAGE_ACCESS_KEY=$GARAGE_ACCESS_KEY
GARAGE_SECRET_KEY=$GARAGE_SECRET_KEY
GARAGE_RPC_SECRET=$GARAGE_RPC_SECRET
GARAGE_ADMIN_TOKEN=$GARAGE_ADMIN_TOKEN
GARAGE_METRICS_TOKEN=$GARAGE_METRICS_TOKEN
GARAGE_BUCKET=siem-raw
EVENT_RETENTION_DAYS=90
ROLLUP_RETENTION_DAYS=730
RAW_RETENTION=720h
RAW_ARCHIVE_ENABLED=true
KAFKA_RETENTION_MS=86400000
KAFKA_PARTITIONS=6
DETECTOR_INTERVAL=30s
DETECTOR_LOOKBACK=20m
UI_QUERY_LIMIT=500
EOT
chmod 600 .env 2>/dev/null || true
get_env() {
key="$1"
awk -F= -v k="$key" '$1==k {sub(/^[^=]*=/, ""); print; exit}' .env 2>/dev/null || true
}
set_env() {
key="$1"; value="$2"; tmp=".env.tmp.$$"
awk -v k="$key" -v v="$value" '
BEGIN {done=0}
index($0,k"=")==1 {print k"="v; done=1; next}
{print}
END {if(!done) print k"="v}
' .env > "$tmp"
mv "$tmp" .env
}
# Start from a complete template, but preserve all values from an existing .env.
[ -f .env ] || : > .env
while IFS= read -r line || [ -n "$line" ]; do
case "$line" in
''|'#'*) continue ;;
esac
key=${line%%=*}
if ! grep -q "^${key}=" .env 2>/dev/null; then
printf '%s\n' "$line" >> .env
fi
done < .env.example
# Generate secrets only if absent/blank/placeholder. Existing installations keep their credentials.
for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD UI_PASSWORD GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do
val="$(get_env "$key")"
if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then
case "$key" in
POSTGRES_PASSWORD|CLICKHOUSE_PASSWORD) val="$(randhex 24)" ;;
UI_PASSWORD) val="$(randhex 18)" ;;
*) val="$(randhex 32)" ;;
esac
set_env "$key" "$val"
fi
done
val="$(get_env GARAGE_ACCESS_KEY)"
if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then
set_env GARAGE_ACCESS_KEY "GK$(randhex 16)"
fi
set -a; . ./.env; set +a
UI_USERNAME="${UI_USERNAME:-admin}"
if [ -z "${UI_PASSWORD:-}" ]; then
UI_PASSWORD="$(randhex 18)"
printf '\nUI_USERNAME=%s\nUI_PASSWORD=%s\n' "$UI_USERNAME" "$UI_PASSWORD" >> .env
export UI_USERNAME UI_PASSWORD
val="$(get_env ENROLLMENT_KEY)"
if [ -n "$PRESET_ENROLLMENT_KEY" ]; then
set_env ENROLLMENT_KEY "$PRESET_ENROLLMENT_KEY"
elif [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then
set_env ENROLLMENT_KEY "$(randhex 32)"
fi
sed -e "s/__GARAGE_RPC_SECRET__/$GARAGE_RPC_SECRET/g" -e "s/__GARAGE_ADMIN_TOKEN__/$GARAGE_ADMIN_TOKEN/g" -e "s/__GARAGE_METRICS_TOKEN__/$GARAGE_METRICS_TOKEN/g" deploy/garage/garage.toml.template > deploy/garage/garage.toml
sed -e "s/__EVENT_RETENTION_DAYS__/${EVENT_RETENTION_DAYS:-90}/g" -e "s/__ROLLUP_RETENTION_DAYS__/${ROLLUP_RETENTION_DAYS:-730}/g" deploy/clickhouse/init.sql.template > deploy/clickhouse/init.sql
chmod 600 deploy/garage/garage.toml 2>/dev/null || true
chmod 600 .env 2>/dev/null || true
# Validate required settings without sourcing .env as shell code. Docker Compose parses
# .env itself; this also keeps values with shell-special characters safe.
for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD UI_PASSWORD ENROLLMENT_KEY GARAGE_ACCESS_KEY GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do
val="$(get_env "$key")"
if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then
echo "Ungültige .env: $key fehlt oder ist noch CHANGE_ME." >&2
exit 1
fi
done
UI_PORT_VALUE="$(get_env UI_PORT)"; [ -n "$UI_PORT_VALUE" ] || UI_PORT_VALUE=8080
INGRESS_PORT_VALUE="$(get_env INGRESS_PORT)"; [ -n "$INGRESS_PORT_VALUE" ] || INGRESS_PORT_VALUE=8090
CLICKHOUSE_USER_VALUE="$(get_env CLICKHOUSE_USER)"; [ -n "$CLICKHOUSE_USER_VALUE" ] || CLICKHOUSE_USER_VALUE=siem
CLICKHOUSE_DB_VALUE="$(get_env CLICKHOUSE_DB)"; [ -n "$CLICKHOUSE_DB_VALUE" ] || CLICKHOUSE_DB_VALUE=siem
CLICKHOUSE_PASSWORD_VALUE="$(get_env CLICKHOUSE_PASSWORD)"
CLICKHOUSE_HTTP_PORT_VALUE="$(get_env CLICKHOUSE_HTTP_PORT)"; [ -n "$CLICKHOUSE_HTTP_PORT_VALUE" ] || CLICKHOUSE_HTTP_PORT_VALUE=8123
CLICKHOUSE_NATIVE_PORT_VALUE="$(get_env CLICKHOUSE_NATIVE_PORT)"; [ -n "$CLICKHOUSE_NATIVE_PORT_VALUE" ] || CLICKHOUSE_NATIVE_PORT_VALUE=9000
UI_USERNAME_VALUE="$(get_env UI_USERNAME)"; [ -n "$UI_USERNAME_VALUE" ] || UI_USERNAME_VALUE=admin
UI_PASSWORD_VALUE="$(get_env UI_PASSWORD)"
ENROLLMENT_KEY_VALUE="$(get_env ENROLLMENT_KEY)"
# Validate the final config before pulling/building anything.
echo "Prüfe Compose-Konfiguration ..."
docker compose config >/dev/null
echo "Baue und starte Greenfield SIEM ..."
docker compose build --pull
docker compose up -d --remove-orphans
echo "Prüfe Dienste ..."
if command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; then
i=0
while [ "$i" -lt 60 ]; do
if http_ok "http://127.0.0.1:${UI_PORT:-8080}/readyz" && http_ok "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz"; then break; fi
i=$((i+1)); sleep 2
done
if [ "$i" -ge 60 ]; then echo "Deployment gestartet, Healthcheck noch nicht grün. Prüfe: docker compose ps && docker compose logs" >&2; exit 1; fi
else
echo "Hinweis: curl/wget fehlt; HTTP-Healthcheck wird übersprungen."
docker compose ps
http_ok() {
url="$1"
if command -v curl >/dev/null 2>&1; then curl -fsS "$url" >/dev/null 2>&1; return $?; fi
if command -v wget >/dev/null 2>&1; then wget -q -O /dev/null "$url" >/dev/null 2>&1; return $?; fi
return 2
}
echo "Prüfe Readiness ..."
i=0
while [ "$i" -lt 90 ]; do
api_ok=0; ingress_ok=0; ch_ok=0; garage_ok=0
http_ok "http://127.0.0.1:${UI_PORT_VALUE}/readyz" && api_ok=1 || true
http_ok "http://127.0.0.1:${INGRESS_PORT_VALUE}/readyz" && ingress_ok=1 || true
docker compose exec -T clickhouse clickhouse-client --user "${CLICKHOUSE_USER_VALUE}" --password "${CLICKHOUSE_PASSWORD_VALUE}" --query 'SELECT 1' >/dev/null 2>&1 && ch_ok=1 || true
docker compose exec -T garage /garage status >/dev/null 2>&1 && garage_ok=1 || true
if [ "$api_ok" -eq 1 ] && [ "$ingress_ok" -eq 1 ] && [ "$ch_ok" -eq 1 ] && [ "$garage_ok" -eq 1 ]; then
break
fi
i=$((i+1))
sleep 2
done
if [ "$i" -ge 90 ]; then
echo "Mindestens ein Dienst wurde nicht bereit. Diagnose:" >&2
docker compose ps >&2 || true
echo >&2
docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor api >&2 || true
exit 1
fi
echo
echo "Greenfield SIEM läuft."
echo "UI: http://127.0.0.1:${UI_PORT:-8080}/ui"
echo "Ingress: http://127.0.0.1:${INGRESS_PORT:-8090}/ingest"
echo "UI-Login: $UI_USERNAME / $UI_PASSWORD"
echo "Enrollment-Key: $ENROLLMENT_KEY"
echo "Secrets stehen in .env (Datei schützen und sichern)."
echo "UI: http://127.0.0.1:${UI_PORT_VALUE}/ui"
echo "Ingress: http://127.0.0.1:${INGRESS_PORT_VALUE}/ingest"
echo "UI-Login: ${UI_USERNAME_VALUE} / ${UI_PASSWORD_VALUE}"
echo "ClickHouse: user=${CLICKHOUSE_USER_VALUE} db=${CLICKHOUSE_DB_VALUE} http=127.0.0.1:${CLICKHOUSE_HTTP_PORT_VALUE} native=127.0.0.1:${CLICKHOUSE_NATIVE_PORT_VALUE}"
echo "Enrollment-Key: ${ENROLLMENT_KEY_VALUE}"
echo "Alle Parameter stehen vollständig in .env."

View File

@@ -1,82 +0,0 @@
CREATE DATABASE IF NOT EXISTS siem;
CREATE TABLE IF NOT EXISTS siem.events
(
event_uid String,
queue_partition Int32,
queue_offset Int64,
tenant_id LowCardinality(String),
event_time DateTime64(3, 'UTC'),
ingest_time DateTime64(3, 'UTC'),
event_date Date MATERIALIZED toDate(event_time),
agent_id String,
host_name LowCardinality(String),
source_type LowCardinality(String),
channel LowCardinality(String),
provider LowCardinality(String),
event_code UInt32,
category LowCardinality(String),
action LowCardinality(String),
outcome LowCardinality(String),
severity UInt8,
user_name String,
user_domain LowCardinality(String),
subject_user String,
subject_domain LowCardinality(String),
target_user String,
target_domain LowCardinality(String),
source_ip String,
source_port UInt16,
destination_ip String,
destination_port UInt16,
workstation String,
logon_type LowCardinality(String),
authentication_package LowCardinality(String),
logon_process LowCardinality(String),
status_code LowCardinality(String),
sub_status_code LowCardinality(String),
failure_reason LowCardinality(String),
process_path String,
parent_process_path String,
command_line String CODEC(ZSTD(3)),
message String CODEC(ZSTD(3)),
attributes Map(String, String) CODEC(ZSTD(3)),
raw_object_key String,
raw_index UInt32,
payload_hash FixedString(64),
schema_version UInt16,
parser_version UInt16,
ingest_delay_ms Int64,
INDEX idx_host host_name TYPE set(10000) GRANULARITY 4,
INDEX idx_event_code event_code TYPE set(256) GRANULARITY 4,
INDEX idx_user user_name TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_target target_user TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_source_ip source_ip TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_process process_path TYPE bloom_filter(0.01) GRANULARITY 8
)
ENGINE = ReplacingMergeTree(ingest_time)
PARTITION BY toYYYYMM(event_time)
ORDER BY (tenant_id, event_time, host_name, event_code, event_uid)
TTL event_time + INTERVAL 90 DAY DELETE
SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS siem.events_5m
(
tenant_id LowCardinality(String),
bucket DateTime('UTC'),
host_name LowCardinality(String),
event_code UInt32,
category LowCardinality(String),
action LowCardinality(String),
outcome LowCardinality(String),
cnt_state AggregateFunction(uniqExact, String)
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(bucket)
ORDER BY (tenant_id, bucket, host_name, event_code, category, action, outcome)
TTL bucket + INTERVAL 730 DAY DELETE;
CREATE MATERIALIZED VIEW IF NOT EXISTS siem.events_5m_mv TO siem.events_5m AS
SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state
FROM siem.events
GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome;

View File

@@ -1,6 +1,6 @@
CREATE DATABASE IF NOT EXISTS siem;
CREATE DATABASE IF NOT EXISTS __CLICKHOUSE_DB__;
CREATE TABLE IF NOT EXISTS siem.events
CREATE TABLE IF NOT EXISTS __CLICKHOUSE_DB__.events
(
event_uid String,
queue_partition Int32,
@@ -60,7 +60,7 @@ ORDER BY (tenant_id, event_time, host_name, event_code, event_uid)
TTL event_time + INTERVAL __EVENT_RETENTION_DAYS__ DAY DELETE
SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS siem.events_5m
CREATE TABLE IF NOT EXISTS __CLICKHOUSE_DB__.events_5m
(
tenant_id LowCardinality(String),
bucket DateTime('UTC'),
@@ -76,7 +76,7 @@ PARTITION BY toYYYYMM(bucket)
ORDER BY (tenant_id, bucket, host_name, event_code, category, action, outcome)
TTL bucket + INTERVAL __ROLLUP_RETENTION_DAYS__ DAY DELETE;
CREATE MATERIALIZED VIEW IF NOT EXISTS siem.events_5m_mv TO siem.events_5m AS
CREATE MATERIALIZED VIEW IF NOT EXISTS __CLICKHOUSE_DB__.events_5m_mv TO __CLICKHOUSE_DB__.events_5m AS
SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state
FROM siem.events
FROM __CLICKHOUSE_DB__.events
GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome;

View File

@@ -1,17 +0,0 @@
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "garage:3901"
rpc_secret = "__GARAGE_RPC_SECRET__"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage.localhost"
[admin]
api_bind_addr = "[::]:3903"
admin_token = "__GARAGE_ADMIN_TOKEN__"
metrics_token = "__GARAGE_METRICS_TOKEN__"

View File

@@ -3,13 +3,15 @@ set -u
cd "$(dirname "$0")"
[ -f .env ] && { set -a; . ./.env; set +a; }
echo "== Docker =="
section(){ printf '\n== %s ==\n' "$1"; }
section "Compose config"
docker compose config >/dev/null 2>&1 && echo "OK" || echo "FEHLER: docker compose config"
section "Docker"
docker compose ps || true
echo "\n== Host filesystem =="
df -h . 2>/dev/null || true
echo "\n== HTTP readiness =="
section "HTTP readiness"
if command -v curl >/dev/null 2>&1; then
printf "Ingress: "; curl -fsS "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz" || true; echo
printf "API: "; curl -fsS "http://127.0.0.1:${UI_PORT:-8080}/readyz" || true; echo
@@ -17,19 +19,31 @@ else
echo "curl fehlt; HTTP-Prüfung übersprungen."
fi
echo "\n== Redpanda =="
section "Garage"
docker compose exec -T garage /garage status 2>/dev/null || true
printf "S3 archive: "
docker compose exec -T archive-uploader rclone lsd "garage:${GARAGE_BUCKET:-siem-raw}" 2>/dev/null && echo "OK" || echo "nicht erreichbar"
section "ClickHouse"
docker compose exec -T clickhouse clickhouse-client \
--user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" \
--query "SELECT version(), currentUser(), currentDatabase()" 2>/dev/null || true
docker compose exec -T clickhouse clickhouse-client \
--user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" \
--database "${CLICKHOUSE_DB:-siem}" \
--query "SELECT formatReadableQuantity(sum(rows)) AS rows, formatReadableSize(sum(bytes_on_disk)) AS disk, count() AS active_parts FROM system.parts WHERE database='${CLICKHOUSE_DB:-siem}' AND table='events' AND active" 2>/dev/null || true
section "Redpanda"
docker compose exec -T redpanda rpk cluster health 2>/dev/null || true
docker compose exec -T redpanda rpk group describe siem-processor-v1 2>/dev/null || true
docker compose exec -T redpanda rpk group describe "${KAFKA_GROUP:-siem-processor-v1}" 2>/dev/null || true
echo "\n== ClickHouse events =="
docker compose exec -T clickhouse clickhouse-client --user siem --password "${CLICKHOUSE_PASSWORD:-}" --query "SELECT formatReadableQuantity(sum(rows)) AS rows, formatReadableSize(sum(bytes_on_disk)) AS disk, count() AS active_parts FROM system.parts WHERE database='siem' AND table='events' AND active" 2>/dev/null || true
docker compose exec -T clickhouse clickhouse-client --user siem --password "${CLICKHOUSE_PASSWORD:-}" --query "SELECT max(event_time) AS latest_event, max(ingest_time) AS latest_ingest, quantile(0.95)(abs(ingest_delay_ms))/1000 AS p95_ingest_delay_seconds FROM siem.events WHERE event_time >= now() - INTERVAL 1 DAY" 2>/dev/null || true
section "PostgreSQL control plane"
docker compose exec -T postgres psql -U "${POSTGRES_USER:-siem}" -d "${POSTGRES_DB:-siem}" -Atc \
"SELECT 'agents='||count(*) FROM agents; SELECT 'open_detections='||count(*) FROM detections WHERE status IN ('open','investigating');" 2>/dev/null || true
echo "\n== PostgreSQL control plane =="
docker compose exec -T postgres psql -U siem -d siem -Atc "SELECT 'agents='||count(*) FROM agents; SELECT 'open_detections='||count(*) FROM detections WHERE status IN ('open','investigating');" 2>/dev/null || true
section "Raw spool"
docker compose exec -T archive-uploader sh -c 'du -sh /spool 2>/dev/null || true; printf "files="; find /spool -type f 2>/dev/null | wc -l' 2>/dev/null || true
echo "\n== Raw spool =="
docker compose exec -T archive-uploader sh -c 'du -sh /spool 2>/dev/null || true; find /spool -type f 2>/dev/null | wc -l' 2>/dev/null || true
echo "\n== Recent errors =="
docker compose logs --since=15m ingress processor detector api archive-uploader 2>&1 | grep -Ei 'error|fatal|failed|panic|not_ready' | tail -80 || true
section "Recent errors"
docker compose logs --since=15m garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor detector api archive-uploader 2>&1 \
| grep -Ei 'error|fatal|failed|panic|not_ready|exception' | tail -120 || true

10
go.mod
View File

@@ -6,13 +6,3 @@ require (
github.com/jackc/pgx/v5 v5.9.2
github.com/segmentio/kafka-go v0.4.51
)
require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
golang.org/x/sync v0.17.0 // indirect
golang.org/x/text v0.29.0 // indirect
)

40
go.sum
View File

@@ -1,40 +0,0 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/segmentio/kafka-go v0.4.51 h1:JgDPPG75tC1rWIS2Me6MwcvXJ6f49UQ4HjAOef71Hno=
github.com/segmentio/kafka-go v0.4.51/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -82,7 +82,7 @@ func Run(ctx context.Context, cfg config.Config) error {
func summary(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) {
ctx, cancel := context.WithTimeout(r.Context(), 8*time.Second)
defer cancel()
q := fmt.Sprintf(`SELECT uniqExactMerge(cnt_state) events_24h, uniqExact(host_name) active_hosts FROM siem.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL 24 HOUR`, clickhouse.Q(cfg.TenantID))
q := fmt.Sprintf(`SELECT uniqExactMerge(cnt_state) events_24h, uniqExact(host_name) active_hosts FROM %s.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL 24 HOUR`, clickhouse.Ident(cfg.ClickHouseDB), clickhouse.Q(cfg.TenantID))
rows, e := ch.QueryJSON(ctx, q)
if e != nil {
j(w, 500, map[string]string{"error": e.Error()})
@@ -118,7 +118,7 @@ func events(w http.ResponseWriter, r *http.Request, cfg config.Config, ch *click
where = append(where, "event_code="+v)
}
}
q := fmt.Sprintf(`SELECT event_uid,event_time,host_name,channel,event_code,category,action,outcome,severity,user_name,subject_user,target_user,source_ip,workstation,process_path,message,raw_object_key,raw_index FROM siem.events WHERE %s ORDER BY event_time DESC, ingest_time DESC LIMIT 1 BY event_uid LIMIT %d`, strings.Join(where, " AND "), limit)
q := fmt.Sprintf(`SELECT event_uid,event_time,host_name,channel,event_code,category,action,outcome,severity,user_name,subject_user,target_user,source_ip,workstation,process_path,message,raw_object_key,raw_index FROM %s.events WHERE %s ORDER BY event_time DESC, ingest_time DESC LIMIT 1 BY event_uid LIMIT %d`, clickhouse.Ident(cfg.ClickHouseDB), strings.Join(where, " AND "), limit)
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
defer cancel()
rows, e := ch.QueryJSON(ctx, q)

View File

@@ -17,12 +17,12 @@ import (
)
type Client struct {
base, user, pass string
hc *http.Client
base, db, user, pass string
hc *http.Client
}
func New(cfg config.Config) *Client {
return &Client{base: cfg.ClickHouseURL, user: cfg.ClickHouseUser, pass: cfg.ClickHousePassword, hc: &http.Client{Timeout: 30 * time.Second}}
return &Client{base: cfg.ClickHouseURL, db: cfg.ClickHouseDB, user: cfg.ClickHouseUser, pass: cfg.ClickHousePassword, hc: &http.Client{Timeout: 30 * time.Second}}
}
func (c *Client) request(ctx context.Context, query string, body io.Reader) (*http.Response, error) {
u := c.base + "/?query=" + url.QueryEscape(query)
@@ -57,7 +57,7 @@ func (c *Client) InsertEvents(ctx context.Context, events []contracts.CanonicalE
return e
}
}
q := `INSERT INTO siem.events FORMAT JSONEachRow`
q := `INSERT INTO ` + Ident(c.db) + `.events FORMAT JSONEachRow`
r, e := c.request(ctx, q, &b)
if e != nil {
return e
@@ -95,4 +95,5 @@ func (c *Client) QueryJSON(ctx context.Context, q string) ([]map[string]any, err
}
return out, s.Err()
}
func Q(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" }
func Q(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" }
func Ident(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" }

View File

@@ -11,6 +11,7 @@ type Config struct {
ServiceAddr string
PostgresURL string
ClickHouseURL string
ClickHouseDB string
ClickHouseUser string
ClickHousePassword string
KafkaBrokers []string
@@ -23,7 +24,6 @@ type Config struct {
RawArchiveEnabled bool
RawSpoolDir string
RawRetention time.Duration
EventRetention time.Duration
DetectorInterval time.Duration
DetectorLookback time.Duration
UIQueryLimit int
@@ -36,6 +36,7 @@ func Load() Config {
ServiceAddr: env("SERVICE_ADDR", ":8080"),
PostgresURL: env("POSTGRES_URL", "postgres://siem:siem@postgres:5432/siem?sslmode=disable"),
ClickHouseURL: strings.TrimRight(env("CLICKHOUSE_URL", "http://clickhouse:8123"), "/"),
ClickHouseDB: env("CLICKHOUSE_DB", "siem"),
ClickHouseUser: env("CLICKHOUSE_USER", "siem"),
ClickHousePassword: env("CLICKHOUSE_PASSWORD", "siem"),
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "redpanda:9092")),
@@ -48,7 +49,6 @@ func Load() Config {
RawArchiveEnabled: envBool("RAW_ARCHIVE_ENABLED", true),
RawSpoolDir: env("RAW_SPOOL_DIR", "/var/spool/siem-raw"),
RawRetention: envDuration("RAW_RETENTION", 30*24*time.Hour),
EventRetention: envDuration("EVENT_RETENTION", 90*24*time.Hour),
DetectorInterval: envDuration("DETECTOR_INTERVAL", 30*time.Second),
DetectorLookback: envDuration("DETECTOR_LOOKBACK", 20*time.Minute),
UIQueryLimit: envInt("UI_QUERY_LIMIT", 500),

View File

@@ -50,7 +50,7 @@ func Run(ctx context.Context, cfg config.Config) error {
func runAll(ctx context.Context, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) error {
end := time.Now().UTC()
start := end.Add(-cfg.DetectorLookback)
for _, r := range rules() {
for _, r := range rules(cfg.ClickHouseDB) {
q := r.query(start, end, cfg.TenantID)
rows, e := ch.QueryJSON(ctx, q)
if e != nil {
@@ -73,39 +73,40 @@ func runAll(ctx context.Context, cfg config.Config, pg *postgres.Store, ch *clic
}
return nil
}
func rules() []rule {
func rules(db string) []rule {
table := clickhouse.Ident(db) + ".events"
return []rule{
{name: "audit_log_cleared", severity: "critical", eventCode: 1102, score: 9.8, query: simpleEvent(1102, 1), summary: func(m map[string]any) string {
{name: "audit_log_cleared", severity: "critical", eventCode: 1102, score: 9.8, query: simpleEvent(table, 1102, 1), summary: func(m map[string]any) string {
return fmt.Sprintf("Audit-Log auf %s wurde gelöscht", str(m["host_name"]))
}},
{name: "service_installed", severity: "high", eventCode: 7045, score: 8.0, query: simpleEvent(7045, 1), summary: func(m map[string]any) string {
{name: "service_installed", severity: "high", eventCode: 7045, score: 8.0, query: simpleEvent(table, 7045, 1), summary: func(m map[string]any) string {
return fmt.Sprintf("Neuer Dienst auf %s installiert", str(m["host_name"]))
}},
{name: "account_lockout", severity: "medium", eventCode: 4740, score: 5.5, query: func(s, e time.Time, t string) string {
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4740 GROUP BY host_name,user_name,workstation HAVING cnt>=1`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4740 GROUP BY host_name,user_name,workstation HAVING cnt>=1`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
}, summary: func(m map[string]any) string {
return fmt.Sprintf("Account-Lockout: %s; Caller %s; DC/Host %s (%d×)", str(m["user_name"]), fallback(str(m["workstation"]), "unbekannt"), str(m["host_name"]), int64(num(m["cnt"])))
}},
{name: "failed_logon_burst", severity: "high", eventCode: 4625, score: 7.5, query: func(s, e time.Time, t string) string {
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND target_user!='' GROUP BY host_name,user_name,source_ip,workstation HAVING cnt>=20`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND target_user!='' GROUP BY host_name,user_name,source_ip,workstation HAVING cnt>=20`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
}, summary: func(m map[string]any) string {
return fmt.Sprintf("%d fehlgeschlagene Logons für %s auf %s", int64(num(m["cnt"])), str(m["user_name"]), str(m["host_name"]))
}},
{name: "password_spray", severity: "high", eventCode: 4625, score: 8.5, query: func(s, e time.Time, t string) string {
return fmt.Sprintf(`SELECT '' AS host_name, '' AS user_name, source_ip, '' AS workstation, uniqExact(target_user) users, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND source_ip!='' AND target_user!='' GROUP BY source_ip HAVING users>=10 AND cnt>=20`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
return fmt.Sprintf(`SELECT '' AS host_name, '' AS user_name, source_ip, '' AS workstation, uniqExact(target_user) users, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND source_ip!='' AND target_user!='' GROUP BY source_ip HAVING users>=10 AND cnt>=20`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
}, summary: func(m map[string]any) string {
return fmt.Sprintf("Password-Spray von %s gegen %.0f Benutzer (%d Versuche)", str(m["source_ip"]), num(m["users"]), int64(num(m["cnt"])))
}},
{name: "privileged_group_change", severity: "critical", eventCode: 4728, score: 9.2, query: func(s, e time.Time, t string) string {
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code IN (4728,4732,4756) GROUP BY host_name,user_name,workstation HAVING cnt>=1`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code IN (4728,4732,4756) GROUP BY host_name,user_name,workstation HAVING cnt>=1`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
}, summary: func(m map[string]any) string {
return fmt.Sprintf("Privilegierte Gruppenmitgliedschaft geändert: %s auf %s", str(m["user_name"]), str(m["host_name"]))
}},
}
}
func simpleEvent(id uint32, min int) func(time.Time, time.Time, string) string {
func simpleEvent(table string, id uint32, min int) func(time.Time, time.Time, string) string {
return func(s, e time.Time, t string) string {
return fmt.Sprintf(`SELECT host_name, '' AS user_name, '' AS source_ip, '' AS workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=%d GROUP BY host_name HAVING cnt>=%d`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)), id, min)
return fmt.Sprintf(`SELECT host_name, '' AS user_name, '' AS source_ip, '' AS workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=%d GROUP BY host_name HAVING cnt>=%d`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)), id, min)
}
}
func ts(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000") }
@@ -143,7 +144,12 @@ func fingerprint(v ...string) string {
h := sha256.Sum256([]byte(strings.Join(v, "|")))
return hex.EncodeToString(h[:])
}
func fallback(v, d string) string { if strings.TrimSpace(v) == "" { return d }; return v }
func fallback(v, d string) string {
if strings.TrimSpace(v) == "" {
return d
}
return v
}
func max64(a, b int64) int64 {
if a > b {
return a

View File

@@ -1,8 +1,7 @@
#!/usr/bin/env sh
set -eu
cd "$(dirname "$0")"
printf 'ACHTUNG: Alle SIEM-Daten löschen. Tippe DELETE: '
printf 'ACHTUNG: Alle SIEM-Daten und Docker-Volumes löschen. Tippe DELETE: '
read x
[ "$x" = "DELETE" ] || exit 1
docker compose down -v --remove-orphans
rm -f deploy/garage/garage.toml

View File

@@ -1,8 +1,16 @@
#!/usr/bin/env sh
set -eu
cd "$(dirname "$0")"
[ -f .env ] && { set -a; . ./.env; set +a; }
docker compose ps
printf '\nIngress readiness:\n'
curl -fsS http://127.0.0.1:$(grep '^INGRESS_PORT=' .env | cut -d= -f2-)/readyz || true
printf '\n\nRecent errors:\n'
docker compose logs --since=10m ingress processor detector api 2>&1 | grep -Ei 'error|fatal|failed|panic' | tail -50 || true
if command -v curl >/dev/null 2>&1; then curl -fsS "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz" || true; fi
printf '\nAPI readiness:\n'
if command -v curl >/dev/null 2>&1; then curl -fsS "http://127.0.0.1:${UI_PORT:-8080}/readyz" || true; fi
printf '\nGarage:\n'
docker compose exec -T garage /garage status 2>/dev/null || true
printf '\nClickHouse:\n'
docker compose exec -T clickhouse clickhouse-client --user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" --query 'SELECT version(), currentUser()' 2>/dev/null || true
printf '\nRecent errors:\n'
docker compose logs --since=10m garage clickhouse clickhouse-schema postgres postgres-schema redpanda ingress processor detector api archive-uploader 2>&1 \
| grep -Ei 'error|fatal|failed|panic|exception' | tail -80 || true