This commit is contained in:
4
.dockerignore
Normal file
4
.dockerignore
Normal file
@@ -0,0 +1,4 @@
|
||||
.git
|
||||
.env
|
||||
spool
|
||||
*.zip
|
||||
144
.env
144
.env
@@ -1,77 +1,85 @@
|
||||
TZ=UTC
|
||||
# ============================================================
|
||||
# Greenfield SIEM - complete configuration
|
||||
# deploy.sh / deploy.ps1 creates .env and replaces CHANGE_ME.
|
||||
# ============================================================
|
||||
|
||||
LISTEN_ADDR=:8080
|
||||
DB_DSN=eventuser:DEINPASSWORT@tcp(mariadb:3306)/eventcollector?parseTime=true&charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&loc=UTC
|
||||
# Public SIEM endpoints
|
||||
INGRESS_BIND=0.0.0.0
|
||||
INGRESS_PORT=8090
|
||||
UI_BIND=0.0.0.0
|
||||
UI_PORT=8080
|
||||
TENANT_ID=default
|
||||
UI_USERNAME=admin
|
||||
UI_PASSWORD=CHANGE_ME
|
||||
ENROLLMENT_KEY=CHANGE_ME
|
||||
|
||||
DB_MAX_OPEN_CONNS=50
|
||||
DB_MAX_IDLE_CONNS=25
|
||||
DB_CONN_MAX_LIFETIME=3m
|
||||
DB_CONN_MAX_IDLE_TIME=1m
|
||||
# Ingress limits
|
||||
MAX_BODY_BYTES=8388608
|
||||
MAX_BATCH_EVENTS=1000
|
||||
|
||||
MAX_BODY_BYTES=10485760
|
||||
HTTP_READ_TIMEOUT=15s
|
||||
HTTP_WRITE_TIMEOUT=30s
|
||||
HTTP_IDLE_TIMEOUT=60s
|
||||
# PostgreSQL control plane
|
||||
POSTGRES_IMAGE=postgres:18.4-alpine
|
||||
POSTGRES_DB=siem
|
||||
POSTGRES_USER=siem
|
||||
POSTGRES_PASSWORD=CHANGE_ME
|
||||
POSTGRES_BIND=127.0.0.1
|
||||
POSTGRES_PORT=5432
|
||||
|
||||
DETECTION_INTERVAL=1m
|
||||
OFFLINE_AFTER=10m
|
||||
OFFLINE_ALERT_MAX=120m
|
||||
FAILED_LOGON_WINDOW=5m
|
||||
FAILED_LOGON_THRESHOLD=25
|
||||
REBOOT_WINDOW=15m
|
||||
REBOOT_THRESHOLD=3
|
||||
PASSWORD_SPRAY_WINDOW=5m
|
||||
PASSWORD_SPRAY_MIN_USERS=5
|
||||
PASSWORD_SPRAY_MIN_ATTEMPTS=15
|
||||
SUCCESS_AFTER_FAILURE_WINDOW=10m
|
||||
NEW_SOURCE_IP_LOOKBACK=720h
|
||||
NEW_SOURCE_IP_WINDOW=10m
|
||||
DETECTIONS_LIMIT=100
|
||||
# ClickHouse event store
|
||||
CLICKHOUSE_IMAGE=clickhouse/clickhouse-server:26.6.2.81
|
||||
CLICKHOUSE_DB=siem
|
||||
CLICKHOUSE_USER=siem
|
||||
CLICKHOUSE_PASSWORD=CHANGE_ME
|
||||
CLICKHOUSE_HTTP_BIND=127.0.0.1
|
||||
CLICKHOUSE_HTTP_PORT=8123
|
||||
CLICKHOUSE_NATIVE_BIND=127.0.0.1
|
||||
CLICKHOUSE_NATIVE_PORT=9000
|
||||
EVENT_RETENTION_DAYS=90
|
||||
ROLLUP_RETENTION_DAYS=730
|
||||
|
||||
MARIADB_DATABASE=eventcollector
|
||||
MARIADB_USER=eventuser
|
||||
MARIADB_PASSWORD=DEINPASSWORT
|
||||
MARIADB_ROOT_PASSWORD=ROOTPASSWORT
|
||||
# Redpanda durable ingest queue
|
||||
REDPANDA_IMAGE=docker.redpanda.com/redpandadata/redpanda:v26.1.13
|
||||
REDPANDA_CONSOLE_IMAGE=docker.redpanda.com/redpandadata/console:v3.8.0
|
||||
REDPANDA_BIND=127.0.0.1
|
||||
REDPANDA_KAFKA_PORT=19092
|
||||
REDPANDA_ADMIN_PORT=19644
|
||||
REDPANDA_CONSOLE_BIND=127.0.0.1
|
||||
REDPANDA_CONSOLE_PORT=8081
|
||||
REDPANDA_SMP=2
|
||||
REDPANDA_LOG_LEVEL=info
|
||||
KAFKA_TOPIC=siem-events
|
||||
KAFKA_GROUP=siem-processor-v1
|
||||
KAFKA_PARTITIONS=6
|
||||
KAFKA_RETENTION_MS=86400000
|
||||
|
||||
GRAFANA_ADMIN_USER=admin
|
||||
GRAFANA_ADMIN_PASSWORD=admin
|
||||
# Garage S3-compatible raw archive
|
||||
GARAGE_IMAGE=dxflrs/garage:v2.3.0
|
||||
ALPINE_IMAGE=alpine:3.22
|
||||
GARAGE_BIND=127.0.0.1
|
||||
GARAGE_S3_PORT=3900
|
||||
GARAGE_ADMIN_PORT=3903
|
||||
GARAGE_BUCKET=siem-raw
|
||||
GARAGE_ACCESS_KEY=CHANGE_ME
|
||||
GARAGE_SECRET_KEY=CHANGE_ME
|
||||
GARAGE_RPC_SECRET=CHANGE_ME
|
||||
GARAGE_ADMIN_TOKEN=CHANGE_ME
|
||||
GARAGE_METRICS_TOKEN=CHANGE_ME
|
||||
GARAGE_LOG_LEVEL=garage=info
|
||||
RAW_ARCHIVE_ENABLED=true
|
||||
RAW_RETENTION=720h
|
||||
ARCHIVE_UPLOAD_INTERVAL_SECONDS=30
|
||||
|
||||
ENROLLMENT_KEY=BITTE_SEHR_LANG_UND_ZUFAELLIG
|
||||
# Detector / API
|
||||
DETECTOR_INTERVAL=30s
|
||||
DETECTOR_LOOKBACK=20m
|
||||
UI_QUERY_LIMIT=500
|
||||
|
||||
#Woche 1
|
||||
BASELINE_WINDOW=15m
|
||||
BASELINE_MIN_SAMPLES=72
|
||||
BASELINE_MIN_COUNT=30
|
||||
BASELINE_MEDIUM_Z=3.5
|
||||
BASELINE_HIGH_Z=6.0
|
||||
BASELINE_SUPPRESS_FOR=6h
|
||||
# Prometheus
|
||||
PROMETHEUS_IMAGE=prom/prometheus:v3.13.1
|
||||
PROMETHEUS_BIND=127.0.0.1
|
||||
PROMETHEUS_PORT=9090
|
||||
PROMETHEUS_RETENTION=30d
|
||||
|
||||
#Woche 2-4
|
||||
#BASELINE_WINDOW=15m
|
||||
#BASELINE_MIN_SAMPLES=48
|
||||
#BASELINE_MIN_COUNT=20
|
||||
#BASELINE_MEDIUM_Z=3.0
|
||||
#BASELINE_HIGH_Z=5.0
|
||||
#BASELINE_SUPPRESS_FOR=4h
|
||||
|
||||
#Dauerlauf Aggressiv
|
||||
#BASELINE_WINDOW=10m
|
||||
#BASELINE_MIN_SAMPLES=36
|
||||
#BASELINE_MIN_COUNT=15
|
||||
#BASELINE_MEDIUM_Z=2.8
|
||||
#BASELINE_HIGH_Z=4.5
|
||||
#BASELINE_SUPPRESS_FOR=2h
|
||||
|
||||
#Dauerlauf Passiv
|
||||
#BASELINE_WINDOW=15m
|
||||
#BASELINE_MIN_SAMPLES=48
|
||||
#BASELINE_MIN_COUNT=20
|
||||
#BASELINE_MEDIUM_Z=3.0
|
||||
#BASELINE_HIGH_Z=5.0
|
||||
#BASELINE_SUPPRESS_FOR=4h
|
||||
|
||||
PARTITION_MAINTENANCE_ENABLED=true
|
||||
PARTITION_MAINTENANCE_INTERVAL=15m
|
||||
PARTITION_INTERVAL=3h
|
||||
PARTITION_AHEAD=24h
|
||||
PARTITION_BEHIND=6h
|
||||
# Raw uploader and application image
|
||||
RCLONE_IMAGE=rclone/rclone:1.74.4
|
||||
APP_IMAGE=greenfield-siem-app:local
|
||||
|
||||
85
.env.example
Normal file
85
.env.example
Normal file
@@ -0,0 +1,85 @@
|
||||
# ============================================================
|
||||
# Greenfield SIEM - complete configuration
|
||||
# deploy.sh / deploy.ps1 creates .env and replaces CHANGE_ME.
|
||||
# ============================================================
|
||||
|
||||
# Public SIEM endpoints
|
||||
INGRESS_BIND=0.0.0.0
|
||||
INGRESS_PORT=8090
|
||||
UI_BIND=0.0.0.0
|
||||
UI_PORT=8080
|
||||
TENANT_ID=default
|
||||
UI_USERNAME=admin
|
||||
UI_PASSWORD=CHANGE_ME
|
||||
ENROLLMENT_KEY=CHANGE_ME
|
||||
|
||||
# Ingress limits
|
||||
MAX_BODY_BYTES=8388608
|
||||
MAX_BATCH_EVENTS=1000
|
||||
|
||||
# PostgreSQL control plane
|
||||
POSTGRES_IMAGE=postgres:18.4-alpine
|
||||
POSTGRES_DB=siem
|
||||
POSTGRES_USER=siem
|
||||
POSTGRES_PASSWORD=CHANGE_ME
|
||||
POSTGRES_BIND=127.0.0.1
|
||||
POSTGRES_PORT=5432
|
||||
|
||||
# ClickHouse event store
|
||||
CLICKHOUSE_IMAGE=clickhouse/clickhouse-server:26.6.2.81
|
||||
CLICKHOUSE_DB=siem
|
||||
CLICKHOUSE_USER=siem
|
||||
CLICKHOUSE_PASSWORD=CHANGE_ME
|
||||
CLICKHOUSE_HTTP_BIND=127.0.0.1
|
||||
CLICKHOUSE_HTTP_PORT=8123
|
||||
CLICKHOUSE_NATIVE_BIND=127.0.0.1
|
||||
CLICKHOUSE_NATIVE_PORT=9000
|
||||
EVENT_RETENTION_DAYS=90
|
||||
ROLLUP_RETENTION_DAYS=730
|
||||
|
||||
# Redpanda durable ingest queue
|
||||
REDPANDA_IMAGE=docker.redpanda.com/redpandadata/redpanda:v26.1.13
|
||||
REDPANDA_CONSOLE_IMAGE=docker.redpanda.com/redpandadata/console:v3.8.0
|
||||
REDPANDA_BIND=127.0.0.1
|
||||
REDPANDA_KAFKA_PORT=19092
|
||||
REDPANDA_ADMIN_PORT=19644
|
||||
REDPANDA_CONSOLE_BIND=127.0.0.1
|
||||
REDPANDA_CONSOLE_PORT=8081
|
||||
REDPANDA_SMP=2
|
||||
REDPANDA_LOG_LEVEL=info
|
||||
KAFKA_TOPIC=siem-events
|
||||
KAFKA_GROUP=siem-processor-v1
|
||||
KAFKA_PARTITIONS=6
|
||||
KAFKA_RETENTION_MS=86400000
|
||||
|
||||
# Garage S3-compatible raw archive
|
||||
GARAGE_IMAGE=dxflrs/garage:v2.3.0
|
||||
ALPINE_IMAGE=alpine:3.22
|
||||
GARAGE_BIND=127.0.0.1
|
||||
GARAGE_S3_PORT=3900
|
||||
GARAGE_ADMIN_PORT=3903
|
||||
GARAGE_BUCKET=siem-raw
|
||||
GARAGE_ACCESS_KEY=CHANGE_ME
|
||||
GARAGE_SECRET_KEY=CHANGE_ME
|
||||
GARAGE_RPC_SECRET=CHANGE_ME
|
||||
GARAGE_ADMIN_TOKEN=CHANGE_ME
|
||||
GARAGE_METRICS_TOKEN=CHANGE_ME
|
||||
GARAGE_LOG_LEVEL=garage=info
|
||||
RAW_ARCHIVE_ENABLED=true
|
||||
RAW_RETENTION=720h
|
||||
ARCHIVE_UPLOAD_INTERVAL_SECONDS=30
|
||||
|
||||
# Detector / API
|
||||
DETECTOR_INTERVAL=30s
|
||||
DETECTOR_LOOKBACK=20m
|
||||
UI_QUERY_LIMIT=500
|
||||
|
||||
# Prometheus
|
||||
PROMETHEUS_IMAGE=prom/prometheus:v3.13.1
|
||||
PROMETHEUS_BIND=127.0.0.1
|
||||
PROMETHEUS_PORT=9090
|
||||
PROMETHEUS_RETENTION=30d
|
||||
|
||||
# Raw uploader and application image
|
||||
RCLONE_IMAGE=rclone/rclone:1.74.4
|
||||
APP_IMAGE=greenfield-siem-app:local
|
||||
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
.env
|
||||
deploy/garage/garage.toml
|
||||
spool/
|
||||
*.zip
|
||||
42
DEPLOYMENT-FIXES.md
Normal file
42
DEPLOYMENT-FIXES.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# Deployment-Fixes – Release 1.1
|
||||
|
||||
Stand: 2026-07-23
|
||||
|
||||
Diese Revision repariert die Startprobleme der ersten Greenfield-Ausgabe, ohne die Event-Architektur wieder zu verändern.
|
||||
|
||||
## Behobene Ursachen
|
||||
|
||||
1. **Garage-Konfiguration war nicht Bestandteil des eigentlichen Compose-Lebenszyklus.**
|
||||
Die alte Ausgabe erwartete eine zuvor auf dem Host erzeugte `deploy/garage/garage.toml`. Jetzt erzeugt der einmalige Service `garage-config` die Konfiguration in einem Docker-Volume. `garage` startet erst nach erfolgreicher Erzeugung.
|
||||
|
||||
2. **Garage Single-Node RPC-Adresse war ungeeignet.**
|
||||
`rpc_public_addr` ist jetzt `127.0.0.1:3901`; der Server bindet intern weiterhin auf `0.0.0.0:3901`.
|
||||
|
||||
3. **ClickHouse hatte nur HTTP 8123 exponiert.**
|
||||
Zusätzlich ist jetzt der native Client-Port 9000 konfigurierbar und standardmäßig nur auf `127.0.0.1` gebunden.
|
||||
|
||||
4. **`.env.example` war unvollständig.**
|
||||
Sie enthält jetzt sämtliche 63 von Compose verwendeten Laufzeitparameter. `deploy.sh` und `deploy.ps1` ergänzen fehlende Schlüssel in einer bestehenden `.env`, ohne bestehende Secrets zu ersetzen. Eigene zusätzliche Schlüssel bleiben erhalten.
|
||||
|
||||
5. **Schema-Setup hing vom ersten Containerstart ab.**
|
||||
`postgres-schema` und `clickhouse-schema` laufen idempotent nach erfolgreicher Datenbank-Readiness. Damit wird ein teilweise abgebrochener Erststart nicht mehr stillschweigend als vollständig initialisiert behandelt.
|
||||
|
||||
6. **ClickHouse-Datenbankname war im Go-Code teilweise fest auf `siem` verdrahtet.**
|
||||
API, Detector und Inserts verwenden jetzt konsistent `CLICKHOUSE_DB`.
|
||||
|
||||
7. **Deployment-Fehler waren zu wenig sichtbar.**
|
||||
Der One-Click-Start validiert zuerst `docker compose config`, prüft nach dem Start API, Ingress, ClickHouse und Garage und gibt bei Fehlschlag automatisch die relevanten Logs aus.
|
||||
|
||||
## Sichere Reparatur einer vorhandenen Installation
|
||||
|
||||
Die Daten-Volumes werden durch `./deploy.sh` **nicht** gelöscht.
|
||||
|
||||
Am sichersten ist es, die bestehende `.env` in die korrigierte Projektversion zu übernehmen und danach `./deploy.sh` auszuführen. So bleiben die bereits verwendeten PostgreSQL-/ClickHouse-/Garage-Secrets erhalten.
|
||||
|
||||
`./reset-DANGEROUS.sh` löscht dagegen die Docker-Volumes und darf bei erhaltenswerten Daten nicht verwendet werden.
|
||||
|
||||
## Diagnose
|
||||
|
||||
- `./credentials.sh` zeigt die tatsächlich verwendeten Zugangsdaten und Ports.
|
||||
- `./doctor.sh` prüft Compose, API/Ingress, Garage, ClickHouse, Redpanda, PostgreSQL und den Raw-Spool.
|
||||
- `docker compose logs --tail=200 garage garage-config clickhouse clickhouse-schema` zeigt die wichtigsten Startfehler.
|
||||
73
README.md
73
README.md
@@ -1,5 +1,7 @@
|
||||
# Greenfield SIEM
|
||||
|
||||
**Release 1.1 – Deployment-Fix:** Garage-Bootstrap, vollständige `.env`, ClickHouse-Native-Port und idempotentes ClickHouse-Schema wurden gegenüber der ersten Greenfield-Version korrigiert.
|
||||
|
||||
Kompletter Neuaufbau des bisherigen Projekts. Vom Altprojekt bleibt absichtlich nur der HTTP-Ingress-Vertrag erhalten.
|
||||
|
||||
## Was dieses Projekt löst
|
||||
@@ -45,7 +47,7 @@ chmod +x deploy.sh
|
||||
./deploy.ps1
|
||||
```
|
||||
|
||||
Das Deployment erzeugt beim ersten Start kryptographisch zufällige Secrets in `.env`, erzeugt die Garage-Konfiguration, baut die SIEM-Binaries und startet alle Dienste.
|
||||
Das Deployment vervollständigt bei jedem Start die `.env`, ohne vorhandene Secrets zu überschreiben. Fehlende Secrets werden kryptographisch zufällig erzeugt. Die Garage-Konfiguration wird anschließend automatisch von einem Compose-Init-Service erzeugt; ClickHouse-Schema und TTLs werden bei jedem Start idempotent geprüft. Danach werden die SIEM-Binaries gebaut und alle Dienste gestartet.
|
||||
|
||||
Danach:
|
||||
|
||||
@@ -56,10 +58,72 @@ Danach:
|
||||
|
||||
Beim ersten Start werden `UI_USERNAME=admin` und ein zufälliges `UI_PASSWORD` erzeugt. Browser fragen diese Zugangsdaten beim Öffnen von `/ui` ab.
|
||||
|
||||
Alle konfigurierbaren Parameter stehen in `.env.example`. `deploy.sh`/`deploy.ps1` ergänzt fehlende Schlüssel automatisch in einer vorhandenen `.env`; damit werden auch unvollständige `.env`-Dateien aus der ersten Greenfield-Version repariert. Zugangsdaten lassen sich jederzeit mit `./credentials.sh` anzeigen.
|
||||
|
||||
UI-Benutzer/Passwort und Enrollment-Key werden am Ende des Deployments ausgegeben und stehen außerdem in der geschützten `.env`. Die UI/API ist standardmäßig per HTTP Basic Auth geschützt; für Zugriff über ein fremdes Netz gehört zusätzlich TLS davor.
|
||||
|
||||
> Bestehende Collector können den bisherigen Enrollment-Key weiterverwenden, ohne eine `.env` von Hand anzulegen. Linux/macOS/WSL: `ENROLLMENT_KEY="ALTER_KEY" ./deploy.sh`. PowerShell: `$env:ENROLLMENT_KEY="ALTER_KEY"; ./deploy.ps1`. Alle anderen Secrets werden trotzdem zufällig erzeugt.
|
||||
|
||||
|
||||
## Upgrade / Reparatur der ersten Greenfield-Version
|
||||
|
||||
Die erste ausgelieferte Greenfield-Version hatte drei Deployment-Probleme:
|
||||
|
||||
1. `deploy/garage/garage.toml` wurde erst auf dem Host erzeugt und fehlte im ZIP; außerdem war `rpc_public_addr` für das Single-Node-Setup ungünstig gesetzt.
|
||||
2. ClickHouse exponierte nur HTTP `8123`, nicht den nativen Client-Port `9000`.
|
||||
3. `.env` enthielt nur einen Teil der tatsächlich konfigurierbaren Parameter.
|
||||
|
||||
Mit dieser Version reicht im bestehenden Projektverzeichnis:
|
||||
|
||||
```bash
|
||||
./deploy.sh
|
||||
./doctor.sh
|
||||
```
|
||||
|
||||
Das Deploy-Skript behält bestehende Secrets, ergänzt fehlende `.env`-Werte und erzeugt Garage-Konfiguration sowie ClickHouse-Schema über Compose-Init-Services neu. **Die Daten-Volumes werden dabei nicht gelöscht.**
|
||||
|
||||
Falls `.env` gelöscht wurde, aber alte PostgreSQL-Volumes mit einem unbekannten alten Passwort weiter existieren, kann dieses Passwort nicht automatisch rekonstruiert werden. In einer reinen Testinstallation ist dann `./reset-DANGEROUS.sh` der saubere Neustart; bei produktiven Daten zuerst Backup beziehungsweise Passwort-Recovery durchführen.
|
||||
|
||||
## ClickHouse-Zugang
|
||||
|
||||
ClickHouse besitzt zwei verschiedene Schnittstellen:
|
||||
|
||||
- HTTP: `127.0.0.1:8123`
|
||||
- nativer ClickHouse-Client: `127.0.0.1:9000`
|
||||
|
||||
Benutzer, Passwort und Datenbank stehen vollständig in `.env`:
|
||||
|
||||
```env
|
||||
CLICKHOUSE_DB=siem
|
||||
CLICKHOUSE_USER=siem
|
||||
CLICKHOUSE_PASSWORD=...
|
||||
CLICKHOUSE_HTTP_PORT=8123
|
||||
CLICKHOUSE_NATIVE_PORT=9000
|
||||
```
|
||||
|
||||
Am einfachsten:
|
||||
|
||||
```bash
|
||||
./credentials.sh
|
||||
docker compose exec clickhouse clickhouse-client \
|
||||
--user "$(grep '^CLICKHOUSE_USER=' .env | cut -d= -f2-)" \
|
||||
--password "$(grep '^CLICKHOUSE_PASSWORD=' .env | cut -d= -f2-)" \
|
||||
--database "$(grep '^CLICKHOUSE_DB=' .env | cut -d= -f2-)"
|
||||
```
|
||||
|
||||
Für DBeaver/DataGrip oder einen lokalen `clickhouse-client` den nativen Port `9000` verwenden. Port `8123` ist die HTTP-Schnittstelle, keine Login-Weboberfläche.
|
||||
|
||||
## Garage-Zugang / Diagnose
|
||||
|
||||
Garage wird im Single-Node-Modus automatisch konfiguriert. Die Konfiguration liegt in einem Docker-Volume und muss nicht mehr als Host-Datei erzeugt werden. Prüfung:
|
||||
|
||||
```bash
|
||||
docker compose exec garage /garage status
|
||||
docker compose logs --tail=100 garage garage-config
|
||||
```
|
||||
|
||||
S3 läuft lokal auf `127.0.0.1:3900`. Access-Key, Secret-Key und Bucket stehen in `.env`.
|
||||
|
||||
## Bestehender Ingress bleibt kompatibel
|
||||
|
||||
```http
|
||||
@@ -253,8 +317,9 @@ ClickHouse Zeilenzahl / Speicher:
|
||||
|
||||
```bash
|
||||
docker compose exec clickhouse clickhouse-client \
|
||||
--user siem --password "$(grep '^CLICKHOUSE_PASSWORD=' .env | cut -d= -f2-)" \
|
||||
--query "SELECT formatReadableQuantity(sum(rows)), formatReadableSize(sum(bytes_on_disk)) FROM system.parts WHERE database='siem' AND table='events' AND active"
|
||||
--user "$(grep '^CLICKHOUSE_USER=' .env | cut -d= -f2-)" \
|
||||
--password "$(grep '^CLICKHOUSE_PASSWORD=' .env | cut -d= -f2-)" \
|
||||
--query "SELECT formatReadableQuantity(sum(rows)), formatReadableSize(sum(bytes_on_disk)) FROM system.parts WHERE database='$(grep '^CLICKHOUSE_DB=' .env | cut -d= -f2-)' AND table='events' AND active"
|
||||
```
|
||||
|
||||
Queue:
|
||||
@@ -320,7 +385,7 @@ doctor.sh Pipeline-/Kapazitätsdiagnose
|
||||
raw-event.sh Raw-Archiv abrufen
|
||||
deploy/clickhouse analytisches Schema
|
||||
deploy/postgres Control-Plane-Schema
|
||||
deploy/garage S3-Archivkonfiguration
|
||||
compose.yml erzeugt Garage-Konfiguration zur Laufzeit aus `.env`
|
||||
deploy/prometheus Betriebsmetriken
|
||||
```
|
||||
|
||||
|
||||
@@ -1,24 +1,39 @@
|
||||
# Release checks
|
||||
|
||||
Stand: 2026-07-23
|
||||
Stand: 2026-07-23 – Deployment-Fix 1.1
|
||||
|
||||
Durchgeführt:
|
||||
|
||||
- `compose.yml` mit YAML-Parser geladen: 15 Services, erfolgreich.
|
||||
- Alle 63 nicht-escaped Compose-Variablen mit `.env.example` abgeglichen: **0 fehlende, 0 überzählige**.
|
||||
- Sämtliche Host-Bind-Mount-Quellen aus `compose.yml` auf Existenz geprüft: **0 fehlende Pfade**.
|
||||
- Prometheus-YAML geparst: erfolgreich.
|
||||
- ClickHouse-Prometheus-XML geparst: erfolgreich.
|
||||
- Garage-Konfigurationskommando aus Compose isoliert ausgeführt und resultierendes TOML mit `tomllib` geparst: erfolgreich.
|
||||
- ClickHouse-Schema-Rendering aus Compose isoliert ausgeführt: Datenbankname und TTL-Platzhalter vollständig ersetzt; Schema-/TTL-Kommandos wurden aufgerufen.
|
||||
- `deploy.sh` mit simuliertem Docker/HTTP end-to-end ausgeführt: erfolgreich.
|
||||
- frische `.env`: 63 Schlüssel, keine `CHANGE_ME`-Secrets;
|
||||
- externer bestehender Enrollment-Key bleibt erhalten;
|
||||
- partielle Alt-`.env` wird vollständig ergänzt;
|
||||
- vorhandene UI-/PostgreSQL-/ClickHouse-/Enrollment-Secrets bleiben unverändert;
|
||||
- eigener unbekannter `.env`-Schlüssel bleibt erhalten.
|
||||
- POSIX-Shellskripte mit `sh -n` geprüft: erfolgreich.
|
||||
- `go test ./...` gegen lokale Interface-Stubs für pgx/kafka-go mit der vorhandenen Go-1.23.2-Toolchain: erfolgreich.
|
||||
- `go vet ./...` unter denselben isolierten Bedingungen: erfolgreich.
|
||||
- Ingress-Tests: Batch-UID/Retry-Idempotenz und Validierung: erfolgreich.
|
||||
- Normalizer-Tests: Windows-Metadaten/XML und Lockout-Workstation: erfolgreich.
|
||||
- `compose.yml` und Prometheus-YAML syntaktisch geparst: erfolgreich.
|
||||
- ClickHouse-Prometheus-XML geparst: erfolgreich.
|
||||
- POSIX-Shellskripte mit `sh -n` geprüft: erfolgreich.
|
||||
- `deploy.sh` mit simuliertem Docker/Healthcheck end-to-end ausgeführt: Secret-Erzeugung, bestehender Enrollment-Key, Template-Rendering und Compose-Aufrufe erfolgreich.
|
||||
- ZIP-Integrität mit `unzip -t`: erfolgreich.
|
||||
|
||||
Zusätzlich gegen aktuelle Primärquellen geprüft:
|
||||
|
||||
- Garage v2.3 Single-Node/Default-Bucket-Start und die erwarteten Default-Credential-Umgebungsvariablen.
|
||||
- ClickHouse Docker: HTTP-Port 8123, nativer Client-Port 9000 sowie `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD` und Access-Management-Initialisierung.
|
||||
- Redpanda Single-Broker Docker-Beispiel und aktuelle Broker-/Console-Versionen.
|
||||
|
||||
Nicht in dieser Ausführungsumgebung möglich:
|
||||
|
||||
- echter Docker-Compose-Start der Datenbanken/Broker/Object-Storage-Komponenten;
|
||||
- echter Docker-Compose-Start der Datenbanken/Broker/Object-Storage-Komponenten, da hier kein Docker-Daemon/CLI verfügbar ist;
|
||||
- Integrationstest gegen reale ClickHouse-, PostgreSQL-, Redpanda- und Garage-Container;
|
||||
- Ausführung von `deploy.ps1`, da PowerShell hier nicht installiert ist;
|
||||
- Build mit Go 1.26.5, da die lokale Umgebung keinen Netzwerkzugriff für die Toolchain/Module besitzt. Das Dockerfile verwendet Go 1.26.5.
|
||||
- regulärer Build mit den realen externen Go-Modulen; die lokale Prüfung verwendete Schnittstellen-Stubs, da die vorhandene lokale Go-Version 1.23.2 ist, während das Dockerfile Go 1.26.5 verwendet.
|
||||
|
||||
Der erste reale Deployment-Lauf sollte daher anschließend mit `./doctor.sh` und einem begrenzten Stress-Test geprüft werden.
|
||||
Der erste reale Lauf sollte deshalb direkt mit `./doctor.sh` verifiziert werden. Bei einem Startfehler gibt `deploy.sh` bereits automatisch die relevanten Service-Logs aus.
|
||||
|
||||
231
compose.yml
231
compose.yml
@@ -2,50 +2,97 @@ name: greenfield-siem
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18.4-alpine
|
||||
image: ${POSTGRES_IMAGE:-postgres:18.4-alpine}
|
||||
environment:
|
||||
POSTGRES_DB: siem
|
||||
POSTGRES_USER: siem
|
||||
POSTGRES_DB: ${POSTGRES_DB:-siem}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-siem}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql
|
||||
- ./deploy/postgres/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U siem -d siem"]
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
|
||||
- "${POSTGRES_BIND:-127.0.0.1}:${POSTGRES_PORT:-5432}:5432"
|
||||
|
||||
postgres-schema:
|
||||
image: ${POSTGRES_IMAGE:-postgres:18.4-alpine}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
PGHOST: postgres
|
||||
PGPORT: "5432"
|
||||
PGDATABASE: ${POSTGRES_DB:-siem}
|
||||
PGUSER: ${POSTGRES_USER:-siem}
|
||||
PGPASSWORD: ${POSTGRES_PASSWORD}
|
||||
volumes:
|
||||
- ./deploy/postgres/init.sql:/schema/init.sql:ro
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
command: ["psql -v ON_ERROR_STOP=1 -f /schema/init.sql"]
|
||||
restart: "no"
|
||||
|
||||
clickhouse:
|
||||
image: clickhouse/clickhouse-server:26.6.2.81
|
||||
image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81}
|
||||
environment:
|
||||
CLICKHOUSE_DB: siem
|
||||
CLICKHOUSE_USER: siem
|
||||
CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem}
|
||||
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
|
||||
volumes:
|
||||
- clickhouse_data:/var/lib/clickhouse
|
||||
- clickhouse_logs:/var/log/clickhouse-server
|
||||
- ./deploy/clickhouse/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
|
||||
- ./deploy/clickhouse/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 262144
|
||||
hard: 262144
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "clickhouse-client --user siem --password $$CLICKHOUSE_PASSWORD --query 'SELECT 1' >/dev/null"]
|
||||
test: ["CMD-SHELL", "clickhouse-client --host 127.0.0.1 --user $${CLICKHOUSE_USER} --password $${CLICKHOUSE_PASSWORD} --query 'SELECT 1' >/dev/null"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
retries: 40
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${CLICKHOUSE_HTTP_PORT:-8123}:8123"
|
||||
- "${CLICKHOUSE_HTTP_BIND:-127.0.0.1}:${CLICKHOUSE_HTTP_PORT:-8123}:8123"
|
||||
- "${CLICKHOUSE_NATIVE_BIND:-127.0.0.1}:${CLICKHOUSE_NATIVE_PORT:-9000}:9000"
|
||||
|
||||
clickhouse-schema:
|
||||
image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81}
|
||||
depends_on:
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/bin/bash", "-ec"]
|
||||
environment:
|
||||
CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem}
|
||||
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
|
||||
EVENT_RETENTION_DAYS: ${EVENT_RETENTION_DAYS:-90}
|
||||
ROLLUP_RETENTION_DAYS: ${ROLLUP_RETENTION_DAYS:-730}
|
||||
volumes:
|
||||
- ./deploy/clickhouse/init.sql.template:/schema/init.sql.template:ro
|
||||
command:
|
||||
- |
|
||||
sed \
|
||||
-e "s/__CLICKHOUSE_DB__/$${CLICKHOUSE_DB}/g" \
|
||||
-e "s/__EVENT_RETENTION_DAYS__/$${EVENT_RETENTION_DAYS}/g" \
|
||||
-e "s/__ROLLUP_RETENTION_DAYS__/$${ROLLUP_RETENTION_DAYS}/g" \
|
||||
/schema/init.sql.template > /tmp/init.sql
|
||||
clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --multiquery < /tmp/init.sql
|
||||
clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \
|
||||
"ALTER TABLE $${CLICKHOUSE_DB}.events MODIFY TTL event_time + INTERVAL $${EVENT_RETENTION_DAYS} DAY DELETE"
|
||||
clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \
|
||||
"ALTER TABLE $${CLICKHOUSE_DB}.events_5m MODIFY TTL bucket + INTERVAL $${ROLLUP_RETENTION_DAYS} DAY DELETE"
|
||||
restart: "no"
|
||||
|
||||
redpanda:
|
||||
image: docker.redpanda.com/redpandadata/redpanda:v26.1.13
|
||||
image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13}
|
||||
command:
|
||||
- redpanda
|
||||
- start
|
||||
@@ -60,14 +107,14 @@ services:
|
||||
- --schema-registry-addr
|
||||
- internal://0.0.0.0:8081,external://0.0.0.0:18081
|
||||
- --rpc-addr
|
||||
- redpanda:33145
|
||||
- 0.0.0.0:33145
|
||||
- --advertise-rpc-addr
|
||||
- redpanda:33145
|
||||
- --mode
|
||||
- dev-container
|
||||
- --smp
|
||||
- "2"
|
||||
- --default-log-level=info
|
||||
- "${REDPANDA_SMP:-2}"
|
||||
- --default-log-level=${REDPANDA_LOG_LEVEL:-info}
|
||||
volumes:
|
||||
- redpanda_data:/var/lib/redpanda/data
|
||||
healthcheck:
|
||||
@@ -75,32 +122,36 @@ services:
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 40
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${REDPANDA_KAFKA_PORT:-19092}:19092"
|
||||
- "127.0.0.1:${REDPANDA_ADMIN_PORT:-19644}:9644"
|
||||
- "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_KAFKA_PORT:-19092}:19092"
|
||||
- "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_ADMIN_PORT:-19644}:9644"
|
||||
|
||||
redpanda-init:
|
||||
image: docker.redpanda.com/redpandadata/redpanda:v26.1.13
|
||||
image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13}
|
||||
depends_on:
|
||||
redpanda:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
environment:
|
||||
RPK_BROKERS: redpanda:9092
|
||||
command: >-
|
||||
rpk topic create --if-not-exists -p ${KAFKA_PARTITIONS:-6} -r 1
|
||||
-c retention.ms=${KAFKA_RETENTION_MS:-86400000}
|
||||
siem-events
|
||||
KAFKA_PARTITIONS: ${KAFKA_PARTITIONS:-6}
|
||||
KAFKA_RETENTION_MS: ${KAFKA_RETENTION_MS:-86400000}
|
||||
KAFKA_TOPIC: ${KAFKA_TOPIC:-siem-events}
|
||||
command:
|
||||
- |
|
||||
rpk topic create --if-not-exists -p "$${KAFKA_PARTITIONS}" -r 1 \
|
||||
-c retention.ms="$${KAFKA_RETENTION_MS}" "$${KAFKA_TOPIC}"
|
||||
restart: "no"
|
||||
|
||||
redpanda-console:
|
||||
image: docker.redpanda.com/redpandadata/console:v3.8.0
|
||||
image: ${REDPANDA_CONSOLE_IMAGE:-docker.redpanda.com/redpandadata/console:v3.8.0}
|
||||
depends_on:
|
||||
redpanda:
|
||||
condition: service_healthy
|
||||
entrypoint: /bin/sh
|
||||
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
|
||||
command: -c 'printf "%s" "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
|
||||
environment:
|
||||
CONFIG_FILEPATH: /tmp/config.yml
|
||||
CONSOLE_CONFIG_FILE: |
|
||||
@@ -115,46 +166,93 @@ services:
|
||||
urls: ["http://redpanda:9644"]
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${REDPANDA_CONSOLE_PORT:-8081}:8080"
|
||||
- "${REDPANDA_CONSOLE_BIND:-127.0.0.1}:${REDPANDA_CONSOLE_PORT:-8081}:8080"
|
||||
|
||||
garage-config:
|
||||
image: ${ALPINE_IMAGE:-alpine:3.22}
|
||||
environment:
|
||||
GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET}
|
||||
GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN}
|
||||
GARAGE_METRICS_TOKEN: ${GARAGE_METRICS_TOKEN}
|
||||
volumes:
|
||||
- garage_config:/config
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
command:
|
||||
- |
|
||||
umask 077
|
||||
cat > /config/garage.toml <<CFG
|
||||
metadata_dir = "/var/lib/garage/meta"
|
||||
data_dir = "/var/lib/garage/data"
|
||||
db_engine = "sqlite"
|
||||
replication_factor = 1
|
||||
rpc_bind_addr = "0.0.0.0:3901"
|
||||
rpc_public_addr = "127.0.0.1:3901"
|
||||
rpc_secret = "$${GARAGE_RPC_SECRET}"
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "0.0.0.0:3900"
|
||||
root_domain = ".s3.garage.localhost"
|
||||
|
||||
[admin]
|
||||
api_bind_addr = "0.0.0.0:3903"
|
||||
admin_token = "$${GARAGE_ADMIN_TOKEN}"
|
||||
metrics_token = "$${GARAGE_METRICS_TOKEN}"
|
||||
CFG
|
||||
test -s /config/garage.toml
|
||||
restart: "no"
|
||||
|
||||
garage:
|
||||
image: dxflrs/garage:v2.3.0
|
||||
image: ${GARAGE_IMAGE:-dxflrs/garage:v2.3.0}
|
||||
command: ["/garage", "server", "--single-node", "--default-bucket"]
|
||||
environment:
|
||||
GARAGE_CONFIG_FILE: /config/garage.toml
|
||||
GARAGE_DEFAULT_ACCESS_KEY: ${GARAGE_ACCESS_KEY}
|
||||
GARAGE_DEFAULT_SECRET_KEY: ${GARAGE_SECRET_KEY}
|
||||
GARAGE_DEFAULT_BUCKET: ${GARAGE_BUCKET:-siem-raw}
|
||||
RUST_LOG: ${GARAGE_LOG_LEVEL:-garage=info}
|
||||
volumes:
|
||||
- garage_config:/config:ro
|
||||
- garage_meta:/var/lib/garage/meta
|
||||
- garage_data:/var/lib/garage/data
|
||||
- ./deploy/garage/garage.toml:/etc/garage.toml:ro
|
||||
depends_on:
|
||||
garage-config:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "/garage", "status"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 40
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${GARAGE_S3_PORT:-3900}:3900"
|
||||
- "127.0.0.1:${GARAGE_ADMIN_PORT:-3903}:3903"
|
||||
- "${GARAGE_BIND:-127.0.0.1}:${GARAGE_S3_PORT:-3900}:3900"
|
||||
- "${GARAGE_BIND:-127.0.0.1}:${GARAGE_ADMIN_PORT:-3903}:3903"
|
||||
|
||||
ingress:
|
||||
image: greenfield-siem-app:local
|
||||
image: ${APP_IMAGE:-greenfield-siem-app:local}
|
||||
build: .
|
||||
command: ["ingress"]
|
||||
environment: &app-env
|
||||
SERVICE_ADDR: :8080
|
||||
POSTGRES_URL: postgres://siem:${POSTGRES_PASSWORD}@postgres:5432/siem?sslmode=disable
|
||||
POSTGRES_URL: postgres://${POSTGRES_USER:-siem}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-siem}?sslmode=disable
|
||||
CLICKHOUSE_URL: http://clickhouse:8123
|
||||
CLICKHOUSE_USER: siem
|
||||
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
|
||||
KAFKA_BROKERS: redpanda:9092
|
||||
KAFKA_TOPIC: siem-events
|
||||
KAFKA_TOPIC: ${KAFKA_TOPIC:-siem-events}
|
||||
TENANT_ID: ${TENANT_ID:-default}
|
||||
UI_USERNAME: ${UI_USERNAME:-admin}
|
||||
UI_PASSWORD: ${UI_PASSWORD}
|
||||
ENROLLMENT_KEY: ${ENROLLMENT_KEY}
|
||||
MAX_BODY_BYTES: ${MAX_BODY_BYTES:-8388608}
|
||||
MAX_BATCH_EVENTS: ${MAX_BATCH_EVENTS:-1000}
|
||||
RAW_ARCHIVE_ENABLED: ${RAW_ARCHIVE_ENABLED:-true}
|
||||
RAW_SPOOL_DIR: /var/spool/siem-raw
|
||||
RAW_RETENTION: ${RAW_RETENTION:-720h}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
postgres-schema:
|
||||
condition: service_completed_successfully
|
||||
redpanda-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
@@ -162,61 +260,65 @@ services:
|
||||
- "${INGRESS_BIND:-0.0.0.0}:${INGRESS_PORT:-8090}:8080"
|
||||
|
||||
processor:
|
||||
image: greenfield-siem-app:local
|
||||
image: ${APP_IMAGE:-greenfield-siem-app:local}
|
||||
build: .
|
||||
command: ["processor"]
|
||||
user: "0:0"
|
||||
environment:
|
||||
<<: *app-env
|
||||
KAFKA_GROUP: siem-processor-v1
|
||||
KAFKA_GROUP: ${KAFKA_GROUP:-siem-processor-v1}
|
||||
volumes:
|
||||
- raw_spool:/var/spool/siem-raw
|
||||
depends_on:
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
clickhouse-schema:
|
||||
condition: service_completed_successfully
|
||||
redpanda-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
detector:
|
||||
image: greenfield-siem-app:local
|
||||
image: ${APP_IMAGE:-greenfield-siem-app:local}
|
||||
build: .
|
||||
command: ["detector"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
DETECTOR_INTERVAL: ${DETECTOR_INTERVAL:-30s}
|
||||
DETECTOR_LOOKBACK: ${DETECTOR_LOOKBACK:-20m}
|
||||
depends_on:
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
clickhouse-schema:
|
||||
condition: service_completed_successfully
|
||||
postgres-schema:
|
||||
condition: service_completed_successfully
|
||||
processor:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
|
||||
api:
|
||||
image: greenfield-siem-app:local
|
||||
image: ${APP_IMAGE:-greenfield-siem-app:local}
|
||||
build: .
|
||||
command: ["api"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
UI_QUERY_LIMIT: ${UI_QUERY_LIMIT:-500}
|
||||
depends_on:
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
clickhouse-schema:
|
||||
condition: service_completed_successfully
|
||||
postgres-schema:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${UI_BIND:-0.0.0.0}:${UI_PORT:-8080}:8080"
|
||||
|
||||
archive-uploader:
|
||||
image: rclone/rclone:1.74.4
|
||||
image: ${RCLONE_IMAGE:-rclone/rclone:1.74.4}
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
command: |
|
||||
while true; do
|
||||
rclone move /spool garage:${GARAGE_BUCKET:-siem-raw} --exclude '*.tmp' --create-empty-src-dirs --transfers 8 --checkers 8 --retries 5 || true
|
||||
rclone delete garage:${GARAGE_BUCKET:-siem-raw} --min-age ${RAW_RETENTION:-720h} --retries 5 || true
|
||||
sleep 30
|
||||
done
|
||||
command:
|
||||
- |
|
||||
while true; do
|
||||
rclone move /spool garage:${GARAGE_BUCKET:-siem-raw} --exclude '*.tmp' --create-empty-src-dirs --transfers 8 --checkers 8 --retries 5 || true
|
||||
rclone delete garage:${GARAGE_BUCKET:-siem-raw} --min-age ${RAW_RETENTION:-720h} --retries 5 || true
|
||||
sleep ${ARCHIVE_UPLOAD_INTERVAL_SECONDS:-30}
|
||||
done
|
||||
environment:
|
||||
RCLONE_CONFIG_GARAGE_TYPE: s3
|
||||
RCLONE_CONFIG_GARAGE_PROVIDER: Other
|
||||
@@ -228,13 +330,17 @@ services:
|
||||
volumes:
|
||||
- raw_spool:/spool
|
||||
depends_on:
|
||||
- garage
|
||||
- processor
|
||||
garage:
|
||||
condition: service_healthy
|
||||
processor:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.13.1
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.retention.time=30d"]
|
||||
image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.13.1}
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d}
|
||||
volumes:
|
||||
- ./deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
@@ -243,13 +349,14 @@ services:
|
||||
- redpanda
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
|
||||
- "${PROMETHEUS_BIND:-127.0.0.1}:${PROMETHEUS_PORT:-9090}:9090"
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
clickhouse_data:
|
||||
clickhouse_logs:
|
||||
redpanda_data:
|
||||
garage_config:
|
||||
garage_meta:
|
||||
garage_data:
|
||||
raw_spool:
|
||||
|
||||
30
credentials.sh
Normal file
30
credentials.sh
Normal file
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
[ -f .env ] || { echo ".env fehlt. Zuerst ./deploy.sh ausführen." >&2; exit 1; }
|
||||
set -a; . ./.env; set +a
|
||||
cat <<EOT
|
||||
UI
|
||||
URL: http://127.0.0.1:${UI_PORT:-8080}/ui
|
||||
User: ${UI_USERNAME:-admin}
|
||||
Passwort: ${UI_PASSWORD}
|
||||
|
||||
ClickHouse
|
||||
Datenbank: ${CLICKHOUSE_DB:-siem}
|
||||
User: ${CLICKHOUSE_USER:-siem}
|
||||
Passwort: ${CLICKHOUSE_PASSWORD}
|
||||
HTTP: http://127.0.0.1:${CLICKHOUSE_HTTP_PORT:-8123}
|
||||
Native: 127.0.0.1:${CLICKHOUSE_NATIVE_PORT:-9000}
|
||||
|
||||
CLI im Container:
|
||||
docker compose exec clickhouse clickhouse-client --user ${CLICKHOUSE_USER:-siem} --password '${CLICKHOUSE_PASSWORD}' --database ${CLICKHOUSE_DB:-siem}
|
||||
|
||||
PostgreSQL
|
||||
Datenbank: ${POSTGRES_DB:-siem}
|
||||
User: ${POSTGRES_USER:-siem}
|
||||
Passwort: ${POSTGRES_PASSWORD}
|
||||
Host: 127.0.0.1:${POSTGRES_PORT:-5432}
|
||||
|
||||
Ingress
|
||||
Enrollment-Key: ${ENROLLMENT_KEY}
|
||||
EOT
|
||||
153
deploy.ps1
153
deploy.ps1
@@ -2,69 +2,108 @@ $ErrorActionPreference = "Stop"
|
||||
Set-Location $PSScriptRoot
|
||||
if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { throw "Docker fehlt." }
|
||||
docker compose version | Out-Null
|
||||
function Hex([int]$bytes) { $b = New-Object byte[] $bytes; [Security.Cryptography.RandomNumberGenerator]::Fill($b); return ([BitConverter]::ToString($b)).Replace("-","").ToLower() }
|
||||
if (-not (Test-Path .env)) {
|
||||
$presetEnrollment = $env:ENROLLMENT_KEY
|
||||
$enrollment = if ([string]::IsNullOrWhiteSpace($presetEnrollment)) { Hex 32 } else { $presetEnrollment }
|
||||
$vals = @{
|
||||
POSTGRES_PASSWORD=Hex 24; CLICKHOUSE_PASSWORD=Hex 24; ENROLLMENT_KEY=$enrollment; UI_PASSWORD=Hex 18;
|
||||
GARAGE_ACCESS_KEY="GK$(Hex 16)"; GARAGE_SECRET_KEY=Hex 32; GARAGE_RPC_SECRET=Hex 32;
|
||||
GARAGE_ADMIN_TOKEN=Hex 32; GARAGE_METRICS_TOKEN=Hex 32
|
||||
}
|
||||
@"
|
||||
INGRESS_BIND=0.0.0.0
|
||||
INGRESS_PORT=8090
|
||||
UI_BIND=0.0.0.0
|
||||
UI_PORT=8080
|
||||
TENANT_ID=default
|
||||
UI_USERNAME=admin
|
||||
UI_PASSWORD=$($vals.UI_PASSWORD)
|
||||
POSTGRES_PASSWORD=$($vals.POSTGRES_PASSWORD)
|
||||
CLICKHOUSE_PASSWORD=$($vals.CLICKHOUSE_PASSWORD)
|
||||
ENROLLMENT_KEY=$($vals.ENROLLMENT_KEY)
|
||||
GARAGE_ACCESS_KEY=$($vals.GARAGE_ACCESS_KEY)
|
||||
GARAGE_SECRET_KEY=$($vals.GARAGE_SECRET_KEY)
|
||||
GARAGE_RPC_SECRET=$($vals.GARAGE_RPC_SECRET)
|
||||
GARAGE_ADMIN_TOKEN=$($vals.GARAGE_ADMIN_TOKEN)
|
||||
GARAGE_METRICS_TOKEN=$($vals.GARAGE_METRICS_TOKEN)
|
||||
GARAGE_BUCKET=siem-raw
|
||||
EVENT_RETENTION_DAYS=90
|
||||
ROLLUP_RETENTION_DAYS=730
|
||||
RAW_RETENTION=720h
|
||||
RAW_ARCHIVE_ENABLED=true
|
||||
KAFKA_RETENTION_MS=86400000
|
||||
KAFKA_PARTITIONS=6
|
||||
DETECTOR_INTERVAL=30s
|
||||
DETECTOR_LOOKBACK=20m
|
||||
UI_QUERY_LIMIT=500
|
||||
"@ | Set-Content .env
|
||||
|
||||
function Hex([int]$bytes) {
|
||||
$b = New-Object byte[] $bytes
|
||||
[Security.Cryptography.RandomNumberGenerator]::Fill($b)
|
||||
return ([BitConverter]::ToString($b)).Replace("-","").ToLower()
|
||||
}
|
||||
$envMap=@{}; Get-Content .env | ForEach-Object { if ($_ -match '^([^#=]+)=(.*)$') {$envMap[$matches[1]]=$matches[2]} }
|
||||
if (-not $envMap.ContainsKey('UI_USERNAME')) { $envMap['UI_USERNAME']='admin'; Add-Content .env "UI_USERNAME=admin" }
|
||||
if (-not $envMap.ContainsKey('UI_PASSWORD') -or [string]::IsNullOrWhiteSpace($envMap['UI_PASSWORD'])) { $envMap['UI_PASSWORD']=Hex 18; Add-Content .env "UI_PASSWORD=$($envMap.UI_PASSWORD)" }
|
||||
(Get-Content deploy/garage/garage.toml.template -Raw).Replace('__GARAGE_RPC_SECRET__',$envMap.GARAGE_RPC_SECRET).Replace('__GARAGE_ADMIN_TOKEN__',$envMap.GARAGE_ADMIN_TOKEN).Replace('__GARAGE_METRICS_TOKEN__',$envMap.GARAGE_METRICS_TOKEN) | Set-Content deploy/garage/garage.toml
|
||||
(Get-Content deploy/clickhouse/init.sql.template -Raw).Replace('__EVENT_RETENTION_DAYS__',$envMap.EVENT_RETENTION_DAYS).Replace('__ROLLUP_RETENTION_DAYS__',$envMap.ROLLUP_RETENTION_DAYS) | Set-Content deploy/clickhouse/init.sql
|
||||
|
||||
function Read-EnvFile([string]$path) {
|
||||
$map = [ordered]@{}
|
||||
if (Test-Path $path) {
|
||||
foreach ($line in [IO.File]::ReadAllLines($path)) {
|
||||
if ($line -match '^([^#=][^=]*)=(.*)$') { $map[$matches[1]] = $matches[2] }
|
||||
}
|
||||
}
|
||||
return $map
|
||||
}
|
||||
|
||||
$template = Read-EnvFile ".env.example"
|
||||
$current = Read-EnvFile ".env"
|
||||
foreach ($k in $template.Keys) {
|
||||
if (-not $current.Contains($k)) { $current[$k] = $template[$k] }
|
||||
}
|
||||
|
||||
$presetEnrollment = $env:ENROLLMENT_KEY
|
||||
$secretLengths = @{
|
||||
POSTGRES_PASSWORD=24; CLICKHOUSE_PASSWORD=24; UI_PASSWORD=18;
|
||||
GARAGE_SECRET_KEY=32; GARAGE_RPC_SECRET=32; GARAGE_ADMIN_TOKEN=32; GARAGE_METRICS_TOKEN=32
|
||||
}
|
||||
foreach ($k in $secretLengths.Keys) {
|
||||
if ([string]::IsNullOrWhiteSpace($current[$k]) -or $current[$k] -eq 'CHANGE_ME') {
|
||||
$current[$k] = Hex $secretLengths[$k]
|
||||
}
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($current['GARAGE_ACCESS_KEY']) -or $current['GARAGE_ACCESS_KEY'] -eq 'CHANGE_ME') {
|
||||
$current['GARAGE_ACCESS_KEY'] = "GK$(Hex 16)"
|
||||
}
|
||||
if (-not [string]::IsNullOrWhiteSpace($presetEnrollment)) {
|
||||
$current['ENROLLMENT_KEY'] = $presetEnrollment
|
||||
} elseif ([string]::IsNullOrWhiteSpace($current['ENROLLMENT_KEY']) -or $current['ENROLLMENT_KEY'] -eq 'CHANGE_ME') {
|
||||
$current['ENROLLMENT_KEY'] = Hex 32
|
||||
}
|
||||
|
||||
# Write a complete UTF-8 .env. This fixes partial .env files from earlier releases
|
||||
# and preserves custom keys that are not part of the current template.
|
||||
$lines = New-Object System.Collections.Generic.List[string]
|
||||
$templateKeys = New-Object 'System.Collections.Generic.HashSet[string]'
|
||||
foreach ($line in [IO.File]::ReadAllLines(".env.example")) {
|
||||
if ($line -match '^([^#=][^=]*)=(.*)$') {
|
||||
$key = $matches[1]
|
||||
[void]$templateKeys.Add($key)
|
||||
$lines.Add("$key=$($current[$key])")
|
||||
} else {
|
||||
$lines.Add($line)
|
||||
}
|
||||
}
|
||||
foreach ($key in $current.Keys) {
|
||||
if (-not $templateKeys.Contains([string]$key)) {
|
||||
$lines.Add("$key=$($current[$key])")
|
||||
}
|
||||
}
|
||||
$utf8NoBom = New-Object System.Text.UTF8Encoding($false)
|
||||
[IO.File]::WriteAllLines((Join-Path $PSScriptRoot '.env'), $lines, $utf8NoBom)
|
||||
|
||||
foreach ($key in @('POSTGRES_PASSWORD','CLICKHOUSE_PASSWORD','UI_PASSWORD','ENROLLMENT_KEY','GARAGE_ACCESS_KEY','GARAGE_SECRET_KEY','GARAGE_RPC_SECRET','GARAGE_ADMIN_TOKEN','GARAGE_METRICS_TOKEN')) {
|
||||
if ([string]::IsNullOrWhiteSpace($current[$key]) -or $current[$key] -eq 'CHANGE_ME') {
|
||||
throw "Ungueltige .env: $key fehlt oder ist noch CHANGE_ME."
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Pruefe Compose-Konfiguration ..."
|
||||
docker compose config | Out-Null
|
||||
|
||||
Write-Host "Baue und starte Greenfield SIEM ..."
|
||||
docker compose build --pull
|
||||
docker compose up -d --remove-orphans
|
||||
|
||||
Write-Host "Pruefe Dienste ..."
|
||||
Write-Host "Pruefe Readiness ..."
|
||||
$healthy = $false
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($envMap.UI_PORT)/readyz" | Out-Null
|
||||
Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($envMap.INGRESS_PORT)/readyz" | Out-Null
|
||||
$healthy = $true
|
||||
break
|
||||
} catch {
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
for ($i = 0; $i -lt 90; $i++) {
|
||||
$apiOk = $false; $ingressOk = $false; $chOk = $false; $garageOk = $false
|
||||
try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.UI_PORT)/readyz" | Out-Null; $apiOk = $true } catch {}
|
||||
try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.INGRESS_PORT)/readyz" | Out-Null; $ingressOk = $true } catch {}
|
||||
docker compose exec -T clickhouse clickhouse-client --user $current.CLICKHOUSE_USER --password $current.CLICKHOUSE_PASSWORD --query "SELECT 1" *> $null
|
||||
if ($LASTEXITCODE -eq 0) { $chOk = $true }
|
||||
docker compose exec -T garage /garage status *> $null
|
||||
if ($LASTEXITCODE -eq 0) { $garageOk = $true }
|
||||
if ($apiOk -and $ingressOk -and $chOk -and $garageOk) { $healthy = $true; break }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
if (-not $healthy) { throw "Deployment gestartet, Healthcheck noch nicht gruen. Pruefe: docker compose ps; docker compose logs" }
|
||||
|
||||
if (-not $healthy) {
|
||||
Write-Host "Mindestens ein Dienst wurde nicht bereit. Diagnose:" -ForegroundColor Red
|
||||
docker compose ps
|
||||
docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor api
|
||||
throw "Deployment nicht vollstaendig bereit."
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Greenfield SIEM laeuft."
|
||||
Write-Host "UI: http://127.0.0.1:$($envMap.UI_PORT)/ui"
|
||||
Write-Host "Ingress: http://127.0.0.1:$($envMap.INGRESS_PORT)/ingest"
|
||||
Write-Host "UI-Login: $($envMap.UI_USERNAME) / $($envMap.UI_PASSWORD)"
|
||||
Write-Host "Enrollment-Key: $($envMap.ENROLLMENT_KEY)"
|
||||
Write-Host "Secrets stehen in .env (Datei schuetzen und sichern)."
|
||||
Write-Host "UI: http://127.0.0.1:$($current.UI_PORT)/ui"
|
||||
Write-Host "Ingress: http://127.0.0.1:$($current.INGRESS_PORT)/ingest"
|
||||
Write-Host "UI-Login: $($current.UI_USERNAME) / $($current.UI_PASSWORD)"
|
||||
Write-Host "ClickHouse: user=$($current.CLICKHOUSE_USER) db=$($current.CLICKHOUSE_DB) http=127.0.0.1:$($current.CLICKHOUSE_HTTP_PORT) native=127.0.0.1:$($current.CLICKHOUSE_NATIVE_PORT)"
|
||||
Write-Host "Enrollment-Key: $($current.ENROLLMENT_KEY)"
|
||||
Write-Host "Alle Parameter stehen vollstaendig in .env."
|
||||
|
||||
190
deploy.sh
190
deploy.sh
@@ -1,88 +1,140 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
command -v docker >/dev/null 2>&1 || { echo "Docker fehlt." >&2; exit 1; }
|
||||
docker compose version >/dev/null 2>&1 || { echo "Docker Compose Plugin fehlt." >&2; exit 1; }
|
||||
|
||||
PRESET_ENROLLMENT_KEY="${ENROLLMENT_KEY:-}"
|
||||
randhex(){ n="$1"; if command -v openssl >/dev/null 2>&1; then openssl rand -hex "$n"; else dd if=/dev/urandom bs="$n" count=1 2>/dev/null | od -An -tx1 | tr -d ' \n'; fi; }
|
||||
http_ok(){
|
||||
url="$1"
|
||||
if command -v curl >/dev/null 2>&1; then curl -fsS "$url" >/dev/null 2>&1; return $?; fi
|
||||
if command -v wget >/dev/null 2>&1; then wget -q -O /dev/null "$url" >/dev/null 2>&1; return $?; fi
|
||||
return 2
|
||||
|
||||
randhex() {
|
||||
n="$1"
|
||||
if command -v openssl >/dev/null 2>&1; then
|
||||
openssl rand -hex "$n"
|
||||
else
|
||||
dd if=/dev/urandom bs="$n" count=1 2>/dev/null | od -An -tx1 | tr -d ' \n'
|
||||
fi
|
||||
}
|
||||
if [ ! -f .env ]; then
|
||||
POSTGRES_PASSWORD="$(randhex 24)"
|
||||
CLICKHOUSE_PASSWORD="$(randhex 24)"
|
||||
ENROLLMENT_KEY="${PRESET_ENROLLMENT_KEY:-$(randhex 32)}"
|
||||
UI_PASSWORD="$(randhex 18)"
|
||||
GARAGE_ACCESS_KEY="GK$(randhex 16)"
|
||||
GARAGE_SECRET_KEY="$(randhex 32)"
|
||||
GARAGE_RPC_SECRET="$(randhex 32)"
|
||||
GARAGE_ADMIN_TOKEN="$(randhex 32)"
|
||||
GARAGE_METRICS_TOKEN="$(randhex 32)"
|
||||
cat > .env <<EOT
|
||||
INGRESS_BIND=0.0.0.0
|
||||
INGRESS_PORT=8090
|
||||
UI_BIND=0.0.0.0
|
||||
UI_PORT=8080
|
||||
TENANT_ID=default
|
||||
UI_USERNAME=admin
|
||||
UI_PASSWORD=$UI_PASSWORD
|
||||
POSTGRES_PASSWORD=$POSTGRES_PASSWORD
|
||||
CLICKHOUSE_PASSWORD=$CLICKHOUSE_PASSWORD
|
||||
ENROLLMENT_KEY=$ENROLLMENT_KEY
|
||||
GARAGE_ACCESS_KEY=$GARAGE_ACCESS_KEY
|
||||
GARAGE_SECRET_KEY=$GARAGE_SECRET_KEY
|
||||
GARAGE_RPC_SECRET=$GARAGE_RPC_SECRET
|
||||
GARAGE_ADMIN_TOKEN=$GARAGE_ADMIN_TOKEN
|
||||
GARAGE_METRICS_TOKEN=$GARAGE_METRICS_TOKEN
|
||||
GARAGE_BUCKET=siem-raw
|
||||
EVENT_RETENTION_DAYS=90
|
||||
ROLLUP_RETENTION_DAYS=730
|
||||
RAW_RETENTION=720h
|
||||
RAW_ARCHIVE_ENABLED=true
|
||||
KAFKA_RETENTION_MS=86400000
|
||||
KAFKA_PARTITIONS=6
|
||||
DETECTOR_INTERVAL=30s
|
||||
DETECTOR_LOOKBACK=20m
|
||||
UI_QUERY_LIMIT=500
|
||||
EOT
|
||||
chmod 600 .env 2>/dev/null || true
|
||||
|
||||
get_env() {
|
||||
key="$1"
|
||||
awk -F= -v k="$key" '$1==k {sub(/^[^=]*=/, ""); print; exit}' .env 2>/dev/null || true
|
||||
}
|
||||
|
||||
set_env() {
|
||||
key="$1"; value="$2"; tmp=".env.tmp.$$"
|
||||
awk -v k="$key" -v v="$value" '
|
||||
BEGIN {done=0}
|
||||
index($0,k"=")==1 {print k"="v; done=1; next}
|
||||
{print}
|
||||
END {if(!done) print k"="v}
|
||||
' .env > "$tmp"
|
||||
mv "$tmp" .env
|
||||
}
|
||||
|
||||
# Start from a complete template, but preserve all values from an existing .env.
|
||||
[ -f .env ] || : > .env
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
''|'#'*) continue ;;
|
||||
esac
|
||||
key=${line%%=*}
|
||||
if ! grep -q "^${key}=" .env 2>/dev/null; then
|
||||
printf '%s\n' "$line" >> .env
|
||||
fi
|
||||
done < .env.example
|
||||
|
||||
# Generate secrets only if absent/blank/placeholder. Existing installations keep their credentials.
|
||||
for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD UI_PASSWORD GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do
|
||||
val="$(get_env "$key")"
|
||||
if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then
|
||||
case "$key" in
|
||||
POSTGRES_PASSWORD|CLICKHOUSE_PASSWORD) val="$(randhex 24)" ;;
|
||||
UI_PASSWORD) val="$(randhex 18)" ;;
|
||||
*) val="$(randhex 32)" ;;
|
||||
esac
|
||||
set_env "$key" "$val"
|
||||
fi
|
||||
done
|
||||
|
||||
val="$(get_env GARAGE_ACCESS_KEY)"
|
||||
if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then
|
||||
set_env GARAGE_ACCESS_KEY "GK$(randhex 16)"
|
||||
fi
|
||||
set -a; . ./.env; set +a
|
||||
UI_USERNAME="${UI_USERNAME:-admin}"
|
||||
if [ -z "${UI_PASSWORD:-}" ]; then
|
||||
UI_PASSWORD="$(randhex 18)"
|
||||
printf '\nUI_USERNAME=%s\nUI_PASSWORD=%s\n' "$UI_USERNAME" "$UI_PASSWORD" >> .env
|
||||
export UI_USERNAME UI_PASSWORD
|
||||
|
||||
val="$(get_env ENROLLMENT_KEY)"
|
||||
if [ -n "$PRESET_ENROLLMENT_KEY" ]; then
|
||||
set_env ENROLLMENT_KEY "$PRESET_ENROLLMENT_KEY"
|
||||
elif [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then
|
||||
set_env ENROLLMENT_KEY "$(randhex 32)"
|
||||
fi
|
||||
sed -e "s/__GARAGE_RPC_SECRET__/$GARAGE_RPC_SECRET/g" -e "s/__GARAGE_ADMIN_TOKEN__/$GARAGE_ADMIN_TOKEN/g" -e "s/__GARAGE_METRICS_TOKEN__/$GARAGE_METRICS_TOKEN/g" deploy/garage/garage.toml.template > deploy/garage/garage.toml
|
||||
sed -e "s/__EVENT_RETENTION_DAYS__/${EVENT_RETENTION_DAYS:-90}/g" -e "s/__ROLLUP_RETENTION_DAYS__/${ROLLUP_RETENTION_DAYS:-730}/g" deploy/clickhouse/init.sql.template > deploy/clickhouse/init.sql
|
||||
chmod 600 deploy/garage/garage.toml 2>/dev/null || true
|
||||
|
||||
chmod 600 .env 2>/dev/null || true
|
||||
|
||||
# Validate required settings without sourcing .env as shell code. Docker Compose parses
|
||||
# .env itself; this also keeps values with shell-special characters safe.
|
||||
for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD UI_PASSWORD ENROLLMENT_KEY GARAGE_ACCESS_KEY GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do
|
||||
val="$(get_env "$key")"
|
||||
if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then
|
||||
echo "Ungültige .env: $key fehlt oder ist noch CHANGE_ME." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
UI_PORT_VALUE="$(get_env UI_PORT)"; [ -n "$UI_PORT_VALUE" ] || UI_PORT_VALUE=8080
|
||||
INGRESS_PORT_VALUE="$(get_env INGRESS_PORT)"; [ -n "$INGRESS_PORT_VALUE" ] || INGRESS_PORT_VALUE=8090
|
||||
CLICKHOUSE_USER_VALUE="$(get_env CLICKHOUSE_USER)"; [ -n "$CLICKHOUSE_USER_VALUE" ] || CLICKHOUSE_USER_VALUE=siem
|
||||
CLICKHOUSE_DB_VALUE="$(get_env CLICKHOUSE_DB)"; [ -n "$CLICKHOUSE_DB_VALUE" ] || CLICKHOUSE_DB_VALUE=siem
|
||||
CLICKHOUSE_PASSWORD_VALUE="$(get_env CLICKHOUSE_PASSWORD)"
|
||||
CLICKHOUSE_HTTP_PORT_VALUE="$(get_env CLICKHOUSE_HTTP_PORT)"; [ -n "$CLICKHOUSE_HTTP_PORT_VALUE" ] || CLICKHOUSE_HTTP_PORT_VALUE=8123
|
||||
CLICKHOUSE_NATIVE_PORT_VALUE="$(get_env CLICKHOUSE_NATIVE_PORT)"; [ -n "$CLICKHOUSE_NATIVE_PORT_VALUE" ] || CLICKHOUSE_NATIVE_PORT_VALUE=9000
|
||||
UI_USERNAME_VALUE="$(get_env UI_USERNAME)"; [ -n "$UI_USERNAME_VALUE" ] || UI_USERNAME_VALUE=admin
|
||||
UI_PASSWORD_VALUE="$(get_env UI_PASSWORD)"
|
||||
ENROLLMENT_KEY_VALUE="$(get_env ENROLLMENT_KEY)"
|
||||
|
||||
# Validate the final config before pulling/building anything.
|
||||
echo "Prüfe Compose-Konfiguration ..."
|
||||
docker compose config >/dev/null
|
||||
|
||||
echo "Baue und starte Greenfield SIEM ..."
|
||||
docker compose build --pull
|
||||
docker compose up -d --remove-orphans
|
||||
|
||||
echo "Prüfe Dienste ..."
|
||||
if command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; then
|
||||
i=0
|
||||
while [ "$i" -lt 60 ]; do
|
||||
if http_ok "http://127.0.0.1:${UI_PORT:-8080}/readyz" && http_ok "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz"; then break; fi
|
||||
i=$((i+1)); sleep 2
|
||||
done
|
||||
if [ "$i" -ge 60 ]; then echo "Deployment gestartet, Healthcheck noch nicht grün. Prüfe: docker compose ps && docker compose logs" >&2; exit 1; fi
|
||||
else
|
||||
echo "Hinweis: curl/wget fehlt; HTTP-Healthcheck wird übersprungen."
|
||||
docker compose ps
|
||||
http_ok() {
|
||||
url="$1"
|
||||
if command -v curl >/dev/null 2>&1; then curl -fsS "$url" >/dev/null 2>&1; return $?; fi
|
||||
if command -v wget >/dev/null 2>&1; then wget -q -O /dev/null "$url" >/dev/null 2>&1; return $?; fi
|
||||
return 2
|
||||
}
|
||||
|
||||
echo "Prüfe Readiness ..."
|
||||
i=0
|
||||
while [ "$i" -lt 90 ]; do
|
||||
api_ok=0; ingress_ok=0; ch_ok=0; garage_ok=0
|
||||
http_ok "http://127.0.0.1:${UI_PORT_VALUE}/readyz" && api_ok=1 || true
|
||||
http_ok "http://127.0.0.1:${INGRESS_PORT_VALUE}/readyz" && ingress_ok=1 || true
|
||||
docker compose exec -T clickhouse clickhouse-client --user "${CLICKHOUSE_USER_VALUE}" --password "${CLICKHOUSE_PASSWORD_VALUE}" --query 'SELECT 1' >/dev/null 2>&1 && ch_ok=1 || true
|
||||
docker compose exec -T garage /garage status >/dev/null 2>&1 && garage_ok=1 || true
|
||||
if [ "$api_ok" -eq 1 ] && [ "$ingress_ok" -eq 1 ] && [ "$ch_ok" -eq 1 ] && [ "$garage_ok" -eq 1 ]; then
|
||||
break
|
||||
fi
|
||||
i=$((i+1))
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ "$i" -ge 90 ]; then
|
||||
echo "Mindestens ein Dienst wurde nicht bereit. Diagnose:" >&2
|
||||
docker compose ps >&2 || true
|
||||
echo >&2
|
||||
docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor api >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Greenfield SIEM läuft."
|
||||
echo "UI: http://127.0.0.1:${UI_PORT:-8080}/ui"
|
||||
echo "Ingress: http://127.0.0.1:${INGRESS_PORT:-8090}/ingest"
|
||||
echo "UI-Login: $UI_USERNAME / $UI_PASSWORD"
|
||||
echo "Enrollment-Key: $ENROLLMENT_KEY"
|
||||
echo "Secrets stehen in .env (Datei schützen und sichern)."
|
||||
echo "UI: http://127.0.0.1:${UI_PORT_VALUE}/ui"
|
||||
echo "Ingress: http://127.0.0.1:${INGRESS_PORT_VALUE}/ingest"
|
||||
echo "UI-Login: ${UI_USERNAME_VALUE} / ${UI_PASSWORD_VALUE}"
|
||||
echo "ClickHouse: user=${CLICKHOUSE_USER_VALUE} db=${CLICKHOUSE_DB_VALUE} http=127.0.0.1:${CLICKHOUSE_HTTP_PORT_VALUE} native=127.0.0.1:${CLICKHOUSE_NATIVE_PORT_VALUE}"
|
||||
echo "Enrollment-Key: ${ENROLLMENT_KEY_VALUE}"
|
||||
echo "Alle Parameter stehen vollständig in .env."
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
CREATE DATABASE IF NOT EXISTS siem;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS siem.events
|
||||
(
|
||||
event_uid String,
|
||||
queue_partition Int32,
|
||||
queue_offset Int64,
|
||||
tenant_id LowCardinality(String),
|
||||
event_time DateTime64(3, 'UTC'),
|
||||
ingest_time DateTime64(3, 'UTC'),
|
||||
event_date Date MATERIALIZED toDate(event_time),
|
||||
agent_id String,
|
||||
host_name LowCardinality(String),
|
||||
source_type LowCardinality(String),
|
||||
channel LowCardinality(String),
|
||||
provider LowCardinality(String),
|
||||
event_code UInt32,
|
||||
category LowCardinality(String),
|
||||
action LowCardinality(String),
|
||||
outcome LowCardinality(String),
|
||||
severity UInt8,
|
||||
user_name String,
|
||||
user_domain LowCardinality(String),
|
||||
subject_user String,
|
||||
subject_domain LowCardinality(String),
|
||||
target_user String,
|
||||
target_domain LowCardinality(String),
|
||||
source_ip String,
|
||||
source_port UInt16,
|
||||
destination_ip String,
|
||||
destination_port UInt16,
|
||||
workstation String,
|
||||
logon_type LowCardinality(String),
|
||||
authentication_package LowCardinality(String),
|
||||
logon_process LowCardinality(String),
|
||||
status_code LowCardinality(String),
|
||||
sub_status_code LowCardinality(String),
|
||||
failure_reason LowCardinality(String),
|
||||
process_path String,
|
||||
parent_process_path String,
|
||||
command_line String CODEC(ZSTD(3)),
|
||||
message String CODEC(ZSTD(3)),
|
||||
attributes Map(String, String) CODEC(ZSTD(3)),
|
||||
raw_object_key String,
|
||||
raw_index UInt32,
|
||||
payload_hash FixedString(64),
|
||||
schema_version UInt16,
|
||||
parser_version UInt16,
|
||||
ingest_delay_ms Int64,
|
||||
INDEX idx_host host_name TYPE set(10000) GRANULARITY 4,
|
||||
INDEX idx_event_code event_code TYPE set(256) GRANULARITY 4,
|
||||
INDEX idx_user user_name TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||
INDEX idx_target target_user TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||
INDEX idx_source_ip source_ip TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||
INDEX idx_process process_path TYPE bloom_filter(0.01) GRANULARITY 8
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(ingest_time)
|
||||
PARTITION BY toYYYYMM(event_time)
|
||||
ORDER BY (tenant_id, event_time, host_name, event_code, event_uid)
|
||||
TTL event_time + INTERVAL 90 DAY DELETE
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS siem.events_5m
|
||||
(
|
||||
tenant_id LowCardinality(String),
|
||||
bucket DateTime('UTC'),
|
||||
host_name LowCardinality(String),
|
||||
event_code UInt32,
|
||||
category LowCardinality(String),
|
||||
action LowCardinality(String),
|
||||
outcome LowCardinality(String),
|
||||
cnt_state AggregateFunction(uniqExact, String)
|
||||
)
|
||||
ENGINE = AggregatingMergeTree
|
||||
PARTITION BY toYYYYMM(bucket)
|
||||
ORDER BY (tenant_id, bucket, host_name, event_code, category, action, outcome)
|
||||
TTL bucket + INTERVAL 730 DAY DELETE;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS siem.events_5m_mv TO siem.events_5m AS
|
||||
SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state
|
||||
FROM siem.events
|
||||
GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome;
|
||||
@@ -1,6 +1,6 @@
|
||||
CREATE DATABASE IF NOT EXISTS siem;
|
||||
CREATE DATABASE IF NOT EXISTS __CLICKHOUSE_DB__;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS siem.events
|
||||
CREATE TABLE IF NOT EXISTS __CLICKHOUSE_DB__.events
|
||||
(
|
||||
event_uid String,
|
||||
queue_partition Int32,
|
||||
@@ -60,7 +60,7 @@ ORDER BY (tenant_id, event_time, host_name, event_code, event_uid)
|
||||
TTL event_time + INTERVAL __EVENT_RETENTION_DAYS__ DAY DELETE
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS siem.events_5m
|
||||
CREATE TABLE IF NOT EXISTS __CLICKHOUSE_DB__.events_5m
|
||||
(
|
||||
tenant_id LowCardinality(String),
|
||||
bucket DateTime('UTC'),
|
||||
@@ -76,7 +76,7 @@ PARTITION BY toYYYYMM(bucket)
|
||||
ORDER BY (tenant_id, bucket, host_name, event_code, category, action, outcome)
|
||||
TTL bucket + INTERVAL __ROLLUP_RETENTION_DAYS__ DAY DELETE;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS siem.events_5m_mv TO siem.events_5m AS
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS __CLICKHOUSE_DB__.events_5m_mv TO __CLICKHOUSE_DB__.events_5m AS
|
||||
SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state
|
||||
FROM siem.events
|
||||
FROM __CLICKHOUSE_DB__.events
|
||||
GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome;
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
metadata_dir = "/var/lib/garage/meta"
|
||||
data_dir = "/var/lib/garage/data"
|
||||
db_engine = "sqlite"
|
||||
replication_factor = 1
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_public_addr = "garage:3901"
|
||||
rpc_secret = "__GARAGE_RPC_SECRET__"
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "[::]:3900"
|
||||
root_domain = ".s3.garage.localhost"
|
||||
|
||||
[admin]
|
||||
api_bind_addr = "[::]:3903"
|
||||
admin_token = "__GARAGE_ADMIN_TOKEN__"
|
||||
metrics_token = "__GARAGE_METRICS_TOKEN__"
|
||||
48
doctor.sh
48
doctor.sh
@@ -3,13 +3,15 @@ set -u
|
||||
cd "$(dirname "$0")"
|
||||
[ -f .env ] && { set -a; . ./.env; set +a; }
|
||||
|
||||
echo "== Docker =="
|
||||
section(){ printf '\n== %s ==\n' "$1"; }
|
||||
|
||||
section "Compose config"
|
||||
docker compose config >/dev/null 2>&1 && echo "OK" || echo "FEHLER: docker compose config"
|
||||
|
||||
section "Docker"
|
||||
docker compose ps || true
|
||||
|
||||
echo "\n== Host filesystem =="
|
||||
df -h . 2>/dev/null || true
|
||||
|
||||
echo "\n== HTTP readiness =="
|
||||
section "HTTP readiness"
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
printf "Ingress: "; curl -fsS "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz" || true; echo
|
||||
printf "API: "; curl -fsS "http://127.0.0.1:${UI_PORT:-8080}/readyz" || true; echo
|
||||
@@ -17,19 +19,31 @@ else
|
||||
echo "curl fehlt; HTTP-Prüfung übersprungen."
|
||||
fi
|
||||
|
||||
echo "\n== Redpanda =="
|
||||
section "Garage"
|
||||
docker compose exec -T garage /garage status 2>/dev/null || true
|
||||
printf "S3 archive: "
|
||||
docker compose exec -T archive-uploader rclone lsd "garage:${GARAGE_BUCKET:-siem-raw}" 2>/dev/null && echo "OK" || echo "nicht erreichbar"
|
||||
|
||||
section "ClickHouse"
|
||||
docker compose exec -T clickhouse clickhouse-client \
|
||||
--user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" \
|
||||
--query "SELECT version(), currentUser(), currentDatabase()" 2>/dev/null || true
|
||||
docker compose exec -T clickhouse clickhouse-client \
|
||||
--user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" \
|
||||
--database "${CLICKHOUSE_DB:-siem}" \
|
||||
--query "SELECT formatReadableQuantity(sum(rows)) AS rows, formatReadableSize(sum(bytes_on_disk)) AS disk, count() AS active_parts FROM system.parts WHERE database='${CLICKHOUSE_DB:-siem}' AND table='events' AND active" 2>/dev/null || true
|
||||
|
||||
section "Redpanda"
|
||||
docker compose exec -T redpanda rpk cluster health 2>/dev/null || true
|
||||
docker compose exec -T redpanda rpk group describe siem-processor-v1 2>/dev/null || true
|
||||
docker compose exec -T redpanda rpk group describe "${KAFKA_GROUP:-siem-processor-v1}" 2>/dev/null || true
|
||||
|
||||
echo "\n== ClickHouse events =="
|
||||
docker compose exec -T clickhouse clickhouse-client --user siem --password "${CLICKHOUSE_PASSWORD:-}" --query "SELECT formatReadableQuantity(sum(rows)) AS rows, formatReadableSize(sum(bytes_on_disk)) AS disk, count() AS active_parts FROM system.parts WHERE database='siem' AND table='events' AND active" 2>/dev/null || true
|
||||
docker compose exec -T clickhouse clickhouse-client --user siem --password "${CLICKHOUSE_PASSWORD:-}" --query "SELECT max(event_time) AS latest_event, max(ingest_time) AS latest_ingest, quantile(0.95)(abs(ingest_delay_ms))/1000 AS p95_ingest_delay_seconds FROM siem.events WHERE event_time >= now() - INTERVAL 1 DAY" 2>/dev/null || true
|
||||
section "PostgreSQL control plane"
|
||||
docker compose exec -T postgres psql -U "${POSTGRES_USER:-siem}" -d "${POSTGRES_DB:-siem}" -Atc \
|
||||
"SELECT 'agents='||count(*) FROM agents; SELECT 'open_detections='||count(*) FROM detections WHERE status IN ('open','investigating');" 2>/dev/null || true
|
||||
|
||||
echo "\n== PostgreSQL control plane =="
|
||||
docker compose exec -T postgres psql -U siem -d siem -Atc "SELECT 'agents='||count(*) FROM agents; SELECT 'open_detections='||count(*) FROM detections WHERE status IN ('open','investigating');" 2>/dev/null || true
|
||||
section "Raw spool"
|
||||
docker compose exec -T archive-uploader sh -c 'du -sh /spool 2>/dev/null || true; printf "files="; find /spool -type f 2>/dev/null | wc -l' 2>/dev/null || true
|
||||
|
||||
echo "\n== Raw spool =="
|
||||
docker compose exec -T archive-uploader sh -c 'du -sh /spool 2>/dev/null || true; find /spool -type f 2>/dev/null | wc -l' 2>/dev/null || true
|
||||
|
||||
echo "\n== Recent errors =="
|
||||
docker compose logs --since=15m ingress processor detector api archive-uploader 2>&1 | grep -Ei 'error|fatal|failed|panic|not_ready' | tail -80 || true
|
||||
section "Recent errors"
|
||||
docker compose logs --since=15m garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor detector api archive-uploader 2>&1 \
|
||||
| grep -Ei 'error|fatal|failed|panic|not_ready|exception' | tail -120 || true
|
||||
|
||||
10
go.mod
10
go.mod
@@ -6,13 +6,3 @@ require (
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/segmentio/kafka-go v0.4.51
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.15.9 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
golang.org/x/sync v0.17.0 // indirect
|
||||
golang.org/x/text v0.29.0 // indirect
|
||||
)
|
||||
|
||||
40
go.sum
40
go.sum
@@ -1,40 +0,0 @@
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
||||
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/segmentio/kafka-go v0.4.51 h1:JgDPPG75tC1rWIS2Me6MwcvXJ6f49UQ4HjAOef71Hno=
|
||||
github.com/segmentio/kafka-go v0.4.51/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
||||
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -82,7 +82,7 @@ func Run(ctx context.Context, cfg config.Config) error {
|
||||
func summary(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 8*time.Second)
|
||||
defer cancel()
|
||||
q := fmt.Sprintf(`SELECT uniqExactMerge(cnt_state) events_24h, uniqExact(host_name) active_hosts FROM siem.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL 24 HOUR`, clickhouse.Q(cfg.TenantID))
|
||||
q := fmt.Sprintf(`SELECT uniqExactMerge(cnt_state) events_24h, uniqExact(host_name) active_hosts FROM %s.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL 24 HOUR`, clickhouse.Ident(cfg.ClickHouseDB), clickhouse.Q(cfg.TenantID))
|
||||
rows, e := ch.QueryJSON(ctx, q)
|
||||
if e != nil {
|
||||
j(w, 500, map[string]string{"error": e.Error()})
|
||||
@@ -118,7 +118,7 @@ func events(w http.ResponseWriter, r *http.Request, cfg config.Config, ch *click
|
||||
where = append(where, "event_code="+v)
|
||||
}
|
||||
}
|
||||
q := fmt.Sprintf(`SELECT event_uid,event_time,host_name,channel,event_code,category,action,outcome,severity,user_name,subject_user,target_user,source_ip,workstation,process_path,message,raw_object_key,raw_index FROM siem.events WHERE %s ORDER BY event_time DESC, ingest_time DESC LIMIT 1 BY event_uid LIMIT %d`, strings.Join(where, " AND "), limit)
|
||||
q := fmt.Sprintf(`SELECT event_uid,event_time,host_name,channel,event_code,category,action,outcome,severity,user_name,subject_user,target_user,source_ip,workstation,process_path,message,raw_object_key,raw_index FROM %s.events WHERE %s ORDER BY event_time DESC, ingest_time DESC LIMIT 1 BY event_uid LIMIT %d`, clickhouse.Ident(cfg.ClickHouseDB), strings.Join(where, " AND "), limit)
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
|
||||
defer cancel()
|
||||
rows, e := ch.QueryJSON(ctx, q)
|
||||
|
||||
@@ -17,12 +17,12 @@ import (
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
base, user, pass string
|
||||
hc *http.Client
|
||||
base, db, user, pass string
|
||||
hc *http.Client
|
||||
}
|
||||
|
||||
func New(cfg config.Config) *Client {
|
||||
return &Client{base: cfg.ClickHouseURL, user: cfg.ClickHouseUser, pass: cfg.ClickHousePassword, hc: &http.Client{Timeout: 30 * time.Second}}
|
||||
return &Client{base: cfg.ClickHouseURL, db: cfg.ClickHouseDB, user: cfg.ClickHouseUser, pass: cfg.ClickHousePassword, hc: &http.Client{Timeout: 30 * time.Second}}
|
||||
}
|
||||
func (c *Client) request(ctx context.Context, query string, body io.Reader) (*http.Response, error) {
|
||||
u := c.base + "/?query=" + url.QueryEscape(query)
|
||||
@@ -57,7 +57,7 @@ func (c *Client) InsertEvents(ctx context.Context, events []contracts.CanonicalE
|
||||
return e
|
||||
}
|
||||
}
|
||||
q := `INSERT INTO siem.events FORMAT JSONEachRow`
|
||||
q := `INSERT INTO ` + Ident(c.db) + `.events FORMAT JSONEachRow`
|
||||
r, e := c.request(ctx, q, &b)
|
||||
if e != nil {
|
||||
return e
|
||||
@@ -95,4 +95,5 @@ func (c *Client) QueryJSON(ctx context.Context, q string) ([]map[string]any, err
|
||||
}
|
||||
return out, s.Err()
|
||||
}
|
||||
func Q(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" }
|
||||
func Q(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" }
|
||||
func Ident(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" }
|
||||
|
||||
@@ -11,6 +11,7 @@ type Config struct {
|
||||
ServiceAddr string
|
||||
PostgresURL string
|
||||
ClickHouseURL string
|
||||
ClickHouseDB string
|
||||
ClickHouseUser string
|
||||
ClickHousePassword string
|
||||
KafkaBrokers []string
|
||||
@@ -23,7 +24,6 @@ type Config struct {
|
||||
RawArchiveEnabled bool
|
||||
RawSpoolDir string
|
||||
RawRetention time.Duration
|
||||
EventRetention time.Duration
|
||||
DetectorInterval time.Duration
|
||||
DetectorLookback time.Duration
|
||||
UIQueryLimit int
|
||||
@@ -36,6 +36,7 @@ func Load() Config {
|
||||
ServiceAddr: env("SERVICE_ADDR", ":8080"),
|
||||
PostgresURL: env("POSTGRES_URL", "postgres://siem:siem@postgres:5432/siem?sslmode=disable"),
|
||||
ClickHouseURL: strings.TrimRight(env("CLICKHOUSE_URL", "http://clickhouse:8123"), "/"),
|
||||
ClickHouseDB: env("CLICKHOUSE_DB", "siem"),
|
||||
ClickHouseUser: env("CLICKHOUSE_USER", "siem"),
|
||||
ClickHousePassword: env("CLICKHOUSE_PASSWORD", "siem"),
|
||||
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "redpanda:9092")),
|
||||
@@ -48,7 +49,6 @@ func Load() Config {
|
||||
RawArchiveEnabled: envBool("RAW_ARCHIVE_ENABLED", true),
|
||||
RawSpoolDir: env("RAW_SPOOL_DIR", "/var/spool/siem-raw"),
|
||||
RawRetention: envDuration("RAW_RETENTION", 30*24*time.Hour),
|
||||
EventRetention: envDuration("EVENT_RETENTION", 90*24*time.Hour),
|
||||
DetectorInterval: envDuration("DETECTOR_INTERVAL", 30*time.Second),
|
||||
DetectorLookback: envDuration("DETECTOR_LOOKBACK", 20*time.Minute),
|
||||
UIQueryLimit: envInt("UI_QUERY_LIMIT", 500),
|
||||
|
||||
@@ -50,7 +50,7 @@ func Run(ctx context.Context, cfg config.Config) error {
|
||||
func runAll(ctx context.Context, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) error {
|
||||
end := time.Now().UTC()
|
||||
start := end.Add(-cfg.DetectorLookback)
|
||||
for _, r := range rules() {
|
||||
for _, r := range rules(cfg.ClickHouseDB) {
|
||||
q := r.query(start, end, cfg.TenantID)
|
||||
rows, e := ch.QueryJSON(ctx, q)
|
||||
if e != nil {
|
||||
@@ -73,39 +73,40 @@ func runAll(ctx context.Context, cfg config.Config, pg *postgres.Store, ch *clic
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func rules() []rule {
|
||||
func rules(db string) []rule {
|
||||
table := clickhouse.Ident(db) + ".events"
|
||||
return []rule{
|
||||
{name: "audit_log_cleared", severity: "critical", eventCode: 1102, score: 9.8, query: simpleEvent(1102, 1), summary: func(m map[string]any) string {
|
||||
{name: "audit_log_cleared", severity: "critical", eventCode: 1102, score: 9.8, query: simpleEvent(table, 1102, 1), summary: func(m map[string]any) string {
|
||||
return fmt.Sprintf("Audit-Log auf %s wurde gelöscht", str(m["host_name"]))
|
||||
}},
|
||||
{name: "service_installed", severity: "high", eventCode: 7045, score: 8.0, query: simpleEvent(7045, 1), summary: func(m map[string]any) string {
|
||||
{name: "service_installed", severity: "high", eventCode: 7045, score: 8.0, query: simpleEvent(table, 7045, 1), summary: func(m map[string]any) string {
|
||||
return fmt.Sprintf("Neuer Dienst auf %s installiert", str(m["host_name"]))
|
||||
}},
|
||||
{name: "account_lockout", severity: "medium", eventCode: 4740, score: 5.5, query: func(s, e time.Time, t string) string {
|
||||
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4740 GROUP BY host_name,user_name,workstation HAVING cnt>=1`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
|
||||
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4740 GROUP BY host_name,user_name,workstation HAVING cnt>=1`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
|
||||
}, summary: func(m map[string]any) string {
|
||||
return fmt.Sprintf("Account-Lockout: %s; Caller %s; DC/Host %s (%d×)", str(m["user_name"]), fallback(str(m["workstation"]), "unbekannt"), str(m["host_name"]), int64(num(m["cnt"])))
|
||||
}},
|
||||
{name: "failed_logon_burst", severity: "high", eventCode: 4625, score: 7.5, query: func(s, e time.Time, t string) string {
|
||||
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND target_user!='' GROUP BY host_name,user_name,source_ip,workstation HAVING cnt>=20`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
|
||||
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND target_user!='' GROUP BY host_name,user_name,source_ip,workstation HAVING cnt>=20`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
|
||||
}, summary: func(m map[string]any) string {
|
||||
return fmt.Sprintf("%d fehlgeschlagene Logons für %s auf %s", int64(num(m["cnt"])), str(m["user_name"]), str(m["host_name"]))
|
||||
}},
|
||||
{name: "password_spray", severity: "high", eventCode: 4625, score: 8.5, query: func(s, e time.Time, t string) string {
|
||||
return fmt.Sprintf(`SELECT '' AS host_name, '' AS user_name, source_ip, '' AS workstation, uniqExact(target_user) users, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND source_ip!='' AND target_user!='' GROUP BY source_ip HAVING users>=10 AND cnt>=20`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
|
||||
return fmt.Sprintf(`SELECT '' AS host_name, '' AS user_name, source_ip, '' AS workstation, uniqExact(target_user) users, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND source_ip!='' AND target_user!='' GROUP BY source_ip HAVING users>=10 AND cnt>=20`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
|
||||
}, summary: func(m map[string]any) string {
|
||||
return fmt.Sprintf("Password-Spray von %s gegen %.0f Benutzer (%d Versuche)", str(m["source_ip"]), num(m["users"]), int64(num(m["cnt"])))
|
||||
}},
|
||||
{name: "privileged_group_change", severity: "critical", eventCode: 4728, score: 9.2, query: func(s, e time.Time, t string) string {
|
||||
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code IN (4728,4732,4756) GROUP BY host_name,user_name,workstation HAVING cnt>=1`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
|
||||
return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code IN (4728,4732,4756) GROUP BY host_name,user_name,workstation HAVING cnt>=1`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
|
||||
}, summary: func(m map[string]any) string {
|
||||
return fmt.Sprintf("Privilegierte Gruppenmitgliedschaft geändert: %s auf %s", str(m["user_name"]), str(m["host_name"]))
|
||||
}},
|
||||
}
|
||||
}
|
||||
func simpleEvent(id uint32, min int) func(time.Time, time.Time, string) string {
|
||||
func simpleEvent(table string, id uint32, min int) func(time.Time, time.Time, string) string {
|
||||
return func(s, e time.Time, t string) string {
|
||||
return fmt.Sprintf(`SELECT host_name, '' AS user_name, '' AS source_ip, '' AS workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=%d GROUP BY host_name HAVING cnt>=%d`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)), id, min)
|
||||
return fmt.Sprintf(`SELECT host_name, '' AS user_name, '' AS source_ip, '' AS workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=%d GROUP BY host_name HAVING cnt>=%d`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)), id, min)
|
||||
}
|
||||
}
|
||||
func ts(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000") }
|
||||
@@ -143,7 +144,12 @@ func fingerprint(v ...string) string {
|
||||
h := sha256.Sum256([]byte(strings.Join(v, "|")))
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
func fallback(v, d string) string { if strings.TrimSpace(v) == "" { return d }; return v }
|
||||
func fallback(v, d string) string {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return d
|
||||
}
|
||||
return v
|
||||
}
|
||||
func max64(a, b int64) int64 {
|
||||
if a > b {
|
||||
return a
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
printf 'ACHTUNG: Alle SIEM-Daten löschen. Tippe DELETE: '
|
||||
printf 'ACHTUNG: Alle SIEM-Daten und Docker-Volumes löschen. Tippe DELETE: '
|
||||
read x
|
||||
[ "$x" = "DELETE" ] || exit 1
|
||||
docker compose down -v --remove-orphans
|
||||
rm -f deploy/garage/garage.toml
|
||||
|
||||
14
status.sh
14
status.sh
@@ -1,8 +1,16 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
[ -f .env ] && { set -a; . ./.env; set +a; }
|
||||
docker compose ps
|
||||
printf '\nIngress readiness:\n'
|
||||
curl -fsS http://127.0.0.1:$(grep '^INGRESS_PORT=' .env | cut -d= -f2-)/readyz || true
|
||||
printf '\n\nRecent errors:\n'
|
||||
docker compose logs --since=10m ingress processor detector api 2>&1 | grep -Ei 'error|fatal|failed|panic' | tail -50 || true
|
||||
if command -v curl >/dev/null 2>&1; then curl -fsS "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz" || true; fi
|
||||
printf '\nAPI readiness:\n'
|
||||
if command -v curl >/dev/null 2>&1; then curl -fsS "http://127.0.0.1:${UI_PORT:-8080}/readyz" || true; fi
|
||||
printf '\nGarage:\n'
|
||||
docker compose exec -T garage /garage status 2>/dev/null || true
|
||||
printf '\nClickHouse:\n'
|
||||
docker compose exec -T clickhouse clickhouse-client --user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" --query 'SELECT version(), currentUser()' 2>/dev/null || true
|
||||
printf '\nRecent errors:\n'
|
||||
docker compose logs --since=10m garage clickhouse clickhouse-schema postgres postgres-schema redpanda ingress processor detector api archive-uploader 2>&1 \
|
||||
| grep -Ei 'error|fatal|failed|panic|exception' | tail -80 || true
|
||||
|
||||
Reference in New Issue
Block a user