From 9e2f48ceaee91a61974f7b33471d30d4f8c316e2 Mon Sep 17 00:00:00 2001 From: jbergner Date: Thu, 23 Jul 2026 17:33:00 +0200 Subject: [PATCH] Bugfix 2.0 --- .dockerignore | 4 + .env | 144 +++++++++-------- .env.example | 85 ++++++++++ .gitignore | 4 + DEPLOYMENT-FIXES.md | 42 +++++ README.md | 73 ++++++++- RELEASE-CHECKS.md | 33 ++-- VERSION | 1 + compose.yml | 231 ++++++++++++++++++++-------- credentials.sh | 30 ++++ deploy.ps1 | 153 +++++++++++------- deploy.sh | 190 ++++++++++++++--------- deploy/clickhouse/init.sql | 82 ---------- deploy/clickhouse/init.sql.template | 10 +- deploy/garage/garage.toml.template | 17 -- doctor.sh | 48 ++++-- go.mod | 10 -- go.sum | 40 ----- internal/api/server.go | 4 +- internal/clickhouse/client.go | 11 +- internal/config/config.go | 4 +- internal/detector/detector.go | 28 ++-- reset-DANGEROUS.sh | 3 +- status.sh | 14 +- 24 files changed, 796 insertions(+), 465 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 DEPLOYMENT-FIXES.md create mode 100644 VERSION create mode 100644 credentials.sh delete mode 100644 deploy/clickhouse/init.sql delete mode 100644 deploy/garage/garage.toml.template delete mode 100644 go.sum diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6c510b7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +.env +spool +*.zip diff --git a/.env b/.env index 62b57b8..2180051 100644 --- a/.env +++ b/.env @@ -1,77 +1,85 @@ -TZ=UTC +# ============================================================ +# Greenfield SIEM - complete configuration +# deploy.sh / deploy.ps1 creates .env and replaces CHANGE_ME. +# ============================================================ -LISTEN_ADDR=:8080 -DB_DSN=eventuser:DEINPASSWORT@tcp(mariadb:3306)/eventcollector?parseTime=true&charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&loc=UTC +# Public SIEM endpoints +INGRESS_BIND=0.0.0.0 +INGRESS_PORT=8090 +UI_BIND=0.0.0.0 +UI_PORT=8080 +TENANT_ID=default +UI_USERNAME=admin +UI_PASSWORD=CHANGE_ME +ENROLLMENT_KEY=CHANGE_ME -DB_MAX_OPEN_CONNS=50 -DB_MAX_IDLE_CONNS=25 -DB_CONN_MAX_LIFETIME=3m -DB_CONN_MAX_IDLE_TIME=1m +# Ingress limits +MAX_BODY_BYTES=8388608 +MAX_BATCH_EVENTS=1000 -MAX_BODY_BYTES=10485760 -HTTP_READ_TIMEOUT=15s -HTTP_WRITE_TIMEOUT=30s -HTTP_IDLE_TIMEOUT=60s +# PostgreSQL control plane +POSTGRES_IMAGE=postgres:18.4-alpine +POSTGRES_DB=siem +POSTGRES_USER=siem +POSTGRES_PASSWORD=CHANGE_ME +POSTGRES_BIND=127.0.0.1 +POSTGRES_PORT=5432 -DETECTION_INTERVAL=1m -OFFLINE_AFTER=10m -OFFLINE_ALERT_MAX=120m -FAILED_LOGON_WINDOW=5m -FAILED_LOGON_THRESHOLD=25 -REBOOT_WINDOW=15m -REBOOT_THRESHOLD=3 -PASSWORD_SPRAY_WINDOW=5m -PASSWORD_SPRAY_MIN_USERS=5 -PASSWORD_SPRAY_MIN_ATTEMPTS=15 -SUCCESS_AFTER_FAILURE_WINDOW=10m -NEW_SOURCE_IP_LOOKBACK=720h -NEW_SOURCE_IP_WINDOW=10m -DETECTIONS_LIMIT=100 +# ClickHouse event store +CLICKHOUSE_IMAGE=clickhouse/clickhouse-server:26.6.2.81 +CLICKHOUSE_DB=siem +CLICKHOUSE_USER=siem +CLICKHOUSE_PASSWORD=CHANGE_ME +CLICKHOUSE_HTTP_BIND=127.0.0.1 +CLICKHOUSE_HTTP_PORT=8123 +CLICKHOUSE_NATIVE_BIND=127.0.0.1 +CLICKHOUSE_NATIVE_PORT=9000 +EVENT_RETENTION_DAYS=90 +ROLLUP_RETENTION_DAYS=730 -MARIADB_DATABASE=eventcollector -MARIADB_USER=eventuser -MARIADB_PASSWORD=DEINPASSWORT -MARIADB_ROOT_PASSWORD=ROOTPASSWORT +# Redpanda durable ingest queue +REDPANDA_IMAGE=docker.redpanda.com/redpandadata/redpanda:v26.1.13 +REDPANDA_CONSOLE_IMAGE=docker.redpanda.com/redpandadata/console:v3.8.0 +REDPANDA_BIND=127.0.0.1 +REDPANDA_KAFKA_PORT=19092 +REDPANDA_ADMIN_PORT=19644 +REDPANDA_CONSOLE_BIND=127.0.0.1 +REDPANDA_CONSOLE_PORT=8081 +REDPANDA_SMP=2 +REDPANDA_LOG_LEVEL=info +KAFKA_TOPIC=siem-events +KAFKA_GROUP=siem-processor-v1 +KAFKA_PARTITIONS=6 +KAFKA_RETENTION_MS=86400000 -GRAFANA_ADMIN_USER=admin -GRAFANA_ADMIN_PASSWORD=admin +# Garage S3-compatible raw archive +GARAGE_IMAGE=dxflrs/garage:v2.3.0 +ALPINE_IMAGE=alpine:3.22 +GARAGE_BIND=127.0.0.1 +GARAGE_S3_PORT=3900 +GARAGE_ADMIN_PORT=3903 +GARAGE_BUCKET=siem-raw +GARAGE_ACCESS_KEY=CHANGE_ME +GARAGE_SECRET_KEY=CHANGE_ME +GARAGE_RPC_SECRET=CHANGE_ME +GARAGE_ADMIN_TOKEN=CHANGE_ME +GARAGE_METRICS_TOKEN=CHANGE_ME +GARAGE_LOG_LEVEL=garage=info +RAW_ARCHIVE_ENABLED=true +RAW_RETENTION=720h +ARCHIVE_UPLOAD_INTERVAL_SECONDS=30 -ENROLLMENT_KEY=BITTE_SEHR_LANG_UND_ZUFAELLIG +# Detector / API +DETECTOR_INTERVAL=30s +DETECTOR_LOOKBACK=20m +UI_QUERY_LIMIT=500 -#Woche 1 -BASELINE_WINDOW=15m -BASELINE_MIN_SAMPLES=72 -BASELINE_MIN_COUNT=30 -BASELINE_MEDIUM_Z=3.5 -BASELINE_HIGH_Z=6.0 -BASELINE_SUPPRESS_FOR=6h +# Prometheus +PROMETHEUS_IMAGE=prom/prometheus:v3.13.1 +PROMETHEUS_BIND=127.0.0.1 +PROMETHEUS_PORT=9090 +PROMETHEUS_RETENTION=30d -#Woche 2-4 -#BASELINE_WINDOW=15m -#BASELINE_MIN_SAMPLES=48 -#BASELINE_MIN_COUNT=20 -#BASELINE_MEDIUM_Z=3.0 -#BASELINE_HIGH_Z=5.0 -#BASELINE_SUPPRESS_FOR=4h - -#Dauerlauf Aggressiv -#BASELINE_WINDOW=10m -#BASELINE_MIN_SAMPLES=36 -#BASELINE_MIN_COUNT=15 -#BASELINE_MEDIUM_Z=2.8 -#BASELINE_HIGH_Z=4.5 -#BASELINE_SUPPRESS_FOR=2h - -#Dauerlauf Passiv -#BASELINE_WINDOW=15m -#BASELINE_MIN_SAMPLES=48 -#BASELINE_MIN_COUNT=20 -#BASELINE_MEDIUM_Z=3.0 -#BASELINE_HIGH_Z=5.0 -#BASELINE_SUPPRESS_FOR=4h - -PARTITION_MAINTENANCE_ENABLED=true -PARTITION_MAINTENANCE_INTERVAL=15m -PARTITION_INTERVAL=3h -PARTITION_AHEAD=24h -PARTITION_BEHIND=6h \ No newline at end of file +# Raw uploader and application image +RCLONE_IMAGE=rclone/rclone:1.74.4 +APP_IMAGE=greenfield-siem-app:local diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2180051 --- /dev/null +++ b/.env.example @@ -0,0 +1,85 @@ +# ============================================================ +# Greenfield SIEM - complete configuration +# deploy.sh / deploy.ps1 creates .env and replaces CHANGE_ME. +# ============================================================ + +# Public SIEM endpoints +INGRESS_BIND=0.0.0.0 +INGRESS_PORT=8090 +UI_BIND=0.0.0.0 +UI_PORT=8080 +TENANT_ID=default +UI_USERNAME=admin +UI_PASSWORD=CHANGE_ME +ENROLLMENT_KEY=CHANGE_ME + +# Ingress limits +MAX_BODY_BYTES=8388608 +MAX_BATCH_EVENTS=1000 + +# PostgreSQL control plane +POSTGRES_IMAGE=postgres:18.4-alpine +POSTGRES_DB=siem +POSTGRES_USER=siem +POSTGRES_PASSWORD=CHANGE_ME +POSTGRES_BIND=127.0.0.1 +POSTGRES_PORT=5432 + +# ClickHouse event store +CLICKHOUSE_IMAGE=clickhouse/clickhouse-server:26.6.2.81 +CLICKHOUSE_DB=siem +CLICKHOUSE_USER=siem +CLICKHOUSE_PASSWORD=CHANGE_ME +CLICKHOUSE_HTTP_BIND=127.0.0.1 +CLICKHOUSE_HTTP_PORT=8123 +CLICKHOUSE_NATIVE_BIND=127.0.0.1 +CLICKHOUSE_NATIVE_PORT=9000 +EVENT_RETENTION_DAYS=90 +ROLLUP_RETENTION_DAYS=730 + +# Redpanda durable ingest queue +REDPANDA_IMAGE=docker.redpanda.com/redpandadata/redpanda:v26.1.13 +REDPANDA_CONSOLE_IMAGE=docker.redpanda.com/redpandadata/console:v3.8.0 +REDPANDA_BIND=127.0.0.1 +REDPANDA_KAFKA_PORT=19092 +REDPANDA_ADMIN_PORT=19644 +REDPANDA_CONSOLE_BIND=127.0.0.1 +REDPANDA_CONSOLE_PORT=8081 +REDPANDA_SMP=2 +REDPANDA_LOG_LEVEL=info +KAFKA_TOPIC=siem-events +KAFKA_GROUP=siem-processor-v1 +KAFKA_PARTITIONS=6 +KAFKA_RETENTION_MS=86400000 + +# Garage S3-compatible raw archive +GARAGE_IMAGE=dxflrs/garage:v2.3.0 +ALPINE_IMAGE=alpine:3.22 +GARAGE_BIND=127.0.0.1 +GARAGE_S3_PORT=3900 +GARAGE_ADMIN_PORT=3903 +GARAGE_BUCKET=siem-raw +GARAGE_ACCESS_KEY=CHANGE_ME +GARAGE_SECRET_KEY=CHANGE_ME +GARAGE_RPC_SECRET=CHANGE_ME +GARAGE_ADMIN_TOKEN=CHANGE_ME +GARAGE_METRICS_TOKEN=CHANGE_ME +GARAGE_LOG_LEVEL=garage=info +RAW_ARCHIVE_ENABLED=true +RAW_RETENTION=720h +ARCHIVE_UPLOAD_INTERVAL_SECONDS=30 + +# Detector / API +DETECTOR_INTERVAL=30s +DETECTOR_LOOKBACK=20m +UI_QUERY_LIMIT=500 + +# Prometheus +PROMETHEUS_IMAGE=prom/prometheus:v3.13.1 +PROMETHEUS_BIND=127.0.0.1 +PROMETHEUS_PORT=9090 +PROMETHEUS_RETENTION=30d + +# Raw uploader and application image +RCLONE_IMAGE=rclone/rclone:1.74.4 +APP_IMAGE=greenfield-siem-app:local diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..819623a --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +deploy/garage/garage.toml +spool/ +*.zip diff --git a/DEPLOYMENT-FIXES.md b/DEPLOYMENT-FIXES.md new file mode 100644 index 0000000..aee9de2 --- /dev/null +++ b/DEPLOYMENT-FIXES.md @@ -0,0 +1,42 @@ +# Deployment-Fixes – Release 1.1 + +Stand: 2026-07-23 + +Diese Revision repariert die Startprobleme der ersten Greenfield-Ausgabe, ohne die Event-Architektur wieder zu verändern. + +## Behobene Ursachen + +1. **Garage-Konfiguration war nicht Bestandteil des eigentlichen Compose-Lebenszyklus.** + Die alte Ausgabe erwartete eine zuvor auf dem Host erzeugte `deploy/garage/garage.toml`. Jetzt erzeugt der einmalige Service `garage-config` die Konfiguration in einem Docker-Volume. `garage` startet erst nach erfolgreicher Erzeugung. + +2. **Garage Single-Node RPC-Adresse war ungeeignet.** + `rpc_public_addr` ist jetzt `127.0.0.1:3901`; der Server bindet intern weiterhin auf `0.0.0.0:3901`. + +3. **ClickHouse hatte nur HTTP 8123 exponiert.** + Zusätzlich ist jetzt der native Client-Port 9000 konfigurierbar und standardmäßig nur auf `127.0.0.1` gebunden. + +4. **`.env.example` war unvollständig.** + Sie enthält jetzt sämtliche 63 von Compose verwendeten Laufzeitparameter. `deploy.sh` und `deploy.ps1` ergänzen fehlende Schlüssel in einer bestehenden `.env`, ohne bestehende Secrets zu ersetzen. Eigene zusätzliche Schlüssel bleiben erhalten. + +5. **Schema-Setup hing vom ersten Containerstart ab.** + `postgres-schema` und `clickhouse-schema` laufen idempotent nach erfolgreicher Datenbank-Readiness. Damit wird ein teilweise abgebrochener Erststart nicht mehr stillschweigend als vollständig initialisiert behandelt. + +6. **ClickHouse-Datenbankname war im Go-Code teilweise fest auf `siem` verdrahtet.** + API, Detector und Inserts verwenden jetzt konsistent `CLICKHOUSE_DB`. + +7. **Deployment-Fehler waren zu wenig sichtbar.** + Der One-Click-Start validiert zuerst `docker compose config`, prüft nach dem Start API, Ingress, ClickHouse und Garage und gibt bei Fehlschlag automatisch die relevanten Logs aus. + +## Sichere Reparatur einer vorhandenen Installation + +Die Daten-Volumes werden durch `./deploy.sh` **nicht** gelöscht. + +Am sichersten ist es, die bestehende `.env` in die korrigierte Projektversion zu übernehmen und danach `./deploy.sh` auszuführen. So bleiben die bereits verwendeten PostgreSQL-/ClickHouse-/Garage-Secrets erhalten. + +`./reset-DANGEROUS.sh` löscht dagegen die Docker-Volumes und darf bei erhaltenswerten Daten nicht verwendet werden. + +## Diagnose + +- `./credentials.sh` zeigt die tatsächlich verwendeten Zugangsdaten und Ports. +- `./doctor.sh` prüft Compose, API/Ingress, Garage, ClickHouse, Redpanda, PostgreSQL und den Raw-Spool. +- `docker compose logs --tail=200 garage garage-config clickhouse clickhouse-schema` zeigt die wichtigsten Startfehler. diff --git a/README.md b/README.md index 072dfb8..19628ea 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Greenfield SIEM +**Release 1.1 – Deployment-Fix:** Garage-Bootstrap, vollständige `.env`, ClickHouse-Native-Port und idempotentes ClickHouse-Schema wurden gegenüber der ersten Greenfield-Version korrigiert. + Kompletter Neuaufbau des bisherigen Projekts. Vom Altprojekt bleibt absichtlich nur der HTTP-Ingress-Vertrag erhalten. ## Was dieses Projekt löst @@ -45,7 +47,7 @@ chmod +x deploy.sh ./deploy.ps1 ``` -Das Deployment erzeugt beim ersten Start kryptographisch zufällige Secrets in `.env`, erzeugt die Garage-Konfiguration, baut die SIEM-Binaries und startet alle Dienste. +Das Deployment vervollständigt bei jedem Start die `.env`, ohne vorhandene Secrets zu überschreiben. Fehlende Secrets werden kryptographisch zufällig erzeugt. Die Garage-Konfiguration wird anschließend automatisch von einem Compose-Init-Service erzeugt; ClickHouse-Schema und TTLs werden bei jedem Start idempotent geprüft. Danach werden die SIEM-Binaries gebaut und alle Dienste gestartet. Danach: @@ -56,10 +58,72 @@ Danach: Beim ersten Start werden `UI_USERNAME=admin` und ein zufälliges `UI_PASSWORD` erzeugt. Browser fragen diese Zugangsdaten beim Öffnen von `/ui` ab. +Alle konfigurierbaren Parameter stehen in `.env.example`. `deploy.sh`/`deploy.ps1` ergänzt fehlende Schlüssel automatisch in einer vorhandenen `.env`; damit werden auch unvollständige `.env`-Dateien aus der ersten Greenfield-Version repariert. Zugangsdaten lassen sich jederzeit mit `./credentials.sh` anzeigen. + UI-Benutzer/Passwort und Enrollment-Key werden am Ende des Deployments ausgegeben und stehen außerdem in der geschützten `.env`. Die UI/API ist standardmäßig per HTTP Basic Auth geschützt; für Zugriff über ein fremdes Netz gehört zusätzlich TLS davor. > Bestehende Collector können den bisherigen Enrollment-Key weiterverwenden, ohne eine `.env` von Hand anzulegen. Linux/macOS/WSL: `ENROLLMENT_KEY="ALTER_KEY" ./deploy.sh`. PowerShell: `$env:ENROLLMENT_KEY="ALTER_KEY"; ./deploy.ps1`. Alle anderen Secrets werden trotzdem zufällig erzeugt. + +## Upgrade / Reparatur der ersten Greenfield-Version + +Die erste ausgelieferte Greenfield-Version hatte drei Deployment-Probleme: + +1. `deploy/garage/garage.toml` wurde erst auf dem Host erzeugt und fehlte im ZIP; außerdem war `rpc_public_addr` für das Single-Node-Setup ungünstig gesetzt. +2. ClickHouse exponierte nur HTTP `8123`, nicht den nativen Client-Port `9000`. +3. `.env` enthielt nur einen Teil der tatsächlich konfigurierbaren Parameter. + +Mit dieser Version reicht im bestehenden Projektverzeichnis: + +```bash +./deploy.sh +./doctor.sh +``` + +Das Deploy-Skript behält bestehende Secrets, ergänzt fehlende `.env`-Werte und erzeugt Garage-Konfiguration sowie ClickHouse-Schema über Compose-Init-Services neu. **Die Daten-Volumes werden dabei nicht gelöscht.** + +Falls `.env` gelöscht wurde, aber alte PostgreSQL-Volumes mit einem unbekannten alten Passwort weiter existieren, kann dieses Passwort nicht automatisch rekonstruiert werden. In einer reinen Testinstallation ist dann `./reset-DANGEROUS.sh` der saubere Neustart; bei produktiven Daten zuerst Backup beziehungsweise Passwort-Recovery durchführen. + +## ClickHouse-Zugang + +ClickHouse besitzt zwei verschiedene Schnittstellen: + +- HTTP: `127.0.0.1:8123` +- nativer ClickHouse-Client: `127.0.0.1:9000` + +Benutzer, Passwort und Datenbank stehen vollständig in `.env`: + +```env +CLICKHOUSE_DB=siem +CLICKHOUSE_USER=siem +CLICKHOUSE_PASSWORD=... +CLICKHOUSE_HTTP_PORT=8123 +CLICKHOUSE_NATIVE_PORT=9000 +``` + +Am einfachsten: + +```bash +./credentials.sh +docker compose exec clickhouse clickhouse-client \ + --user "$(grep '^CLICKHOUSE_USER=' .env | cut -d= -f2-)" \ + --password "$(grep '^CLICKHOUSE_PASSWORD=' .env | cut -d= -f2-)" \ + --database "$(grep '^CLICKHOUSE_DB=' .env | cut -d= -f2-)" +``` + +Für DBeaver/DataGrip oder einen lokalen `clickhouse-client` den nativen Port `9000` verwenden. Port `8123` ist die HTTP-Schnittstelle, keine Login-Weboberfläche. + +## Garage-Zugang / Diagnose + +Garage wird im Single-Node-Modus automatisch konfiguriert. Die Konfiguration liegt in einem Docker-Volume und muss nicht mehr als Host-Datei erzeugt werden. Prüfung: + +```bash +docker compose exec garage /garage status +docker compose logs --tail=100 garage garage-config +``` + +S3 läuft lokal auf `127.0.0.1:3900`. Access-Key, Secret-Key und Bucket stehen in `.env`. + ## Bestehender Ingress bleibt kompatibel ```http @@ -253,8 +317,9 @@ ClickHouse Zeilenzahl / Speicher: ```bash docker compose exec clickhouse clickhouse-client \ - --user siem --password "$(grep '^CLICKHOUSE_PASSWORD=' .env | cut -d= -f2-)" \ - --query "SELECT formatReadableQuantity(sum(rows)), formatReadableSize(sum(bytes_on_disk)) FROM system.parts WHERE database='siem' AND table='events' AND active" + --user "$(grep '^CLICKHOUSE_USER=' .env | cut -d= -f2-)" \ + --password "$(grep '^CLICKHOUSE_PASSWORD=' .env | cut -d= -f2-)" \ + --query "SELECT formatReadableQuantity(sum(rows)), formatReadableSize(sum(bytes_on_disk)) FROM system.parts WHERE database='$(grep '^CLICKHOUSE_DB=' .env | cut -d= -f2-)' AND table='events' AND active" ``` Queue: @@ -320,7 +385,7 @@ doctor.sh Pipeline-/Kapazitätsdiagnose raw-event.sh Raw-Archiv abrufen deploy/clickhouse analytisches Schema deploy/postgres Control-Plane-Schema -deploy/garage S3-Archivkonfiguration +compose.yml erzeugt Garage-Konfiguration zur Laufzeit aus `.env` deploy/prometheus Betriebsmetriken ``` diff --git a/RELEASE-CHECKS.md b/RELEASE-CHECKS.md index 245e283..3c98cc5 100644 --- a/RELEASE-CHECKS.md +++ b/RELEASE-CHECKS.md @@ -1,24 +1,39 @@ # Release checks -Stand: 2026-07-23 +Stand: 2026-07-23 – Deployment-Fix 1.1 Durchgeführt: +- `compose.yml` mit YAML-Parser geladen: 15 Services, erfolgreich. +- Alle 63 nicht-escaped Compose-Variablen mit `.env.example` abgeglichen: **0 fehlende, 0 überzählige**. +- Sämtliche Host-Bind-Mount-Quellen aus `compose.yml` auf Existenz geprüft: **0 fehlende Pfade**. +- Prometheus-YAML geparst: erfolgreich. +- ClickHouse-Prometheus-XML geparst: erfolgreich. +- Garage-Konfigurationskommando aus Compose isoliert ausgeführt und resultierendes TOML mit `tomllib` geparst: erfolgreich. +- ClickHouse-Schema-Rendering aus Compose isoliert ausgeführt: Datenbankname und TTL-Platzhalter vollständig ersetzt; Schema-/TTL-Kommandos wurden aufgerufen. +- `deploy.sh` mit simuliertem Docker/HTTP end-to-end ausgeführt: erfolgreich. + - frische `.env`: 63 Schlüssel, keine `CHANGE_ME`-Secrets; + - externer bestehender Enrollment-Key bleibt erhalten; + - partielle Alt-`.env` wird vollständig ergänzt; + - vorhandene UI-/PostgreSQL-/ClickHouse-/Enrollment-Secrets bleiben unverändert; + - eigener unbekannter `.env`-Schlüssel bleibt erhalten. +- POSIX-Shellskripte mit `sh -n` geprüft: erfolgreich. - `go test ./...` gegen lokale Interface-Stubs für pgx/kafka-go mit der vorhandenen Go-1.23.2-Toolchain: erfolgreich. - `go vet ./...` unter denselben isolierten Bedingungen: erfolgreich. - Ingress-Tests: Batch-UID/Retry-Idempotenz und Validierung: erfolgreich. - Normalizer-Tests: Windows-Metadaten/XML und Lockout-Workstation: erfolgreich. -- `compose.yml` und Prometheus-YAML syntaktisch geparst: erfolgreich. -- ClickHouse-Prometheus-XML geparst: erfolgreich. -- POSIX-Shellskripte mit `sh -n` geprüft: erfolgreich. -- `deploy.sh` mit simuliertem Docker/Healthcheck end-to-end ausgeführt: Secret-Erzeugung, bestehender Enrollment-Key, Template-Rendering und Compose-Aufrufe erfolgreich. -- ZIP-Integrität mit `unzip -t`: erfolgreich. + +Zusätzlich gegen aktuelle Primärquellen geprüft: + +- Garage v2.3 Single-Node/Default-Bucket-Start und die erwarteten Default-Credential-Umgebungsvariablen. +- ClickHouse Docker: HTTP-Port 8123, nativer Client-Port 9000 sowie `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD` und Access-Management-Initialisierung. +- Redpanda Single-Broker Docker-Beispiel und aktuelle Broker-/Console-Versionen. Nicht in dieser Ausführungsumgebung möglich: -- echter Docker-Compose-Start der Datenbanken/Broker/Object-Storage-Komponenten; +- echter Docker-Compose-Start der Datenbanken/Broker/Object-Storage-Komponenten, da hier kein Docker-Daemon/CLI verfügbar ist; - Integrationstest gegen reale ClickHouse-, PostgreSQL-, Redpanda- und Garage-Container; - Ausführung von `deploy.ps1`, da PowerShell hier nicht installiert ist; -- Build mit Go 1.26.5, da die lokale Umgebung keinen Netzwerkzugriff für die Toolchain/Module besitzt. Das Dockerfile verwendet Go 1.26.5. +- regulärer Build mit den realen externen Go-Modulen; die lokale Prüfung verwendete Schnittstellen-Stubs, da die vorhandene lokale Go-Version 1.23.2 ist, während das Dockerfile Go 1.26.5 verwendet. -Der erste reale Deployment-Lauf sollte daher anschließend mit `./doctor.sh` und einem begrenzten Stress-Test geprüft werden. +Der erste reale Lauf sollte deshalb direkt mit `./doctor.sh` verifiziert werden. Bei einem Startfehler gibt `deploy.sh` bereits automatisch die relevanten Service-Logs aus. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..5aa3ff1 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.1.0-deployment-fix diff --git a/compose.yml b/compose.yml index 4518f8c..34bcc28 100644 --- a/compose.yml +++ b/compose.yml @@ -2,50 +2,97 @@ name: greenfield-siem services: postgres: - image: postgres:18.4-alpine + image: ${POSTGRES_IMAGE:-postgres:18.4-alpine} environment: - POSTGRES_DB: siem - POSTGRES_USER: siem + POSTGRES_DB: ${POSTGRES_DB:-siem} + POSTGRES_USER: ${POSTGRES_USER:-siem} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} volumes: - postgres_data:/var/lib/postgresql - ./deploy/postgres/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro healthcheck: - test: ["CMD-SHELL", "pg_isready -U siem -d siem"] + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] interval: 5s timeout: 3s retries: 30 + start_period: 5s restart: unless-stopped ports: - - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" + - "${POSTGRES_BIND:-127.0.0.1}:${POSTGRES_PORT:-5432}:5432" + + postgres-schema: + image: ${POSTGRES_IMAGE:-postgres:18.4-alpine} + depends_on: + postgres: + condition: service_healthy + environment: + PGHOST: postgres + PGPORT: "5432" + PGDATABASE: ${POSTGRES_DB:-siem} + PGUSER: ${POSTGRES_USER:-siem} + PGPASSWORD: ${POSTGRES_PASSWORD} + volumes: + - ./deploy/postgres/init.sql:/schema/init.sql:ro + entrypoint: ["/bin/sh", "-ec"] + command: ["psql -v ON_ERROR_STOP=1 -f /schema/init.sql"] + restart: "no" clickhouse: - image: clickhouse/clickhouse-server:26.6.2.81 + image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81} environment: - CLICKHOUSE_DB: siem - CLICKHOUSE_USER: siem + CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem} + CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem} CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" volumes: - clickhouse_data:/var/lib/clickhouse - clickhouse_logs:/var/log/clickhouse-server - - ./deploy/clickhouse/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro - ./deploy/clickhouse/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro ulimits: nofile: soft: 262144 hard: 262144 healthcheck: - test: ["CMD-SHELL", "clickhouse-client --user siem --password $$CLICKHOUSE_PASSWORD --query 'SELECT 1' >/dev/null"] + test: ["CMD-SHELL", "clickhouse-client --host 127.0.0.1 --user $${CLICKHOUSE_USER} --password $${CLICKHOUSE_PASSWORD} --query 'SELECT 1' >/dev/null"] interval: 5s timeout: 5s - retries: 30 + retries: 40 + start_period: 10s restart: unless-stopped ports: - - "127.0.0.1:${CLICKHOUSE_HTTP_PORT:-8123}:8123" + - "${CLICKHOUSE_HTTP_BIND:-127.0.0.1}:${CLICKHOUSE_HTTP_PORT:-8123}:8123" + - "${CLICKHOUSE_NATIVE_BIND:-127.0.0.1}:${CLICKHOUSE_NATIVE_PORT:-9000}:9000" + + clickhouse-schema: + image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81} + depends_on: + clickhouse: + condition: service_healthy + entrypoint: ["/bin/bash", "-ec"] + environment: + CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem} + CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + EVENT_RETENTION_DAYS: ${EVENT_RETENTION_DAYS:-90} + ROLLUP_RETENTION_DAYS: ${ROLLUP_RETENTION_DAYS:-730} + volumes: + - ./deploy/clickhouse/init.sql.template:/schema/init.sql.template:ro + command: + - | + sed \ + -e "s/__CLICKHOUSE_DB__/$${CLICKHOUSE_DB}/g" \ + -e "s/__EVENT_RETENTION_DAYS__/$${EVENT_RETENTION_DAYS}/g" \ + -e "s/__ROLLUP_RETENTION_DAYS__/$${ROLLUP_RETENTION_DAYS}/g" \ + /schema/init.sql.template > /tmp/init.sql + clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --multiquery < /tmp/init.sql + clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \ + "ALTER TABLE $${CLICKHOUSE_DB}.events MODIFY TTL event_time + INTERVAL $${EVENT_RETENTION_DAYS} DAY DELETE" + clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \ + "ALTER TABLE $${CLICKHOUSE_DB}.events_5m MODIFY TTL bucket + INTERVAL $${ROLLUP_RETENTION_DAYS} DAY DELETE" + restart: "no" redpanda: - image: docker.redpanda.com/redpandadata/redpanda:v26.1.13 + image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13} command: - redpanda - start @@ -60,14 +107,14 @@ services: - --schema-registry-addr - internal://0.0.0.0:8081,external://0.0.0.0:18081 - --rpc-addr - - redpanda:33145 + - 0.0.0.0:33145 - --advertise-rpc-addr - redpanda:33145 - --mode - dev-container - --smp - - "2" - - --default-log-level=info + - "${REDPANDA_SMP:-2}" + - --default-log-level=${REDPANDA_LOG_LEVEL:-info} volumes: - redpanda_data:/var/lib/redpanda/data healthcheck: @@ -75,32 +122,36 @@ services: interval: 5s timeout: 5s retries: 40 + start_period: 10s restart: unless-stopped ports: - - "127.0.0.1:${REDPANDA_KAFKA_PORT:-19092}:19092" - - "127.0.0.1:${REDPANDA_ADMIN_PORT:-19644}:9644" + - "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_KAFKA_PORT:-19092}:19092" + - "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_ADMIN_PORT:-19644}:9644" redpanda-init: - image: docker.redpanda.com/redpandadata/redpanda:v26.1.13 + image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13} depends_on: redpanda: condition: service_healthy entrypoint: ["/bin/sh", "-ec"] environment: RPK_BROKERS: redpanda:9092 - command: >- - rpk topic create --if-not-exists -p ${KAFKA_PARTITIONS:-6} -r 1 - -c retention.ms=${KAFKA_RETENTION_MS:-86400000} - siem-events + KAFKA_PARTITIONS: ${KAFKA_PARTITIONS:-6} + KAFKA_RETENTION_MS: ${KAFKA_RETENTION_MS:-86400000} + KAFKA_TOPIC: ${KAFKA_TOPIC:-siem-events} + command: + - | + rpk topic create --if-not-exists -p "$${KAFKA_PARTITIONS}" -r 1 \ + -c retention.ms="$${KAFKA_RETENTION_MS}" "$${KAFKA_TOPIC}" restart: "no" redpanda-console: - image: docker.redpanda.com/redpandadata/console:v3.8.0 + image: ${REDPANDA_CONSOLE_IMAGE:-docker.redpanda.com/redpandadata/console:v3.8.0} depends_on: redpanda: condition: service_healthy entrypoint: /bin/sh - command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console' + command: -c 'printf "%s" "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console' environment: CONFIG_FILEPATH: /tmp/config.yml CONSOLE_CONFIG_FILE: | @@ -115,46 +166,93 @@ services: urls: ["http://redpanda:9644"] restart: unless-stopped ports: - - "127.0.0.1:${REDPANDA_CONSOLE_PORT:-8081}:8080" + - "${REDPANDA_CONSOLE_BIND:-127.0.0.1}:${REDPANDA_CONSOLE_PORT:-8081}:8080" + + garage-config: + image: ${ALPINE_IMAGE:-alpine:3.22} + environment: + GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET} + GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN} + GARAGE_METRICS_TOKEN: ${GARAGE_METRICS_TOKEN} + volumes: + - garage_config:/config + entrypoint: ["/bin/sh", "-ec"] + command: + - | + umask 077 + cat > /config/garage.toml <&2; exit 1; } +set -a; . ./.env; set +a +cat < $null + if ($LASTEXITCODE -eq 0) { $chOk = $true } + docker compose exec -T garage /garage status *> $null + if ($LASTEXITCODE -eq 0) { $garageOk = $true } + if ($apiOk -and $ingressOk -and $chOk -and $garageOk) { $healthy = $true; break } + Start-Sleep -Seconds 2 } -if (-not $healthy) { throw "Deployment gestartet, Healthcheck noch nicht gruen. Pruefe: docker compose ps; docker compose logs" } + +if (-not $healthy) { + Write-Host "Mindestens ein Dienst wurde nicht bereit. Diagnose:" -ForegroundColor Red + docker compose ps + docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor api + throw "Deployment nicht vollstaendig bereit." +} + Write-Host "" Write-Host "Greenfield SIEM laeuft." -Write-Host "UI: http://127.0.0.1:$($envMap.UI_PORT)/ui" -Write-Host "Ingress: http://127.0.0.1:$($envMap.INGRESS_PORT)/ingest" -Write-Host "UI-Login: $($envMap.UI_USERNAME) / $($envMap.UI_PASSWORD)" -Write-Host "Enrollment-Key: $($envMap.ENROLLMENT_KEY)" -Write-Host "Secrets stehen in .env (Datei schuetzen und sichern)." +Write-Host "UI: http://127.0.0.1:$($current.UI_PORT)/ui" +Write-Host "Ingress: http://127.0.0.1:$($current.INGRESS_PORT)/ingest" +Write-Host "UI-Login: $($current.UI_USERNAME) / $($current.UI_PASSWORD)" +Write-Host "ClickHouse: user=$($current.CLICKHOUSE_USER) db=$($current.CLICKHOUSE_DB) http=127.0.0.1:$($current.CLICKHOUSE_HTTP_PORT) native=127.0.0.1:$($current.CLICKHOUSE_NATIVE_PORT)" +Write-Host "Enrollment-Key: $($current.ENROLLMENT_KEY)" +Write-Host "Alle Parameter stehen vollstaendig in .env." diff --git a/deploy.sh b/deploy.sh index 46f16b7..9050552 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,88 +1,140 @@ #!/usr/bin/env sh set -eu cd "$(dirname "$0")" + command -v docker >/dev/null 2>&1 || { echo "Docker fehlt." >&2; exit 1; } docker compose version >/dev/null 2>&1 || { echo "Docker Compose Plugin fehlt." >&2; exit 1; } PRESET_ENROLLMENT_KEY="${ENROLLMENT_KEY:-}" -randhex(){ n="$1"; if command -v openssl >/dev/null 2>&1; then openssl rand -hex "$n"; else dd if=/dev/urandom bs="$n" count=1 2>/dev/null | od -An -tx1 | tr -d ' \n'; fi; } -http_ok(){ - url="$1" - if command -v curl >/dev/null 2>&1; then curl -fsS "$url" >/dev/null 2>&1; return $?; fi - if command -v wget >/dev/null 2>&1; then wget -q -O /dev/null "$url" >/dev/null 2>&1; return $?; fi - return 2 + +randhex() { + n="$1" + if command -v openssl >/dev/null 2>&1; then + openssl rand -hex "$n" + else + dd if=/dev/urandom bs="$n" count=1 2>/dev/null | od -An -tx1 | tr -d ' \n' + fi } -if [ ! -f .env ]; then - POSTGRES_PASSWORD="$(randhex 24)" - CLICKHOUSE_PASSWORD="$(randhex 24)" - ENROLLMENT_KEY="${PRESET_ENROLLMENT_KEY:-$(randhex 32)}" - UI_PASSWORD="$(randhex 18)" - GARAGE_ACCESS_KEY="GK$(randhex 16)" - GARAGE_SECRET_KEY="$(randhex 32)" - GARAGE_RPC_SECRET="$(randhex 32)" - GARAGE_ADMIN_TOKEN="$(randhex 32)" - GARAGE_METRICS_TOKEN="$(randhex 32)" - cat > .env </dev/null || true + +get_env() { + key="$1" + awk -F= -v k="$key" '$1==k {sub(/^[^=]*=/, ""); print; exit}' .env 2>/dev/null || true +} + +set_env() { + key="$1"; value="$2"; tmp=".env.tmp.$$" + awk -v k="$key" -v v="$value" ' + BEGIN {done=0} + index($0,k"=")==1 {print k"="v; done=1; next} + {print} + END {if(!done) print k"="v} + ' .env > "$tmp" + mv "$tmp" .env +} + +# Start from a complete template, but preserve all values from an existing .env. +[ -f .env ] || : > .env +while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|'#'*) continue ;; + esac + key=${line%%=*} + if ! grep -q "^${key}=" .env 2>/dev/null; then + printf '%s\n' "$line" >> .env + fi +done < .env.example + +# Generate secrets only if absent/blank/placeholder. Existing installations keep their credentials. +for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD UI_PASSWORD GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do + val="$(get_env "$key")" + if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then + case "$key" in + POSTGRES_PASSWORD|CLICKHOUSE_PASSWORD) val="$(randhex 24)" ;; + UI_PASSWORD) val="$(randhex 18)" ;; + *) val="$(randhex 32)" ;; + esac + set_env "$key" "$val" + fi +done + +val="$(get_env GARAGE_ACCESS_KEY)" +if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then + set_env GARAGE_ACCESS_KEY "GK$(randhex 16)" fi -set -a; . ./.env; set +a -UI_USERNAME="${UI_USERNAME:-admin}" -if [ -z "${UI_PASSWORD:-}" ]; then - UI_PASSWORD="$(randhex 18)" - printf '\nUI_USERNAME=%s\nUI_PASSWORD=%s\n' "$UI_USERNAME" "$UI_PASSWORD" >> .env - export UI_USERNAME UI_PASSWORD + +val="$(get_env ENROLLMENT_KEY)" +if [ -n "$PRESET_ENROLLMENT_KEY" ]; then + set_env ENROLLMENT_KEY "$PRESET_ENROLLMENT_KEY" +elif [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then + set_env ENROLLMENT_KEY "$(randhex 32)" fi -sed -e "s/__GARAGE_RPC_SECRET__/$GARAGE_RPC_SECRET/g" -e "s/__GARAGE_ADMIN_TOKEN__/$GARAGE_ADMIN_TOKEN/g" -e "s/__GARAGE_METRICS_TOKEN__/$GARAGE_METRICS_TOKEN/g" deploy/garage/garage.toml.template > deploy/garage/garage.toml -sed -e "s/__EVENT_RETENTION_DAYS__/${EVENT_RETENTION_DAYS:-90}/g" -e "s/__ROLLUP_RETENTION_DAYS__/${ROLLUP_RETENTION_DAYS:-730}/g" deploy/clickhouse/init.sql.template > deploy/clickhouse/init.sql -chmod 600 deploy/garage/garage.toml 2>/dev/null || true + +chmod 600 .env 2>/dev/null || true + +# Validate required settings without sourcing .env as shell code. Docker Compose parses +# .env itself; this also keeps values with shell-special characters safe. +for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD UI_PASSWORD ENROLLMENT_KEY GARAGE_ACCESS_KEY GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do + val="$(get_env "$key")" + if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then + echo "Ungültige .env: $key fehlt oder ist noch CHANGE_ME." >&2 + exit 1 + fi +done + +UI_PORT_VALUE="$(get_env UI_PORT)"; [ -n "$UI_PORT_VALUE" ] || UI_PORT_VALUE=8080 +INGRESS_PORT_VALUE="$(get_env INGRESS_PORT)"; [ -n "$INGRESS_PORT_VALUE" ] || INGRESS_PORT_VALUE=8090 +CLICKHOUSE_USER_VALUE="$(get_env CLICKHOUSE_USER)"; [ -n "$CLICKHOUSE_USER_VALUE" ] || CLICKHOUSE_USER_VALUE=siem +CLICKHOUSE_DB_VALUE="$(get_env CLICKHOUSE_DB)"; [ -n "$CLICKHOUSE_DB_VALUE" ] || CLICKHOUSE_DB_VALUE=siem +CLICKHOUSE_PASSWORD_VALUE="$(get_env CLICKHOUSE_PASSWORD)" +CLICKHOUSE_HTTP_PORT_VALUE="$(get_env CLICKHOUSE_HTTP_PORT)"; [ -n "$CLICKHOUSE_HTTP_PORT_VALUE" ] || CLICKHOUSE_HTTP_PORT_VALUE=8123 +CLICKHOUSE_NATIVE_PORT_VALUE="$(get_env CLICKHOUSE_NATIVE_PORT)"; [ -n "$CLICKHOUSE_NATIVE_PORT_VALUE" ] || CLICKHOUSE_NATIVE_PORT_VALUE=9000 +UI_USERNAME_VALUE="$(get_env UI_USERNAME)"; [ -n "$UI_USERNAME_VALUE" ] || UI_USERNAME_VALUE=admin +UI_PASSWORD_VALUE="$(get_env UI_PASSWORD)" +ENROLLMENT_KEY_VALUE="$(get_env ENROLLMENT_KEY)" + +# Validate the final config before pulling/building anything. +echo "Prüfe Compose-Konfiguration ..." +docker compose config >/dev/null echo "Baue und starte Greenfield SIEM ..." docker compose build --pull docker compose up -d --remove-orphans -echo "Prüfe Dienste ..." -if command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; then - i=0 - while [ "$i" -lt 60 ]; do - if http_ok "http://127.0.0.1:${UI_PORT:-8080}/readyz" && http_ok "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz"; then break; fi - i=$((i+1)); sleep 2 - done - if [ "$i" -ge 60 ]; then echo "Deployment gestartet, Healthcheck noch nicht grün. Prüfe: docker compose ps && docker compose logs" >&2; exit 1; fi -else - echo "Hinweis: curl/wget fehlt; HTTP-Healthcheck wird übersprungen." - docker compose ps +http_ok() { + url="$1" + if command -v curl >/dev/null 2>&1; then curl -fsS "$url" >/dev/null 2>&1; return $?; fi + if command -v wget >/dev/null 2>&1; then wget -q -O /dev/null "$url" >/dev/null 2>&1; return $?; fi + return 2 +} + +echo "Prüfe Readiness ..." +i=0 +while [ "$i" -lt 90 ]; do + api_ok=0; ingress_ok=0; ch_ok=0; garage_ok=0 + http_ok "http://127.0.0.1:${UI_PORT_VALUE}/readyz" && api_ok=1 || true + http_ok "http://127.0.0.1:${INGRESS_PORT_VALUE}/readyz" && ingress_ok=1 || true + docker compose exec -T clickhouse clickhouse-client --user "${CLICKHOUSE_USER_VALUE}" --password "${CLICKHOUSE_PASSWORD_VALUE}" --query 'SELECT 1' >/dev/null 2>&1 && ch_ok=1 || true + docker compose exec -T garage /garage status >/dev/null 2>&1 && garage_ok=1 || true + if [ "$api_ok" -eq 1 ] && [ "$ingress_ok" -eq 1 ] && [ "$ch_ok" -eq 1 ] && [ "$garage_ok" -eq 1 ]; then + break + fi + i=$((i+1)) + sleep 2 +done + +if [ "$i" -ge 90 ]; then + echo "Mindestens ein Dienst wurde nicht bereit. Diagnose:" >&2 + docker compose ps >&2 || true + echo >&2 + docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor api >&2 || true + exit 1 fi echo echo "Greenfield SIEM läuft." -echo "UI: http://127.0.0.1:${UI_PORT:-8080}/ui" -echo "Ingress: http://127.0.0.1:${INGRESS_PORT:-8090}/ingest" -echo "UI-Login: $UI_USERNAME / $UI_PASSWORD" -echo "Enrollment-Key: $ENROLLMENT_KEY" -echo "Secrets stehen in .env (Datei schützen und sichern)." +echo "UI: http://127.0.0.1:${UI_PORT_VALUE}/ui" +echo "Ingress: http://127.0.0.1:${INGRESS_PORT_VALUE}/ingest" +echo "UI-Login: ${UI_USERNAME_VALUE} / ${UI_PASSWORD_VALUE}" +echo "ClickHouse: user=${CLICKHOUSE_USER_VALUE} db=${CLICKHOUSE_DB_VALUE} http=127.0.0.1:${CLICKHOUSE_HTTP_PORT_VALUE} native=127.0.0.1:${CLICKHOUSE_NATIVE_PORT_VALUE}" +echo "Enrollment-Key: ${ENROLLMENT_KEY_VALUE}" +echo "Alle Parameter stehen vollständig in .env." diff --git a/deploy/clickhouse/init.sql b/deploy/clickhouse/init.sql deleted file mode 100644 index ed67cc7..0000000 --- a/deploy/clickhouse/init.sql +++ /dev/null @@ -1,82 +0,0 @@ -CREATE DATABASE IF NOT EXISTS siem; - -CREATE TABLE IF NOT EXISTS siem.events -( - event_uid String, - queue_partition Int32, - queue_offset Int64, - tenant_id LowCardinality(String), - event_time DateTime64(3, 'UTC'), - ingest_time DateTime64(3, 'UTC'), - event_date Date MATERIALIZED toDate(event_time), - agent_id String, - host_name LowCardinality(String), - source_type LowCardinality(String), - channel LowCardinality(String), - provider LowCardinality(String), - event_code UInt32, - category LowCardinality(String), - action LowCardinality(String), - outcome LowCardinality(String), - severity UInt8, - user_name String, - user_domain LowCardinality(String), - subject_user String, - subject_domain LowCardinality(String), - target_user String, - target_domain LowCardinality(String), - source_ip String, - source_port UInt16, - destination_ip String, - destination_port UInt16, - workstation String, - logon_type LowCardinality(String), - authentication_package LowCardinality(String), - logon_process LowCardinality(String), - status_code LowCardinality(String), - sub_status_code LowCardinality(String), - failure_reason LowCardinality(String), - process_path String, - parent_process_path String, - command_line String CODEC(ZSTD(3)), - message String CODEC(ZSTD(3)), - attributes Map(String, String) CODEC(ZSTD(3)), - raw_object_key String, - raw_index UInt32, - payload_hash FixedString(64), - schema_version UInt16, - parser_version UInt16, - ingest_delay_ms Int64, - INDEX idx_host host_name TYPE set(10000) GRANULARITY 4, - INDEX idx_event_code event_code TYPE set(256) GRANULARITY 4, - INDEX idx_user user_name TYPE bloom_filter(0.01) GRANULARITY 4, - INDEX idx_target target_user TYPE bloom_filter(0.01) GRANULARITY 4, - INDEX idx_source_ip source_ip TYPE bloom_filter(0.01) GRANULARITY 4, - INDEX idx_process process_path TYPE bloom_filter(0.01) GRANULARITY 8 -) -ENGINE = ReplacingMergeTree(ingest_time) -PARTITION BY toYYYYMM(event_time) -ORDER BY (tenant_id, event_time, host_name, event_code, event_uid) -TTL event_time + INTERVAL 90 DAY DELETE -SETTINGS index_granularity = 8192; - -CREATE TABLE IF NOT EXISTS siem.events_5m -( - tenant_id LowCardinality(String), - bucket DateTime('UTC'), - host_name LowCardinality(String), - event_code UInt32, - category LowCardinality(String), - action LowCardinality(String), - outcome LowCardinality(String), - cnt_state AggregateFunction(uniqExact, String) -) -ENGINE = AggregatingMergeTree -PARTITION BY toYYYYMM(bucket) -ORDER BY (tenant_id, bucket, host_name, event_code, category, action, outcome) -TTL bucket + INTERVAL 730 DAY DELETE; - -CREATE MATERIALIZED VIEW IF NOT EXISTS siem.events_5m_mv TO siem.events_5m AS -SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state -FROM siem.events -GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome; diff --git a/deploy/clickhouse/init.sql.template b/deploy/clickhouse/init.sql.template index 433779b..0cc8890 100644 --- a/deploy/clickhouse/init.sql.template +++ b/deploy/clickhouse/init.sql.template @@ -1,6 +1,6 @@ -CREATE DATABASE IF NOT EXISTS siem; +CREATE DATABASE IF NOT EXISTS __CLICKHOUSE_DB__; -CREATE TABLE IF NOT EXISTS siem.events +CREATE TABLE IF NOT EXISTS __CLICKHOUSE_DB__.events ( event_uid String, queue_partition Int32, @@ -60,7 +60,7 @@ ORDER BY (tenant_id, event_time, host_name, event_code, event_uid) TTL event_time + INTERVAL __EVENT_RETENTION_DAYS__ DAY DELETE SETTINGS index_granularity = 8192; -CREATE TABLE IF NOT EXISTS siem.events_5m +CREATE TABLE IF NOT EXISTS __CLICKHOUSE_DB__.events_5m ( tenant_id LowCardinality(String), bucket DateTime('UTC'), @@ -76,7 +76,7 @@ PARTITION BY toYYYYMM(bucket) ORDER BY (tenant_id, bucket, host_name, event_code, category, action, outcome) TTL bucket + INTERVAL __ROLLUP_RETENTION_DAYS__ DAY DELETE; -CREATE MATERIALIZED VIEW IF NOT EXISTS siem.events_5m_mv TO siem.events_5m AS +CREATE MATERIALIZED VIEW IF NOT EXISTS __CLICKHOUSE_DB__.events_5m_mv TO __CLICKHOUSE_DB__.events_5m AS SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state -FROM siem.events +FROM __CLICKHOUSE_DB__.events GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome; diff --git a/deploy/garage/garage.toml.template b/deploy/garage/garage.toml.template deleted file mode 100644 index e1d1bbb..0000000 --- a/deploy/garage/garage.toml.template +++ /dev/null @@ -1,17 +0,0 @@ -metadata_dir = "/var/lib/garage/meta" -data_dir = "/var/lib/garage/data" -db_engine = "sqlite" -replication_factor = 1 -rpc_bind_addr = "[::]:3901" -rpc_public_addr = "garage:3901" -rpc_secret = "__GARAGE_RPC_SECRET__" - -[s3_api] -s3_region = "garage" -api_bind_addr = "[::]:3900" -root_domain = ".s3.garage.localhost" - -[admin] -api_bind_addr = "[::]:3903" -admin_token = "__GARAGE_ADMIN_TOKEN__" -metrics_token = "__GARAGE_METRICS_TOKEN__" diff --git a/doctor.sh b/doctor.sh index a78b2de..ca8e139 100755 --- a/doctor.sh +++ b/doctor.sh @@ -3,13 +3,15 @@ set -u cd "$(dirname "$0")" [ -f .env ] && { set -a; . ./.env; set +a; } -echo "== Docker ==" +section(){ printf '\n== %s ==\n' "$1"; } + +section "Compose config" +docker compose config >/dev/null 2>&1 && echo "OK" || echo "FEHLER: docker compose config" + +section "Docker" docker compose ps || true -echo "\n== Host filesystem ==" -df -h . 2>/dev/null || true - -echo "\n== HTTP readiness ==" +section "HTTP readiness" if command -v curl >/dev/null 2>&1; then printf "Ingress: "; curl -fsS "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz" || true; echo printf "API: "; curl -fsS "http://127.0.0.1:${UI_PORT:-8080}/readyz" || true; echo @@ -17,19 +19,31 @@ else echo "curl fehlt; HTTP-Prüfung übersprungen." fi -echo "\n== Redpanda ==" +section "Garage" +docker compose exec -T garage /garage status 2>/dev/null || true +printf "S3 archive: " +docker compose exec -T archive-uploader rclone lsd "garage:${GARAGE_BUCKET:-siem-raw}" 2>/dev/null && echo "OK" || echo "nicht erreichbar" + +section "ClickHouse" +docker compose exec -T clickhouse clickhouse-client \ + --user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" \ + --query "SELECT version(), currentUser(), currentDatabase()" 2>/dev/null || true +docker compose exec -T clickhouse clickhouse-client \ + --user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" \ + --database "${CLICKHOUSE_DB:-siem}" \ + --query "SELECT formatReadableQuantity(sum(rows)) AS rows, formatReadableSize(sum(bytes_on_disk)) AS disk, count() AS active_parts FROM system.parts WHERE database='${CLICKHOUSE_DB:-siem}' AND table='events' AND active" 2>/dev/null || true + +section "Redpanda" docker compose exec -T redpanda rpk cluster health 2>/dev/null || true -docker compose exec -T redpanda rpk group describe siem-processor-v1 2>/dev/null || true +docker compose exec -T redpanda rpk group describe "${KAFKA_GROUP:-siem-processor-v1}" 2>/dev/null || true -echo "\n== ClickHouse events ==" -docker compose exec -T clickhouse clickhouse-client --user siem --password "${CLICKHOUSE_PASSWORD:-}" --query "SELECT formatReadableQuantity(sum(rows)) AS rows, formatReadableSize(sum(bytes_on_disk)) AS disk, count() AS active_parts FROM system.parts WHERE database='siem' AND table='events' AND active" 2>/dev/null || true -docker compose exec -T clickhouse clickhouse-client --user siem --password "${CLICKHOUSE_PASSWORD:-}" --query "SELECT max(event_time) AS latest_event, max(ingest_time) AS latest_ingest, quantile(0.95)(abs(ingest_delay_ms))/1000 AS p95_ingest_delay_seconds FROM siem.events WHERE event_time >= now() - INTERVAL 1 DAY" 2>/dev/null || true +section "PostgreSQL control plane" +docker compose exec -T postgres psql -U "${POSTGRES_USER:-siem}" -d "${POSTGRES_DB:-siem}" -Atc \ + "SELECT 'agents='||count(*) FROM agents; SELECT 'open_detections='||count(*) FROM detections WHERE status IN ('open','investigating');" 2>/dev/null || true -echo "\n== PostgreSQL control plane ==" -docker compose exec -T postgres psql -U siem -d siem -Atc "SELECT 'agents='||count(*) FROM agents; SELECT 'open_detections='||count(*) FROM detections WHERE status IN ('open','investigating');" 2>/dev/null || true +section "Raw spool" +docker compose exec -T archive-uploader sh -c 'du -sh /spool 2>/dev/null || true; printf "files="; find /spool -type f 2>/dev/null | wc -l' 2>/dev/null || true -echo "\n== Raw spool ==" -docker compose exec -T archive-uploader sh -c 'du -sh /spool 2>/dev/null || true; find /spool -type f 2>/dev/null | wc -l' 2>/dev/null || true - -echo "\n== Recent errors ==" -docker compose logs --since=15m ingress processor detector api archive-uploader 2>&1 | grep -Ei 'error|fatal|failed|panic|not_ready' | tail -80 || true +section "Recent errors" +docker compose logs --since=15m garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor detector api archive-uploader 2>&1 \ + | grep -Ei 'error|fatal|failed|panic|not_ready|exception' | tail -120 || true diff --git a/go.mod b/go.mod index 9d68300..a326dd4 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,3 @@ require ( github.com/jackc/pgx/v5 v5.9.2 github.com/segmentio/kafka-go v0.4.51 ) - -require ( - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/text v0.29.0 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 54a764e..0000000 --- a/go.sum +++ /dev/null @@ -1,40 +0,0 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= -github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= -github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= -github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/segmentio/kafka-go v0.4.51 h1:JgDPPG75tC1rWIS2Me6MwcvXJ6f49UQ4HjAOef71Hno= -github.com/segmentio/kafka-go v0.4.51/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/server.go b/internal/api/server.go index b3cc9c1..d919cbc 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -82,7 +82,7 @@ func Run(ctx context.Context, cfg config.Config) error { func summary(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) { ctx, cancel := context.WithTimeout(r.Context(), 8*time.Second) defer cancel() - q := fmt.Sprintf(`SELECT uniqExactMerge(cnt_state) events_24h, uniqExact(host_name) active_hosts FROM siem.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL 24 HOUR`, clickhouse.Q(cfg.TenantID)) + q := fmt.Sprintf(`SELECT uniqExactMerge(cnt_state) events_24h, uniqExact(host_name) active_hosts FROM %s.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL 24 HOUR`, clickhouse.Ident(cfg.ClickHouseDB), clickhouse.Q(cfg.TenantID)) rows, e := ch.QueryJSON(ctx, q) if e != nil { j(w, 500, map[string]string{"error": e.Error()}) @@ -118,7 +118,7 @@ func events(w http.ResponseWriter, r *http.Request, cfg config.Config, ch *click where = append(where, "event_code="+v) } } - q := fmt.Sprintf(`SELECT event_uid,event_time,host_name,channel,event_code,category,action,outcome,severity,user_name,subject_user,target_user,source_ip,workstation,process_path,message,raw_object_key,raw_index FROM siem.events WHERE %s ORDER BY event_time DESC, ingest_time DESC LIMIT 1 BY event_uid LIMIT %d`, strings.Join(where, " AND "), limit) + q := fmt.Sprintf(`SELECT event_uid,event_time,host_name,channel,event_code,category,action,outcome,severity,user_name,subject_user,target_user,source_ip,workstation,process_path,message,raw_object_key,raw_index FROM %s.events WHERE %s ORDER BY event_time DESC, ingest_time DESC LIMIT 1 BY event_uid LIMIT %d`, clickhouse.Ident(cfg.ClickHouseDB), strings.Join(where, " AND "), limit) ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second) defer cancel() rows, e := ch.QueryJSON(ctx, q) diff --git a/internal/clickhouse/client.go b/internal/clickhouse/client.go index 0eed875..0f76c74 100644 --- a/internal/clickhouse/client.go +++ b/internal/clickhouse/client.go @@ -17,12 +17,12 @@ import ( ) type Client struct { - base, user, pass string - hc *http.Client + base, db, user, pass string + hc *http.Client } func New(cfg config.Config) *Client { - return &Client{base: cfg.ClickHouseURL, user: cfg.ClickHouseUser, pass: cfg.ClickHousePassword, hc: &http.Client{Timeout: 30 * time.Second}} + return &Client{base: cfg.ClickHouseURL, db: cfg.ClickHouseDB, user: cfg.ClickHouseUser, pass: cfg.ClickHousePassword, hc: &http.Client{Timeout: 30 * time.Second}} } func (c *Client) request(ctx context.Context, query string, body io.Reader) (*http.Response, error) { u := c.base + "/?query=" + url.QueryEscape(query) @@ -57,7 +57,7 @@ func (c *Client) InsertEvents(ctx context.Context, events []contracts.CanonicalE return e } } - q := `INSERT INTO siem.events FORMAT JSONEachRow` + q := `INSERT INTO ` + Ident(c.db) + `.events FORMAT JSONEachRow` r, e := c.request(ctx, q, &b) if e != nil { return e @@ -95,4 +95,5 @@ func (c *Client) QueryJSON(ctx context.Context, q string) ([]map[string]any, err } return out, s.Err() } -func Q(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" } +func Q(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" } +func Ident(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } diff --git a/internal/config/config.go b/internal/config/config.go index 8066560..cee5c56 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,7 @@ type Config struct { ServiceAddr string PostgresURL string ClickHouseURL string + ClickHouseDB string ClickHouseUser string ClickHousePassword string KafkaBrokers []string @@ -23,7 +24,6 @@ type Config struct { RawArchiveEnabled bool RawSpoolDir string RawRetention time.Duration - EventRetention time.Duration DetectorInterval time.Duration DetectorLookback time.Duration UIQueryLimit int @@ -36,6 +36,7 @@ func Load() Config { ServiceAddr: env("SERVICE_ADDR", ":8080"), PostgresURL: env("POSTGRES_URL", "postgres://siem:siem@postgres:5432/siem?sslmode=disable"), ClickHouseURL: strings.TrimRight(env("CLICKHOUSE_URL", "http://clickhouse:8123"), "/"), + ClickHouseDB: env("CLICKHOUSE_DB", "siem"), ClickHouseUser: env("CLICKHOUSE_USER", "siem"), ClickHousePassword: env("CLICKHOUSE_PASSWORD", "siem"), KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "redpanda:9092")), @@ -48,7 +49,6 @@ func Load() Config { RawArchiveEnabled: envBool("RAW_ARCHIVE_ENABLED", true), RawSpoolDir: env("RAW_SPOOL_DIR", "/var/spool/siem-raw"), RawRetention: envDuration("RAW_RETENTION", 30*24*time.Hour), - EventRetention: envDuration("EVENT_RETENTION", 90*24*time.Hour), DetectorInterval: envDuration("DETECTOR_INTERVAL", 30*time.Second), DetectorLookback: envDuration("DETECTOR_LOOKBACK", 20*time.Minute), UIQueryLimit: envInt("UI_QUERY_LIMIT", 500), diff --git a/internal/detector/detector.go b/internal/detector/detector.go index b2ef709..c7bb7be 100644 --- a/internal/detector/detector.go +++ b/internal/detector/detector.go @@ -50,7 +50,7 @@ func Run(ctx context.Context, cfg config.Config) error { func runAll(ctx context.Context, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) error { end := time.Now().UTC() start := end.Add(-cfg.DetectorLookback) - for _, r := range rules() { + for _, r := range rules(cfg.ClickHouseDB) { q := r.query(start, end, cfg.TenantID) rows, e := ch.QueryJSON(ctx, q) if e != nil { @@ -73,39 +73,40 @@ func runAll(ctx context.Context, cfg config.Config, pg *postgres.Store, ch *clic } return nil } -func rules() []rule { +func rules(db string) []rule { + table := clickhouse.Ident(db) + ".events" return []rule{ - {name: "audit_log_cleared", severity: "critical", eventCode: 1102, score: 9.8, query: simpleEvent(1102, 1), summary: func(m map[string]any) string { + {name: "audit_log_cleared", severity: "critical", eventCode: 1102, score: 9.8, query: simpleEvent(table, 1102, 1), summary: func(m map[string]any) string { return fmt.Sprintf("Audit-Log auf %s wurde gelöscht", str(m["host_name"])) }}, - {name: "service_installed", severity: "high", eventCode: 7045, score: 8.0, query: simpleEvent(7045, 1), summary: func(m map[string]any) string { + {name: "service_installed", severity: "high", eventCode: 7045, score: 8.0, query: simpleEvent(table, 7045, 1), summary: func(m map[string]any) string { return fmt.Sprintf("Neuer Dienst auf %s installiert", str(m["host_name"])) }}, {name: "account_lockout", severity: "medium", eventCode: 4740, score: 5.5, query: func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4740 GROUP BY host_name,user_name,workstation HAVING cnt>=1`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) + return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4740 GROUP BY host_name,user_name,workstation HAVING cnt>=1`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) }, summary: func(m map[string]any) string { return fmt.Sprintf("Account-Lockout: %s; Caller %s; DC/Host %s (%d×)", str(m["user_name"]), fallback(str(m["workstation"]), "unbekannt"), str(m["host_name"]), int64(num(m["cnt"]))) }}, {name: "failed_logon_burst", severity: "high", eventCode: 4625, score: 7.5, query: func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT host_name, target_user AS user_name, source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND target_user!='' GROUP BY host_name,user_name,source_ip,workstation HAVING cnt>=20`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) + return fmt.Sprintf(`SELECT host_name, target_user AS user_name, source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND target_user!='' GROUP BY host_name,user_name,source_ip,workstation HAVING cnt>=20`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) }, summary: func(m map[string]any) string { return fmt.Sprintf("%d fehlgeschlagene Logons für %s auf %s", int64(num(m["cnt"])), str(m["user_name"]), str(m["host_name"])) }}, {name: "password_spray", severity: "high", eventCode: 4625, score: 8.5, query: func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT '' AS host_name, '' AS user_name, source_ip, '' AS workstation, uniqExact(target_user) users, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND source_ip!='' AND target_user!='' GROUP BY source_ip HAVING users>=10 AND cnt>=20`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) + return fmt.Sprintf(`SELECT '' AS host_name, '' AS user_name, source_ip, '' AS workstation, uniqExact(target_user) users, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND source_ip!='' AND target_user!='' GROUP BY source_ip HAVING users>=10 AND cnt>=20`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) }, summary: func(m map[string]any) string { return fmt.Sprintf("Password-Spray von %s gegen %.0f Benutzer (%d Versuche)", str(m["source_ip"]), num(m["users"]), int64(num(m["cnt"]))) }}, {name: "privileged_group_change", severity: "critical", eventCode: 4728, score: 9.2, query: func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code IN (4728,4732,4756) GROUP BY host_name,user_name,workstation HAVING cnt>=1`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) + return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code IN (4728,4732,4756) GROUP BY host_name,user_name,workstation HAVING cnt>=1`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) }, summary: func(m map[string]any) string { return fmt.Sprintf("Privilegierte Gruppenmitgliedschaft geändert: %s auf %s", str(m["user_name"]), str(m["host_name"])) }}, } } -func simpleEvent(id uint32, min int) func(time.Time, time.Time, string) string { +func simpleEvent(table string, id uint32, min int) func(time.Time, time.Time, string) string { return func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT host_name, '' AS user_name, '' AS source_ip, '' AS workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=%d GROUP BY host_name HAVING cnt>=%d`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)), id, min) + return fmt.Sprintf(`SELECT host_name, '' AS user_name, '' AS source_ip, '' AS workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=%d GROUP BY host_name HAVING cnt>=%d`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)), id, min) } } func ts(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000") } @@ -143,7 +144,12 @@ func fingerprint(v ...string) string { h := sha256.Sum256([]byte(strings.Join(v, "|"))) return hex.EncodeToString(h[:]) } -func fallback(v, d string) string { if strings.TrimSpace(v) == "" { return d }; return v } +func fallback(v, d string) string { + if strings.TrimSpace(v) == "" { + return d + } + return v +} func max64(a, b int64) int64 { if a > b { return a diff --git a/reset-DANGEROUS.sh b/reset-DANGEROUS.sh index d7af9f0..af98597 100755 --- a/reset-DANGEROUS.sh +++ b/reset-DANGEROUS.sh @@ -1,8 +1,7 @@ #!/usr/bin/env sh set -eu cd "$(dirname "$0")" -printf 'ACHTUNG: Alle SIEM-Daten löschen. Tippe DELETE: ' +printf 'ACHTUNG: Alle SIEM-Daten und Docker-Volumes löschen. Tippe DELETE: ' read x [ "$x" = "DELETE" ] || exit 1 docker compose down -v --remove-orphans -rm -f deploy/garage/garage.toml diff --git a/status.sh b/status.sh index 82d0dd2..5393709 100755 --- a/status.sh +++ b/status.sh @@ -1,8 +1,16 @@ #!/usr/bin/env sh set -eu cd "$(dirname "$0")" +[ -f .env ] && { set -a; . ./.env; set +a; } docker compose ps printf '\nIngress readiness:\n' -curl -fsS http://127.0.0.1:$(grep '^INGRESS_PORT=' .env | cut -d= -f2-)/readyz || true -printf '\n\nRecent errors:\n' -docker compose logs --since=10m ingress processor detector api 2>&1 | grep -Ei 'error|fatal|failed|panic' | tail -50 || true +if command -v curl >/dev/null 2>&1; then curl -fsS "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz" || true; fi +printf '\nAPI readiness:\n' +if command -v curl >/dev/null 2>&1; then curl -fsS "http://127.0.0.1:${UI_PORT:-8080}/readyz" || true; fi +printf '\nGarage:\n' +docker compose exec -T garage /garage status 2>/dev/null || true +printf '\nClickHouse:\n' +docker compose exec -T clickhouse clickhouse-client --user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" --query 'SELECT version(), currentUser()' 2>/dev/null || true +printf '\nRecent errors:\n' +docker compose logs --since=10m garage clickhouse clickhouse-schema postgres postgres-schema redpanda ingress processor detector api archive-uploader 2>&1 \ + | grep -Ei 'error|fatal|failed|panic|exception' | tail -80 || true