From e9d9583f2842e208a844293c4e7c828492a671be Mon Sep 17 00:00:00 2001 From: jbergner Date: Fri, 24 Jul 2026 07:17:38 +0200 Subject: [PATCH] Funktionsrollback --- .env.example | 18 +- ARCHITECTURE.md | 104 +++- CPU-COMPATIBILITY-FIX.md | 17 + Dockerfile | 1 + PRODUCT-RESTORATION.md | 27 + README.md | 90 ++- RELEASE-CHECKS.md | 61 +- RULES.md | 75 +++ VERSION | 2 +- compose.yml | 38 +- credentials.sh | 10 +- deploy.ps1 | 153 +++-- deploy.sh | 27 +- deploy/clickhouse/init.sql.template | 11 + .../grafana/dashboards/pipeline-health.json | 461 +++++++++++++++ .../grafana/dashboards/security-overview.json | 550 ++++++++++++++++++ .../grafana/provisioning/dashboards/siem.yml | 12 + .../grafana/provisioning/datasources/siem.yml | 29 + deploy/postgres/init.sql | 56 ++ deploy/rules/windows-account-admin.json | 99 ++++ deploy/rules/windows-authentication.json | 102 ++++ deploy/rules/windows-core.json | 98 ++++ deploy/rules/windows-defender-powershell.json | 81 +++ doctor.sh | 12 +- go.mod | 10 - go.sum | 40 -- host-info.sh | 37 ++ internal/api/server.go | 254 +++++++- internal/config/config.go | 6 + internal/contracts/event.go | 88 +-- internal/detector/detector.go | 125 ++-- internal/ingress/server_test.go | 10 +- internal/postgres/store.go | 248 +++++++- internal/rules/rules.go | 355 +++++++++++ internal/rules/rules_test.go | 26 + preflight.sh | 137 +++++ status.sh | 2 +- web/index.html | 135 ++++- 38 files changed, 3243 insertions(+), 364 deletions(-) create mode 100644 CPU-COMPATIBILITY-FIX.md create mode 100644 PRODUCT-RESTORATION.md create mode 100644 RULES.md create mode 100644 deploy/grafana/dashboards/pipeline-health.json create mode 100644 deploy/grafana/dashboards/security-overview.json create mode 100644 deploy/grafana/provisioning/dashboards/siem.yml create mode 100644 deploy/grafana/provisioning/datasources/siem.yml create mode 100644 deploy/rules/windows-account-admin.json create mode 100644 deploy/rules/windows-authentication.json create mode 100644 deploy/rules/windows-core.json create mode 100644 deploy/rules/windows-defender-powershell.json delete mode 100644 go.sum create mode 100644 host-info.sh create mode 100644 internal/rules/rules.go create mode 100644 internal/rules/rules_test.go create mode 100644 preflight.sh diff --git a/.env.example b/.env.example index 2180051..754628b 100644 --- a/.env.example +++ b/.env.example @@ -26,10 +26,11 @@ POSTGRES_BIND=127.0.0.1 POSTGRES_PORT=5432 # ClickHouse event store -CLICKHOUSE_IMAGE=clickhouse/clickhouse-server:26.6.2.81 +CLICKHOUSE_IMAGE=clickhouse/clickhouse-server:26.3.17.56 CLICKHOUSE_DB=siem CLICKHOUSE_USER=siem CLICKHOUSE_PASSWORD=CHANGE_ME +CLICKHOUSE_GRAFANA_PASSWORD=CHANGE_ME CLICKHOUSE_HTTP_BIND=127.0.0.1 CLICKHOUSE_HTTP_PORT=8123 CLICKHOUSE_NATIVE_BIND=127.0.0.1 @@ -74,6 +75,16 @@ DETECTOR_INTERVAL=30s DETECTOR_LOOKBACK=20m UI_QUERY_LIMIT=500 + +# Grafana analyst dashboards +GRAFANA_IMAGE=grafana/grafana:13.1 +GRAFANA_BIND=0.0.0.0 +GRAFANA_PORT=3000 +GRAFANA_PUBLIC_URL= +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=CHANGE_ME +GRAFANA_CLICKHOUSE_PLUGIN_VERSION=4.20.0 + # Prometheus PROMETHEUS_IMAGE=prom/prometheus:v3.13.1 PROMETHEUS_BIND=127.0.0.1 @@ -83,3 +94,8 @@ PROMETHEUS_RETENTION=30d # Raw uploader and application image RCLONE_IMAGE=rclone/rclone:1.74.4 APP_IMAGE=greenfield-siem-app:local + +# Deployment compatibility / automatic conflict handling +AUTO_PORTS=true +CLICKHOUSE_CPU_PROBE=true +CLICKHOUSE_FALLBACK_IMAGE=clickhouse/clickhouse-server:26.3.17.56 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3983797..44ca287 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,42 +1,100 @@ -# Architekturentscheidungen +# Greenfield SIEM – Architektur 1.2 -## 1. MariaDB ist kein Event Store mehr +## Leitprinzip -PostgreSQL ist ausschließlich Control Plane. ClickHouse ist ausschließlich Event-/Analytics-Plane. Dadurch konkurrieren Agent-Updates, Incident-Status und Benutzeraktionen nicht mit Milliarden append-only Events. +Der skalierbare Eventpfad und die SIEM-Produktebene sind getrennt. Ein langsames Dashboard, eine Rule-Auswertung oder ein Grafana-Query darf niemals den HTTP-Ingress blockieren. -## 2. Queue vor Datenbank +```text +Collectors + │ + ▼ +HTTP Ingress ───────────────► PostgreSQL (Agent-Auth) + │ 202 Accepted + ▼ +Redpanda + │ + ▼ +Processor ─────► ClickHouse events ─────► Rule Engine + │ │ │ + │ │ ▼ + │ │ PostgreSQL Findings + │ │ + │ ├────► Analyst API/UI + │ └────► Grafana (read-only) + │ + └────► gzip spool ─────► Garage/S3 -Der Ingress quittiert erst, nachdem Redpanda den Batch bestätigt hat. ClickHouse-Ausfälle führen damit nicht zu HTTP-Timeout-Kaskaden bei den Collectoren. Der Processor kann nachholen, solange die Queue-Retention nicht überschritten wird. +Prometheus ◄──── Ingress / Redpanda / ClickHouse + │ + └─────────────────────► Grafana Pipeline Health +``` -## 3. Ein kanonisches Event +## Stores -Es gibt keine parallelen `event_logs`, `event_occurrences`, `event_count_buckets`, `raw_event`- und Rule-Helper-Kopien mehr. Die einzige langfristige Eventtabelle ist `siem.events`. Das Dashboard-Rollup ist eine explizite, kleine Materialized View. +### ClickHouse -## 4. Raw getrennt von Analytics +Einzige kanonische, vollständig durchsuchbare Eventtabelle. Normalisierte Felder werden spaltenorientiert gespeichert; unbekannte Restattribute liegen begrenzt in einer Map. Vollständige XML-/Raw-Batches werden nicht dupliziert. -Raw-Payloads sind für Parserfehler und Forensik wertvoll, aber ungeeignet als primäre Analytics-Zeile. Sie werden gzip-komprimiert in S3-kompatiblen Object Storage verschoben. ClickHouse speichert nur den Object-Key und den Index im Batch. +- `events`: 90 Tage Standard-TTL +- `events_5m`: 5-Minuten-Rollup, 730 Tage Standard-TTL +- `ReplacingMergeTree` + stabile `event_uid` gegen Retry-Duplikate -## 5. Kein Graph als Primärspeicher +Auf dem bekannten KVM-Host wird `26.3.17.56` verwendet, weil diese Version mit dem präsentierten QEMU-CPU-Modell läuft. -Graph-Sichten können später aus ClickHouse abgeleitet werden, z. B. `(User)-[:LOGGED_ON_TO]->(Host)` mit `first_seen`, `last_seen`, `count`. Jedes einzelne Event als Graph-Knoten würde die gleiche Explosion nur in einer anderen Engine wiederholen. +### PostgreSQL -## 6. Keine TSDB für Security-Events +Nur Control Plane und Workflow: -Prometheus überwacht die Pipeline, speichert aber keine Benutzer-/Host-/IP-Eventdimensionen. Hochkardinale Security-Attribute gehören nach ClickHouse. +- Agents / Enrollment +- Rule-Sets und Rule-Registry +- Suppressions +- Detections / Status -## 7. Idempotenz +Keine Eventhistorie. -Redpanda/Kafka-Consumer liefern at-least-once und auch ein Collector kann nach einem HTTP-Timeout denselben Batch erneut senden. Deshalb berechnet der Ingress `batch_uid = SHA-256(agent_id + Batch-Inhalt)` und der Processor verwendet `event_uid = batch_uid:index`. ReplacingMergeTree und `uniqExact`-States verhindern damit Zählfehler bei Queue- und identischen HTTP-Retries. +### Redpanda -## 8. UI ist kein Compute-Job +Persistenter Recovery-Puffer und Entkopplung zwischen Ingress und Verarbeitung. Standard-Retention 24 Stunden. -Die Startseite liest `events_5m` und PostgreSQL-Findings. Sie startet keine Detection-Regeln. Die Event-Timeline hat immer ein begrenztes Zeitfenster und Limit. +### Garage/S3 -## 9. Retention nach Datenklasse +Komprimiertes Rohdatenarchiv. Standard-Retention 30 Tage. -- normalisierte Events: 90 Tage -- Raw: 30 Tage -- Rollups: 730 Tage -- Queue: 24 Stunden +### Prometheus -Diese Werte sind Defaults, keine Compliance-Aussage. Rechtliche und organisatorische Anforderungen haben Vorrang. +Nur Betriebsmetriken. Keine Benutzer/IP/Event-ID-Kardinalität als Labels. + +## Rule Engine + +Built-in-Rule-Sets liegen unter `deploy/rules/*.json`. Der Detector synchronisiert sie nach PostgreSQL. Das ermöglicht Versionierung im Repository und Laufzeitsteuerung über das UI. + +Die Rule-Definition enthält kein freies SQL. Der Compiler akzeptiert nur erlaubte Felder, Operatoren, Gruppierungen und Regeltypen und erzeugt daraus begrenzte ClickHouse-Abfragen. + +Regeltypen: + +- Event Match +- Threshold +- Distinct/Spread + +Rule-Set- und Regel-Aktivierungszustände werden nicht durch ein Softwareupdate zurückgesetzt. + +## Analystenoberflächen + +### SIEM Analyst UI + +Für operative Arbeit: + +- Overview +- Event Investigation und Drill-down +- Detections / Status +- Rule-Sets / Custom Rules +- Suppressions +- Agents + +### Grafana + +Für Exploration, Visualisierung und Betriebsüberwachung. Grafana erhält einen eigenen ClickHouse-Account mit ausschließlich SELECT-Rechten. Das mitgelieferte Datasource-Provisioning verbindet Grafana direkt mit ClickHouse; Prometheus ist die zweite provisionierte Datasource. + +## Availability + +Das One-Click-Compose ist ein Single-Node-Deployment und nicht hochverfügbar. Die Architektur erlaubt später getrennte Redpanda-/ClickHouse-/PostgreSQL-/Object-Storage-Nodes, ohne den HTTP-Vertrag oder das Eventmodell zu ändern. diff --git a/CPU-COMPATIBILITY-FIX.md b/CPU-COMPATIBILITY-FIX.md new file mode 100644 index 0000000..fc4548c --- /dev/null +++ b/CPU-COMPATIBILITY-FIX.md @@ -0,0 +1,17 @@ +# CPU compatibility and host-port fix + +This release adds a real ClickHouse binary probe before the SIEM stack starts. + +- `docker run ... clickhouse local --query 'SELECT 1'` validates the configured image on the actual Docker CPU. +- Exit 132 / `Illegal instruction` is treated as a CPU/VM instruction-set incompatibility, not as a credential or schema failure. +- A current 26.3 LTS image is probed as a safe fallback if the configured 26.6 image SIGILLs. +- If both official images fail, deployment stops with exit 42 and instructions for CPU passthrough / hardware compatibility. +- Existing unrelated containers occupying host ports are detected. With `AUTO_PORTS=true`, conflicting ports are moved to free alternatives and persisted in `.env`. +- `host-info.sh` prints host architecture, CPU features, Docker architecture, published ports and performs the direct ClickHouse CPU probe. + +On the reported host, the supplied `docker ps` already shows two conflicts: + +- host port 8080 is occupied by `tradingdiscordbot-starauftrag-bot-1` +- host port 9000 is occupied by `peertube-nginx-1` + +The updated preflight therefore moves these defaults automatically when deploying on that host. diff --git a/Dockerfile b/Dockerfile index c8dac4f..43775f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,5 +11,6 @@ WORKDIR /app COPY --from=build /out/siem /siem COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY web /app/web +COPY deploy/rules /app/rules USER 65532:65532 ENTRYPOINT ["/siem"] diff --git a/PRODUCT-RESTORATION.md b/PRODUCT-RESTORATION.md new file mode 100644 index 0000000..3f15337 --- /dev/null +++ b/PRODUCT-RESTORATION.md @@ -0,0 +1,27 @@ +# Product Restoration – Rule-Sets, Grafana und Analyst UI + +Diese Version korrigiert einen funktionalen Rückschritt des ursprünglichen Greenfield-Redesigns. Die skalierbare Datenpipeline bleibt bestehen, aber die produktrelevanten SIEM-Funktionen wurden wieder ergänzt. + +## Wiederhergestellt / neu gebaut + +- datengetriebene, versionierte Rule-Sets statt sechs hart codierter Go-Regeln +- 19 eingebaute Windows-Regeln in vier Rule-Sets +- PostgreSQL-Registry für Rule-Sets, Regeln und Suppressions +- Aktivieren/Deaktivieren einzelner Regeln und kompletter Rule-Sets +- eigener Rule-Editor für Custom Rules +- sichere Übersetzung der Regeln in ClickHouse-Abfragen über Allowlist statt beliebigem SQL +- Grafana inklusive provisioniertem ClickHouse-Datasource und zwei Dashboards +- separater read-only ClickHouse-Account für Grafana +- neues Analysten-UI mit Overview, Investigation, Detections, Rule-Sets, Suppressions und Agents +- Event-Detailansicht mit Commandline, Message, Attributen und Raw-Archiv-Referenz +- Detections mit Rule-ID, Rule-Set, Tags und MITRE-Techniken +- CPU-kompatibles ClickHouse `26.3.17.56` als Standard für den bekannten KVM-Host +- automatische Portkonfliktbehandlung weiterhin enthalten + +## Architektur bleibt unverändert sinnvoll + +Ingress → Redpanda → Processor → ClickHouse bleibt der Eventpfad. PostgreSQL bleibt Control Plane und Finding Store. Garage bleibt Raw-Archiv. Grafana und das Analysten-UI lesen aus diesen spezialisierten Stores, ohne den Ingest zu blockieren. + +## Upgrade + +Eine bestehende Greenfield-`.env` kann übernommen werden. `deploy.sh` ergänzt die neuen Grafana-Secrets automatisch, behält vorhandene Passwörter und Enrollment-Keys und synchronisiert das erweiterte PostgreSQL-Schema idempotent. diff --git a/README.md b/README.md index 19628ea..bbc42a2 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Greenfield SIEM -**Release 1.1 – Deployment-Fix:** Garage-Bootstrap, vollständige `.env`, ClickHouse-Native-Port und idempotentes ClickHouse-Schema wurden gegenüber der ersten Greenfield-Version korrigiert. +**Release 1.2 – Product Restoration:** Die skalierbare Greenfield-Pipeline bleibt bestehen; Rule-Sets, Suppressions, Grafana und ein deutlich umfangreicheres Analysten-UI sind wieder Bestandteil des Produkts. ClickHouse ist für den bekannten KVM-Host standardmäßig auf `26.3.17.56` gepinnt. -Kompletter Neuaufbau des bisherigen Projekts. Vom Altprojekt bleibt absichtlich nur der HTTP-Ingress-Vertrag erhalten. +Kompletter Neuaufbau des bisherigen Eventpfads. Der HTTP-Ingress-Vertrag bleibt kompatibel, die SIEM-Produktebene wurde gegenüber der ersten Greenfield-Version wieder vollständig ausgebaut. ## Was dieses Projekt löst @@ -24,8 +24,9 @@ Processor │ └── rclone übernimmt Upload + Retention -ClickHouse ──► Detector ──► PostgreSQL (Detections / Status) -ClickHouse + PostgreSQL ──► API/UI +ClickHouse ──► Rule Engine ──► PostgreSQL (Rules / Suppressions / Findings) +ClickHouse + PostgreSQL ──► Analyst API/UI +ClickHouse + Prometheus ──► Grafana ``` **Keine Event-Zeilen in PostgreSQL. Keine MariaDB. Keine synchronen Detection-Abfragen im Ingress. Keine Raw-XML-Duplikate in ClickHouse.** @@ -53,6 +54,7 @@ Danach: - UI: `http://SERVER:8080/ui` - Ingress: `http://SERVER:8090/ingest` +- Grafana: `http://SERVER:3000` (Port wird bei Konflikten automatisch verschoben) - Redpanda Console: lokal `http://127.0.0.1:8081` - Prometheus: lokal `http://127.0.0.1:9090` @@ -86,6 +88,8 @@ Falls `.env` gelöscht wurde, aber alte PostgreSQL-Volumes mit einem unbekannten ## ClickHouse-Zugang +Für den bekannten KVM-Host mit `QEMU Virtual CPU version 2.5+` ist `clickhouse/clickhouse-server:26.3.17.56` der Standard, weil diese Version auf der VM nachweislich startet, während 26.6 dort mit SIGILL/Exit 132 abbricht. Der Preflight testet das Binary vor dem Stackstart. + ClickHouse besitzt zwei verschiedene Schnittstellen: - HTTP: `127.0.0.1:8123` @@ -229,33 +233,59 @@ RAW_RETENTION=720h KAFKA_RETENTION_MS=86400000 ``` -## Detections in Version 1 +## Rule-Sets und Detection Engine -Der Detector läuft unabhängig vom UI alle 30 Sekunden mit einem begrenzten Lookback und schreibt nur Findings in PostgreSQL. +Der Detector ist nicht mehr auf einige hart codierte Go-Regeln beschränkt. Beim Start werden versionierte Rule-Sets aus `deploy/rules/` nach PostgreSQL synchronisiert. Aktivierungszustände aus dem UI bleiben bei Updates erhalten. Eigene Regeln werden im Rule-Set `custom` gespeichert. -Enthalten: +Mitgeliefert werden 19 Regeln in vier Rule-Sets: -- Audit Log gelöscht (1102) -- Dienst installiert (7045) -- Account Lockout (4740) -- Failed Logon Burst (4625) -- Password Spray (4625 gegen viele Benutzer) -- privilegierte Gruppenmitgliedschaft geändert (4728 / 4732 / 4756) +| Rule-Set | Regeln | Beispiele | +|---|---:|---| +| Windows Core | 5 | Audit Log gelöscht, Dienst installiert, Scheduled Task, Audit Policy, Firewall | +| Windows Authentication | 5 | Lockout, Failed-Logon-Burst, Password Spray, Kerberos, NTLM | +| Windows Account & Admin | 5 | privilegierte Gruppen, Benutzer angelegt/aktiviert/gelöscht, Passwort-Reset | +| Defender & PowerShell | 4 | Malware, Defender deaktiviert, EncodedCommand, verdächtige Download-/Memory-Muster | -**Es gibt bewusst keine pauschale `new_event_id`-Detection mehr.** Event 5857 und vergleichbare Betriebsereignisse sind normale Events, keine Incidents. +Regeltypen sind `event`, `threshold` und `distinct`. Bedingungen werden über eine Feld-/Operator-Allowlist in ClickHouse-SQL kompiliert; Rule-Dateien enthalten kein frei ausführbares SQL. Einzelne Regeln und komplette Rule-Sets können aktiviert/deaktiviert werden. Suppressions können nach Rule-ID, Host, User oder Source-IP begrenzt und zeitlich befristet werden. -## UI +**Es gibt weiterhin keine pauschale `new_event_id`-Detection.** Event 5857 und vergleichbare Betriebsereignisse sind normale Events. Details zum Format stehen in [`RULES.md`](RULES.md). -`/ui` enthält: +## Analysten-UI -- Eventzahl 24h aus Rollup -- aktive Hosts 24h -- offene High-/Critical-Detections -- Eventsuche nach Zeitraum, Host, User, IP, Event-ID und Channel -- Detection-Liste mit Statusänderung -- Agent-Liste / Last Seen +`/ui` ist wieder eine eigentliche SIEM-Oberfläche und nicht nur eine einfache Eventtabelle. Enthalten sind: -Die initiale Eventsuche ist auf 24 Stunden und 500 Ergebnisse begrenzt. Der ClickHouse-Sortierschlüssel beginnt mit Tenant und Event-Zeit; alle UI-Suchen besitzen zusätzlich ein hart begrenztes Zeitfenster und Ergebnislimit. Das verhindert wiederkehrende Vollscans über die gesamte Historie. +- **Overview:** 24h-KPIs, Eventvolumen, Top Hosts/Event-IDs und Authentication-Übersicht +- **Investigation:** Zeit-, Host-, User-, IP-, Event-ID-, Channel- und Textfilter mit Drill-down +- **Event Details:** Message, Commandline, relevante Attribute, Raw-Archiv-Referenz und Normalisierungsfelder +- **Detections:** Severity, Status, Rule/Rule-Set, MITRE-Tags und Status-Workflow +- **Rule-Sets:** Set- und Regel-Toggles sowie Custom-Rule-Editor +- **Suppressions:** anlegen und entfernen, optional mit Ablaufzeit +- **Agents:** Enrollment-/Last-Seen-Übersicht und Aktivierung + +Die initiale Investigation ist zeitlich und per Ergebnislimit begrenzt. Das Overview verwendet kleine Rollups beziehungsweise begrenzte Aggregationen statt unkontrollierter Vollscans. + +## Grafana + +Grafana ist wieder Bestandteil des Standard-Stacks. Es wird automatisch mit einem dedizierten, **SELECT-only** ClickHouse-Benutzer und dem ClickHouse-Datasource provisioniert. Das administrative SIEM-UI und Grafana haben bewusst getrennte Aufgaben: UI für Workflow/Rules/Findings, Grafana für freie Visualisierung und Explore. + +Mitgelieferte Dashboards: + +- **SIEM Security Overview:** Eventvolumen, aktive Hosts, Failed Logons, Lockouts, Top Hosts/Event-IDs, Authentication, Top Failed Users und Source-IPs +- **SIEM Pipeline Health:** Ingress/Redpanda/ClickHouse-Erreichbarkeit, Event-/Batch-Rate, Rejections und Scrape-Latenzen + +Zugangsdaten: + +```bash +./credentials.sh +``` + +Standardmäßig: + +```text +http://SERVER:3000 +``` + +`GRAFANA_PORT` wird wie die anderen Host-Ports durch `preflight.sh` auf Konflikte geprüft. ## Raw-Event wiederherstellen @@ -398,3 +428,17 @@ Nur wenn wirklich alle Daten weg sollen: ``` Das Skript verlangt zusätzlich die Eingabe `DELETE` und entfernt anschließend die Docker-Volumes. + +## CPU-Kompatibilität und Portkonflikte + +`deploy.sh` führt vor dem eigentlichen Start einen echten ClickHouse-Binary-Probe in einem kurzlebigen Container aus. Ein `Illegal instruction`/Exit 132 wird dadurch erkannt, bevor der Stack in eine Restart-Schleife gerät. + +Bei `amd64` benötigt das offizielle ClickHouse-Image mindestens SSE3. Bei `arm64` setzt das offizielle Image ARMv8.2-A plus RCpc voraus. Bei VMs sollte deshalb nach Möglichkeit das CPU-Modell `host` beziehungsweise CPU-Passthrough verwendet werden. + +Zusätzlich erkennt `deploy.sh` bereits belegte Host-Ports. Wenn `AUTO_PORTS=true` ist, werden nur kollidierende Ports auf freie Ersatzports verschoben und in `.env` gespeichert. Die tatsächlich verwendeten Werte zeigt anschließend `./credentials.sh`. + +Für eine vollständige Hostdiagnose: + +```bash +./host-info.sh +``` diff --git a/RELEASE-CHECKS.md b/RELEASE-CHECKS.md index 3c98cc5..498116c 100644 --- a/RELEASE-CHECKS.md +++ b/RELEASE-CHECKS.md @@ -1,39 +1,36 @@ -# Release checks +# Release Checks – 1.2.0 Product Restoration -Stand: 2026-07-23 – Deployment-Fix 1.1 +## Geprüft -Durchgeführt: +- vollständiger Go-Typcheck aller Pakete mit lokalen Schnittstellen-Stubs für `pgx` und `kafka-go` +- `go test` für alle Pakete unter dieser Stub-Umgebung +- `go vet` für alle Pakete unter dieser Stub-Umgebung +- reale Unit-Tests der Rule-Compiler- und Normalizer-Pakete +- alle vier eingebauten Rule-Set-Dateien mit dem echten Rule-Loader/Validator geladen: 19 Regeln +- Analysten-UI JavaScript mit Node `--check` geprüft +- alle Grafana-Dashboard-JSON-Dateien geparst +- Compose-, Prometheus- und Grafana-Provisioning-YAML geparst +- alle Shell-Skripte mit `sh -n` geprüft +- Compose-Variablen gegen `.env.example` abgeglichen: kein referenzierter Schlüssel fehlt +- alle Compose-Bind-Mount-Dateien geprüft +- simuliertes Upgrade-Deployment mit vorhandener partieller `.env` +- simuliert: Port 8080 belegt → automatische Verschiebung auf 18080 +- simuliert: Port 9000 belegt → automatische Verschiebung auf 19000 +- simuliert: ClickHouse 26.6 CPU-Probe Exit 132 → automatischer Wechsel auf 26.3.17.56 +- neue Grafana-/ClickHouse-Reader-Secrets werden ergänzt; vorhandene eigene `.env`-Werte bleiben erhalten +- Pipeline-Health-Dashboard verwendet die tatsächlich exportierten `siem_ingress_*`-Metriken -- `compose.yml` mit YAML-Parser geladen: 15 Services, erfolgreich. -- Alle 63 nicht-escaped Compose-Variablen mit `.env.example` abgeglichen: **0 fehlende, 0 überzählige**. -- Sämtliche Host-Bind-Mount-Quellen aus `compose.yml` auf Existenz geprüft: **0 fehlende Pfade**. -- Prometheus-YAML geparst: erfolgreich. -- ClickHouse-Prometheus-XML geparst: erfolgreich. -- Garage-Konfigurationskommando aus Compose isoliert ausgeführt und resultierendes TOML mit `tomllib` geparst: erfolgreich. -- ClickHouse-Schema-Rendering aus Compose isoliert ausgeführt: Datenbankname und TTL-Platzhalter vollständig ersetzt; Schema-/TTL-Kommandos wurden aufgerufen. -- `deploy.sh` mit simuliertem Docker/HTTP end-to-end ausgeführt: erfolgreich. - - frische `.env`: 63 Schlüssel, keine `CHANGE_ME`-Secrets; - - externer bestehender Enrollment-Key bleibt erhalten; - - partielle Alt-`.env` wird vollständig ergänzt; - - vorhandene UI-/PostgreSQL-/ClickHouse-/Enrollment-Secrets bleiben unverändert; - - eigener unbekannter `.env`-Schlüssel bleibt erhalten. -- POSIX-Shellskripte mit `sh -n` geprüft: erfolgreich. -- `go test ./...` gegen lokale Interface-Stubs für pgx/kafka-go mit der vorhandenen Go-1.23.2-Toolchain: erfolgreich. -- `go vet ./...` unter denselben isolierten Bedingungen: erfolgreich. -- Ingress-Tests: Batch-UID/Retry-Idempotenz und Validierung: erfolgreich. -- Normalizer-Tests: Windows-Metadaten/XML und Lockout-Workstation: erfolgreich. +## Sicherheitschecks -Zusätzlich gegen aktuelle Primärquellen geprüft: +- Grafana verwendet einen eigenen ClickHouse-User `grafana_reader` +- dieser erhält ausschließlich `SELECT` auf die SIEM-Datenbank +- Custom Rules enthalten kein frei ausführbares SQL; der Compiler verwendet Feld-/Operator-Allowlists +- Custom Rules können Built-in-Regeln mit derselben ID nicht überschreiben +- entfernte Built-in-Regeln und Rule-Sets werden beim Synchronisieren bereinigt +- vorhandene Aktivierungszustände von Rule-Sets und Built-in-Regeln bleiben bei Updates erhalten -- Garage v2.3 Single-Node/Default-Bucket-Start und die erwarteten Default-Credential-Umgebungsvariablen. -- ClickHouse Docker: HTTP-Port 8123, nativer Client-Port 9000 sowie `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD` und Access-Management-Initialisierung. -- Redpanda Single-Broker Docker-Beispiel und aktuelle Broker-/Console-Versionen. +## Nicht in dieser Umgebung möglich -Nicht in dieser Ausführungsumgebung möglich: +Es steht kein echter Docker-Daemon zur Verfügung. Daher konnte der komplette Stack hier nicht real mit PostgreSQL, ClickHouse, Redpanda, Garage und Grafana gestartet werden. Das Deployment wurde mit einem Docker-/HTTP-Simulator end-to-end durchlaufen; der Anwendungscode wurde separat typgeprüft. -- echter Docker-Compose-Start der Datenbanken/Broker/Object-Storage-Komponenten, da hier kein Docker-Daemon/CLI verfügbar ist; -- Integrationstest gegen reale ClickHouse-, PostgreSQL-, Redpanda- und Garage-Container; -- Ausführung von `deploy.ps1`, da PowerShell hier nicht installiert ist; -- regulärer Build mit den realen externen Go-Modulen; die lokale Prüfung verwendete Schnittstellen-Stubs, da die vorhandene lokale Go-Version 1.23.2 ist, während das Dockerfile Go 1.26.5 verwendet. - -Der erste reale Lauf sollte deshalb direkt mit `./doctor.sh` verifiziert werden. Bei einem Startfehler gibt `deploy.sh` bereits automatisch die relevanten Service-Logs aus. +Auf dem Zielhost des Nutzers wurde ClickHouse `26.3.17.56` bereits direkt im offiziellen Container erfolgreich ausgeführt, während `26.6.2.81` wegen der von KVM präsentierten CPU-Features mit Exit 132 abbricht. Deshalb ist `26.3.17.56` der Default dieser Ausgabe. diff --git a/RULES.md b/RULES.md new file mode 100644 index 0000000..914d5f8 --- /dev/null +++ b/RULES.md @@ -0,0 +1,75 @@ +# Rule-Sets und Detection Engine + +Die Detection Engine ist wieder datengetrieben. Eingebaute Rule-Sets liegen als versionierte JSON-Dateien unter `deploy/rules/` und werden beim Start des Detectors nach PostgreSQL synchronisiert. Aktivierungszustände, die im UI geändert wurden, bleiben dabei erhalten. + +## Mitgelieferte Rule-Sets + +| Rule-Set | Regeln | Schwerpunkt | +|---|---:|---| +| `windows-core` | 5 | Audit, Dienste, Scheduled Tasks, Firewall | +| `windows-authentication` | 5 | Lockout, Brute Force, Password Spray, Kerberos, NTLM | +| `windows-account-admin` | 5 | Benutzer- und Gruppenänderungen | +| `windows-defender-powershell` | 4 | Defender und PowerShell | + +Insgesamt werden 19 Regeln mitgeliefert. Hinzu kommt das PostgreSQL-Rule-Set `custom` für eigene Regeln aus dem Analysten-UI. + +## Regeltypen + +- `event`: Match auf ein oder mehrere Events im Zeitfenster. +- `threshold`: Mindestanzahl passender Events, optional gruppiert nach Host/User/IP/etc. +- `distinct`: Mindestanzahl unterschiedlicher Werte, z. B. viele Zielbenutzer von derselben Source-IP. + +## Erlaubte Bedingungen + +Die UI beziehungsweise JSON-Regeln dürfen nur freigegebene kanonische Felder verwenden. Beliebiges SQL wird nicht ausgeführt. + +Operatoren: + +- `equals`, `not_equals` +- `contains`, `not_contains` +- `regex` +- `exists`, `not_exists` +- `in` + +Typische Felder: + +- `channel`, `provider`, `event_code` +- `host`, `user`, `target_user`, `subject_user` +- `source_ip`, `destination_ip`, `workstation` +- `process_path`, `parent_process_path`, `command_line` +- `message`, `category`, `action`, `outcome` +- `status_code`, `failure_reason`, `logon_type` + +## Beispiel: fehlgeschlagene Logons + +```json +{ + "id": "windows-failed-logon-burst", + "title": "Viele fehlgeschlagene Anmeldungen", + "severity": "high", + "score": 75, + "enabled": true, + "kind": "threshold", + "channels": ["Security"], + "event_codes": [4625], + "group_by": ["host", "user", "source_ip"], + "threshold": 20, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "{count} fehlgeschlagene Logons für {user} auf {host} von {source_ip}", + "tags": ["authentication", "bruteforce"], + "mitre": ["T1110"] +} +``` + +## Suppressions + +Suppressions werden separat in PostgreSQL gespeichert und können nach Rule-ID, Host, User und Source-IP eingeschränkt werden. `*` ist als Wildcard möglich. Eine optionale Ablaufzeit verhindert dauerhafte versehentliche Ausnahmen. + +## Built-ins versus Custom Rules + +Built-in-Regeln werden aus Dateien synchronisiert und können im UI ein-/ausgeschaltet werden, aber nicht durch eine Custom Rule mit derselben ID überschrieben werden. Eigene Regeln landen im Rule-Set `custom` und können über die UI gepflegt werden. + +## Sicherheit + +Der Rule-Compiler baut selbst ClickHouse-SQL aus einer Feld-/Operator-Allowlist. Rule-JSON enthält kein direkt ausführbares SQL. Zeitfenster und Ergebnisanzahl sind begrenzt; der Detector verwendet zusätzlich einen Query-Timeout. diff --git a/VERSION b/VERSION index 5aa3ff1..5a27181 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0-deployment-fix +1.2.0-product-restoration diff --git a/compose.yml b/compose.yml index 34bcc28..8ae5b47 100644 --- a/compose.yml +++ b/compose.yml @@ -38,7 +38,7 @@ services: restart: "no" clickhouse: - image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81} + image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56} environment: CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem} CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem} @@ -64,7 +64,7 @@ services: - "${CLICKHOUSE_NATIVE_BIND:-127.0.0.1}:${CLICKHOUSE_NATIVE_PORT:-9000}:9000" clickhouse-schema: - image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81} + image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56} depends_on: clickhouse: condition: service_healthy @@ -73,6 +73,7 @@ services: CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem} CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem} CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + CLICKHOUSE_GRAFANA_PASSWORD: ${CLICKHOUSE_GRAFANA_PASSWORD} EVENT_RETENTION_DAYS: ${EVENT_RETENTION_DAYS:-90} ROLLUP_RETENTION_DAYS: ${ROLLUP_RETENTION_DAYS:-730} volumes: @@ -83,6 +84,7 @@ services: -e "s/__CLICKHOUSE_DB__/$${CLICKHOUSE_DB}/g" \ -e "s/__EVENT_RETENTION_DAYS__/$${EVENT_RETENTION_DAYS}/g" \ -e "s/__ROLLUP_RETENTION_DAYS__/$${ROLLUP_RETENTION_DAYS}/g" \ + -e "s/__GRAFANA_CLICKHOUSE_PASSWORD__/$${CLICKHOUSE_GRAFANA_PASSWORD}/g" \ /schema/init.sql.template > /tmp/init.sql clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --multiquery < /tmp/init.sql clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \ @@ -250,6 +252,8 @@ services: RAW_ARCHIVE_ENABLED: ${RAW_ARCHIVE_ENABLED:-true} RAW_SPOOL_DIR: /var/spool/siem-raw RAW_RETENTION: ${RAW_RETENTION:-720h} + GRAFANA_PORT: ${GRAFANA_PORT:-3000} + GRAFANA_PUBLIC_URL: ${GRAFANA_PUBLIC_URL:-} depends_on: postgres-schema: condition: service_completed_successfully @@ -351,6 +355,35 @@ services: ports: - "${PROMETHEUS_BIND:-127.0.0.1}:${PROMETHEUS_PORT:-9090}:9090" + grafana: + image: ${GRAFANA_IMAGE:-grafana/grafana:13.1} + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_PLUGINS_PREINSTALL: grafana-clickhouse-datasource@${GRAFANA_CLICKHOUSE_PLUGIN_VERSION:-4.20.0} + CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem} + CLICKHOUSE_GRAFANA_PASSWORD: ${CLICKHOUSE_GRAFANA_PASSWORD} + volumes: + - grafana_data:/var/lib/grafana + - ./deploy/grafana/provisioning:/etc/grafana/provisioning:ro + - ./deploy/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + clickhouse-schema: + condition: service_completed_successfully + prometheus: + condition: service_started + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"] + interval: 10s + timeout: 5s + retries: 30 + start_period: 20s + restart: unless-stopped + ports: + - "${GRAFANA_BIND:-0.0.0.0}:${GRAFANA_PORT:-3000}:3000" + volumes: postgres_data: clickhouse_data: @@ -361,3 +394,4 @@ volumes: garage_data: raw_spool: prometheus_data: + grafana_data: diff --git a/credentials.sh b/credentials.sh index a4c2157..f28f091 100644 --- a/credentials.sh +++ b/credentials.sh @@ -4,11 +4,17 @@ cd "$(dirname "$0")" [ -f .env ] || { echo ".env fehlt. Zuerst ./deploy.sh ausführen." >&2; exit 1; } set -a; . ./.env; set +a cat <$null)) { + if ($line.Contains($needle)) { return ($line -split ' ')[0] } + } + return $null +} + +function Ensure-Port($current, [string]$Key, [int]$Fallback) { + $value = [int]$current[$Key] + $owner = Port-Owner $value + if ([string]::IsNullOrWhiteSpace($owner) -or $owner.StartsWith('greenfield-siem-')) { return } + $auto = "$($current['AUTO_PORTS'])".ToLower() -in @('1','true','yes','on') + if (-not $auto) { throw "$Key=$value ist bereits durch '$owner' belegt." } + for ($p=$Fallback; $p -lt ($Fallback+100); $p++) { + if ([string]::IsNullOrWhiteSpace((Port-Owner $p))) { + Write-Host "Portkonflikt: $Key=$value wird von '$owner' benutzt -> verwende $p." + $current[$Key] = "$p" + return + } + } + throw "Kein freier Ersatzport fuer $Key gefunden." +} + +function Test-ClickHouseImage([string]$Image) { + Write-Host "Pruefe ClickHouse-Binary '$Image' auf dieser CPU ..." + & docker pull $Image *> $null + if ($LASTEXITCODE -ne 0) { return $false } + & docker run --rm --entrypoint clickhouse $Image local --query 'SELECT 1' *> $null + return ($LASTEXITCODE -eq 0) +} + $template = Read-EnvFile ".env.example" $current = Read-EnvFile ".env" -foreach ($k in $template.Keys) { - if (-not $current.Contains($k)) { $current[$k] = $template[$k] } -} +foreach ($k in $template.Keys) { if (-not $current.Contains($k)) { $current[$k] = $template[$k] } } $presetEnrollment = $env:ENROLLMENT_KEY $secretLengths = @{ - POSTGRES_PASSWORD=24; CLICKHOUSE_PASSWORD=24; UI_PASSWORD=18; + POSTGRES_PASSWORD=24; CLICKHOUSE_PASSWORD=24; CLICKHOUSE_GRAFANA_PASSWORD=24; + UI_PASSWORD=18; GRAFANA_ADMIN_PASSWORD=18; GARAGE_SECRET_KEY=32; GARAGE_RPC_SECRET=32; GARAGE_ADMIN_TOKEN=32; GARAGE_METRICS_TOKEN=32 } foreach ($k in $secretLengths.Keys) { - if ([string]::IsNullOrWhiteSpace($current[$k]) -or $current[$k] -eq 'CHANGE_ME') { - $current[$k] = Hex $secretLengths[$k] + if ([string]::IsNullOrWhiteSpace($current[$k]) -or $current[$k] -eq 'CHANGE_ME') { $current[$k] = Hex $secretLengths[$k] } +} +if ([string]::IsNullOrWhiteSpace($current['GARAGE_ACCESS_KEY']) -or $current['GARAGE_ACCESS_KEY'] -eq 'CHANGE_ME') { $current['GARAGE_ACCESS_KEY'] = "GK$(Hex 16)" } +if (-not [string]::IsNullOrWhiteSpace($presetEnrollment)) { $current['ENROLLMENT_KEY'] = $presetEnrollment } +elseif ([string]::IsNullOrWhiteSpace($current['ENROLLMENT_KEY']) -or $current['ENROLLMENT_KEY'] -eq 'CHANGE_ME') { $current['ENROLLMENT_KEY'] = Hex 32 } + +foreach ($key in @('UI_PORT','INGRESS_PORT','CLICKHOUSE_HTTP_PORT','CLICKHOUSE_NATIVE_PORT','POSTGRES_PORT','REDPANDA_KAFKA_PORT','REDPANDA_ADMIN_PORT','REDPANDA_CONSOLE_PORT','GARAGE_S3_PORT','GARAGE_ADMIN_PORT','PROMETHEUS_PORT','GRAFANA_PORT')) { + if ([string]::IsNullOrWhiteSpace($current[$key])) { throw ".env: $key fehlt." } +} +Ensure-Port $current 'UI_PORT' 18080 +Ensure-Port $current 'INGRESS_PORT' 18090 +Ensure-Port $current 'CLICKHOUSE_HTTP_PORT' 18123 +Ensure-Port $current 'CLICKHOUSE_NATIVE_PORT' 19000 +Ensure-Port $current 'POSTGRES_PORT' 15432 +Ensure-Port $current 'REDPANDA_KAFKA_PORT' 29092 +Ensure-Port $current 'REDPANDA_ADMIN_PORT' 29644 +Ensure-Port $current 'REDPANDA_CONSOLE_PORT' 18081 +Ensure-Port $current 'GARAGE_S3_PORT' 13900 +Ensure-Port $current 'GARAGE_ADMIN_PORT' 13903 +Ensure-Port $current 'PROMETHEUS_PORT' 19090 +Ensure-Port $current 'GRAFANA_PORT' 13000 + +$probe = "$($current['CLICKHOUSE_CPU_PROBE'])".ToLower() -in @('1','true','yes','on') +if ($probe) { + if (-not (Test-ClickHouseImage $current['CLICKHOUSE_IMAGE'])) { + $fallback = $current['CLICKHOUSE_FALLBACK_IMAGE'] + if (-not [string]::IsNullOrWhiteSpace($fallback) -and $fallback -ne $current['CLICKHOUSE_IMAGE'] -and (Test-ClickHouseImage $fallback)) { + Write-Host "ClickHouse-Fallback laeuft. Verwende $fallback." + $current['CLICKHOUSE_IMAGE'] = $fallback + } else { + throw "ClickHouse-Binary ist mit der CPU/VM inkompatibel. Auf KVM/Proxmox CPU-Passthrough bzw. CPU-Modell 'host' pruefen." + } } } -if ([string]::IsNullOrWhiteSpace($current['GARAGE_ACCESS_KEY']) -or $current['GARAGE_ACCESS_KEY'] -eq 'CHANGE_ME') { - $current['GARAGE_ACCESS_KEY'] = "GK$(Hex 16)" -} -if (-not [string]::IsNullOrWhiteSpace($presetEnrollment)) { - $current['ENROLLMENT_KEY'] = $presetEnrollment -} elseif ([string]::IsNullOrWhiteSpace($current['ENROLLMENT_KEY']) -or $current['ENROLLMENT_KEY'] -eq 'CHANGE_ME') { - $current['ENROLLMENT_KEY'] = Hex 32 -} -# Write a complete UTF-8 .env. This fixes partial .env files from earlier releases -# and preserves custom keys that are not part of the current template. -$lines = New-Object System.Collections.Generic.List[string] -$templateKeys = New-Object 'System.Collections.Generic.HashSet[string]' -foreach ($line in [IO.File]::ReadAllLines(".env.example")) { - if ($line -match '^([^#=][^=]*)=(.*)$') { - $key = $matches[1] - [void]$templateKeys.Add($key) - $lines.Add("$key=$($current[$key])") - } else { - $lines.Add($line) - } -} -foreach ($key in $current.Keys) { - if (-not $templateKeys.Contains([string]$key)) { - $lines.Add("$key=$($current[$key])") - } -} -$utf8NoBom = New-Object System.Text.UTF8Encoding($false) -[IO.File]::WriteAllLines((Join-Path $PSScriptRoot '.env'), $lines, $utf8NoBom) +Write-EnvFile $template $current -foreach ($key in @('POSTGRES_PASSWORD','CLICKHOUSE_PASSWORD','UI_PASSWORD','ENROLLMENT_KEY','GARAGE_ACCESS_KEY','GARAGE_SECRET_KEY','GARAGE_RPC_SECRET','GARAGE_ADMIN_TOKEN','GARAGE_METRICS_TOKEN')) { - if ([string]::IsNullOrWhiteSpace($current[$key]) -or $current[$key] -eq 'CHANGE_ME') { - throw "Ungueltige .env: $key fehlt oder ist noch CHANGE_ME." - } +foreach ($key in @('POSTGRES_PASSWORD','CLICKHOUSE_PASSWORD','CLICKHOUSE_GRAFANA_PASSWORD','UI_PASSWORD','GRAFANA_ADMIN_PASSWORD','ENROLLMENT_KEY','GARAGE_ACCESS_KEY','GARAGE_SECRET_KEY','GARAGE_RPC_SECRET','GARAGE_ADMIN_TOKEN','GARAGE_METRICS_TOKEN')) { + if ([string]::IsNullOrWhiteSpace($current[$key]) -or $current[$key] -eq 'CHANGE_ME') { throw "Ungueltige .env: $key fehlt oder ist noch CHANGE_ME." } } Write-Host "Pruefe Compose-Konfiguration ..." docker compose config | Out-Null - Write-Host "Baue und starte Greenfield SIEM ..." docker compose build --pull docker compose up -d --remove-orphans @@ -81,29 +129,32 @@ docker compose up -d --remove-orphans Write-Host "Pruefe Readiness ..." $healthy = $false for ($i = 0; $i -lt 90; $i++) { - $apiOk = $false; $ingressOk = $false; $chOk = $false; $garageOk = $false - try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.UI_PORT)/readyz" | Out-Null; $apiOk = $true } catch {} - try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.INGRESS_PORT)/readyz" | Out-Null; $ingressOk = $true } catch {} - docker compose exec -T clickhouse clickhouse-client --user $current.CLICKHOUSE_USER --password $current.CLICKHOUSE_PASSWORD --query "SELECT 1" *> $null - if ($LASTEXITCODE -eq 0) { $chOk = $true } - docker compose exec -T garage /garage status *> $null - if ($LASTEXITCODE -eq 0) { $garageOk = $true } - if ($apiOk -and $ingressOk -and $chOk -and $garageOk) { $healthy = $true; break } + $apiOk=$false; $ingressOk=$false; $chOk=$false; $garageOk=$false; $grafanaOk=$false + try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.UI_PORT)/readyz" | Out-Null; $apiOk=$true } catch {} + try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.INGRESS_PORT)/readyz" | Out-Null; $ingressOk=$true } catch {} + try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.GRAFANA_PORT)/api/health" | Out-Null; $grafanaOk=$true } catch {} + & docker compose exec -T clickhouse clickhouse-client --user $current.CLICKHOUSE_USER --password $current.CLICKHOUSE_PASSWORD --query "SELECT 1" *> $null + if ($LASTEXITCODE -eq 0) { $chOk=$true } + & docker compose exec -T garage /garage status *> $null + if ($LASTEXITCODE -eq 0) { $garageOk=$true } + if ($apiOk -and $ingressOk -and $chOk -and $garageOk -and $grafanaOk) { $healthy=$true; break } Start-Sleep -Seconds 2 } if (-not $healthy) { Write-Host "Mindestens ein Dienst wurde nicht bereit. Diagnose:" -ForegroundColor Red docker compose ps - docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor api + docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor detector api grafana throw "Deployment nicht vollstaendig bereit." } Write-Host "" Write-Host "Greenfield SIEM laeuft." -Write-Host "UI: http://127.0.0.1:$($current.UI_PORT)/ui" -Write-Host "Ingress: http://127.0.0.1:$($current.INGRESS_PORT)/ingest" -Write-Host "UI-Login: $($current.UI_USERNAME) / $($current.UI_PASSWORD)" +Write-Host "Analyst UI: http://127.0.0.1:$($current.UI_PORT)/ui" +Write-Host "Ingress: http://127.0.0.1:$($current.INGRESS_PORT)/ingest" +Write-Host "Grafana: http://127.0.0.1:$($current.GRAFANA_PORT)/" +Write-Host "UI-Login: $($current.UI_USERNAME) / $($current.UI_PASSWORD)" +Write-Host "Grafana-Login: $($current.GRAFANA_ADMIN_USER) / $($current.GRAFANA_ADMIN_PASSWORD)" Write-Host "ClickHouse: user=$($current.CLICKHOUSE_USER) db=$($current.CLICKHOUSE_DB) http=127.0.0.1:$($current.CLICKHOUSE_HTTP_PORT) native=127.0.0.1:$($current.CLICKHOUSE_NATIVE_PORT)" Write-Host "Enrollment-Key: $($current.ENROLLMENT_KEY)" Write-Host "Alle Parameter stehen vollstaendig in .env." diff --git a/deploy.sh b/deploy.sh index 9050552..041e820 100755 --- a/deploy.sh +++ b/deploy.sh @@ -45,12 +45,12 @@ while IFS= read -r line || [ -n "$line" ]; do done < .env.example # Generate secrets only if absent/blank/placeholder. Existing installations keep their credentials. -for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD UI_PASSWORD GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do +for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD CLICKHOUSE_GRAFANA_PASSWORD UI_PASSWORD GRAFANA_ADMIN_PASSWORD GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do val="$(get_env "$key")" if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then case "$key" in - POSTGRES_PASSWORD|CLICKHOUSE_PASSWORD) val="$(randhex 24)" ;; - UI_PASSWORD) val="$(randhex 18)" ;; + POSTGRES_PASSWORD|CLICKHOUSE_PASSWORD|CLICKHOUSE_GRAFANA_PASSWORD) val="$(randhex 24)" ;; + UI_PASSWORD|GRAFANA_ADMIN_PASSWORD) val="$(randhex 18)" ;; *) val="$(randhex 32)" ;; esac set_env "$key" "$val" @@ -73,7 +73,7 @@ chmod 600 .env 2>/dev/null || true # Validate required settings without sourcing .env as shell code. Docker Compose parses # .env itself; this also keeps values with shell-special characters safe. -for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD UI_PASSWORD ENROLLMENT_KEY GARAGE_ACCESS_KEY GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do +for key in POSTGRES_PASSWORD CLICKHOUSE_PASSWORD CLICKHOUSE_GRAFANA_PASSWORD UI_PASSWORD GRAFANA_ADMIN_PASSWORD ENROLLMENT_KEY GARAGE_ACCESS_KEY GARAGE_SECRET_KEY GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN; do val="$(get_env "$key")" if [ -z "$val" ] || [ "$val" = "CHANGE_ME" ]; then echo "Ungültige .env: $key fehlt oder ist noch CHANGE_ME." >&2 @@ -92,6 +92,16 @@ UI_USERNAME_VALUE="$(get_env UI_USERNAME)"; [ -n "$UI_USERNAME_VALUE" ] || UI_US UI_PASSWORD_VALUE="$(get_env UI_PASSWORD)" ENROLLMENT_KEY_VALUE="$(get_env ENROLLMENT_KEY)" +echo "Prüfe Host-Kompatibilität und Portbelegung ..." +./preflight.sh + +# preflight may have changed ports or the ClickHouse image. Read final values again. +UI_PORT_VALUE="$(get_env UI_PORT)"; [ -n "$UI_PORT_VALUE" ] || UI_PORT_VALUE=8080 +INGRESS_PORT_VALUE="$(get_env INGRESS_PORT)"; [ -n "$INGRESS_PORT_VALUE" ] || INGRESS_PORT_VALUE=8090 +GRAFANA_PORT_VALUE="$(get_env GRAFANA_PORT)"; [ -n "$GRAFANA_PORT_VALUE" ] || GRAFANA_PORT_VALUE=3000 +CLICKHOUSE_HTTP_PORT_VALUE="$(get_env CLICKHOUSE_HTTP_PORT)"; [ -n "$CLICKHOUSE_HTTP_PORT_VALUE" ] || CLICKHOUSE_HTTP_PORT_VALUE=8123 +CLICKHOUSE_NATIVE_PORT_VALUE="$(get_env CLICKHOUSE_NATIVE_PORT)"; [ -n "$CLICKHOUSE_NATIVE_PORT_VALUE" ] || CLICKHOUSE_NATIVE_PORT_VALUE=9000 + # Validate the final config before pulling/building anything. echo "Prüfe Compose-Konfiguration ..." docker compose config >/dev/null @@ -110,12 +120,13 @@ http_ok() { echo "Prüfe Readiness ..." i=0 while [ "$i" -lt 90 ]; do - api_ok=0; ingress_ok=0; ch_ok=0; garage_ok=0 + api_ok=0; ingress_ok=0; ch_ok=0; garage_ok=0; grafana_ok=0 http_ok "http://127.0.0.1:${UI_PORT_VALUE}/readyz" && api_ok=1 || true http_ok "http://127.0.0.1:${INGRESS_PORT_VALUE}/readyz" && ingress_ok=1 || true docker compose exec -T clickhouse clickhouse-client --user "${CLICKHOUSE_USER_VALUE}" --password "${CLICKHOUSE_PASSWORD_VALUE}" --query 'SELECT 1' >/dev/null 2>&1 && ch_ok=1 || true docker compose exec -T garage /garage status >/dev/null 2>&1 && garage_ok=1 || true - if [ "$api_ok" -eq 1 ] && [ "$ingress_ok" -eq 1 ] && [ "$ch_ok" -eq 1 ] && [ "$garage_ok" -eq 1 ]; then + http_ok "http://127.0.0.1:${GRAFANA_PORT_VALUE}/api/health" && grafana_ok=1 || true + if [ "$api_ok" -eq 1 ] && [ "$ingress_ok" -eq 1 ] && [ "$ch_ok" -eq 1 ] && [ "$garage_ok" -eq 1 ] && [ "$grafana_ok" -eq 1 ]; then break fi i=$((i+1)) @@ -126,7 +137,7 @@ if [ "$i" -ge 90 ]; then echo "Mindestens ein Dienst wurde nicht bereit. Diagnose:" >&2 docker compose ps >&2 || true echo >&2 - docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor api >&2 || true + docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor detector api grafana >&2 || true exit 1 fi @@ -134,6 +145,8 @@ echo echo "Greenfield SIEM läuft." echo "UI: http://127.0.0.1:${UI_PORT_VALUE}/ui" echo "Ingress: http://127.0.0.1:${INGRESS_PORT_VALUE}/ingest" +echo "Grafana: http://127.0.0.1:${GRAFANA_PORT_VALUE}/" +echo "Grafana-Login: $(get_env GRAFANA_ADMIN_USER) / $(get_env GRAFANA_ADMIN_PASSWORD)" echo "UI-Login: ${UI_USERNAME_VALUE} / ${UI_PASSWORD_VALUE}" echo "ClickHouse: user=${CLICKHOUSE_USER_VALUE} db=${CLICKHOUSE_DB_VALUE} http=127.0.0.1:${CLICKHOUSE_HTTP_PORT_VALUE} native=127.0.0.1:${CLICKHOUSE_NATIVE_PORT_VALUE}" echo "Enrollment-Key: ${ENROLLMENT_KEY_VALUE}" diff --git a/deploy/clickhouse/init.sql.template b/deploy/clickhouse/init.sql.template index 0cc8890..bf39749 100644 --- a/deploy/clickhouse/init.sql.template +++ b/deploy/clickhouse/init.sql.template @@ -80,3 +80,14 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS __CLICKHOUSE_DB__.events_5m_mv TO __CLICK SELECT tenant_id, toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket, host_name, event_code, category, action, outcome, uniqExactState(event_uid) AS cnt_state FROM __CLICKHOUSE_DB__.events GROUP BY tenant_id, bucket, host_name, event_code, category, action, outcome; + +-- Dedicated Grafana account. It receives SELECT only. readonly=2 lets the +-- datasource client change query settings such as max_execution_time; ClickHouse itself +-- remains bound to localhost by default and the role has no write privileges. +CREATE ROLE IF NOT EXISTS siem_grafana_role; +ALTER ROLE siem_grafana_role SETTINGS readonly = 2, max_execution_time = 30, max_threads = 4, max_memory_usage = 2000000000; +GRANT SELECT ON __CLICKHOUSE_DB__.* TO siem_grafana_role; +CREATE USER IF NOT EXISTS grafana_reader IDENTIFIED WITH sha256_password BY '__GRAFANA_CLICKHOUSE_PASSWORD__'; +ALTER USER grafana_reader IDENTIFIED WITH sha256_password BY '__GRAFANA_CLICKHOUSE_PASSWORD__'; +GRANT siem_grafana_role TO grafana_reader; +ALTER USER grafana_reader DEFAULT ROLE siem_grafana_role; diff --git a/deploy/grafana/dashboards/pipeline-health.json b/deploy/grafana/dashboards/pipeline-health.json new file mode 100644 index 0000000..4adb480 --- /dev/null +++ b/deploy/grafana/dashboards/pipeline-health.json @@ -0,0 +1,461 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "id": 1, + "title": "Ingress erreichbar", + "type": "stat", + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "expr": "up{job=\"siem-ingress\"}", + "legendFormat": "", + "refId": "A", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 2, + "title": "Redpanda erreichbar", + "type": "stat", + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "expr": "up{job=\"redpanda\"}", + "legendFormat": "", + "refId": "A", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 3, + "title": "ClickHouse erreichbar", + "type": "stat", + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "expr": "up{job=\"clickhouse\"}", + "legendFormat": "", + "refId": "A", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 4, + "title": "Akzeptierte Events / s", + "type": "stat", + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "expr": "rate(siem_ingress_events_accepted_total[5m])", + "legendFormat": "", + "refId": "A", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 5, + "title": "Ingest-Batches / s", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "expr": "rate(siem_ingress_batches_total[5m])", + "legendFormat": "", + "refId": "A", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "eps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "title": "Abgewiesene Requests / s", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "expr": "rate(siem_ingress_requests_rejected_total[5m])", + "legendFormat": "", + "refId": "A", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "title": "ClickHouse Scrape-Latenz", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "expr": "scrape_duration_seconds{job=\"clickhouse\"}", + "legendFormat": "", + "refId": "A", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "title": "Redpanda Scrape-Latenz", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "siem-prometheus" + }, + "expr": "scrape_duration_seconds{job=\"redpanda\"}", + "legendFormat": "", + "refId": "A", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ], + "refresh": "15s", + "schemaVersion": 42, + "tags": [ + "siem", + "pipeline", + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "title": "SIEM Pipeline Health", + "uid": "siem-pipeline-health", + "version": 2 +} diff --git a/deploy/grafana/dashboards/security-overview.json b/deploy/grafana/dashboards/security-overview.json new file mode 100644 index 0000000..c0ce336 --- /dev/null +++ b/deploy/grafana/dashboards/security-overview.json @@ -0,0 +1,550 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "id": 1, + "title": "Events im Zeitraum", + "type": "stat", + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "table", + "rawSql": "SELECT uniqExact(event_uid) AS value FROM siem.events WHERE $__timeFilter(event_time)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 2, + "title": "Aktive Hosts", + "type": "stat", + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "table", + "rawSql": "SELECT uniqExact(host_name) AS value FROM siem.events WHERE $__timeFilter(event_time)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 3, + "title": "Failed Logons", + "type": "stat", + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "table", + "rawSql": "SELECT uniqExact(event_uid) AS value FROM siem.events WHERE $__timeFilter(event_time) AND event_code=4625", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 4, + "title": "Account Lockouts", + "type": "stat", + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "table", + "rawSql": "SELECT uniqExact(event_uid) AS value FROM siem.events WHERE $__timeFilter(event_time) AND event_code=4740", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 5, + "title": "Event-Volumen", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 4, + "w": 24, + "h": 8 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "timeseries", + "rawSql": "SELECT $__timeInterval(event_time) AS time, uniqExact(event_uid) AS events FROM siem.events WHERE $__timeFilter(event_time) GROUP BY time ORDER BY time", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "title": "Top Hosts", + "type": "table", + "gridPos": { + "x": 0, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "table", + "rawSql": "SELECT host_name AS Host, uniqExact(event_uid) AS Events FROM siem.events WHERE $__timeFilter(event_time) GROUP BY host_name ORDER BY Events DESC LIMIT 20", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm" + } + }, + { + "id": 7, + "title": "Top Event-IDs", + "type": "table", + "gridPos": { + "x": 12, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "table", + "rawSql": "SELECT event_code AS EventID, uniqExact(event_uid) AS Events FROM siem.events WHERE $__timeFilter(event_time) GROUP BY event_code ORDER BY Events DESC LIMIT 20", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm" + } + }, + { + "id": 8, + "title": "Authentication Events", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 20, + "w": 24, + "h": 8 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "timeseries", + "rawSql": "SELECT $__timeInterval(event_time) AS time, toString(event_code) AS event, uniqExact(event_uid) AS events FROM siem.events WHERE $__timeFilter(event_time) AND event_code IN (4624,4625,4740,4771,4776) GROUP BY time,event ORDER BY time", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "title": "Top fehlgeschlagene Benutzer", + "type": "table", + "gridPos": { + "x": 0, + "y": 28, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "table", + "rawSql": "SELECT target_user AS User, uniqExact(event_uid) AS Failures, uniqExact(source_ip) AS SourceIPs FROM siem.events WHERE $__timeFilter(event_time) AND event_code=4625 AND target_user!='' GROUP BY target_user ORDER BY Failures DESC LIMIT 25", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm" + } + }, + { + "id": 10, + "title": "Top Source-IPs bei 4625", + "type": "table", + "gridPos": { + "x": 12, + "y": 28, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "siem-clickhouse" + }, + "editorType": "sql", + "format": 0, + "queryType": "table", + "rawSql": "SELECT source_ip AS SourceIP, uniqExact(event_uid) AS Failures, uniqExact(target_user) AS Users FROM siem.events WHERE $__timeFilter(event_time) AND event_code=4625 AND source_ip!='' GROUP BY source_ip ORDER BY Failures DESC LIMIT 25", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm" + } + } + ], + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "siem", + "security", + "clickhouse" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "browser", + "title": "SIEM Security Overview", + "uid": "siem-security-overview", + "version": 1 +} \ No newline at end of file diff --git a/deploy/grafana/provisioning/dashboards/siem.yml b/deploy/grafana/provisioning/dashboards/siem.yml new file mode 100644 index 0000000..586b961 --- /dev/null +++ b/deploy/grafana/provisioning/dashboards/siem.yml @@ -0,0 +1,12 @@ +apiVersion: 1 +providers: + - name: Greenfield SIEM + orgId: 1 + folder: Greenfield SIEM + folderUid: greenfield-siem + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/deploy/grafana/provisioning/datasources/siem.yml b/deploy/grafana/provisioning/datasources/siem.yml new file mode 100644 index 0000000..eddc960 --- /dev/null +++ b/deploy/grafana/provisioning/datasources/siem.yml @@ -0,0 +1,29 @@ +apiVersion: 1 +prune: true +datasources: + - name: SIEM ClickHouse + uid: siem-clickhouse + type: grafana-clickhouse-datasource + access: proxy + isDefault: true + editable: false + jsonData: + host: clickhouse + port: 9000 + protocol: native + username: grafana_reader + defaultDatabase: $CLICKHOUSE_DB + defaultTable: events + queryTimeout: 30 + dialTimeout: 10 + validateSql: true + enableRowLimit: true + secureJsonData: + password: $CLICKHOUSE_GRAFANA_PASSWORD + - name: SIEM Prometheus + uid: siem-prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: false + editable: false diff --git a/deploy/postgres/init.sql b/deploy/postgres/init.sql index ac21565..d0c151c 100644 --- a/deploy/postgres/init.sql +++ b/deploy/postgres/init.sql @@ -1,4 +1,5 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; + CREATE TABLE IF NOT EXISTS agents ( id uuid PRIMARY KEY DEFAULT gen_random_uuid(), tenant_id text NOT NULL, @@ -12,10 +13,58 @@ CREATE TABLE IF NOT EXISTS agents ( ); CREATE INDEX IF NOT EXISTS agents_last_seen_idx ON agents(tenant_id,last_seen DESC); +CREATE TABLE IF NOT EXISTS rule_sets ( + tenant_id text NOT NULL, + id text NOT NULL, + name text NOT NULL, + description text NOT NULL DEFAULT '', + version integer NOT NULL DEFAULT 1, + enabled boolean NOT NULL DEFAULT true, + source text NOT NULL DEFAULT 'builtin', + locked boolean NOT NULL DEFAULT false, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (tenant_id,id) +); + +CREATE TABLE IF NOT EXISTS detection_rules ( + tenant_id text NOT NULL, + id text NOT NULL, + rule_set_id text NOT NULL, + title text NOT NULL, + severity text NOT NULL, + score double precision NOT NULL DEFAULT 0, + enabled boolean NOT NULL DEFAULT true, + source text NOT NULL DEFAULT 'builtin', + definition jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (tenant_id,id), + FOREIGN KEY (tenant_id,rule_set_id) REFERENCES rule_sets(tenant_id,id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS detection_rules_set_idx ON detection_rules(tenant_id,rule_set_id,enabled); +CREATE INDEX IF NOT EXISTS detection_rules_severity_idx ON detection_rules(tenant_id,severity,enabled); + +CREATE TABLE IF NOT EXISTS detection_suppressions ( + id bigserial PRIMARY KEY, + tenant_id text NOT NULL, + rule_id text NOT NULL DEFAULT '', + host_pattern text NOT NULL DEFAULT '', + user_pattern text NOT NULL DEFAULT '', + source_ip_pattern text NOT NULL DEFAULT '', + reason text NOT NULL DEFAULT '', + enabled boolean NOT NULL DEFAULT true, + expires_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS detection_suppressions_active_idx ON detection_suppressions(tenant_id,enabled,expires_at); + CREATE TABLE IF NOT EXISTS detections ( id bigserial PRIMARY KEY, tenant_id text NOT NULL, fingerprint char(64) NOT NULL, + rule_id text NOT NULL DEFAULT '', + rule_set_id text NOT NULL DEFAULT '', rule_name text NOT NULL, severity text NOT NULL, status text NOT NULL DEFAULT 'open', @@ -29,10 +78,17 @@ CREATE TABLE IF NOT EXISTS detections ( window_end timestamptz NOT NULL, summary text NOT NULL, hit_count bigint NOT NULL DEFAULT 1, + tags jsonb NOT NULL DEFAULT '[]'::jsonb, + mitre jsonb NOT NULL DEFAULT '[]'::jsonb, first_seen timestamptz NOT NULL DEFAULT now(), last_seen timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now(), UNIQUE (tenant_id, fingerprint) ); +ALTER TABLE detections ADD COLUMN IF NOT EXISTS rule_id text NOT NULL DEFAULT ''; +ALTER TABLE detections ADD COLUMN IF NOT EXISTS rule_set_id text NOT NULL DEFAULT ''; +ALTER TABLE detections ADD COLUMN IF NOT EXISTS tags jsonb NOT NULL DEFAULT '[]'::jsonb; +ALTER TABLE detections ADD COLUMN IF NOT EXISTS mitre jsonb NOT NULL DEFAULT '[]'::jsonb; CREATE INDEX IF NOT EXISTS detections_open_idx ON detections(tenant_id,status,last_seen DESC); CREATE INDEX IF NOT EXISTS detections_rule_idx ON detections(tenant_id,rule_name,last_seen DESC); +CREATE INDEX IF NOT EXISTS detections_rule_id_idx ON detections(tenant_id,rule_id,last_seen DESC); diff --git a/deploy/rules/windows-account-admin.json b/deploy/rules/windows-account-admin.json new file mode 100644 index 0000000..ad9065b --- /dev/null +++ b/deploy/rules/windows-account-admin.json @@ -0,0 +1,99 @@ +{ + "id": "windows-account-admin", + "name": "Windows Account & Privilege Management", + "description": "Konten, Gruppen und privilegierte Änderungen.", + "version": 1, + "enabled": true, + "rules": [ + { + "id": "win-privileged-group-change", + "title": "Privilegierte Gruppenmitgliedschaft geändert", + "description": "Mitglied zu privilegierter lokaler, globaler oder universeller Gruppe hinzugefügt.", + "severity": "critical", + "score": 9.2, + "enabled": true, + "kind": "event", + "channels": ["Security"], + "event_codes": [4728, 4732, 4756], + "group_by": ["host", "user", "workstation"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Privilegierte Gruppenmitgliedschaft geändert: {user} auf {host}", + "tags": ["windows", "privilege", "account-management"], + "mitre": ["T1098"] + }, + { + "id": "win-user-created", + "title": "Benutzerkonto erstellt", + "description": "Windows Security Event 4720.", + "severity": "medium", + "score": 6.0, + "enabled": true, + "kind": "event", + "channels": ["Security"], + "event_codes": [4720], + "group_by": ["host", "user"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Benutzerkonto {user} auf {host} erstellt", + "tags": ["windows", "account-management"], + "mitre": ["T1136.001"] + }, + { + "id": "win-user-enabled", + "title": "Benutzerkonto aktiviert", + "description": "Windows Security Event 4722.", + "severity": "medium", + "score": 5.5, + "enabled": true, + "kind": "event", + "channels": ["Security"], + "event_codes": [4722], + "group_by": ["host", "user"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Benutzerkonto {user} auf {host} aktiviert", + "tags": ["windows", "account-management"], + "mitre": [] + }, + { + "id": "win-password-reset", + "title": "Passwort eines Kontos zurückgesetzt", + "description": "Windows Security Event 4724.", + "severity": "medium", + "score": 6.2, + "enabled": true, + "kind": "event", + "channels": ["Security"], + "event_codes": [4724], + "group_by": ["host", "user"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Passwort für {user} auf {host} zurückgesetzt", + "tags": ["windows", "account-management", "credential"], + "mitre": ["T1098"] + }, + { + "id": "win-user-deleted", + "title": "Benutzerkonto gelöscht", + "description": "Windows Security Event 4726.", + "severity": "medium", + "score": 5.8, + "enabled": true, + "kind": "event", + "channels": ["Security"], + "event_codes": [4726], + "group_by": ["host", "user"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Benutzerkonto {user} auf {host} gelöscht", + "tags": ["windows", "account-management"], + "mitre": [] + } + ] +} diff --git a/deploy/rules/windows-authentication.json b/deploy/rules/windows-authentication.json new file mode 100644 index 0000000..fa3c299 --- /dev/null +++ b/deploy/rules/windows-authentication.json @@ -0,0 +1,102 @@ +{ + "id": "windows-authentication", + "name": "Windows Authentication", + "description": "Anmelde-, Lockout-, Spray- und Kerberos/NTLM-Erkennungen.", + "version": 1, + "enabled": true, + "rules": [ + { + "id": "win-account-lockout", + "title": "Account Lockout", + "description": "Windows Security Event 4740.", + "severity": "medium", + "score": 5.5, + "enabled": true, + "kind": "event", + "channels": ["Security"], + "event_codes": [4740], + "group_by": ["host", "user", "workstation"], + "threshold": 1, + "window_seconds": 600, + "suppress_seconds": 900, + "summary": "Account-Lockout: {user}; Caller {workstation}; DC/Host {host} ({count}×)", + "tags": ["windows", "authentication", "lockout"], + "mitre": [] + }, + { + "id": "win-failed-logon-burst", + "title": "Viele fehlgeschlagene Anmeldungen", + "description": "Mindestens 20 Event-4625-Ereignisse für denselben Kontext in fünf Minuten.", + "severity": "high", + "score": 7.5, + "enabled": true, + "kind": "threshold", + "channels": ["Security"], + "event_codes": [4625], + "group_by": ["host", "user", "source_ip"], + "threshold": 20, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "{count} fehlgeschlagene Logons für {user} auf {host} von {source_ip}", + "tags": ["windows", "authentication", "brute-force"], + "mitre": ["T1110"] + }, + { + "id": "win-password-spray", + "title": "Password Spray", + "description": "Eine Source-IP versucht viele unterschiedliche Konten anzumelden.", + "severity": "high", + "score": 8.5, + "enabled": true, + "kind": "distinct", + "channels": ["Security"], + "event_codes": [4625], + "conditions": [{"field": "source_ip", "operator": "exists"}, {"field": "target_user", "operator": "exists"}], + "group_by": ["source_ip"], + "threshold": 20, + "distinct_field": "target_user", + "distinct_threshold": 10, + "window_seconds": 600, + "suppress_seconds": 1800, + "summary": "Password-Spray von {source_ip}: {count} Versuche gegen {distinct} Benutzer", + "tags": ["windows", "authentication", "password-spray"], + "mitre": ["T1110.003"] + }, + { + "id": "win-kerberos-preauth-burst", + "title": "Kerberos Pre-Auth Fehler-Burst", + "description": "Viele Event-4771-Ereignisse für denselben Benutzer oder Quellkontext.", + "severity": "high", + "score": 7.4, + "enabled": true, + "kind": "threshold", + "channels": ["Security"], + "event_codes": [4771], + "group_by": ["host", "user", "source_ip"], + "threshold": 15, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "{count} Kerberos Pre-Auth Fehler für {user} auf {host} von {source_ip}", + "tags": ["windows", "kerberos", "authentication"], + "mitre": ["T1110"] + }, + { + "id": "win-ntlm-auth-failure-burst", + "title": "NTLM Authentifizierungsfehler-Burst", + "description": "Viele Event-4776-Ereignisse im kurzen Zeitraum.", + "severity": "high", + "score": 7.2, + "enabled": true, + "kind": "threshold", + "channels": ["Security"], + "event_codes": [4776], + "group_by": ["host", "user", "workstation"], + "threshold": 15, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "{count} NTLM-Authentifizierungsfehler für {user} über {workstation}", + "tags": ["windows", "ntlm", "authentication"], + "mitre": ["T1110"] + } + ] +} diff --git a/deploy/rules/windows-core.json b/deploy/rules/windows-core.json new file mode 100644 index 0000000..b5c76da --- /dev/null +++ b/deploy/rules/windows-core.json @@ -0,0 +1,98 @@ +{ + "id": "windows-core", + "name": "Windows Core Security", + "description": "Hochwertige Windows-System- und Audit-Ereignisse mit geringer Grundlautstärke.", + "version": 1, + "enabled": true, + "rules": [ + { + "id": "win-audit-log-cleared", + "title": "Security Audit Log gelöscht", + "description": "Windows Security Event 1102.", + "severity": "critical", + "score": 9.8, + "enabled": true, + "kind": "event", + "channels": ["Security"], + "event_codes": [1102], + "group_by": ["host"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Security Audit Log auf {host} wurde gelöscht", + "tags": ["windows", "audit", "defense-evasion"], + "mitre": ["T1070.001"] + }, + { + "id": "win-service-installed", + "title": "Neuer Windows-Dienst installiert", + "description": "Service Control Manager Event 7045.", + "severity": "high", + "score": 8.0, + "enabled": true, + "kind": "event", + "event_codes": [7045], + "group_by": ["host", "process_path"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Neuer Dienst auf {host} installiert: {process}", + "tags": ["windows", "persistence", "service"], + "mitre": ["T1543.003"] + }, + { + "id": "win-scheduled-task-created", + "title": "Scheduled Task erstellt", + "description": "Windows Security Event 4698.", + "severity": "high", + "score": 7.8, + "enabled": true, + "kind": "event", + "channels": ["Security"], + "event_codes": [4698], + "group_by": ["host", "user"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Scheduled Task auf {host} durch {user} erstellt", + "tags": ["windows", "persistence", "scheduled-task"], + "mitre": ["T1053.005"] + }, + { + "id": "win-audit-policy-changed", + "title": "Audit Policy geändert", + "description": "Windows Security Event 4719.", + "severity": "high", + "score": 8.2, + "enabled": true, + "kind": "event", + "channels": ["Security"], + "event_codes": [4719], + "group_by": ["host", "user"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Audit Policy auf {host} durch {user} geändert", + "tags": ["windows", "audit", "policy"], + "mitre": ["T1562.002"] + }, + { + "id": "win-firewall-rule-change", + "title": "Windows Firewall-Regel geändert", + "description": "Firewall-Regeln hinzugefügt, geändert oder gelöscht.", + "severity": "medium", + "score": 5.5, + "enabled": true, + "kind": "threshold", + "channels": ["Security"], + "event_codes": [4946, 4947, 4948], + "group_by": ["host", "user"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Windows Firewall-Regel auf {host} geändert ({count} Ereignisse)", + "tags": ["windows", "firewall", "configuration"], + "mitre": ["T1562.004"] + } + ] +} diff --git a/deploy/rules/windows-defender-powershell.json b/deploy/rules/windows-defender-powershell.json new file mode 100644 index 0000000..3ef6cd8 --- /dev/null +++ b/deploy/rules/windows-defender-powershell.json @@ -0,0 +1,81 @@ +{ + "id": "windows-defender-powershell", + "name": "Windows Defender & PowerShell", + "description": "Malware-, Defender- und auffällige PowerShell-Signale.", + "version": 1, + "enabled": true, + "rules": [ + { + "id": "defender-malware-detected", + "title": "Microsoft Defender Malware erkannt", + "description": "Defender Operational Event 1116.", + "severity": "high", + "score": 8.8, + "enabled": true, + "kind": "event", + "event_codes": [1116], + "conditions": [{"field": "provider", "operator": "contains", "value": "Defender"}], + "group_by": ["host"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 1800, + "summary": "Microsoft Defender meldet Malware auf {host}", + "tags": ["windows", "defender", "malware"], + "mitre": [] + }, + { + "id": "defender-realtime-protection-disabled", + "title": "Defender Echtzeitschutz deaktiviert", + "description": "Defender Operational Event 5001.", + "severity": "critical", + "score": 9.0, + "enabled": true, + "kind": "event", + "event_codes": [5001], + "conditions": [{"field": "provider", "operator": "contains", "value": "Defender"}], + "group_by": ["host"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 1800, + "summary": "Microsoft Defender Echtzeitschutz auf {host} deaktiviert", + "tags": ["windows", "defender", "defense-evasion"], + "mitre": ["T1562.001"] + }, + { + "id": "powershell-encoded-command", + "title": "PowerShell EncodedCommand", + "description": "PowerShell-Nachricht oder Kommandozeile enthält EncodedCommand.", + "severity": "high", + "score": 8.0, + "enabled": true, + "kind": "event", + "event_codes": [4104, 4688], + "conditions": [{"field": "message", "operator": "contains", "value": "EncodedCommand"}], + "group_by": ["host", "user"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "PowerShell EncodedCommand auf {host} durch {user}", + "tags": ["windows", "powershell", "execution"], + "mitre": ["T1059.001", "T1027"] + }, + { + "id": "powershell-download-cradle", + "title": "PowerShell Download-/Execution-Muster", + "description": "PowerShell ScriptBlock enthält typische Download- oder In-Memory-Execution-Muster.", + "severity": "high", + "score": 8.4, + "enabled": true, + "kind": "event", + "event_codes": [4104], + "conditions": [{"field": "message", "operator": "regex", "value": "(?i)(DownloadString|DownloadFile|Invoke-WebRequest|FromBase64String|IEX\\s*\\()"}], + "group_by": ["host", "user"], + "threshold": 1, + "window_seconds": 300, + "suppress_seconds": 900, + "summary": "Auffälliges PowerShell Download-/Execution-Muster auf {host} durch {user}", + "tags": ["windows", "powershell", "execution"], + "mitre": ["T1059.001", "T1105"] + } + ] +} diff --git a/doctor.sh b/doctor.sh index ca8e139..7c25769 100755 --- a/doctor.sh +++ b/doctor.sh @@ -5,6 +5,13 @@ cd "$(dirname "$0")" section(){ printf '\n== %s ==\n' "$1"; } +section "Host / CPU" +echo "Architektur: $(uname -m 2>/dev/null || echo unknown)" +if docker compose ps clickhouse 2>/dev/null | grep -q "Restarting"; then + echo "WARNUNG: ClickHouse startet wiederholt neu." + docker compose logs --tail=30 clickhouse 2>&1 | grep -q "Illegal instruction" && echo "URSACHE: ClickHouse SIGILL / CPU-Instruktionssatz inkompatibel. ./host-info.sh ausführen." || true +fi + section "Compose config" docker compose config >/dev/null 2>&1 && echo "OK" || echo "FEHLER: docker compose config" @@ -15,6 +22,7 @@ section "HTTP readiness" if command -v curl >/dev/null 2>&1; then printf "Ingress: "; curl -fsS "http://127.0.0.1:${INGRESS_PORT:-8090}/readyz" || true; echo printf "API: "; curl -fsS "http://127.0.0.1:${UI_PORT:-8080}/readyz" || true; echo + printf "Grafana: "; curl -fsS "http://127.0.0.1:${GRAFANA_PORT:-3000}/api/health" || true; echo else echo "curl fehlt; HTTP-Prüfung übersprungen." fi @@ -39,11 +47,11 @@ docker compose exec -T redpanda rpk group describe "${KAFKA_GROUP:-siem-processo section "PostgreSQL control plane" docker compose exec -T postgres psql -U "${POSTGRES_USER:-siem}" -d "${POSTGRES_DB:-siem}" -Atc \ - "SELECT 'agents='||count(*) FROM agents; SELECT 'open_detections='||count(*) FROM detections WHERE status IN ('open','investigating');" 2>/dev/null || true + "SELECT 'agents='||count(*) FROM agents; SELECT 'rule_sets='||count(*) FROM rule_sets; SELECT 'enabled_rules='||count(*) FROM detection_rules WHERE enabled; SELECT 'suppressions='||count(*) FROM detection_suppressions WHERE enabled AND (expires_at IS NULL OR expires_at>now()); SELECT 'open_detections='||count(*) FROM detections WHERE status IN ('open','investigating');" 2>/dev/null || true section "Raw spool" docker compose exec -T archive-uploader sh -c 'du -sh /spool 2>/dev/null || true; printf "files="; find /spool -type f 2>/dev/null | wc -l' 2>/dev/null || true section "Recent errors" -docker compose logs --since=15m garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor detector api archive-uploader 2>&1 \ +docker compose logs --since=15m garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor detector api archive-uploader grafana 2>&1 \ | grep -Ei 'error|fatal|failed|panic|not_ready|exception' | tail -120 || true diff --git a/go.mod b/go.mod index 9d68300..a326dd4 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,3 @@ require ( github.com/jackc/pgx/v5 v5.9.2 github.com/segmentio/kafka-go v0.4.51 ) - -require ( - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/text v0.29.0 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 54a764e..0000000 --- a/go.sum +++ /dev/null @@ -1,40 +0,0 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= -github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= -github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= -github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/segmentio/kafka-go v0.4.51 h1:JgDPPG75tC1rWIS2Me6MwcvXJ6f49UQ4HjAOef71Hno= -github.com/segmentio/kafka-go v0.4.51/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/host-info.sh b/host-info.sh new file mode 100644 index 0000000..da06493 --- /dev/null +++ b/host-info.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env sh +set -u +cd "$(dirname "$0")" +[ -f .env ] && { set -a; . ./.env; set +a; } + +echo '== Host ==' +uname -a || true +command -v lscpu >/dev/null 2>&1 && lscpu || true + +echo +echo '== CPU flags/features ==' +if [ -r /proc/cpuinfo ]; then + case "$(uname -m 2>/dev/null)" in + x86_64|amd64) grep -m1 -E '^flags[[:space:]]*:' /proc/cpuinfo || true ;; + aarch64|arm64) grep -m1 -E '^Features[[:space:]]*:' /proc/cpuinfo || true ;; + esac +fi + +echo +echo '== Docker ==' +docker version 2>/dev/null || true +docker info --format 'Architecture={{.Architecture}} OS={{.OSType}} Kernel={{.KernelVersion}}' 2>/dev/null || true + +echo +echo '== Published ports ==' +docker ps --format 'table {{.Names}}\t{{.Ports}}' || true + +echo +echo '== ClickHouse direct CPU probe ==' +image="${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81}" +echo "Image: $image" +set +e +docker run --rm --entrypoint clickhouse "$image" local --query 'SELECT version(), 1' +rc=$? +set -e +echo "Exit-Code: $rc" +[ "$rc" -eq 132 ] && echo 'Exit 132 = SIGILL / Illegal instruction: CPU-Instruktionssatz passt nicht zum Binary.' diff --git a/internal/api/server.go b/internal/api/server.go index d919cbc..01ecaaf 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -15,6 +15,7 @@ import ( "example.com/siem-greenfield/internal/clickhouse" "example.com/siem-greenfield/internal/config" "example.com/siem-greenfield/internal/postgres" + "example.com/siem-greenfield/internal/rules" ) func Run(ctx context.Context, cfg config.Config) error { @@ -31,20 +32,20 @@ func Run(ctx context.Context, cfg config.Config) error { mux := http.NewServeMux() mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { j(w, 200, map[string]string{"status": "ok"}) }) mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) { - cctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + c, cancel := context.WithTimeout(r.Context(), 3*time.Second) defer cancel() - if e := pg.Pool.Ping(cctx); e != nil { + if e := pg.Pool.Ping(c); e != nil { j(w, 503, map[string]string{"status": "not_ready", "component": "postgres"}) return } - if e := ch.Exec(cctx, "SELECT 1"); e != nil { + if e := ch.Exec(c, "SELECT 1"); e != nil { j(w, 503, map[string]string{"status": "not_ready", "component": "clickhouse"}) return } j(w, 200, map[string]string{"status": "ready"}) }) mux.HandleFunc("/ui", func(w http.ResponseWriter, r *http.Request) { - _ = tpl.Execute(w, map[string]string{"Tenant": cfg.TenantID}) + _ = tpl.Execute(w, map[string]string{"Tenant": cfg.TenantID, "GrafanaPort": cfg.GrafanaPort, "GrafanaURL": cfg.GrafanaPublicURL}) }) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/" { @@ -53,19 +54,44 @@ func Run(ctx context.Context, cfg config.Config) error { } http.NotFound(w, r) }) + mux.HandleFunc("/api/summary", func(w http.ResponseWriter, r *http.Request) { summary(w, r, cfg, pg, ch) }) + mux.HandleFunc("/api/analytics", func(w http.ResponseWriter, r *http.Request) { analytics(w, r, cfg, ch) }) mux.HandleFunc("/api/events", func(w http.ResponseWriter, r *http.Request) { events(w, r, cfg, ch) }) mux.HandleFunc("/api/detections", func(w http.ResponseWriter, r *http.Request) { detections(w, r, cfg, pg) }) mux.HandleFunc("/api/detections/status", func(w http.ResponseWriter, r *http.Request) { detectionStatus(w, r, cfg, pg) }) mux.HandleFunc("/api/agents", func(w http.ResponseWriter, r *http.Request) { x, e := pg.ListAgents(r.Context(), cfg.TenantID) if e != nil { - j(w, 500, map[string]string{"error": e.Error()}) + j(w, 500, errj(e)) return } j(w, 200, x) }) - srv := &http.Server{Addr: cfg.ServiceAddr, Handler: security(basicAuth(mux, cfg.UIUsername, cfg.UIPassword)), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 30 * time.Second, IdleTimeout: 60 * time.Second} + mux.HandleFunc("/api/agents/toggle", func(w http.ResponseWriter, r *http.Request) { agentToggle(w, r, cfg, pg) }) + mux.HandleFunc("/api/rule-sets", func(w http.ResponseWriter, r *http.Request) { + x, e := pg.ListRuleSets(r.Context(), cfg.TenantID) + if e != nil { + j(w, 500, errj(e)) + return + } + j(w, 200, x) + }) + mux.HandleFunc("/api/rule-sets/toggle", func(w http.ResponseWriter, r *http.Request) { ruleSetToggle(w, r, cfg, pg) }) + mux.HandleFunc("/api/rules", func(w http.ResponseWriter, r *http.Request) { + x, e := pg.ListRules(r.Context(), cfg.TenantID, false) + if e != nil { + j(w, 500, errj(e)) + return + } + j(w, 200, x) + }) + mux.HandleFunc("/api/rules/toggle", func(w http.ResponseWriter, r *http.Request) { ruleToggle(w, r, cfg, pg) }) + mux.HandleFunc("/api/rules/save", func(w http.ResponseWriter, r *http.Request) { ruleSave(w, r, cfg, pg) }) + mux.HandleFunc("/api/suppressions", func(w http.ResponseWriter, r *http.Request) { suppressions(w, r, cfg, pg) }) + mux.HandleFunc("/api/suppressions/delete", func(w http.ResponseWriter, r *http.Request) { suppressionDelete(w, r, cfg, pg) }) + + srv := &http.Server{Addr: cfg.ServiceAddr, Handler: security(basicAuth(mux, cfg.UIUsername, cfg.UIPassword)), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 20 * time.Second, WriteTimeout: 35 * time.Second, IdleTimeout: 60 * time.Second} go func() { <-ctx.Done() c, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -79,13 +105,14 @@ func Run(ctx context.Context, cfg config.Config) error { } return e } + func summary(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) { ctx, cancel := context.WithTimeout(r.Context(), 8*time.Second) defer cancel() q := fmt.Sprintf(`SELECT uniqExactMerge(cnt_state) events_24h, uniqExact(host_name) active_hosts FROM %s.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL 24 HOUR`, clickhouse.Ident(cfg.ClickHouseDB), clickhouse.Q(cfg.TenantID)) rows, e := ch.QueryJSON(ctx, q) if e != nil { - j(w, 500, map[string]string{"error": e.Error()}) + j(w, 500, errj(e)) return } out := map[string]any{"events_24h": 0, "active_hosts": 0} @@ -96,65 +123,237 @@ func summary(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *post } dc, _ := pg.DetectionCounts(ctx, cfg.TenantID) out["detections"] = dc + rs, _ := pg.ListRuleSets(ctx, cfg.TenantID) + var enabledRules int64 + var setsOn int64 + for _, x := range rs { + if x.Enabled { + setsOn++ + enabledRules += x.EnabledRules + } + } + out["rule_sets_enabled"] = setsOn + out["rules_enabled"] = enabledRules j(w, 200, out) } + +func analytics(w http.ResponseWriter, r *http.Request, cfg config.Config, ch *clickhouse.Client) { + hours := queryInt(r, "hours", 24, 1, 2160) + db := clickhouse.Ident(cfg.ClickHouseDB) + tenant := clickhouse.Q(cfg.TenantID) + ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second) + defer cancel() + series, e := ch.QueryJSON(ctx, fmt.Sprintf(`SELECT bucket,uniqExactMerge(cnt_state) events FROM %s.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL %d HOUR GROUP BY bucket ORDER BY bucket`, db, tenant, hours)) + if e != nil { + j(w, 500, errj(e)) + return + } + hosts, e := ch.QueryJSON(ctx, fmt.Sprintf(`SELECT host_name,uniqExactMerge(cnt_state) events FROM %s.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL %d HOUR GROUP BY host_name ORDER BY events DESC LIMIT 10`, db, tenant, hours)) + if e != nil { + j(w, 500, errj(e)) + return + } + codes, e := ch.QueryJSON(ctx, fmt.Sprintf(`SELECT event_code,uniqExactMerge(cnt_state) events FROM %s.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL %d HOUR GROUP BY event_code ORDER BY events DESC LIMIT 10`, db, tenant, hours)) + if e != nil { + j(w, 500, errj(e)) + return + } + auth, e := ch.QueryJSON(ctx, fmt.Sprintf(`SELECT event_code,uniqExactMerge(cnt_state) events FROM %s.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL %d HOUR AND event_code IN (4624,4625,4740,4771,4776) GROUP BY event_code ORDER BY event_code`, db, tenant, hours)) + if e != nil { + j(w, 500, errj(e)) + return + } + j(w, 200, map[string]any{"series": series, "top_hosts": hosts, "top_event_codes": codes, "authentication": auth}) +} + func events(w http.ResponseWriter, r *http.Request, cfg config.Config, ch *clickhouse.Client) { - limit := cfg.UIQueryLimit - if n, e := strconv.Atoi(r.URL.Query().Get("limit")); e == nil && n > 0 && n <= 2000 { - limit = n - } - hours := 24 - if n, e := strconv.Atoi(r.URL.Query().Get("hours")); e == nil && n > 0 && n <= 2160 { - hours = n - } + limit := queryInt(r, "limit", cfg.UIQueryLimit, 1, 2000) + hours := queryInt(r, "hours", 24, 1, 2160) where := []string{"tenant_id=" + clickhouse.Q(cfg.TenantID), fmt.Sprintf("event_time>=now()-INTERVAL %d HOUR", hours)} - for key, col := range map[string]string{"host": "host_name", "user": "user_name", "ip": "source_ip", "channel": "channel", "action": "action"} { + exact := map[string]string{"host": "host_name", "ip": "source_ip", "channel": "channel", "action": "action", "outcome": "outcome"} + for key, col := range exact { if v := strings.TrimSpace(r.URL.Query().Get(key)); v != "" { where = append(where, col+"="+clickhouse.Q(v)) } } + if v := strings.TrimSpace(r.URL.Query().Get("user")); v != "" { + q := clickhouse.Q(v) + where = append(where, "(user_name="+q+" OR target_user="+q+" OR subject_user="+q+")") + } if v := strings.TrimSpace(r.URL.Query().Get("event_code")); v != "" { if _, e := strconv.ParseUint(v, 10, 32); e == nil { where = append(where, "event_code="+v) } } - q := fmt.Sprintf(`SELECT event_uid,event_time,host_name,channel,event_code,category,action,outcome,severity,user_name,subject_user,target_user,source_ip,workstation,process_path,message,raw_object_key,raw_index FROM %s.events WHERE %s ORDER BY event_time DESC, ingest_time DESC LIMIT 1 BY event_uid LIMIT %d`, clickhouse.Ident(cfg.ClickHouseDB), strings.Join(where, " AND "), limit) - ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second) + if v := strings.TrimSpace(r.URL.Query().Get("query")); v != "" { + q := clickhouse.Q(v) + where = append(where, "(positionCaseInsensitiveUTF8(message,"+q+")>0 OR positionCaseInsensitiveUTF8(command_line,"+q+")>0 OR positionCaseInsensitiveUTF8(process_path,"+q+")>0)") + } + q := fmt.Sprintf(`SELECT event_uid,event_time,ingest_time,host_name,channel,provider,event_code,category,action,outcome,severity,user_name,subject_user,target_user,source_ip,destination_ip,workstation,logon_type,authentication_package,status_code,failure_reason,process_path,parent_process_path,command_line,message,attributes,raw_object_key,raw_index FROM %s.events WHERE %s ORDER BY event_time DESC,ingest_time DESC LIMIT 1 BY event_uid LIMIT %d`, clickhouse.Ident(cfg.ClickHouseDB), strings.Join(where, " AND "), limit) + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) defer cancel() rows, e := ch.QueryJSON(ctx, q) if e != nil { - j(w, 500, map[string]string{"error": e.Error()}) + j(w, 500, errj(e)) return } j(w, 200, rows) } func detections(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) { - x, e := pg.ListDetections(r.Context(), cfg.TenantID, 500) + x, e := pg.ListDetections(r.Context(), cfg.TenantID, queryInt(r, "limit", 500, 1, 1000)) if e != nil { - j(w, 500, map[string]string{"error": e.Error()}) + j(w, 500, errj(e)) return } j(w, 200, x) } func detectionStatus(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) { - if r.Method != http.MethodPost { - j(w, 405, map[string]string{"error": "method not allowed"}) + if !post(w, r) { return } var v struct { ID int64 `json:"id"` Status string `json:"status"` } - if e := json.NewDecoder(r.Body).Decode(&v); e != nil { - j(w, 400, map[string]string{"error": "invalid json"}) + if decode(w, r, &v) != nil { return } if e := pg.UpdateDetectionStatus(r.Context(), cfg.TenantID, v.ID, v.Status); e != nil { - j(w, 400, map[string]string{"error": e.Error()}) + j(w, 400, errj(e)) return } - j(w, 200, map[string]string{"status": "ok"}) + j(w, 200, okj()) } +func agentToggle(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) { + if !post(w, r) { + return + } + var v struct { + ID string `json:"id"` + Enabled bool `json:"enabled"` + } + if decode(w, r, &v) != nil { + return + } + if e := pg.SetAgentEnabled(r.Context(), cfg.TenantID, v.ID, v.Enabled); e != nil { + j(w, 400, errj(e)) + return + } + j(w, 200, okj()) +} +func ruleToggle(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) { + if !post(w, r) { + return + } + var v struct { + ID string `json:"id"` + Enabled bool `json:"enabled"` + } + if decode(w, r, &v) != nil { + return + } + if e := pg.SetRuleEnabled(r.Context(), cfg.TenantID, v.ID, v.Enabled); e != nil { + j(w, 400, errj(e)) + return + } + j(w, 200, okj()) +} +func ruleSetToggle(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) { + if !post(w, r) { + return + } + var v struct { + ID string `json:"id"` + Enabled bool `json:"enabled"` + } + if decode(w, r, &v) != nil { + return + } + if e := pg.SetRuleSetEnabled(r.Context(), cfg.TenantID, v.ID, v.Enabled); e != nil { + j(w, 400, errj(e)) + return + } + j(w, 200, okj()) +} +func ruleSave(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) { + if !post(w, r) { + return + } + var v rules.Rule + if decode(w, r, &v) != nil { + return + } + if e := pg.SaveCustomRule(r.Context(), cfg.TenantID, v); e != nil { + j(w, 400, errj(e)) + return + } + j(w, 200, okj()) +} +func suppressions(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) { + if r.Method == http.MethodGet { + x, e := pg.ListSuppressions(r.Context(), cfg.TenantID) + if e != nil { + j(w, 500, errj(e)) + return + } + j(w, 200, x) + return + } + if !post(w, r) { + return + } + var v postgres.Suppression + if decode(w, r, &v) != nil { + return + } + if e := pg.CreateSuppression(r.Context(), cfg.TenantID, v); e != nil { + j(w, 400, errj(e)) + return + } + j(w, 200, okj()) +} +func suppressionDelete(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) { + if !post(w, r) { + return + } + var v struct { + ID int64 `json:"id"` + } + if decode(w, r, &v) != nil { + return + } + if e := pg.DeleteSuppression(r.Context(), cfg.TenantID, v.ID); e != nil { + j(w, 400, errj(e)) + return + } + j(w, 200, okj()) +} + +func queryInt(r *http.Request, key string, def, min, max int) int { + n, e := strconv.Atoi(r.URL.Query().Get(key)) + if e != nil || n < min || n > max { + return def + } + return n +} +func decode(w http.ResponseWriter, r *http.Request, v any) error { + d := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)) + d.DisallowUnknownFields() + if e := d.Decode(v); e != nil { + j(w, 400, map[string]string{"error": "invalid json: " + e.Error()}) + return e + } + return nil +} +func post(w http.ResponseWriter, r *http.Request) bool { + if r.Method != http.MethodPost { + j(w, 405, map[string]string{"error": "method not allowed"}) + return false + } + return true +} +func errj(e error) map[string]string { return map[string]string{"error": e.Error()} } +func okj() map[string]string { return map[string]string{"status": "ok"} } func j(w http.ResponseWriter, s int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(s) @@ -175,7 +374,6 @@ func basicAuth(next http.Handler, user, pass string) http.Handler { next.ServeHTTP(w, r) }) } - func security(n http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-Content-Type-Options", "nosniff") diff --git a/internal/config/config.go b/internal/config/config.go index cee5c56..9d9f6d3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -29,6 +29,9 @@ type Config struct { UIQueryLimit int UIUsername string UIPassword string + RulesDir string + GrafanaPort string + GrafanaPublicURL string } func Load() Config { @@ -54,6 +57,9 @@ func Load() Config { UIQueryLimit: envInt("UI_QUERY_LIMIT", 500), UIUsername: env("UI_USERNAME", "admin"), UIPassword: env("UI_PASSWORD", "change-me"), + RulesDir: env("RULES_DIR", "/app/rules"), + GrafanaPort: env("GRAFANA_PORT", "3000"), + GrafanaPublicURL: env("GRAFANA_PUBLIC_URL", ""), } } diff --git a/internal/contracts/event.go b/internal/contracts/event.go index 24bcc4e..cafda74 100644 --- a/internal/contracts/event.go +++ b/internal/contracts/event.go @@ -47,48 +47,48 @@ type IngestEnvelope struct { } type CanonicalEvent struct { - EventUID string `json:"event_uid"` - QueuePartition int32 `json:"queue_partition"` - QueueOffset int64 `json:"queue_offset"` - TenantID string `json:"tenant_id"` - EventTime string `json:"event_time"` - IngestTime string `json:"ingest_time"` - AgentID string `json:"agent_id"` - HostName string `json:"host_name"` - SourceType string `json:"source_type"` - Channel string `json:"channel"` - Provider string `json:"provider"` - EventCode uint32 `json:"event_code"` - Category string `json:"category"` - Action string `json:"action"` - Outcome string `json:"outcome"` - Severity uint8 `json:"severity"` - UserName string `json:"user_name"` - UserDomain string `json:"user_domain"` - SubjectUser string `json:"subject_user"` - SubjectDomain string `json:"subject_domain"` - TargetUser string `json:"target_user"` - TargetDomain string `json:"target_domain"` - SourceIP string `json:"source_ip"` - SourcePort uint16 `json:"source_port"` - DestinationIP string `json:"destination_ip"` - DestinationPort uint16 `json:"destination_port"` - Workstation string `json:"workstation"` - LogonType string `json:"logon_type"` - AuthenticationPackage string `json:"authentication_package"` - LogonProcess string `json:"logon_process"` - StatusCode string `json:"status_code"` - SubStatusCode string `json:"sub_status_code"` - FailureReason string `json:"failure_reason"` - ProcessPath string `json:"process_path"` - ParentProcessPath string `json:"parent_process_path"` - CommandLine string `json:"command_line"` - Message string `json:"message"` - Attributes map[string]string `json:"attributes"` - RawObjectKey string `json:"raw_object_key"` - RawIndex uint32 `json:"raw_index"` - PayloadHash string `json:"payload_hash"` - SchemaVersion uint16 `json:"schema_version"` - ParserVersion uint16 `json:"parser_version"` - IngestDelayMS int64 `json:"ingest_delay_ms"` + EventUID string `json:"event_uid"` + QueuePartition int32 `json:"queue_partition"` + QueueOffset int64 `json:"queue_offset"` + TenantID string `json:"tenant_id"` + EventTime string `json:"event_time"` + IngestTime string `json:"ingest_time"` + AgentID string `json:"agent_id"` + HostName string `json:"host_name"` + SourceType string `json:"source_type"` + Channel string `json:"channel"` + Provider string `json:"provider"` + EventCode uint32 `json:"event_code"` + Category string `json:"category"` + Action string `json:"action"` + Outcome string `json:"outcome"` + Severity uint8 `json:"severity"` + UserName string `json:"user_name"` + UserDomain string `json:"user_domain"` + SubjectUser string `json:"subject_user"` + SubjectDomain string `json:"subject_domain"` + TargetUser string `json:"target_user"` + TargetDomain string `json:"target_domain"` + SourceIP string `json:"source_ip"` + SourcePort uint16 `json:"source_port"` + DestinationIP string `json:"destination_ip"` + DestinationPort uint16 `json:"destination_port"` + Workstation string `json:"workstation"` + LogonType string `json:"logon_type"` + AuthenticationPackage string `json:"authentication_package"` + LogonProcess string `json:"logon_process"` + StatusCode string `json:"status_code"` + SubStatusCode string `json:"sub_status_code"` + FailureReason string `json:"failure_reason"` + ProcessPath string `json:"process_path"` + ParentProcessPath string `json:"parent_process_path"` + CommandLine string `json:"command_line"` + Message string `json:"message"` + Attributes map[string]string `json:"attributes"` + RawObjectKey string `json:"raw_object_key"` + RawIndex uint32 `json:"raw_index"` + PayloadHash string `json:"payload_hash"` + SchemaVersion uint16 `json:"schema_version"` + ParserVersion uint16 `json:"parser_version"` + IngestDelayMS int64 `json:"ingest_delay_ms"` } diff --git a/internal/detector/detector.go b/internal/detector/detector.go index c7bb7be..df0b774 100644 --- a/internal/detector/detector.go +++ b/internal/detector/detector.go @@ -13,22 +13,21 @@ import ( "example.com/siem-greenfield/internal/clickhouse" "example.com/siem-greenfield/internal/config" "example.com/siem-greenfield/internal/postgres" + "example.com/siem-greenfield/internal/rules" ) -type rule struct { - name, severity string - eventCode uint32 - score float64 - query func(time.Time, time.Time, string) string - summary func(map[string]any) string -} - func Run(ctx context.Context, cfg config.Config) error { pg, e := postgres.Open(ctx, cfg.PostgresURL) if e != nil { return e } defer pg.Close() + if e := syncBuiltins(ctx, cfg, pg); e != nil { + return fmt.Errorf("sync built-in rules: %w", e) + } + if e := pg.EnsureCustomRuleSet(ctx, cfg.TenantID); e != nil { + return e + } ch := clickhouse.New(cfg) ticker := time.NewTicker(cfg.DetectorInterval) defer ticker.Stop() @@ -47,69 +46,79 @@ func Run(ctx context.Context, cfg config.Config) error { } } } + +func syncBuiltins(ctx context.Context, cfg config.Config, pg *postgres.Store) error { + sets, e := rules.LoadDir(cfg.RulesDir) + if e != nil { + return e + } + keep := make([]string, 0, len(sets)) + for _, rs := range sets { + keep = append(keep, rs.ID) + if e := pg.SyncRuleSet(ctx, cfg.TenantID, rs); e != nil { + return fmt.Errorf("%s: %w", rs.ID, e) + } + } + if e := pg.PruneBuiltinRuleSets(ctx, cfg.TenantID, keep); e != nil { + return fmt.Errorf("prune built-in rule sets: %w", e) + } + log.Printf("rule engine: synchronized %d built-in rule sets", len(sets)) + return nil +} + func runAll(ctx context.Context, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) error { + enabled, e := pg.ListRules(ctx, cfg.TenantID, true) + if e != nil { + return e + } end := time.Now().UTC() - start := end.Add(-cfg.DetectorLookback) - for _, r := range rules(cfg.ClickHouseDB) { - q := r.query(start, end, cfg.TenantID) - rows, e := ch.QueryJSON(ctx, q) + for _, sr := range enabled { + q, _, e := rules.Compile(sr.Rule, cfg.ClickHouseDB, cfg.TenantID, end) if e != nil { - return fmt.Errorf("%s: %w", r.name, e) + log.Printf("rule %s compile: %v", sr.ID, e) + continue + } + qctx, cancel := context.WithTimeout(ctx, 20*time.Second) + rows, e := ch.QueryJSON(qctx, q) + cancel() + if e != nil { + log.Printf("rule %s query: %v", sr.ID, e) + continue } for _, row := range rows { host := str(row["host_name"]) user := str(row["user_name"]) ip := str(row["source_ip"]) workstation := str(row["workstation"]) - count := int64(num(row["cnt"])) - ws := timeVal(row["window_start"], start) + ws := timeVal(row["window_start"], end.Add(-time.Duration(sr.WindowSeconds)*time.Second)) we := timeVal(row["window_end"], end) - fp := fingerprint(r.name, host, user, ip, workstation, strconv.FormatInt(ws.Unix()/300, 10)) - d := postgres.Detection{Fingerprint: fp, RuleName: r.name, Severity: r.severity, Hostname: host, UserName: user, SourceIP: ip, Workstation: workstation, EventCode: r.eventCode, Score: r.score, WindowStart: ws, WindowEnd: we, Summary: r.summary(row), Count: max64(1, count)} + suppressed, e := pg.IsSuppressed(ctx, cfg.TenantID, sr.ID, host, user, ip, we) + if e != nil { + log.Printf("rule %s suppression: %v", sr.ID, e) + continue + } + if suppressed { + continue + } + bucket := sr.SuppressSeconds + if bucket <= 0 { + bucket = sr.WindowSeconds + } + if bucket < 60 { + bucket = 60 + } + fp := fingerprint(sr.ID, host, user, ip, workstation, strconv.FormatInt(ws.Unix()/int64(bucket), 10)) + count := int64(num(row["cnt"])) + eventCode := uint32(num(row["event_code"])) + d := postgres.Detection{Fingerprint: fp, RuleID: sr.ID, RuleSetID: sr.RuleSetID, RuleName: sr.Title, Severity: sr.Severity, Hostname: host, UserName: user, SourceIP: ip, Workstation: workstation, EventCode: eventCode, Score: sr.Score, WindowStart: ws, WindowEnd: we, Summary: rules.RenderSummary(sr.Summary, row), Count: max64(1, count), Tags: sr.Tags, MITRE: sr.MITRE} if e := pg.UpsertDetection(ctx, d, cfg.TenantID); e != nil { - return e + log.Printf("rule %s detection: %v", sr.ID, e) } } } return nil } -func rules(db string) []rule { - table := clickhouse.Ident(db) + ".events" - return []rule{ - {name: "audit_log_cleared", severity: "critical", eventCode: 1102, score: 9.8, query: simpleEvent(table, 1102, 1), summary: func(m map[string]any) string { - return fmt.Sprintf("Audit-Log auf %s wurde gelöscht", str(m["host_name"])) - }}, - {name: "service_installed", severity: "high", eventCode: 7045, score: 8.0, query: simpleEvent(table, 7045, 1), summary: func(m map[string]any) string { - return fmt.Sprintf("Neuer Dienst auf %s installiert", str(m["host_name"])) - }}, - {name: "account_lockout", severity: "medium", eventCode: 4740, score: 5.5, query: func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4740 GROUP BY host_name,user_name,workstation HAVING cnt>=1`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) - }, summary: func(m map[string]any) string { - return fmt.Sprintf("Account-Lockout: %s; Caller %s; DC/Host %s (%d×)", str(m["user_name"]), fallback(str(m["workstation"]), "unbekannt"), str(m["host_name"]), int64(num(m["cnt"]))) - }}, - {name: "failed_logon_burst", severity: "high", eventCode: 4625, score: 7.5, query: func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT host_name, target_user AS user_name, source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND target_user!='' GROUP BY host_name,user_name,source_ip,workstation HAVING cnt>=20`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) - }, summary: func(m map[string]any) string { - return fmt.Sprintf("%d fehlgeschlagene Logons für %s auf %s", int64(num(m["cnt"])), str(m["user_name"]), str(m["host_name"])) - }}, - {name: "password_spray", severity: "high", eventCode: 4625, score: 8.5, query: func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT '' AS host_name, '' AS user_name, source_ip, '' AS workstation, uniqExact(target_user) users, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND source_ip!='' AND target_user!='' GROUP BY source_ip HAVING users>=10 AND cnt>=20`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) - }, summary: func(m map[string]any) string { - return fmt.Sprintf("Password-Spray von %s gegen %.0f Benutzer (%d Versuche)", str(m["source_ip"]), num(m["users"]), int64(num(m["cnt"]))) - }}, - {name: "privileged_group_change", severity: "critical", eventCode: 4728, score: 9.2, query: func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code IN (4728,4732,4756) GROUP BY host_name,user_name,workstation HAVING cnt>=1`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e))) - }, summary: func(m map[string]any) string { - return fmt.Sprintf("Privilegierte Gruppenmitgliedschaft geändert: %s auf %s", str(m["user_name"]), str(m["host_name"])) - }}, - } -} -func simpleEvent(table string, id uint32, min int) func(time.Time, time.Time, string) string { - return func(s, e time.Time, t string) string { - return fmt.Sprintf(`SELECT host_name, '' AS user_name, '' AS source_ip, '' AS workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM %s WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=%d GROUP BY host_name HAVING cnt>=%d`, table, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)), id, min) - } -} -func ts(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000") } + func str(v any) string { if v == nil { return "" @@ -144,12 +153,6 @@ func fingerprint(v ...string) string { h := sha256.Sum256([]byte(strings.Join(v, "|"))) return hex.EncodeToString(h[:]) } -func fallback(v, d string) string { - if strings.TrimSpace(v) == "" { - return d - } - return v -} func max64(a, b int64) int64 { if a > b { return a diff --git a/internal/ingress/server_test.go b/internal/ingress/server_test.go index 9c8938b..9e42cad 100644 --- a/internal/ingress/server_test.go +++ b/internal/ingress/server_test.go @@ -22,8 +22,12 @@ func TestValidateRequiresMessageOrMetadata(t *testing.T) { } func TestBatchUIDIsDeterministic(t *testing.T) { - b := []contracts.LogPayload{{Hostname: "PC01", Channel: "Security", EventID: 4625, Source: "agent", Time: time.Date(2026,7,23,12,0,0,0,time.UTC), Metadata: &contracts.EventMetadataPayload{TargetUser: "alice"}}} + b := []contracts.LogPayload{{Hostname: "PC01", Channel: "Security", EventID: 4625, Source: "agent", Time: time.Date(2026, 7, 23, 12, 0, 0, 0, time.UTC), Metadata: &contracts.EventMetadataPayload{TargetUser: "alice"}}} a := batchUID("agent-1", b) - if a == "" || a != batchUID("agent-1", b) { t.Fatalf("batch uid is not deterministic") } - if a == batchUID("agent-2", b) { t.Fatalf("batch uid must be scoped to agent") } + if a == "" || a != batchUID("agent-1", b) { + t.Fatalf("batch uid is not deterministic") + } + if a == batchUID("agent-2", b) { + t.Fatalf("batch uid must be scoped to agent") + } } diff --git a/internal/postgres/store.go b/internal/postgres/store.go index 4a18f78..91e1354 100644 --- a/internal/postgres/store.go +++ b/internal/postgres/store.go @@ -5,12 +5,14 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/hex" + "encoding/json" "errors" "fmt" "net" "strings" "time" + "example.com/siem-greenfield/internal/rules" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) @@ -27,9 +29,12 @@ type Agent struct { FirstSeen time.Time `json:"first_seen"` LastSeen time.Time `json:"last_seen"` } + type Detection struct { ID int64 `json:"id"` Fingerprint string `json:"fingerprint"` + RuleID string `json:"rule_id"` + RuleSetID string `json:"rule_set_id"` RuleName string `json:"rule_name"` Severity string `json:"severity"` Status string `json:"status"` @@ -45,6 +50,33 @@ type Detection struct { FirstSeen time.Time `json:"first_seen"` LastSeen time.Time `json:"last_seen"` Count int64 `json:"count"` + Tags []string `json:"tags"` + MITRE []string `json:"mitre"` +} + +type RuleSetRecord struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Version int `json:"version"` + Enabled bool `json:"enabled"` + Source string `json:"source"` + Locked bool `json:"locked"` + RuleCount int64 `json:"rule_count"` + EnabledRules int64 `json:"enabled_rules"` + UpdatedAt time.Time `json:"updated_at"` +} + +type Suppression struct { + ID int64 `json:"id"` + RuleID string `json:"rule_id"` + HostPattern string `json:"host_pattern"` + UserPattern string `json:"user_pattern"` + SourceIPPattern string `json:"source_ip_pattern"` + Reason string `json:"reason"` + Enabled bool `json:"enabled"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + CreatedAt time.Time `json:"created_at"` } func Open(ctx context.Context, url string) (*Store, error) { @@ -82,7 +114,6 @@ func (s *Store) AuthenticateOrEnroll(ctx context.Context, tenant, hostname, apiK if !errors.Is(err, pgx.ErrNoRows) { return "", err } - // A concurrent first request may have enrolled the host between SELECT and INSERT. err = s.Pool.QueryRow(ctx, `SELECT id::text, api_key_hash, enabled FROM agents WHERE tenant_id=$1 AND hostname=$2`, tenant, hostname).Scan(&id, &hash, &enabled) if err != nil || !enabled || !secureEqual(strings.ToLower(hash), hashHex(apiKey)) { return "", ErrUnauthorized @@ -112,23 +143,209 @@ func (s *Store) ListAgents(ctx context.Context, tenant string) ([]Agent, error) } return out, rows.Err() } +func (s *Store) SetAgentEnabled(ctx context.Context, tenant, id string, enabled bool) error { + tag, e := s.Pool.Exec(ctx, `UPDATE agents SET enabled=$3 WHERE tenant_id=$1 AND id=$2::uuid`, tenant, id, enabled) + if e != nil { + return e + } + if tag.RowsAffected() == 0 { + return pgx.ErrNoRows + } + return nil +} + +func (s *Store) SyncRuleSet(ctx context.Context, tenant string, rs rules.RuleSet) error { + tx, e := s.Pool.Begin(ctx) + if e != nil { + return e + } + defer tx.Rollback(ctx) + _, e = tx.Exec(ctx, `INSERT INTO rule_sets(tenant_id,id,name,description,version,enabled,source,locked) +VALUES($1,$2,$3,$4,$5,$6,'builtin',true) +ON CONFLICT(tenant_id,id) DO UPDATE SET name=EXCLUDED.name,description=EXCLUDED.description,version=GREATEST(rule_sets.version,EXCLUDED.version),source='builtin',locked=true,updated_at=now()`, tenant, rs.ID, rs.Name, rs.Description, rs.Version, rs.Enabled) + if e != nil { + return e + } + ids := make([]string, 0, len(rs.Rules)) + for _, r := range rs.Rules { + ids = append(ids, r.ID) + b, e := json.Marshal(r) + if e != nil { + return e + } + _, e = tx.Exec(ctx, `INSERT INTO detection_rules(tenant_id,id,rule_set_id,title,severity,score,enabled,source,definition) +VALUES($1,$2,$3,$4,$5,$6,$7,'builtin',$8::jsonb) +ON CONFLICT(tenant_id,id) DO UPDATE SET rule_set_id=EXCLUDED.rule_set_id,title=EXCLUDED.title,severity=EXCLUDED.severity,score=EXCLUDED.score,source='builtin',definition=EXCLUDED.definition,updated_at=now()`, tenant, r.ID, rs.ID, r.Title, r.Severity, r.Score, r.Enabled, string(b)) + if e != nil { + return e + } + } + if len(ids) == 0 { + _, e = tx.Exec(ctx, `DELETE FROM detection_rules WHERE tenant_id=$1 AND rule_set_id=$2 AND source='builtin'`, tenant, rs.ID) + } else { + _, e = tx.Exec(ctx, `DELETE FROM detection_rules WHERE tenant_id=$1 AND rule_set_id=$2 AND source='builtin' AND NOT (id = ANY($3))`, tenant, rs.ID, ids) + } + if e != nil { + return e + } + return tx.Commit(ctx) +} + +func (s *Store) PruneBuiltinRuleSets(ctx context.Context, tenant string, keep []string) error { + if len(keep) == 0 { + _, e := s.Pool.Exec(ctx, `DELETE FROM rule_sets WHERE tenant_id=$1 AND source='builtin'`, tenant) + return e + } + _, e := s.Pool.Exec(ctx, `DELETE FROM rule_sets WHERE tenant_id=$1 AND source='builtin' AND NOT (id = ANY($2))`, tenant, keep) + return e +} + +func (s *Store) EnsureCustomRuleSet(ctx context.Context, tenant string) error { + _, e := s.Pool.Exec(ctx, `INSERT INTO rule_sets(tenant_id,id,name,description,version,enabled,source,locked) +VALUES($1,'custom','Eigene Regeln','Über die SIEM-Oberfläche verwaltete Regeln',1,true,'custom',false) +ON CONFLICT(tenant_id,id) DO NOTHING`, tenant) + return e +} + +func (s *Store) ListRuleSets(ctx context.Context, tenant string) ([]RuleSetRecord, error) { + rows, e := s.Pool.Query(ctx, `SELECT rs.id,rs.name,rs.description,rs.version,rs.enabled,rs.source,rs.locked,rs.updated_at, +count(r.id),count(r.id) FILTER (WHERE r.enabled) +FROM rule_sets rs LEFT JOIN detection_rules r ON r.tenant_id=rs.tenant_id AND r.rule_set_id=rs.id +WHERE rs.tenant_id=$1 GROUP BY rs.id,rs.name,rs.description,rs.version,rs.enabled,rs.source,rs.locked,rs.updated_at ORDER BY rs.name`, tenant) + if e != nil { + return nil, e + } + defer rows.Close() + var out []RuleSetRecord + for rows.Next() { + var x RuleSetRecord + if e := rows.Scan(&x.ID, &x.Name, &x.Description, &x.Version, &x.Enabled, &x.Source, &x.Locked, &x.UpdatedAt, &x.RuleCount, &x.EnabledRules); e != nil { + return nil, e + } + out = append(out, x) + } + return out, rows.Err() +} + +func (s *Store) ListRules(ctx context.Context, tenant string, enabledOnly bool) ([]rules.StoredRule, error) { + q := `SELECT r.rule_set_id,rs.name,rs.enabled,r.source,r.enabled,r.definition FROM detection_rules r JOIN rule_sets rs ON rs.tenant_id=r.tenant_id AND rs.id=r.rule_set_id WHERE r.tenant_id=$1` + if enabledOnly { + q += ` AND r.enabled AND rs.enabled` + } + q += ` ORDER BY rs.name,r.severity DESC,r.title` + rows, e := s.Pool.Query(ctx, q, tenant) + if e != nil { + return nil, e + } + defer rows.Close() + var out []rules.StoredRule + for rows.Next() { + var x rules.StoredRule + var b []byte + var enabled bool + if e := rows.Scan(&x.RuleSetID, &x.RuleSetName, &x.RuleSetOn, &x.Source, &enabled, &b); e != nil { + return nil, e + } + if e := json.Unmarshal(b, &x.Rule); e != nil { + return nil, e + } + x.Enabled = enabled + out = append(out, x) + } + return out, rows.Err() +} +func (s *Store) SetRuleEnabled(ctx context.Context, tenant, id string, enabled bool) error { + tag, e := s.Pool.Exec(ctx, `UPDATE detection_rules SET enabled=$3,updated_at=now() WHERE tenant_id=$1 AND id=$2`, tenant, id, enabled) + if e != nil { + return e + } + if tag.RowsAffected() == 0 { + return pgx.ErrNoRows + } + return nil +} +func (s *Store) SetRuleSetEnabled(ctx context.Context, tenant, id string, enabled bool) error { + tag, e := s.Pool.Exec(ctx, `UPDATE rule_sets SET enabled=$3,updated_at=now() WHERE tenant_id=$1 AND id=$2`, tenant, id, enabled) + if e != nil { + return e + } + if tag.RowsAffected() == 0 { + return pgx.ErrNoRows + } + return nil +} +func (s *Store) SaveCustomRule(ctx context.Context, tenant string, r rules.Rule) error { + if err := rules.Validate(r); err != nil { + return err + } + if err := s.EnsureCustomRuleSet(ctx, tenant); err != nil { + return err + } + b, e := json.Marshal(r) + if e != nil { + return e + } + tag, e := s.Pool.Exec(ctx, `INSERT INTO detection_rules(tenant_id,id,rule_set_id,title,severity,score,enabled,source,definition) +VALUES($1,$2,'custom',$3,$4,$5,$6,'custom',$7::jsonb) +ON CONFLICT(tenant_id,id) DO UPDATE SET rule_set_id='custom',title=EXCLUDED.title,severity=EXCLUDED.severity,score=EXCLUDED.score,enabled=EXCLUDED.enabled,definition=EXCLUDED.definition,updated_at=now() +WHERE detection_rules.source='custom'`, tenant, r.ID, r.Title, r.Severity, r.Score, r.Enabled, string(b)) + if e != nil { + return e + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("rule id %q belongs to a built-in rule set", r.ID) + } + return nil +} + +func (s *Store) CreateSuppression(ctx context.Context, tenant string, x Suppression) error { + if x.RuleID == "" && x.HostPattern == "" && x.UserPattern == "" && x.SourceIPPattern == "" { + return fmt.Errorf("suppression must constrain rule or entity") + } + _, e := s.Pool.Exec(ctx, `INSERT INTO detection_suppressions(tenant_id,rule_id,host_pattern,user_pattern,source_ip_pattern,reason,enabled,expires_at) VALUES($1,$2,$3,$4,$5,$6,true,$7)`, tenant, x.RuleID, x.HostPattern, x.UserPattern, x.SourceIPPattern, x.Reason, x.ExpiresAt) + return e +} +func (s *Store) DeleteSuppression(ctx context.Context, tenant string, id int64) error { + _, e := s.Pool.Exec(ctx, `DELETE FROM detection_suppressions WHERE tenant_id=$1 AND id=$2`, tenant, id) + return e +} +func (s *Store) ListSuppressions(ctx context.Context, tenant string) ([]Suppression, error) { + rows, e := s.Pool.Query(ctx, `SELECT id,rule_id,host_pattern,user_pattern,source_ip_pattern,reason,enabled,expires_at,created_at FROM detection_suppressions WHERE tenant_id=$1 ORDER BY created_at DESC`, tenant) + if e != nil { + return nil, e + } + defer rows.Close() + var out []Suppression + for rows.Next() { + var x Suppression + if e := rows.Scan(&x.ID, &x.RuleID, &x.HostPattern, &x.UserPattern, &x.SourceIPPattern, &x.Reason, &x.Enabled, &x.ExpiresAt, &x.CreatedAt); e != nil { + return nil, e + } + out = append(out, x) + } + return out, rows.Err() +} +func (s *Store) IsSuppressed(ctx context.Context, tenant, ruleID, host, user, ip string, at time.Time) (bool, error) { + var ok bool + e := s.Pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM detection_suppressions WHERE tenant_id=$1 AND enabled AND (expires_at IS NULL OR expires_at>$6) +AND (rule_id='' OR rule_id=$2) +AND (host_pattern='' OR $3 LIKE replace(host_pattern,'*','%')) +AND (user_pattern='' OR $4 LIKE replace(user_pattern,'*','%')) +AND (source_ip_pattern='' OR $5 LIKE replace(source_ip_pattern,'*','%')))`, tenant, ruleID, host, user, ip, at).Scan(&ok) + return ok, e +} func (s *Store) UpsertDetection(ctx context.Context, d Detection, tenant string) error { - _, e := s.Pool.Exec(ctx, `INSERT INTO detections(tenant_id,fingerprint,rule_name,severity,status,hostname,user_name,source_ip,workstation,event_code,score,window_start,window_end,summary,hit_count,first_seen,last_seen) -VALUES($1,$2,$3,$4,'open',$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$11,$12) -ON CONFLICT(tenant_id,fingerprint) DO UPDATE SET - first_seen=LEAST(detections.first_seen,EXCLUDED.first_seen), - last_seen=GREATEST(detections.last_seen,EXCLUDED.last_seen), - window_start=LEAST(detections.window_start,EXCLUDED.window_start), - window_end=GREATEST(detections.window_end,EXCLUDED.window_end), - hit_count=GREATEST(detections.hit_count,EXCLUDED.hit_count), - score=GREATEST(detections.score,EXCLUDED.score), - summary=EXCLUDED.summary, workstation=EXCLUDED.workstation, updated_at=now()`, tenant, d.Fingerprint, d.RuleName, d.Severity, d.Hostname, d.UserName, d.SourceIP, d.Workstation, d.EventCode, d.Score, d.WindowStart, d.WindowEnd, d.Summary, d.Count) + tags, _ := json.Marshal(d.Tags) + mitre, _ := json.Marshal(d.MITRE) + _, e := s.Pool.Exec(ctx, `INSERT INTO detections(tenant_id,fingerprint,rule_id,rule_set_id,rule_name,severity,status,hostname,user_name,source_ip,workstation,event_code,score,window_start,window_end,summary,hit_count,tags,mitre,first_seen,last_seen) +VALUES($1,$2,$3,$4,$5,$6,'open',$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17::jsonb,$18::jsonb,$13,$14) +ON CONFLICT(tenant_id,fingerprint) DO UPDATE SET first_seen=LEAST(detections.first_seen,EXCLUDED.first_seen),last_seen=GREATEST(detections.last_seen,EXCLUDED.last_seen),window_start=LEAST(detections.window_start,EXCLUDED.window_start),window_end=GREATEST(detections.window_end,EXCLUDED.window_end),hit_count=GREATEST(detections.hit_count,EXCLUDED.hit_count),score=GREATEST(detections.score,EXCLUDED.score),summary=EXCLUDED.summary,workstation=EXCLUDED.workstation,tags=EXCLUDED.tags,mitre=EXCLUDED.mitre,updated_at=now()`, tenant, d.Fingerprint, d.RuleID, d.RuleSetID, d.RuleName, d.Severity, d.Hostname, d.UserName, d.SourceIP, d.Workstation, d.EventCode, d.Score, d.WindowStart, d.WindowEnd, d.Summary, d.Count, string(tags), string(mitre)) return e } func (s *Store) ListDetections(ctx context.Context, tenant string, limit int) ([]Detection, error) { - rows, e := s.Pool.Query(ctx, `SELECT id,fingerprint,rule_name,severity,status,hostname,user_name,source_ip,workstation,event_code,score,window_start,window_end,summary,hit_count,first_seen,last_seen FROM detections WHERE tenant_id=$1 ORDER BY last_seen DESC LIMIT $2`, tenant, limit) + rows, e := s.Pool.Query(ctx, `SELECT id,fingerprint,rule_id,rule_set_id,rule_name,severity,status,hostname,user_name,source_ip,workstation,event_code,score,window_start,window_end,summary,hit_count,tags,mitre,first_seen,last_seen FROM detections WHERE tenant_id=$1 ORDER BY last_seen DESC LIMIT $2`, tenant, limit) if e != nil { return nil, e } @@ -136,9 +353,12 @@ func (s *Store) ListDetections(ctx context.Context, tenant string, limit int) ([ var out []Detection for rows.Next() { var d Detection - if e := rows.Scan(&d.ID, &d.Fingerprint, &d.RuleName, &d.Severity, &d.Status, &d.Hostname, &d.UserName, &d.SourceIP, &d.Workstation, &d.EventCode, &d.Score, &d.WindowStart, &d.WindowEnd, &d.Summary, &d.Count, &d.FirstSeen, &d.LastSeen); e != nil { + var tags, mitre []byte + if e := rows.Scan(&d.ID, &d.Fingerprint, &d.RuleID, &d.RuleSetID, &d.RuleName, &d.Severity, &d.Status, &d.Hostname, &d.UserName, &d.SourceIP, &d.Workstation, &d.EventCode, &d.Score, &d.WindowStart, &d.WindowEnd, &d.Summary, &d.Count, &tags, &mitre, &d.FirstSeen, &d.LastSeen); e != nil { return nil, e } + _ = json.Unmarshal(tags, &d.Tags) + _ = json.Unmarshal(mitre, &d.MITRE) out = append(out, d) } return out, rows.Err() @@ -147,7 +367,7 @@ func (s *Store) UpdateDetectionStatus(ctx context.Context, tenant string, id int if status != "open" && status != "investigating" && status != "closed" && status != "false_positive" { return fmt.Errorf("invalid status") } - _, e := s.Pool.Exec(ctx, `UPDATE detections SET status=$3, updated_at=now() WHERE tenant_id=$1 AND id=$2`, tenant, id, status) + _, e := s.Pool.Exec(ctx, `UPDATE detections SET status=$3,updated_at=now() WHERE tenant_id=$1 AND id=$2`, tenant, id, status) return e } func (s *Store) DetectionCounts(ctx context.Context, tenant string) (map[string]int64, error) { diff --git a/internal/rules/rules.go b/internal/rules/rules.go new file mode 100644 index 0000000..90fa0c1 --- /dev/null +++ b/internal/rules/rules.go @@ -0,0 +1,355 @@ +package rules + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "example.com/siem-greenfield/internal/clickhouse" +) + +type RuleSet struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Version int `json:"version"` + Enabled bool `json:"enabled"` + Rules []Rule `json:"rules"` +} + +type Rule struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Severity string `json:"severity"` + Score float64 `json:"score"` + Enabled bool `json:"enabled"` + Kind string `json:"kind"` + Channels []string `json:"channels,omitempty"` + EventCodes []uint32 `json:"event_codes,omitempty"` + Conditions []Condition `json:"conditions,omitempty"` + GroupBy []string `json:"group_by,omitempty"` + Threshold int64 `json:"threshold,omitempty"` + DistinctField string `json:"distinct_field,omitempty"` + DistinctThreshold int64 `json:"distinct_threshold,omitempty"` + WindowSeconds int `json:"window_seconds"` + SuppressSeconds int `json:"suppress_seconds,omitempty"` + Summary string `json:"summary"` + Tags []string `json:"tags,omitempty"` + MITRE []string `json:"mitre,omitempty"` +} + +type Condition struct { + Field string `json:"field"` + Operator string `json:"operator"` + Value string `json:"value,omitempty"` +} + +type StoredRule struct { + RuleSetID string `json:"rule_set_id"` + RuleSetName string `json:"rule_set_name"` + RuleSetOn bool `json:"rule_set_enabled"` + Source string `json:"source"` + Rule +} + +func LoadDir(dir string) ([]RuleSet, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var out []RuleSet + for _, ent := range entries { + if ent.IsDir() || !strings.HasSuffix(strings.ToLower(ent.Name()), ".json") { + continue + } + b, err := os.ReadFile(filepath.Join(dir, ent.Name())) + if err != nil { + return nil, err + } + var rs RuleSet + if err := json.Unmarshal(b, &rs); err != nil { + return nil, fmt.Errorf("%s: %w", ent.Name(), err) + } + if err := ValidateSet(rs); err != nil { + return nil, fmt.Errorf("%s: %w", ent.Name(), err) + } + out = append(out, rs) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func ValidateSet(rs RuleSet) error { + if !validID(rs.ID) || strings.TrimSpace(rs.Name) == "" { + return fmt.Errorf("invalid rule-set id/name") + } + if rs.Version <= 0 { + return fmt.Errorf("rule-set %s: version must be positive", rs.ID) + } + seen := map[string]bool{} + for i := range rs.Rules { + if err := Validate(rs.Rules[i]); err != nil { + return fmt.Errorf("rule %d: %w", i+1, err) + } + if seen[rs.Rules[i].ID] { + return fmt.Errorf("duplicate rule id %q", rs.Rules[i].ID) + } + seen[rs.Rules[i].ID] = true + } + return nil +} + +func Validate(r Rule) error { + if !validID(r.ID) || strings.TrimSpace(r.Title) == "" { + return fmt.Errorf("invalid id/title") + } + switch r.Severity { + case "info", "low", "medium", "high", "critical": + default: + return fmt.Errorf("invalid severity %q", r.Severity) + } + if r.Score < 0 || r.Score > 100 { + return fmt.Errorf("score must be between 0 and 100") + } + switch r.Kind { + case "event", "threshold", "distinct": + default: + return fmt.Errorf("invalid kind %q", r.Kind) + } + if len(r.EventCodes) == 0 && len(r.Conditions) == 0 { + return fmt.Errorf("rule must constrain event_codes or conditions") + } + if r.WindowSeconds < 30 || r.WindowSeconds > 86400 { + return fmt.Errorf("window_seconds must be between 30 and 86400") + } + if r.Threshold <= 0 { + r.Threshold = 1 + } + if r.Kind == "distinct" { + if _, ok := fieldExpr(r.DistinctField); !ok { + return fmt.Errorf("invalid distinct_field %q", r.DistinctField) + } + if r.DistinctThreshold <= 0 { + return fmt.Errorf("distinct_threshold must be positive") + } + } + for _, g := range r.GroupBy { + if _, ok := groupExpr(g); !ok { + return fmt.Errorf("invalid group_by field %q", g) + } + } + for _, c := range r.Conditions { + if _, ok := fieldExpr(c.Field); !ok { + return fmt.Errorf("invalid condition field %q", c.Field) + } + switch c.Operator { + case "equals", "not_equals", "contains", "not_contains", "regex", "exists", "not_exists", "in": + default: + return fmt.Errorf("invalid operator %q", c.Operator) + } + } + return nil +} + +func Compile(r Rule, db, tenant string, end time.Time) (string, time.Time, error) { + if err := Validate(r); err != nil { + return "", time.Time{}, err + } + start := end.Add(-time.Duration(r.WindowSeconds) * time.Second) + table := clickhouse.Ident(db) + ".events" + where := []string{ + "tenant_id=" + clickhouse.Q(tenant), + "event_time>=" + clickhouse.Q(ts(start)), + "event_time<" + clickhouse.Q(ts(end)), + } + if len(r.Channels) > 0 { + where = append(where, "channel IN ("+quoteStrings(r.Channels)+")") + } + if len(r.EventCodes) > 0 { + vals := make([]string, 0, len(r.EventCodes)) + for _, id := range r.EventCodes { + vals = append(vals, strconv.FormatUint(uint64(id), 10)) + } + where = append(where, "event_code IN ("+strings.Join(vals, ",")+")") + } + for _, c := range r.Conditions { + x, err := compileCondition(c) + if err != nil { + return "", time.Time{}, err + } + where = append(where, x) + } + + groupFields := make([]string, 0, len(r.GroupBy)) + selectFields := make([]string, 0, len(r.GroupBy)+8) + selected := map[string]bool{} + for _, g := range r.GroupBy { + expr, _ := groupExpr(g) + alias := groupAlias(g) + selectFields = append(selectFields, expr+" AS "+alias) + groupFields = append(groupFields, expr) + selected[g] = true + } + addContext := func(key, alias string) { + if !selected[key] { + selectFields = append(selectFields, "'' AS "+alias) + } + } + addContext("host", "host_name") + addContext("user", "user_name") + addContext("source_ip", "source_ip") + addContext("workstation", "workstation") + addContext("process_path", "process_path") + if selected["event_code"] { + // already selected under event_code + } else if len(r.EventCodes) == 1 { + selectFields = append(selectFields, strconv.FormatUint(uint64(r.EventCodes[0]), 10)+" AS event_code") + } else { + selectFields = append(selectFields, "toUInt32(0) AS event_code") + } + selectFields = append(selectFields, + "uniqExact(event_uid) AS cnt", + "min(event_time) AS window_start", + "max(event_time) AS window_end", + ) + if r.Kind == "distinct" { + expr, _ := fieldExpr(r.DistinctField) + selectFields = append(selectFields, "uniqExact("+expr+") AS distinct_cnt") + } + having := fmt.Sprintf("cnt >= %d", max64(1, r.Threshold)) + if r.Kind == "distinct" { + having += fmt.Sprintf(" AND distinct_cnt >= %d", r.DistinctThreshold) + } + q := "SELECT " + strings.Join(selectFields, ", ") + " FROM " + table + " WHERE " + strings.Join(where, " AND ") + if len(groupFields) > 0 { + q += " GROUP BY " + strings.Join(groupFields, ", ") + } + q += " HAVING " + having + return q, start, nil +} + +func RenderSummary(tpl string, row map[string]any) string { + vals := map[string]string{ + "host": value(row["host_name"]), + "user": value(row["user_name"]), + "source_ip": value(row["source_ip"]), + "workstation": value(row["workstation"]), + "process": value(row["process_path"]), + "event_code": value(row["event_code"]), + "count": value(row["cnt"]), + "distinct": value(row["distinct_cnt"]), + } + out := tpl + for k, v := range vals { + out = strings.ReplaceAll(out, "{"+k+"}", v) + } + if strings.TrimSpace(out) == "" { + out = "Rule matched on " + vals["host"] + } + return out +} + +func fieldExpr(name string) (string, bool) { + fields := map[string]string{ + "host": "host_name", "host_name": "host_name", + "user": "multiIf(target_user!='',target_user,user_name!='',user_name,subject_user)", + "user_name": "user_name", "target_user": "target_user", "subject_user": "subject_user", + "source_ip": "source_ip", "destination_ip": "destination_ip", "workstation": "workstation", + "process_path": "process_path", "parent_process_path": "parent_process_path", "command_line": "command_line", + "message": "message", "channel": "channel", "provider": "provider", "category": "category", + "action": "action", "outcome": "outcome", "logon_type": "logon_type", + "authentication_package": "authentication_package", "status_code": "status_code", + "failure_reason": "failure_reason", "event_code": "event_code", + } + v, ok := fields[name] + return v, ok +} + +func groupExpr(name string) (string, bool) { return fieldExpr(name) } +func groupAlias(name string) string { + switch name { + case "host", "host_name": + return "host_name" + case "user", "user_name", "target_user", "subject_user": + return "user_name" + default: + return name + } +} + +func compileCondition(c Condition) (string, error) { + expr, ok := fieldExpr(c.Field) + if !ok { + return "", fmt.Errorf("invalid field %q", c.Field) + } + q := clickhouse.Q(c.Value) + switch c.Operator { + case "equals": + return expr + "=" + q, nil + case "not_equals": + return expr + "!=" + q, nil + case "contains": + return "positionCaseInsensitiveUTF8(" + expr + "," + q + ")>0", nil + case "not_contains": + return "positionCaseInsensitiveUTF8(" + expr + "," + q + ")=0", nil + case "regex": + return "match(" + expr + "," + q + ")", nil + case "exists": + return expr + "!=''", nil + case "not_exists": + return expr + "=''", nil + case "in": + parts := strings.Split(c.Value, ",") + vals := make([]string, 0, len(parts)) + for _, p := range parts { + if x := strings.TrimSpace(p); x != "" { + vals = append(vals, clickhouse.Q(x)) + } + } + if len(vals) == 0 { + return "", fmt.Errorf("empty in condition") + } + return expr + " IN (" + strings.Join(vals, ",") + ")", nil + default: + return "", fmt.Errorf("unsupported operator %q", c.Operator) + } +} + +func quoteStrings(v []string) string { + out := make([]string, 0, len(v)) + for _, x := range v { + out = append(out, clickhouse.Q(x)) + } + return strings.Join(out, ",") +} +func validID(s string) bool { + if s == "" || len(s) > 128 { + return false + } + for _, r := range s { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-' || r == '_' || r == '.' { + continue + } + return false + } + return true +} +func ts(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000") } +func value(v any) string { + if v == nil { + return "" + } + return fmt.Sprint(v) +} +func max64(a, b int64) int64 { + if a > b { + return a + } + return b +} diff --git a/internal/rules/rules_test.go b/internal/rules/rules_test.go new file mode 100644 index 0000000..7daf131 --- /dev/null +++ b/internal/rules/rules_test.go @@ -0,0 +1,26 @@ +package rules + +import ( + "strings" + "testing" + "time" +) + +func TestCompileThreshold(t *testing.T) { + r := Rule{ID: "failed", Title: "Failed", Severity: "high", Kind: "threshold", Enabled: true, EventCodes: []uint32{4625}, GroupBy: []string{"host", "user", "source_ip"}, Threshold: 20, WindowSeconds: 300, Summary: "{count} failures for {user}"} + q, _, err := Compile(r, "siem", "default", time.Date(2026, 7, 23, 12, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"event_code IN (4625)", "GROUP BY host_name", "HAVING cnt >= 20"} { + if !strings.Contains(q, want) { + t.Fatalf("query missing %q: %s", want, q) + } + } +} +func TestRejectUnknownField(t *testing.T) { + r := Rule{ID: "bad", Title: "Bad", Severity: "high", Kind: "event", EventCodes: []uint32{1}, Conditions: []Condition{{Field: "DROP TABLE", Operator: "equals", Value: "x"}}, WindowSeconds: 300} + if err := Validate(r); err == nil { + t.Fatal("expected validation error") + } +} diff --git a/preflight.sh b/preflight.sh new file mode 100644 index 0000000..838f434 --- /dev/null +++ b/preflight.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env sh +set -u +cd "$(dirname "$0")" + +get_env() { + key="$1" + awk -F= -v k="$key" '$1==k {sub(/^[^=]*=/, ""); print; exit}' .env 2>/dev/null || true +} + +set_env() { + key="$1"; value="$2"; tmp=".env.tmp.$$" + awk -v k="$key" -v v="$value" ' + BEGIN {done=0} + index($0,k"=")==1 {print k"="v; done=1; next} + {print} + END {if(!done) print k"="v} + ' .env > "$tmp" && mv "$tmp" .env +} + +bool_true() { + case "${1:-}" in 1|true|TRUE|yes|YES|on|ON) return 0 ;; *) return 1 ;; esac +} + +port_owner() { + port="$1" + docker ps --format '{{.Names}} {{.Ports}}' 2>/dev/null | awk -v p=":${port}->" 'index($0,p){print $1; exit}' +} + +port_free() { + [ -z "$(port_owner "$1")" ] +} + +choose_free_port() { + start="$1" + p="$start" + n=0 + while [ "$n" -lt 100 ]; do + if port_free "$p"; then printf '%s' "$p"; return 0; fi + p=$((p+1)); n=$((n+1)) + done + return 1 +} + +ensure_port() { + key="$1"; fallback="$2" + val="$(get_env "$key")" + [ -n "$val" ] || return 0 + owner="$(port_owner "$val")" + [ -z "$owner" ] && return 0 + case "$owner" in greenfield-siem-*) return 0 ;; esac + + if bool_true "$(get_env AUTO_PORTS)"; then + new="$(choose_free_port "$fallback")" || { + echo "FEHLER: Kein freier Ersatzport für $key gefunden." >&2 + exit 1 + } + echo "Portkonflikt: $key=$val wird bereits von '$owner' benutzt -> verwende $new." + set_env "$key" "$new" + else + echo "FEHLER: $key=$val ist bereits durch '$owner' belegt. AUTO_PORTS=true aktivieren oder .env anpassen." >&2 + exit 1 + fi +} + +probe_clickhouse() { + image="$1" + echo "Prüfe ClickHouse-Binary '$image' auf dieser CPU ..." + docker pull "$image" >/dev/null 2>&1 || return 125 + set +e + docker run --rm --entrypoint clickhouse "$image" local --query 'SELECT 1' >/tmp/siem-clickhouse-probe.out 2>/tmp/siem-clickhouse-probe.err + rc=$? + set -e + return "$rc" +} + +[ -f .env ] || { echo "FEHLER: .env fehlt." >&2; exit 1; } + +ARCH="$(uname -m 2>/dev/null || echo unknown)" +echo "Host-Architektur: $ARCH" +if command -v lscpu >/dev/null 2>&1; then + lscpu | awk -F: '/Architecture|Model name|Vendor ID|CPU op-mode/{gsub(/^[ \t]+/,"",$2); print $1 ": " $2}' || true +fi + +# Resolve host-port collisions before Compose creates containers. +ensure_port UI_PORT 18080 +ensure_port INGRESS_PORT 18090 +ensure_port CLICKHOUSE_HTTP_PORT 18123 +ensure_port CLICKHOUSE_NATIVE_PORT 19000 +ensure_port POSTGRES_PORT 15432 +ensure_port REDPANDA_KAFKA_PORT 29092 +ensure_port REDPANDA_ADMIN_PORT 29644 +ensure_port REDPANDA_CONSOLE_PORT 18081 +ensure_port GARAGE_S3_PORT 13900 +ensure_port GARAGE_ADMIN_PORT 13903 +ensure_port PROMETHEUS_PORT 19090 +ensure_port GRAFANA_PORT 13000 + +if bool_true "$(get_env CLICKHOUSE_CPU_PROBE)"; then + image="$(get_env CLICKHOUSE_IMAGE)" + [ -n "$image" ] || image='clickhouse/clickhouse-server:26.3.17.56' + if probe_clickhouse "$image"; then + echo "ClickHouse CPU-Probe: OK ($image)" + else + rc=$? + echo "ClickHouse CPU-Probe fehlgeschlagen (Exit $rc)." >&2 + cat /tmp/siem-clickhouse-probe.err >&2 2>/dev/null || true + + fallback="$(get_env CLICKHOUSE_FALLBACK_IMAGE)" + if [ -n "$fallback" ] && [ "$fallback" != "$image" ]; then + echo "Teste LTS-Fallback: $fallback" + if probe_clickhouse "$fallback"; then + echo "LTS-Fallback läuft auf dieser CPU. CLICKHOUSE_IMAGE wird auf $fallback gesetzt." + set_env CLICKHOUSE_IMAGE "$fallback" + exit 0 + else + frc=$? + echo "Auch LTS-Fallback fehlgeschlagen (Exit $frc)." >&2 + cat /tmp/siem-clickhouse-probe.err >&2 2>/dev/null || true + fi + fi + + echo >&2 + echo "ClickHouse kann auf dieser CPU/VM mit den offiziellen Images nicht ausgeführt werden." >&2 + case "$ARCH" in + x86_64|amd64) + echo "Das offizielle amd64-Image benötigt mindestens SSE3. Prüfe: grep -m1 '^flags' /proc/cpuinfo" >&2 + echo "Bei einer VM: CPU-Modell auf 'host' / CPU-Passthrough stellen, sofern die physische CPU SSE3 unterstützt." >&2 + ;; + aarch64|arm64) + echo "Das offizielle arm64-Image benötigt ARMv8.2-A plus RCpc-Unterstützung." >&2 + echo "Bei einer VM: CPU-Passthrough aktivieren. Auf älterer ARMv8.0/8.1-Hardware ist ein offizielles aktuelles Image nicht kompatibel." >&2 + ;; + esac + echo "Diagnose: ./host-info.sh" >&2 + exit 42 + fi +fi diff --git a/status.sh b/status.sh index 5393709..0b9bd53 100755 --- a/status.sh +++ b/status.sh @@ -12,5 +12,5 @@ docker compose exec -T garage /garage status 2>/dev/null || true printf '\nClickHouse:\n' docker compose exec -T clickhouse clickhouse-client --user "${CLICKHOUSE_USER:-siem}" --password "${CLICKHOUSE_PASSWORD:-}" --query 'SELECT version(), currentUser()' 2>/dev/null || true printf '\nRecent errors:\n' -docker compose logs --since=10m garage clickhouse clickhouse-schema postgres postgres-schema redpanda ingress processor detector api archive-uploader 2>&1 \ +docker compose logs --since=10m garage clickhouse clickhouse-schema postgres postgres-schema redpanda ingress processor detector api archive-uploader grafana 2>&1 \ | grep -Ei 'error|fatal|failed|panic|exception' | tail -80 || true diff --git a/web/index.html b/web/index.html index 98b194c..0db5000 100644 --- a/web/index.html +++ b/web/index.html @@ -1,22 +1,119 @@ -Greenfield SIEM + + + +Greenfield SIEM -
Greenfield SIEMTenant: {{.Tenant}}
-
Events / 24h
–
Aktive Hosts / 24h
–
Offene High/Critical
–
Pipeline
OK
-
-
ZeitHostEventAktionUserSource-IPWorkstationProzess
- -
+:root{--bg:#0b1020;--side:#0f172a;--panel:#111a2e;--panel2:#16213a;--line:#263653;--text:#e8eefb;--muted:#91a2bd;--brand:#5b8cff;--brand2:#7aa2ff;--green:#37c99b;--yellow:#f1b84b;--orange:#f17c4a;--red:#f05d6f;--purple:#ad7cff;--shadow:0 18px 50px rgba(0,0,0,.22)}*{box-sizing:border-box}html,body{height:100%}body{margin:0;background:var(--bg);color:var(--text);font:14px/1.45 Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif}button,input,select,textarea{font:inherit}.app{min-height:100%;display:grid;grid-template-columns:250px 1fr}.sidebar{background:linear-gradient(180deg,#0e172b,#0b1325);border-right:1px solid var(--line);padding:22px 16px;position:sticky;top:0;height:100vh}.brand{display:flex;gap:11px;align-items:center;padding:0 8px 24px}.brandmark{width:34px;height:34px;border-radius:10px;background:linear-gradient(135deg,var(--brand),var(--purple));box-shadow:0 8px 26px #5b8cff44}.brand h1{font-size:16px;margin:0}.brand small{display:block;color:var(--muted);font-size:11px;margin-top:2px}.nav{display:grid;gap:5px}.nav button{border:0;background:transparent;color:var(--muted);padding:10px 12px;text-align:left;border-radius:9px;cursor:pointer;display:flex;align-items:center;gap:10px}.nav button:hover{background:#17223a;color:var(--text)}.nav button.active{background:#1b2a4a;color:#fff;box-shadow:inset 3px 0 var(--brand)}.nav .ico{width:22px;text-align:center;color:var(--brand2)}.sidebarfoot{position:absolute;left:16px;right:16px;bottom:18px;padding:12px;border:1px solid var(--line);border-radius:10px;background:#121d33;color:var(--muted);font-size:12px}.main{min-width:0}.topbar{height:70px;padding:0 26px;border-bottom:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;background:#0c1323cc;backdrop-filter:blur(10px);position:sticky;top:0;z-index:20}.title h2{margin:0;font-size:18px}.title span{color:var(--muted);font-size:12px}.actions{display:flex;gap:8px;align-items:center}.btn{border:1px solid var(--line);color:var(--text);background:#152039;padding:8px 12px;border-radius:8px;cursor:pointer;text-decoration:none}.btn:hover{border-color:#45638f}.btn.primary{background:var(--brand);border-color:var(--brand);color:#fff}.btn.danger{background:#371b28;border-color:#613044;color:#ffabb6}.content{padding:24px 26px 50px;max-width:1900px;margin:0 auto}.view{display:none}.view.active{display:block}.kpis{display:grid;grid-template-columns:repeat(5,minmax(150px,1fr));gap:12px}.kpi,.panel{background:linear-gradient(180deg,var(--panel),#10192c);border:1px solid var(--line);border-radius:12px;box-shadow:var(--shadow)}.kpi{padding:16px}.kpi .label{color:var(--muted);font-size:12px}.kpi .value{font-size:27px;font-weight:750;margin-top:7px;letter-spacing:-.03em}.kpi .hint{font-size:11px;color:var(--muted);margin-top:4px}.grid2{display:grid;grid-template-columns:1.35fr 1fr;gap:14px;margin-top:14px}.grid3{display:grid;grid-template-columns:1fr 1fr 1fr;gap:14px;margin-top:14px}.panel{padding:16px;min-width:0}.panelhead{display:flex;justify-content:space-between;align-items:center;margin-bottom:12px}.panelhead h3{font-size:14px;margin:0}.panelhead span{color:var(--muted);font-size:11px}.spark{height:200px;width:100%;display:block}.barlist{display:grid;gap:9px}.barrow{display:grid;grid-template-columns:minmax(80px,1.2fr) 3fr 80px;gap:10px;align-items:center}.barlabel{white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.bartrack{height:8px;background:#202d47;border-radius:10px;overflow:hidden}.barfill{height:100%;background:linear-gradient(90deg,var(--brand),var(--purple));border-radius:10px}.barvalue{text-align:right;color:var(--muted);font-variant-numeric:tabular-nums}.filters{display:flex;flex-wrap:wrap;gap:8px;margin-bottom:12px}.field{background:#0e1729;border:1px solid var(--line);color:var(--text);border-radius:8px;padding:8px 10px;min-width:130px}.field::placeholder{color:#62728c}.tablewrap{overflow:auto;max-height:68vh;border:1px solid var(--line);border-radius:10px}table{border-collapse:collapse;width:100%;font-size:12px}th,td{padding:10px 11px;border-bottom:1px solid #20304c;text-align:left;vertical-align:top}th{position:sticky;top:0;background:#14213a;color:#a9b7cb;font-weight:600;z-index:2}tbody tr{background:#101a2e}tbody tr:nth-child(even){background:#0e182a}tbody tr.clickable:hover{background:#182744;cursor:pointer}.muted{color:var(--muted)}.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace}.pill{display:inline-flex;align-items:center;gap:5px;padding:3px 7px;border-radius:999px;background:#20304b;color:#cbd7e9;font-size:11px;white-space:nowrap}.sev{font-weight:700;text-transform:uppercase;font-size:10px;letter-spacing:.04em}.sev-critical{color:#ff7c8b}.sev-high{color:#ff9a6b}.sev-medium{color:#ffd06b}.sev-low{color:#7bc4ff}.sev-info{color:#9fb0c8}.dot{width:7px;height:7px;border-radius:50%;display:inline-block;background:var(--green)}.switch{position:relative;width:38px;height:21px;display:inline-block}.switch input{display:none}.switch span{position:absolute;inset:0;border-radius:20px;background:#33435f;cursor:pointer}.switch span:after{content:"";position:absolute;width:15px;height:15px;left:3px;top:3px;background:white;border-radius:50%;transition:.15s}.switch input:checked+span{background:var(--brand)}.switch input:checked+span:after{transform:translateX(17px)}.rulesets{display:grid;grid-template-columns:repeat(4,minmax(210px,1fr));gap:10px;margin-bottom:14px}.ruleset{padding:13px;border:1px solid var(--line);background:#101b30;border-radius:10px}.ruleset h4{margin:0 0 3px;font-size:13px}.ruleset p{color:var(--muted);font-size:11px;height:33px;overflow:hidden;margin:5px 0 9px}.rulesetfoot{display:flex;justify-content:space-between;align-items:center}.tag{display:inline-block;font-size:10px;padding:2px 5px;border-radius:5px;background:#202e49;color:#b9c8df;margin:1px}.empty{padding:35px;text-align:center;color:var(--muted)}.alert{padding:10px 12px;border-radius:8px;margin-bottom:12px;background:#3a1d29;color:#ffb8c1;border:1px solid #643244}.success{background:#123326;color:#9ae3c4;border-color:#225841}.drawer{position:fixed;right:0;top:0;bottom:0;width:min(660px,92vw);background:#0d1729;border-left:1px solid var(--line);z-index:50;box-shadow:-20px 0 50px #0008;transform:translateX(105%);transition:.2s;padding:22px;overflow:auto}.drawer.open{transform:none}.drawer h3{margin:0}.drawergrid{display:grid;grid-template-columns:145px 1fr;gap:8px;margin-top:18px;font-size:12px}.drawergrid .k{color:var(--muted)}pre{white-space:pre-wrap;word-break:break-word;background:#08101f;border:1px solid var(--line);padding:12px;border-radius:8px;color:#cbd8ec}.modalback{position:fixed;inset:0;background:#0009;z-index:60;display:none;align-items:center;justify-content:center;padding:20px}.modalback.open{display:flex}.modal{width:min(760px,96vw);max-height:92vh;overflow:auto;background:#101a2d;border:1px solid var(--line);border-radius:14px;box-shadow:0 30px 80px #000b;padding:20px}.modal h3{margin-top:0}.formgrid{display:grid;grid-template-columns:1fr 1fr;gap:10px}.formgrid .wide{grid-column:1/-1}.formgrid label{font-size:11px;color:var(--muted);display:block;margin-bottom:4px}.formgrid input,.formgrid select,.formgrid textarea{width:100%;background:#0a1425;border:1px solid var(--line);color:var(--text);padding:9px;border-radius:8px}.formgrid textarea{min-height:75px}.modalactions{display:flex;justify-content:flex-end;gap:8px;margin-top:16px}.sectionintro{display:flex;justify-content:space-between;gap:20px;align-items:end;margin-bottom:14px}.sectionintro h3{margin:0 0 3px;font-size:17px}.sectionintro p{margin:0;color:var(--muted);font-size:12px}.status-open{color:#ff9a6b}.status-investigating{color:#ffd06b}.status-closed{color:#7be2bb}.status-false_positive{color:#90a0ba}@media(max-width:1200px){.app{grid-template-columns:78px 1fr}.sidebar{padding:20px 9px}.brand h1,.brand small,.nav .label,.sidebarfoot{display:none}.brand{justify-content:center}.nav button{justify-content:center}.kpis{grid-template-columns:repeat(3,1fr)}.rulesets{grid-template-columns:1fr 1fr}}@media(max-width:800px){.app{display:block}.sidebar{height:auto;position:static;border-right:0;border-bottom:1px solid var(--line)}.brand,.sidebarfoot{display:none}.nav{grid-template-columns:repeat(6,1fr)}.nav button{padding:10px 4px}.main{width:100%}.content{padding:16px}.topbar{padding:0 15px}.kpis{grid-template-columns:1fr 1fr}.grid2,.grid3{grid-template-columns:1fr}.rulesets{grid-template-columns:1fr}.formgrid{grid-template-columns:1fr}.formgrid .wide{grid-column:auto}} + + + +
+ +
+

Security Overview

Ereignisse, Detections und Pipeline-Zustand
Grafana ↗
+
+
+
+
Events / 24h
–
dedupliziert
+
Aktive Hosts / 24h
–
mit Event-Aktivität
+
High / Critical offen
–
offen + investigating
+
Aktive Regeln
–
– Rule-Sets
+
Pipeline
OK
API → Queue → ClickHouse
+
+

Event-Volumen

24 Stunden / 5-Minuten-Buckets

Top Hosts

Events / 24h
+

Top Event-IDs

Events / 24h

Authentifizierung

4624 / 4625 / 4740 / 4771 / 4776
+
+ +
+

Event Investigation

Drill-down direkt auf die kanonischen ClickHouse-Events.

+
ZeitHostEventUserSourceWorkstationProzess / Nachricht
+
+ +
+

Detections

Ergebnisse der Rule-Engine mit Status, MITRE-Kontext und Evidence-Zeitfenster.

+
ZuletztSeverityRuleEntitätZusammenfassungMITREStatus
+
+ +
+

Rule-Sets

Versionierte Built-ins plus eigene Regeln. Built-ins bleiben updatefähig; Aktivierungszustände werden in PostgreSQL erhalten.

+
+
OnSeverityRuleSetTypEvent-IDsFensterMITRE
+
+ +
+

Suppressions

Gezielte Ausnahmen nach Rule, Host, Benutzer oder Source-IP. Wildcards mit *.

+
RuleHostUserSource-IPBisGrund
+
+ +
+

Agents

Enrollment, letzter Kontakt und Aktivierungsstatus.

+
OnHostIPFirst seenLast seenZustand
+
+
+ +

Event Details

+

Eigene Detection-Regel

+
+
+
+
+
+
+
+
+
+
+
+
+
+

Suppression erstellen

+ +const $=id=>document.getElementById(id), esc=s=>String(s??'').replace(/[&<>'"]/g,c=>({'&':'&','<':'<','>':'>',"'":''','"':'"'}[c])); +let rulesCache=[], currentView='dashboard'; +const titles={dashboard:['Security Overview','Ereignisse, Detections und Pipeline-Zustand'],events:['Event Investigation','Suche und Drill-down'],detections:['Detections','Priorisieren, untersuchen und schließen'],rules:['Rule-Sets','Detection Content verwalten'],suppressions:['Suppressions','False Positives kontrolliert ausnehmen'],agents:['Agents','Collector-Zustand und Enrollment']}; +const explicitGrafana='{{.GrafanaURL}}', grafanaPort='{{.GrafanaPort}}'; $('grafanaLink').href=explicitGrafana||('http://'+location.hostname+':'+grafanaPort+'/'); +async function api(url,opt){const r=await fetch(url,opt);let d={};try{d=await r.json()}catch{}if(!r.ok)throw new Error(d.error||r.statusText);return d} +function notice(msg,type='alert'){ $('notice').innerHTML=`
${esc(msg)}
`;setTimeout(()=>{if($('notice').textContent===msg)$('notice').innerHTML=''},5000)} +function fmt(n){return Number(n||0).toLocaleString('de-DE')} function dt(s){if(!s)return '';const d=new Date(s);return isNaN(d)?esc(s):d.toLocaleString('de-DE')} +function sev(s){return `${esc(s)}`} +function switcher(on,call){return ``} +function setView(v){currentView=v;document.querySelectorAll('.view').forEach(x=>x.classList.remove('active'));$('view-'+v).classList.add('active');document.querySelectorAll('.nav button').forEach(x=>x.classList.toggle('active',x.dataset.view===v));$('viewTitle').textContent=titles[v][0];$('viewSub').textContent=titles[v][1];refreshCurrent()} +document.querySelectorAll('.nav button').forEach(b=>b.onclick=()=>setView(b.dataset.view)); +function refreshCurrent(){({dashboard:loadDashboard,events:loadEvents,detections:loadDetections,rules:loadRules,suppressions:loadSuppressions,agents:loadAgents}[currentView]||(()=>{}))()} +async function loadDashboard(){try{const [s,a]=await Promise.all([api('/api/summary'),api('/api/analytics?hours=24')]);$('kEvents').textContent=fmt(s.events_24h);$('kHosts').textContent=fmt(s.active_hosts);$('kHigh').textContent=fmt((s.detections?.high||0)+(s.detections?.critical||0));$('kRules').textContent=fmt(s.rules_enabled);$('kRuleSets').textContent=fmt(s.rule_sets_enabled)+' Rule-Sets';$('kPipeline').textContent='OK';drawSpark(a.series||[]);bars('topHosts',a.top_hosts||[],'host_name','events');bars('topCodes',a.top_event_codes||[],'event_code','events',x=>'Event '+x);bars('authBars',a.authentication||[],'event_code','events',authName)}catch(e){$('kPipeline').textContent='FEHLER';$('kPipeline').style.color='var(--red)';notice(e.message)}} +function authName(x){return ({4624:'4624 Successful Logon',4625:'4625 Failed Logon',4740:'4740 Lockout',4771:'4771 Kerberos Failure',4776:'4776 NTLM'}[x]||('Event '+x))} +function bars(id,rows,key,val,labelFn){const max=Math.max(1,...rows.map(x=>Number(x[val]||0)));$(id).innerHTML=rows.length?rows.map(x=>`
${esc(labelFn?labelFn(x[key]):x[key])}
${fmt(x[val])}
`).join(''):'
Keine Daten
'} +function drawSpark(rows){const svg=$('eventSpark');if(!rows.length){svg.innerHTML='Keine Daten';return}const vals=rows.map(x=>Number(x.events||0)),max=Math.max(1,...vals),w=900,h=190;const pts=vals.map((v,i)=>`${i/(Math.max(1,vals.length-1))*w},${h-v/max*(h-20)}`).join(' ');svg.innerHTML=``} +async function loadEvents(){try{const q=new URLSearchParams({hours:$('eHours').value,limit:'500'});[['query','eQuery'],['host','eHost'],['user','eUser'],['ip','eIP'],['event_code','eCode'],['channel','eChannel']].forEach(([k,id])=>{const v=$(id).value.trim();if(v)q.set(k,v)});const d=await api('/api/events?'+q);window._events=d;$('eventRows').innerHTML=d.length?d.map((x,i)=>`${dt(x.event_time)}${esc(x.host_name)}
${esc(x.provider)}${esc(x.event_code)}
${esc(x.channel)}${esc(x.target_user||x.user_name||x.subject_user)}${esc(x.source_ip)}${esc(x.workstation)}${esc((x.process_path||x.command_line||x.message||'').slice(0,180))}`).join(''):'Keine Events im gewählten Zeitraum'}catch(e){notice(e.message)}} +function showEvent(i){const x=window._events[i]||{};const attrs=typeof x.attributes==='object'?JSON.stringify(x.attributes,null,2):String(x.attributes||'');$('drawerBody').innerHTML=`
${[['Zeit',dt(x.event_time)],['Host',x.host_name],['Provider',x.provider],['Channel',x.channel],['Event-ID',x.event_code],['Action',(x.category||'')+' / '+(x.action||'')+' / '+(x.outcome||'')],['User',x.target_user||x.user_name||x.subject_user],['Source-IP',x.source_ip],['Destination-IP',x.destination_ip],['Workstation',x.workstation],['Logon Type',x.logon_type],['Auth Package',x.authentication_package],['Status',x.status_code],['Failure Reason',x.failure_reason],['Process',x.process_path],['Parent',x.parent_process_path],['Raw Object',x.raw_object_key]].map(([k,v])=>`
${k}
${esc(v)}
`).join('')}

Command line

${esc(x.command_line)}

Message

${esc(x.message)}

Restattribute

${esc(attrs)}
`;$('drawer').classList.add('open')} +function closeDrawer(){$('drawer').classList.remove('open')} +async function loadDetections(){try{const d=await api('/api/detections?limit=700');$('detectionRows').innerHTML=d.length?d.map(x=>`${dt(x.last_seen)}
${fmt(x.count)} Treffer${sev(x.severity)}${esc(x.rule_name)}
${esc(x.rule_id)}${esc(x.hostname)}
${esc(x.user_name)}
${esc(x.source_ip||x.workstation)}${esc(x.summary)}${(x.mitre||[]).map(t=>`${esc(t)}`).join('')}`).join(''):'Keine Detections'}catch(e){notice(e.message)}} +async function setDetectionStatus(id,status){try{await api('/api/detections/status',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({id,status})});loadDashboard()}catch(e){notice(e.message)}} +async function loadRules(){try{const [sets,rules]=await Promise.all([api('/api/rule-sets'),api('/api/rules')]);rulesCache=rules;$('ruleSets').innerHTML=sets.map(x=>`

${esc(x.name)}

${esc(x.id)} · v${esc(x.version)}

${esc(x.description)}

${fmt(x.enabled_rules)}/${fmt(x.rule_count)} Regeln${switcher(x.enabled,`toggleRuleSet('${js(x.id)}',this.checked)`)}
`).join('');renderRules()}catch(e){notice(e.message)}} +function renderRules(){const q=$('ruleSearch').value.toLowerCase(),sv=$('ruleSeverity').value;const r=rulesCache.filter(x=>(!q||(x.title+' '+x.id+' '+x.rule_set_name).toLowerCase().includes(q))&&(!sv||x.severity===sv));$('ruleRows').innerHTML=r.length?r.map(x=>`${switcher(x.enabled,`toggleRule('${js(x.id)}',this.checked)`)}${sev(x.severity)}${esc(x.title)}
${esc(x.id)}
${esc(x.description)}${esc(x.rule_set_name)} ${x.source==='custom'?'custom':''}${esc(x.kind)}${(x.event_codes||[]).map(v=>`${esc(v)}`).join('')}${fmt(x.window_seconds)}s / ≥${fmt(x.threshold||1)}${(x.mitre||[]).map(v=>`${esc(v)}`).join('')}`).join(''):'Keine Regeln'} +async function toggleRule(id,enabled){try{await api('/api/rules/toggle',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({id,enabled})});await loadRules()}catch(e){notice(e.message)}} +async function toggleRuleSet(id,enabled){try{await api('/api/rule-sets/toggle',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({id,enabled})});await loadRules()}catch(e){notice(e.message)}} +function openRuleModal(){$('ruleModal').classList.add('open')} function openSuppressionModal(){$('suppModal').classList.add('open')} function closeModal(id){$(id).classList.remove('open')} +function csv(v){return v.split(',').map(x=>x.trim()).filter(Boolean)} function js(v){return String(v).replace(/\\/g,'\\\\').replace(/'/g,"\\'")} +async function saveRule(){try{const cond=$('rCondField').value.trim()? [{field:$('rCondField').value.trim(),operator:$('rCondOp').value,value:$('rCondValue').value}]:[];const data={id:$('rID').value.trim(),title:$('rTitle').value.trim(),description:$('rDesc').value.trim(),severity:$('rSeverity').value,score:Number($('rScore').value||0),enabled:true,kind:$('rKind').value,channels:csv($('rChannels').value),event_codes:csv($('rCodes').value).map(Number).filter(Number.isFinite),conditions:cond,group_by:csv($('rGroup').value),threshold:Number($('rThreshold').value||1),distinct_field:$('rDistinctField').value.trim(),distinct_threshold:Number($('rDistinctThreshold').value||0),window_seconds:Number($('rWindow').value||300),suppress_seconds:Number($('rSuppress').value||0),summary:$('rSummary').value.trim(),tags:csv($('rTags').value).length?csv($('rTags').value):['custom'],mitre:csv($('rMitre').value)};await api('/api/rules/save',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(data)});closeModal('ruleModal');notice('Regel gespeichert','success');loadRules()}catch(e){notice(e.message)}} +async function loadSuppressions(){try{const d=await api('/api/suppressions');$('suppRows').innerHTML=d.length?d.map(x=>`${esc(x.rule_id||'*')}${esc(x.host_pattern||'*')}${esc(x.user_pattern||'*')}${esc(x.source_ip_pattern||'*')}${x.expires_at?dt(x.expires_at):'unbegrenzt'}${esc(x.reason)}`).join(''):'Keine Suppressions'}catch(e){notice(e.message)}} +async function saveSuppression(){try{const exp=$('sExpires').value?new Date($('sExpires').value).toISOString():null;await api('/api/suppressions',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({rule_id:$('sRule').value.trim(),host_pattern:$('sHost').value.trim(),user_pattern:$('sUser').value.trim(),source_ip_pattern:$('sIP').value.trim(),reason:$('sReason').value.trim(),enabled:true,expires_at:exp})});closeModal('suppModal');notice('Suppression erstellt','success');loadSuppressions()}catch(e){notice(e.message)}} +async function deleteSuppression(id){if(!confirm('Suppression wirklich löschen?'))return;try{await api('/api/suppressions/delete',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({id})});loadSuppressions()}catch(e){notice(e.message)}} +async function loadAgents(){try{const d=await api('/api/agents');const now=Date.now();$('agentRows').innerHTML=d.length?d.map(x=>{const online=x.enabled&&(now-new Date(x.last_seen).getTime()<15*60e3);return `${switcher(x.enabled,`toggleAgent('${js(x.id)}',this.checked)`)}${esc(x.hostname)}
${esc(x.id)}${esc(x.last_ip)}${dt(x.first_seen)}${dt(x.last_seen)}${online?'online':'offline'}`}).join(''):'Keine Agents'}catch(e){notice(e.message)}} +async function toggleAgent(id,enabled){try{await api('/api/agents/toggle',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({id,enabled})});loadAgents()}catch(e){notice(e.message)}} +loadDashboard();setInterval(()=>{if(currentView==='dashboard')loadDashboard()},30000); + +