Funktionsrollback

2026-07-24 07:17:38 +02:00
parent a4ff984914
commit e9d9583f28
38 changed files with 3243 additions and 364 deletions
--- a/deploy.ps1
+++ b/deploy.ps1
@@ -19,61 +19,109 @@ function Read-EnvFile([string]$path) {
  return $map
 }

+function Write-EnvFile($template, $current) {
+  $lines = New-Object System.Collections.Generic.List[string]
+  $templateKeys = New-Object 'System.Collections.Generic.HashSet[string]'
+  foreach ($line in [IO.File]::ReadAllLines(".env.example")) {
+    if ($line -match '^([^#=][^=]*)=(.*)$') {
+      $key = $matches[1]
+      [void]$templateKeys.Add($key)
+      $lines.Add("$key=$($current[$key])")
+    } else { $lines.Add($line) }
+  }
+  foreach ($key in $current.Keys) {
+    if (-not $templateKeys.Contains([string]$key)) { $lines.Add("$key=$($current[$key])") }
+  }
+  $utf8NoBom = New-Object System.Text.UTF8Encoding($false)
+  [IO.File]::WriteAllLines((Join-Path $PSScriptRoot '.env'), $lines, $utf8NoBom)
+}
+
+function Port-Owner([int]$Port) {
+  $needle = ":$Port->"
+  foreach ($line in (& docker ps --format '{{.Names}} {{.Ports}}' 2>$null)) {
+    if ($line.Contains($needle)) { return ($line -split ' ')[0] }
+  }
+  return $null
+}
+
+function Ensure-Port($current, [string]$Key, [int]$Fallback) {
+  $value = [int]$current[$Key]
+  $owner = Port-Owner $value
+  if ([string]::IsNullOrWhiteSpace($owner) -or $owner.StartsWith('greenfield-siem-')) { return }
+  $auto = "$($current['AUTO_PORTS'])".ToLower() -in @('1','true','yes','on')
+  if (-not $auto) { throw "$Key=$value ist bereits durch '$owner' belegt." }
+  for ($p=$Fallback; $p -lt ($Fallback+100); $p++) {
+    if ([string]::IsNullOrWhiteSpace((Port-Owner $p))) {
+      Write-Host "Portkonflikt: $Key=$value wird von '$owner' benutzt -> verwende $p."
+      $current[$Key] = "$p"
+      return
+    }
+  }
+  throw "Kein freier Ersatzport fuer $Key gefunden."
+}
+
+function Test-ClickHouseImage([string]$Image) {
+  Write-Host "Pruefe ClickHouse-Binary '$Image' auf dieser CPU ..."
+  & docker pull $Image *> $null
+  if ($LASTEXITCODE -ne 0) { return $false }
+  & docker run --rm --entrypoint clickhouse $Image local --query 'SELECT 1' *> $null
+  return ($LASTEXITCODE -eq 0)
+}
+
 $template = Read-EnvFile ".env.example"
 $current = Read-EnvFile ".env"
-foreach ($k in $template.Keys) {
-  if (-not $current.Contains($k)) { $current[$k] = $template[$k] }
-}
+foreach ($k in $template.Keys) { if (-not $current.Contains($k)) { $current[$k] = $template[$k] } }

 $presetEnrollment = $env:ENROLLMENT_KEY
 $secretLengths = @{
-  POSTGRES_PASSWORD=24; CLICKHOUSE_PASSWORD=24; UI_PASSWORD=18;
+  POSTGRES_PASSWORD=24; CLICKHOUSE_PASSWORD=24; CLICKHOUSE_GRAFANA_PASSWORD=24;
+  UI_PASSWORD=18; GRAFANA_ADMIN_PASSWORD=18;
  GARAGE_SECRET_KEY=32; GARAGE_RPC_SECRET=32; GARAGE_ADMIN_TOKEN=32; GARAGE_METRICS_TOKEN=32
 }
 foreach ($k in $secretLengths.Keys) {
-  if ([string]::IsNullOrWhiteSpace($current[$k]) -or $current[$k] -eq 'CHANGE_ME') {
-    $current[$k] = Hex $secretLengths[$k]
+  if ([string]::IsNullOrWhiteSpace($current[$k]) -or $current[$k] -eq 'CHANGE_ME') { $current[$k] = Hex $secretLengths[$k] }
+}
+if ([string]::IsNullOrWhiteSpace($current['GARAGE_ACCESS_KEY']) -or $current['GARAGE_ACCESS_KEY'] -eq 'CHANGE_ME') { $current['GARAGE_ACCESS_KEY'] = "GK$(Hex 16)" }
+if (-not [string]::IsNullOrWhiteSpace($presetEnrollment)) { $current['ENROLLMENT_KEY'] = $presetEnrollment }
+elseif ([string]::IsNullOrWhiteSpace($current['ENROLLMENT_KEY']) -or $current['ENROLLMENT_KEY'] -eq 'CHANGE_ME') { $current['ENROLLMENT_KEY'] = Hex 32 }
+
+foreach ($key in @('UI_PORT','INGRESS_PORT','CLICKHOUSE_HTTP_PORT','CLICKHOUSE_NATIVE_PORT','POSTGRES_PORT','REDPANDA_KAFKA_PORT','REDPANDA_ADMIN_PORT','REDPANDA_CONSOLE_PORT','GARAGE_S3_PORT','GARAGE_ADMIN_PORT','PROMETHEUS_PORT','GRAFANA_PORT')) {
+  if ([string]::IsNullOrWhiteSpace($current[$key])) { throw ".env: $key fehlt." }
+}
+Ensure-Port $current 'UI_PORT' 18080
+Ensure-Port $current 'INGRESS_PORT' 18090
+Ensure-Port $current 'CLICKHOUSE_HTTP_PORT' 18123
+Ensure-Port $current 'CLICKHOUSE_NATIVE_PORT' 19000
+Ensure-Port $current 'POSTGRES_PORT' 15432
+Ensure-Port $current 'REDPANDA_KAFKA_PORT' 29092
+Ensure-Port $current 'REDPANDA_ADMIN_PORT' 29644
+Ensure-Port $current 'REDPANDA_CONSOLE_PORT' 18081
+Ensure-Port $current 'GARAGE_S3_PORT' 13900
+Ensure-Port $current 'GARAGE_ADMIN_PORT' 13903
+Ensure-Port $current 'PROMETHEUS_PORT' 19090
+Ensure-Port $current 'GRAFANA_PORT' 13000
+
+$probe = "$($current['CLICKHOUSE_CPU_PROBE'])".ToLower() -in @('1','true','yes','on')
+if ($probe) {
+  if (-not (Test-ClickHouseImage $current['CLICKHOUSE_IMAGE'])) {
+    $fallback = $current['CLICKHOUSE_FALLBACK_IMAGE']
+    if (-not [string]::IsNullOrWhiteSpace($fallback) -and $fallback -ne $current['CLICKHOUSE_IMAGE'] -and (Test-ClickHouseImage $fallback)) {
+      Write-Host "ClickHouse-Fallback laeuft. Verwende $fallback."
+      $current['CLICKHOUSE_IMAGE'] = $fallback
+    } else {
+      throw "ClickHouse-Binary ist mit der CPU/VM inkompatibel. Auf KVM/Proxmox CPU-Passthrough bzw. CPU-Modell 'host' pruefen."
+    }
  }
 }
-if ([string]::IsNullOrWhiteSpace($current['GARAGE_ACCESS_KEY']) -or $current['GARAGE_ACCESS_KEY'] -eq 'CHANGE_ME') {
-  $current['GARAGE_ACCESS_KEY'] = "GK$(Hex 16)"
-}
-if (-not [string]::IsNullOrWhiteSpace($presetEnrollment)) {
-  $current['ENROLLMENT_KEY'] = $presetEnrollment
-} elseif ([string]::IsNullOrWhiteSpace($current['ENROLLMENT_KEY']) -or $current['ENROLLMENT_KEY'] -eq 'CHANGE_ME') {
-  $current['ENROLLMENT_KEY'] = Hex 32
-}

-# Write a complete UTF-8 .env. This fixes partial .env files from earlier releases
-# and preserves custom keys that are not part of the current template.
-$lines = New-Object System.Collections.Generic.List[string]
-$templateKeys = New-Object 'System.Collections.Generic.HashSet[string]'
-foreach ($line in [IO.File]::ReadAllLines(".env.example")) {
-  if ($line -match '^([^#=][^=]*)=(.*)$') {
-    $key = $matches[1]
-    [void]$templateKeys.Add($key)
-    $lines.Add("$key=$($current[$key])")
-  } else {
-    $lines.Add($line)
-  }
-}
-foreach ($key in $current.Keys) {
-  if (-not $templateKeys.Contains([string]$key)) {
-    $lines.Add("$key=$($current[$key])")
-  }
-}
-$utf8NoBom = New-Object System.Text.UTF8Encoding($false)
-[IO.File]::WriteAllLines((Join-Path $PSScriptRoot '.env'), $lines, $utf8NoBom)
+Write-EnvFile $template $current

-foreach ($key in @('POSTGRES_PASSWORD','CLICKHOUSE_PASSWORD','UI_PASSWORD','ENROLLMENT_KEY','GARAGE_ACCESS_KEY','GARAGE_SECRET_KEY','GARAGE_RPC_SECRET','GARAGE_ADMIN_TOKEN','GARAGE_METRICS_TOKEN')) {
-  if ([string]::IsNullOrWhiteSpace($current[$key]) -or $current[$key] -eq 'CHANGE_ME') {
-    throw "Ungueltige .env: $key fehlt oder ist noch CHANGE_ME."
-  }
+foreach ($key in @('POSTGRES_PASSWORD','CLICKHOUSE_PASSWORD','CLICKHOUSE_GRAFANA_PASSWORD','UI_PASSWORD','GRAFANA_ADMIN_PASSWORD','ENROLLMENT_KEY','GARAGE_ACCESS_KEY','GARAGE_SECRET_KEY','GARAGE_RPC_SECRET','GARAGE_ADMIN_TOKEN','GARAGE_METRICS_TOKEN')) {
+  if ([string]::IsNullOrWhiteSpace($current[$key]) -or $current[$key] -eq 'CHANGE_ME') { throw "Ungueltige .env: $key fehlt oder ist noch CHANGE_ME." }
 }

 Write-Host "Pruefe Compose-Konfiguration ..."
 docker compose config | Out-Null
-
 Write-Host "Baue und starte Greenfield SIEM ..."
 docker compose build --pull
 docker compose up -d --remove-orphans
@@ -81,29 +129,32 @@ docker compose up -d --remove-orphans
 Write-Host "Pruefe Readiness ..."
 $healthy = $false
 for ($i = 0; $i -lt 90; $i++) {
-  $apiOk = $false; $ingressOk = $false; $chOk = $false; $garageOk = $false
-  try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.UI_PORT)/readyz" | Out-Null; $apiOk = $true } catch {}
-  try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.INGRESS_PORT)/readyz" | Out-Null; $ingressOk = $true } catch {}
-  docker compose exec -T clickhouse clickhouse-client --user $current.CLICKHOUSE_USER --password $current.CLICKHOUSE_PASSWORD --query "SELECT 1" *> $null
-  if ($LASTEXITCODE -eq 0) { $chOk = $true }
-  docker compose exec -T garage /garage status *> $null
-  if ($LASTEXITCODE -eq 0) { $garageOk = $true }
-  if ($apiOk -and $ingressOk -and $chOk -and $garageOk) { $healthy = $true; break }
+  $apiOk=$false; $ingressOk=$false; $chOk=$false; $garageOk=$false; $grafanaOk=$false
+  try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.UI_PORT)/readyz" | Out-Null; $apiOk=$true } catch {}
+  try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.INGRESS_PORT)/readyz" | Out-Null; $ingressOk=$true } catch {}
+  try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 "http://127.0.0.1:$($current.GRAFANA_PORT)/api/health" | Out-Null; $grafanaOk=$true } catch {}
+  & docker compose exec -T clickhouse clickhouse-client --user $current.CLICKHOUSE_USER --password $current.CLICKHOUSE_PASSWORD --query "SELECT 1" *> $null
+  if ($LASTEXITCODE -eq 0) { $chOk=$true }
+  & docker compose exec -T garage /garage status *> $null
+  if ($LASTEXITCODE -eq 0) { $garageOk=$true }
+  if ($apiOk -and $ingressOk -and $chOk -and $garageOk -and $grafanaOk) { $healthy=$true; break }
  Start-Sleep -Seconds 2
 }

 if (-not $healthy) {
  Write-Host "Mindestens ein Dienst wurde nicht bereit. Diagnose:" -ForegroundColor Red
  docker compose ps
-  docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor api
+  docker compose logs --tail=120 garage clickhouse clickhouse-schema postgres postgres-schema redpanda redpanda-init ingress processor detector api grafana
  throw "Deployment nicht vollstaendig bereit."
 }

 Write-Host ""
 Write-Host "Greenfield SIEM laeuft."
-Write-Host "UI:      http://127.0.0.1:$($current.UI_PORT)/ui"
-Write-Host "Ingress: http://127.0.0.1:$($current.INGRESS_PORT)/ingest"
-Write-Host "UI-Login: $($current.UI_USERNAME) / $($current.UI_PASSWORD)"
+Write-Host "Analyst UI: http://127.0.0.1:$($current.UI_PORT)/ui"
+Write-Host "Ingress:    http://127.0.0.1:$($current.INGRESS_PORT)/ingest"
+Write-Host "Grafana:    http://127.0.0.1:$($current.GRAFANA_PORT)/"
+Write-Host "UI-Login:      $($current.UI_USERNAME) / $($current.UI_PASSWORD)"
+Write-Host "Grafana-Login: $($current.GRAFANA_ADMIN_USER) / $($current.GRAFANA_ADMIN_PASSWORD)"
 Write-Host "ClickHouse: user=$($current.CLICKHOUSE_USER) db=$($current.CLICKHOUSE_DB) http=127.0.0.1:$($current.CLICKHOUSE_HTTP_PORT) native=127.0.0.1:$($current.CLICKHOUSE_NATIVE_PORT)"
 Write-Host "Enrollment-Key: $($current.ENROLLMENT_KEY)"
 Write-Host "Alle Parameter stehen vollstaendig in .env."