Bugfix 2.0

2026-07-23 17:33:00 +02:00
parent 466ca83238
commit 9e2f48ceae
24 changed files with 796 additions and 465 deletions
@@ -2,50 +2,97 @@ name: greenfield-siem

 services:
  postgres:
-    image: postgres:18.4-alpine
+    image: ${POSTGRES_IMAGE:-postgres:18.4-alpine}
    environment:
-      POSTGRES_DB: siem
-      POSTGRES_USER: siem
+      POSTGRES_DB: ${POSTGRES_DB:-siem}
+      POSTGRES_USER: ${POSTGRES_USER:-siem}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
    volumes:
      - postgres_data:/var/lib/postgresql
      - ./deploy/postgres/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U siem -d siem"]
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
      interval: 5s
      timeout: 3s
      retries: 30
+      start_period: 5s
    restart: unless-stopped
    ports:
-      - "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
+      - "${POSTGRES_BIND:-127.0.0.1}:${POSTGRES_PORT:-5432}:5432"
+
+  postgres-schema:
+    image: ${POSTGRES_IMAGE:-postgres:18.4-alpine}
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PGHOST: postgres
+      PGPORT: "5432"
+      PGDATABASE: ${POSTGRES_DB:-siem}
+      PGUSER: ${POSTGRES_USER:-siem}
+      PGPASSWORD: ${POSTGRES_PASSWORD}
+    volumes:
+      - ./deploy/postgres/init.sql:/schema/init.sql:ro
+    entrypoint: ["/bin/sh", "-ec"]
+    command: ["psql -v ON_ERROR_STOP=1 -f /schema/init.sql"]
+    restart: "no"

  clickhouse:
-    image: clickhouse/clickhouse-server:26.6.2.81
+    image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81}
    environment:
-      CLICKHOUSE_DB: siem
-      CLICKHOUSE_USER: siem
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
    volumes:
      - clickhouse_data:/var/lib/clickhouse
      - clickhouse_logs:/var/log/clickhouse-server
-      - ./deploy/clickhouse/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
      - ./deploy/clickhouse/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    healthcheck:
-      test: ["CMD-SHELL", "clickhouse-client --user siem --password $$CLICKHOUSE_PASSWORD --query 'SELECT 1' >/dev/null"]
+      test: ["CMD-SHELL", "clickhouse-client --host 127.0.0.1 --user $${CLICKHOUSE_USER} --password $${CLICKHOUSE_PASSWORD} --query 'SELECT 1' >/dev/null"]
      interval: 5s
      timeout: 5s
-      retries: 30
+      retries: 40
+      start_period: 10s
    restart: unless-stopped
    ports:
-      - "127.0.0.1:${CLICKHOUSE_HTTP_PORT:-8123}:8123"
+      - "${CLICKHOUSE_HTTP_BIND:-127.0.0.1}:${CLICKHOUSE_HTTP_PORT:-8123}:8123"
+      - "${CLICKHOUSE_NATIVE_BIND:-127.0.0.1}:${CLICKHOUSE_NATIVE_PORT:-9000}:9000"
+
+  clickhouse-schema:
+    image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.6.2.81}
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+    entrypoint: ["/bin/bash", "-ec"]
+    environment:
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
+      EVENT_RETENTION_DAYS: ${EVENT_RETENTION_DAYS:-90}
+      ROLLUP_RETENTION_DAYS: ${ROLLUP_RETENTION_DAYS:-730}
+    volumes:
+      - ./deploy/clickhouse/init.sql.template:/schema/init.sql.template:ro
+    command:
+      - |
+        sed \
+          -e "s/__CLICKHOUSE_DB__/$${CLICKHOUSE_DB}/g" \
+          -e "s/__EVENT_RETENTION_DAYS__/$${EVENT_RETENTION_DAYS}/g" \
+          -e "s/__ROLLUP_RETENTION_DAYS__/$${ROLLUP_RETENTION_DAYS}/g" \
+          /schema/init.sql.template > /tmp/init.sql
+        clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --multiquery < /tmp/init.sql
+        clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \
+          "ALTER TABLE $${CLICKHOUSE_DB}.events MODIFY TTL event_time + INTERVAL $${EVENT_RETENTION_DAYS} DAY DELETE"
+        clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \
+          "ALTER TABLE $${CLICKHOUSE_DB}.events_5m MODIFY TTL bucket + INTERVAL $${ROLLUP_RETENTION_DAYS} DAY DELETE"
+    restart: "no"

  redpanda:
-    image: docker.redpanda.com/redpandadata/redpanda:v26.1.13
+    image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13}
    command:
      - redpanda
      - start
@@ -60,14 +107,14 @@ services:
      - --schema-registry-addr
      - internal://0.0.0.0:8081,external://0.0.0.0:18081
      - --rpc-addr
-      - redpanda:33145
+      - 0.0.0.0:33145
      - --advertise-rpc-addr
      - redpanda:33145
      - --mode
      - dev-container
      - --smp
-      - "2"
-      - --default-log-level=info
+      - "${REDPANDA_SMP:-2}"
+      - --default-log-level=${REDPANDA_LOG_LEVEL:-info}
    volumes:
      - redpanda_data:/var/lib/redpanda/data
    healthcheck:
@@ -75,32 +122,36 @@ services:
      interval: 5s
      timeout: 5s
      retries: 40
+      start_period: 10s
    restart: unless-stopped
    ports:
-      - "127.0.0.1:${REDPANDA_KAFKA_PORT:-19092}:19092"
-      - "127.0.0.1:${REDPANDA_ADMIN_PORT:-19644}:9644"
+      - "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_KAFKA_PORT:-19092}:19092"
+      - "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_ADMIN_PORT:-19644}:9644"

  redpanda-init:
-    image: docker.redpanda.com/redpandadata/redpanda:v26.1.13
+    image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13}
    depends_on:
      redpanda:
        condition: service_healthy
    entrypoint: ["/bin/sh", "-ec"]
    environment:
      RPK_BROKERS: redpanda:9092
-    command: >-
-      rpk topic create --if-not-exists -p ${KAFKA_PARTITIONS:-6} -r 1
-      -c retention.ms=${KAFKA_RETENTION_MS:-86400000}
-      siem-events
+      KAFKA_PARTITIONS: ${KAFKA_PARTITIONS:-6}
+      KAFKA_RETENTION_MS: ${KAFKA_RETENTION_MS:-86400000}
+      KAFKA_TOPIC: ${KAFKA_TOPIC:-siem-events}
+    command:
+      - |
+        rpk topic create --if-not-exists -p "$${KAFKA_PARTITIONS}" -r 1 \
+          -c retention.ms="$${KAFKA_RETENTION_MS}" "$${KAFKA_TOPIC}"
    restart: "no"

  redpanda-console:
-    image: docker.redpanda.com/redpandadata/console:v3.8.0
+    image: ${REDPANDA_CONSOLE_IMAGE:-docker.redpanda.com/redpandadata/console:v3.8.0}
    depends_on:
      redpanda:
        condition: service_healthy
    entrypoint: /bin/sh
-    command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
+    command: -c 'printf "%s" "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
    environment:
      CONFIG_FILEPATH: /tmp/config.yml
      CONSOLE_CONFIG_FILE: |
@@ -115,46 +166,93 @@ services:
            urls: ["http://redpanda:9644"]
    restart: unless-stopped
    ports:
-      - "127.0.0.1:${REDPANDA_CONSOLE_PORT:-8081}:8080"
+      - "${REDPANDA_CONSOLE_BIND:-127.0.0.1}:${REDPANDA_CONSOLE_PORT:-8081}:8080"
+
+  garage-config:
+    image: ${ALPINE_IMAGE:-alpine:3.22}
+    environment:
+      GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET}
+      GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN}
+      GARAGE_METRICS_TOKEN: ${GARAGE_METRICS_TOKEN}
+    volumes:
+      - garage_config:/config
+    entrypoint: ["/bin/sh", "-ec"]
+    command:
+      - |
+        umask 077
+        cat > /config/garage.toml <<CFG
+        metadata_dir = "/var/lib/garage/meta"
+        data_dir = "/var/lib/garage/data"
+        db_engine = "sqlite"
+        replication_factor = 1
+        rpc_bind_addr = "0.0.0.0:3901"
+        rpc_public_addr = "127.0.0.1:3901"
+        rpc_secret = "$${GARAGE_RPC_SECRET}"
+
+        [s3_api]
+        s3_region = "garage"
+        api_bind_addr = "0.0.0.0:3900"
+        root_domain = ".s3.garage.localhost"
+
+        [admin]
+        api_bind_addr = "0.0.0.0:3903"
+        admin_token = "$${GARAGE_ADMIN_TOKEN}"
+        metrics_token = "$${GARAGE_METRICS_TOKEN}"
+        CFG
+        test -s /config/garage.toml
+    restart: "no"

  garage:
-    image: dxflrs/garage:v2.3.0
+    image: ${GARAGE_IMAGE:-dxflrs/garage:v2.3.0}
    command: ["/garage", "server", "--single-node", "--default-bucket"]
    environment:
+      GARAGE_CONFIG_FILE: /config/garage.toml
      GARAGE_DEFAULT_ACCESS_KEY: ${GARAGE_ACCESS_KEY}
      GARAGE_DEFAULT_SECRET_KEY: ${GARAGE_SECRET_KEY}
      GARAGE_DEFAULT_BUCKET: ${GARAGE_BUCKET:-siem-raw}
+      RUST_LOG: ${GARAGE_LOG_LEVEL:-garage=info}
    volumes:
+      - garage_config:/config:ro
      - garage_meta:/var/lib/garage/meta
      - garage_data:/var/lib/garage/data
-      - ./deploy/garage/garage.toml:/etc/garage.toml:ro
+    depends_on:
+      garage-config:
+        condition: service_completed_successfully
+    healthcheck:
+      test: ["CMD", "/garage", "status"]
+      interval: 5s
+      timeout: 5s
+      retries: 40
+      start_period: 10s
    restart: unless-stopped
    ports:
-      - "127.0.0.1:${GARAGE_S3_PORT:-3900}:3900"
-      - "127.0.0.1:${GARAGE_ADMIN_PORT:-3903}:3903"
+      - "${GARAGE_BIND:-127.0.0.1}:${GARAGE_S3_PORT:-3900}:3900"
+      - "${GARAGE_BIND:-127.0.0.1}:${GARAGE_ADMIN_PORT:-3903}:3903"

  ingress:
-    image: greenfield-siem-app:local
+    image: ${APP_IMAGE:-greenfield-siem-app:local}
    build: .
    command: ["ingress"]
    environment: &app-env
      SERVICE_ADDR: :8080
-      POSTGRES_URL: postgres://siem:${POSTGRES_PASSWORD}@postgres:5432/siem?sslmode=disable
+      POSTGRES_URL: postgres://${POSTGRES_USER:-siem}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-siem}?sslmode=disable
      CLICKHOUSE_URL: http://clickhouse:8123
-      CLICKHOUSE_USER: siem
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
      KAFKA_BROKERS: redpanda:9092
-      KAFKA_TOPIC: siem-events
+      KAFKA_TOPIC: ${KAFKA_TOPIC:-siem-events}
      TENANT_ID: ${TENANT_ID:-default}
      UI_USERNAME: ${UI_USERNAME:-admin}
      UI_PASSWORD: ${UI_PASSWORD}
      ENROLLMENT_KEY: ${ENROLLMENT_KEY}
+      MAX_BODY_BYTES: ${MAX_BODY_BYTES:-8388608}
+      MAX_BATCH_EVENTS: ${MAX_BATCH_EVENTS:-1000}
      RAW_ARCHIVE_ENABLED: ${RAW_ARCHIVE_ENABLED:-true}
      RAW_SPOOL_DIR: /var/spool/siem-raw
      RAW_RETENTION: ${RAW_RETENTION:-720h}
    depends_on:
-      postgres:
-        condition: service_healthy
+      postgres-schema:
+        condition: service_completed_successfully
      redpanda-init:
        condition: service_completed_successfully
    restart: unless-stopped
@@ -162,61 +260,65 @@ services:
      - "${INGRESS_BIND:-0.0.0.0}:${INGRESS_PORT:-8090}:8080"

  processor:
-    image: greenfield-siem-app:local
+    image: ${APP_IMAGE:-greenfield-siem-app:local}
+    build: .
    command: ["processor"]
    user: "0:0"
    environment:
      <<: *app-env
-      KAFKA_GROUP: siem-processor-v1
+      KAFKA_GROUP: ${KAFKA_GROUP:-siem-processor-v1}
    volumes:
      - raw_spool:/var/spool/siem-raw
    depends_on:
-      clickhouse:
-        condition: service_healthy
+      clickhouse-schema:
+        condition: service_completed_successfully
      redpanda-init:
        condition: service_completed_successfully
    restart: unless-stopped

  detector:
-    image: greenfield-siem-app:local
+    image: ${APP_IMAGE:-greenfield-siem-app:local}
+    build: .
    command: ["detector"]
    environment:
      <<: *app-env
      DETECTOR_INTERVAL: ${DETECTOR_INTERVAL:-30s}
      DETECTOR_LOOKBACK: ${DETECTOR_LOOKBACK:-20m}
    depends_on:
-      clickhouse:
-        condition: service_healthy
-      postgres:
-        condition: service_healthy
+      clickhouse-schema:
+        condition: service_completed_successfully
+      postgres-schema:
+        condition: service_completed_successfully
      processor:
        condition: service_started
    restart: unless-stopped

  api:
-    image: greenfield-siem-app:local
+    image: ${APP_IMAGE:-greenfield-siem-app:local}
+    build: .
    command: ["api"]
    environment:
      <<: *app-env
      UI_QUERY_LIMIT: ${UI_QUERY_LIMIT:-500}
    depends_on:
-      clickhouse:
-        condition: service_healthy
-      postgres:
-        condition: service_healthy
+      clickhouse-schema:
+        condition: service_completed_successfully
+      postgres-schema:
+        condition: service_completed_successfully
    restart: unless-stopped
    ports:
      - "${UI_BIND:-0.0.0.0}:${UI_PORT:-8080}:8080"

  archive-uploader:
-    image: rclone/rclone:1.74.4
+    image: ${RCLONE_IMAGE:-rclone/rclone:1.74.4}
    entrypoint: ["/bin/sh", "-ec"]
-    command: |
-      while true; do
-        rclone move /spool garage:${GARAGE_BUCKET:-siem-raw} --exclude '*.tmp' --create-empty-src-dirs --transfers 8 --checkers 8 --retries 5 || true
-        rclone delete garage:${GARAGE_BUCKET:-siem-raw} --min-age ${RAW_RETENTION:-720h} --retries 5 || true
-        sleep 30
-      done
+    command:
+      - |
+        while true; do
+          rclone move /spool garage:${GARAGE_BUCKET:-siem-raw} --exclude '*.tmp' --create-empty-src-dirs --transfers 8 --checkers 8 --retries 5 || true
+          rclone delete garage:${GARAGE_BUCKET:-siem-raw} --min-age ${RAW_RETENTION:-720h} --retries 5 || true
+          sleep ${ARCHIVE_UPLOAD_INTERVAL_SECONDS:-30}
+        done
    environment:
      RCLONE_CONFIG_GARAGE_TYPE: s3
      RCLONE_CONFIG_GARAGE_PROVIDER: Other
@@ -228,13 +330,17 @@ services:
    volumes:
      - raw_spool:/spool
    depends_on:
-      - garage
-      - processor
+      garage:
+        condition: service_healthy
+      processor:
+        condition: service_started
    restart: unless-stopped

  prometheus:
-    image: prom/prometheus:v3.13.1
-    command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.retention.time=30d"]
+    image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.13.1}
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d}
    volumes:
      - ./deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus_data:/prometheus
@@ -243,13 +349,14 @@ services:
      - redpanda
    restart: unless-stopped
    ports:
-      - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
+      - "${PROMETHEUS_BIND:-127.0.0.1}:${PROMETHEUS_PORT:-9090}:9090"

 volumes:
  postgres_data:
  clickhouse_data:
  clickhouse_logs:
  redpanda_data:
+  garage_config:
  garage_meta:
  garage_data:
  raw_spool: