siem-backend/compose.yml

name: greenfield-siem

services:
  postgres:
    image: ${POSTGRES_IMAGE:-postgres:18.4-alpine}
    environment:
      POSTGRES_DB: ${POSTGRES_DB:-siem}
      POSTGRES_USER: ${POSTGRES_USER:-siem}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
    volumes:
      - postgres_data:/var/lib/postgresql
      - ./deploy/postgres/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
      interval: 5s
      timeout: 3s
      retries: 30
      start_period: 5s
    restart: unless-stopped
    ports:
      - "${POSTGRES_BIND:-127.0.0.1}:${POSTGRES_PORT:-5432}:5432"

  postgres-schema:
    image: ${POSTGRES_IMAGE:-postgres:18.4-alpine}
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      PGHOST: postgres
      PGPORT: "5432"
      PGDATABASE: ${POSTGRES_DB:-siem}
      PGUSER: ${POSTGRES_USER:-siem}
      PGPASSWORD: ${POSTGRES_PASSWORD}
    volumes:
      - ./deploy/postgres/init.sql:/schema/init.sql:ro
    entrypoint: ["/bin/sh", "-ec"]
    command: ["psql -v ON_ERROR_STOP=1 -f /schema/init.sql"]
    restart: "no"

  clickhouse:
    image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56}
    environment:
      CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem}
      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
    volumes:
      - clickhouse_data:/var/lib/clickhouse
      - clickhouse_logs:/var/log/clickhouse-server
      - ./deploy/clickhouse/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    healthcheck:
      test: ["CMD-SHELL", "clickhouse-client --host 127.0.0.1 --user $${CLICKHOUSE_USER} --password $${CLICKHOUSE_PASSWORD} --query 'SELECT 1' >/dev/null"]
      interval: 5s
      timeout: 5s
      retries: 40
      start_period: 10s
    restart: unless-stopped
    ports:
      - "${CLICKHOUSE_HTTP_BIND:-127.0.0.1}:${CLICKHOUSE_HTTP_PORT:-8123}:8123"
      - "${CLICKHOUSE_NATIVE_BIND:-127.0.0.1}:${CLICKHOUSE_NATIVE_PORT:-9000}:9000"

  clickhouse-schema:
    image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56}
    depends_on:
      clickhouse:
        condition: service_healthy
    entrypoint: ["/bin/bash", "-ec"]
    environment:
      CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem}
      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
      CLICKHOUSE_GRAFANA_PASSWORD: ${CLICKHOUSE_GRAFANA_PASSWORD}
      EVENT_RETENTION_DAYS: ${EVENT_RETENTION_DAYS:-90}
      ROLLUP_RETENTION_DAYS: ${ROLLUP_RETENTION_DAYS:-730}
    volumes:
      - ./deploy/clickhouse/init.sql.template:/schema/init.sql.template:ro
    command:
      - |
        sed \
          -e "s/__CLICKHOUSE_DB__/$${CLICKHOUSE_DB}/g" \
          -e "s/__EVENT_RETENTION_DAYS__/$${EVENT_RETENTION_DAYS}/g" \
          -e "s/__ROLLUP_RETENTION_DAYS__/$${ROLLUP_RETENTION_DAYS}/g" \
          -e "s/__GRAFANA_CLICKHOUSE_PASSWORD__/$${CLICKHOUSE_GRAFANA_PASSWORD}/g" \
          /schema/init.sql.template > /tmp/init.sql
        clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --multiquery < /tmp/init.sql
        clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \
          "ALTER TABLE $${CLICKHOUSE_DB}.events MODIFY TTL event_time + INTERVAL $${EVENT_RETENTION_DAYS} DAY DELETE"
        clickhouse-client --host clickhouse --user "$${CLICKHOUSE_USER}" --password "$${CLICKHOUSE_PASSWORD}" --query \
          "ALTER TABLE $${CLICKHOUSE_DB}.events_5m MODIFY TTL bucket + INTERVAL $${ROLLUP_RETENTION_DAYS} DAY DELETE"
    restart: "no"

  redpanda:
    image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13}
    command:
      - redpanda
      - start
      - --kafka-addr
      - internal://0.0.0.0:9092,external://0.0.0.0:19092
      - --advertise-kafka-addr
      - internal://redpanda:9092,external://localhost:19092
      - --pandaproxy-addr
      - internal://0.0.0.0:8082,external://0.0.0.0:18082
      - --advertise-pandaproxy-addr
      - internal://redpanda:8082,external://localhost:18082
      - --schema-registry-addr
      - internal://0.0.0.0:8081,external://0.0.0.0:18081
      - --rpc-addr
      - 0.0.0.0:33145
      - --advertise-rpc-addr
      - redpanda:33145
      - --mode
      - dev-container
      - --smp
      - "${REDPANDA_SMP:-2}"
      - --default-log-level=${REDPANDA_LOG_LEVEL:-info}
    volumes:
      - redpanda_data:/var/lib/redpanda/data
    healthcheck:
      test: ["CMD-SHELL", "RPK_BROKERS=localhost:9092 rpk cluster health | grep -q 'Healthy:.*true'"]
      interval: 5s
      timeout: 5s
      retries: 40
      start_period: 10s
    restart: unless-stopped
    ports:
      - "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_KAFKA_PORT:-19092}:19092"
      - "${REDPANDA_BIND:-127.0.0.1}:${REDPANDA_ADMIN_PORT:-19644}:9644"

  redpanda-init:
    image: ${REDPANDA_IMAGE:-docker.redpanda.com/redpandadata/redpanda:v26.1.13}
    depends_on:
      redpanda:
        condition: service_healthy
    entrypoint: ["/bin/sh", "-ec"]
    environment:
      RPK_BROKERS: redpanda:9092
      KAFKA_PARTITIONS: ${KAFKA_PARTITIONS:-6}
      KAFKA_RETENTION_MS: ${KAFKA_RETENTION_MS:-86400000}
      KAFKA_TOPIC: ${KAFKA_TOPIC:-siem-events}
    command:
      - |
        rpk topic create --if-not-exists -p "$${KAFKA_PARTITIONS}" -r 1 \
          -c retention.ms="$${KAFKA_RETENTION_MS}" "$${KAFKA_TOPIC}"
    restart: "no"

  redpanda-console:
    image: ${REDPANDA_CONSOLE_IMAGE:-docker.redpanda.com/redpandadata/console:v3.8.0}
    depends_on:
      redpanda:
        condition: service_healthy
    entrypoint: /bin/sh
    command: -c 'printf "%s" "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
    environment:
      CONFIG_FILEPATH: /tmp/config.yml
      CONSOLE_CONFIG_FILE: |
        kafka:
          brokers: ["redpanda:9092"]
        schemaRegistry:
          enabled: true
          urls: ["http://redpanda:8081"]
        redpanda:
          adminApi:
            enabled: true
            urls: ["http://redpanda:9644"]
    restart: unless-stopped
    ports:
      - "${REDPANDA_CONSOLE_BIND:-127.0.0.1}:${REDPANDA_CONSOLE_PORT:-8081}:8080"

  garage-config:
    image: ${ALPINE_IMAGE:-alpine:3.22}
    environment:
      GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET}
      GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN}
      GARAGE_METRICS_TOKEN: ${GARAGE_METRICS_TOKEN}
    volumes:
      - garage_config:/config
    entrypoint: ["/bin/sh", "-ec"]
    command:
      - |
        umask 077
        cat > /config/garage.toml <<CFG
        metadata_dir = "/var/lib/garage/meta"
        data_dir = "/var/lib/garage/data"
        db_engine = "sqlite"
        replication_factor = 1
        rpc_bind_addr = "0.0.0.0:3901"
        rpc_public_addr = "127.0.0.1:3901"
        rpc_secret = "$${GARAGE_RPC_SECRET}"

        [s3_api]
        s3_region = "garage"
        api_bind_addr = "0.0.0.0:3900"
        root_domain = ".s3.garage.localhost"

        [admin]
        api_bind_addr = "0.0.0.0:3903"
        admin_token = "$${GARAGE_ADMIN_TOKEN}"
        metrics_token = "$${GARAGE_METRICS_TOKEN}"
        CFG
        test -s /config/garage.toml
    restart: "no"

  garage:
    image: ${GARAGE_IMAGE:-dxflrs/garage:v2.3.0}
    command: ["/garage", "server", "--single-node", "--default-bucket"]
    environment:
      GARAGE_CONFIG_FILE: /config/garage.toml
      GARAGE_DEFAULT_ACCESS_KEY: ${GARAGE_ACCESS_KEY}
      GARAGE_DEFAULT_SECRET_KEY: ${GARAGE_SECRET_KEY}
      GARAGE_DEFAULT_BUCKET: ${GARAGE_BUCKET:-siem-raw}
      RUST_LOG: ${GARAGE_LOG_LEVEL:-garage=info}
    volumes:
      - garage_config:/config:ro
      - garage_meta:/var/lib/garage/meta
      - garage_data:/var/lib/garage/data
    depends_on:
      garage-config:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD", "/garage", "status"]
      interval: 5s
      timeout: 5s
      retries: 40
      start_period: 10s
    restart: unless-stopped
    ports:
      - "${GARAGE_BIND:-127.0.0.1}:${GARAGE_S3_PORT:-3900}:3900"
      - "${GARAGE_BIND:-127.0.0.1}:${GARAGE_ADMIN_PORT:-3903}:3903"

  ingress:
    image: ${APP_IMAGE:-greenfield-siem-app:local}
    build: .
    command: ["ingress"]
    environment: &app-env
      SERVICE_ADDR: :8080
      POSTGRES_URL: postgres://${POSTGRES_USER:-siem}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-siem}?sslmode=disable
      CLICKHOUSE_URL: http://clickhouse:8123
      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-siem}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
      KAFKA_BROKERS: redpanda:9092
      KAFKA_TOPIC: ${KAFKA_TOPIC:-siem-events}
      TENANT_ID: ${TENANT_ID:-default}
      UI_USERNAME: ${UI_USERNAME:-admin}
      UI_PASSWORD: ${UI_PASSWORD}
      ENROLLMENT_KEY: ${ENROLLMENT_KEY}
      MAX_BODY_BYTES: ${MAX_BODY_BYTES:-8388608}
      MAX_BATCH_EVENTS: ${MAX_BATCH_EVENTS:-1000}
      RAW_ARCHIVE_ENABLED: ${RAW_ARCHIVE_ENABLED:-true}
      RAW_SPOOL_DIR: /var/spool/siem-raw
      RAW_RETENTION: ${RAW_RETENTION:-720h}
      GRAFANA_PORT: ${GRAFANA_PORT:-3000}
      GRAFANA_PUBLIC_URL: ${GRAFANA_PUBLIC_URL:-}
    depends_on:
      postgres-schema:
        condition: service_completed_successfully
      redpanda-init:
        condition: service_completed_successfully
    restart: unless-stopped
    ports:
      - "${INGRESS_BIND:-0.0.0.0}:${INGRESS_PORT:-8090}:8080"

  processor:
    image: ${APP_IMAGE:-greenfield-siem-app:local}
    build: .
    command: ["processor"]
    user: "0:0"
    environment:
      <<: *app-env
      KAFKA_GROUP: ${KAFKA_GROUP:-siem-processor-v1}
    volumes:
      - raw_spool:/var/spool/siem-raw
    depends_on:
      clickhouse-schema:
        condition: service_completed_successfully
      redpanda-init:
        condition: service_completed_successfully
    restart: unless-stopped

  detector:
    image: ${APP_IMAGE:-greenfield-siem-app:local}
    build: .
    command: ["detector"]
    environment:
      <<: *app-env
      DETECTOR_INTERVAL: ${DETECTOR_INTERVAL:-30s}
      DETECTOR_LOOKBACK: ${DETECTOR_LOOKBACK:-20m}
    depends_on:
      clickhouse-schema:
        condition: service_completed_successfully
      postgres-schema:
        condition: service_completed_successfully
      processor:
        condition: service_started
    restart: unless-stopped

  api:
    image: ${APP_IMAGE:-greenfield-siem-app:local}
    build: .
    command: ["api"]
    environment:
      <<: *app-env
      UI_QUERY_LIMIT: ${UI_QUERY_LIMIT:-500}
    depends_on:
      clickhouse-schema:
        condition: service_completed_successfully
      postgres-schema:
        condition: service_completed_successfully
    restart: unless-stopped
    ports:
      - "${UI_BIND:-0.0.0.0}:${UI_PORT:-8080}:8080"

  archive-uploader:
    image: ${RCLONE_IMAGE:-rclone/rclone:1.74.4}
    entrypoint: ["/bin/sh", "-ec"]
    command:
      - |
        while true; do
          rclone move /spool garage:${GARAGE_BUCKET:-siem-raw} --exclude '*.tmp' --create-empty-src-dirs --transfers 8 --checkers 8 --retries 5 || true
          rclone delete garage:${GARAGE_BUCKET:-siem-raw} --min-age ${RAW_RETENTION:-720h} --retries 5 || true
          sleep ${ARCHIVE_UPLOAD_INTERVAL_SECONDS:-30}
        done
    environment:
      RCLONE_CONFIG_GARAGE_TYPE: s3
      RCLONE_CONFIG_GARAGE_PROVIDER: Other
      RCLONE_CONFIG_GARAGE_ACCESS_KEY_ID: ${GARAGE_ACCESS_KEY}
      RCLONE_CONFIG_GARAGE_SECRET_ACCESS_KEY: ${GARAGE_SECRET_KEY}
      RCLONE_CONFIG_GARAGE_ENDPOINT: http://garage:3900
      RCLONE_CONFIG_GARAGE_REGION: garage
      RCLONE_CONFIG_GARAGE_FORCE_PATH_STYLE: "true"
    volumes:
      - raw_spool:/spool
    depends_on:
      garage:
        condition: service_healthy
      processor:
        condition: service_started
    restart: unless-stopped

  prometheus:
    image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.13.1}
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d}
    volumes:
      - ./deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus_data:/prometheus
    depends_on:
      - ingress
      - redpanda
    restart: unless-stopped
    ports:
      - "${PROMETHEUS_BIND:-127.0.0.1}:${PROMETHEUS_PORT:-9090}:9090"

  grafana:
    image: ${GRAFANA_IMAGE:-grafana/grafana:13.1}
    environment:
      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
      GF_USERS_ALLOW_SIGN_UP: "false"
      GF_AUTH_ANONYMOUS_ENABLED: "false"
      GF_PLUGINS_PREINSTALL: grafana-clickhouse-datasource@${GRAFANA_CLICKHOUSE_PLUGIN_VERSION:-4.20.0}
      CLICKHOUSE_DB: ${CLICKHOUSE_DB:-siem}
      CLICKHOUSE_GRAFANA_PASSWORD: ${CLICKHOUSE_GRAFANA_PASSWORD}
    volumes:
      - grafana_data:/var/lib/grafana
      - ./deploy/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./deploy/grafana/dashboards:/var/lib/grafana/dashboards:ro
    depends_on:
      clickhouse-schema:
        condition: service_completed_successfully
      prometheus:
        condition: service_started
    healthcheck:
      test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"]
      interval: 10s
      timeout: 5s
      retries: 30
      start_period: 20s
    restart: unless-stopped
    ports:
      - "${GRAFANA_BIND:-0.0.0.0}:${GRAFANA_PORT:-3000}:3000"

volumes:
  postgres_data:
  clickhouse_data:
  clickhouse_logs:
  redpanda_data:
  garage_config:
  garage_meta:
  garage_data:
  raw_spool:
  prometheus_data:
  grafana_data: