sendnrw/siem-backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Vollständiges Redesign
release-tag / release-image (push) Failing after 5m54s
Details

Browse Source
This commit is contained in:
groot
2026-07-23 14:45:35 +02:00
parent 42db7d18f6
commit c49bfa1c34
58 changed files with 2781 additions and 18124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose.yml
+239 -162
View File
@@ -1,179 +1,256 @@
name: greenfield-siem
services:
mariadb:
image: mariadb:11.8
container_name: siem-mariadb
restart: unless-stopped
ports:
- 3307:3306
env_file:
- .env
postgres:
image: postgres:18.4-alpine
environment:
MARIADB_DATABASE: ${MARIADB_DATABASE}
MARIADB_USER: ${MARIADB_USER}
MARIADB_PASSWORD: ${MARIADB_PASSWORD}
MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD}
TZ: ${TZ}
command:
- --character-set-server=utf8mb4
- --collation-server=utf8mb4_unicode_ci
- --innodb-buffer-pool-size=512M
- --max-connections=300
POSTGRES_DB: siem
POSTGRES_USER: siem
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
volumes:
- mariadb_data:/var/lib/mysql
- ./deploy/mariadb/init:/docker-entrypoint-initdb.d:ro
- postgres_data:/var/lib/postgresql
- ./deploy/postgres/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
healthcheck:
test:
- CMD-SHELL
- mariadb-admin ping -h 127.0.0.1 -u root -p$$MARIADB_ROOT_PASSWORD
--silent
interval: 20s
timeout: 5s
retries: 10
start_period: 30s
networks:
- dockge_default
siem-backend:
image: git.send.nrw/sendnrw/siem-backend:latest
container_name: siem-backend
test: ["CMD-SHELL", "pg_isready -U siem -d siem"]
interval: 5s
timeout: 3s
retries: 30
restart: unless-stopped
env_file:
- .env
environment:
LISTEN_ADDR: ${LISTEN_ADDR}
DB_DSN: ${DB_DSN}
DB_MAX_OPEN_CONNS: ${DB_MAX_OPEN_CONNS}
DB_MAX_IDLE_CONNS: ${DB_MAX_IDLE_CONNS}
DB_CONN_MAX_LIFETIME: ${DB_CONN_MAX_LIFETIME}
DB_CONN_MAX_IDLE_TIME: ${DB_CONN_MAX_IDLE_TIME}
MAX_BODY_BYTES: ${MAX_BODY_BYTES}
HTTP_READ_TIMEOUT: ${HTTP_READ_TIMEOUT}
HTTP_WRITE_TIMEOUT: ${HTTP_WRITE_TIMEOUT}
HTTP_IDLE_TIMEOUT: ${HTTP_IDLE_TIMEOUT}
DETECTION_INTERVAL: ${DETECTION_INTERVAL}
OFFLINE_AFTER: ${OFFLINE_AFTER}
FAILED_LOGON_WINDOW: ${FAILED_LOGON_WINDOW}
FAILED_LOGON_THRESHOLD: ${FAILED_LOGON_THRESHOLD}
REBOOT_WINDOW: ${REBOOT_WINDOW}
REBOOT_THRESHOLD: ${REBOOT_THRESHOLD}
PASSWORD_SPRAY_WINDOW: ${PASSWORD_SPRAY_WINDOW}
PASSWORD_SPRAY_MIN_USERS: ${PASSWORD_SPRAY_MIN_USERS}
PASSWORD_SPRAY_MIN_ATTEMPTS: ${PASSWORD_SPRAY_MIN_ATTEMPTS}
SUCCESS_AFTER_FAILURE_WINDOW: ${SUCCESS_AFTER_FAILURE_WINDOW}
NEW_SOURCE_IP_LOOKBACK: ${NEW_SOURCE_IP_LOOKBACK}
NEW_SOURCE_IP_WINDOW: ${NEW_SOURCE_IP_WINDOW}
DETECTIONS_LIMIT: ${DETECTIONS_LIMIT}
NEW_EVENT_ID_MODE: ${NEW_EVENT_ID_MODE}
NEW_EVENT_ID_LEARNING_PERIOD: ${NEW_EVENT_ID_LEARNING_PERIOD}
NEW_EVENT_ID_CONFIRM_WINDOW: ${NEW_EVENT_ID_CONFIRM_WINDOW}
NEW_EVENT_ID_MIN_COUNT: ${NEW_EVENT_ID_MIN_COUNT}
NEW_EVENT_ID_IGNORE_CHANNELS: ${NEW_EVENT_ID_IGNORE_CHANNELS}
NEW_EVENT_ID_ALERT_CHANNELS: ${NEW_EVENT_ID_ALERT_CHANNELS}
NEW_EVENT_ID_HIGH_RISK_IDS: ${NEW_EVENT_ID_HIGH_RISK_IDS}
STORE_EVENT_ROWS: ${STORE_EVENT_ROWS}
STORE_RAW_XML: ${STORE_RAW_XML}
METADATA_CONTEXT_EVENT_IDS: ${METADATA_CONTEXT_EVENT_IDS}
METADATA_BUCKET: ${METADATA_BUCKET}
EVENT_RETENTION: ${EVENT_RETENTION}
RAW_RETENTION: ${RAW_RETENTION}
METADATA_RETENTION: ${METADATA_RETENTION}
PARTITION_MAINTENANCE_ENABLED: ${PARTITION_MAINTENANCE_ENABLED}
PARTITION_MAINTENANCE_INTERVAL: ${PARTITION_MAINTENANCE_INTERVAL}
PARTITION_INTERVAL: ${PARTITION_INTERVAL}
PARTITION_AHEAD: ${PARTITION_AHEAD}
PARTITION_BEHIND: ${PARTITION_BEHIND}
PARTITION_RETENTION: ${PARTITION_RETENTION}
TZ: ${TZ}
depends_on:
mariadb:
condition: service_healthy
ports:
- 8090:8080
healthcheck:
test:
- CMD-SHELL
- wget -qO- http://127.0.0.1:8080/healthz >/dev/null 2>&1 || exit 1
interval: 30s
timeout: 5s
retries: 5
start_period: 20s
networks:
- dockge_default
stress-agent:
profiles: ["stress"]
build:
context: .
dockerfile: Dockerfile.stress
container_name: siem-stress-agent
restart: "no"
- "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
clickhouse:
image: clickhouse/clickhouse-server:26.6.2.81
environment:
SIEM_STRESS_URL: ${SIEM_STRESS_URL}
SIEM_STRESS_HOST: ${SIEM_STRESS_HOST}
SIEM_STRESS_API_KEY: ${SIEM_STRESS_API_KEY}
SIEM_STRESS_ENROLLMENT_KEY: ${ENROLLMENT_KEY}
SIEM_STRESS_SCENARIO: ${SIEM_STRESS_SCENARIO}
SIEM_STRESS_RATE: ${SIEM_STRESS_RATE}
SIEM_STRESS_BATCH: ${SIEM_STRESS_BATCH}
SIEM_STRESS_WORKERS: ${SIEM_STRESS_WORKERS}
SIEM_STRESS_DURATION: ${SIEM_STRESS_DURATION}
SIEM_STRESS_MAX_EVENTS: ${SIEM_STRESS_MAX_EVENTS}
SIEM_STRESS_TIMEOUT: ${SIEM_STRESS_TIMEOUT}
SIEM_STRESS_CONFIRM: ${SIEM_STRESS_CONFIRM}
CLICKHOUSE_DB: siem
CLICKHOUSE_USER: siem
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
volumes:
- clickhouse_data:/var/lib/clickhouse
- clickhouse_logs:/var/log/clickhouse-server
- ./deploy/clickhouse/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
- ./deploy/clickhouse/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
ulimits:
nofile:
soft: 262144
hard: 262144
healthcheck:
test: ["CMD-SHELL", "clickhouse-client --user siem --password $$CLICKHOUSE_PASSWORD --query 'SELECT 1' >/dev/null"]
interval: 5s
timeout: 5s
retries: 30
restart: unless-stopped
ports:
- "127.0.0.1:${CLICKHOUSE_HTTP_PORT:-8123}:8123"
redpanda:
image: docker.redpanda.com/redpandadata/redpanda:v26.1.13
command:
- redpanda
- start
- --kafka-addr
- internal://0.0.0.0:9092,external://0.0.0.0:19092
- --advertise-kafka-addr
- internal://redpanda:9092,external://localhost:19092
- --pandaproxy-addr
- internal://0.0.0.0:8082,external://0.0.0.0:18082
- --advertise-pandaproxy-addr
- internal://redpanda:8082,external://localhost:18082
- --schema-registry-addr
- internal://0.0.0.0:8081,external://0.0.0.0:18081
- --rpc-addr
- redpanda:33145
- --advertise-rpc-addr
- redpanda:33145
- --mode
- dev-container
- --smp
- "2"
- --default-log-level=info
volumes:
- redpanda_data:/var/lib/redpanda/data
healthcheck:
test: ["CMD-SHELL", "RPK_BROKERS=localhost:9092 rpk cluster health | grep -q 'Healthy:.*true'"]
interval: 5s
timeout: 5s
retries: 40
restart: unless-stopped
ports:
- "127.0.0.1:${REDPANDA_KAFKA_PORT:-19092}:19092"
- "127.0.0.1:${REDPANDA_ADMIN_PORT:-19644}:9644"
redpanda-init:
image: docker.redpanda.com/redpandadata/redpanda:v26.1.13
depends_on:
siem-backend:
redpanda:
condition: service_healthy
networks:
- dockge_default
entrypoint: ["/bin/sh", "-ec"]
environment:
RPK_BROKERS: redpanda:9092
command: >-
rpk topic create --if-not-exists -p ${KAFKA_PARTITIONS:-6} -r 1
-c retention.ms=${KAFKA_RETENTION_MS:-86400000}
siem-events
restart: "no"
redpanda-console:
image: docker.redpanda.com/redpandadata/console:v3.8.0
depends_on:
redpanda:
condition: service_healthy
entrypoint: /bin/sh
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
environment:
CONFIG_FILEPATH: /tmp/config.yml
CONSOLE_CONFIG_FILE: |
kafka:
brokers: ["redpanda:9092"]
schemaRegistry:
enabled: true
urls: ["http://redpanda:8081"]
redpanda:
adminApi:
enabled: true
urls: ["http://redpanda:9644"]
restart: unless-stopped
ports:
- "127.0.0.1:${REDPANDA_CONSOLE_PORT:-8081}:8080"
garage:
image: dxflrs/garage:v2.3.0
command: ["/garage", "server", "--single-node", "--default-bucket"]
environment:
GARAGE_DEFAULT_ACCESS_KEY: ${GARAGE_ACCESS_KEY}
GARAGE_DEFAULT_SECRET_KEY: ${GARAGE_SECRET_KEY}
GARAGE_DEFAULT_BUCKET: ${GARAGE_BUCKET:-siem-raw}
volumes:
- garage_meta:/var/lib/garage/meta
- garage_data:/var/lib/garage/data
- ./deploy/garage/garage.toml:/etc/garage.toml:ro
restart: unless-stopped
ports:
- "127.0.0.1:${GARAGE_S3_PORT:-3900}:3900"
- "127.0.0.1:${GARAGE_ADMIN_PORT:-3903}:3903"
ingress:
image: greenfield-siem-app:local
build: .
command: ["ingress"]
environment: &app-env
SERVICE_ADDR: :8080
POSTGRES_URL: postgres://siem:${POSTGRES_PASSWORD}@postgres:5432/siem?sslmode=disable
CLICKHOUSE_URL: http://clickhouse:8123
CLICKHOUSE_USER: siem
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
KAFKA_BROKERS: redpanda:9092
KAFKA_TOPIC: siem-events
TENANT_ID: ${TENANT_ID:-default}
UI_USERNAME: ${UI_USERNAME:-admin}
UI_PASSWORD: ${UI_PASSWORD}
ENROLLMENT_KEY: ${ENROLLMENT_KEY}
RAW_ARCHIVE_ENABLED: ${RAW_ARCHIVE_ENABLED:-true}
RAW_SPOOL_DIR: /var/spool/siem-raw
RAW_RETENTION: ${RAW_RETENTION:-720h}
depends_on:
postgres:
condition: service_healthy
redpanda-init:
condition: service_completed_successfully
restart: unless-stopped
ports:
- "${INGRESS_BIND:-0.0.0.0}:${INGRESS_PORT:-8090}:8080"
processor:
image: greenfield-siem-app:local
command: ["processor"]
user: "0:0"
environment:
<<: *app-env
KAFKA_GROUP: siem-processor-v1
volumes:
- raw_spool:/var/spool/siem-raw
depends_on:
clickhouse:
condition: service_healthy
redpanda-init:
condition: service_completed_successfully
restart: unless-stopped
detector:
image: greenfield-siem-app:local
command: ["detector"]
environment:
<<: *app-env
DETECTOR_INTERVAL: ${DETECTOR_INTERVAL:-30s}
DETECTOR_LOOKBACK: ${DETECTOR_LOOKBACK:-20m}
depends_on:
clickhouse:
condition: service_healthy
postgres:
condition: service_healthy
processor:
condition: service_started
restart: unless-stopped
api:
image: greenfield-siem-app:local
command: ["api"]
environment:
<<: *app-env
UI_QUERY_LIMIT: ${UI_QUERY_LIMIT:-500}
depends_on:
clickhouse:
condition: service_healthy
postgres:
condition: service_healthy
restart: unless-stopped
ports:
- "${UI_BIND:-0.0.0.0}:${UI_PORT:-8080}:8080"
archive-uploader:
image: rclone/rclone:1.74.4
entrypoint: ["/bin/sh", "-ec"]
command: |
while true; do
rclone move /spool garage:${GARAGE_BUCKET:-siem-raw} --exclude '*.tmp' --create-empty-src-dirs --transfers 8 --checkers 8 --retries 5 || true
rclone delete garage:${GARAGE_BUCKET:-siem-raw} --min-age ${RAW_RETENTION:-720h} --retries 5 || true
sleep 30
done
environment:
RCLONE_CONFIG_GARAGE_TYPE: s3
RCLONE_CONFIG_GARAGE_PROVIDER: Other
RCLONE_CONFIG_GARAGE_ACCESS_KEY_ID: ${GARAGE_ACCESS_KEY}
RCLONE_CONFIG_GARAGE_SECRET_ACCESS_KEY: ${GARAGE_SECRET_KEY}
RCLONE_CONFIG_GARAGE_ENDPOINT: http://garage:3900
RCLONE_CONFIG_GARAGE_REGION: garage
RCLONE_CONFIG_GARAGE_FORCE_PATH_STYLE: "true"
volumes:
- raw_spool:/spool
depends_on:
- garage
- processor
restart: unless-stopped
prometheus:
image: prom/prometheus:latest
container_name: siem-prometheus
restart: unless-stopped
env_file:
- .env
environment:
TZ: ${TZ}
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=30d
- --web.enable-lifecycle
depends_on:
siem-backend:
condition: service_healthy
image: prom/prometheus:v3.13.1
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.retention.time=30d"]
volumes:
- ./deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./deploy/prometheus/rules:/etc/prometheus/rules:ro
- prometheus_data:/prometheus
ports:
- 9090:9090
networks:
- dockge_default
grafana:
image: grafana/grafana:latest
container_name: siem-grafana
restart: unless-stopped
env_file:
- .env
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER}
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
GF_USERS_ALLOW_SIGN_UP: "false"
GF_SERVER_ROOT_URL: http://localhost:3000
TZ: ${TZ}
depends_on:
- prometheus
volumes:
- grafana_data:/var/lib/grafana
- ./deploy/grafana/provisioning:/etc/grafana/provisioning:ro
- ./deploy/grafana/dashboards:/var/lib/grafana/dashboards:ro
- ingress
- redpanda
restart: unless-stopped
ports:
- 3090:3000
networks:
- dockge_default
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
volumes:
mariadb_data: null
prometheus_data: null
grafana_data: null
networks:
dockge_default:
external: true
postgres_data:
clickhouse_data:
clickhouse_logs:
redpanda_data:
garage_meta:
garage_data:
raw_spool:
prometheus_data: