v1.6.0

deployments/master/monitoring/README.md

@@ -0,0 +1,29 @@
+# Prometheus / Grafana Monitoring
+
+The `monitoring` profile starts:
+- Prometheus 3.14.0
+- Grafana 13.2.0
+- node-exporter 1.12.1 on the Master
+
+Prometheus also scrapes the CPU/GPU subagent node-exporters and NVIDIA DCGM exporter using targets from the Master `.env`. NeuroForge `/metrics` is authenticated with the metrics bearer token; an init container writes that token into a Docker volume instead of embedding it in the Prometheus config.
+
+Provisioned dashboard panels include:
+- Master up, memories, synapses
+- online CPU/GPU workers
+- durable jobs by status
+- inflight/capacity by worker
+- linked/isolated/multi-linked memories
+- graph degree/components/largest component
+- HNSW and Disk-PQ population
+- HTTP request rate and P95 latency
+- Master heap
+- Master/CPU/GPU host CPU and memory
+- NVIDIA GPU utilization, framebuffer and power (when DCGM exporter is enabled)
+
+The bundled Prometheus alert rules cover Master down, missing CPU/GPU workers, failed jobs, stuck `apply_wait`, large queue, and a persistently highly-isolated graph.
+
+Validation:
+```bash
+./validate.sh
+```
+This renders the Prometheus template with the targets from the Master `.env` and runs the pinned Prometheus image's own `promtool check config`.

deployments/master/monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+providers:
+  - name: NeuroForge
+    orgId: 1
+    folder: NeuroForge
+    type: file
+    disableDeletion: true
+    updateIntervalSeconds: 30
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards

deployments/master/monitoring/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false

deployments/master/monitoring/prometheus/alerts.yml

@@ -0,0 +1,58 @@
+groups:
+  - name: neuroforge-production
+    rules:
+      - alert: NeuroForgeMasterDown
+        expr: up{job="neuroforge"} == 0
+        for: 1m
+        labels: {severity: critical}
+        annotations:
+          summary: "NeuroForge Master is down"
+          description: "Prometheus cannot scrape the NeuroForge Master for more than 1 minute."
+
+      - alert: NeuroForgeCPUWorkerMissing
+        expr: sum(neuroforge_workers{resource="cpu",status="online"}) < 1
+        for: 2m
+        labels: {severity: critical}
+        annotations:
+          summary: "No online CPU subagent"
+          description: "Graph relink/backfill jobs cannot converge without a CPU worker."
+
+      - alert: NeuroForgeGPUWorkerMissing
+        expr: sum(neuroforge_workers{resource="gpu",status="online"}) < 1
+        for: 2m
+        labels: {severity: warning}
+        annotations:
+          summary: "No online GPU subagent"
+          description: "Distributed chat/embed jobs have no online GPU worker."
+
+      - alert: NeuroForgeFailedJobs
+        expr: neuroforge_jobs{status="failed"} > 0
+        for: 5m
+        labels: {severity: warning}
+        annotations:
+          summary: "NeuroForge has failed durable jobs"
+          description: "At least one orchestrator job has remained failed for 5 minutes."
+
+      - alert: NeuroForgeApplyWaitStuck
+        expr: neuroforge_jobs{status="apply_wait"} > 0
+        for: 10m
+        labels: {severity: critical}
+        annotations:
+          summary: "Master apply phase appears stuck"
+          description: "A durable job has remained in apply_wait for more than 10 minutes."
+
+      - alert: NeuroForgeQueueHigh
+        expr: neuroforge_jobs{status="queued"} > 1000
+        for: 10m
+        labels: {severity: warning}
+        annotations:
+          summary: "NeuroForge queue is high"
+          description: "More than 1000 jobs have remained queued for 10 minutes."
+
+      - alert: NeuroForgeGraphHighlyIsolated
+        expr: (neuroforge_graph_memories{state="isolated"} / clamp_min(neuroforge_memories, 1)) > 0.80 and on() neuroforge_memories > 1000
+        for: 30m
+        labels: {severity: warning}
+        annotations:
+          summary: "Knowledge graph remains highly isolated"
+          description: "More than 80% of memories are still isolated after 30 minutes."

deployments/master/monitoring/prometheus/prometheus.yml.template

@@ -0,0 +1,31 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - /etc/prometheus-alerts/alerts.yml
+
+scrape_configs:
+  - job_name: neuroforge
+    metrics_path: /metrics
+    authorization:
+      type: Bearer
+      credentials_file: /etc/prometheus-secrets/neuroforge_metrics.token
+    static_configs:
+      - targets: ["neuroforge:8080"]
+
+  - job_name: master_node
+    static_configs:
+      - targets: ["node-exporter:9100"]
+
+  - job_name: cpu_subagent_node
+    static_configs:
+      - targets: ["__CPU_TARGET__"]
+
+  - job_name: gpu_subagent_node
+    static_configs:
+      - targets: ["__GPU_TARGET__"]
+
+  - job_name: gpu_dcgm
+    static_configs:
+      - targets: ["__GPU_DCGM_TARGET__"]

deployments/master/monitoring/validate.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+set -eu
+cd "$(dirname "$0")"
+command -v docker >/dev/null 2>&1 || { echo "FEHLT: docker"; exit 1; }
+ENV_FILE=../.env
+getenv() { grep -E "^$1=" "$ENV_FILE" | head -1 | cut -d= -f2-; }
+CPU_TARGET=$(getenv CPU_SUBAGENT_NODE_EXPORTER_TARGET)
+GPU_TARGET=$(getenv GPU_SUBAGENT_NODE_EXPORTER_TARGET)
+GPU_DCGM_TARGET=$(getenv GPU_SUBAGENT_DCGM_EXPORTER_TARGET)
+METRICS_TOKEN=$(getenv NEUROFORGE_METRICS_TOKEN)
+PROM_IMAGE=$(getenv PROMETHEUS_IMAGE)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+sed -e "s|__CPU_TARGET__|${CPU_TARGET}|g" \
+    -e "s|__GPU_TARGET__|${GPU_TARGET}|g" \
+    -e "s|__GPU_DCGM_TARGET__|${GPU_DCGM_TARGET}|g" \
+    prometheus/prometheus.yml.template > "$TMP/prometheus.yml"
+printf '%s' "$METRICS_TOKEN" > "$TMP/neuroforge_metrics.token"
+docker run --rm --entrypoint=/bin/promtool \
+  -v "$TMP/prometheus.yml:/etc/prometheus/prometheus.yml:ro" \
+  -v "$PWD/prometheus/alerts.yml:/etc/prometheus-alerts/alerts.yml:ro" \
+  -v "$TMP/neuroforge_metrics.token:/etc/prometheus-secrets/neuroforge_metrics.token:ro" \
+  "$PROM_IMAGE" check config /etc/prometheus/prometheus.yml
+echo "Prometheus promtool validation: OK"