groot
e0bf42bf32
mega-ci / static-release-gates (push) Failing after 11s
release-tag / release-image (push) Successful in 6m46s
mega-ci / go-quality (services/control) (push) Successful in 10m7s
mega-ci / go-quality (platform/neuroforge) (push) Successful in 10m20s
mega-ci / go-quality (services/agent) (push) Successful in 11m1s
mega-ci / go-quality (services/knowledge) (push) Successful in 11m13s
mega-ci / docker-build (push) Has been skipped
75 lines
3.1 KiB
YAML
75 lines
3.1 KiB
YAML
groups:
|
|
- name: neuroforge-production
|
|
rules:
|
|
- alert: NeuroForgeMasterDown
|
|
expr: up{job="neuroforge"} == 0
|
|
for: 1m
|
|
labels: {severity: critical}
|
|
annotations:
|
|
summary: "NeuroForge Master is down"
|
|
description: "Prometheus cannot scrape the NeuroForge Master for more than 1 minute."
|
|
|
|
- alert: NeuroForgeCPUWorkerMissing
|
|
expr: sum(neuroforge_workers{resource="cpu",status="online"}) < 1
|
|
for: 2m
|
|
labels: {severity: critical}
|
|
annotations:
|
|
summary: "No online CPU subagent"
|
|
description: "Graph relink/backfill jobs cannot converge without a CPU worker."
|
|
|
|
- alert: NeuroForgeGPUWorkerMissing
|
|
expr: sum(neuroforge_workers{resource="gpu",status="online"}) < 1
|
|
for: 2m
|
|
labels: {severity: warning}
|
|
annotations:
|
|
summary: "No online GPU subagent"
|
|
description: "Distributed chat/embed jobs have no online GPU worker."
|
|
|
|
- alert: NeuroForgeFailedJobs
|
|
expr: neuroforge_jobs{status="failed"} > 0
|
|
for: 5m
|
|
labels: {severity: warning}
|
|
annotations:
|
|
summary: "NeuroForge has failed durable jobs"
|
|
description: "At least one orchestrator job has remained failed for 5 minutes."
|
|
|
|
- alert: NeuroForgeApplyWaitStuck
|
|
expr: neuroforge_jobs{status="apply_wait"} > 0
|
|
for: 10m
|
|
labels: {severity: critical}
|
|
annotations:
|
|
summary: "Master apply phase appears stuck"
|
|
description: "A durable job has remained in apply_wait for more than 10 minutes."
|
|
|
|
- alert: NeuroForgeQueueHigh
|
|
expr: neuroforge_jobs{status="queued"} > 1000
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations:
|
|
summary: "NeuroForge queue is high"
|
|
description: "More than 1000 jobs have remained queued for 10 minutes."
|
|
|
|
- alert: NeuroForgeGraphHighlyIsolated
|
|
expr: (neuroforge_graph_memories{state="isolated"} / clamp_min(neuroforge_memories, 1)) > 0.80 and on() neuroforge_memories > 1000
|
|
for: 30m
|
|
labels: {severity: warning}
|
|
annotations:
|
|
summary: "Knowledge graph remains highly isolated"
|
|
description: "More than 80% of memories are still isolated after 30 minutes."
|
|
|
|
- alert: NeuroForgePendingPayloadHigh
|
|
expr: neuroforge_job_payload_bytes{state="pending"} > 104857600
|
|
for: 5m
|
|
labels: {severity: warning}
|
|
annotations:
|
|
summary: "NeuroForge pending job payload memory is high"
|
|
description: "Pending durable job payload/result bytes exceed 100 MiB for 5 minutes; backfill should remain below the v1.6.2 safety budget."
|
|
|
|
- alert: NeuroForgeTerminalPayloadHigh
|
|
expr: neuroforge_job_payload_bytes{state="terminal"} > 67108864
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations:
|
|
summary: "NeuroForge terminal job payload retention is high"
|
|
description: "Terminal job payload/result bytes exceed 64 MiB. Completed vector.relink jobs should be compacted automatically in v1.6.2."
|