v1.6.0

2026-09-02 10:26:50 +02:00
parent 8c4ce2d6c2
commit 9a4370e4df
272 changed files with 31244 additions and 5980 deletions
@@ -92,6 +92,8 @@ func (s *Server) routes() {
 	s.mux.Handle("POST /api/v1/integrations/outcomes/search", s.integrationAuth(http.HandlerFunc(s.integrationValidatedOutcomeSearch)))
 	s.mux.Handle("GET /api/v1/integrations/graph/research", s.controlReadAuth(http.HandlerFunc(s.integrationResearchGraph)))
 	s.mux.Handle("GET /api/v1/integrations/graph/brain", s.controlReadAuth(http.HandlerFunc(s.integrationBrainGraph)))
+	s.mux.Handle("GET /api/v1/integrations/graph/status", s.controlReadAuth(http.HandlerFunc(s.integrationGraphStatus)))
+	s.mux.Handle("GET /api/v1/integrations/orchestrator/status", s.controlReadAuth(http.HandlerFunc(s.integrationOrchestratorStatus)))

 	s.mux.Handle("POST /internal/v1/cluster/request-vote", s.clusterAuth(http.HandlerFunc(s.clusterRequestVote)))
 	s.mux.Handle("POST /internal/v1/cluster/heartbeat", s.clusterAuth(http.HandlerFunc(s.clusterHeartbeat)))
@@ -102,6 +104,8 @@ func (s *Server) routes() {
 	s.mux.Handle("GET /internal/v1/cluster/decision/{id}", s.clusterAuth(http.HandlerFunc(s.clusterDecision)))
 	s.mux.Handle("GET /internal/v1/cluster/status", s.clusterAuth(http.HandlerFunc(s.clusterStatus)))

+	s.mux.Handle("POST /api/v1/worker/register", s.workerAuth(http.HandlerFunc(s.workerRegister)))
+	s.mux.Handle("POST /api/v1/worker/heartbeat", s.workerAuth(http.HandlerFunc(s.workerHeartbeat)))
 	s.mux.Handle("POST /api/v1/worker/claim", s.workerAuth(http.HandlerFunc(s.workerClaim)))
 	s.mux.Handle("POST /api/v1/worker/complete", s.workerAuth(http.HandlerFunc(s.workerComplete)))

@@ -138,6 +142,12 @@ func (s *Server) routes() {
 	s.mux.Handle("GET /admin/api/knowledge/memories", s.adminAuth(http.HandlerFunc(s.adminKnowledgeMemories)))
 	s.mux.Handle("GET /admin/api/knowledge/memory/{id}", s.adminAuth(http.HandlerFunc(s.adminKnowledgeMemory)))
 	s.mux.Handle("GET /admin/api/knowledge/graph", s.adminAuth(http.HandlerFunc(s.adminKnowledgeGraph)))
+	s.mux.Handle("GET /admin/api/graph/status", s.adminAuth(http.HandlerFunc(s.adminGraphStatus)))
+	s.mux.Handle("POST /admin/api/graph/backfill", s.adminAuth(http.HandlerFunc(s.adminGraphBackfill)))
+	s.mux.Handle("GET /admin/api/orchestrator/status", s.adminAuth(http.HandlerFunc(s.adminOrchestratorStatus)))
+	s.mux.Handle("GET /admin/api/orchestrator/jobs", s.adminAuth(http.HandlerFunc(s.adminOrchestratorJobs)))
+	s.mux.Handle("POST /admin/api/orchestrator/jobs/{id}/retry", s.adminAuth(http.HandlerFunc(s.adminOrchestratorRetry)))
+	s.mux.Handle("POST /admin/api/orchestrator/jobs/{id}/cancel", s.adminAuth(http.HandlerFunc(s.adminOrchestratorCancel)))
 	s.mux.Handle("GET /admin/api/knowledge/events", s.adminAuth(http.HandlerFunc(s.adminKnowledgeEvents)))
 	s.mux.Handle("POST /admin/api/knowledge/search", s.adminAuth(http.HandlerFunc(s.adminKnowledgeSearch)))
 	s.mux.Handle("GET /admin/api/learning-policy", s.adminAuth(http.HandlerFunc(s.adminGetLearningPolicy)))
@@ -418,10 +428,84 @@ func (s *Server) stats(w http.ResponseWriter, r *http.Request) {
 	s.json(w, 200, map[string]any{"stats": s.store.Stats(), "cost": s.cost.Totals()})
 }

-func (s *Server) workerClaim(w http.ResponseWriter, r *http.Request) {
-	var q struct {
-		WorkerID string `json:"worker_id"`
+func (s *Server) integrationGraphStatus(w http.ResponseWriter, r *http.Request) {
+	cfg := s.store.Config()
+	s.json(w, http.StatusOK, map[string]any{
+		"stats": s.store.GraphStats(),
+		"config": map[string]any{
+			"max_hops": cfg.Brain.GraphMaxHops, "hop_decay": cfg.Brain.GraphHopDecay,
+			"max_expansion": cfg.Brain.GraphMaxExpansion, "min_edge_weight": cfg.Brain.GraphMinEdgeWeight,
+			"backfill_enabled": cfg.Worker.GraphBackfillEnabled, "backfill_min_degree": cfg.Worker.GraphBackfillMinDegree,
+		},
+	})
+}
+
+func (s *Server) integrationOrchestratorStatus(w http.ResponseWriter, r *http.Request) {
+	cfg := s.store.Config()
+	s.json(w, http.StatusOK, map[string]any{
+		"leader": s.store.IsOrchestratorLeader(),
+		"status": s.store.OrchestratorStatus(),
+		"config": map[string]any{
+			"lease_seconds": cfg.Worker.LeaseSeconds, "heartbeat_seconds": cfg.Worker.HeartbeatSeconds,
+			"stale_after_seconds": cfg.Worker.StaleAfterSeconds, "max_queued_jobs": cfg.Worker.MaxQueuedJobs,
+			"offload_chat": cfg.Worker.OffloadChat, "offload_embeddings": cfg.Worker.OffloadEmbeddings,
+		},
+	})
+}
+
+type workerRequest struct {
+	WorkerID       string            `json:"worker_id"`
+	ResourceClass  string            `json:"resource_class,omitempty"`
+	Capabilities   []string          `json:"capabilities,omitempty"`
+	Labels         map[string]string `json:"labels,omitempty"`
+	MaxConcurrency int               `json:"max_concurrency,omitempty"`
+	Version        string            `json:"version,omitempty"`
+	Hostname       string            `json:"hostname,omitempty"`
+	ActiveLeases   map[string]string `json:"active_leases,omitempty"`
+}
+
+func workerHeartbeatFromRequest(q workerRequest) store.WorkerHeartbeat {
+	return store.WorkerHeartbeat{
+		ID: q.WorkerID, ResourceClass: q.ResourceClass, Capabilities: q.Capabilities,
+		Labels: q.Labels, MaxConcurrency: q.MaxConcurrency, Version: q.Version,
+		Hostname: q.Hostname, ActiveLeases: q.ActiveLeases,
 	}
+}
+
+func (s *Server) workerRegister(w http.ResponseWriter, r *http.Request) {
+	var q workerRequest
+	if err := decode(r, &q); err != nil {
+		s.err(w, http.StatusBadRequest, err)
+		return
+	}
+	state, err := s.store.RegisterWorker(workerHeartbeatFromRequest(q))
+	if err != nil {
+		s.err(w, http.StatusBadRequest, err)
+		return
+	}
+	s.json(w, http.StatusOK, state)
+}
+
+func (s *Server) workerHeartbeat(w http.ResponseWriter, r *http.Request) {
+	var q workerRequest
+	if err := decode(r, &q); err != nil {
+		s.err(w, http.StatusBadRequest, err)
+		return
+	}
+	lease := s.store.Config().Worker.LeaseSeconds
+	if lease < 10 {
+		lease = 120
+	}
+	state, err := s.store.HeartbeatWorker(workerHeartbeatFromRequest(q), time.Duration(lease)*time.Second)
+	if err != nil {
+		s.err(w, http.StatusBadRequest, err)
+		return
+	}
+	s.json(w, http.StatusOK, state)
+}
+
+func (s *Server) workerClaim(w http.ResponseWriter, r *http.Request) {
+	var q workerRequest
 	if err := decode(r, &q); err != nil {
 		s.err(w, 400, err)
 		return
@@ -434,7 +518,7 @@ func (s *Server) workerClaim(w http.ResponseWriter, r *http.Request) {
 	if lease < 10 {
 		lease = 120
 	}
-	j, err := s.store.ClaimJob(q.WorkerID, time.Duration(lease)*time.Second)
+	j, err := s.store.ClaimJobForWorker(workerHeartbeatFromRequest(q), time.Duration(lease)*time.Second)
 	if err != nil {
 		s.err(w, 500, err)
 		return
@@ -445,27 +529,33 @@ func (s *Server) workerClaim(w http.ResponseWriter, r *http.Request) {
 	}
 	s.json(w, 200, j)
 }
+
 func (s *Server) workerComplete(w http.ResponseWriter, r *http.Request) {
 	var q struct {
-		WorkerID string          `json:"worker_id"`
-		JobID    string          `json:"job_id"`
-		Result   json.RawMessage `json:"result"`
-		Error    string          `json:"error"`
+		WorkerID   string          `json:"worker_id"`
+		JobID      string          `json:"job_id"`
+		LeaseToken string          `json:"lease_token,omitempty"`
+		Result     json.RawMessage `json:"result"`
+		Error      string          `json:"error"`
 	}
 	if err := decode(r, &q); err != nil {
 		s.err(w, 400, err)
 		return
 	}
-	j, err := s.store.CompleteJob(q.JobID, q.WorkerID, q.Result, q.Error)
+	j, err := s.store.CompleteJobLease(q.JobID, q.WorkerID, q.LeaseToken, q.Result, q.Error)
 	if err != nil {
-		s.err(w, 400, err)
+		s.err(w, 409, err)
 		return
 	}
-	if err := s.brain.ApplyJobResult(j); err != nil {
-		s.err(w, 500, err)
-		return
+	if j.Status == "apply_wait" && s.store.IsOrchestratorLeader() {
+		// Best-effort eager second phase. Any failure remains durable as apply_wait
+		// and is retried by the orchestrator; the worker must not recompute it.
+		_ = s.brain.ApplyAndFinalizeJob(j)
+		if cur, ok := s.store.Job(j.ID); ok {
+			j = cur
+		}
 	}
-	s.json(w, 200, map[string]bool{"ok": true})
+	s.json(w, 200, map[string]any{"ok": true, "status": j.Status, "attempts": j.Attempts, "apply_attempts": j.ApplyAttempts, "apply_error": j.ApplyError, "next_attempt_at": j.NextAttemptAt, "apply_next_attempt_at": j.ApplyNextAttemptAt})
 }

 func (s *Server) adminStatus(w http.ResponseWriter, r *http.Request) {
@@ -115,3 +115,69 @@ func (s *Server) adminPutLearningPolicy(w http.ResponseWriter, r *http.Request)
 	})
 	s.json(w, http.StatusOK, q)
 }
+
+func (s *Server) adminGraphStatus(w http.ResponseWriter, r *http.Request) {
+	s.json(w, http.StatusOK, map[string]any{
+		"graph":        s.store.GraphStats(),
+		"orchestrator": s.store.OrchestratorStatus(),
+		"config": map[string]any{
+			"max_hops":            s.store.Config().Brain.GraphMaxHops,
+			"hop_decay":           s.store.Config().Brain.GraphHopDecay,
+			"max_expansion":       s.store.Config().Brain.GraphMaxExpansion,
+			"min_edge_weight":     s.store.Config().Brain.GraphMinEdgeWeight,
+			"backfill_enabled":    s.store.Config().Worker.GraphBackfillEnabled,
+			"backfill_min_degree": s.store.Config().Worker.GraphBackfillMinDegree,
+			"backfill_max_queued": s.store.Config().Worker.GraphBackfillMaxQueued,
+		},
+	})
+}
+
+func (s *Server) adminGraphBackfill(w http.ResponseWriter, r *http.Request) {
+	planned, err := s.brain.PlanGraphBackfill()
+	if err != nil {
+		s.err(w, http.StatusInternalServerError, err)
+		return
+	}
+	s.json(w, http.StatusAccepted, map[string]any{"ok": true, "planned": planned, "graph": s.store.GraphStats()})
+}
+
+func (s *Server) adminOrchestratorStatus(w http.ResponseWriter, r *http.Request) {
+	s.json(w, http.StatusOK, s.store.OrchestratorStatus())
+}
+
+func (s *Server) adminOrchestratorJobs(w http.ResponseWriter, r *http.Request) {
+	limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
+	s.json(w, http.StatusOK, s.store.JobsSnapshot(limit, r.URL.Query().Get("status"), r.URL.Query().Get("type")))
+}
+
+func (s *Server) adminOrchestratorRetry(w http.ResponseWriter, r *http.Request) {
+	job, err := s.store.RetryJob(r.PathValue("id"))
+	if err != nil {
+		s.err(w, http.StatusConflict, err)
+		return
+	}
+	s.json(w, http.StatusOK, job)
+}
+
+func (s *Server) adminOrchestratorCancel(w http.ResponseWriter, r *http.Request) {
+	var q struct {
+		Reason string `json:"reason"`
+	}
+	// Empty bodies are valid for an operator cancellation; malformed non-empty
+	// JSON is not.
+	if r.ContentLength != 0 {
+		if err := decode(r, &q); err != nil {
+			s.err(w, http.StatusBadRequest, err)
+			return
+		}
+	}
+	if strings.TrimSpace(q.Reason) == "" {
+		q.Reason = "canceled by administrator"
+	}
+	if err := s.store.CancelJob(r.PathValue("id"), q.Reason); err != nil {
+		s.err(w, http.StatusNotFound, err)
+		return
+	}
+	job, _ := s.store.Job(r.PathValue("id"))
+	s.json(w, http.StatusOK, job)
+}
@@ -242,6 +242,8 @@ func (s *Server) metricsEndpoint(w http.ResponseWriter, r *http.Request) {
 	}

 	st := s.store.ObservabilitySnapshot()
+	orch := s.store.OrchestratorStatus()
+	graph := s.store.GraphStats()
 	costs := s.cost.Totals()
 	cfg := s.store.Config()
 	rt := currentRuntimeSnapshot()
@@ -269,11 +271,43 @@ func (s *Server) metricsEndpoint(w http.ResponseWriter, r *http.Request) {
 	promHeader(&b, "neuroforge_knowledge_events", "Current number of retained explainability/knowledge events.", "gauge")
 	promSample(&b, "neuroforge_knowledge_events", st.KnowledgeEvents)

-	promHeader(&b, "neuroforge_jobs", "Current worker jobs by bounded status class.", "gauge")
-	promSample(&b, "neuroforge_jobs", st.JobsQueued, "status", "queued")
-	promSample(&b, "neuroforge_jobs", st.JobsClaimed, "status", "claimed")
-	promSample(&b, "neuroforge_jobs", st.JobsDone, "status", "done")
-	promSample(&b, "neuroforge_jobs", st.JobsFailed, "status", "failed")
+	promHeader(&b, "neuroforge_jobs", "Current durable orchestrator jobs by status.", "gauge")
+	statuses := []string{"queued", "claimed", "retry_wait", "blocked", "apply_wait", "done", "failed", "canceled"}
+	for _, status := range statuses {
+		promSample(&b, "neuroforge_jobs", orch.JobsByStatus[status], "status", status)
+	}
+	promHeader(&b, "neuroforge_jobs_by_resource", "Current durable orchestrator jobs by resource class.", "gauge")
+	for resource, n := range orch.JobsByResource {
+		promSample(&b, "neuroforge_jobs_by_resource", n, "resource", resource)
+	}
+	promHeader(&b, "neuroforge_workers", "Registered subagents by resource class and liveness state.", "gauge")
+	workerCounts := map[string]int{}
+	for _, worker := range orch.Workers {
+		workerCounts[worker.ResourceClass+"\x00"+worker.Status]++
+	}
+	for key, n := range workerCounts {
+		parts := strings.SplitN(key, "\x00", 2)
+		promSample(&b, "neuroforge_workers", n, "resource", parts[0], "status", parts[1])
+	}
+	promHeader(&b, "neuroforge_worker_inflight", "Current claimed jobs per registered subagent.", "gauge")
+	promHeader(&b, "neuroforge_worker_capacity", "Configured concurrency capacity per registered subagent.", "gauge")
+	for _, worker := range orch.Workers {
+		promSample(&b, "neuroforge_worker_inflight", worker.Inflight, "worker", worker.ID, "resource", worker.ResourceClass)
+		promSample(&b, "neuroforge_worker_capacity", worker.MaxConcurrency, "worker", worker.ID, "resource", worker.ResourceClass)
+	}
+
+	promHeader(&b, "neuroforge_graph_memories", "Knowledge memories by graph linkage class.", "gauge")
+	promSample(&b, "neuroforge_graph_memories", graph.LinkedMemories, "state", "linked")
+	promSample(&b, "neuroforge_graph_memories", graph.IsolatedMemories, "state", "isolated")
+	promSample(&b, "neuroforge_graph_memories", graph.MultiLinked, "state", "multi_linked")
+	promHeader(&b, "neuroforge_graph_max_degree", "Maximum associative graph degree.", "gauge")
+	promSample(&b, "neuroforge_graph_max_degree", graph.MaxDegree)
+	promHeader(&b, "neuroforge_graph_average_degree", "Average associative graph degree.", "gauge")
+	promSample(&b, "neuroforge_graph_average_degree", strconv.FormatFloat(graph.AverageDegree, 'f', 6, 64))
+	promHeader(&b, "neuroforge_graph_connected_components", "Number of connected components in the associative graph.", "gauge")
+	promSample(&b, "neuroforge_graph_connected_components", graph.ConnectedComponents)
+	promHeader(&b, "neuroforge_graph_largest_component", "Memories in the largest connected component.", "gauge")
+	promSample(&b, "neuroforge_graph_largest_component", graph.LargestComponent)

 	promHeader(&b, "neuroforge_hnsw_nodes", "Current number of vectors in hot HNSW indexes.", "gauge")
 	promSample(&b, "neuroforge_hnsw_nodes", st.HNSWNodes)