RC-4
release-tag / release-image (push) Successful in 2m32s

This commit is contained in:
2026-08-09 18:41:47 +02:00
parent 440423c5b6
commit 94dbd4ccab
195 changed files with 3335 additions and 3501 deletions
+63 -8
View File
@@ -257,11 +257,15 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
var quality model.ArticleQualityDecision
var finalReviewEvidence []model.ResearchResult
var finalCPUQuality articlequality.Result
var bestCPUDraft model.KnowledgeArticleDraft
var bestCPUQuality articlequality.Result
hasBestCPUQuality := false
var reviewFeedback *model.ArticleQualityDecision
rewritten := false
repairAttempts := 0
cpuRepairAttempts := 0
authorResearchAttempted := false
articleTypeReconsidered := false
for {
content, wasRewritten, generationErr := e.generateArticleContent(ctx, articleRunID, selected, plan, brief, researchResults, reviewFeedback)
if generationErr != nil {
@@ -311,6 +315,40 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
metadata["trigger"] = trigger
metadata["article_type"] = normalizeArticleType(plan.ArticleType)
metadata["author_research_attempted"] = authorResearchAttempted
var validationErr *articleDraftValidationError
reconsiderable := errors.As(structureErr, &validationErr) && isOperationalArticleType(plan.ArticleType) && !articleTypeReconsidered && (validationErr.Code == "insufficient_solution_steps" || validationErr.Code == "missing_validation_steps")
if reconsiderable {
articleTypeReconsidered = true
previousType := normalizeArticleType(plan.ArticleType)
reconsideration, reconsiderErr := e.reconsiderOperationalArticleType(ctx, plan, content, selected, researchResults)
if reconsiderErr != nil {
e.Broker.Publish(model.Activity{Type: "article.type.reconsideration.failed", Source: "brain", Phase: "knowledge-planning", NodeIDs: plan.SourceNodeIDs, Message: "Die einmalige Artikeltyp-Neubewertung ist fehlgeschlagen", Strength: .34, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "previous_article_type": previousType, "reason": validationErr.Code, "error": reconsiderErr.Error()})})
} else if reconsideration.Action == "reclassify" {
plan.ArticleType = reconsideration.ArticleType
plan.Reason = strings.TrimSpace(strings.Join([]string{plan.Reason, "v10 type reconsideration: " + reconsideration.Reason}, " · "))
// Type is part of article identity. Recompute the fingerprint and avoid
// generating a duplicate non-operational draft if one already exists.
sourceFingerprint = articleSourceFingerprint(selected, plan, selectedPlanningResearch, e.articlePipelineFingerprintIdentity())
if e.hasArticleSourceFingerprint(sourceFingerprint) || e.hasEquivalentArticleDraft(selected, plan, sourceFingerprint) {
e.Broker.Publish(model.Activity{Type: "article.duplicate", Source: "brain", Phase: "knowledge-planning", NodeIDs: plan.SourceNodeIDs, Message: "Die korrigierte Artikeltyp-Variante existiert bereits", Strength: .36, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "reason": "reclassified_article_duplicate", "previous_article_type": previousType, "article_type": plan.ArticleType, "source_fingerprint": sourceFingerprint})})
return articleSynthesisOutcome{Skipped: true, Reason: "reclassified_article_duplicate", Action: plan.Action}, nil
}
if e.RuntimeSettings().ProcessingMode == "clustered" {
brief = fallbackSynthesisBrief(plan, selected)
} else if rebuiltBrief, briefErr := e.buildKnowledgeBrief(ctx, selected, researchResults, plan.ArticleType); briefErr == nil {
brief = rebuiltBrief
}
reviewFeedback = nil
cpuRepairAttempts = 0
e.Broker.Publish(model.Activity{Type: "article.type.reconsidered", Source: "brain", Phase: "knowledge-planning", NodeIDs: plan.SourceNodeIDs, Message: fmt.Sprintf("Operationaler Artikeltyp wurde einmalig von %s auf %s korrigiert", previousType, plan.ArticleType), Strength: .7, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "previous_article_type": previousType, "article_type": plan.ArticleType, "reason": reconsideration.Reason, "structure_failure": validationErr.Code, "author_research_attempted": authorResearchAttempted})})
continue
} else {
metadata["reason"] = "wrong_article_type_or_insufficient_operational_evidence"
metadata["reconsideration_reason"] = reconsideration.Reason
e.Broker.Publish(model.Activity{Type: "article.type.reconsidered", Source: "brain", Phase: "knowledge-planning", NodeIDs: plan.SourceNodeIDs, Message: "Operationaler Artikeltyp trägt trotz Evidenzsuche keinen belastbaren Lösungsablauf und wird nicht künstlich umgedeutet", Strength: .46, Metadata: articleRunMetadata(articleRunID, metadata)})
return articleSynthesisOutcome{Skipped: true, Reason: "wrong_article_type_or_insufficient_operational_evidence", Action: plan.Action}, nil
}
}
e.Broker.Publish(model.Activity{Type: "article.draft.rejected", Source: "brain", Phase: "quality-gate", NodeIDs: plan.SourceNodeIDs, Message: "Der Entwurf erfüllt die Mindeststruktur seines Artikeltyps nicht", Strength: .4, Metadata: articleRunMetadata(articleRunID, metadata)})
return articleSynthesisOutcome{Skipped: true, Reason: metadata["reason"].(string), Action: plan.Action}, nil
}
@@ -323,13 +361,18 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
return articleSynthesisOutcome{}, fmt.Errorf("deterministic article quality evaluation failed: %w", cpuErr)
}
finalCPUQuality = cpuQuality.Result
if !hasBestCPUQuality || articleCPUQualityBetter(cpuQuality.Result, bestCPUQuality) {
bestCPUDraft = draft
bestCPUQuality = cpuQuality.Result
hasBestCPUQuality = true
}
e.Broker.Publish(model.Activity{Type: "article.cpu_quality.completed", Source: map[bool]string{true: "agent", false: "brain"}[cpuQuality.Offloaded], Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: fmt.Sprintf("Modellfreie Artikelprüfung: Score %.2f · %d Wörter · %d Abschnitte", cpuQuality.Result.Score, cpuQuality.Result.WordCount, cpuQuality.Result.SectionCount), Strength: .64, Metadata: articleRunMetadata(articleRunID, map[string]any{
"trigger": trigger, "article_type": normalizeArticleType(plan.ArticleType), "passed": cpuQuality.Result.Passed, "score": cpuQuality.Result.Score,
"word_count": cpuQuality.Result.WordCount, "content_word_count": cpuQuality.Result.ContentWordCount, "section_count": cpuQuality.Result.SectionCount,
"lexical_diversity": cpuQuality.Result.LexicalDiversity, "redundancy": cpuQuality.Result.Redundancy, "evidence_alignment": cpuQuality.Result.EvidenceAlignment,
"source_utilization": cpuQuality.Result.SourceUtilization, "technical_specificity": cpuQuality.Result.TechnicalSpecificity, "type_depth_score": cpuQuality.Result.TypeDepthScore,
"hard_failures": cpuQuality.Result.HardFailures, "recommendations": cpuQuality.Result.Recommendations, "algorithm": cpuQuality.Result.Algorithm,
"no_model_call": true, "agent_offloaded": cpuQuality.Offloaded, "agent_id": cpuQuality.AgentID, "compute_ms": cpuQuality.ComputeMS, "fallback_reason": cpuQuality.FallbackReason,
"evidence_alignment_mode": "diagnostic_only_semantic_grounding_by_qwen", "no_model_call": true, "agent_offloaded": cpuQuality.Offloaded, "agent_id": cpuQuality.AgentID, "compute_ms": cpuQuality.ComputeMS, "fallback_reason": cpuQuality.FallbackReason,
})})
if !cpuQuality.Result.Passed {
if cpuRepairAttempts < 1 {
@@ -339,7 +382,18 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
e.Broker.Publish(model.Activity{Type: "article.cpu_quality.revision", Source: "brain", Phase: "knowledge-synthesis", NodeIDs: draft.SourceNodeIDs, Message: "Der modellfreie Quality-Layer fordert vor dem LLM-Review eine substanziellere Fassung an", Strength: .72, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "score": cpuQuality.Result.Score, "hard_failures": cpuQuality.Result.HardFailures, "recommendations": cpuQuality.Result.Recommendations, "repair_attempt": cpuRepairAttempts})})
continue
}
e.Broker.Publish(model.Activity{Type: "article.draft.rejected", Source: "brain", Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: "Der Artikel ist mathematisch/strukturell noch zu kurz, redundant oder nutzt die Evidenz nicht ausreichend", Strength: .46, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "reason": "article_cpu_quality_rejected", "score": cpuQuality.Result.Score, "hard_failures": cpuQuality.Result.HardFailures, "recommendations": cpuQuality.Result.Recommendations, "agent_offloaded": cpuQuality.Offloaded})})
// A repair must never erase a stronger previous draft. This mattered in
// analysis(21), where several CPU-requested rewrites became materially
// shorter. Restore the strongest structurally evaluated version for final
// diagnostics/rejection instead of silently keeping the regression.
if hasBestCPUQuality && articleCPUQualityBetter(bestCPUQuality, cpuQuality.Result) {
previousWords, previousScore := cpuQuality.Result.WordCount, cpuQuality.Result.Score
draft = bestCPUDraft
cpuQuality.Result = bestCPUQuality
finalCPUQuality = bestCPUQuality
e.Broker.Publish(model.Activity{Type: "article.cpu_quality.revision.regressed", Source: "brain", Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: "Die CPU-Revision war schwächer als die Vorfassung; die bessere Fassung bleibt erhalten", Strength: .5, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "repair_word_count": previousWords, "repair_score": previousScore, "restored_word_count": bestCPUQuality.WordCount, "restored_score": bestCPUQuality.Score, "hard_failures": bestCPUQuality.HardFailures})})
}
e.Broker.Publish(model.Activity{Type: "article.draft.rejected", Source: "brain", Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: "Der Artikel verfehlt nach einer gezielten Revision weiterhin harte mathematische Struktur-/Dichtegrenzen", Strength: .46, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "reason": "article_cpu_quality_rejected", "score": cpuQuality.Result.Score, "hard_failures": cpuQuality.Result.HardFailures, "recommendations": cpuQuality.Result.Recommendations, "agent_offloaded": cpuQuality.Offloaded})})
return articleSynthesisOutcome{Skipped: true, Reason: "article_cpu_quality_rejected", Action: plan.Action, Title: draft.Title}, nil
}
@@ -1265,11 +1319,12 @@ merge: Mehrere produktive Artikel überschneiden sich und sollten als Staging-En
skip: Kein echter Mehrwert, bloße Dublette oder ein Thema, das auch nach realistischer Recherche keinen eigenständigen Helpdesk-Nutzen hätte. Fehlende recherchierbare Fakten sind allein kein skip-Grund: Wähle in diesem Fall create, update oder merge und setze needs_research=true mit einer präzisen ersten Suchfrage.
Artikeltyp-Regeln:
- troubleshooting: konkreter Fehlerzustand, Fehlercode, Ausfall, Diagnose oder Wiederherstellung. Ein einzelner operativer Fehlercode ist troubleshooting, nicht how_to.
- how_to: bewusst auszuführende Einrichtung, Konfiguration oder Prozedur ohne primären Fehlerzustand.
- reference: mehrere Fehlercodes, Mechanismen, Statuswerte oder technische Zuordnungen, wenn kein einzelner Lösungsablauf im Mittelpunkt steht.
- concept: technische Grundlagen und Zusammenhänge ohne operativen Ablauf.
- troubleshooting: nur wenn ein konkreter Fehlerzustand, Fehlercode, Ausfall, Diagnose oder Wiederherstellung im Mittelpunkt steht UND die Quellen bereits einen realistischen ausführbaren Diagnose-/Recovery-Pfad tragen. Ein einzelner operativer Fehlercode ist troubleshooting, nicht how_to.
- how_to: nur für eine bewusst auszuführende Einrichtung, Konfiguration oder Prozedur, deren konkrete Schritte aus den Quellen ableitbar sind.
- reference: mehrere Fehlercodes, Mechanismen, Statuswerte, Profile, TTPs, Artefakte oder technische Zuordnungen, wenn kein einzelner Lösungsablauf im Mittelpunkt steht.
- concept: technische Grundlagen, Architektur und Zusammenhänge ohne operativen Ablauf.
- decision_guide: Auswahl oder Abgrenzung anhand belastbarer Kriterien.
WICHTIG: Allgemeine Härtung, Prävention, Framework-/Standard-Zuordnung, Threat-Intelligence-Profile oder technische Mechanismen sind nicht automatisch troubleshooting. Wenn die Quellen keine mindestens drei konkreten belegbaren Schritte erwarten lassen, bevorzuge reference/concept/decision_guide statt einen künstlichen operationalen Artikel zu erzwingen.
Erfinde keine Fakten. Bevorzuge konkrete Problemlösung gegenüber technischer Meta-Analyse. target_article_id ist bei update/merge zwingend eine SOURCE_NODE_ID einer produktiven Quelle. source_node_ids dürfen nur IDs aus dem Kontext enthalten. Wenn notwendige Fakten fehlen, setze needs_research=true. Gib ausschließlich JSON nach Schema zurück.`
}
@@ -2233,7 +2288,7 @@ func (e *Engine) writeKnowledgeArticleDraft(sources []articleSource, plan model.
"solution_step_count": countMarkdownNumberedSteps(draft.Answer), "validation_step_count": len(cleanArticleItems(draft.Validation)),
"productive_source_count": productionCount, "ai_source_count": aiCount, "production_ratio": productionRatio,
"source_fingerprint": fingerprint, "synthesis_model": e.Cfg.ArticleSynthesisModel, "review_model": e.Cfg.ArticleReviewModel,
"pipeline": "adaptive_generate_review/v3-quality-v2",
"pipeline": "adaptive_generate_review/v5-quality-gate-v12",
"cpu_quality_algorithm": cpuQuality.Algorithm, "cpu_quality_score": cpuQuality.Score, "cpu_quality_passed": cpuQuality.Passed,
"cpu_quality_word_count": cpuQuality.WordCount, "cpu_quality_section_count": cpuQuality.SectionCount,
},
@@ -2267,7 +2322,7 @@ func (e *Engine) writeKnowledgeArticleDraft(sources []articleSource, plan model.
"source_nodes": externalIDsFromArticleSources(sources), "source_node_ids": sourceIDs,
"productive_source_count": productionCount, "ai_source_count": aiCount, "production_ratio": productionRatio,
"generation_depth": generationDepth, "confidence": draft.Confidence, "open_questions": draft.OpenQuestions, "language": articleLanguageTag(e.Cfg.ArticleLanguage), "source_fingerprint": fingerprint,
"synthesis_model": e.Cfg.ArticleSynthesisModel, "review_model": e.Cfg.ArticleReviewModel, "pipeline": "adaptive_generate_review/v3-quality-v2",
"synthesis_model": e.Cfg.ArticleSynthesisModel, "review_model": e.Cfg.ArticleReviewModel, "pipeline": "adaptive_generate_review/v5-quality-gate-v12",
"knowledge_brief": brief, "research_query": plan.ResearchQuery, "research_material": evidence,
"grounded_research_evidence": researchEvidenceMetadata(groundedResearch),
"article_review": quality, "article_cpu_quality": cpuQuality, "review_repair_attempts": repairAttempts,
+1 -1
View File
@@ -179,7 +179,7 @@ func (e *Engine) collectAdaptiveInitialResearch(ctx context.Context, trigger str
}
func (e *Engine) articlePipelineFingerprintIdentity() string {
return fmt.Sprintf("adaptive_generate_review/v3-quality-v2|topic-guard=strict-v2|provenance=v2|lang=%s|author=%s|reviewer=%s|research=%s|repair=%d", articleLanguageTag(e.Cfg.ArticleLanguage), strings.TrimSpace(e.Cfg.ArticleSynthesisModel), strings.TrimSpace(e.Cfg.ArticleReviewModel), e.effectiveArticleResearchStrategy(), e.Cfg.ArticleReviewRepairRounds)
return fmt.Sprintf("adaptive_generate_review/v5-quality-gate-v12|topic-guard=strict-v3-relation-evidence|provenance=v2|lang=%s|author=%s|reviewer=%s|research=%s|repair=%d", articleLanguageTag(e.Cfg.ArticleLanguage), strings.TrimSpace(e.Cfg.ArticleSynthesisModel), strings.TrimSpace(e.Cfg.ArticleReviewModel), e.effectiveArticleResearchStrategy(), e.Cfg.ArticleReviewRepairRounds)
}
func articleWorkFingerprint(sources []articleSource, relation model.RelationDecision, research []model.ResearchResult, pipelineIdentity string) string {
+19
View File
@@ -128,3 +128,22 @@ func (e *Engine) evaluateArticleCPUQuality(ctx context.Context, articleRunID str
}
return articleCPUQualityExecution{Result: result.Quality, Offloaded: true, AgentID: result.AgentID, ComputeMS: result.DurationMS}, nil
}
func articleCPUQualityBetter(a, b articlequality.Result) bool {
if a.Passed != b.Passed {
return a.Passed
}
if len(a.HardFailures) != len(b.HardFailures) {
return len(a.HardFailures) < len(b.HardFailures)
}
if math.Abs(a.Score-b.Score) > 0.000001 {
return a.Score > b.Score
}
if a.WordCount != b.WordCount {
return a.WordCount > b.WordCount
}
if a.SectionCount != b.SectionCount {
return a.SectionCount > b.SectionCount
}
return a.SourceUtilization > b.SourceUtilization
}
@@ -0,0 +1,31 @@
package engine
import (
"testing"
"github.com/local/glpi-neural-brain/internal/articlequality"
)
func TestArticleCPUQualityBetterPrefersFewerHardFailures(t *testing.T) {
better := articlequality.Result{Score: .61, WordCount: 620, HardFailures: []string{"low_information_density"}}
worse := articlequality.Result{Score: .70, WordCount: 700, HardFailures: []string{"article_too_short", "low_information_density"}}
if !articleCPUQualityBetter(better, worse) {
t.Fatal("fewer hard failures must outrank a numerically higher score")
}
}
func TestArticleCPUQualityBetterPrefersHigherScoreWhenFailuresEqual(t *testing.T) {
better := articlequality.Result{Score: .72, WordCount: 640, HardFailures: []string{"article_too_short"}}
worse := articlequality.Result{Score: .66, WordCount: 700, HardFailures: []string{"article_too_short"}}
if !articleCPUQualityBetter(better, worse) {
t.Fatal("higher score should win when pass/fail and hard-failure count are equal")
}
}
func TestArticleCPUQualityBetterAlwaysPrefersPassingDraft(t *testing.T) {
better := articlequality.Result{Passed: true, Score: .60, WordCount: 600}
worse := articlequality.Result{Passed: false, Score: .90, WordCount: 900, HardFailures: []string{"high_redundancy"}}
if !articleCPUQualityBetter(better, worse) {
t.Fatal("a passing draft must always outrank a failing draft")
}
}
@@ -0,0 +1,79 @@
package engine
import (
"context"
"fmt"
"strings"
"github.com/local/glpi-neural-brain/internal/model"
)
type articleTypeReconsideration struct {
Action string `json:"action"`
ArticleType string `json:"article_type"`
Reason string `json:"reason"`
}
func articleTypeReconsiderationSchema() map[string]any {
return map[string]any{"type": "object", "properties": map[string]any{
"action": map[string]any{"type": "string", "enum": []string{"reclassify", "skip"}},
"article_type": map[string]any{"type": "string", "enum": []string{"reference", "concept", "decision_guide"}},
"reason": map[string]any{"type": "string"},
}, "required": []string{"action", "article_type", "reason"}}
}
func articleTypeReconsiderationContext(plan model.ArticlePlanDecision, content model.KnowledgeArticleContent, sources []articleSource, evidence []model.ResearchResult) string {
var b strings.Builder
fmt.Fprintf(&b, "BISHERIGER_ARTIKELTYP: %s\n", normalizeArticleType(plan.ArticleType))
fmt.Fprintf(&b, "PLANNER_ZIEL: %s\n", plan.ExpectedValue)
fmt.Fprintf(&b, "TITEL_ENTWURF: %s\n", content.Title)
fmt.Fprintf(&b, "PROBLEM: %s\n", clamp(content.ProblemDescription, 1000))
fmt.Fprintf(&b, "SCHRITTE: %d\nVALIDIERUNGEN: %d\n", len(cleanArticleItems(content.SolutionSteps)), len(cleanArticleItems(content.ValidationSteps)))
fmt.Fprintf(&b, "TECHNISCHER_HINTERGRUND: %d\nTECHNISCHE_DETAILS: %d\nMAPPINGS: %d\nBEISPIELE: %d\nGRENZEN: %d\n", len(cleanArticleItems(content.TechnicalBackground)), len(cleanArticleItems(content.TechnicalDetails)), len(cleanArticleItems(content.Mappings)), len(cleanArticleItems(content.Examples)), len(cleanArticleItems(content.Limitations)))
b.WriteString("\nINTERNE QUELLENTITEL:\n")
for _, source := range sources {
fmt.Fprintf(&b, "- %s\n", source.Node.Label)
}
if len(evidence) > 0 {
b.WriteString("\nGEPRÜFTE/VERFÜGBARE WEB-EVIDENZ:\n")
for _, item := range firstResearchResults(evidence, 8) {
fmt.Fprintf(&b, "- %s | quality=%s %.2f | relevance=%.2f\n", item.Title, item.SourceQuality, item.SourceQualityScore, item.Relevance)
}
}
return b.String()
}
func firstResearchResults(values []model.ResearchResult, limit int) []model.ResearchResult {
if limit > 0 && len(values) > limit {
return values[:limit]
}
return values
}
func (e *Engine) reconsiderOperationalArticleType(ctx context.Context, plan model.ArticlePlanDecision, content model.KnowledgeArticleContent, sources []articleSource, evidence []model.ResearchResult) (articleTypeReconsideration, error) {
result := articleTypeReconsideration{Action: "skip", ArticleType: "reference", Reason: "operational evidence is insufficient"}
system := `Du korrigierst ausschließlich die Artikeltyp-Routingentscheidung einer Wissensdatenbank. Der bisherige how_to/troubleshooting-Entwurf hat trotz gezielter Evidenzsuche weniger als drei belastbare ausführbare Schritte oder keine belastbare Validierung. Wähle NICHT erneut how_to oder troubleshooting.
Wähle genau eine Option:
- reference: wenn die Evidenz belastbare technische Zuordnungen, mehrere Profile/Techniken/Fehlercodes, Artefakte, Statuswerte oder technische Details trägt.
- concept: wenn Mechanismen, Grundlagen, Architektur oder Zusammenhänge der eigentliche Wissenswert sind.
- decision_guide: wenn belastbare Auswahl-/Abgrenzungskriterien vorliegen.
- skip: wenn die Evidenz auch für keinen dieser nicht-operationalen Typen genügend eigenständigen Wissensnutzen trägt.
Erfinde keine fehlenden Schritte. Das Ziel ist keine Umgehung des Quality-Gates, sondern eine einmalige Korrektur eines offenbar falsch gewählten operationalen Artikeltyps. Gib ausschließlich JSON nach Schema zurück.`
if err := e.Ollama.ChatJSON(ctx, system, articleTypeReconsiderationContext(plan, content, sources, evidence), articleTypeReconsiderationSchema(), &result); err != nil {
return result, err
}
result.Action = strings.ToLower(strings.TrimSpace(result.Action))
result.ArticleType = normalizeArticleType(result.ArticleType)
if result.Action != "reclassify" {
result.Action = "skip"
return result, nil
}
if result.ArticleType != "reference" && result.ArticleType != "concept" && result.ArticleType != "decision_guide" {
result.Action = "skip"
result.ArticleType = "reference"
result.Reason = nonempty(result.Reason, "reclassification did not produce a supported non-operational article type")
}
return result, nil
}
@@ -0,0 +1,61 @@
package engine
import (
"context"
"testing"
"time"
"github.com/local/glpi-neural-brain/internal/graph"
"github.com/local/glpi-neural-brain/internal/model"
)
func TestAutonomousResearchSemanticDuplicateSurvivesChangingSeedMembership(t *testing.T) {
a := model.ResearchTask{Topic: "T1133 / T1078 / T1588.001", SeedNodeIDs: []string{"a", "b", "c"}, Questions: []string{"Wie hängen T1133 T1078 und T1588.001 zusammen?"}}
b := model.ResearchTask{Topic: "T1133 / T1078 / T1588.001", SeedNodeIDs: []string{"a", "d"}, Questions: []string{"Welche Abwehrmaßnahmen betreffen T1133, T1078 und T1588.001?"}}
if !autonomousResearchSemanticDuplicate(a, b) {
t.Fatalf("same ATT&CK opportunity with changed seeds should dedupe, similarity=%.3f", autonomousResearchTaskSimilarity(a, b))
}
c := model.ResearchTask{Topic: "Windows Update CBS 0x80D02002", SeedNodeIDs: []string{"x"}}
if autonomousResearchSemanticDuplicate(a, c) {
t.Fatalf("unrelated opportunities should not dedupe, similarity=%.3f", autonomousResearchTaskSimilarity(a, c))
}
}
func TestMergeQueuedResearchTaskAndPersistentTotals(t *testing.T) {
store, err := graph.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
defer store.Close()
ctx := context.Background()
queued, created, err := store.EnqueueResearchTask(ctx, model.ResearchTask{DedupeKey: "a", Topic: "ATT&CK cluster", RequestedBy: "autonomous-scanner", Priority: .7, SeedNodeIDs: []string{"a"}, Questions: []string{"q1"}}, time.Hour)
if err != nil || !created {
t.Fatalf("enqueue failed created=%v err=%v", created, err)
}
merged, changed, err := store.MergeQueuedResearchTask(ctx, queued.ID, model.ResearchTask{Topic: "ATT&CK cluster", Priority: .9, SeedNodeIDs: []string{"b"}, Questions: []string{"q2"}})
if err != nil || !changed {
t.Fatalf("merge failed changed=%v err=%v", changed, err)
}
if merged.Priority != .9 || len(merged.SeedNodeIDs) != 2 || len(merged.Questions) != 2 {
t.Fatalf("unexpected merged task: %+v", merged)
}
leased, ok, err := store.LeaseNextResearchTask(ctx, 0, time.Minute)
if err != nil || !ok {
t.Fatalf("lease failed ok=%v err=%v", ok, err)
}
if err := store.MarkResearchTaskRunning(ctx, leased.ID); err != nil {
t.Fatal(err)
}
leased.EvidenceCount = 3
leased.ArticleCreated = true
if err := store.CompleteResearchTask(ctx, leased); err != nil {
t.Fatal(err)
}
totals, err := store.ResearchTaskPersistentTotals(ctx)
if err != nil {
t.Fatal(err)
}
if totals["completed"] != 1 || totals["evidence"] != 3 || totals["articles"] != 1 {
t.Fatalf("unexpected persistent totals: %+v", totals)
}
}
+273 -18
View File
@@ -15,7 +15,6 @@ import (
"github.com/local/glpi-neural-brain/internal/graph"
"github.com/local/glpi-neural-brain/internal/model"
"github.com/local/glpi-neural-brain/internal/ollama"
)
type autonomousCandidate struct {
@@ -48,6 +47,9 @@ func (e *Engine) startAutonomousResearch(ctx context.Context) {
if _, err := e.Graph.ResetExpiredResearchTaskLeases(ctx); err != nil {
slog.Warn("reset expired autonomous research leases failed", "error", err)
}
if _, err := e.consolidateAutonomousResearchQueue(ctx); err != nil {
slog.Warn("consolidate autonomous research queue failed", "error", err)
}
go e.autonomousResearchScanner(ctx)
go e.autonomousResearchWorker(ctx)
// Existing queued work is resumed after every restart. The scheduler scan is
@@ -55,6 +57,9 @@ func (e *Engine) startAutonomousResearch(ctx context.Context) {
e.signalAutonomousResearch()
go func() {
delay := 45 * time.Second
if e.SpeedModeEnabled() {
delay = 0
}
timer := time.NewTimer(delay)
defer timer.Stop()
select {
@@ -128,7 +133,7 @@ func (e *Engine) autonomousResearchScanner(ctx context.Context) {
}
func (e *Engine) scanAutonomousResearchOpportunities(ctx context.Context, trigger string) error {
ctx = ollama.WithLowPriority(ctx)
ctx = e.backgroundOllamaContext(ctx)
settings := e.RuntimeSettings()
candidates := buildAutonomousCandidates(e.Graph.Snapshot(), e.effectiveThinkingFilter(), e.Cfg.AutonomousResearchOpportunityLimit)
e.beginAutonomousOpportunityScan(trigger, len(candidates))
@@ -137,13 +142,38 @@ func (e *Engine) scanAutonomousResearchOpportunities(ctx context.Context, trigge
e.Broker.Publish(model.Activity{Type: "autonomous.research.scan.completed", Source: "brain", Phase: "autonomous-research", Message: "Der Graph enthält aktuell keine ausreichend starke autonome Recherchechance", Strength: .24, Metadata: map[string]any{"trigger": trigger, "candidate_count": 0, "created": 0, "decisions": []any{}, "rejection_counts": map[string]int{}}})
return nil
}
if _, err := e.consolidateAutonomousResearchQueue(ctx); err != nil {
slog.Warn("autonomous queue consolidation before scan failed", "error", err)
}
completedToday, activeQueued, remainingBudget, budgetErr := e.autonomousResearchDailyBudget(ctx, settings)
if budgetErr != nil {
return budgetErr
}
if remainingBudget <= 0 {
decisions := make([]autonomousOpportunityDecision, 0, len(candidates))
for _, candidate := range candidates {
decision := newAutonomousOpportunityDecision(candidate)
decision.RejectionReason = "daily_budget_exhausted"
decision.RecommendedAction = "wait_for_next_utc_day_or_finish_queue"
decisions = append(decisions, decision)
}
e.finishAutonomousOpportunityScan(trigger, 0, decisions)
e.Broker.Publish(model.Activity{Type: "autonomous.research.scan.completed", Source: "brain", Phase: "autonomous-research", Message: "Autonome Graphanalyse abgeschlossen · Tagesbudget beziehungsweise bereits eingeplante Arbeit schöpft die aktuelle Kapazität aus", Strength: .34, Metadata: map[string]any{"trigger": trigger, "candidate_count": len(candidates), "created": 0, "completed_today": completedToday, "active_research_tasks": activeQueued, "daily_limit": settings.AutonomousResearchMaxTasksPerDay, "remaining_budget_slots": 0, "decisions": decisions, "rejection_counts": autonomousDecisionRejectionCounts(decisions)}})
return nil
}
limit := settings.AutonomousResearchTasksPerCycle
if settings.SpeedMode {
limit = remainingBudget
}
if limit < 1 {
limit = 1
}
if limit > remainingBudget {
limit = remainingBudget
}
created := 0
decisions := make([]autonomousOpportunityDecision, 0, len(candidates))
e.Broker.Publish(model.Activity{Type: "autonomous.research.scan.started", Source: "brain", Phase: "autonomous-research", Message: fmt.Sprintf("%d Graphsignale werden als mögliche Wissenslücken bewertet", len(candidates)), Strength: .66, Metadata: map[string]any{"trigger": trigger, "candidate_count": len(candidates), "task_limit": limit}})
e.Broker.Publish(model.Activity{Type: "autonomous.research.scan.started", Source: "brain", Phase: "autonomous-research", Message: fmt.Sprintf("%d Graphsignale werden als mögliche Wissenslücken bewertet", len(candidates)), Strength: .66, Metadata: map[string]any{"trigger": trigger, "candidate_count": len(candidates), "task_limit": limit, "completed_today": completedToday, "active_research_tasks": activeQueued, "daily_limit": settings.AutonomousResearchMaxTasksPerDay, "remaining_budget_slots": remainingBudget}})
for _, candidate := range candidates {
decision := newAutonomousOpportunityDecision(candidate)
if created >= limit {
@@ -206,6 +236,30 @@ func (e *Engine) scanAutonomousResearchOpportunities(ctx context.Context, trigge
"signals": candidate.Signals,
},
}
if duplicate, found, duplicateErr := e.findAutonomousResearchSemanticDuplicate(ctx, task); duplicateErr != nil {
decision.RejectionReason = "semantic_dedupe_error"
decision.KnowledgeGap = duplicateErr.Error()
decision.RecommendedAction = "retry_next_scan"
decisions = append(decisions, decision)
continue
} else if found {
merged, changed, mergeErr := e.Graph.MergeQueuedResearchTask(ctx, duplicate.ID, task)
if mergeErr != nil {
decision.RejectionReason = "semantic_dedupe_error"
decision.KnowledgeGap = mergeErr.Error()
decision.RecommendedAction = "retry_next_scan"
decisions = append(decisions, decision)
continue
}
decision.RejectionReason = "semantic_queue_duplicate"
decision.RecommendedAction = "merged_into_existing_task"
decision.SourceNodeIDs = append([]string(nil), merged.SeedNodeIDs...)
decisions = append(decisions, decision)
if changed {
e.Broker.Publish(model.Activity{Type: "autonomous.research.task.merged", Source: "brain", Phase: "autonomous-research-queue", NodeIDs: merged.SeedNodeIDs, Message: fmt.Sprintf("Semantisch gleiche Wissenslücke wurde in vorhandene Rechercheaufgabe zusammengeführt · %s", merged.Topic), Strength: .54, Metadata: map[string]any{"task_id": merged.ID, "incoming_topic": task.Topic, "similarity": autonomousResearchTaskSimilarity(duplicate, task), "algorithm": "topic-seed-question-jaccard-v1"}})
}
continue
}
queued, wasCreated, err := e.Graph.EnqueueResearchTask(ctx, task, e.Cfg.AutonomousResearchCooldown)
if err != nil {
decision.RejectionReason = "enqueue_error"
@@ -444,7 +498,7 @@ func (e *Engine) autonomousMayUseOllama(_ bool) bool {
if !autonomousResearchRuntimeAllowed(settings, e.ResearchEnabledForRuntime()) {
return false
}
if settings.AutonomousResearchIdleOnly {
if settings.AutonomousResearchIdleOnly && !settings.SpeedMode {
if e.interactiveInflight.Load() > 0 {
return false
}
@@ -461,7 +515,7 @@ func (e *Engine) autonomousMayUseOllama(_ bool) bool {
}
}
for _, node := range e.Ollama.NodeStatuses() {
if node.Healthy && node.Compatible && node.Inflight < e.Cfg.OllamaNodeMaxInflight && time.Now().After(node.CooldownUntil) {
if node.Healthy && node.Compatible && node.Inflight < e.Ollama.NodeMaxInflight() && time.Now().After(node.CooldownUntil) {
return true
}
}
@@ -471,7 +525,7 @@ func (e *Engine) autonomousMayUseOllama(_ bool) bool {
func (e *Engine) runAutonomousResearchTask(parent context.Context, task model.ResearchTask) {
ctx, cancel := context.WithTimeout(parent, maxDuration(e.Cfg.OllamaRequestTimeout*3, 20*time.Minute))
defer cancel()
ctx = ollama.WithLowPriority(ctx)
ctx = e.backgroundOllamaContext(ctx)
if err := e.Graph.MarkResearchTaskRunning(ctx, task.ID); err != nil {
slog.Warn("mark autonomous research task running failed", "task_id", task.ID, "error", err)
return
@@ -534,6 +588,9 @@ func (e *Engine) runAutonomousResearchTask(parent context.Context, task model.Re
message = fmt.Sprintf("Autonome Recherche hat einen KB-Entwurf erstellt · %s", outcome.ArticleTitle)
}
e.Broker.Publish(model.Activity{Type: "autonomous.research.task.completed", Source: "brain", Phase: "autonomous-research", NodeIDs: task.SeedNodeIDs, Message: message, Strength: 1, Metadata: map[string]any{"task_id": task.ID, "outcome": outcome.Outcome, "evidence_count": outcome.EvidenceCount, "queries_executed": outcome.QueriesExecuted, "pages_fetched": outcome.PagesFetched, "article_created": outcome.ArticleCreated, "article_title": outcome.ArticleTitle, "article_path": outcome.ArticlePath}})
if e.SpeedModeEnabled() {
e.RequestAutonomousResearchScan("speed-drain")
}
}
type autonomousTaskOutcome struct {
@@ -638,6 +695,16 @@ func (e *Engine) executeAutonomousResearchTask(ctx context.Context, task model.R
if err != nil {
return outcome, err
}
// Autonomous synthesis bypasses EnrichOne's process counters. Keep the
// runtime dashboard truthful for autonomous article outcomes as well.
e.stateMu.Lock()
if article.Created {
e.articlesCreated++
}
if article.Skipped {
e.articlesSkipped++
}
e.stateMu.Unlock()
outcome.ArticleReason = article.Reason
if article.Created {
outcome.Outcome = "article_created"
@@ -1329,6 +1396,186 @@ func autonomousNodeSetJaccard(a, b []string) float64 {
return float64(intersection) / float64(union)
}
func autonomousResearchActiveTasks(ctx context.Context, store *graph.Store) ([]model.ResearchTask, error) {
tasks, err := store.ListResearchTasks(ctx, 500, "queued", "deferred", "reserved", "running")
if err != nil {
return nil, err
}
out := make([]model.ResearchTask, 0, len(tasks))
for _, task := range tasks {
if task.RequestedBy == "autonomous-scanner" {
out = append(out, task)
}
}
return out, nil
}
func stringSetJaccard(a, b []string) float64 {
left := map[string]bool{}
right := map[string]bool{}
for _, value := range a {
value = strings.TrimSpace(strings.ToLower(value))
if value != "" {
left[value] = true
}
}
for _, value := range b {
value = strings.TrimSpace(strings.ToLower(value))
if value != "" {
right[value] = true
}
}
if len(left) == 0 && len(right) == 0 {
return 0
}
intersection := 0
union := map[string]bool{}
for value := range left {
union[value] = true
if right[value] {
intersection++
}
}
for value := range right {
union[value] = true
}
return float64(intersection) / float64(len(union))
}
func researchTermSlice(value string) []string {
terms := researchTerms(value)
out := make([]string, 0, len(terms))
for term := range terms {
out = append(out, term)
}
sort.Strings(out)
return out
}
func autonomousResearchTaskSimilarity(a, b model.ResearchTask) float64 {
if strings.TrimSpace(a.DedupeKey) != "" && a.DedupeKey == b.DedupeKey {
return 1
}
topic := stringSetJaccard(researchTermSlice(a.Topic), researchTermSlice(b.Topic))
seed := stringSetJaccard(a.SeedNodeIDs, b.SeedNodeIDs)
questions := stringSetJaccard(researchTermSlice(strings.Join(a.Questions, " ")), researchTermSlice(strings.Join(b.Questions, " ")))
// Topic identity dominates, but changing orphan-cluster membership must not
// create a fresh queue entry for the same semantic gap.
return clamp01(topic*.62 + seed*.23 + questions*.15)
}
func autonomousResearchSemanticDuplicate(a, b model.ResearchTask) bool {
if strings.TrimSpace(a.DedupeKey) != "" && a.DedupeKey == b.DedupeKey {
return true
}
topic := stringSetJaccard(researchTermSlice(a.Topic), researchTermSlice(b.Topic))
seed := stringSetJaccard(a.SeedNodeIDs, b.SeedNodeIDs)
questions := stringSetJaccard(researchTermSlice(strings.Join(a.Questions, " ")), researchTermSlice(strings.Join(b.Questions, " ")))
if topic >= .72 {
return true
}
if topic >= .42 && seed >= .45 {
return true
}
return topic >= .50 && questions >= .55
}
func (e *Engine) findAutonomousResearchSemanticDuplicate(ctx context.Context, incoming model.ResearchTask) (model.ResearchTask, bool, error) {
active, err := autonomousResearchActiveTasks(ctx, e.Graph)
if err != nil {
return model.ResearchTask{}, false, err
}
bestScore := 0.0
var best model.ResearchTask
for _, task := range active {
if !autonomousResearchSemanticDuplicate(task, incoming) {
continue
}
score := autonomousResearchTaskSimilarity(task, incoming)
if score > bestScore {
bestScore = score
best = task
}
}
return best, best.ID != "", nil
}
// consolidateAutonomousResearchQueue collapses semantically equivalent queued
// work after upgrades/restarts and before new opportunity scans. Running work is
// never cancelled. The highest-priority/oldest queued task becomes the keeper.
func (e *Engine) consolidateAutonomousResearchQueue(ctx context.Context) (int, error) {
tasks, err := e.Graph.ListResearchTasks(ctx, 500, "queued", "deferred")
if err != nil {
return 0, err
}
filtered := make([]model.ResearchTask, 0, len(tasks))
for _, task := range tasks {
if task.RequestedBy == "autonomous-scanner" {
filtered = append(filtered, task)
}
}
sort.SliceStable(filtered, func(i, j int) bool {
if filtered[i].Priority != filtered[j].Priority {
return filtered[i].Priority > filtered[j].Priority
}
return filtered[i].CreatedAt.Before(filtered[j].CreatedAt)
})
merged := 0
cancelled := map[string]bool{}
for i := 0; i < len(filtered); i++ {
keeper := filtered[i]
if cancelled[keeper.ID] {
continue
}
for j := i + 1; j < len(filtered); j++ {
candidate := filtered[j]
if cancelled[candidate.ID] || !autonomousResearchSemanticDuplicate(keeper, candidate) {
continue
}
updated, changed, mergeErr := e.Graph.MergeQueuedResearchTask(ctx, keeper.ID, candidate)
if mergeErr != nil {
return merged, mergeErr
}
if changed {
keeper = updated
}
ok, cancelErr := e.Graph.CancelResearchTask(ctx, candidate.ID)
if cancelErr != nil {
return merged, cancelErr
}
if ok {
cancelled[candidate.ID] = true
merged++
}
}
}
if merged > 0 && e.Broker != nil {
e.Broker.Publish(model.Activity{Type: "autonomous.research.queue.consolidated", Source: "brain", Phase: "autonomous-research-queue", Message: fmt.Sprintf("%d semantisch redundante Rechercheaufgaben wurden in vorhandene Queue-Einträge zusammengeführt", merged), Strength: .52, Metadata: map[string]any{"merged_tasks": merged, "algorithm": "topic-seed-question-jaccard-v1"}})
}
return merged, nil
}
func (e *Engine) autonomousResearchDailyBudget(ctx context.Context, settings RuntimeSettings) (completed, active, remaining int, err error) {
startOfDay := time.Now().UTC().Truncate(24 * time.Hour)
completed, err = e.Graph.CountResearchTasksCompletedSince(ctx, startOfDay)
if err != nil {
return 0, 0, 0, err
}
// The same worker leases API/manual and autonomous-scanner tasks. Reserve
// daily capacity against the whole active queue so a fast opportunity scan
// cannot create a backlog that the worker cannot consume inside the budget.
activeTasks, err := e.Graph.ListResearchTasks(ctx, 500, "queued", "deferred", "reserved", "running")
if err != nil {
return 0, 0, 0, err
}
active = len(activeTasks)
remaining = settings.AutonomousResearchMaxTasksPerDay - completed - active
if remaining < 0 {
remaining = 0
}
return completed, active, remaining, nil
}
func autonomousDedupeKey(topic string, seedIDs []string) string {
ids := append([]string(nil), seedIDs...)
sort.Strings(ids)
@@ -1399,21 +1646,29 @@ func (e *Engine) AutonomousResearchStatus(ctx context.Context) map[string]any {
if err != nil {
counts = map[string]int{}
}
persistentTotals, totalsErr := e.Graph.ResearchTaskPersistentTotals(ctx)
if totalsErr != nil {
persistentTotals = map[string]int{}
}
e.stateMu.RLock()
lastScanDecisions := append([]autonomousOpportunityDecision(nil), e.autonomousLastScanDecisions...)
settings := e.RuntimeSettings()
status := map[string]any{
"enabled": e.RuntimeSettings().AutonomousResearchEnabled,
"idle_only": e.RuntimeSettings().AutonomousResearchIdleOnly,
"running": e.autonomousRunning,
"task_id": e.autonomousTaskID,
"task_topic": e.autonomousTaskTopic,
"last_started": e.autonomousLastStarted,
"last_completed": e.autonomousLastCompleted,
"last_error": e.autonomousLastError,
"completed_total": e.autonomousCompleted,
"failed_total": e.autonomousFailed,
"evidence_total": e.autonomousEvidence,
"articles_total": e.autonomousArticles,
"enabled": settings.AutonomousResearchEnabled,
"idle_only": settings.AutonomousResearchIdleOnly,
"idle_only_effective": settings.AutonomousResearchIdleOnly && !settings.SpeedMode,
"speed_mode": settings.SpeedMode,
"running": e.autonomousRunning,
"task_id": e.autonomousTaskID,
"task_topic": e.autonomousTaskTopic,
"last_started": e.autonomousLastStarted,
"last_completed": e.autonomousLastCompleted,
"last_error": e.autonomousLastError,
"completed_total": persistentTotals["completed"],
"failed_total": persistentTotals["failed"],
"evidence_total": persistentTotals["evidence"],
"articles_total": persistentTotals["articles"],
"process_totals": map[string]any{"completed": e.autonomousCompleted, "failed": e.autonomousFailed, "evidence": e.autonomousEvidence, "articles": e.autonomousArticles},
"last_scan": map[string]any{
"started": e.autonomousLastScanStarted,
"completed": e.autonomousLastScanCompleted,
+77 -99
View File
@@ -75,6 +75,7 @@ type Engine struct {
lastScan time.Time
lastVectorGraph time.Time
lastVectorLayout time.Time
vectorMaintenanceStartedAt time.Time
bootstrapComplete bool
bootstrapCompletedAt time.Time
bootstrapError string
@@ -108,6 +109,7 @@ type Engine struct {
interactiveInflight atomic.Int64
autonomousWake chan struct{}
autonomousScanRequests chan string
sourceInboxWake chan struct{}
autonomousRunning bool
autonomousTaskID string
autonomousTaskTopic string
@@ -312,13 +314,9 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
sharedWork := workqueue.New(cfg.ResearchOllamaMaxInflight, cfg.ResearchOllamaQueueSize)
pool.SetSharedLimiter(sharedWork)
persistence := persist.New(g, b, cfg.PersistInterval)
e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: pool, Persistence: persistence, Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs, FullVerifyInterval: cfg.KnowledgeFullVerifyInterval}, bootstrapReady: make(chan struct{}), enrichRequests: make(chan string, 1), autonomousWake: make(chan struct{}, 1), autonomousScanRequests: make(chan string, 1), runtimePath: filepath.Join(cfg.DataDir, "runtime-settings.json"), researchEvidenceCache: map[string]researchEvidenceRecord{}, sharedWork: sharedWork, researchDedupe: map[string]*researchDedupeEntry{}}
// Slow relaxation is maintenance, not a startup animation. Delaying its first
// cycle prevents frequent process restarts from moving the cloud repeatedly.
if cfg.VectorGraphRelaxLayout && !cfg.VectorGraphLayout {
e.lastVectorLayout = time.Now().UTC()
}
e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: pool, Persistence: persistence, vectorMaintenanceStartedAt: time.Now().UTC(), Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs, FullVerifyInterval: cfg.KnowledgeFullVerifyInterval}, bootstrapReady: make(chan struct{}), enrichRequests: make(chan string, 1), autonomousWake: make(chan struct{}, 1), autonomousScanRequests: make(chan string, 1), sourceInboxWake: make(chan struct{}, 1), runtimePath: filepath.Join(cfg.DataDir, "runtime-settings.json"), researchEvidenceCache: map[string]researchEvidenceRecord{}, sharedWork: sharedWork, researchDedupe: map[string]*researchDedupeEntry{}}
e.loadRuntimeSettings()
e.applyRuntimePerformance(e.RuntimeSettings())
if cfg.SearXNGURL != "" {
e.Research = research.New(cfg.SearXNGURL)
}
@@ -327,7 +325,7 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
e.GLPIKB = ingest.NewGLPIKBSyncer(ingest.GLPIKBConfig{Enabled: true, Path: cfg.GLPIKBPath, Filter: cfg.GLPIKBFilter, Limit: cfg.GLPIKBLimit, SyncInterval: cfg.GLPIKBSyncInterval, Source: cfg.GLPIKBSource, CachePath: filepath.Join(cfg.DataDir, "glpi-kb-cache.json"), ShouldSync: e.LearningEnabled}, client, g, b, persistence)
}
if b != nil {
b.Publish(model.Activity{Type: "system.started", Source: "brain", Phase: "startup", Message: "Neural Brain wurde gestartet; das Analyseprotokoll zeichnet Läufe und Graphänderungen auf", Strength: .3, Metadata: map[string]any{"chat_model": cfg.ChatModel, "embedding_model": cfg.EmbeddingModel, "article_language": cfg.ArticleLanguage, "article_synthesis_model": cfg.ArticleSynthesisModel, "article_review_model": cfg.ArticleReviewModel, "article_review_repair_rounds": cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v3-quality-v2", "article_research_strategy": cfg.ArticleResearchStrategy, "cluster_article_batching": cfg.ClusterArticleBatching, "vector_graph_enabled": cfg.VectorGraphEnabled, "vector_graph_layout": cfg.VectorGraphLayout, "vector_graph_reevaluate_interval": cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_interval": cfg.VectorGraphLayoutRelaxInterval.String(), "article_cpu_quality_enabled": cfg.ArticleCPUQualityEnabled, "article_cpu_quality_agent_offload": cfg.ArticleCPUQualityAgentOffload, "research_ollama_max_inflight": cfg.ResearchOllamaMaxInflight, "research_ollama_queue_size": cfg.ResearchOllamaQueueSize, "graph_version": g.Version()}})
b.Publish(model.Activity{Type: "system.started", Source: "brain", Phase: "startup", Message: "Neural Brain wurde gestartet; das Analyseprotokoll zeichnet Läufe und Graphänderungen auf", Strength: .3, Metadata: map[string]any{"chat_model": cfg.ChatModel, "embedding_model": cfg.EmbeddingModel, "article_language": cfg.ArticleLanguage, "article_synthesis_model": cfg.ArticleSynthesisModel, "article_review_model": cfg.ArticleReviewModel, "article_review_repair_rounds": cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v5-quality-gate-v12", "article_research_strategy": cfg.ArticleResearchStrategy, "cluster_article_batching": cfg.ClusterArticleBatching, "vector_graph_enabled": cfg.VectorGraphEnabled, "vector_graph_layout": cfg.VectorGraphLayout, "vector_graph_layout_mode": e.effectiveVectorLayoutMode(), "vector_graph_reevaluate_interval": cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_effective": cfg.VectorGraphRelaxLayout && !cfg.VectorGraphLayout, "vector_graph_layout_relax_interval": cfg.VectorGraphLayoutRelaxInterval.String(), "article_cpu_quality_enabled": cfg.ArticleCPUQualityEnabled, "article_cpu_quality_agent_offload": cfg.ArticleCPUQualityAgentOffload, "research_ollama_max_inflight": cfg.ResearchOllamaMaxInflight, "research_ollama_queue_size": cfg.ResearchOllamaQueueSize, "speed_mode": e.RuntimeSettings().SpeedMode, "speed_cpu_tasks": e.RuntimeSettings().SpeedCPUWorkers, "speed_gpu_tasks": e.RuntimeSettings().SpeedGPUInflight, "graph_version": g.Version()}})
}
return e
}
@@ -378,6 +376,12 @@ func (e *Engine) Start(ctx context.Context) {
}
}()
go func() {
if e.waitBootstrap(ctx) {
e.vectorMaintenanceLoop(ctx)
}
}()
go e.enrichmentWorker(ctx)
if e.Cfg.AutoEnrich {
go e.enrichmentScheduler(ctx)
@@ -461,6 +465,9 @@ func (e *Engine) waitBootstrap(ctx context.Context) bool {
func (e *Engine) enrichmentScheduler(ctx context.Context) {
firstDelay := 12 * time.Second
if e.SpeedModeEnabled() {
firstDelay = 0
}
e.setNextEnrich(time.Now().Add(firstDelay))
timer := time.NewTimer(firstDelay)
defer timer.Stop()
@@ -488,7 +495,10 @@ func (e *Engine) enrichmentWorker(ctx context.Context) {
if !e.waitBootstrap(ctx) {
return
}
e.runEnrichmentCycle(ctx, trigger)
more := e.runEnrichmentCycle(ctx, trigger)
if more && e.SpeedModeEnabled() && e.ThinkingEnabled() {
e.RequestEnrich("speed-drain")
}
}
}
}
@@ -528,7 +538,7 @@ func (e *Engine) RequestEnrich(trigger string) bool {
}
}
func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) bool {
started := time.Now()
e.stateMu.Lock()
e.enrichRunning = true
@@ -588,13 +598,13 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
if outcome.Rejected {
rejected++
}
if step+1 < e.Cfg.EnrichBatchSize && e.Cfg.EnrichStepDelay > 0 {
if delay := e.effectiveEnrichStepDelay(); step+1 < e.Cfg.EnrichBatchSize && delay > 0 {
select {
case <-ctx.Done():
cycleErr = ctx.Err()
result = "cancelled"
step = e.Cfg.EnrichBatchSize
case <-time.After(e.Cfg.EnrichStepDelay):
case <-time.After(delay):
}
}
}
@@ -624,13 +634,14 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
if cycleErr != nil {
e.Broker.Publish(model.Activity{Type: "think.cycle.failed", Source: "brain", Phase: "autonomous", Message: "AI-THINK-Zyklus wurde mit Fehler beendet", Strength: .45, Metadata: metadata})
slog.Warn("enrichment cycle failed", "trigger", trigger, "error", cycleErr)
return
return false
}
message := fmt.Sprintf("AI-THINK-Zyklus abgeschlossen · %d Relationen · %d Artikel · %d verworfen", relationsCreated, articlesCreated, rejected)
if result == "no_candidate" {
message = "AI-THINK hat im aktuell geprüften Graphbereich keinen Kandidaten oberhalb des Schwellwerts gefunden"
}
e.Broker.Publish(model.Activity{Type: "think.cycle.completed", Source: "brain", Phase: "autonomous", Message: message, Strength: .58, Metadata: metadata})
return result == "completed" && checked > 0
}
func (e *Engine) setNextEnrich(t time.Time) {
@@ -770,11 +781,11 @@ func (e *Engine) Scan(ctx context.Context) error {
// locally on the Brain CPU or be claimed by a compute-capable Source Agent.
// Only 256-dimensional deterministic fallback vectors are excluded because
// mixing them with the configured embedding space would make thresholds lie.
e.stateMu.RLock()
lastVectorGraph := e.lastVectorGraph
e.stateMu.RUnlock()
periodicVectorRefresh := lastVectorGraph.IsZero() || time.Since(lastVectorGraph) >= e.Cfg.VectorGraphReevaluateInterval
vectorLayerNeeded := !scanResult.FastPath || needsEmbeddingWork || !e.Graph.HasEdgesByOrigin(graph.VectorMathOrigin) || periodicVectorRefresh
// Scheduled reevaluation/layout maintenance has its own loop and therefore
// continues even while Learning is paused. Learning scans rebuild the vector
// layer only when their own ingest/embedding work changed the knowledge space
// or when no vector layer exists yet.
vectorLayerNeeded := !scanResult.FastPath || needsEmbeddingWork || !e.Graph.HasEdgesByOrigin(graph.VectorMathOrigin)
vectorReady := e.Graph.CountVectorsByDimension(256) == 0
if e.Cfg.VectorGraphEnabled && vectorReady && vectorLayerNeeded {
startedVectorGraph := time.Now()
@@ -794,8 +805,9 @@ func (e *Engine) Scan(ctx context.Context) error {
"candidate_pairs": stats.CandidatePairs, "exact_comparisons": stats.ExactComparisons,
"orphan_pass_enabled": e.Cfg.VectorGraphOrphanPass, "orphan_focus": stats.OrphanFocus, "orphan_links": stats.OrphanLinks,
"orphan_exact_comparisons": stats.OrphanStats.ExactComparisons, "orphan_candidate_pairs": stats.OrphanStats.CandidatePairs,
"position_updates": stats.PositionUpdates, "layout_enabled": e.Cfg.VectorGraphLayout, "layout_relaxation": e.Cfg.VectorGraphRelaxLayout, "periodic_refresh": periodicVectorRefresh, "reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(),
"position_updates": stats.PositionUpdates, "layout_enabled": e.Cfg.VectorGraphLayout, "layout_relaxation_configured": e.Cfg.VectorGraphRelaxLayout, "layout_mode": execution.LayoutMode, "layout_applied": execution.LayoutDue && stats.PositionUpdates > 0, "periodic_refresh": false, "maintenance_owned": true, "reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(),
"agent_offloaded": execution.Offloaded, "agent_id": execution.AgentID, "agent_compute_ms": execution.ComputeMS, "agent_fallback_reason": execution.FallbackReason,
"speed_mode": e.SpeedModeEnabled(), "cpu_workers": e.vectorPrimaryConfig(execution.LayoutDue).Workers,
"duration_ms": time.Since(startedVectorGraph).Milliseconds(),
}, vectorMutations)})
}
@@ -993,7 +1005,9 @@ func (e *Engine) Query(ctx context.Context, q string) (model.QueryResponse, erro
for i, h := range hits {
nodeIDs = append(nodeIDs, h.NodeID)
e.Broker.Publish(model.Activity{Type: "node.activated", Source: "brain", Phase: "retrieval", Query: q, NodeIDs: []string{h.NodeID}, Message: fmt.Sprintf("Treffer %d · %.0f%% · %s", i+1, h.Score*100, h.Label), Strength: math.Max(.25, h.Score), Metadata: queryMeta(nil)})
time.Sleep(55 * time.Millisecond)
if !e.SpeedModeEnabled() {
time.Sleep(55 * time.Millisecond)
}
}
edgeIDs := e.Graph.ConnectingEdges(nodeIDs)
if len(edgeIDs) > 0 {
@@ -1156,7 +1170,7 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
e.stateMu.Unlock()
e.Broker.Publish(model.Activity{Type: "think.started", Source: "brain", Phase: "association", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Verwandtschaft wird geprüft · %.0f%% semantische Nähe", sim*100), Strength: .88, Metadata: map[string]any{"trigger": trigger, "semantic_similarity": sim, "source_label": a.Label, "target_label": b.Label, "model": e.Cfg.ChatModel, "candidate_comparisons": comparisons, "exact_comparisons": comparisons, "coarse_comparisons": coarseComparisons, "indexed_nodes": indexedNodes, "candidate_pool": candidatePool, "processing_mode": processingMode, "candidate_source": candidateSource}})
system := "Du führst ausschließlich eine Relationserkennung für einen Wissensgraphen durch. Analysiere zwei interne Wissenseinträge, erfinde keine Fakten und entscheide, ob eine belastbare Beziehung besteht. Schreibe keinen Artikel und keine technische Synthese. Wenn externe Fakten zur Relationsentscheidung fehlen, setze needs_research=true. Gib ausschließlich JSON nach Schema zurück."
system := "Du führst ausschließlich eine Relationserkennung für einen Wissensgraphen durch. Analysiere zwei interne Wissenseinträge, erfinde keine Fakten und entscheide, ob eine belastbare Beziehung besteht. Schreibe keinen Artikel und keine technische Synthese. Bei sehr hoher semantischer Nähe und same_topic/related_to ist normalerweise keine Webrecherche nötig. Wenn externe Fakten für eine kausale, abhängige, widersprüchliche oder zeitkritische Relationsentscheidung fehlen, setze needs_research=true. research_query darf ausschließlich fachliche Begriffe und sichtbare Titel enthalten, niemals interne Node-IDs, Hashes oder Datenbankkennungen. Gib ausschließlich JSON nach Schema zurück."
var decision model.RelationDecision
if err := e.Ollama.ChatJSON(ctx, system, relationContext(a, b, sim), relationSchema(), &decision); err != nil {
e.Broker.Publish(model.Activity{Type: "think.failed", Source: "brain", Phase: "inference", NodeIDs: []string{a.ID, b.ID}, Message: "Qwen-Beziehungsanalyse ist fehlgeschlagen; es wurde nichts gespeichert", Strength: .35, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "model": e.Cfg.ChatModel}})
@@ -1172,88 +1186,42 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
decision.ResearchQuery = ""
}
if decision.NeedsResearch && e.ResearchEnabledForRuntime() && strings.TrimSpace(decision.ResearchQuery) != "" {
decision.ResearchQuery = sanitizeSearchQuerySiteFilters(decision.ResearchQuery)
if strings.TrimSpace(decision.ResearchQuery) == "" {
decision.NeedsResearch = false
}
}
if decision.NeedsResearch && e.ResearchEnabledForRuntime() && strings.TrimSpace(decision.ResearchQuery) != "" {
researchID := newResearchRunID("relation-research", decision.ResearchQuery)
researchID := newResearchRunID("relation-research-v10", decision.ResearchQuery)
researchStarted := time.Now()
startMetadata := map[string]any{"trigger": trigger, "research_id": researchID, "research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label, "animation_min_ms": 2000}
e.Broker.Publish(model.Activity{Type: "research.started", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · SearXNG durchsucht externe Quellen", Strength: .9, Metadata: startMetadata})
lease, reused, dedupeErr := e.beginResearchIntent(ctx, "relation", decision.ResearchQuery)
var results []model.ResearchResult
var diagnostic research.Diagnostic
var researchErr error
if dedupeErr != nil {
researchErr = dedupeErr
} else if !lease.owner {
results = cloneResearchResults(reused)
metadata := mergeResearchMetadata(startMetadata, map[string]any{"similarity": lease.similarity, "reused_results": len(results), "dedupe_threshold": e.Cfg.ResearchDedupeThreshold})
for key, value := range researchDedupeLeaseMetadata(lease) {
metadata[key] = value
}
e.Broker.Publish(model.Activity{Type: "research.deduplicated", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Semantisch gleiche Relationsrecherche wurde wiederverwendet · %d Treffer", len(results)), Strength: .76, Metadata: metadata})
} else {
// Relation research intentionally considers more than the old four
// snippets. The full article pipeline remains the final quality gate.
relationResultLimit := 8
if e.Cfg.ArticleResearchResults > relationResultLimit {
relationResultLimit = e.Cfg.ArticleResearchResults
}
if relationResultLimit > 12 {
relationResultLimit = 12
}
researchErr = e.withSharedResearchWork(ctx, "searxng.relation_search", func() error {
var searchErr error
results, diagnostic, searchErr = e.Research.SearchDetailed(ctx, decision.ResearchQuery, relationResultLimit)
return searchErr
})
e.completeResearchIntent(lease, results, researchErr)
}
e.Broker.Publish(model.Activity{Type: "research.started", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · Relationsrecherche prüft externe Volltextbelege einzeln", Strength: .9, Metadata: map[string]any{"trigger": trigger, "research_id": researchID, "research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label, "evidence_gate": "relation-fulltext-v10", "animation_min_ms": 2000}})
acceptedEvidence, researchMetadata, researchErr := e.collectRelationResearchEvidence(ctx, trigger, a, b, decision)
researchMetadata["research_id"] = researchID
researchMetadata["duration_ms"] = time.Since(researchStarted).Milliseconds()
if researchErr != nil {
metadata := mergeResearchMetadata(startMetadata, researchDiagnosticMetadata(diagnostic))
metadata["error"] = researchErr.Error()
metadata["duration_ms"] = time.Since(researchStarted).Milliseconds()
slog.Warn("research failed", "query", decision.ResearchQuery, "base_url", diagnostic.BaseURL, "kind", diagnostic.ErrorKind, "http_status", diagnostic.HTTPStatus, "duration_ms", diagnostic.DurationMS, "error", researchErr)
e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "SearXNG-Recherche ist fehlgeschlagen", Strength: .35, Metadata: metadata})
researchMetadata["error"] = researchErr.Error()
decision.Related = false
decision.Confidence = math.Min(decision.Confidence, math.Max(0, e.Cfg.RelationThreshold-.01))
decision.Explanation = strings.TrimSpace(decision.Explanation + " · erforderliche externe Relationsrecherche fehlgeschlagen")
e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Relationsrecherche ist fehlgeschlagen; die beweisabhängige Relation wird nicht übernommen", Strength: .35, Metadata: researchMetadata})
} else {
allowedResults := e.filterResearchEvidenceForThinking(results, unique(append(append([]string{}, a.Categories...), b.Categories...)))
resultMetadata := mergeResearchMetadata(researchEventMetadata(trigger, researchID, decision.ResearchQuery, allowedResults, time.Since(researchStarted)), researchDiagnosticMetadata(diagnostic))
resultMetadata["unfiltered_result_count"] = len(results)
resultMetadata["source_filter_rejected_count"] = len(results) - len(allowedResults)
resultMetadata["deduplicated"] = !lease.owner
message := fmt.Sprintf("SearXNG hat %d durch den Thinking-Filter erlaubte Webquellen geliefert", len(allowedResults))
if len(allowedResults) == 0 {
message = "SearXNG-Treffer lagen außerhalb des wirksamen Thinking-Quellenfilters"
}
e.Broker.Publish(model.Activity{Type: "research.results", Source: "searxng", Phase: "research-results", NodeIDs: []string{a.ID, b.ID}, Message: message, Strength: .92, Metadata: resultMetadata})
if len(allowedResults) > 0 {
// Keep relation-search hits in memory until the relation itself has
// passed the second model review. Rejected relation research must not
// permanently inflate the graph with unused external nodes.
researchResults = allowedResults
researchResults = acceptedEvidence
e.Broker.Publish(model.Activity{Type: "research.results", Source: "brain", Phase: "research-results", NodeIDs: []string{a.ID, b.ID}, Message: relationResearchResultMessage(researchMetadata), Strength: .82, Metadata: researchMetadata})
if len(acceptedEvidence) == 0 {
decision.Related = false
decision.Confidence = math.Min(decision.Confidence, math.Max(0, e.Cfg.RelationThreshold-.01))
decision.Explanation = strings.TrimSpace(decision.Explanation + " · erforderliche externe Relationsrecherche lieferte keinen belastbaren Volltextbeleg")
e.Broker.Publish(model.Activity{Type: "think.research.insufficient", Source: "brain", Phase: "research-routing", NodeIDs: []string{a.ID, b.ID}, Message: "Die Relation benötigte externe Fakten, aber kein Suchtreffer bestand die Volltext-Evidenzprüfung", Strength: .4, Metadata: researchMetadata})
} else {
var reviewed model.RelationDecision
reviewSystem := "Bewerte die Beziehung erneut anhand der zwei internen Wissenseinträge und der beigefügten Web-Suchergebnisse. Suchtreffer sind Hinweise, keine garantierten Fakten. Erfinde nichts, kennzeichne verbleibende Unsicherheit und gib ausschließlich JSON nach Schema zurück."
if err := e.Ollama.ChatJSON(ctx, reviewSystem, relationContextWithResearch(a, b, sim, allowedResults), relationSchema(), &reviewed); err != nil {
slog.Warn("research review failed; keeping pre-research decision", "error", err)
reviewSystem := "Bewerte die Beziehung erneut anhand der zwei internen Wissenseinträge und ausschließlich der beigefügten, einzeln geprüften Volltextbelege. Die Belege haben bereits Topic-, Quellenqualitäts- und Volltext-Relevanz-Gates bestanden. Erfinde nichts. Interne Node-IDs sind keine Websuchbegriffe. Wenn die Relation weiterhin nicht belastbar ist, setze related=false. Gib ausschließlich JSON nach Schema zurück."
if err := e.Ollama.ChatJSON(ctx, reviewSystem, relationContextWithResearch(a, b, sim, acceptedEvidence), relationSchema(), &reviewed); err != nil {
slog.Warn("relation evidence review failed; rejecting evidence-dependent relation", "error", err)
researchResults = nil
decision.Related = false
decision.Confidence = math.Min(decision.Confidence, math.Max(0, e.Cfg.RelationThreshold-.01))
decision.Explanation = strings.TrimSpace(decision.Explanation + " · zweiter Relationsreview mit geprüfter Evidenz fehlgeschlagen")
} else {
decision = reviewed
}
}
// Always close the relation-research lifecycle explicitly. Older code
// only emitted research.ingested when at least one source passed the
// Thinking filter, leaving zero-result and filtered-out searches stuck
// as "running" forever in the analysis history.
completedMetadata := mergeResearchMetadata(resultMetadata, map[string]any{
"duration_ms": time.Since(researchStarted).Milliseconds(),
"accepted_count": len(allowedResults),
"result": "completed",
})
e.Broker.Publish(model.Activity{Type: "research.completed", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Relationsrecherche beendet · %d verwendbare Treffer", len(allowedResults)), Strength: .72, Metadata: completedMetadata})
researchMetadata["result"] = "completed"
e.Broker.Publish(model.Activity{Type: "research.completed", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: relationResearchResultMessage(researchMetadata), Strength: .72, Metadata: researchMetadata})
}
}
@@ -1304,11 +1272,18 @@ func (e *Engine) addResearch(ctx context.Context, a, b model.Node, results []mod
refs := researchGraphRefs{}
for _, r := range results {
id := graph.ID("external", r.URL)
n := model.Node{ID: id, Kind: "external", Label: r.Title, Summary: clamp(r.Content, 700), Status: "research", Origin: "research", ExternalID: r.URL, URI: r.URL, Categories: unique(append(append([]string{}, a.Categories...), b.Categories...)), Weight: .8, Metadata: map[string]any{"source": graph.SourceFromURL(r.URL), "query_pair": []string{a.ID, b.ID}}, UpdatedAt: time.Now().UTC()}
metadata := map[string]any{"source": graph.SourceFromURL(r.URL), "query_pair": []string{a.ID, b.ID}, "validation_state": "relation_fulltext_gate_v10", "relevance": r.Relevance, "source_quality": r.SourceQuality, "source_quality_score": r.SourceQualityScore, "assessment_reason": r.AssessmentReason}
if r.Fetched && r.Relevant {
if relPath, contentHash, err := e.queueResearchEvidence(r); err == nil {
metadata["evidence_path"] = relPath
metadata["evidence_sha256"] = contentHash
}
}
n := model.Node{ID: id, Kind: "external", Label: r.Title, Summary: clamp(r.Content, 700), Status: "research", Origin: "research", ExternalID: r.URL, URI: r.URL, Categories: unique(append(append([]string{}, a.Categories...), b.Categories...)), Weight: math.Max(.8, r.SourceQualityScore), Metadata: metadata, UpdatedAt: time.Now().UTC()}
refs.Mutations.Add(e.Graph.UpsertNodeWithStats(n))
refs.NodeIDs = append(refs.NodeIDs, id)
for _, targetID := range []string{a.ID, b.ID} {
edge := model.Edge{Source: id, Target: targetID, Type: "research_evidence", Origin: "research", Status: "staging", Confidence: .55, Weight: .4}
edge := model.Edge{Source: id, Target: targetID, Type: "research_evidence", Origin: "research", Status: "staging", Confidence: math.Max(.65, r.Relevance), Weight: math.Max(.5, r.SourceQualityScore*.7), Explanation: r.AssessmentReason, Metadata: map[string]any{"validation_state": "relation_fulltext_gate_v10", "source_quality": r.SourceQuality, "relevance": r.Relevance}}
refs.Mutations.Add(e.Graph.UpsertEdgeWithStats(edge))
refs.EdgeIDs = append(refs.EdgeIDs, graph.EdgeID(edge.Source, edge.Target, edge.Type, edge.Origin))
}
@@ -1381,14 +1356,14 @@ func (e *Engine) Status() map[string]any {
"article_research_min_relevance": e.Cfg.ArticleResearchMinRelevance, "article_research_min_quality": e.Cfg.ArticleResearchMinQuality,
"article_research_page_max_bytes": e.Cfg.ArticleResearchPageMaxBytes, "article_research_page_max_chars": e.Cfg.ArticleResearchPageMaxChars,
"article_research_fetch_timeout": e.Cfg.ArticleResearchFetchTimeout.String(), "article_research_allow_private": e.Cfg.ArticleResearchAllowPrivate,
"article_language": e.Cfg.ArticleLanguage, "article_synthesis_model": e.Cfg.ArticleSynthesisModel, "article_review_model": e.Cfg.ArticleReviewModel, "article_review_repair_rounds": e.Cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v3-quality-v2", "article_research_strategy": e.Cfg.ArticleResearchStrategy, "article_effective_research_strategy": e.effectiveArticleResearchStrategy(), "article_adaptive_initial_queries": e.Cfg.ArticleAdaptiveInitialQueries, "article_adaptive_initial_fetch": e.Cfg.ArticleAdaptiveInitialFetch, "research_dedupe": e.researchDedupeStatus(),
"article_language": e.Cfg.ArticleLanguage, "article_synthesis_model": e.Cfg.ArticleSynthesisModel, "article_review_model": e.Cfg.ArticleReviewModel, "article_review_repair_rounds": e.Cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v5-quality-gate-v12", "article_research_strategy": e.Cfg.ArticleResearchStrategy, "article_effective_research_strategy": e.effectiveArticleResearchStrategy(), "article_adaptive_initial_queries": e.Cfg.ArticleAdaptiveInitialQueries, "article_adaptive_initial_fetch": e.Cfg.ArticleAdaptiveInitialFetch, "research_dedupe": e.researchDedupeStatus(),
"scan_interval": e.Cfg.ScanInterval.String(), "knowledge_full_verify_interval": e.Cfg.KnowledgeFullVerifyInterval.String(),
"enrich_interval": e.Cfg.EnrichInterval.String(),
"enrich_batch_size": e.Cfg.EnrichBatchSize, "enrich_anchors": e.Cfg.EnrichAnchors,
"processing_mode": e.RuntimeSettings().ProcessingMode, "cluster_hash_bits": e.Cfg.ClusterHashBits, "cluster_hash_tables": e.Cfg.ClusterHashTables,
"processing_mode": e.RuntimeSettings().ProcessingMode, "speed_mode": e.RuntimeSettings().SpeedMode, "speed_cpu_tasks": e.RuntimeSettings().SpeedCPUWorkers, "speed_gpu_tasks": e.RuntimeSettings().SpeedGPUInflight, "speed_algorithm": speedModeVersion, "cluster_hash_bits": e.Cfg.ClusterHashBits, "cluster_hash_tables": e.Cfg.ClusterHashTables,
"cluster_candidates_per_anchor": e.Cfg.ClusterCandidatesPerAnchor, "cluster_article_candidates": e.Cfg.ClusterArticleCandidates,
"cluster_review_evidence": e.Cfg.ClusterReviewEvidence, "cluster_review_context_chars": e.Cfg.ClusterReviewContextChars, "cluster_article_batching": e.Cfg.ClusterArticleBatching,
"vector_graph_enabled": e.Cfg.VectorGraphEnabled, "vector_graph_neighbors": e.Cfg.VectorGraphNeighbors, "vector_graph_candidates": e.Cfg.VectorGraphCandidates, "vector_graph_min_similarity": e.Cfg.VectorGraphMinSimilarity, "vector_graph_min_affinity": e.Cfg.VectorGraphMinAffinity, "vector_graph_layout": e.Cfg.VectorGraphLayout, "vector_graph_reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": e.Cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_interval": e.Cfg.VectorGraphLayoutRelaxInterval.String(), "vector_graph_layout_blend": e.Cfg.VectorGraphLayoutBlend, "vector_graph_layout_max_shift": e.Cfg.VectorGraphLayoutMaxShift, "last_vector_graph": e.lastVectorGraph, "last_vector_layout": e.lastVectorLayout,
"vector_graph_enabled": e.Cfg.VectorGraphEnabled, "vector_graph_neighbors": e.Cfg.VectorGraphNeighbors, "vector_graph_candidates": e.Cfg.VectorGraphCandidates, "vector_graph_min_similarity": e.Cfg.VectorGraphMinSimilarity, "vector_graph_min_affinity": e.Cfg.VectorGraphMinAffinity, "vector_graph_layout": e.Cfg.VectorGraphLayout, "vector_graph_layout_mode": e.effectiveVectorLayoutMode(), "vector_graph_reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": e.Cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_effective": e.Cfg.VectorGraphRelaxLayout && !e.Cfg.VectorGraphLayout, "vector_graph_layout_relax_interval": e.Cfg.VectorGraphLayoutRelaxInterval.String(), "vector_graph_layout_blend": e.Cfg.VectorGraphLayoutBlend, "vector_graph_layout_max_shift": e.Cfg.VectorGraphLayoutMaxShift, "last_vector_graph": e.lastVectorGraph, "last_vector_layout": e.lastVectorLayout,
"vector_graph_orphan_pass": e.Cfg.VectorGraphOrphanPass, "vector_graph_orphan_neighbors": e.Cfg.VectorGraphOrphanNeighbors, "vector_graph_orphan_candidates": e.Cfg.VectorGraphOrphanCandidates, "vector_graph_orphan_min_similarity": e.Cfg.VectorGraphOrphanMinSimilarity, "vector_graph_orphan_min_affinity": e.Cfg.VectorGraphOrphanMinAffinity,
"vector_graph_agent_offload": e.Cfg.VectorGraphAgentOffload, "vector_graph_agent_required": e.Cfg.VectorGraphAgentRequired, "vector_graph_agent_wait": e.Cfg.VectorGraphAgentWait.String(), "thinking_vector_guided": e.Cfg.ThinkingVectorGuided,
"research_enabled": e.ResearchEnabledForRuntime(), "chat_model": e.Cfg.ChatModel, "embedding_model": e.Cfg.EmbeddingModel,
@@ -1475,7 +1450,10 @@ func relationContextWithResearch(a, b model.Node, sim float64, results []model.R
}
func relationContext(a, b model.Node, sim float64) string {
return fmt.Sprintf("SEMANTISCHE_NÄHE: %.4f\n\nA\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s\n\nB\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s", sim, a.ID, a.Label, strings.Join(a.Categories, ", "), a.Summary, b.ID, b.Label, strings.Join(b.Categories, ", "), b.Summary)
// Internal IDs are intentionally omitted from the model-visible relation
// context. They are routing metadata, not semantic search terms, and older
// prompts occasionally copied them into public SearXNG queries.
return fmt.Sprintf("SEMANTISCHE_NÄHE: %.4f\n\nA\nTitel: %s\nKategorien: %s\nInhalt: %s\n\nB\nTitel: %s\nKategorien: %s\nInhalt: %s", sim, a.Label, strings.Join(a.Categories, ", "), a.Summary, b.Label, strings.Join(b.Categories, ", "), b.Summary)
}
func relationSchema() map[string]any {
return map[string]any{"type": "object", "properties": map[string]any{"related": map[string]any{"type": "boolean"}, "relation_type": map[string]any{"type": "string", "enum": []string{"related_to", "depends_on", "supports", "contradicts", "extends", "same_topic", "caused_by"}}, "confidence": map[string]any{"type": "number", "minimum": 0, "maximum": 1}, "explanation": map[string]any{"type": "string"}, "needs_research": map[string]any{"type": "boolean"}, "research_query": map[string]any{"type": "string"}, "topic_label": map[string]any{"type": "string"}, "keywords": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}}, "required": []string{"related", "relation_type", "confidence", "explanation", "needs_research", "research_query", "topic_label", "keywords"}}
+257
View File
@@ -0,0 +1,257 @@
package engine
import (
"context"
"fmt"
"math"
"regexp"
"sort"
"strings"
"time"
"github.com/local/glpi-neural-brain/internal/model"
"github.com/local/glpi-neural-brain/internal/research"
)
var internalNodeIDPattern = regexp.MustCompile(`(?i)\b[0-9a-f]{20,64}\b`)
// normalizeRelationResearchQuery prevents internal graph identifiers and generic
// bookkeeping language from leaking into public Web searches. If the planner's
// query does not contain a meaningful term from either source label/topic, the
// Brain deterministically rebuilds it from human-readable source labels.
func normalizeRelationResearchQuery(a, b model.Node, decision model.RelationDecision) (string, bool, string) {
original := strings.TrimSpace(sanitizeSearchQuerySiteFilters(decision.ResearchQuery))
if original == "" {
return "", false, "empty"
}
labels := strings.TrimSpace(strings.Join([]string{a.Label, b.Label, decision.TopicLabel}, " "))
anchorTerms := researchTerms(labels)
queryTerms := researchTerms(original)
matched := 0
for term := range anchorTerms {
if queryTerms[term] {
matched++
}
}
containsInternalID := (strings.TrimSpace(a.ID) != "" && strings.Contains(original, a.ID)) || (strings.TrimSpace(b.ID) != "" && strings.Contains(original, b.ID)) || internalNodeIDPattern.MatchString(original)
genericOnly := len(anchorTerms) > 0 && matched == 0
if !containsInternalID && !genericOnly {
return original, false, ""
}
relationIntent := map[string]string{
"same_topic": "Gemeinsamkeiten Unterschiede fachliche Einordnung",
"related_to": "fachlicher Zusammenhang Abgrenzung",
"depends_on": "Abhängigkeit Voraussetzung Zusammenhang",
"supports": "Unterstützung Zusammenhang Nachweis",
"contradicts": "Widerspruch Unterschiede Nachweis",
"extends": "Erweiterung Zusammenhang Abgrenzung",
"caused_by": "Ursache Zusammenhang Nachweis",
}[safeRelation(decision.RelationType)]
if relationIntent == "" {
relationIntent = "fachlicher Zusammenhang Nachweis"
}
parts := []string{strings.TrimSpace(a.Label), strings.TrimSpace(b.Label), strings.TrimSpace(decision.TopicLabel), relationIntent, "offizielle Dokumentation"}
rebuilt := strings.TrimSpace(strings.Join(unique(parts), " "))
reason := "missing_topic_anchor"
if containsInternalID {
reason = "internal_node_id_removed"
}
return rebuilt, true, reason
}
func relationResearchQuestion(a, b model.Node, decision model.RelationDecision, query string) model.ResearchQuestion {
question := strings.TrimSpace(strings.Join([]string{
"Prüfe den fachlichen Zusammenhang zwischen",
a.Label,
"und",
b.Label + ".",
"Zu bewertende Relation:",
safeRelation(decision.RelationType) + ".",
strings.TrimSpace(decision.Explanation),
}, " "))
return model.ResearchQuestion{
GapID: "relation-evidence",
Question: question,
Critical: true,
ExpectActionable: false,
QueriesDE: []string{query},
}
}
// collectRelationResearchEvidence performs a strict, per-source evidence path.
// Search snippets remain ephemeral. Only fetched full text that passes the
// deterministic topic guard, source filter and full-text relevance/quality gate
// is returned to the relation reviewer. Nothing is written to the graph here.
func (e *Engine) collectRelationResearchEvidence(ctx context.Context, trigger string, a, b model.Node, decision model.RelationDecision) ([]model.ResearchResult, map[string]any, error) {
query, rebuilt, rebuildReason := normalizeRelationResearchQuery(a, b, decision)
metadata := map[string]any{
"trigger": trigger,
"research_query": query,
"source_label": a.Label,
"target_label": b.Label,
"query_rebuilt": rebuilt,
"query_rebuild_reason": rebuildReason,
}
if query == "" {
return nil, metadata, nil
}
if rebuilt && e.Broker != nil {
e.Broker.Publish(model.Activity{Type: "think.research.query.rebuilt", Source: "brain", Phase: "research-routing", NodeIDs: []string{a.ID, b.ID}, Message: "Relationsrecherche wurde aus den sichtbaren Themenbegriffen neu aufgebaut; interne Node-IDs werden nicht ins Web gesendet", Strength: .56, Metadata: map[string]any{
"trigger": trigger, "original_query": decision.ResearchQuery, "research_query": query, "reason": rebuildReason, "source_label": a.Label, "target_label": b.Label,
}})
}
question := relationResearchQuestion(a, b, decision, query)
lease, reused, err := e.beginResearchIntent(ctx, "relation-v10", query)
if err != nil {
return nil, metadata, err
}
if !lease.owner {
validated, rejected := e.revalidateReusableResearchEvidence(ctx, question, reused)
metadata["deduplicated"] = true
metadata["reused_results"] = len(reused)
metadata["reused_rejected"] = rejected
metadata["accepted_count"] = len(validated)
return validated, metadata, nil
}
resultLimit := e.Cfg.ArticleResearchResults
if resultLimit < 8 {
resultLimit = 8
}
if resultLimit > 12 {
resultLimit = 12
}
var results []model.ResearchResult
var diagnostic research.Diagnostic
searchStarted := time.Now()
searchErr := e.withSharedResearchWork(ctx, "searxng.relation_search_v10", func() error {
var inner error
results, diagnostic, inner = e.Research.SearchDetailed(ctx, query, resultLimit)
return inner
})
if searchErr != nil {
e.completeResearchIntent(lease, nil, searchErr)
metadata = mergeResearchMetadata(metadata, researchDiagnosticMetadata(diagnostic))
metadata["duration_ms"] = time.Since(searchStarted).Milliseconds()
return nil, metadata, searchErr
}
for i := range results {
results[i].Query = query
results[i].Round = 1
}
metadata["unfiltered_result_count"] = len(results)
categories := unique(append(append([]string{}, a.Categories...), b.Categories...))
allowed := e.filterResearchEvidenceForThinking(results, categories)
metadata["source_filter_rejected_count"] = len(results) - len(allowed)
metadata["source_filter_allowed_count"] = len(allowed)
if len(allowed) == 0 {
e.completeResearchIntent(lease, nil, nil)
metadata["accepted_count"] = 0
return nil, metadata, nil
}
// Snippet selection is deterministic/heuristic; semantic model budget is
// spent only after full text has been fetched.
ranked := rankResearchCandidatesHeuristic(question, allowed, false)
fetchLimit := e.Cfg.ArticleResearchFetchResults
if fetchLimit < 1 {
fetchLimit = 4
}
if fetchLimit > 4 {
fetchLimit = 4
}
prefetchMin := math.Max(.35, e.Cfg.ArticleResearchPrefetchMinRelevance)
finalMin := math.Max(.60, e.Cfg.ArticleResearchMinRelevance)
minQuality := math.Max(.55, e.Cfg.ArticleResearchMinQuality)
selection := selectResearchCandidates(question, ranked, map[string]bool{}, fetchLimit, 1, prefetchMin, finalMin, minQuality)
metadata["snippet_gate_selected_count"] = len(selection.Selected)
metadata["snippet_gate_rejected_count"] = selection.GateRejected
metadata["authoritative_exploration_selected_count"] = countSelectedMode(selection.Decisions, "authoritative_exploration")
if len(selection.Selected) == 0 {
e.completeResearchIntent(lease, nil, nil)
metadata["accepted_count"] = 0
return nil, metadata, nil
}
fetched := make([]model.ResearchResult, 0, len(selection.Selected))
fetchFailures := 0
for _, candidate := range selection.Selected {
var page research.FetchedPage
fetchErr := e.withSharedResearchWork(ctx, "web.relation_fetch_v10", func() error {
var inner error
page, _, inner = e.Research.FetchPage(ctx, candidate.Result.URL, research.FetchOptions{
MaxBytes: e.Cfg.ArticleResearchPageMaxBytes, MaxChars: e.Cfg.ArticleResearchPageMaxChars,
Timeout: e.Cfg.ArticleResearchFetchTimeout, AllowPrivate: e.Cfg.ArticleResearchAllowPrivate,
})
return inner
})
if fetchErr != nil {
fetchFailures++
continue
}
item := candidate.Result
item.URL = nonempty(canonicalResearchURL(page.URL), page.URL)
if strings.TrimSpace(page.Title) != "" {
item.Title = page.Title
}
item.Content = page.Content
item.ContentType = page.ContentType
item.Fetched = true
item.Query = query
item.Round = 1
fetched = append(fetched, item)
}
metadata["fetched_count"] = len(fetched)
metadata["fetch_failed_count"] = fetchFailures
if len(fetched) == 0 {
e.completeResearchIntent(lease, nil, nil)
metadata["accepted_count"] = 0
return nil, metadata, nil
}
assessed := e.rankResearchCandidates(ctx, question, fetched, true)
accepted := make([]model.ResearchResult, 0, len(assessed))
fulltextRejected := 0
for _, candidate := range assessed {
assessment := candidate.Assessment
strict := candidate.TopicGuardPassed && assessment.Relevant && assessment.Relevance >= finalMin && assessment.SourceQualityScore >= minQuality
if !strict {
fulltextRejected++
continue
}
item := candidate.Result
item.Relevant = true
item.Relevance = assessment.Relevance
item.SourceQuality = assessment.SourceQuality
item.SourceQualityScore = assessment.SourceQualityScore
item.Actionable = assessment.Actionable
item.CoveredGapIDs = unique(append(assessment.CoveredGapIDs, question.GapID))
item.AssessmentReason = assessment.Reason
accepted = append(accepted, item)
}
// Stable ordering makes relation-review prompts and dedupe cache deterministic.
sort.SliceStable(accepted, func(i, j int) bool {
if accepted[i].Relevance != accepted[j].Relevance {
return accepted[i].Relevance > accepted[j].Relevance
}
return accepted[i].URL < accepted[j].URL
})
metadata["fulltext_rejected_count"] = fulltextRejected
metadata["accepted_count"] = len(accepted)
metadata["accepted_titles"] = researchTitles(accepted)
metadata["minimum_relevance"] = finalMin
metadata["minimum_quality"] = minQuality
metadata["duration_ms"] = time.Since(searchStarted).Milliseconds()
e.completeResearchIntent(lease, accepted, nil)
return accepted, metadata, nil
}
func relationResearchResultMessage(metadata map[string]any) string {
accepted := 0
if value, ok := metadata["accepted_count"].(int); ok {
accepted = value
}
return fmt.Sprintf("Relationsrecherche beendet · %d einzeln geprüfte Volltextbelege", accepted)
}
@@ -0,0 +1,90 @@
package engine
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/local/glpi-neural-brain/internal/model"
"github.com/local/glpi-neural-brain/internal/ollama"
)
func TestNormalizeRelationResearchQueryRemovesInternalNodeIDs(t *testing.T) {
a := model.Node{ID: "8c68b454ce45705357ae1831", Label: "AI Safety Guardrails – sicher gestalten und härten"}
b := model.Node{ID: "a0808bae9ec114a144632262", Label: "AI Security Guardrails – sicher gestalten und härten"}
decision := model.RelationDecision{RelationType: "same_topic", TopicLabel: "AI Guardrails", ResearchQuery: "Überprüfung der Einträge 8c68b454ce45705357ae1831 und a0808bae9ec114a144632262 auf Duplikat oder unterschiedliche Quellen"}
query, rebuilt, reason := normalizeRelationResearchQuery(a, b, decision)
if !rebuilt || reason != "internal_node_id_removed" {
t.Fatalf("expected deterministic rebuild, rebuilt=%v reason=%q query=%q", rebuilt, reason, query)
}
if strings.Contains(query, a.ID) || strings.Contains(query, b.ID) || internalNodeIDPattern.MatchString(query) {
t.Fatalf("internal IDs leaked into rebuilt query: %q", query)
}
for _, want := range []string{"AI Safety Guardrails", "AI Security Guardrails"} {
if !strings.Contains(query, want) {
t.Fatalf("rebuilt query lost visible topic %q: %q", want, query)
}
}
}
func TestNormalizeRelationResearchQueryRebuildsGenericQueryWithoutTopicAnchor(t *testing.T) {
a := model.Node{Label: "Kubernetes Restore Testing"}
b := model.Node{Label: "Kubernetes Backup Validation"}
decision := model.RelationDecision{RelationType: "supports", ResearchQuery: "Einträge prüfen und Unterschiede validieren"}
query, rebuilt, reason := normalizeRelationResearchQuery(a, b, decision)
if !rebuilt || reason != "missing_topic_anchor" || !strings.Contains(strings.ToLower(query), "kubernetes") {
t.Fatalf("expected topic-anchored rebuild, got rebuilt=%v reason=%q query=%q", rebuilt, reason, query)
}
}
func TestRelationSnippetGateRejectsUnrelatedDuplicateTools(t *testing.T) {
question := model.ResearchQuestion{GapID: "relation-evidence", Question: "AI Safety Guardrails und AI Security Guardrails fachlich vergleichen"}
results := []model.ResearchResult{
{Title: "IBAN auf Fehler prüfen und Bankverbindung identifizieren", URL: "https://example.org/iban", Snippet: "IBAN prüfen und Duplikate finden"},
{Title: "Duplikate in Excel finden", URL: "https://example.org/excel", Snippet: "Doppelte Einträge über mehrere Spalten"},
{Title: "AI Security Guardrails technical guidance", URL: "https://docs.example.org/ai-security/guardrails", Snippet: "AI safety and security guardrails, controls and validation"},
}
ranked := rankResearchCandidatesHeuristic(question, results, false)
selection := selectResearchCandidates(question, ranked, map[string]bool{}, 3, 1, .35, .60, .55)
for _, candidate := range selection.Selected {
if strings.Contains(strings.ToLower(candidate.Result.Title), "iban") || strings.Contains(strings.ToLower(candidate.Result.Title), "excel") {
t.Fatalf("unrelated duplicate-tool result passed relation topic gate: %+v", candidate.Result)
}
}
}
func TestReconsiderOperationalArticleTypeCanChooseReference(t *testing.T) {
mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/tags":
_ = json.NewEncoder(w).Encode(map[string]any{"models": []map[string]any{{"name": "qwen3:8b", "digest": "chat"}, {"name": "embeddinggemma", "digest": "embed"}}})
case "/api/chat":
_ = json.NewEncoder(w).Encode(map[string]any{"message": map[string]any{"content": `{"action":"reclassify","article_type":"reference","reason":"Die Evidenz trägt technische Zuordnungen, aber keinen belastbaren Lösungsablauf."}`}})
default:
http.NotFound(w, r)
}
}))
defer mock.Close()
client := ollama.New(mock.URL, "qwen3:8b", "embeddinggemma")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client.Start(ctx)
deadline := time.Now().Add(time.Second)
for client.PoolStatus()["healthy_nodes"].(int) < 1 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
e := &Engine{Ollama: client}
plan := model.ArticlePlanDecision{ArticleType: "troubleshooting", ExpectedValue: "ATT&CK-Techniken einordnen"}
content := model.KnowledgeArticleContent{Title: "ATT&CK-Techniken", TechnicalDetails: []string{"T1018"}, Mappings: []string{"Profil → T1018"}}
result, err := e.reconsiderOperationalArticleType(context.Background(), plan, content, nil, nil)
if err != nil {
t.Fatal(err)
}
if result.Action != "reclassify" || result.ArticleType != "reference" {
t.Fatalf("unexpected reconsideration: %+v", result)
}
}
+1 -2
View File
@@ -8,7 +8,6 @@ import (
"time"
"github.com/local/glpi-neural-brain/internal/model"
"github.com/local/glpi-neural-brain/internal/ollama"
)
type researchIntentProfile struct {
@@ -83,7 +82,7 @@ func (e *Engine) beginResearchIntentMode(ctx context.Context, kind, rawIntent st
// Embed the natural-language intent rather than the sorted normalization.
// Word order and phrases such as "volatile evidence before reboot" carry
// information that is lost by a bag-of-terms representation.
vectors, err := e.Ollama.Embed(ollama.WithLowPriority(ctx), []string{rawIntent})
vectors, err := e.Ollama.Embed(e.backgroundOllamaContext(ctx), []string{rawIntent})
if err == nil && len(vectors) == 1 {
vector = vectors[0]
}
+54
View File
@@ -21,6 +21,9 @@ type RuntimeSettings struct {
ViewMode string `json:"view_mode"`
MaxDisplayNodes int `json:"max_display_nodes"`
LowPowerMode bool `json:"low_power_mode"`
SpeedMode bool `json:"speed_mode"`
SpeedCPUWorkers int `json:"speed_cpu_tasks"`
SpeedGPUInflight int `json:"speed_gpu_tasks"`
ProcessingMode string `json:"processing_mode"`
AutonomousResearchEnabled bool `json:"autonomous_research_enabled"`
AutonomousResearchIdleOnly bool `json:"autonomous_research_idle_only"`
@@ -53,6 +56,9 @@ func (e *Engine) defaultRuntimeSettings() RuntimeSettings {
ViewMode: e.Cfg.DefaultView,
MaxDisplayNodes: e.Cfg.MaxDisplayNodes,
LowPowerMode: e.Cfg.LowPowerMode,
SpeedMode: e.Cfg.SpeedMode,
SpeedCPUWorkers: e.Cfg.SpeedCPUWorkers,
SpeedGPUInflight: e.Cfg.SpeedGPUInflight,
ProcessingMode: e.Cfg.ProcessingMode,
AutonomousResearchEnabled: e.Cfg.AutonomousResearchEnabled,
AutonomousResearchIdleOnly: e.Cfg.AutonomousResearchIdleOnly,
@@ -86,6 +92,23 @@ func normalizeRuntimeSettings(in RuntimeSettings) RuntimeSettings {
if in.MaxDisplayNodes > 500000 {
in.MaxDisplayNodes = 500000
}
if in.SpeedCPUWorkers < 1 {
in.SpeedCPUWorkers = 1
}
if in.SpeedCPUWorkers > 256 {
in.SpeedCPUWorkers = 256
}
if in.SpeedGPUInflight < 1 {
in.SpeedGPUInflight = 1
}
if in.SpeedGPUInflight > 64 {
in.SpeedGPUInflight = 64
}
if in.SpeedMode {
// Eco and Speed have opposite scheduling/rendering goals. Speed wins when
// an older client accidentally submits both flags.
in.LowPowerMode = false
}
if in.AutonomousResearchMinPriority < 0 {
in.AutonomousResearchMinPriority = 0
}
@@ -172,6 +195,9 @@ func mergeRuntimeSettingsJSON(settings *RuntimeSettings, data []byte) {
decode("view_mode", &settings.ViewMode)
decode("max_display_nodes", &settings.MaxDisplayNodes)
decode("low_power_mode", &settings.LowPowerMode)
decode("speed_mode", &settings.SpeedMode)
decode("speed_cpu_tasks", &settings.SpeedCPUWorkers)
decode("speed_gpu_tasks", &settings.SpeedGPUInflight)
decode("processing_mode", &settings.ProcessingMode)
decode("autonomous_research_enabled", &settings.AutonomousResearchEnabled)
decode("autonomous_research_idle_only", &settings.AutonomousResearchIdleOnly)
@@ -217,6 +243,12 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
if settings.AutonomousResearchTasksPerCycle == 0 {
settings.AutonomousResearchTasksPerCycle = previous.AutonomousResearchTasksPerCycle
}
if settings.SpeedCPUWorkers == 0 {
settings.SpeedCPUWorkers = previous.SpeedCPUWorkers
}
if settings.SpeedGPUInflight == 0 {
settings.SpeedGPUInflight = previous.SpeedGPUInflight
}
if settings.ProcessingMode == "" {
settings.ProcessingMode = previous.ProcessingMode
}
@@ -226,6 +258,12 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
if settings.MaxDisplayNodes < 0 || settings.MaxDisplayNodes > 500000 {
return e.RuntimeSettings(), fmt.Errorf("max_display_nodes must be between 0 and 500000")
}
if settings.SpeedCPUWorkers < 1 || settings.SpeedCPUWorkers > 256 {
return e.RuntimeSettings(), fmt.Errorf("speed_cpu_tasks must be between 1 and 256")
}
if settings.SpeedGPUInflight < 1 || settings.SpeedGPUInflight > 64 {
return e.RuntimeSettings(), fmt.Errorf("speed_gpu_tasks must be between 1 and 64")
}
if settings.AutonomousResearchMinPriority < 0 || settings.AutonomousResearchMinPriority > 1 {
return e.RuntimeSettings(), fmt.Errorf("autonomous_research_min_priority must be between 0 and 1")
}
@@ -243,6 +281,11 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
previous = e.runtime
e.runtime = settings
e.runtimeMu.Unlock()
e.applyRuntimePerformance(settings)
if settings.SpeedMode && !previous.SpeedMode {
e.signalSpeedWork()
}
if settings.AutonomousResearchEnabled && !previous.AutonomousResearchEnabled {
e.signalAutonomousResearch()
@@ -265,6 +308,7 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
e.runtimeMu.Lock()
e.runtime = previous
e.runtimeMu.Unlock()
e.applyRuntimePerformance(previous)
return previous, err
}
}
@@ -285,6 +329,9 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
"view_mode": settings.ViewMode,
"max_display_nodes": settings.MaxDisplayNodes,
"low_power_mode": settings.LowPowerMode,
"speed_mode": settings.SpeedMode,
"speed_cpu_tasks": settings.SpeedCPUWorkers,
"speed_gpu_tasks": settings.SpeedGPUInflight,
"processing_mode": settings.ProcessingMode,
"autonomous_research_enabled": settings.AutonomousResearchEnabled,
"autonomous_research_idle_only": settings.AutonomousResearchIdleOnly,
@@ -296,6 +343,13 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
return settings, nil
}
func (e *Engine) SpeedModeEnabled() bool {
e.runtimeMu.RLock()
enabled := e.runtime.SpeedMode
e.runtimeMu.RUnlock()
return enabled
}
func (e *Engine) LearningEnabled() bool {
e.runtimeMu.RLock()
enabled := e.runtime.LearningEnabled
+17
View File
@@ -100,3 +100,20 @@ func TestNormalizeRuntimeSettingsProcessingMode(t *testing.T) {
t.Fatalf("invalid mode should fall back to precise: %+v", settings)
}
}
func TestNormalizeRuntimeSettingsSpeedWinsOverEco(t *testing.T) {
settings := normalizeRuntimeSettings(RuntimeSettings{
LowPowerMode: true,
SpeedMode: true,
SpeedCPUWorkers: 999,
SpeedGPUInflight: 999,
AutonomousResearchMaxTasksPerDay: 1,
AutonomousResearchTasksPerCycle: 1,
})
if settings.LowPowerMode {
t.Fatal("speed mode must disable eco mode")
}
if settings.SpeedCPUWorkers != 256 || settings.SpeedGPUInflight != 64 {
t.Fatalf("speed limits were not clamped: %+v", settings)
}
}
+44 -21
View File
@@ -11,7 +11,6 @@ import (
"github.com/local/glpi-neural-brain/internal/graph"
"github.com/local/glpi-neural-brain/internal/model"
"github.com/local/glpi-neural-brain/internal/ollama"
"github.com/local/glpi-neural-brain/internal/research"
"github.com/local/glpi-neural-brain/internal/sourceagent"
)
@@ -58,11 +57,23 @@ func (e *Engine) evidenceAcquisitionEnabled() bool {
func (e *Engine) sourceInboxLoop(ctx context.Context) {
e.reconcileProactiveSecurityMaterialization(ctx)
e.queueExistingProactiveSecurityCandidates(ctx)
// Classify once immediately so newly ingested or migration-requeued documents
// do not wait a full interval after startup.
e.processSourceInbox(ctx)
e.queueExistingProactiveSecurityCandidates(ctx)
e.processProactiveSecurityInbox(ctx)
run := func() int {
processed := e.processSourceInbox(ctx)
e.queueExistingProactiveSecurityCandidates(ctx)
processed += e.processProactiveSecurityInbox(ctx)
return processed
}
drain := func() {
for {
processed := run()
if !e.SpeedModeEnabled() || processed == 0 || ctx.Err() != nil {
return
}
}
}
// Always classify once immediately. In Speed mode the loop keeps claiming
// batches until the queues are empty instead of sleeping between batches.
drain()
ticker := time.NewTicker(e.Cfg.SourceInboxInterval)
defer ticker.Stop()
for {
@@ -70,24 +81,34 @@ func (e *Engine) sourceInboxLoop(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
e.processSourceInbox(ctx)
e.queueExistingProactiveSecurityCandidates(ctx)
e.processProactiveSecurityInbox(ctx)
drain()
case <-e.sourceInboxWake:
drain()
}
}
}
func (e *Engine) processSourceInbox(ctx context.Context) {
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled {
func (e *Engine) WakeSourceInbox() {
if e == nil || e.sourceInboxWake == nil {
return
}
select {
case e.sourceInboxWake <- struct{}{}:
default:
}
}
func (e *Engine) processSourceInbox(ctx context.Context) int {
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled {
return 0
}
items, err := e.SourceInbox.ClaimInbox(ctx, e.Cfg.SourceInboxBatchSize)
if err != nil {
slog.Warn("source inbox claim failed", "error", err)
return
return 0
}
if len(items) == 0 {
return
return 0
}
texts := make([]string, 0, len(items))
for _, item := range items {
@@ -97,7 +118,7 @@ func (e *Engine) processSourceInbox(ctx context.Context) {
}
texts = append(texts, text)
}
cctx, cancel := context.WithTimeout(ollama.WithLowPriority(ctx), 4*time.Minute)
cctx, cancel := context.WithTimeout(e.backgroundOllamaContext(ctx), 4*time.Minute)
vectors, embedErr := e.Ollama.Embed(cctx, texts)
cancel()
if embedErr != nil || len(vectors) != len(items) {
@@ -110,7 +131,7 @@ func (e *Engine) processSourceInbox(ctx context.Context) {
slog.Error("source inbox release after embedding failure failed", "inbox_id", item.ID, "error", releaseErr)
}
}
return
return 0
}
candidateCount := 0
for i, item := range items {
@@ -158,6 +179,7 @@ func (e *Engine) processSourceInbox(ctx context.Context) {
if e.Broker != nil {
e.Broker.Publish(model.Activity{Type: "source.inbox.classified", Source: "brain", Phase: "source-inbox", Message: fmt.Sprintf("Source-Inbox: %d Dokumente geprüft · %d als Wissenskandidaten vorgemerkt", len(items), candidateCount), Strength: .36, Metadata: map[string]any{"documents": len(items), "candidates": candidateCount, "minimum_similarity": e.Cfg.SourceInboxMinSimilarity, "minimum_priority": e.Cfg.SourceInboxMinPriority, "novelty_floor": e.Cfg.SourceInboxNoveltyFloor, "classifier_version": SourceInboxClassifierVersion}})
}
return len(items)
}
func (e *Engine) sourceInboxKnowledgeHit(hits []model.Hit) (model.Hit, bool) {
@@ -335,17 +357,17 @@ func (e *Engine) queueExistingProactiveSecurityCandidates(ctx context.Context) {
}
}
func (e *Engine) processProactiveSecurityInbox(ctx context.Context) {
func (e *Engine) processProactiveSecurityInbox(ctx context.Context) int {
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled || !e.Cfg.SourceInboxSecurityProactiveEnabled || e.Ollama == nil {
return
return 0
}
items, err := e.SourceInbox.ClaimProactiveSecurity(ctx, e.Cfg.SourceInboxSecurityBatchSize)
if err != nil {
slog.Warn("source inbox proactive security claim failed", "error", err)
return
return 0
}
if len(items) == 0 {
return
return 0
}
materialized := 0
for _, item := range items {
@@ -400,6 +422,7 @@ func (e *Engine) processProactiveSecurityInbox(ctx context.Context) {
if materialized > 0 {
e.RequestEnrich("source-security")
}
return len(items)
}
func (e *Engine) processProactiveSecurityItem(ctx context.Context, item sourceagent.InboxDocument, started time.Time) (bool, graph.MutationStats, error) {
@@ -610,7 +633,7 @@ func (e *Engine) assessSecurityInbox(ctx context.Context, item sourceagent.Inbox
if modelName == "" {
modelName = e.Cfg.ChatModel
}
cctx, cancel := context.WithTimeout(ollama.WithLowPriority(ctx), 6*time.Minute)
cctx, cancel := context.WithTimeout(e.backgroundOllamaContext(ctx), 6*time.Minute)
err := e.Ollama.ChatJSONModel(cctx, modelName, securityInboxSystemPrompt(), b.String(), securityInboxSchema(), &out)
cancel()
out = normalizeSecurityAssessment(out, item.Document.Title)
@@ -821,7 +844,7 @@ func (e *Engine) learnProactiveSecurityNode(ctx context.Context, node model.Node
if text == "" {
return graph.MutationStats{}
}
cctx, cancel := context.WithTimeout(ollama.WithLowPriority(ctx), 3*time.Minute)
cctx, cancel := context.WithTimeout(e.backgroundOllamaContext(ctx), 3*time.Minute)
vectors, err := e.Ollama.Embed(cctx, []string{text})
cancel()
if err != nil || len(vectors) != 1 || len(vectors[0]) == 0 {
+115
View File
@@ -0,0 +1,115 @@
package engine
import (
"context"
"runtime"
"time"
"github.com/local/glpi-neural-brain/internal/model"
"github.com/local/glpi-neural-brain/internal/ollama"
)
const speedModeVersion = "throughput-v1"
func (e *Engine) effectiveSpeedCPUWorkers(settings RuntimeSettings) int {
workers := settings.SpeedCPUWorkers
if workers < 1 {
workers = runtime.NumCPU()
}
if workers < 1 {
workers = 1
}
if workers > 256 {
workers = 256
}
return workers
}
func (e *Engine) effectiveSpeedGPUInflight(settings RuntimeSettings) int {
limit := settings.SpeedGPUInflight
if limit < 1 {
limit = 1
}
if limit > 64 {
limit = 64
}
return limit
}
func (e *Engine) applyRuntimePerformance(settings RuntimeSettings) {
if e == nil {
return
}
sharedInflight := e.Cfg.ResearchOllamaMaxInflight
queueSize := e.Cfg.ResearchOllamaQueueSize
gpuInflight := e.Cfg.OllamaNodeMaxInflight
if sharedInflight < 1 {
sharedInflight = 1
}
if queueSize < 1 {
queueSize = 1
}
if gpuInflight < 1 {
gpuInflight = 1
}
if settings.SpeedMode {
cpu := e.effectiveSpeedCPUWorkers(settings)
gpu := e.effectiveSpeedGPUInflight(settings)
// The shared limiter covers model calls and outbound research work. Do not
// let it become the bottleneck below either requested CPU fan-out or total
// per-node GPU fan-out.
nodeCount := 1
if e.Ollama != nil {
nodeCount = len(e.Ollama.NodeStatuses())
}
if nodeCount < 1 {
nodeCount = 1
}
sharedInflight = maxInt(cpu, gpu*nodeCount)
queueSize = maxInt(queueSize, sharedInflight*16)
gpuInflight = gpu
}
if e.sharedWork != nil {
e.sharedWork.SetLimits(sharedInflight, queueSize)
}
if e.Ollama != nil {
e.Ollama.SetNodeMaxInflight(gpuInflight)
}
}
func (e *Engine) signalSpeedWork() {
if e == nil || !e.SpeedModeEnabled() {
return
}
if e.ThinkingEnabled() {
e.RequestEnrich("speed-mode")
}
if e.AutonomousResearchEnabled() {
e.RequestAutonomousResearchScan("speed-mode")
e.signalAutonomousResearch()
}
if e.sourceInboxWake != nil {
select {
case e.sourceInboxWake <- struct{}{}:
default:
}
}
if e.Broker != nil {
settings := e.RuntimeSettings()
e.Broker.Publish(model.Activity{Type: "performance.speed.pulse", Source: "brain", Phase: "performance", Message: "Speed-Modus treibt vorhandene CPU-/GPU-Arbeit ohne künstliche Pacing-Pausen weiter", Strength: .58, Metadata: map[string]any{"algorithm": speedModeVersion, "cpu_tasks": settings.SpeedCPUWorkers, "gpu_tasks_per_node": settings.SpeedGPUInflight}})
}
}
func (e *Engine) backgroundOllamaContext(ctx context.Context) context.Context {
if e != nil && e.SpeedModeEnabled() {
return ctx
}
return ollama.WithLowPriority(ctx)
}
func (e *Engine) effectiveEnrichStepDelay() time.Duration {
if e != nil && e.SpeedModeEnabled() {
return 0
}
return e.Cfg.EnrichStepDelay
}
+35
View File
@@ -0,0 +1,35 @@
package engine
import (
"testing"
"github.com/local/glpi-neural-brain/internal/config"
"github.com/local/glpi-neural-brain/internal/ollama"
"github.com/local/glpi-neural-brain/internal/workqueue"
)
func TestApplyRuntimePerformanceRaisesAndRestoresLimits(t *testing.T) {
pool := ollama.NewPool(ollama.PoolConfig{
Nodes: []ollama.NodeConfig{{Name: "gpu-a", URL: "http://unused"}, {Name: "gpu-b", URL: "http://unused"}},
NodeMaxInflight: 1,
}, "qwen3:8b", "embeddinggemma")
limiter := workqueue.New(2, 8)
e := &Engine{
Cfg: config.Config{ResearchOllamaMaxInflight: 2, ResearchOllamaQueueSize: 8, OllamaNodeMaxInflight: 1},
Ollama: pool,
sharedWork: limiter,
}
e.applyRuntimePerformance(RuntimeSettings{SpeedMode: true, SpeedCPUWorkers: 6, SpeedGPUInflight: 3})
status := limiter.Status()
if status.MaxInflight != 6 || status.QueueSize < 96 {
t.Fatalf("unexpected speed limiter: %+v", status)
}
if got := pool.NodeMaxInflight(); got != 3 {
t.Fatalf("gpu limit=%d want 3", got)
}
e.applyRuntimePerformance(RuntimeSettings{SpeedMode: false})
status = limiter.Status()
if status.MaxInflight != 2 || status.QueueSize != 8 || pool.NodeMaxInflight() != 1 {
t.Fatalf("normal limits not restored: limiter=%+v gpu=%d", status, pool.NodeMaxInflight())
}
}
+34 -11
View File
@@ -21,11 +21,17 @@ type vectorLayerExecution struct {
AgentID string
ComputeMS int64
FallbackReason string
LayoutMode string
LayoutDue bool
}
func (e *Engine) vectorLayerConfig() graph.VectorSemanticLayerConfig {
workers := 1
if settings := e.RuntimeSettings(); settings.SpeedMode {
workers = e.effectiveSpeedCPUWorkers(settings)
}
return graph.VectorSemanticLayerConfig{
Neighbors: e.Cfg.VectorGraphNeighbors, CandidateLimit: e.Cfg.VectorGraphCandidates,
Workers: workers, Neighbors: e.Cfg.VectorGraphNeighbors, CandidateLimit: e.Cfg.VectorGraphCandidates,
HashBits: e.Cfg.ClusterHashBits, HashTables: e.Cfg.ClusterHashTables,
MinSimilarity: e.Cfg.VectorGraphMinSimilarity, MinAffinity: e.Cfg.VectorGraphMinAffinity,
Layout: e.Cfg.VectorGraphLayout, LayoutRelax: false, LayoutBlend: e.Cfg.VectorGraphLayoutBlend, LayoutMaxShift: e.Cfg.VectorGraphLayoutMaxShift,
@@ -36,8 +42,12 @@ func (e *Engine) vectorLayerConfig() graph.VectorSemanticLayerConfig {
}
func (e *Engine) vectorPrimaryConfig(layout bool) vectorgraph.Config {
workers := 1
if settings := e.RuntimeSettings(); settings.SpeedMode {
workers = e.effectiveSpeedCPUWorkers(settings)
}
return vectorgraph.Config{
Neighbors: e.Cfg.VectorGraphNeighbors, CandidateLimit: e.Cfg.VectorGraphCandidates,
Workers: workers, Neighbors: e.Cfg.VectorGraphNeighbors, CandidateLimit: e.Cfg.VectorGraphCandidates,
HashBits: e.Cfg.ClusterHashBits, HashTables: e.Cfg.ClusterHashTables, BandBits: 8,
MinSimilarity: e.Cfg.VectorGraphMinSimilarity, MinAffinity: e.Cfg.VectorGraphMinAffinity,
Layout: layout, Smoothing: .22,
@@ -45,8 +55,12 @@ func (e *Engine) vectorPrimaryConfig(layout bool) vectorgraph.Config {
}
func (e *Engine) vectorOrphanConfig() vectorgraph.Config {
workers := 1
if settings := e.RuntimeSettings(); settings.SpeedMode {
workers = e.effectiveSpeedCPUWorkers(settings)
}
return vectorgraph.Config{
Neighbors: e.Cfg.VectorGraphOrphanNeighbors, CandidateLimit: e.Cfg.VectorGraphOrphanCandidates,
Workers: workers, Neighbors: e.Cfg.VectorGraphOrphanNeighbors, CandidateLimit: e.Cfg.VectorGraphOrphanCandidates,
HashBits: e.Cfg.ClusterHashBits, HashTables: e.Cfg.ClusterHashTables, BandBits: 8,
MinSimilarity: e.Cfg.VectorGraphOrphanMinSimilarity, MinAffinity: e.Cfg.VectorGraphOrphanMinAffinity,
Layout: false,
@@ -104,15 +118,24 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
layoutDue := e.Cfg.VectorGraphLayout || (e.Cfg.VectorGraphRelaxLayout && (lastVectorLayout.IsZero() || time.Since(lastVectorLayout) >= e.Cfg.VectorGraphLayoutRelaxInterval))
cfg.Layout = layoutDue
cfg.LayoutRelax = !e.Cfg.VectorGraphLayout && layoutDue
layoutMode := "off"
if e.Cfg.VectorGraphLayout && layoutDue {
layoutMode = "full"
} else if cfg.LayoutRelax {
layoutMode = "relax"
}
if !e.Cfg.VectorGraphAgentOffload || e.SourceInbox == nil {
primary, orphan, focus := e.Graph.BuildVectorSemanticLayerLocal(cfg, filter)
stats, mutations := e.Graph.ApplyVectorSemanticLayer(cfg, primary, orphan, focus)
e.noteVectorLayoutApplied(layoutDue, stats.PositionUpdates)
return vectorLayerExecution{Stats: stats, Mutations: mutations}, nil
return vectorLayerExecution{Stats: stats, Mutations: mutations, LayoutMode: layoutMode, LayoutDue: layoutDue}, nil
}
hasAgent, availabilityErr := e.checkOnlineVectorComputeAgent(ctx)
startupGrace := vectorAgentStartupGrace(!e.bootstrapIsComplete(), e.Cfg.VectorGraphAgentWait)
if e.SpeedModeEnabled() && !e.Cfg.VectorGraphAgentRequired {
startupGrace = 0
}
if availabilityErr == nil && !hasAgent && startupGrace > 0 {
_ = e.requestControllerComputeCapacity(ctx, sourceagent.ComputeKindVectorGraph)
e.Broker.Publish(model.Activity{Type: "vector.graph.agent.waiting", Source: "brain", Phase: "semantic-linking", Message: fmt.Sprintf("Initialer Vector-Graph wartet bis zu %s auf die Compute-Agent-Registrierung", startupGrace), Strength: .34, Metadata: map[string]any{"run_id": runID, "kind": sourceagent.ComputeKindVectorGraph, "startup_grace": startupGrace.String()}})
@@ -127,12 +150,12 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
reason = "compute_agent_check_failed: " + availabilityErr.Error()
}
if e.Cfg.VectorGraphAgentRequired {
return vectorLayerExecution{FallbackReason: reason}, fmt.Errorf("vector graph agent offload required but unavailable: %s", reason)
return vectorLayerExecution{FallbackReason: reason, LayoutMode: layoutMode, LayoutDue: layoutDue}, fmt.Errorf("vector graph agent offload required but unavailable: %s", reason)
}
primary, orphan, focus := e.Graph.BuildVectorSemanticLayerLocal(cfg, filter)
stats, mutations := e.Graph.ApplyVectorSemanticLayer(cfg, primary, orphan, focus)
e.noteVectorLayoutApplied(layoutDue, stats.PositionUpdates)
return vectorLayerExecution{Stats: stats, Mutations: mutations, FallbackReason: reason}, nil
return vectorLayerExecution{Stats: stats, Mutations: mutations, FallbackReason: reason, LayoutMode: layoutMode, LayoutDue: layoutDue}, nil
}
entries := e.Graph.VectorSemanticEntries(filter)
@@ -146,7 +169,7 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
Entries: entries,
}
e.Broker.Publish(model.Activity{Type: "vector.graph.agent.queued", Source: "brain", Phase: "semantic-linking", Message: fmt.Sprintf("Vector-Graph-CPU-Job für Agent bereitgestellt · %d Embeddings", len(entries)), Strength: .46, Metadata: map[string]any{
"run_id": runID, "kind": sourceagent.ComputeKindVectorGraph, "indexed": len(entries), "orphan_focus_candidates": len(baseOrphans), "graph_version": inputVersion,
"run_id": runID, "kind": sourceagent.ComputeKindVectorGraph, "indexed": len(entries), "orphan_focus_candidates": len(baseOrphans), "graph_version": inputVersion, "cpu_workers": request.Header.Primary.Workers, "speed_mode": e.SpeedModeEnabled(),
}})
jobCtx, cancelJob := context.WithTimeout(ctx, e.Cfg.VectorGraphAgentWait)
result, err := e.SourceInbox.SubmitVectorGraphJob(jobCtx, request)
@@ -161,12 +184,12 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
reason := err.Error()
e.Broker.Publish(model.Activity{Type: "vector.graph.agent.fallback", Source: "brain", Phase: "semantic-linking", Message: "Agent-Vectorjob konnte nicht sicher übernommen werden; lokale CPU-Berechnung wird verwendet", Strength: .38, Metadata: map[string]any{"run_id": runID, "reason": reason, "graph_version": inputVersion}})
if e.Cfg.VectorGraphAgentRequired {
return vectorLayerExecution{FallbackReason: reason}, err
return vectorLayerExecution{FallbackReason: reason, LayoutMode: layoutMode, LayoutDue: layoutDue}, err
}
primary, orphan, focus := e.Graph.BuildVectorSemanticLayerLocal(cfg, filter)
stats, mutations := e.Graph.ApplyVectorSemanticLayer(cfg, primary, orphan, focus)
e.noteVectorLayoutApplied(layoutDue, stats.PositionUpdates)
return vectorLayerExecution{Stats: stats, Mutations: mutations, FallbackReason: reason}, nil
return vectorLayerExecution{Stats: stats, Mutations: mutations, FallbackReason: reason, LayoutMode: layoutMode, LayoutDue: layoutDue}, nil
}
focus := remainingOrphanFocus(baseOrphans, result.Primary.Links)
@@ -174,9 +197,9 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
e.noteVectorLayoutApplied(layoutDue, stats.PositionUpdates)
e.Broker.Publish(model.Activity{Type: "vector.graph.agent.completed", Source: "agent", Phase: "semantic-linking", Message: fmt.Sprintf("Agent hat Vector-Graph-CPU-Job abgeschlossen · %d + %d Kanten", len(result.Primary.Links), len(result.Orphan.Links)), Strength: .68, Metadata: withRunMutations(map[string]any{
"run_id": runID, "agent_id": result.AgentID, "compute_duration_ms": result.DurationMS, "indexed": len(entries),
"primary_links": len(result.Primary.Links), "orphan_links": len(result.Orphan.Links), "orphan_focus": len(focus), "no_model_call": true,
"primary_links": len(result.Primary.Links), "orphan_links": len(result.Orphan.Links), "orphan_focus": len(focus), "no_model_call": true, "cpu_workers": request.Header.Primary.Workers, "speed_mode": e.SpeedModeEnabled(),
}, mutations)})
return vectorLayerExecution{Stats: stats, Mutations: mutations, Offloaded: true, AgentID: result.AgentID, ComputeMS: result.DurationMS}, nil
return vectorLayerExecution{Stats: stats, Mutations: mutations, Offloaded: true, AgentID: result.AgentID, ComputeMS: result.DurationMS, LayoutMode: layoutMode, LayoutDue: layoutDue}, nil
}
func (e *Engine) noteVectorLayoutApplied(layoutDue bool, updates uint64) {
+134
View File
@@ -0,0 +1,134 @@
package engine
import (
"context"
"fmt"
"log/slog"
"time"
"github.com/local/glpi-neural-brain/internal/graph"
"github.com/local/glpi-neural-brain/internal/model"
)
func (e *Engine) effectiveVectorLayoutMode() string {
if e.Cfg.VectorGraphLayout {
return "full"
}
if e.Cfg.VectorGraphRelaxLayout {
return "relax"
}
return "off"
}
func (e *Engine) vectorMaintenanceDue(now time.Time) (reevaluate bool, layout bool) {
if !e.Cfg.VectorGraphEnabled {
return false, false
}
e.stateMu.RLock()
lastGraph := e.lastVectorGraph
lastLayout := e.lastVectorLayout
maintenanceStartedAt := e.vectorMaintenanceStartedAt
e.stateMu.RUnlock()
reevaluate = !e.Graph.HasEdgesByOrigin(graph.VectorMathOrigin) || lastGraph.IsZero() || now.Sub(lastGraph) >= e.Cfg.VectorGraphReevaluateInterval
// Full layout is intentionally coupled to semantic reevaluation. Relax mode
// may run on its own slower/faster cadence without enabling the hard layout.
if !e.Cfg.VectorGraphLayout && e.Cfg.VectorGraphRelaxLayout {
if lastLayout.IsZero() {
// Do not fake last_vector_layout merely to delay the first relaxation.
// Keep telemetry truthful and use the process-start marker as a separate
// not-before timestamp.
layout = maintenanceStartedAt.IsZero() || now.Sub(maintenanceStartedAt) >= e.Cfg.VectorGraphLayoutRelaxInterval
} else {
layout = now.Sub(lastLayout) >= e.Cfg.VectorGraphLayoutRelaxInterval
}
}
return reevaluate, layout
}
func (e *Engine) vectorMaintenanceLoop(ctx context.Context) {
// Give bootstrap-owned scans a short head start. If Learning is disabled the
// loop still runs and can maintain an already persisted vector layer.
initialDelay := 5 * time.Second
if e.SpeedModeEnabled() {
initialDelay = 0
}
initial := time.NewTimer(initialDelay)
defer initial.Stop()
select {
case <-ctx.Done():
return
case <-initial.C:
}
if err := e.runVectorMaintenance(ctx, "startup-maintenance"); err != nil {
slog.Warn("vector maintenance startup run failed", "error", err)
}
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := e.runVectorMaintenance(ctx, "scheduled-maintenance"); err != nil {
slog.Warn("vector maintenance run failed", "error", err)
}
}
}
}
func (e *Engine) runVectorMaintenance(ctx context.Context, trigger string) error {
reevaluateDue, layoutDue := e.vectorMaintenanceDue(time.Now().UTC())
if !reevaluateDue && !layoutDue {
return nil
}
// Serialize with Learning Scan. Both operations may replace vector-math
// edges/positions and must never race against each other.
e.mu.Lock()
defer e.mu.Unlock()
reevaluateDue, layoutDue = e.vectorMaintenanceDue(time.Now().UTC())
if !reevaluateDue && !layoutDue {
return nil
}
if e.Graph.CountVectorsByDimension(256) != 0 {
if e.Broker != nil {
e.Broker.Publish(model.Activity{Type: "vector.graph.maintenance.skipped", Source: "brain", Phase: "semantic-linking", Message: "Vector-Maintenance wartet auf echte Embeddings; 256D-Fallback-Vektoren werden nicht mit dem Produktionsraum gemischt", Strength: .28, Metadata: map[string]any{"trigger": trigger, "reason": "fallback_vectors_present"}})
}
return nil
}
entries := e.Graph.VectorSemanticEntries(e.effectiveLearningFilter())
if len(entries) < 2 {
return nil
}
started := time.Now().UTC()
runID := fmt.Sprintf("vector-maintenance-%d", started.UnixNano())
if e.Broker != nil {
e.Broker.Publish(model.Activity{Type: "vector.graph.maintenance.started", Source: "brain", Phase: "semantic-linking", Message: "Periodische mathematische Nähepflege wurde gestartet", Strength: .4, Metadata: map[string]any{
"run_id": runID, "trigger": trigger, "reevaluate_due": reevaluateDue, "layout_due": layoutDue, "effective_layout_mode": e.effectiveVectorLayoutMode(), "learning_enabled": e.LearningEnabled(),
}})
}
execution, err := e.rebuildVectorSemanticLayer(ctx, runID, e.effectiveLearningFilter())
if err != nil {
if e.Broker != nil {
e.Broker.Publish(model.Activity{Type: "vector.graph.maintenance.failed", Source: "brain", Phase: "semantic-linking", Message: "Periodische mathematische Nähepflege ist fehlgeschlagen", Strength: .32, Metadata: map[string]any{"run_id": runID, "trigger": trigger, "error": err.Error(), "agent_required": e.Cfg.VectorGraphAgentRequired}})
}
return err
}
now := time.Now().UTC()
e.stateMu.Lock()
e.lastVectorGraph = now
e.stateMu.Unlock()
stats := execution.Stats
if e.Broker != nil {
meta := withRunMutations(map[string]any{
"run_id": runID, "trigger": trigger, "algorithm": "mutual-knn-local-scaling-v1", "orphan_algorithm": "orphan-knn-local-scaling-v1", "no_model_call": true,
"indexed": stats.Indexed, "links": stats.Links, "reciprocal_links": stats.ReciprocalLinks, "candidate_pairs": stats.CandidatePairs, "exact_comparisons": stats.ExactComparisons,
"orphan_pass_enabled": e.Cfg.VectorGraphOrphanPass, "orphan_focus": stats.OrphanFocus, "orphan_links": stats.OrphanLinks, "orphan_exact_comparisons": stats.OrphanStats.ExactComparisons, "orphan_candidate_pairs": stats.OrphanStats.CandidatePairs,
"position_updates": stats.PositionUpdates, "layout_mode": execution.LayoutMode, "layout_applied": execution.LayoutDue && stats.PositionUpdates > 0, "periodic_refresh": reevaluateDue, "layout_refresh": layoutDue,
"reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(), "layout_relax_interval": e.Cfg.VectorGraphLayoutRelaxInterval.String(), "learning_enabled": e.LearningEnabled(),
"agent_offloaded": execution.Offloaded, "agent_id": execution.AgentID, "agent_compute_ms": execution.ComputeMS, "agent_fallback_reason": execution.FallbackReason, "speed_mode": e.SpeedModeEnabled(), "cpu_workers": e.vectorPrimaryConfig(execution.LayoutDue).Workers, "duration_ms": time.Since(started).Milliseconds(),
}, execution.Mutations)
e.Broker.Publish(model.Activity{Type: "vector.graph.rebuilt", Source: "brain", Phase: "semantic-linking", Message: fmt.Sprintf("Periodische Vektorpflege: %d Primär- + %d Orphan-Kanten aus %d Embeddings", stats.Links, stats.OrphanLinks, stats.Indexed), Strength: .58, Metadata: meta})
e.Broker.Publish(model.Activity{Type: "vector.graph.maintenance.completed", Source: "brain", Phase: "semantic-linking", Message: fmt.Sprintf("Mathematische Nähepflege abgeschlossen · Layoutmodus %s · %d Positionsupdates", execution.LayoutMode, stats.PositionUpdates), Strength: .5, Metadata: meta})
}
return nil
}
@@ -0,0 +1,22 @@
package engine
import (
"testing"
"github.com/local/glpi-neural-brain/internal/config"
)
func TestEffectiveVectorLayoutModeFullOverridesRelax(t *testing.T) {
e := &Engine{Cfg: config.Config{VectorGraphLayout: true, VectorGraphRelaxLayout: true}}
if got := e.effectiveVectorLayoutMode(); got != "full" {
t.Fatalf("full layout must win when both flags are true, got %q", got)
}
e.Cfg.VectorGraphLayout = false
if got := e.effectiveVectorLayoutMode(); got != "relax" {
t.Fatalf("expected relax mode, got %q", got)
}
e.Cfg.VectorGraphRelaxLayout = false
if got := e.effectiveVectorLayoutMode(); got != "off" {
t.Fatalf("expected off mode, got %q", got)
}
}