@@ -257,11 +257,15 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
|
||||
var quality model.ArticleQualityDecision
|
||||
var finalReviewEvidence []model.ResearchResult
|
||||
var finalCPUQuality articlequality.Result
|
||||
var bestCPUDraft model.KnowledgeArticleDraft
|
||||
var bestCPUQuality articlequality.Result
|
||||
hasBestCPUQuality := false
|
||||
var reviewFeedback *model.ArticleQualityDecision
|
||||
rewritten := false
|
||||
repairAttempts := 0
|
||||
cpuRepairAttempts := 0
|
||||
authorResearchAttempted := false
|
||||
articleTypeReconsidered := false
|
||||
for {
|
||||
content, wasRewritten, generationErr := e.generateArticleContent(ctx, articleRunID, selected, plan, brief, researchResults, reviewFeedback)
|
||||
if generationErr != nil {
|
||||
@@ -311,6 +315,40 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
|
||||
metadata["trigger"] = trigger
|
||||
metadata["article_type"] = normalizeArticleType(plan.ArticleType)
|
||||
metadata["author_research_attempted"] = authorResearchAttempted
|
||||
var validationErr *articleDraftValidationError
|
||||
reconsiderable := errors.As(structureErr, &validationErr) && isOperationalArticleType(plan.ArticleType) && !articleTypeReconsidered && (validationErr.Code == "insufficient_solution_steps" || validationErr.Code == "missing_validation_steps")
|
||||
if reconsiderable {
|
||||
articleTypeReconsidered = true
|
||||
previousType := normalizeArticleType(plan.ArticleType)
|
||||
reconsideration, reconsiderErr := e.reconsiderOperationalArticleType(ctx, plan, content, selected, researchResults)
|
||||
if reconsiderErr != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "article.type.reconsideration.failed", Source: "brain", Phase: "knowledge-planning", NodeIDs: plan.SourceNodeIDs, Message: "Die einmalige Artikeltyp-Neubewertung ist fehlgeschlagen", Strength: .34, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "previous_article_type": previousType, "reason": validationErr.Code, "error": reconsiderErr.Error()})})
|
||||
} else if reconsideration.Action == "reclassify" {
|
||||
plan.ArticleType = reconsideration.ArticleType
|
||||
plan.Reason = strings.TrimSpace(strings.Join([]string{plan.Reason, "v10 type reconsideration: " + reconsideration.Reason}, " · "))
|
||||
// Type is part of article identity. Recompute the fingerprint and avoid
|
||||
// generating a duplicate non-operational draft if one already exists.
|
||||
sourceFingerprint = articleSourceFingerprint(selected, plan, selectedPlanningResearch, e.articlePipelineFingerprintIdentity())
|
||||
if e.hasArticleSourceFingerprint(sourceFingerprint) || e.hasEquivalentArticleDraft(selected, plan, sourceFingerprint) {
|
||||
e.Broker.Publish(model.Activity{Type: "article.duplicate", Source: "brain", Phase: "knowledge-planning", NodeIDs: plan.SourceNodeIDs, Message: "Die korrigierte Artikeltyp-Variante existiert bereits", Strength: .36, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "reason": "reclassified_article_duplicate", "previous_article_type": previousType, "article_type": plan.ArticleType, "source_fingerprint": sourceFingerprint})})
|
||||
return articleSynthesisOutcome{Skipped: true, Reason: "reclassified_article_duplicate", Action: plan.Action}, nil
|
||||
}
|
||||
if e.RuntimeSettings().ProcessingMode == "clustered" {
|
||||
brief = fallbackSynthesisBrief(plan, selected)
|
||||
} else if rebuiltBrief, briefErr := e.buildKnowledgeBrief(ctx, selected, researchResults, plan.ArticleType); briefErr == nil {
|
||||
brief = rebuiltBrief
|
||||
}
|
||||
reviewFeedback = nil
|
||||
cpuRepairAttempts = 0
|
||||
e.Broker.Publish(model.Activity{Type: "article.type.reconsidered", Source: "brain", Phase: "knowledge-planning", NodeIDs: plan.SourceNodeIDs, Message: fmt.Sprintf("Operationaler Artikeltyp wurde einmalig von %s auf %s korrigiert", previousType, plan.ArticleType), Strength: .7, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "previous_article_type": previousType, "article_type": plan.ArticleType, "reason": reconsideration.Reason, "structure_failure": validationErr.Code, "author_research_attempted": authorResearchAttempted})})
|
||||
continue
|
||||
} else {
|
||||
metadata["reason"] = "wrong_article_type_or_insufficient_operational_evidence"
|
||||
metadata["reconsideration_reason"] = reconsideration.Reason
|
||||
e.Broker.Publish(model.Activity{Type: "article.type.reconsidered", Source: "brain", Phase: "knowledge-planning", NodeIDs: plan.SourceNodeIDs, Message: "Operationaler Artikeltyp trägt trotz Evidenzsuche keinen belastbaren Lösungsablauf und wird nicht künstlich umgedeutet", Strength: .46, Metadata: articleRunMetadata(articleRunID, metadata)})
|
||||
return articleSynthesisOutcome{Skipped: true, Reason: "wrong_article_type_or_insufficient_operational_evidence", Action: plan.Action}, nil
|
||||
}
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "article.draft.rejected", Source: "brain", Phase: "quality-gate", NodeIDs: plan.SourceNodeIDs, Message: "Der Entwurf erfüllt die Mindeststruktur seines Artikeltyps nicht", Strength: .4, Metadata: articleRunMetadata(articleRunID, metadata)})
|
||||
return articleSynthesisOutcome{Skipped: true, Reason: metadata["reason"].(string), Action: plan.Action}, nil
|
||||
}
|
||||
@@ -323,13 +361,18 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
|
||||
return articleSynthesisOutcome{}, fmt.Errorf("deterministic article quality evaluation failed: %w", cpuErr)
|
||||
}
|
||||
finalCPUQuality = cpuQuality.Result
|
||||
if !hasBestCPUQuality || articleCPUQualityBetter(cpuQuality.Result, bestCPUQuality) {
|
||||
bestCPUDraft = draft
|
||||
bestCPUQuality = cpuQuality.Result
|
||||
hasBestCPUQuality = true
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "article.cpu_quality.completed", Source: map[bool]string{true: "agent", false: "brain"}[cpuQuality.Offloaded], Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: fmt.Sprintf("Modellfreie Artikelprüfung: Score %.2f · %d Wörter · %d Abschnitte", cpuQuality.Result.Score, cpuQuality.Result.WordCount, cpuQuality.Result.SectionCount), Strength: .64, Metadata: articleRunMetadata(articleRunID, map[string]any{
|
||||
"trigger": trigger, "article_type": normalizeArticleType(plan.ArticleType), "passed": cpuQuality.Result.Passed, "score": cpuQuality.Result.Score,
|
||||
"word_count": cpuQuality.Result.WordCount, "content_word_count": cpuQuality.Result.ContentWordCount, "section_count": cpuQuality.Result.SectionCount,
|
||||
"lexical_diversity": cpuQuality.Result.LexicalDiversity, "redundancy": cpuQuality.Result.Redundancy, "evidence_alignment": cpuQuality.Result.EvidenceAlignment,
|
||||
"source_utilization": cpuQuality.Result.SourceUtilization, "technical_specificity": cpuQuality.Result.TechnicalSpecificity, "type_depth_score": cpuQuality.Result.TypeDepthScore,
|
||||
"hard_failures": cpuQuality.Result.HardFailures, "recommendations": cpuQuality.Result.Recommendations, "algorithm": cpuQuality.Result.Algorithm,
|
||||
"no_model_call": true, "agent_offloaded": cpuQuality.Offloaded, "agent_id": cpuQuality.AgentID, "compute_ms": cpuQuality.ComputeMS, "fallback_reason": cpuQuality.FallbackReason,
|
||||
"evidence_alignment_mode": "diagnostic_only_semantic_grounding_by_qwen", "no_model_call": true, "agent_offloaded": cpuQuality.Offloaded, "agent_id": cpuQuality.AgentID, "compute_ms": cpuQuality.ComputeMS, "fallback_reason": cpuQuality.FallbackReason,
|
||||
})})
|
||||
if !cpuQuality.Result.Passed {
|
||||
if cpuRepairAttempts < 1 {
|
||||
@@ -339,7 +382,18 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
|
||||
e.Broker.Publish(model.Activity{Type: "article.cpu_quality.revision", Source: "brain", Phase: "knowledge-synthesis", NodeIDs: draft.SourceNodeIDs, Message: "Der modellfreie Quality-Layer fordert vor dem LLM-Review eine substanziellere Fassung an", Strength: .72, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "score": cpuQuality.Result.Score, "hard_failures": cpuQuality.Result.HardFailures, "recommendations": cpuQuality.Result.Recommendations, "repair_attempt": cpuRepairAttempts})})
|
||||
continue
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "article.draft.rejected", Source: "brain", Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: "Der Artikel ist mathematisch/strukturell noch zu kurz, redundant oder nutzt die Evidenz nicht ausreichend", Strength: .46, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "reason": "article_cpu_quality_rejected", "score": cpuQuality.Result.Score, "hard_failures": cpuQuality.Result.HardFailures, "recommendations": cpuQuality.Result.Recommendations, "agent_offloaded": cpuQuality.Offloaded})})
|
||||
// A repair must never erase a stronger previous draft. This mattered in
|
||||
// analysis(21), where several CPU-requested rewrites became materially
|
||||
// shorter. Restore the strongest structurally evaluated version for final
|
||||
// diagnostics/rejection instead of silently keeping the regression.
|
||||
if hasBestCPUQuality && articleCPUQualityBetter(bestCPUQuality, cpuQuality.Result) {
|
||||
previousWords, previousScore := cpuQuality.Result.WordCount, cpuQuality.Result.Score
|
||||
draft = bestCPUDraft
|
||||
cpuQuality.Result = bestCPUQuality
|
||||
finalCPUQuality = bestCPUQuality
|
||||
e.Broker.Publish(model.Activity{Type: "article.cpu_quality.revision.regressed", Source: "brain", Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: "Die CPU-Revision war schwächer als die Vorfassung; die bessere Fassung bleibt erhalten", Strength: .5, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "repair_word_count": previousWords, "repair_score": previousScore, "restored_word_count": bestCPUQuality.WordCount, "restored_score": bestCPUQuality.Score, "hard_failures": bestCPUQuality.HardFailures})})
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "article.draft.rejected", Source: "brain", Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: "Der Artikel verfehlt nach einer gezielten Revision weiterhin harte mathematische Struktur-/Dichtegrenzen", Strength: .46, Metadata: articleRunMetadata(articleRunID, map[string]any{"trigger": trigger, "reason": "article_cpu_quality_rejected", "score": cpuQuality.Result.Score, "hard_failures": cpuQuality.Result.HardFailures, "recommendations": cpuQuality.Result.Recommendations, "agent_offloaded": cpuQuality.Offloaded})})
|
||||
return articleSynthesisOutcome{Skipped: true, Reason: "article_cpu_quality_rejected", Action: plan.Action, Title: draft.Title}, nil
|
||||
}
|
||||
|
||||
@@ -1265,11 +1319,12 @@ merge: Mehrere produktive Artikel überschneiden sich und sollten als Staging-En
|
||||
skip: Kein echter Mehrwert, bloße Dublette oder ein Thema, das auch nach realistischer Recherche keinen eigenständigen Helpdesk-Nutzen hätte. Fehlende recherchierbare Fakten sind allein kein skip-Grund: Wähle in diesem Fall create, update oder merge und setze needs_research=true mit einer präzisen ersten Suchfrage.
|
||||
|
||||
Artikeltyp-Regeln:
|
||||
- troubleshooting: konkreter Fehlerzustand, Fehlercode, Ausfall, Diagnose oder Wiederherstellung. Ein einzelner operativer Fehlercode ist troubleshooting, nicht how_to.
|
||||
- how_to: bewusst auszuführende Einrichtung, Konfiguration oder Prozedur ohne primären Fehlerzustand.
|
||||
- reference: mehrere Fehlercodes, Mechanismen, Statuswerte oder technische Zuordnungen, wenn kein einzelner Lösungsablauf im Mittelpunkt steht.
|
||||
- concept: technische Grundlagen und Zusammenhänge ohne operativen Ablauf.
|
||||
- troubleshooting: nur wenn ein konkreter Fehlerzustand, Fehlercode, Ausfall, Diagnose oder Wiederherstellung im Mittelpunkt steht UND die Quellen bereits einen realistischen ausführbaren Diagnose-/Recovery-Pfad tragen. Ein einzelner operativer Fehlercode ist troubleshooting, nicht how_to.
|
||||
- how_to: nur für eine bewusst auszuführende Einrichtung, Konfiguration oder Prozedur, deren konkrete Schritte aus den Quellen ableitbar sind.
|
||||
- reference: mehrere Fehlercodes, Mechanismen, Statuswerte, Profile, TTPs, Artefakte oder technische Zuordnungen, wenn kein einzelner Lösungsablauf im Mittelpunkt steht.
|
||||
- concept: technische Grundlagen, Architektur und Zusammenhänge ohne operativen Ablauf.
|
||||
- decision_guide: Auswahl oder Abgrenzung anhand belastbarer Kriterien.
|
||||
WICHTIG: Allgemeine Härtung, Prävention, Framework-/Standard-Zuordnung, Threat-Intelligence-Profile oder technische Mechanismen sind nicht automatisch troubleshooting. Wenn die Quellen keine mindestens drei konkreten belegbaren Schritte erwarten lassen, bevorzuge reference/concept/decision_guide statt einen künstlichen operationalen Artikel zu erzwingen.
|
||||
|
||||
Erfinde keine Fakten. Bevorzuge konkrete Problemlösung gegenüber technischer Meta-Analyse. target_article_id ist bei update/merge zwingend eine SOURCE_NODE_ID einer produktiven Quelle. source_node_ids dürfen nur IDs aus dem Kontext enthalten. Wenn notwendige Fakten fehlen, setze needs_research=true. Gib ausschließlich JSON nach Schema zurück.`
|
||||
}
|
||||
@@ -2233,7 +2288,7 @@ func (e *Engine) writeKnowledgeArticleDraft(sources []articleSource, plan model.
|
||||
"solution_step_count": countMarkdownNumberedSteps(draft.Answer), "validation_step_count": len(cleanArticleItems(draft.Validation)),
|
||||
"productive_source_count": productionCount, "ai_source_count": aiCount, "production_ratio": productionRatio,
|
||||
"source_fingerprint": fingerprint, "synthesis_model": e.Cfg.ArticleSynthesisModel, "review_model": e.Cfg.ArticleReviewModel,
|
||||
"pipeline": "adaptive_generate_review/v3-quality-v2",
|
||||
"pipeline": "adaptive_generate_review/v5-quality-gate-v12",
|
||||
"cpu_quality_algorithm": cpuQuality.Algorithm, "cpu_quality_score": cpuQuality.Score, "cpu_quality_passed": cpuQuality.Passed,
|
||||
"cpu_quality_word_count": cpuQuality.WordCount, "cpu_quality_section_count": cpuQuality.SectionCount,
|
||||
},
|
||||
@@ -2267,7 +2322,7 @@ func (e *Engine) writeKnowledgeArticleDraft(sources []articleSource, plan model.
|
||||
"source_nodes": externalIDsFromArticleSources(sources), "source_node_ids": sourceIDs,
|
||||
"productive_source_count": productionCount, "ai_source_count": aiCount, "production_ratio": productionRatio,
|
||||
"generation_depth": generationDepth, "confidence": draft.Confidence, "open_questions": draft.OpenQuestions, "language": articleLanguageTag(e.Cfg.ArticleLanguage), "source_fingerprint": fingerprint,
|
||||
"synthesis_model": e.Cfg.ArticleSynthesisModel, "review_model": e.Cfg.ArticleReviewModel, "pipeline": "adaptive_generate_review/v3-quality-v2",
|
||||
"synthesis_model": e.Cfg.ArticleSynthesisModel, "review_model": e.Cfg.ArticleReviewModel, "pipeline": "adaptive_generate_review/v5-quality-gate-v12",
|
||||
"knowledge_brief": brief, "research_query": plan.ResearchQuery, "research_material": evidence,
|
||||
"grounded_research_evidence": researchEvidenceMetadata(groundedResearch),
|
||||
"article_review": quality, "article_cpu_quality": cpuQuality, "review_repair_attempts": repairAttempts,
|
||||
|
||||
@@ -179,7 +179,7 @@ func (e *Engine) collectAdaptiveInitialResearch(ctx context.Context, trigger str
|
||||
}
|
||||
|
||||
func (e *Engine) articlePipelineFingerprintIdentity() string {
|
||||
return fmt.Sprintf("adaptive_generate_review/v3-quality-v2|topic-guard=strict-v2|provenance=v2|lang=%s|author=%s|reviewer=%s|research=%s|repair=%d", articleLanguageTag(e.Cfg.ArticleLanguage), strings.TrimSpace(e.Cfg.ArticleSynthesisModel), strings.TrimSpace(e.Cfg.ArticleReviewModel), e.effectiveArticleResearchStrategy(), e.Cfg.ArticleReviewRepairRounds)
|
||||
return fmt.Sprintf("adaptive_generate_review/v5-quality-gate-v12|topic-guard=strict-v3-relation-evidence|provenance=v2|lang=%s|author=%s|reviewer=%s|research=%s|repair=%d", articleLanguageTag(e.Cfg.ArticleLanguage), strings.TrimSpace(e.Cfg.ArticleSynthesisModel), strings.TrimSpace(e.Cfg.ArticleReviewModel), e.effectiveArticleResearchStrategy(), e.Cfg.ArticleReviewRepairRounds)
|
||||
}
|
||||
|
||||
func articleWorkFingerprint(sources []articleSource, relation model.RelationDecision, research []model.ResearchResult, pipelineIdentity string) string {
|
||||
|
||||
@@ -128,3 +128,22 @@ func (e *Engine) evaluateArticleCPUQuality(ctx context.Context, articleRunID str
|
||||
}
|
||||
return articleCPUQualityExecution{Result: result.Quality, Offloaded: true, AgentID: result.AgentID, ComputeMS: result.DurationMS}, nil
|
||||
}
|
||||
|
||||
func articleCPUQualityBetter(a, b articlequality.Result) bool {
|
||||
if a.Passed != b.Passed {
|
||||
return a.Passed
|
||||
}
|
||||
if len(a.HardFailures) != len(b.HardFailures) {
|
||||
return len(a.HardFailures) < len(b.HardFailures)
|
||||
}
|
||||
if math.Abs(a.Score-b.Score) > 0.000001 {
|
||||
return a.Score > b.Score
|
||||
}
|
||||
if a.WordCount != b.WordCount {
|
||||
return a.WordCount > b.WordCount
|
||||
}
|
||||
if a.SectionCount != b.SectionCount {
|
||||
return a.SectionCount > b.SectionCount
|
||||
}
|
||||
return a.SourceUtilization > b.SourceUtilization
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/articlequality"
|
||||
)
|
||||
|
||||
func TestArticleCPUQualityBetterPrefersFewerHardFailures(t *testing.T) {
|
||||
better := articlequality.Result{Score: .61, WordCount: 620, HardFailures: []string{"low_information_density"}}
|
||||
worse := articlequality.Result{Score: .70, WordCount: 700, HardFailures: []string{"article_too_short", "low_information_density"}}
|
||||
if !articleCPUQualityBetter(better, worse) {
|
||||
t.Fatal("fewer hard failures must outrank a numerically higher score")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArticleCPUQualityBetterPrefersHigherScoreWhenFailuresEqual(t *testing.T) {
|
||||
better := articlequality.Result{Score: .72, WordCount: 640, HardFailures: []string{"article_too_short"}}
|
||||
worse := articlequality.Result{Score: .66, WordCount: 700, HardFailures: []string{"article_too_short"}}
|
||||
if !articleCPUQualityBetter(better, worse) {
|
||||
t.Fatal("higher score should win when pass/fail and hard-failure count are equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArticleCPUQualityBetterAlwaysPrefersPassingDraft(t *testing.T) {
|
||||
better := articlequality.Result{Passed: true, Score: .60, WordCount: 600}
|
||||
worse := articlequality.Result{Passed: false, Score: .90, WordCount: 900, HardFailures: []string{"high_redundancy"}}
|
||||
if !articleCPUQualityBetter(better, worse) {
|
||||
t.Fatal("a passing draft must always outrank a failing draft")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
)
|
||||
|
||||
type articleTypeReconsideration struct {
|
||||
Action string `json:"action"`
|
||||
ArticleType string `json:"article_type"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
func articleTypeReconsiderationSchema() map[string]any {
|
||||
return map[string]any{"type": "object", "properties": map[string]any{
|
||||
"action": map[string]any{"type": "string", "enum": []string{"reclassify", "skip"}},
|
||||
"article_type": map[string]any{"type": "string", "enum": []string{"reference", "concept", "decision_guide"}},
|
||||
"reason": map[string]any{"type": "string"},
|
||||
}, "required": []string{"action", "article_type", "reason"}}
|
||||
}
|
||||
|
||||
func articleTypeReconsiderationContext(plan model.ArticlePlanDecision, content model.KnowledgeArticleContent, sources []articleSource, evidence []model.ResearchResult) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "BISHERIGER_ARTIKELTYP: %s\n", normalizeArticleType(plan.ArticleType))
|
||||
fmt.Fprintf(&b, "PLANNER_ZIEL: %s\n", plan.ExpectedValue)
|
||||
fmt.Fprintf(&b, "TITEL_ENTWURF: %s\n", content.Title)
|
||||
fmt.Fprintf(&b, "PROBLEM: %s\n", clamp(content.ProblemDescription, 1000))
|
||||
fmt.Fprintf(&b, "SCHRITTE: %d\nVALIDIERUNGEN: %d\n", len(cleanArticleItems(content.SolutionSteps)), len(cleanArticleItems(content.ValidationSteps)))
|
||||
fmt.Fprintf(&b, "TECHNISCHER_HINTERGRUND: %d\nTECHNISCHE_DETAILS: %d\nMAPPINGS: %d\nBEISPIELE: %d\nGRENZEN: %d\n", len(cleanArticleItems(content.TechnicalBackground)), len(cleanArticleItems(content.TechnicalDetails)), len(cleanArticleItems(content.Mappings)), len(cleanArticleItems(content.Examples)), len(cleanArticleItems(content.Limitations)))
|
||||
b.WriteString("\nINTERNE QUELLENTITEL:\n")
|
||||
for _, source := range sources {
|
||||
fmt.Fprintf(&b, "- %s\n", source.Node.Label)
|
||||
}
|
||||
if len(evidence) > 0 {
|
||||
b.WriteString("\nGEPRÜFTE/VERFÜGBARE WEB-EVIDENZ:\n")
|
||||
for _, item := range firstResearchResults(evidence, 8) {
|
||||
fmt.Fprintf(&b, "- %s | quality=%s %.2f | relevance=%.2f\n", item.Title, item.SourceQuality, item.SourceQualityScore, item.Relevance)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func firstResearchResults(values []model.ResearchResult, limit int) []model.ResearchResult {
|
||||
if limit > 0 && len(values) > limit {
|
||||
return values[:limit]
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func (e *Engine) reconsiderOperationalArticleType(ctx context.Context, plan model.ArticlePlanDecision, content model.KnowledgeArticleContent, sources []articleSource, evidence []model.ResearchResult) (articleTypeReconsideration, error) {
|
||||
result := articleTypeReconsideration{Action: "skip", ArticleType: "reference", Reason: "operational evidence is insufficient"}
|
||||
system := `Du korrigierst ausschließlich die Artikeltyp-Routingentscheidung einer Wissensdatenbank. Der bisherige how_to/troubleshooting-Entwurf hat trotz gezielter Evidenzsuche weniger als drei belastbare ausführbare Schritte oder keine belastbare Validierung. Wähle NICHT erneut how_to oder troubleshooting.
|
||||
|
||||
Wähle genau eine Option:
|
||||
- reference: wenn die Evidenz belastbare technische Zuordnungen, mehrere Profile/Techniken/Fehlercodes, Artefakte, Statuswerte oder technische Details trägt.
|
||||
- concept: wenn Mechanismen, Grundlagen, Architektur oder Zusammenhänge der eigentliche Wissenswert sind.
|
||||
- decision_guide: wenn belastbare Auswahl-/Abgrenzungskriterien vorliegen.
|
||||
- skip: wenn die Evidenz auch für keinen dieser nicht-operationalen Typen genügend eigenständigen Wissensnutzen trägt.
|
||||
|
||||
Erfinde keine fehlenden Schritte. Das Ziel ist keine Umgehung des Quality-Gates, sondern eine einmalige Korrektur eines offenbar falsch gewählten operationalen Artikeltyps. Gib ausschließlich JSON nach Schema zurück.`
|
||||
if err := e.Ollama.ChatJSON(ctx, system, articleTypeReconsiderationContext(plan, content, sources, evidence), articleTypeReconsiderationSchema(), &result); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Action = strings.ToLower(strings.TrimSpace(result.Action))
|
||||
result.ArticleType = normalizeArticleType(result.ArticleType)
|
||||
if result.Action != "reclassify" {
|
||||
result.Action = "skip"
|
||||
return result, nil
|
||||
}
|
||||
if result.ArticleType != "reference" && result.ArticleType != "concept" && result.ArticleType != "decision_guide" {
|
||||
result.Action = "skip"
|
||||
result.ArticleType = "reference"
|
||||
result.Reason = nonempty(result.Reason, "reclassification did not produce a supported non-operational article type")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/graph"
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
)
|
||||
|
||||
func TestAutonomousResearchSemanticDuplicateSurvivesChangingSeedMembership(t *testing.T) {
|
||||
a := model.ResearchTask{Topic: "T1133 / T1078 / T1588.001", SeedNodeIDs: []string{"a", "b", "c"}, Questions: []string{"Wie hängen T1133 T1078 und T1588.001 zusammen?"}}
|
||||
b := model.ResearchTask{Topic: "T1133 / T1078 / T1588.001", SeedNodeIDs: []string{"a", "d"}, Questions: []string{"Welche Abwehrmaßnahmen betreffen T1133, T1078 und T1588.001?"}}
|
||||
if !autonomousResearchSemanticDuplicate(a, b) {
|
||||
t.Fatalf("same ATT&CK opportunity with changed seeds should dedupe, similarity=%.3f", autonomousResearchTaskSimilarity(a, b))
|
||||
}
|
||||
c := model.ResearchTask{Topic: "Windows Update CBS 0x80D02002", SeedNodeIDs: []string{"x"}}
|
||||
if autonomousResearchSemanticDuplicate(a, c) {
|
||||
t.Fatalf("unrelated opportunities should not dedupe, similarity=%.3f", autonomousResearchTaskSimilarity(a, c))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeQueuedResearchTaskAndPersistentTotals(t *testing.T) {
|
||||
store, err := graph.Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
ctx := context.Background()
|
||||
queued, created, err := store.EnqueueResearchTask(ctx, model.ResearchTask{DedupeKey: "a", Topic: "ATT&CK cluster", RequestedBy: "autonomous-scanner", Priority: .7, SeedNodeIDs: []string{"a"}, Questions: []string{"q1"}}, time.Hour)
|
||||
if err != nil || !created {
|
||||
t.Fatalf("enqueue failed created=%v err=%v", created, err)
|
||||
}
|
||||
merged, changed, err := store.MergeQueuedResearchTask(ctx, queued.ID, model.ResearchTask{Topic: "ATT&CK cluster", Priority: .9, SeedNodeIDs: []string{"b"}, Questions: []string{"q2"}})
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("merge failed changed=%v err=%v", changed, err)
|
||||
}
|
||||
if merged.Priority != .9 || len(merged.SeedNodeIDs) != 2 || len(merged.Questions) != 2 {
|
||||
t.Fatalf("unexpected merged task: %+v", merged)
|
||||
}
|
||||
leased, ok, err := store.LeaseNextResearchTask(ctx, 0, time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("lease failed ok=%v err=%v", ok, err)
|
||||
}
|
||||
if err := store.MarkResearchTaskRunning(ctx, leased.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
leased.EvidenceCount = 3
|
||||
leased.ArticleCreated = true
|
||||
if err := store.CompleteResearchTask(ctx, leased); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
totals, err := store.ResearchTaskPersistentTotals(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if totals["completed"] != 1 || totals["evidence"] != 3 || totals["articles"] != 1 {
|
||||
t.Fatalf("unexpected persistent totals: %+v", totals)
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/graph"
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
)
|
||||
|
||||
type autonomousCandidate struct {
|
||||
@@ -48,6 +47,9 @@ func (e *Engine) startAutonomousResearch(ctx context.Context) {
|
||||
if _, err := e.Graph.ResetExpiredResearchTaskLeases(ctx); err != nil {
|
||||
slog.Warn("reset expired autonomous research leases failed", "error", err)
|
||||
}
|
||||
if _, err := e.consolidateAutonomousResearchQueue(ctx); err != nil {
|
||||
slog.Warn("consolidate autonomous research queue failed", "error", err)
|
||||
}
|
||||
go e.autonomousResearchScanner(ctx)
|
||||
go e.autonomousResearchWorker(ctx)
|
||||
// Existing queued work is resumed after every restart. The scheduler scan is
|
||||
@@ -55,6 +57,9 @@ func (e *Engine) startAutonomousResearch(ctx context.Context) {
|
||||
e.signalAutonomousResearch()
|
||||
go func() {
|
||||
delay := 45 * time.Second
|
||||
if e.SpeedModeEnabled() {
|
||||
delay = 0
|
||||
}
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
@@ -128,7 +133,7 @@ func (e *Engine) autonomousResearchScanner(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (e *Engine) scanAutonomousResearchOpportunities(ctx context.Context, trigger string) error {
|
||||
ctx = ollama.WithLowPriority(ctx)
|
||||
ctx = e.backgroundOllamaContext(ctx)
|
||||
settings := e.RuntimeSettings()
|
||||
candidates := buildAutonomousCandidates(e.Graph.Snapshot(), e.effectiveThinkingFilter(), e.Cfg.AutonomousResearchOpportunityLimit)
|
||||
e.beginAutonomousOpportunityScan(trigger, len(candidates))
|
||||
@@ -137,13 +142,38 @@ func (e *Engine) scanAutonomousResearchOpportunities(ctx context.Context, trigge
|
||||
e.Broker.Publish(model.Activity{Type: "autonomous.research.scan.completed", Source: "brain", Phase: "autonomous-research", Message: "Der Graph enthält aktuell keine ausreichend starke autonome Recherchechance", Strength: .24, Metadata: map[string]any{"trigger": trigger, "candidate_count": 0, "created": 0, "decisions": []any{}, "rejection_counts": map[string]int{}}})
|
||||
return nil
|
||||
}
|
||||
if _, err := e.consolidateAutonomousResearchQueue(ctx); err != nil {
|
||||
slog.Warn("autonomous queue consolidation before scan failed", "error", err)
|
||||
}
|
||||
completedToday, activeQueued, remainingBudget, budgetErr := e.autonomousResearchDailyBudget(ctx, settings)
|
||||
if budgetErr != nil {
|
||||
return budgetErr
|
||||
}
|
||||
if remainingBudget <= 0 {
|
||||
decisions := make([]autonomousOpportunityDecision, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
decision := newAutonomousOpportunityDecision(candidate)
|
||||
decision.RejectionReason = "daily_budget_exhausted"
|
||||
decision.RecommendedAction = "wait_for_next_utc_day_or_finish_queue"
|
||||
decisions = append(decisions, decision)
|
||||
}
|
||||
e.finishAutonomousOpportunityScan(trigger, 0, decisions)
|
||||
e.Broker.Publish(model.Activity{Type: "autonomous.research.scan.completed", Source: "brain", Phase: "autonomous-research", Message: "Autonome Graphanalyse abgeschlossen · Tagesbudget beziehungsweise bereits eingeplante Arbeit schöpft die aktuelle Kapazität aus", Strength: .34, Metadata: map[string]any{"trigger": trigger, "candidate_count": len(candidates), "created": 0, "completed_today": completedToday, "active_research_tasks": activeQueued, "daily_limit": settings.AutonomousResearchMaxTasksPerDay, "remaining_budget_slots": 0, "decisions": decisions, "rejection_counts": autonomousDecisionRejectionCounts(decisions)}})
|
||||
return nil
|
||||
}
|
||||
limit := settings.AutonomousResearchTasksPerCycle
|
||||
if settings.SpeedMode {
|
||||
limit = remainingBudget
|
||||
}
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
if limit > remainingBudget {
|
||||
limit = remainingBudget
|
||||
}
|
||||
created := 0
|
||||
decisions := make([]autonomousOpportunityDecision, 0, len(candidates))
|
||||
e.Broker.Publish(model.Activity{Type: "autonomous.research.scan.started", Source: "brain", Phase: "autonomous-research", Message: fmt.Sprintf("%d Graphsignale werden als mögliche Wissenslücken bewertet", len(candidates)), Strength: .66, Metadata: map[string]any{"trigger": trigger, "candidate_count": len(candidates), "task_limit": limit}})
|
||||
e.Broker.Publish(model.Activity{Type: "autonomous.research.scan.started", Source: "brain", Phase: "autonomous-research", Message: fmt.Sprintf("%d Graphsignale werden als mögliche Wissenslücken bewertet", len(candidates)), Strength: .66, Metadata: map[string]any{"trigger": trigger, "candidate_count": len(candidates), "task_limit": limit, "completed_today": completedToday, "active_research_tasks": activeQueued, "daily_limit": settings.AutonomousResearchMaxTasksPerDay, "remaining_budget_slots": remainingBudget}})
|
||||
for _, candidate := range candidates {
|
||||
decision := newAutonomousOpportunityDecision(candidate)
|
||||
if created >= limit {
|
||||
@@ -206,6 +236,30 @@ func (e *Engine) scanAutonomousResearchOpportunities(ctx context.Context, trigge
|
||||
"signals": candidate.Signals,
|
||||
},
|
||||
}
|
||||
if duplicate, found, duplicateErr := e.findAutonomousResearchSemanticDuplicate(ctx, task); duplicateErr != nil {
|
||||
decision.RejectionReason = "semantic_dedupe_error"
|
||||
decision.KnowledgeGap = duplicateErr.Error()
|
||||
decision.RecommendedAction = "retry_next_scan"
|
||||
decisions = append(decisions, decision)
|
||||
continue
|
||||
} else if found {
|
||||
merged, changed, mergeErr := e.Graph.MergeQueuedResearchTask(ctx, duplicate.ID, task)
|
||||
if mergeErr != nil {
|
||||
decision.RejectionReason = "semantic_dedupe_error"
|
||||
decision.KnowledgeGap = mergeErr.Error()
|
||||
decision.RecommendedAction = "retry_next_scan"
|
||||
decisions = append(decisions, decision)
|
||||
continue
|
||||
}
|
||||
decision.RejectionReason = "semantic_queue_duplicate"
|
||||
decision.RecommendedAction = "merged_into_existing_task"
|
||||
decision.SourceNodeIDs = append([]string(nil), merged.SeedNodeIDs...)
|
||||
decisions = append(decisions, decision)
|
||||
if changed {
|
||||
e.Broker.Publish(model.Activity{Type: "autonomous.research.task.merged", Source: "brain", Phase: "autonomous-research-queue", NodeIDs: merged.SeedNodeIDs, Message: fmt.Sprintf("Semantisch gleiche Wissenslücke wurde in vorhandene Rechercheaufgabe zusammengeführt · %s", merged.Topic), Strength: .54, Metadata: map[string]any{"task_id": merged.ID, "incoming_topic": task.Topic, "similarity": autonomousResearchTaskSimilarity(duplicate, task), "algorithm": "topic-seed-question-jaccard-v1"}})
|
||||
}
|
||||
continue
|
||||
}
|
||||
queued, wasCreated, err := e.Graph.EnqueueResearchTask(ctx, task, e.Cfg.AutonomousResearchCooldown)
|
||||
if err != nil {
|
||||
decision.RejectionReason = "enqueue_error"
|
||||
@@ -444,7 +498,7 @@ func (e *Engine) autonomousMayUseOllama(_ bool) bool {
|
||||
if !autonomousResearchRuntimeAllowed(settings, e.ResearchEnabledForRuntime()) {
|
||||
return false
|
||||
}
|
||||
if settings.AutonomousResearchIdleOnly {
|
||||
if settings.AutonomousResearchIdleOnly && !settings.SpeedMode {
|
||||
if e.interactiveInflight.Load() > 0 {
|
||||
return false
|
||||
}
|
||||
@@ -461,7 +515,7 @@ func (e *Engine) autonomousMayUseOllama(_ bool) bool {
|
||||
}
|
||||
}
|
||||
for _, node := range e.Ollama.NodeStatuses() {
|
||||
if node.Healthy && node.Compatible && node.Inflight < e.Cfg.OllamaNodeMaxInflight && time.Now().After(node.CooldownUntil) {
|
||||
if node.Healthy && node.Compatible && node.Inflight < e.Ollama.NodeMaxInflight() && time.Now().After(node.CooldownUntil) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -471,7 +525,7 @@ func (e *Engine) autonomousMayUseOllama(_ bool) bool {
|
||||
func (e *Engine) runAutonomousResearchTask(parent context.Context, task model.ResearchTask) {
|
||||
ctx, cancel := context.WithTimeout(parent, maxDuration(e.Cfg.OllamaRequestTimeout*3, 20*time.Minute))
|
||||
defer cancel()
|
||||
ctx = ollama.WithLowPriority(ctx)
|
||||
ctx = e.backgroundOllamaContext(ctx)
|
||||
if err := e.Graph.MarkResearchTaskRunning(ctx, task.ID); err != nil {
|
||||
slog.Warn("mark autonomous research task running failed", "task_id", task.ID, "error", err)
|
||||
return
|
||||
@@ -534,6 +588,9 @@ func (e *Engine) runAutonomousResearchTask(parent context.Context, task model.Re
|
||||
message = fmt.Sprintf("Autonome Recherche hat einen KB-Entwurf erstellt · %s", outcome.ArticleTitle)
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "autonomous.research.task.completed", Source: "brain", Phase: "autonomous-research", NodeIDs: task.SeedNodeIDs, Message: message, Strength: 1, Metadata: map[string]any{"task_id": task.ID, "outcome": outcome.Outcome, "evidence_count": outcome.EvidenceCount, "queries_executed": outcome.QueriesExecuted, "pages_fetched": outcome.PagesFetched, "article_created": outcome.ArticleCreated, "article_title": outcome.ArticleTitle, "article_path": outcome.ArticlePath}})
|
||||
if e.SpeedModeEnabled() {
|
||||
e.RequestAutonomousResearchScan("speed-drain")
|
||||
}
|
||||
}
|
||||
|
||||
type autonomousTaskOutcome struct {
|
||||
@@ -638,6 +695,16 @@ func (e *Engine) executeAutonomousResearchTask(ctx context.Context, task model.R
|
||||
if err != nil {
|
||||
return outcome, err
|
||||
}
|
||||
// Autonomous synthesis bypasses EnrichOne's process counters. Keep the
|
||||
// runtime dashboard truthful for autonomous article outcomes as well.
|
||||
e.stateMu.Lock()
|
||||
if article.Created {
|
||||
e.articlesCreated++
|
||||
}
|
||||
if article.Skipped {
|
||||
e.articlesSkipped++
|
||||
}
|
||||
e.stateMu.Unlock()
|
||||
outcome.ArticleReason = article.Reason
|
||||
if article.Created {
|
||||
outcome.Outcome = "article_created"
|
||||
@@ -1329,6 +1396,186 @@ func autonomousNodeSetJaccard(a, b []string) float64 {
|
||||
return float64(intersection) / float64(union)
|
||||
}
|
||||
|
||||
func autonomousResearchActiveTasks(ctx context.Context, store *graph.Store) ([]model.ResearchTask, error) {
|
||||
tasks, err := store.ListResearchTasks(ctx, 500, "queued", "deferred", "reserved", "running")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]model.ResearchTask, 0, len(tasks))
|
||||
for _, task := range tasks {
|
||||
if task.RequestedBy == "autonomous-scanner" {
|
||||
out = append(out, task)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func stringSetJaccard(a, b []string) float64 {
|
||||
left := map[string]bool{}
|
||||
right := map[string]bool{}
|
||||
for _, value := range a {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
if value != "" {
|
||||
left[value] = true
|
||||
}
|
||||
}
|
||||
for _, value := range b {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
if value != "" {
|
||||
right[value] = true
|
||||
}
|
||||
}
|
||||
if len(left) == 0 && len(right) == 0 {
|
||||
return 0
|
||||
}
|
||||
intersection := 0
|
||||
union := map[string]bool{}
|
||||
for value := range left {
|
||||
union[value] = true
|
||||
if right[value] {
|
||||
intersection++
|
||||
}
|
||||
}
|
||||
for value := range right {
|
||||
union[value] = true
|
||||
}
|
||||
return float64(intersection) / float64(len(union))
|
||||
}
|
||||
|
||||
func researchTermSlice(value string) []string {
|
||||
terms := researchTerms(value)
|
||||
out := make([]string, 0, len(terms))
|
||||
for term := range terms {
|
||||
out = append(out, term)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func autonomousResearchTaskSimilarity(a, b model.ResearchTask) float64 {
|
||||
if strings.TrimSpace(a.DedupeKey) != "" && a.DedupeKey == b.DedupeKey {
|
||||
return 1
|
||||
}
|
||||
topic := stringSetJaccard(researchTermSlice(a.Topic), researchTermSlice(b.Topic))
|
||||
seed := stringSetJaccard(a.SeedNodeIDs, b.SeedNodeIDs)
|
||||
questions := stringSetJaccard(researchTermSlice(strings.Join(a.Questions, " ")), researchTermSlice(strings.Join(b.Questions, " ")))
|
||||
// Topic identity dominates, but changing orphan-cluster membership must not
|
||||
// create a fresh queue entry for the same semantic gap.
|
||||
return clamp01(topic*.62 + seed*.23 + questions*.15)
|
||||
}
|
||||
|
||||
func autonomousResearchSemanticDuplicate(a, b model.ResearchTask) bool {
|
||||
if strings.TrimSpace(a.DedupeKey) != "" && a.DedupeKey == b.DedupeKey {
|
||||
return true
|
||||
}
|
||||
topic := stringSetJaccard(researchTermSlice(a.Topic), researchTermSlice(b.Topic))
|
||||
seed := stringSetJaccard(a.SeedNodeIDs, b.SeedNodeIDs)
|
||||
questions := stringSetJaccard(researchTermSlice(strings.Join(a.Questions, " ")), researchTermSlice(strings.Join(b.Questions, " ")))
|
||||
if topic >= .72 {
|
||||
return true
|
||||
}
|
||||
if topic >= .42 && seed >= .45 {
|
||||
return true
|
||||
}
|
||||
return topic >= .50 && questions >= .55
|
||||
}
|
||||
|
||||
func (e *Engine) findAutonomousResearchSemanticDuplicate(ctx context.Context, incoming model.ResearchTask) (model.ResearchTask, bool, error) {
|
||||
active, err := autonomousResearchActiveTasks(ctx, e.Graph)
|
||||
if err != nil {
|
||||
return model.ResearchTask{}, false, err
|
||||
}
|
||||
bestScore := 0.0
|
||||
var best model.ResearchTask
|
||||
for _, task := range active {
|
||||
if !autonomousResearchSemanticDuplicate(task, incoming) {
|
||||
continue
|
||||
}
|
||||
score := autonomousResearchTaskSimilarity(task, incoming)
|
||||
if score > bestScore {
|
||||
bestScore = score
|
||||
best = task
|
||||
}
|
||||
}
|
||||
return best, best.ID != "", nil
|
||||
}
|
||||
|
||||
// consolidateAutonomousResearchQueue collapses semantically equivalent queued
|
||||
// work after upgrades/restarts and before new opportunity scans. Running work is
|
||||
// never cancelled. The highest-priority/oldest queued task becomes the keeper.
|
||||
func (e *Engine) consolidateAutonomousResearchQueue(ctx context.Context) (int, error) {
|
||||
tasks, err := e.Graph.ListResearchTasks(ctx, 500, "queued", "deferred")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
filtered := make([]model.ResearchTask, 0, len(tasks))
|
||||
for _, task := range tasks {
|
||||
if task.RequestedBy == "autonomous-scanner" {
|
||||
filtered = append(filtered, task)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(filtered, func(i, j int) bool {
|
||||
if filtered[i].Priority != filtered[j].Priority {
|
||||
return filtered[i].Priority > filtered[j].Priority
|
||||
}
|
||||
return filtered[i].CreatedAt.Before(filtered[j].CreatedAt)
|
||||
})
|
||||
merged := 0
|
||||
cancelled := map[string]bool{}
|
||||
for i := 0; i < len(filtered); i++ {
|
||||
keeper := filtered[i]
|
||||
if cancelled[keeper.ID] {
|
||||
continue
|
||||
}
|
||||
for j := i + 1; j < len(filtered); j++ {
|
||||
candidate := filtered[j]
|
||||
if cancelled[candidate.ID] || !autonomousResearchSemanticDuplicate(keeper, candidate) {
|
||||
continue
|
||||
}
|
||||
updated, changed, mergeErr := e.Graph.MergeQueuedResearchTask(ctx, keeper.ID, candidate)
|
||||
if mergeErr != nil {
|
||||
return merged, mergeErr
|
||||
}
|
||||
if changed {
|
||||
keeper = updated
|
||||
}
|
||||
ok, cancelErr := e.Graph.CancelResearchTask(ctx, candidate.ID)
|
||||
if cancelErr != nil {
|
||||
return merged, cancelErr
|
||||
}
|
||||
if ok {
|
||||
cancelled[candidate.ID] = true
|
||||
merged++
|
||||
}
|
||||
}
|
||||
}
|
||||
if merged > 0 && e.Broker != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "autonomous.research.queue.consolidated", Source: "brain", Phase: "autonomous-research-queue", Message: fmt.Sprintf("%d semantisch redundante Rechercheaufgaben wurden in vorhandene Queue-Einträge zusammengeführt", merged), Strength: .52, Metadata: map[string]any{"merged_tasks": merged, "algorithm": "topic-seed-question-jaccard-v1"}})
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
func (e *Engine) autonomousResearchDailyBudget(ctx context.Context, settings RuntimeSettings) (completed, active, remaining int, err error) {
|
||||
startOfDay := time.Now().UTC().Truncate(24 * time.Hour)
|
||||
completed, err = e.Graph.CountResearchTasksCompletedSince(ctx, startOfDay)
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
// The same worker leases API/manual and autonomous-scanner tasks. Reserve
|
||||
// daily capacity against the whole active queue so a fast opportunity scan
|
||||
// cannot create a backlog that the worker cannot consume inside the budget.
|
||||
activeTasks, err := e.Graph.ListResearchTasks(ctx, 500, "queued", "deferred", "reserved", "running")
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
active = len(activeTasks)
|
||||
remaining = settings.AutonomousResearchMaxTasksPerDay - completed - active
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
return completed, active, remaining, nil
|
||||
}
|
||||
|
||||
func autonomousDedupeKey(topic string, seedIDs []string) string {
|
||||
ids := append([]string(nil), seedIDs...)
|
||||
sort.Strings(ids)
|
||||
@@ -1399,21 +1646,29 @@ func (e *Engine) AutonomousResearchStatus(ctx context.Context) map[string]any {
|
||||
if err != nil {
|
||||
counts = map[string]int{}
|
||||
}
|
||||
persistentTotals, totalsErr := e.Graph.ResearchTaskPersistentTotals(ctx)
|
||||
if totalsErr != nil {
|
||||
persistentTotals = map[string]int{}
|
||||
}
|
||||
e.stateMu.RLock()
|
||||
lastScanDecisions := append([]autonomousOpportunityDecision(nil), e.autonomousLastScanDecisions...)
|
||||
settings := e.RuntimeSettings()
|
||||
status := map[string]any{
|
||||
"enabled": e.RuntimeSettings().AutonomousResearchEnabled,
|
||||
"idle_only": e.RuntimeSettings().AutonomousResearchIdleOnly,
|
||||
"running": e.autonomousRunning,
|
||||
"task_id": e.autonomousTaskID,
|
||||
"task_topic": e.autonomousTaskTopic,
|
||||
"last_started": e.autonomousLastStarted,
|
||||
"last_completed": e.autonomousLastCompleted,
|
||||
"last_error": e.autonomousLastError,
|
||||
"completed_total": e.autonomousCompleted,
|
||||
"failed_total": e.autonomousFailed,
|
||||
"evidence_total": e.autonomousEvidence,
|
||||
"articles_total": e.autonomousArticles,
|
||||
"enabled": settings.AutonomousResearchEnabled,
|
||||
"idle_only": settings.AutonomousResearchIdleOnly,
|
||||
"idle_only_effective": settings.AutonomousResearchIdleOnly && !settings.SpeedMode,
|
||||
"speed_mode": settings.SpeedMode,
|
||||
"running": e.autonomousRunning,
|
||||
"task_id": e.autonomousTaskID,
|
||||
"task_topic": e.autonomousTaskTopic,
|
||||
"last_started": e.autonomousLastStarted,
|
||||
"last_completed": e.autonomousLastCompleted,
|
||||
"last_error": e.autonomousLastError,
|
||||
"completed_total": persistentTotals["completed"],
|
||||
"failed_total": persistentTotals["failed"],
|
||||
"evidence_total": persistentTotals["evidence"],
|
||||
"articles_total": persistentTotals["articles"],
|
||||
"process_totals": map[string]any{"completed": e.autonomousCompleted, "failed": e.autonomousFailed, "evidence": e.autonomousEvidence, "articles": e.autonomousArticles},
|
||||
"last_scan": map[string]any{
|
||||
"started": e.autonomousLastScanStarted,
|
||||
"completed": e.autonomousLastScanCompleted,
|
||||
|
||||
+77
-99
@@ -75,6 +75,7 @@ type Engine struct {
|
||||
lastScan time.Time
|
||||
lastVectorGraph time.Time
|
||||
lastVectorLayout time.Time
|
||||
vectorMaintenanceStartedAt time.Time
|
||||
bootstrapComplete bool
|
||||
bootstrapCompletedAt time.Time
|
||||
bootstrapError string
|
||||
@@ -108,6 +109,7 @@ type Engine struct {
|
||||
interactiveInflight atomic.Int64
|
||||
autonomousWake chan struct{}
|
||||
autonomousScanRequests chan string
|
||||
sourceInboxWake chan struct{}
|
||||
autonomousRunning bool
|
||||
autonomousTaskID string
|
||||
autonomousTaskTopic string
|
||||
@@ -312,13 +314,9 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
|
||||
sharedWork := workqueue.New(cfg.ResearchOllamaMaxInflight, cfg.ResearchOllamaQueueSize)
|
||||
pool.SetSharedLimiter(sharedWork)
|
||||
persistence := persist.New(g, b, cfg.PersistInterval)
|
||||
e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: pool, Persistence: persistence, Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs, FullVerifyInterval: cfg.KnowledgeFullVerifyInterval}, bootstrapReady: make(chan struct{}), enrichRequests: make(chan string, 1), autonomousWake: make(chan struct{}, 1), autonomousScanRequests: make(chan string, 1), runtimePath: filepath.Join(cfg.DataDir, "runtime-settings.json"), researchEvidenceCache: map[string]researchEvidenceRecord{}, sharedWork: sharedWork, researchDedupe: map[string]*researchDedupeEntry{}}
|
||||
// Slow relaxation is maintenance, not a startup animation. Delaying its first
|
||||
// cycle prevents frequent process restarts from moving the cloud repeatedly.
|
||||
if cfg.VectorGraphRelaxLayout && !cfg.VectorGraphLayout {
|
||||
e.lastVectorLayout = time.Now().UTC()
|
||||
}
|
||||
e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: pool, Persistence: persistence, vectorMaintenanceStartedAt: time.Now().UTC(), Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs, FullVerifyInterval: cfg.KnowledgeFullVerifyInterval}, bootstrapReady: make(chan struct{}), enrichRequests: make(chan string, 1), autonomousWake: make(chan struct{}, 1), autonomousScanRequests: make(chan string, 1), sourceInboxWake: make(chan struct{}, 1), runtimePath: filepath.Join(cfg.DataDir, "runtime-settings.json"), researchEvidenceCache: map[string]researchEvidenceRecord{}, sharedWork: sharedWork, researchDedupe: map[string]*researchDedupeEntry{}}
|
||||
e.loadRuntimeSettings()
|
||||
e.applyRuntimePerformance(e.RuntimeSettings())
|
||||
if cfg.SearXNGURL != "" {
|
||||
e.Research = research.New(cfg.SearXNGURL)
|
||||
}
|
||||
@@ -327,7 +325,7 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
|
||||
e.GLPIKB = ingest.NewGLPIKBSyncer(ingest.GLPIKBConfig{Enabled: true, Path: cfg.GLPIKBPath, Filter: cfg.GLPIKBFilter, Limit: cfg.GLPIKBLimit, SyncInterval: cfg.GLPIKBSyncInterval, Source: cfg.GLPIKBSource, CachePath: filepath.Join(cfg.DataDir, "glpi-kb-cache.json"), ShouldSync: e.LearningEnabled}, client, g, b, persistence)
|
||||
}
|
||||
if b != nil {
|
||||
b.Publish(model.Activity{Type: "system.started", Source: "brain", Phase: "startup", Message: "Neural Brain wurde gestartet; das Analyseprotokoll zeichnet Läufe und Graphänderungen auf", Strength: .3, Metadata: map[string]any{"chat_model": cfg.ChatModel, "embedding_model": cfg.EmbeddingModel, "article_language": cfg.ArticleLanguage, "article_synthesis_model": cfg.ArticleSynthesisModel, "article_review_model": cfg.ArticleReviewModel, "article_review_repair_rounds": cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v3-quality-v2", "article_research_strategy": cfg.ArticleResearchStrategy, "cluster_article_batching": cfg.ClusterArticleBatching, "vector_graph_enabled": cfg.VectorGraphEnabled, "vector_graph_layout": cfg.VectorGraphLayout, "vector_graph_reevaluate_interval": cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_interval": cfg.VectorGraphLayoutRelaxInterval.String(), "article_cpu_quality_enabled": cfg.ArticleCPUQualityEnabled, "article_cpu_quality_agent_offload": cfg.ArticleCPUQualityAgentOffload, "research_ollama_max_inflight": cfg.ResearchOllamaMaxInflight, "research_ollama_queue_size": cfg.ResearchOllamaQueueSize, "graph_version": g.Version()}})
|
||||
b.Publish(model.Activity{Type: "system.started", Source: "brain", Phase: "startup", Message: "Neural Brain wurde gestartet; das Analyseprotokoll zeichnet Läufe und Graphänderungen auf", Strength: .3, Metadata: map[string]any{"chat_model": cfg.ChatModel, "embedding_model": cfg.EmbeddingModel, "article_language": cfg.ArticleLanguage, "article_synthesis_model": cfg.ArticleSynthesisModel, "article_review_model": cfg.ArticleReviewModel, "article_review_repair_rounds": cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v5-quality-gate-v12", "article_research_strategy": cfg.ArticleResearchStrategy, "cluster_article_batching": cfg.ClusterArticleBatching, "vector_graph_enabled": cfg.VectorGraphEnabled, "vector_graph_layout": cfg.VectorGraphLayout, "vector_graph_layout_mode": e.effectiveVectorLayoutMode(), "vector_graph_reevaluate_interval": cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_effective": cfg.VectorGraphRelaxLayout && !cfg.VectorGraphLayout, "vector_graph_layout_relax_interval": cfg.VectorGraphLayoutRelaxInterval.String(), "article_cpu_quality_enabled": cfg.ArticleCPUQualityEnabled, "article_cpu_quality_agent_offload": cfg.ArticleCPUQualityAgentOffload, "research_ollama_max_inflight": cfg.ResearchOllamaMaxInflight, "research_ollama_queue_size": cfg.ResearchOllamaQueueSize, "speed_mode": e.RuntimeSettings().SpeedMode, "speed_cpu_tasks": e.RuntimeSettings().SpeedCPUWorkers, "speed_gpu_tasks": e.RuntimeSettings().SpeedGPUInflight, "graph_version": g.Version()}})
|
||||
}
|
||||
return e
|
||||
}
|
||||
@@ -378,6 +376,12 @@ func (e *Engine) Start(ctx context.Context) {
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
if e.waitBootstrap(ctx) {
|
||||
e.vectorMaintenanceLoop(ctx)
|
||||
}
|
||||
}()
|
||||
|
||||
go e.enrichmentWorker(ctx)
|
||||
if e.Cfg.AutoEnrich {
|
||||
go e.enrichmentScheduler(ctx)
|
||||
@@ -461,6 +465,9 @@ func (e *Engine) waitBootstrap(ctx context.Context) bool {
|
||||
|
||||
func (e *Engine) enrichmentScheduler(ctx context.Context) {
|
||||
firstDelay := 12 * time.Second
|
||||
if e.SpeedModeEnabled() {
|
||||
firstDelay = 0
|
||||
}
|
||||
e.setNextEnrich(time.Now().Add(firstDelay))
|
||||
timer := time.NewTimer(firstDelay)
|
||||
defer timer.Stop()
|
||||
@@ -488,7 +495,10 @@ func (e *Engine) enrichmentWorker(ctx context.Context) {
|
||||
if !e.waitBootstrap(ctx) {
|
||||
return
|
||||
}
|
||||
e.runEnrichmentCycle(ctx, trigger)
|
||||
more := e.runEnrichmentCycle(ctx, trigger)
|
||||
if more && e.SpeedModeEnabled() && e.ThinkingEnabled() {
|
||||
e.RequestEnrich("speed-drain")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -528,7 +538,7 @@ func (e *Engine) RequestEnrich(trigger string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
|
||||
func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) bool {
|
||||
started := time.Now()
|
||||
e.stateMu.Lock()
|
||||
e.enrichRunning = true
|
||||
@@ -588,13 +598,13 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
|
||||
if outcome.Rejected {
|
||||
rejected++
|
||||
}
|
||||
if step+1 < e.Cfg.EnrichBatchSize && e.Cfg.EnrichStepDelay > 0 {
|
||||
if delay := e.effectiveEnrichStepDelay(); step+1 < e.Cfg.EnrichBatchSize && delay > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
cycleErr = ctx.Err()
|
||||
result = "cancelled"
|
||||
step = e.Cfg.EnrichBatchSize
|
||||
case <-time.After(e.Cfg.EnrichStepDelay):
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -624,13 +634,14 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
|
||||
if cycleErr != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "think.cycle.failed", Source: "brain", Phase: "autonomous", Message: "AI-THINK-Zyklus wurde mit Fehler beendet", Strength: .45, Metadata: metadata})
|
||||
slog.Warn("enrichment cycle failed", "trigger", trigger, "error", cycleErr)
|
||||
return
|
||||
return false
|
||||
}
|
||||
message := fmt.Sprintf("AI-THINK-Zyklus abgeschlossen · %d Relationen · %d Artikel · %d verworfen", relationsCreated, articlesCreated, rejected)
|
||||
if result == "no_candidate" {
|
||||
message = "AI-THINK hat im aktuell geprüften Graphbereich keinen Kandidaten oberhalb des Schwellwerts gefunden"
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "think.cycle.completed", Source: "brain", Phase: "autonomous", Message: message, Strength: .58, Metadata: metadata})
|
||||
return result == "completed" && checked > 0
|
||||
}
|
||||
|
||||
func (e *Engine) setNextEnrich(t time.Time) {
|
||||
@@ -770,11 +781,11 @@ func (e *Engine) Scan(ctx context.Context) error {
|
||||
// locally on the Brain CPU or be claimed by a compute-capable Source Agent.
|
||||
// Only 256-dimensional deterministic fallback vectors are excluded because
|
||||
// mixing them with the configured embedding space would make thresholds lie.
|
||||
e.stateMu.RLock()
|
||||
lastVectorGraph := e.lastVectorGraph
|
||||
e.stateMu.RUnlock()
|
||||
periodicVectorRefresh := lastVectorGraph.IsZero() || time.Since(lastVectorGraph) >= e.Cfg.VectorGraphReevaluateInterval
|
||||
vectorLayerNeeded := !scanResult.FastPath || needsEmbeddingWork || !e.Graph.HasEdgesByOrigin(graph.VectorMathOrigin) || periodicVectorRefresh
|
||||
// Scheduled reevaluation/layout maintenance has its own loop and therefore
|
||||
// continues even while Learning is paused. Learning scans rebuild the vector
|
||||
// layer only when their own ingest/embedding work changed the knowledge space
|
||||
// or when no vector layer exists yet.
|
||||
vectorLayerNeeded := !scanResult.FastPath || needsEmbeddingWork || !e.Graph.HasEdgesByOrigin(graph.VectorMathOrigin)
|
||||
vectorReady := e.Graph.CountVectorsByDimension(256) == 0
|
||||
if e.Cfg.VectorGraphEnabled && vectorReady && vectorLayerNeeded {
|
||||
startedVectorGraph := time.Now()
|
||||
@@ -794,8 +805,9 @@ func (e *Engine) Scan(ctx context.Context) error {
|
||||
"candidate_pairs": stats.CandidatePairs, "exact_comparisons": stats.ExactComparisons,
|
||||
"orphan_pass_enabled": e.Cfg.VectorGraphOrphanPass, "orphan_focus": stats.OrphanFocus, "orphan_links": stats.OrphanLinks,
|
||||
"orphan_exact_comparisons": stats.OrphanStats.ExactComparisons, "orphan_candidate_pairs": stats.OrphanStats.CandidatePairs,
|
||||
"position_updates": stats.PositionUpdates, "layout_enabled": e.Cfg.VectorGraphLayout, "layout_relaxation": e.Cfg.VectorGraphRelaxLayout, "periodic_refresh": periodicVectorRefresh, "reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(),
|
||||
"position_updates": stats.PositionUpdates, "layout_enabled": e.Cfg.VectorGraphLayout, "layout_relaxation_configured": e.Cfg.VectorGraphRelaxLayout, "layout_mode": execution.LayoutMode, "layout_applied": execution.LayoutDue && stats.PositionUpdates > 0, "periodic_refresh": false, "maintenance_owned": true, "reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(),
|
||||
"agent_offloaded": execution.Offloaded, "agent_id": execution.AgentID, "agent_compute_ms": execution.ComputeMS, "agent_fallback_reason": execution.FallbackReason,
|
||||
"speed_mode": e.SpeedModeEnabled(), "cpu_workers": e.vectorPrimaryConfig(execution.LayoutDue).Workers,
|
||||
"duration_ms": time.Since(startedVectorGraph).Milliseconds(),
|
||||
}, vectorMutations)})
|
||||
}
|
||||
@@ -993,7 +1005,9 @@ func (e *Engine) Query(ctx context.Context, q string) (model.QueryResponse, erro
|
||||
for i, h := range hits {
|
||||
nodeIDs = append(nodeIDs, h.NodeID)
|
||||
e.Broker.Publish(model.Activity{Type: "node.activated", Source: "brain", Phase: "retrieval", Query: q, NodeIDs: []string{h.NodeID}, Message: fmt.Sprintf("Treffer %d · %.0f%% · %s", i+1, h.Score*100, h.Label), Strength: math.Max(.25, h.Score), Metadata: queryMeta(nil)})
|
||||
time.Sleep(55 * time.Millisecond)
|
||||
if !e.SpeedModeEnabled() {
|
||||
time.Sleep(55 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
edgeIDs := e.Graph.ConnectingEdges(nodeIDs)
|
||||
if len(edgeIDs) > 0 {
|
||||
@@ -1156,7 +1170,7 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
|
||||
e.stateMu.Unlock()
|
||||
e.Broker.Publish(model.Activity{Type: "think.started", Source: "brain", Phase: "association", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Verwandtschaft wird geprüft · %.0f%% semantische Nähe", sim*100), Strength: .88, Metadata: map[string]any{"trigger": trigger, "semantic_similarity": sim, "source_label": a.Label, "target_label": b.Label, "model": e.Cfg.ChatModel, "candidate_comparisons": comparisons, "exact_comparisons": comparisons, "coarse_comparisons": coarseComparisons, "indexed_nodes": indexedNodes, "candidate_pool": candidatePool, "processing_mode": processingMode, "candidate_source": candidateSource}})
|
||||
|
||||
system := "Du führst ausschließlich eine Relationserkennung für einen Wissensgraphen durch. Analysiere zwei interne Wissenseinträge, erfinde keine Fakten und entscheide, ob eine belastbare Beziehung besteht. Schreibe keinen Artikel und keine technische Synthese. Wenn externe Fakten zur Relationsentscheidung fehlen, setze needs_research=true. Gib ausschließlich JSON nach Schema zurück."
|
||||
system := "Du führst ausschließlich eine Relationserkennung für einen Wissensgraphen durch. Analysiere zwei interne Wissenseinträge, erfinde keine Fakten und entscheide, ob eine belastbare Beziehung besteht. Schreibe keinen Artikel und keine technische Synthese. Bei sehr hoher semantischer Nähe und same_topic/related_to ist normalerweise keine Webrecherche nötig. Wenn externe Fakten für eine kausale, abhängige, widersprüchliche oder zeitkritische Relationsentscheidung fehlen, setze needs_research=true. research_query darf ausschließlich fachliche Begriffe und sichtbare Titel enthalten, niemals interne Node-IDs, Hashes oder Datenbankkennungen. Gib ausschließlich JSON nach Schema zurück."
|
||||
var decision model.RelationDecision
|
||||
if err := e.Ollama.ChatJSON(ctx, system, relationContext(a, b, sim), relationSchema(), &decision); err != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "think.failed", Source: "brain", Phase: "inference", NodeIDs: []string{a.ID, b.ID}, Message: "Qwen-Beziehungsanalyse ist fehlgeschlagen; es wurde nichts gespeichert", Strength: .35, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "model": e.Cfg.ChatModel}})
|
||||
@@ -1172,88 +1186,42 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
|
||||
decision.ResearchQuery = ""
|
||||
}
|
||||
if decision.NeedsResearch && e.ResearchEnabledForRuntime() && strings.TrimSpace(decision.ResearchQuery) != "" {
|
||||
decision.ResearchQuery = sanitizeSearchQuerySiteFilters(decision.ResearchQuery)
|
||||
if strings.TrimSpace(decision.ResearchQuery) == "" {
|
||||
decision.NeedsResearch = false
|
||||
}
|
||||
}
|
||||
if decision.NeedsResearch && e.ResearchEnabledForRuntime() && strings.TrimSpace(decision.ResearchQuery) != "" {
|
||||
researchID := newResearchRunID("relation-research", decision.ResearchQuery)
|
||||
researchID := newResearchRunID("relation-research-v10", decision.ResearchQuery)
|
||||
researchStarted := time.Now()
|
||||
startMetadata := map[string]any{"trigger": trigger, "research_id": researchID, "research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label, "animation_min_ms": 2000}
|
||||
e.Broker.Publish(model.Activity{Type: "research.started", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · SearXNG durchsucht externe Quellen", Strength: .9, Metadata: startMetadata})
|
||||
|
||||
lease, reused, dedupeErr := e.beginResearchIntent(ctx, "relation", decision.ResearchQuery)
|
||||
var results []model.ResearchResult
|
||||
var diagnostic research.Diagnostic
|
||||
var researchErr error
|
||||
if dedupeErr != nil {
|
||||
researchErr = dedupeErr
|
||||
} else if !lease.owner {
|
||||
results = cloneResearchResults(reused)
|
||||
metadata := mergeResearchMetadata(startMetadata, map[string]any{"similarity": lease.similarity, "reused_results": len(results), "dedupe_threshold": e.Cfg.ResearchDedupeThreshold})
|
||||
for key, value := range researchDedupeLeaseMetadata(lease) {
|
||||
metadata[key] = value
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "research.deduplicated", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Semantisch gleiche Relationsrecherche wurde wiederverwendet · %d Treffer", len(results)), Strength: .76, Metadata: metadata})
|
||||
} else {
|
||||
// Relation research intentionally considers more than the old four
|
||||
// snippets. The full article pipeline remains the final quality gate.
|
||||
relationResultLimit := 8
|
||||
if e.Cfg.ArticleResearchResults > relationResultLimit {
|
||||
relationResultLimit = e.Cfg.ArticleResearchResults
|
||||
}
|
||||
if relationResultLimit > 12 {
|
||||
relationResultLimit = 12
|
||||
}
|
||||
researchErr = e.withSharedResearchWork(ctx, "searxng.relation_search", func() error {
|
||||
var searchErr error
|
||||
results, diagnostic, searchErr = e.Research.SearchDetailed(ctx, decision.ResearchQuery, relationResultLimit)
|
||||
return searchErr
|
||||
})
|
||||
e.completeResearchIntent(lease, results, researchErr)
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "research.started", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · Relationsrecherche prüft externe Volltextbelege einzeln", Strength: .9, Metadata: map[string]any{"trigger": trigger, "research_id": researchID, "research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label, "evidence_gate": "relation-fulltext-v10", "animation_min_ms": 2000}})
|
||||
|
||||
acceptedEvidence, researchMetadata, researchErr := e.collectRelationResearchEvidence(ctx, trigger, a, b, decision)
|
||||
researchMetadata["research_id"] = researchID
|
||||
researchMetadata["duration_ms"] = time.Since(researchStarted).Milliseconds()
|
||||
if researchErr != nil {
|
||||
metadata := mergeResearchMetadata(startMetadata, researchDiagnosticMetadata(diagnostic))
|
||||
metadata["error"] = researchErr.Error()
|
||||
metadata["duration_ms"] = time.Since(researchStarted).Milliseconds()
|
||||
slog.Warn("research failed", "query", decision.ResearchQuery, "base_url", diagnostic.BaseURL, "kind", diagnostic.ErrorKind, "http_status", diagnostic.HTTPStatus, "duration_ms", diagnostic.DurationMS, "error", researchErr)
|
||||
e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "SearXNG-Recherche ist fehlgeschlagen", Strength: .35, Metadata: metadata})
|
||||
researchMetadata["error"] = researchErr.Error()
|
||||
decision.Related = false
|
||||
decision.Confidence = math.Min(decision.Confidence, math.Max(0, e.Cfg.RelationThreshold-.01))
|
||||
decision.Explanation = strings.TrimSpace(decision.Explanation + " · erforderliche externe Relationsrecherche fehlgeschlagen")
|
||||
e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Relationsrecherche ist fehlgeschlagen; die beweisabhängige Relation wird nicht übernommen", Strength: .35, Metadata: researchMetadata})
|
||||
} else {
|
||||
allowedResults := e.filterResearchEvidenceForThinking(results, unique(append(append([]string{}, a.Categories...), b.Categories...)))
|
||||
resultMetadata := mergeResearchMetadata(researchEventMetadata(trigger, researchID, decision.ResearchQuery, allowedResults, time.Since(researchStarted)), researchDiagnosticMetadata(diagnostic))
|
||||
resultMetadata["unfiltered_result_count"] = len(results)
|
||||
resultMetadata["source_filter_rejected_count"] = len(results) - len(allowedResults)
|
||||
resultMetadata["deduplicated"] = !lease.owner
|
||||
message := fmt.Sprintf("SearXNG hat %d durch den Thinking-Filter erlaubte Webquellen geliefert", len(allowedResults))
|
||||
if len(allowedResults) == 0 {
|
||||
message = "SearXNG-Treffer lagen außerhalb des wirksamen Thinking-Quellenfilters"
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "research.results", Source: "searxng", Phase: "research-results", NodeIDs: []string{a.ID, b.ID}, Message: message, Strength: .92, Metadata: resultMetadata})
|
||||
if len(allowedResults) > 0 {
|
||||
// Keep relation-search hits in memory until the relation itself has
|
||||
// passed the second model review. Rejected relation research must not
|
||||
// permanently inflate the graph with unused external nodes.
|
||||
researchResults = allowedResults
|
||||
researchResults = acceptedEvidence
|
||||
e.Broker.Publish(model.Activity{Type: "research.results", Source: "brain", Phase: "research-results", NodeIDs: []string{a.ID, b.ID}, Message: relationResearchResultMessage(researchMetadata), Strength: .82, Metadata: researchMetadata})
|
||||
if len(acceptedEvidence) == 0 {
|
||||
decision.Related = false
|
||||
decision.Confidence = math.Min(decision.Confidence, math.Max(0, e.Cfg.RelationThreshold-.01))
|
||||
decision.Explanation = strings.TrimSpace(decision.Explanation + " · erforderliche externe Relationsrecherche lieferte keinen belastbaren Volltextbeleg")
|
||||
e.Broker.Publish(model.Activity{Type: "think.research.insufficient", Source: "brain", Phase: "research-routing", NodeIDs: []string{a.ID, b.ID}, Message: "Die Relation benötigte externe Fakten, aber kein Suchtreffer bestand die Volltext-Evidenzprüfung", Strength: .4, Metadata: researchMetadata})
|
||||
} else {
|
||||
var reviewed model.RelationDecision
|
||||
reviewSystem := "Bewerte die Beziehung erneut anhand der zwei internen Wissenseinträge und der beigefügten Web-Suchergebnisse. Suchtreffer sind Hinweise, keine garantierten Fakten. Erfinde nichts, kennzeichne verbleibende Unsicherheit und gib ausschließlich JSON nach Schema zurück."
|
||||
if err := e.Ollama.ChatJSON(ctx, reviewSystem, relationContextWithResearch(a, b, sim, allowedResults), relationSchema(), &reviewed); err != nil {
|
||||
slog.Warn("research review failed; keeping pre-research decision", "error", err)
|
||||
reviewSystem := "Bewerte die Beziehung erneut anhand der zwei internen Wissenseinträge und ausschließlich der beigefügten, einzeln geprüften Volltextbelege. Die Belege haben bereits Topic-, Quellenqualitäts- und Volltext-Relevanz-Gates bestanden. Erfinde nichts. Interne Node-IDs sind keine Websuchbegriffe. Wenn die Relation weiterhin nicht belastbar ist, setze related=false. Gib ausschließlich JSON nach Schema zurück."
|
||||
if err := e.Ollama.ChatJSON(ctx, reviewSystem, relationContextWithResearch(a, b, sim, acceptedEvidence), relationSchema(), &reviewed); err != nil {
|
||||
slog.Warn("relation evidence review failed; rejecting evidence-dependent relation", "error", err)
|
||||
researchResults = nil
|
||||
decision.Related = false
|
||||
decision.Confidence = math.Min(decision.Confidence, math.Max(0, e.Cfg.RelationThreshold-.01))
|
||||
decision.Explanation = strings.TrimSpace(decision.Explanation + " · zweiter Relationsreview mit geprüfter Evidenz fehlgeschlagen")
|
||||
} else {
|
||||
decision = reviewed
|
||||
}
|
||||
}
|
||||
// Always close the relation-research lifecycle explicitly. Older code
|
||||
// only emitted research.ingested when at least one source passed the
|
||||
// Thinking filter, leaving zero-result and filtered-out searches stuck
|
||||
// as "running" forever in the analysis history.
|
||||
completedMetadata := mergeResearchMetadata(resultMetadata, map[string]any{
|
||||
"duration_ms": time.Since(researchStarted).Milliseconds(),
|
||||
"accepted_count": len(allowedResults),
|
||||
"result": "completed",
|
||||
})
|
||||
e.Broker.Publish(model.Activity{Type: "research.completed", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Relationsrecherche beendet · %d verwendbare Treffer", len(allowedResults)), Strength: .72, Metadata: completedMetadata})
|
||||
researchMetadata["result"] = "completed"
|
||||
e.Broker.Publish(model.Activity{Type: "research.completed", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: relationResearchResultMessage(researchMetadata), Strength: .72, Metadata: researchMetadata})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1304,11 +1272,18 @@ func (e *Engine) addResearch(ctx context.Context, a, b model.Node, results []mod
|
||||
refs := researchGraphRefs{}
|
||||
for _, r := range results {
|
||||
id := graph.ID("external", r.URL)
|
||||
n := model.Node{ID: id, Kind: "external", Label: r.Title, Summary: clamp(r.Content, 700), Status: "research", Origin: "research", ExternalID: r.URL, URI: r.URL, Categories: unique(append(append([]string{}, a.Categories...), b.Categories...)), Weight: .8, Metadata: map[string]any{"source": graph.SourceFromURL(r.URL), "query_pair": []string{a.ID, b.ID}}, UpdatedAt: time.Now().UTC()}
|
||||
metadata := map[string]any{"source": graph.SourceFromURL(r.URL), "query_pair": []string{a.ID, b.ID}, "validation_state": "relation_fulltext_gate_v10", "relevance": r.Relevance, "source_quality": r.SourceQuality, "source_quality_score": r.SourceQualityScore, "assessment_reason": r.AssessmentReason}
|
||||
if r.Fetched && r.Relevant {
|
||||
if relPath, contentHash, err := e.queueResearchEvidence(r); err == nil {
|
||||
metadata["evidence_path"] = relPath
|
||||
metadata["evidence_sha256"] = contentHash
|
||||
}
|
||||
}
|
||||
n := model.Node{ID: id, Kind: "external", Label: r.Title, Summary: clamp(r.Content, 700), Status: "research", Origin: "research", ExternalID: r.URL, URI: r.URL, Categories: unique(append(append([]string{}, a.Categories...), b.Categories...)), Weight: math.Max(.8, r.SourceQualityScore), Metadata: metadata, UpdatedAt: time.Now().UTC()}
|
||||
refs.Mutations.Add(e.Graph.UpsertNodeWithStats(n))
|
||||
refs.NodeIDs = append(refs.NodeIDs, id)
|
||||
for _, targetID := range []string{a.ID, b.ID} {
|
||||
edge := model.Edge{Source: id, Target: targetID, Type: "research_evidence", Origin: "research", Status: "staging", Confidence: .55, Weight: .4}
|
||||
edge := model.Edge{Source: id, Target: targetID, Type: "research_evidence", Origin: "research", Status: "staging", Confidence: math.Max(.65, r.Relevance), Weight: math.Max(.5, r.SourceQualityScore*.7), Explanation: r.AssessmentReason, Metadata: map[string]any{"validation_state": "relation_fulltext_gate_v10", "source_quality": r.SourceQuality, "relevance": r.Relevance}}
|
||||
refs.Mutations.Add(e.Graph.UpsertEdgeWithStats(edge))
|
||||
refs.EdgeIDs = append(refs.EdgeIDs, graph.EdgeID(edge.Source, edge.Target, edge.Type, edge.Origin))
|
||||
}
|
||||
@@ -1381,14 +1356,14 @@ func (e *Engine) Status() map[string]any {
|
||||
"article_research_min_relevance": e.Cfg.ArticleResearchMinRelevance, "article_research_min_quality": e.Cfg.ArticleResearchMinQuality,
|
||||
"article_research_page_max_bytes": e.Cfg.ArticleResearchPageMaxBytes, "article_research_page_max_chars": e.Cfg.ArticleResearchPageMaxChars,
|
||||
"article_research_fetch_timeout": e.Cfg.ArticleResearchFetchTimeout.String(), "article_research_allow_private": e.Cfg.ArticleResearchAllowPrivate,
|
||||
"article_language": e.Cfg.ArticleLanguage, "article_synthesis_model": e.Cfg.ArticleSynthesisModel, "article_review_model": e.Cfg.ArticleReviewModel, "article_review_repair_rounds": e.Cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v3-quality-v2", "article_research_strategy": e.Cfg.ArticleResearchStrategy, "article_effective_research_strategy": e.effectiveArticleResearchStrategy(), "article_adaptive_initial_queries": e.Cfg.ArticleAdaptiveInitialQueries, "article_adaptive_initial_fetch": e.Cfg.ArticleAdaptiveInitialFetch, "research_dedupe": e.researchDedupeStatus(),
|
||||
"article_language": e.Cfg.ArticleLanguage, "article_synthesis_model": e.Cfg.ArticleSynthesisModel, "article_review_model": e.Cfg.ArticleReviewModel, "article_review_repair_rounds": e.Cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v5-quality-gate-v12", "article_research_strategy": e.Cfg.ArticleResearchStrategy, "article_effective_research_strategy": e.effectiveArticleResearchStrategy(), "article_adaptive_initial_queries": e.Cfg.ArticleAdaptiveInitialQueries, "article_adaptive_initial_fetch": e.Cfg.ArticleAdaptiveInitialFetch, "research_dedupe": e.researchDedupeStatus(),
|
||||
"scan_interval": e.Cfg.ScanInterval.String(), "knowledge_full_verify_interval": e.Cfg.KnowledgeFullVerifyInterval.String(),
|
||||
"enrich_interval": e.Cfg.EnrichInterval.String(),
|
||||
"enrich_batch_size": e.Cfg.EnrichBatchSize, "enrich_anchors": e.Cfg.EnrichAnchors,
|
||||
"processing_mode": e.RuntimeSettings().ProcessingMode, "cluster_hash_bits": e.Cfg.ClusterHashBits, "cluster_hash_tables": e.Cfg.ClusterHashTables,
|
||||
"processing_mode": e.RuntimeSettings().ProcessingMode, "speed_mode": e.RuntimeSettings().SpeedMode, "speed_cpu_tasks": e.RuntimeSettings().SpeedCPUWorkers, "speed_gpu_tasks": e.RuntimeSettings().SpeedGPUInflight, "speed_algorithm": speedModeVersion, "cluster_hash_bits": e.Cfg.ClusterHashBits, "cluster_hash_tables": e.Cfg.ClusterHashTables,
|
||||
"cluster_candidates_per_anchor": e.Cfg.ClusterCandidatesPerAnchor, "cluster_article_candidates": e.Cfg.ClusterArticleCandidates,
|
||||
"cluster_review_evidence": e.Cfg.ClusterReviewEvidence, "cluster_review_context_chars": e.Cfg.ClusterReviewContextChars, "cluster_article_batching": e.Cfg.ClusterArticleBatching,
|
||||
"vector_graph_enabled": e.Cfg.VectorGraphEnabled, "vector_graph_neighbors": e.Cfg.VectorGraphNeighbors, "vector_graph_candidates": e.Cfg.VectorGraphCandidates, "vector_graph_min_similarity": e.Cfg.VectorGraphMinSimilarity, "vector_graph_min_affinity": e.Cfg.VectorGraphMinAffinity, "vector_graph_layout": e.Cfg.VectorGraphLayout, "vector_graph_reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": e.Cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_interval": e.Cfg.VectorGraphLayoutRelaxInterval.String(), "vector_graph_layout_blend": e.Cfg.VectorGraphLayoutBlend, "vector_graph_layout_max_shift": e.Cfg.VectorGraphLayoutMaxShift, "last_vector_graph": e.lastVectorGraph, "last_vector_layout": e.lastVectorLayout,
|
||||
"vector_graph_enabled": e.Cfg.VectorGraphEnabled, "vector_graph_neighbors": e.Cfg.VectorGraphNeighbors, "vector_graph_candidates": e.Cfg.VectorGraphCandidates, "vector_graph_min_similarity": e.Cfg.VectorGraphMinSimilarity, "vector_graph_min_affinity": e.Cfg.VectorGraphMinAffinity, "vector_graph_layout": e.Cfg.VectorGraphLayout, "vector_graph_layout_mode": e.effectiveVectorLayoutMode(), "vector_graph_reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": e.Cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_effective": e.Cfg.VectorGraphRelaxLayout && !e.Cfg.VectorGraphLayout, "vector_graph_layout_relax_interval": e.Cfg.VectorGraphLayoutRelaxInterval.String(), "vector_graph_layout_blend": e.Cfg.VectorGraphLayoutBlend, "vector_graph_layout_max_shift": e.Cfg.VectorGraphLayoutMaxShift, "last_vector_graph": e.lastVectorGraph, "last_vector_layout": e.lastVectorLayout,
|
||||
"vector_graph_orphan_pass": e.Cfg.VectorGraphOrphanPass, "vector_graph_orphan_neighbors": e.Cfg.VectorGraphOrphanNeighbors, "vector_graph_orphan_candidates": e.Cfg.VectorGraphOrphanCandidates, "vector_graph_orphan_min_similarity": e.Cfg.VectorGraphOrphanMinSimilarity, "vector_graph_orphan_min_affinity": e.Cfg.VectorGraphOrphanMinAffinity,
|
||||
"vector_graph_agent_offload": e.Cfg.VectorGraphAgentOffload, "vector_graph_agent_required": e.Cfg.VectorGraphAgentRequired, "vector_graph_agent_wait": e.Cfg.VectorGraphAgentWait.String(), "thinking_vector_guided": e.Cfg.ThinkingVectorGuided,
|
||||
"research_enabled": e.ResearchEnabledForRuntime(), "chat_model": e.Cfg.ChatModel, "embedding_model": e.Cfg.EmbeddingModel,
|
||||
@@ -1475,7 +1450,10 @@ func relationContextWithResearch(a, b model.Node, sim float64, results []model.R
|
||||
}
|
||||
|
||||
func relationContext(a, b model.Node, sim float64) string {
|
||||
return fmt.Sprintf("SEMANTISCHE_NÄHE: %.4f\n\nA\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s\n\nB\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s", sim, a.ID, a.Label, strings.Join(a.Categories, ", "), a.Summary, b.ID, b.Label, strings.Join(b.Categories, ", "), b.Summary)
|
||||
// Internal IDs are intentionally omitted from the model-visible relation
|
||||
// context. They are routing metadata, not semantic search terms, and older
|
||||
// prompts occasionally copied them into public SearXNG queries.
|
||||
return fmt.Sprintf("SEMANTISCHE_NÄHE: %.4f\n\nA\nTitel: %s\nKategorien: %s\nInhalt: %s\n\nB\nTitel: %s\nKategorien: %s\nInhalt: %s", sim, a.Label, strings.Join(a.Categories, ", "), a.Summary, b.Label, strings.Join(b.Categories, ", "), b.Summary)
|
||||
}
|
||||
func relationSchema() map[string]any {
|
||||
return map[string]any{"type": "object", "properties": map[string]any{"related": map[string]any{"type": "boolean"}, "relation_type": map[string]any{"type": "string", "enum": []string{"related_to", "depends_on", "supports", "contradicts", "extends", "same_topic", "caused_by"}}, "confidence": map[string]any{"type": "number", "minimum": 0, "maximum": 1}, "explanation": map[string]any{"type": "string"}, "needs_research": map[string]any{"type": "boolean"}, "research_query": map[string]any{"type": "string"}, "topic_label": map[string]any{"type": "string"}, "keywords": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}}, "required": []string{"related", "relation_type", "confidence", "explanation", "needs_research", "research_query", "topic_label", "keywords"}}
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
"github.com/local/glpi-neural-brain/internal/research"
|
||||
)
|
||||
|
||||
var internalNodeIDPattern = regexp.MustCompile(`(?i)\b[0-9a-f]{20,64}\b`)
|
||||
|
||||
// normalizeRelationResearchQuery prevents internal graph identifiers and generic
|
||||
// bookkeeping language from leaking into public Web searches. If the planner's
|
||||
// query does not contain a meaningful term from either source label/topic, the
|
||||
// Brain deterministically rebuilds it from human-readable source labels.
|
||||
func normalizeRelationResearchQuery(a, b model.Node, decision model.RelationDecision) (string, bool, string) {
|
||||
original := strings.TrimSpace(sanitizeSearchQuerySiteFilters(decision.ResearchQuery))
|
||||
if original == "" {
|
||||
return "", false, "empty"
|
||||
}
|
||||
labels := strings.TrimSpace(strings.Join([]string{a.Label, b.Label, decision.TopicLabel}, " "))
|
||||
anchorTerms := researchTerms(labels)
|
||||
queryTerms := researchTerms(original)
|
||||
matched := 0
|
||||
for term := range anchorTerms {
|
||||
if queryTerms[term] {
|
||||
matched++
|
||||
}
|
||||
}
|
||||
containsInternalID := (strings.TrimSpace(a.ID) != "" && strings.Contains(original, a.ID)) || (strings.TrimSpace(b.ID) != "" && strings.Contains(original, b.ID)) || internalNodeIDPattern.MatchString(original)
|
||||
genericOnly := len(anchorTerms) > 0 && matched == 0
|
||||
if !containsInternalID && !genericOnly {
|
||||
return original, false, ""
|
||||
}
|
||||
|
||||
relationIntent := map[string]string{
|
||||
"same_topic": "Gemeinsamkeiten Unterschiede fachliche Einordnung",
|
||||
"related_to": "fachlicher Zusammenhang Abgrenzung",
|
||||
"depends_on": "Abhängigkeit Voraussetzung Zusammenhang",
|
||||
"supports": "Unterstützung Zusammenhang Nachweis",
|
||||
"contradicts": "Widerspruch Unterschiede Nachweis",
|
||||
"extends": "Erweiterung Zusammenhang Abgrenzung",
|
||||
"caused_by": "Ursache Zusammenhang Nachweis",
|
||||
}[safeRelation(decision.RelationType)]
|
||||
if relationIntent == "" {
|
||||
relationIntent = "fachlicher Zusammenhang Nachweis"
|
||||
}
|
||||
parts := []string{strings.TrimSpace(a.Label), strings.TrimSpace(b.Label), strings.TrimSpace(decision.TopicLabel), relationIntent, "offizielle Dokumentation"}
|
||||
rebuilt := strings.TrimSpace(strings.Join(unique(parts), " "))
|
||||
reason := "missing_topic_anchor"
|
||||
if containsInternalID {
|
||||
reason = "internal_node_id_removed"
|
||||
}
|
||||
return rebuilt, true, reason
|
||||
}
|
||||
|
||||
func relationResearchQuestion(a, b model.Node, decision model.RelationDecision, query string) model.ResearchQuestion {
|
||||
question := strings.TrimSpace(strings.Join([]string{
|
||||
"Prüfe den fachlichen Zusammenhang zwischen",
|
||||
a.Label,
|
||||
"und",
|
||||
b.Label + ".",
|
||||
"Zu bewertende Relation:",
|
||||
safeRelation(decision.RelationType) + ".",
|
||||
strings.TrimSpace(decision.Explanation),
|
||||
}, " "))
|
||||
return model.ResearchQuestion{
|
||||
GapID: "relation-evidence",
|
||||
Question: question,
|
||||
Critical: true,
|
||||
ExpectActionable: false,
|
||||
QueriesDE: []string{query},
|
||||
}
|
||||
}
|
||||
|
||||
// collectRelationResearchEvidence performs a strict, per-source evidence path.
|
||||
// Search snippets remain ephemeral. Only fetched full text that passes the
|
||||
// deterministic topic guard, source filter and full-text relevance/quality gate
|
||||
// is returned to the relation reviewer. Nothing is written to the graph here.
|
||||
func (e *Engine) collectRelationResearchEvidence(ctx context.Context, trigger string, a, b model.Node, decision model.RelationDecision) ([]model.ResearchResult, map[string]any, error) {
|
||||
query, rebuilt, rebuildReason := normalizeRelationResearchQuery(a, b, decision)
|
||||
metadata := map[string]any{
|
||||
"trigger": trigger,
|
||||
"research_query": query,
|
||||
"source_label": a.Label,
|
||||
"target_label": b.Label,
|
||||
"query_rebuilt": rebuilt,
|
||||
"query_rebuild_reason": rebuildReason,
|
||||
}
|
||||
if query == "" {
|
||||
return nil, metadata, nil
|
||||
}
|
||||
if rebuilt && e.Broker != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "think.research.query.rebuilt", Source: "brain", Phase: "research-routing", NodeIDs: []string{a.ID, b.ID}, Message: "Relationsrecherche wurde aus den sichtbaren Themenbegriffen neu aufgebaut; interne Node-IDs werden nicht ins Web gesendet", Strength: .56, Metadata: map[string]any{
|
||||
"trigger": trigger, "original_query": decision.ResearchQuery, "research_query": query, "reason": rebuildReason, "source_label": a.Label, "target_label": b.Label,
|
||||
}})
|
||||
}
|
||||
|
||||
question := relationResearchQuestion(a, b, decision, query)
|
||||
lease, reused, err := e.beginResearchIntent(ctx, "relation-v10", query)
|
||||
if err != nil {
|
||||
return nil, metadata, err
|
||||
}
|
||||
if !lease.owner {
|
||||
validated, rejected := e.revalidateReusableResearchEvidence(ctx, question, reused)
|
||||
metadata["deduplicated"] = true
|
||||
metadata["reused_results"] = len(reused)
|
||||
metadata["reused_rejected"] = rejected
|
||||
metadata["accepted_count"] = len(validated)
|
||||
return validated, metadata, nil
|
||||
}
|
||||
|
||||
resultLimit := e.Cfg.ArticleResearchResults
|
||||
if resultLimit < 8 {
|
||||
resultLimit = 8
|
||||
}
|
||||
if resultLimit > 12 {
|
||||
resultLimit = 12
|
||||
}
|
||||
var results []model.ResearchResult
|
||||
var diagnostic research.Diagnostic
|
||||
searchStarted := time.Now()
|
||||
searchErr := e.withSharedResearchWork(ctx, "searxng.relation_search_v10", func() error {
|
||||
var inner error
|
||||
results, diagnostic, inner = e.Research.SearchDetailed(ctx, query, resultLimit)
|
||||
return inner
|
||||
})
|
||||
if searchErr != nil {
|
||||
e.completeResearchIntent(lease, nil, searchErr)
|
||||
metadata = mergeResearchMetadata(metadata, researchDiagnosticMetadata(diagnostic))
|
||||
metadata["duration_ms"] = time.Since(searchStarted).Milliseconds()
|
||||
return nil, metadata, searchErr
|
||||
}
|
||||
for i := range results {
|
||||
results[i].Query = query
|
||||
results[i].Round = 1
|
||||
}
|
||||
metadata["unfiltered_result_count"] = len(results)
|
||||
categories := unique(append(append([]string{}, a.Categories...), b.Categories...))
|
||||
allowed := e.filterResearchEvidenceForThinking(results, categories)
|
||||
metadata["source_filter_rejected_count"] = len(results) - len(allowed)
|
||||
metadata["source_filter_allowed_count"] = len(allowed)
|
||||
if len(allowed) == 0 {
|
||||
e.completeResearchIntent(lease, nil, nil)
|
||||
metadata["accepted_count"] = 0
|
||||
return nil, metadata, nil
|
||||
}
|
||||
|
||||
// Snippet selection is deterministic/heuristic; semantic model budget is
|
||||
// spent only after full text has been fetched.
|
||||
ranked := rankResearchCandidatesHeuristic(question, allowed, false)
|
||||
fetchLimit := e.Cfg.ArticleResearchFetchResults
|
||||
if fetchLimit < 1 {
|
||||
fetchLimit = 4
|
||||
}
|
||||
if fetchLimit > 4 {
|
||||
fetchLimit = 4
|
||||
}
|
||||
prefetchMin := math.Max(.35, e.Cfg.ArticleResearchPrefetchMinRelevance)
|
||||
finalMin := math.Max(.60, e.Cfg.ArticleResearchMinRelevance)
|
||||
minQuality := math.Max(.55, e.Cfg.ArticleResearchMinQuality)
|
||||
selection := selectResearchCandidates(question, ranked, map[string]bool{}, fetchLimit, 1, prefetchMin, finalMin, minQuality)
|
||||
metadata["snippet_gate_selected_count"] = len(selection.Selected)
|
||||
metadata["snippet_gate_rejected_count"] = selection.GateRejected
|
||||
metadata["authoritative_exploration_selected_count"] = countSelectedMode(selection.Decisions, "authoritative_exploration")
|
||||
if len(selection.Selected) == 0 {
|
||||
e.completeResearchIntent(lease, nil, nil)
|
||||
metadata["accepted_count"] = 0
|
||||
return nil, metadata, nil
|
||||
}
|
||||
|
||||
fetched := make([]model.ResearchResult, 0, len(selection.Selected))
|
||||
fetchFailures := 0
|
||||
for _, candidate := range selection.Selected {
|
||||
var page research.FetchedPage
|
||||
fetchErr := e.withSharedResearchWork(ctx, "web.relation_fetch_v10", func() error {
|
||||
var inner error
|
||||
page, _, inner = e.Research.FetchPage(ctx, candidate.Result.URL, research.FetchOptions{
|
||||
MaxBytes: e.Cfg.ArticleResearchPageMaxBytes, MaxChars: e.Cfg.ArticleResearchPageMaxChars,
|
||||
Timeout: e.Cfg.ArticleResearchFetchTimeout, AllowPrivate: e.Cfg.ArticleResearchAllowPrivate,
|
||||
})
|
||||
return inner
|
||||
})
|
||||
if fetchErr != nil {
|
||||
fetchFailures++
|
||||
continue
|
||||
}
|
||||
item := candidate.Result
|
||||
item.URL = nonempty(canonicalResearchURL(page.URL), page.URL)
|
||||
if strings.TrimSpace(page.Title) != "" {
|
||||
item.Title = page.Title
|
||||
}
|
||||
item.Content = page.Content
|
||||
item.ContentType = page.ContentType
|
||||
item.Fetched = true
|
||||
item.Query = query
|
||||
item.Round = 1
|
||||
fetched = append(fetched, item)
|
||||
}
|
||||
metadata["fetched_count"] = len(fetched)
|
||||
metadata["fetch_failed_count"] = fetchFailures
|
||||
if len(fetched) == 0 {
|
||||
e.completeResearchIntent(lease, nil, nil)
|
||||
metadata["accepted_count"] = 0
|
||||
return nil, metadata, nil
|
||||
}
|
||||
|
||||
assessed := e.rankResearchCandidates(ctx, question, fetched, true)
|
||||
accepted := make([]model.ResearchResult, 0, len(assessed))
|
||||
fulltextRejected := 0
|
||||
for _, candidate := range assessed {
|
||||
assessment := candidate.Assessment
|
||||
strict := candidate.TopicGuardPassed && assessment.Relevant && assessment.Relevance >= finalMin && assessment.SourceQualityScore >= minQuality
|
||||
if !strict {
|
||||
fulltextRejected++
|
||||
continue
|
||||
}
|
||||
item := candidate.Result
|
||||
item.Relevant = true
|
||||
item.Relevance = assessment.Relevance
|
||||
item.SourceQuality = assessment.SourceQuality
|
||||
item.SourceQualityScore = assessment.SourceQualityScore
|
||||
item.Actionable = assessment.Actionable
|
||||
item.CoveredGapIDs = unique(append(assessment.CoveredGapIDs, question.GapID))
|
||||
item.AssessmentReason = assessment.Reason
|
||||
accepted = append(accepted, item)
|
||||
}
|
||||
// Stable ordering makes relation-review prompts and dedupe cache deterministic.
|
||||
sort.SliceStable(accepted, func(i, j int) bool {
|
||||
if accepted[i].Relevance != accepted[j].Relevance {
|
||||
return accepted[i].Relevance > accepted[j].Relevance
|
||||
}
|
||||
return accepted[i].URL < accepted[j].URL
|
||||
})
|
||||
metadata["fulltext_rejected_count"] = fulltextRejected
|
||||
metadata["accepted_count"] = len(accepted)
|
||||
metadata["accepted_titles"] = researchTitles(accepted)
|
||||
metadata["minimum_relevance"] = finalMin
|
||||
metadata["minimum_quality"] = minQuality
|
||||
metadata["duration_ms"] = time.Since(searchStarted).Milliseconds()
|
||||
e.completeResearchIntent(lease, accepted, nil)
|
||||
return accepted, metadata, nil
|
||||
}
|
||||
|
||||
func relationResearchResultMessage(metadata map[string]any) string {
|
||||
accepted := 0
|
||||
if value, ok := metadata["accepted_count"].(int); ok {
|
||||
accepted = value
|
||||
}
|
||||
return fmt.Sprintf("Relationsrecherche beendet · %d einzeln geprüfte Volltextbelege", accepted)
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
)
|
||||
|
||||
func TestNormalizeRelationResearchQueryRemovesInternalNodeIDs(t *testing.T) {
|
||||
a := model.Node{ID: "8c68b454ce45705357ae1831", Label: "AI Safety Guardrails – sicher gestalten und härten"}
|
||||
b := model.Node{ID: "a0808bae9ec114a144632262", Label: "AI Security Guardrails – sicher gestalten und härten"}
|
||||
decision := model.RelationDecision{RelationType: "same_topic", TopicLabel: "AI Guardrails", ResearchQuery: "Überprüfung der Einträge 8c68b454ce45705357ae1831 und a0808bae9ec114a144632262 auf Duplikat oder unterschiedliche Quellen"}
|
||||
query, rebuilt, reason := normalizeRelationResearchQuery(a, b, decision)
|
||||
if !rebuilt || reason != "internal_node_id_removed" {
|
||||
t.Fatalf("expected deterministic rebuild, rebuilt=%v reason=%q query=%q", rebuilt, reason, query)
|
||||
}
|
||||
if strings.Contains(query, a.ID) || strings.Contains(query, b.ID) || internalNodeIDPattern.MatchString(query) {
|
||||
t.Fatalf("internal IDs leaked into rebuilt query: %q", query)
|
||||
}
|
||||
for _, want := range []string{"AI Safety Guardrails", "AI Security Guardrails"} {
|
||||
if !strings.Contains(query, want) {
|
||||
t.Fatalf("rebuilt query lost visible topic %q: %q", want, query)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRelationResearchQueryRebuildsGenericQueryWithoutTopicAnchor(t *testing.T) {
|
||||
a := model.Node{Label: "Kubernetes Restore Testing"}
|
||||
b := model.Node{Label: "Kubernetes Backup Validation"}
|
||||
decision := model.RelationDecision{RelationType: "supports", ResearchQuery: "Einträge prüfen und Unterschiede validieren"}
|
||||
query, rebuilt, reason := normalizeRelationResearchQuery(a, b, decision)
|
||||
if !rebuilt || reason != "missing_topic_anchor" || !strings.Contains(strings.ToLower(query), "kubernetes") {
|
||||
t.Fatalf("expected topic-anchored rebuild, got rebuilt=%v reason=%q query=%q", rebuilt, reason, query)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRelationSnippetGateRejectsUnrelatedDuplicateTools(t *testing.T) {
|
||||
question := model.ResearchQuestion{GapID: "relation-evidence", Question: "AI Safety Guardrails und AI Security Guardrails fachlich vergleichen"}
|
||||
results := []model.ResearchResult{
|
||||
{Title: "IBAN auf Fehler prüfen und Bankverbindung identifizieren", URL: "https://example.org/iban", Snippet: "IBAN prüfen und Duplikate finden"},
|
||||
{Title: "Duplikate in Excel finden", URL: "https://example.org/excel", Snippet: "Doppelte Einträge über mehrere Spalten"},
|
||||
{Title: "AI Security Guardrails technical guidance", URL: "https://docs.example.org/ai-security/guardrails", Snippet: "AI safety and security guardrails, controls and validation"},
|
||||
}
|
||||
ranked := rankResearchCandidatesHeuristic(question, results, false)
|
||||
selection := selectResearchCandidates(question, ranked, map[string]bool{}, 3, 1, .35, .60, .55)
|
||||
for _, candidate := range selection.Selected {
|
||||
if strings.Contains(strings.ToLower(candidate.Result.Title), "iban") || strings.Contains(strings.ToLower(candidate.Result.Title), "excel") {
|
||||
t.Fatalf("unrelated duplicate-tool result passed relation topic gate: %+v", candidate.Result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconsiderOperationalArticleTypeCanChooseReference(t *testing.T) {
|
||||
mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/tags":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"models": []map[string]any{{"name": "qwen3:8b", "digest": "chat"}, {"name": "embeddinggemma", "digest": "embed"}}})
|
||||
case "/api/chat":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"message": map[string]any{"content": `{"action":"reclassify","article_type":"reference","reason":"Die Evidenz trägt technische Zuordnungen, aber keinen belastbaren Lösungsablauf."}`}})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer mock.Close()
|
||||
client := ollama.New(mock.URL, "qwen3:8b", "embeddinggemma")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client.Start(ctx)
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for client.PoolStatus()["healthy_nodes"].(int) < 1 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
e := &Engine{Ollama: client}
|
||||
plan := model.ArticlePlanDecision{ArticleType: "troubleshooting", ExpectedValue: "ATT&CK-Techniken einordnen"}
|
||||
content := model.KnowledgeArticleContent{Title: "ATT&CK-Techniken", TechnicalDetails: []string{"T1018"}, Mappings: []string{"Profil → T1018"}}
|
||||
result, err := e.reconsiderOperationalArticleType(context.Background(), plan, content, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Action != "reclassify" || result.ArticleType != "reference" {
|
||||
t.Fatalf("unexpected reconsideration: %+v", result)
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
)
|
||||
|
||||
type researchIntentProfile struct {
|
||||
@@ -83,7 +82,7 @@ func (e *Engine) beginResearchIntentMode(ctx context.Context, kind, rawIntent st
|
||||
// Embed the natural-language intent rather than the sorted normalization.
|
||||
// Word order and phrases such as "volatile evidence before reboot" carry
|
||||
// information that is lost by a bag-of-terms representation.
|
||||
vectors, err := e.Ollama.Embed(ollama.WithLowPriority(ctx), []string{rawIntent})
|
||||
vectors, err := e.Ollama.Embed(e.backgroundOllamaContext(ctx), []string{rawIntent})
|
||||
if err == nil && len(vectors) == 1 {
|
||||
vector = vectors[0]
|
||||
}
|
||||
|
||||
@@ -21,6 +21,9 @@ type RuntimeSettings struct {
|
||||
ViewMode string `json:"view_mode"`
|
||||
MaxDisplayNodes int `json:"max_display_nodes"`
|
||||
LowPowerMode bool `json:"low_power_mode"`
|
||||
SpeedMode bool `json:"speed_mode"`
|
||||
SpeedCPUWorkers int `json:"speed_cpu_tasks"`
|
||||
SpeedGPUInflight int `json:"speed_gpu_tasks"`
|
||||
ProcessingMode string `json:"processing_mode"`
|
||||
AutonomousResearchEnabled bool `json:"autonomous_research_enabled"`
|
||||
AutonomousResearchIdleOnly bool `json:"autonomous_research_idle_only"`
|
||||
@@ -53,6 +56,9 @@ func (e *Engine) defaultRuntimeSettings() RuntimeSettings {
|
||||
ViewMode: e.Cfg.DefaultView,
|
||||
MaxDisplayNodes: e.Cfg.MaxDisplayNodes,
|
||||
LowPowerMode: e.Cfg.LowPowerMode,
|
||||
SpeedMode: e.Cfg.SpeedMode,
|
||||
SpeedCPUWorkers: e.Cfg.SpeedCPUWorkers,
|
||||
SpeedGPUInflight: e.Cfg.SpeedGPUInflight,
|
||||
ProcessingMode: e.Cfg.ProcessingMode,
|
||||
AutonomousResearchEnabled: e.Cfg.AutonomousResearchEnabled,
|
||||
AutonomousResearchIdleOnly: e.Cfg.AutonomousResearchIdleOnly,
|
||||
@@ -86,6 +92,23 @@ func normalizeRuntimeSettings(in RuntimeSettings) RuntimeSettings {
|
||||
if in.MaxDisplayNodes > 500000 {
|
||||
in.MaxDisplayNodes = 500000
|
||||
}
|
||||
if in.SpeedCPUWorkers < 1 {
|
||||
in.SpeedCPUWorkers = 1
|
||||
}
|
||||
if in.SpeedCPUWorkers > 256 {
|
||||
in.SpeedCPUWorkers = 256
|
||||
}
|
||||
if in.SpeedGPUInflight < 1 {
|
||||
in.SpeedGPUInflight = 1
|
||||
}
|
||||
if in.SpeedGPUInflight > 64 {
|
||||
in.SpeedGPUInflight = 64
|
||||
}
|
||||
if in.SpeedMode {
|
||||
// Eco and Speed have opposite scheduling/rendering goals. Speed wins when
|
||||
// an older client accidentally submits both flags.
|
||||
in.LowPowerMode = false
|
||||
}
|
||||
if in.AutonomousResearchMinPriority < 0 {
|
||||
in.AutonomousResearchMinPriority = 0
|
||||
}
|
||||
@@ -172,6 +195,9 @@ func mergeRuntimeSettingsJSON(settings *RuntimeSettings, data []byte) {
|
||||
decode("view_mode", &settings.ViewMode)
|
||||
decode("max_display_nodes", &settings.MaxDisplayNodes)
|
||||
decode("low_power_mode", &settings.LowPowerMode)
|
||||
decode("speed_mode", &settings.SpeedMode)
|
||||
decode("speed_cpu_tasks", &settings.SpeedCPUWorkers)
|
||||
decode("speed_gpu_tasks", &settings.SpeedGPUInflight)
|
||||
decode("processing_mode", &settings.ProcessingMode)
|
||||
decode("autonomous_research_enabled", &settings.AutonomousResearchEnabled)
|
||||
decode("autonomous_research_idle_only", &settings.AutonomousResearchIdleOnly)
|
||||
@@ -217,6 +243,12 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
|
||||
if settings.AutonomousResearchTasksPerCycle == 0 {
|
||||
settings.AutonomousResearchTasksPerCycle = previous.AutonomousResearchTasksPerCycle
|
||||
}
|
||||
if settings.SpeedCPUWorkers == 0 {
|
||||
settings.SpeedCPUWorkers = previous.SpeedCPUWorkers
|
||||
}
|
||||
if settings.SpeedGPUInflight == 0 {
|
||||
settings.SpeedGPUInflight = previous.SpeedGPUInflight
|
||||
}
|
||||
if settings.ProcessingMode == "" {
|
||||
settings.ProcessingMode = previous.ProcessingMode
|
||||
}
|
||||
@@ -226,6 +258,12 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
|
||||
if settings.MaxDisplayNodes < 0 || settings.MaxDisplayNodes > 500000 {
|
||||
return e.RuntimeSettings(), fmt.Errorf("max_display_nodes must be between 0 and 500000")
|
||||
}
|
||||
if settings.SpeedCPUWorkers < 1 || settings.SpeedCPUWorkers > 256 {
|
||||
return e.RuntimeSettings(), fmt.Errorf("speed_cpu_tasks must be between 1 and 256")
|
||||
}
|
||||
if settings.SpeedGPUInflight < 1 || settings.SpeedGPUInflight > 64 {
|
||||
return e.RuntimeSettings(), fmt.Errorf("speed_gpu_tasks must be between 1 and 64")
|
||||
}
|
||||
if settings.AutonomousResearchMinPriority < 0 || settings.AutonomousResearchMinPriority > 1 {
|
||||
return e.RuntimeSettings(), fmt.Errorf("autonomous_research_min_priority must be between 0 and 1")
|
||||
}
|
||||
@@ -243,6 +281,11 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
|
||||
previous = e.runtime
|
||||
e.runtime = settings
|
||||
e.runtimeMu.Unlock()
|
||||
e.applyRuntimePerformance(settings)
|
||||
|
||||
if settings.SpeedMode && !previous.SpeedMode {
|
||||
e.signalSpeedWork()
|
||||
}
|
||||
|
||||
if settings.AutonomousResearchEnabled && !previous.AutonomousResearchEnabled {
|
||||
e.signalAutonomousResearch()
|
||||
@@ -265,6 +308,7 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
|
||||
e.runtimeMu.Lock()
|
||||
e.runtime = previous
|
||||
e.runtimeMu.Unlock()
|
||||
e.applyRuntimePerformance(previous)
|
||||
return previous, err
|
||||
}
|
||||
}
|
||||
@@ -285,6 +329,9 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
|
||||
"view_mode": settings.ViewMode,
|
||||
"max_display_nodes": settings.MaxDisplayNodes,
|
||||
"low_power_mode": settings.LowPowerMode,
|
||||
"speed_mode": settings.SpeedMode,
|
||||
"speed_cpu_tasks": settings.SpeedCPUWorkers,
|
||||
"speed_gpu_tasks": settings.SpeedGPUInflight,
|
||||
"processing_mode": settings.ProcessingMode,
|
||||
"autonomous_research_enabled": settings.AutonomousResearchEnabled,
|
||||
"autonomous_research_idle_only": settings.AutonomousResearchIdleOnly,
|
||||
@@ -296,6 +343,13 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func (e *Engine) SpeedModeEnabled() bool {
|
||||
e.runtimeMu.RLock()
|
||||
enabled := e.runtime.SpeedMode
|
||||
e.runtimeMu.RUnlock()
|
||||
return enabled
|
||||
}
|
||||
|
||||
func (e *Engine) LearningEnabled() bool {
|
||||
e.runtimeMu.RLock()
|
||||
enabled := e.runtime.LearningEnabled
|
||||
|
||||
@@ -100,3 +100,20 @@ func TestNormalizeRuntimeSettingsProcessingMode(t *testing.T) {
|
||||
t.Fatalf("invalid mode should fall back to precise: %+v", settings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRuntimeSettingsSpeedWinsOverEco(t *testing.T) {
|
||||
settings := normalizeRuntimeSettings(RuntimeSettings{
|
||||
LowPowerMode: true,
|
||||
SpeedMode: true,
|
||||
SpeedCPUWorkers: 999,
|
||||
SpeedGPUInflight: 999,
|
||||
AutonomousResearchMaxTasksPerDay: 1,
|
||||
AutonomousResearchTasksPerCycle: 1,
|
||||
})
|
||||
if settings.LowPowerMode {
|
||||
t.Fatal("speed mode must disable eco mode")
|
||||
}
|
||||
if settings.SpeedCPUWorkers != 256 || settings.SpeedGPUInflight != 64 {
|
||||
t.Fatalf("speed limits were not clamped: %+v", settings)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/graph"
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
"github.com/local/glpi-neural-brain/internal/research"
|
||||
"github.com/local/glpi-neural-brain/internal/sourceagent"
|
||||
)
|
||||
@@ -58,11 +57,23 @@ func (e *Engine) evidenceAcquisitionEnabled() bool {
|
||||
func (e *Engine) sourceInboxLoop(ctx context.Context) {
|
||||
e.reconcileProactiveSecurityMaterialization(ctx)
|
||||
e.queueExistingProactiveSecurityCandidates(ctx)
|
||||
// Classify once immediately so newly ingested or migration-requeued documents
|
||||
// do not wait a full interval after startup.
|
||||
e.processSourceInbox(ctx)
|
||||
e.queueExistingProactiveSecurityCandidates(ctx)
|
||||
e.processProactiveSecurityInbox(ctx)
|
||||
run := func() int {
|
||||
processed := e.processSourceInbox(ctx)
|
||||
e.queueExistingProactiveSecurityCandidates(ctx)
|
||||
processed += e.processProactiveSecurityInbox(ctx)
|
||||
return processed
|
||||
}
|
||||
drain := func() {
|
||||
for {
|
||||
processed := run()
|
||||
if !e.SpeedModeEnabled() || processed == 0 || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Always classify once immediately. In Speed mode the loop keeps claiming
|
||||
// batches until the queues are empty instead of sleeping between batches.
|
||||
drain()
|
||||
ticker := time.NewTicker(e.Cfg.SourceInboxInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
@@ -70,24 +81,34 @@ func (e *Engine) sourceInboxLoop(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
e.processSourceInbox(ctx)
|
||||
e.queueExistingProactiveSecurityCandidates(ctx)
|
||||
e.processProactiveSecurityInbox(ctx)
|
||||
drain()
|
||||
case <-e.sourceInboxWake:
|
||||
drain()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) processSourceInbox(ctx context.Context) {
|
||||
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled {
|
||||
func (e *Engine) WakeSourceInbox() {
|
||||
if e == nil || e.sourceInboxWake == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case e.sourceInboxWake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) processSourceInbox(ctx context.Context) int {
|
||||
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled {
|
||||
return 0
|
||||
}
|
||||
items, err := e.SourceInbox.ClaimInbox(ctx, e.Cfg.SourceInboxBatchSize)
|
||||
if err != nil {
|
||||
slog.Warn("source inbox claim failed", "error", err)
|
||||
return
|
||||
return 0
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return
|
||||
return 0
|
||||
}
|
||||
texts := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
@@ -97,7 +118,7 @@ func (e *Engine) processSourceInbox(ctx context.Context) {
|
||||
}
|
||||
texts = append(texts, text)
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ollama.WithLowPriority(ctx), 4*time.Minute)
|
||||
cctx, cancel := context.WithTimeout(e.backgroundOllamaContext(ctx), 4*time.Minute)
|
||||
vectors, embedErr := e.Ollama.Embed(cctx, texts)
|
||||
cancel()
|
||||
if embedErr != nil || len(vectors) != len(items) {
|
||||
@@ -110,7 +131,7 @@ func (e *Engine) processSourceInbox(ctx context.Context) {
|
||||
slog.Error("source inbox release after embedding failure failed", "inbox_id", item.ID, "error", releaseErr)
|
||||
}
|
||||
}
|
||||
return
|
||||
return 0
|
||||
}
|
||||
candidateCount := 0
|
||||
for i, item := range items {
|
||||
@@ -158,6 +179,7 @@ func (e *Engine) processSourceInbox(ctx context.Context) {
|
||||
if e.Broker != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "source.inbox.classified", Source: "brain", Phase: "source-inbox", Message: fmt.Sprintf("Source-Inbox: %d Dokumente geprüft · %d als Wissenskandidaten vorgemerkt", len(items), candidateCount), Strength: .36, Metadata: map[string]any{"documents": len(items), "candidates": candidateCount, "minimum_similarity": e.Cfg.SourceInboxMinSimilarity, "minimum_priority": e.Cfg.SourceInboxMinPriority, "novelty_floor": e.Cfg.SourceInboxNoveltyFloor, "classifier_version": SourceInboxClassifierVersion}})
|
||||
}
|
||||
return len(items)
|
||||
}
|
||||
|
||||
func (e *Engine) sourceInboxKnowledgeHit(hits []model.Hit) (model.Hit, bool) {
|
||||
@@ -335,17 +357,17 @@ func (e *Engine) queueExistingProactiveSecurityCandidates(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) processProactiveSecurityInbox(ctx context.Context) {
|
||||
func (e *Engine) processProactiveSecurityInbox(ctx context.Context) int {
|
||||
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled || !e.Cfg.SourceInboxSecurityProactiveEnabled || e.Ollama == nil {
|
||||
return
|
||||
return 0
|
||||
}
|
||||
items, err := e.SourceInbox.ClaimProactiveSecurity(ctx, e.Cfg.SourceInboxSecurityBatchSize)
|
||||
if err != nil {
|
||||
slog.Warn("source inbox proactive security claim failed", "error", err)
|
||||
return
|
||||
return 0
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return
|
||||
return 0
|
||||
}
|
||||
materialized := 0
|
||||
for _, item := range items {
|
||||
@@ -400,6 +422,7 @@ func (e *Engine) processProactiveSecurityInbox(ctx context.Context) {
|
||||
if materialized > 0 {
|
||||
e.RequestEnrich("source-security")
|
||||
}
|
||||
return len(items)
|
||||
}
|
||||
|
||||
func (e *Engine) processProactiveSecurityItem(ctx context.Context, item sourceagent.InboxDocument, started time.Time) (bool, graph.MutationStats, error) {
|
||||
@@ -610,7 +633,7 @@ func (e *Engine) assessSecurityInbox(ctx context.Context, item sourceagent.Inbox
|
||||
if modelName == "" {
|
||||
modelName = e.Cfg.ChatModel
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ollama.WithLowPriority(ctx), 6*time.Minute)
|
||||
cctx, cancel := context.WithTimeout(e.backgroundOllamaContext(ctx), 6*time.Minute)
|
||||
err := e.Ollama.ChatJSONModel(cctx, modelName, securityInboxSystemPrompt(), b.String(), securityInboxSchema(), &out)
|
||||
cancel()
|
||||
out = normalizeSecurityAssessment(out, item.Document.Title)
|
||||
@@ -821,7 +844,7 @@ func (e *Engine) learnProactiveSecurityNode(ctx context.Context, node model.Node
|
||||
if text == "" {
|
||||
return graph.MutationStats{}
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ollama.WithLowPriority(ctx), 3*time.Minute)
|
||||
cctx, cancel := context.WithTimeout(e.backgroundOllamaContext(ctx), 3*time.Minute)
|
||||
vectors, err := e.Ollama.Embed(cctx, []string{text})
|
||||
cancel()
|
||||
if err != nil || len(vectors) != 1 || len(vectors[0]) == 0 {
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
)
|
||||
|
||||
const speedModeVersion = "throughput-v1"
|
||||
|
||||
func (e *Engine) effectiveSpeedCPUWorkers(settings RuntimeSettings) int {
|
||||
workers := settings.SpeedCPUWorkers
|
||||
if workers < 1 {
|
||||
workers = runtime.NumCPU()
|
||||
}
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
if workers > 256 {
|
||||
workers = 256
|
||||
}
|
||||
return workers
|
||||
}
|
||||
|
||||
func (e *Engine) effectiveSpeedGPUInflight(settings RuntimeSettings) int {
|
||||
limit := settings.SpeedGPUInflight
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
if limit > 64 {
|
||||
limit = 64
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
func (e *Engine) applyRuntimePerformance(settings RuntimeSettings) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
sharedInflight := e.Cfg.ResearchOllamaMaxInflight
|
||||
queueSize := e.Cfg.ResearchOllamaQueueSize
|
||||
gpuInflight := e.Cfg.OllamaNodeMaxInflight
|
||||
if sharedInflight < 1 {
|
||||
sharedInflight = 1
|
||||
}
|
||||
if queueSize < 1 {
|
||||
queueSize = 1
|
||||
}
|
||||
if gpuInflight < 1 {
|
||||
gpuInflight = 1
|
||||
}
|
||||
if settings.SpeedMode {
|
||||
cpu := e.effectiveSpeedCPUWorkers(settings)
|
||||
gpu := e.effectiveSpeedGPUInflight(settings)
|
||||
// The shared limiter covers model calls and outbound research work. Do not
|
||||
// let it become the bottleneck below either requested CPU fan-out or total
|
||||
// per-node GPU fan-out.
|
||||
nodeCount := 1
|
||||
if e.Ollama != nil {
|
||||
nodeCount = len(e.Ollama.NodeStatuses())
|
||||
}
|
||||
if nodeCount < 1 {
|
||||
nodeCount = 1
|
||||
}
|
||||
sharedInflight = maxInt(cpu, gpu*nodeCount)
|
||||
queueSize = maxInt(queueSize, sharedInflight*16)
|
||||
gpuInflight = gpu
|
||||
}
|
||||
if e.sharedWork != nil {
|
||||
e.sharedWork.SetLimits(sharedInflight, queueSize)
|
||||
}
|
||||
if e.Ollama != nil {
|
||||
e.Ollama.SetNodeMaxInflight(gpuInflight)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) signalSpeedWork() {
|
||||
if e == nil || !e.SpeedModeEnabled() {
|
||||
return
|
||||
}
|
||||
if e.ThinkingEnabled() {
|
||||
e.RequestEnrich("speed-mode")
|
||||
}
|
||||
if e.AutonomousResearchEnabled() {
|
||||
e.RequestAutonomousResearchScan("speed-mode")
|
||||
e.signalAutonomousResearch()
|
||||
}
|
||||
if e.sourceInboxWake != nil {
|
||||
select {
|
||||
case e.sourceInboxWake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
if e.Broker != nil {
|
||||
settings := e.RuntimeSettings()
|
||||
e.Broker.Publish(model.Activity{Type: "performance.speed.pulse", Source: "brain", Phase: "performance", Message: "Speed-Modus treibt vorhandene CPU-/GPU-Arbeit ohne künstliche Pacing-Pausen weiter", Strength: .58, Metadata: map[string]any{"algorithm": speedModeVersion, "cpu_tasks": settings.SpeedCPUWorkers, "gpu_tasks_per_node": settings.SpeedGPUInflight}})
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) backgroundOllamaContext(ctx context.Context) context.Context {
|
||||
if e != nil && e.SpeedModeEnabled() {
|
||||
return ctx
|
||||
}
|
||||
return ollama.WithLowPriority(ctx)
|
||||
}
|
||||
|
||||
func (e *Engine) effectiveEnrichStepDelay() time.Duration {
|
||||
if e != nil && e.SpeedModeEnabled() {
|
||||
return 0
|
||||
}
|
||||
return e.Cfg.EnrichStepDelay
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/config"
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
"github.com/local/glpi-neural-brain/internal/workqueue"
|
||||
)
|
||||
|
||||
func TestApplyRuntimePerformanceRaisesAndRestoresLimits(t *testing.T) {
|
||||
pool := ollama.NewPool(ollama.PoolConfig{
|
||||
Nodes: []ollama.NodeConfig{{Name: "gpu-a", URL: "http://unused"}, {Name: "gpu-b", URL: "http://unused"}},
|
||||
NodeMaxInflight: 1,
|
||||
}, "qwen3:8b", "embeddinggemma")
|
||||
limiter := workqueue.New(2, 8)
|
||||
e := &Engine{
|
||||
Cfg: config.Config{ResearchOllamaMaxInflight: 2, ResearchOllamaQueueSize: 8, OllamaNodeMaxInflight: 1},
|
||||
Ollama: pool,
|
||||
sharedWork: limiter,
|
||||
}
|
||||
e.applyRuntimePerformance(RuntimeSettings{SpeedMode: true, SpeedCPUWorkers: 6, SpeedGPUInflight: 3})
|
||||
status := limiter.Status()
|
||||
if status.MaxInflight != 6 || status.QueueSize < 96 {
|
||||
t.Fatalf("unexpected speed limiter: %+v", status)
|
||||
}
|
||||
if got := pool.NodeMaxInflight(); got != 3 {
|
||||
t.Fatalf("gpu limit=%d want 3", got)
|
||||
}
|
||||
e.applyRuntimePerformance(RuntimeSettings{SpeedMode: false})
|
||||
status = limiter.Status()
|
||||
if status.MaxInflight != 2 || status.QueueSize != 8 || pool.NodeMaxInflight() != 1 {
|
||||
t.Fatalf("normal limits not restored: limiter=%+v gpu=%d", status, pool.NodeMaxInflight())
|
||||
}
|
||||
}
|
||||
@@ -21,11 +21,17 @@ type vectorLayerExecution struct {
|
||||
AgentID string
|
||||
ComputeMS int64
|
||||
FallbackReason string
|
||||
LayoutMode string
|
||||
LayoutDue bool
|
||||
}
|
||||
|
||||
func (e *Engine) vectorLayerConfig() graph.VectorSemanticLayerConfig {
|
||||
workers := 1
|
||||
if settings := e.RuntimeSettings(); settings.SpeedMode {
|
||||
workers = e.effectiveSpeedCPUWorkers(settings)
|
||||
}
|
||||
return graph.VectorSemanticLayerConfig{
|
||||
Neighbors: e.Cfg.VectorGraphNeighbors, CandidateLimit: e.Cfg.VectorGraphCandidates,
|
||||
Workers: workers, Neighbors: e.Cfg.VectorGraphNeighbors, CandidateLimit: e.Cfg.VectorGraphCandidates,
|
||||
HashBits: e.Cfg.ClusterHashBits, HashTables: e.Cfg.ClusterHashTables,
|
||||
MinSimilarity: e.Cfg.VectorGraphMinSimilarity, MinAffinity: e.Cfg.VectorGraphMinAffinity,
|
||||
Layout: e.Cfg.VectorGraphLayout, LayoutRelax: false, LayoutBlend: e.Cfg.VectorGraphLayoutBlend, LayoutMaxShift: e.Cfg.VectorGraphLayoutMaxShift,
|
||||
@@ -36,8 +42,12 @@ func (e *Engine) vectorLayerConfig() graph.VectorSemanticLayerConfig {
|
||||
}
|
||||
|
||||
func (e *Engine) vectorPrimaryConfig(layout bool) vectorgraph.Config {
|
||||
workers := 1
|
||||
if settings := e.RuntimeSettings(); settings.SpeedMode {
|
||||
workers = e.effectiveSpeedCPUWorkers(settings)
|
||||
}
|
||||
return vectorgraph.Config{
|
||||
Neighbors: e.Cfg.VectorGraphNeighbors, CandidateLimit: e.Cfg.VectorGraphCandidates,
|
||||
Workers: workers, Neighbors: e.Cfg.VectorGraphNeighbors, CandidateLimit: e.Cfg.VectorGraphCandidates,
|
||||
HashBits: e.Cfg.ClusterHashBits, HashTables: e.Cfg.ClusterHashTables, BandBits: 8,
|
||||
MinSimilarity: e.Cfg.VectorGraphMinSimilarity, MinAffinity: e.Cfg.VectorGraphMinAffinity,
|
||||
Layout: layout, Smoothing: .22,
|
||||
@@ -45,8 +55,12 @@ func (e *Engine) vectorPrimaryConfig(layout bool) vectorgraph.Config {
|
||||
}
|
||||
|
||||
func (e *Engine) vectorOrphanConfig() vectorgraph.Config {
|
||||
workers := 1
|
||||
if settings := e.RuntimeSettings(); settings.SpeedMode {
|
||||
workers = e.effectiveSpeedCPUWorkers(settings)
|
||||
}
|
||||
return vectorgraph.Config{
|
||||
Neighbors: e.Cfg.VectorGraphOrphanNeighbors, CandidateLimit: e.Cfg.VectorGraphOrphanCandidates,
|
||||
Workers: workers, Neighbors: e.Cfg.VectorGraphOrphanNeighbors, CandidateLimit: e.Cfg.VectorGraphOrphanCandidates,
|
||||
HashBits: e.Cfg.ClusterHashBits, HashTables: e.Cfg.ClusterHashTables, BandBits: 8,
|
||||
MinSimilarity: e.Cfg.VectorGraphOrphanMinSimilarity, MinAffinity: e.Cfg.VectorGraphOrphanMinAffinity,
|
||||
Layout: false,
|
||||
@@ -104,15 +118,24 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
|
||||
layoutDue := e.Cfg.VectorGraphLayout || (e.Cfg.VectorGraphRelaxLayout && (lastVectorLayout.IsZero() || time.Since(lastVectorLayout) >= e.Cfg.VectorGraphLayoutRelaxInterval))
|
||||
cfg.Layout = layoutDue
|
||||
cfg.LayoutRelax = !e.Cfg.VectorGraphLayout && layoutDue
|
||||
layoutMode := "off"
|
||||
if e.Cfg.VectorGraphLayout && layoutDue {
|
||||
layoutMode = "full"
|
||||
} else if cfg.LayoutRelax {
|
||||
layoutMode = "relax"
|
||||
}
|
||||
if !e.Cfg.VectorGraphAgentOffload || e.SourceInbox == nil {
|
||||
primary, orphan, focus := e.Graph.BuildVectorSemanticLayerLocal(cfg, filter)
|
||||
stats, mutations := e.Graph.ApplyVectorSemanticLayer(cfg, primary, orphan, focus)
|
||||
e.noteVectorLayoutApplied(layoutDue, stats.PositionUpdates)
|
||||
return vectorLayerExecution{Stats: stats, Mutations: mutations}, nil
|
||||
return vectorLayerExecution{Stats: stats, Mutations: mutations, LayoutMode: layoutMode, LayoutDue: layoutDue}, nil
|
||||
}
|
||||
|
||||
hasAgent, availabilityErr := e.checkOnlineVectorComputeAgent(ctx)
|
||||
startupGrace := vectorAgentStartupGrace(!e.bootstrapIsComplete(), e.Cfg.VectorGraphAgentWait)
|
||||
if e.SpeedModeEnabled() && !e.Cfg.VectorGraphAgentRequired {
|
||||
startupGrace = 0
|
||||
}
|
||||
if availabilityErr == nil && !hasAgent && startupGrace > 0 {
|
||||
_ = e.requestControllerComputeCapacity(ctx, sourceagent.ComputeKindVectorGraph)
|
||||
e.Broker.Publish(model.Activity{Type: "vector.graph.agent.waiting", Source: "brain", Phase: "semantic-linking", Message: fmt.Sprintf("Initialer Vector-Graph wartet bis zu %s auf die Compute-Agent-Registrierung", startupGrace), Strength: .34, Metadata: map[string]any{"run_id": runID, "kind": sourceagent.ComputeKindVectorGraph, "startup_grace": startupGrace.String()}})
|
||||
@@ -127,12 +150,12 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
|
||||
reason = "compute_agent_check_failed: " + availabilityErr.Error()
|
||||
}
|
||||
if e.Cfg.VectorGraphAgentRequired {
|
||||
return vectorLayerExecution{FallbackReason: reason}, fmt.Errorf("vector graph agent offload required but unavailable: %s", reason)
|
||||
return vectorLayerExecution{FallbackReason: reason, LayoutMode: layoutMode, LayoutDue: layoutDue}, fmt.Errorf("vector graph agent offload required but unavailable: %s", reason)
|
||||
}
|
||||
primary, orphan, focus := e.Graph.BuildVectorSemanticLayerLocal(cfg, filter)
|
||||
stats, mutations := e.Graph.ApplyVectorSemanticLayer(cfg, primary, orphan, focus)
|
||||
e.noteVectorLayoutApplied(layoutDue, stats.PositionUpdates)
|
||||
return vectorLayerExecution{Stats: stats, Mutations: mutations, FallbackReason: reason}, nil
|
||||
return vectorLayerExecution{Stats: stats, Mutations: mutations, FallbackReason: reason, LayoutMode: layoutMode, LayoutDue: layoutDue}, nil
|
||||
}
|
||||
|
||||
entries := e.Graph.VectorSemanticEntries(filter)
|
||||
@@ -146,7 +169,7 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
|
||||
Entries: entries,
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "vector.graph.agent.queued", Source: "brain", Phase: "semantic-linking", Message: fmt.Sprintf("Vector-Graph-CPU-Job für Agent bereitgestellt · %d Embeddings", len(entries)), Strength: .46, Metadata: map[string]any{
|
||||
"run_id": runID, "kind": sourceagent.ComputeKindVectorGraph, "indexed": len(entries), "orphan_focus_candidates": len(baseOrphans), "graph_version": inputVersion,
|
||||
"run_id": runID, "kind": sourceagent.ComputeKindVectorGraph, "indexed": len(entries), "orphan_focus_candidates": len(baseOrphans), "graph_version": inputVersion, "cpu_workers": request.Header.Primary.Workers, "speed_mode": e.SpeedModeEnabled(),
|
||||
}})
|
||||
jobCtx, cancelJob := context.WithTimeout(ctx, e.Cfg.VectorGraphAgentWait)
|
||||
result, err := e.SourceInbox.SubmitVectorGraphJob(jobCtx, request)
|
||||
@@ -161,12 +184,12 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
|
||||
reason := err.Error()
|
||||
e.Broker.Publish(model.Activity{Type: "vector.graph.agent.fallback", Source: "brain", Phase: "semantic-linking", Message: "Agent-Vectorjob konnte nicht sicher übernommen werden; lokale CPU-Berechnung wird verwendet", Strength: .38, Metadata: map[string]any{"run_id": runID, "reason": reason, "graph_version": inputVersion}})
|
||||
if e.Cfg.VectorGraphAgentRequired {
|
||||
return vectorLayerExecution{FallbackReason: reason}, err
|
||||
return vectorLayerExecution{FallbackReason: reason, LayoutMode: layoutMode, LayoutDue: layoutDue}, err
|
||||
}
|
||||
primary, orphan, focus := e.Graph.BuildVectorSemanticLayerLocal(cfg, filter)
|
||||
stats, mutations := e.Graph.ApplyVectorSemanticLayer(cfg, primary, orphan, focus)
|
||||
e.noteVectorLayoutApplied(layoutDue, stats.PositionUpdates)
|
||||
return vectorLayerExecution{Stats: stats, Mutations: mutations, FallbackReason: reason}, nil
|
||||
return vectorLayerExecution{Stats: stats, Mutations: mutations, FallbackReason: reason, LayoutMode: layoutMode, LayoutDue: layoutDue}, nil
|
||||
}
|
||||
|
||||
focus := remainingOrphanFocus(baseOrphans, result.Primary.Links)
|
||||
@@ -174,9 +197,9 @@ func (e *Engine) rebuildVectorSemanticLayer(ctx context.Context, runID string, f
|
||||
e.noteVectorLayoutApplied(layoutDue, stats.PositionUpdates)
|
||||
e.Broker.Publish(model.Activity{Type: "vector.graph.agent.completed", Source: "agent", Phase: "semantic-linking", Message: fmt.Sprintf("Agent hat Vector-Graph-CPU-Job abgeschlossen · %d + %d Kanten", len(result.Primary.Links), len(result.Orphan.Links)), Strength: .68, Metadata: withRunMutations(map[string]any{
|
||||
"run_id": runID, "agent_id": result.AgentID, "compute_duration_ms": result.DurationMS, "indexed": len(entries),
|
||||
"primary_links": len(result.Primary.Links), "orphan_links": len(result.Orphan.Links), "orphan_focus": len(focus), "no_model_call": true,
|
||||
"primary_links": len(result.Primary.Links), "orphan_links": len(result.Orphan.Links), "orphan_focus": len(focus), "no_model_call": true, "cpu_workers": request.Header.Primary.Workers, "speed_mode": e.SpeedModeEnabled(),
|
||||
}, mutations)})
|
||||
return vectorLayerExecution{Stats: stats, Mutations: mutations, Offloaded: true, AgentID: result.AgentID, ComputeMS: result.DurationMS}, nil
|
||||
return vectorLayerExecution{Stats: stats, Mutations: mutations, Offloaded: true, AgentID: result.AgentID, ComputeMS: result.DurationMS, LayoutMode: layoutMode, LayoutDue: layoutDue}, nil
|
||||
}
|
||||
|
||||
func (e *Engine) noteVectorLayoutApplied(layoutDue bool, updates uint64) {
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/graph"
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
)
|
||||
|
||||
func (e *Engine) effectiveVectorLayoutMode() string {
|
||||
if e.Cfg.VectorGraphLayout {
|
||||
return "full"
|
||||
}
|
||||
if e.Cfg.VectorGraphRelaxLayout {
|
||||
return "relax"
|
||||
}
|
||||
return "off"
|
||||
}
|
||||
|
||||
func (e *Engine) vectorMaintenanceDue(now time.Time) (reevaluate bool, layout bool) {
|
||||
if !e.Cfg.VectorGraphEnabled {
|
||||
return false, false
|
||||
}
|
||||
e.stateMu.RLock()
|
||||
lastGraph := e.lastVectorGraph
|
||||
lastLayout := e.lastVectorLayout
|
||||
maintenanceStartedAt := e.vectorMaintenanceStartedAt
|
||||
e.stateMu.RUnlock()
|
||||
reevaluate = !e.Graph.HasEdgesByOrigin(graph.VectorMathOrigin) || lastGraph.IsZero() || now.Sub(lastGraph) >= e.Cfg.VectorGraphReevaluateInterval
|
||||
// Full layout is intentionally coupled to semantic reevaluation. Relax mode
|
||||
// may run on its own slower/faster cadence without enabling the hard layout.
|
||||
if !e.Cfg.VectorGraphLayout && e.Cfg.VectorGraphRelaxLayout {
|
||||
if lastLayout.IsZero() {
|
||||
// Do not fake last_vector_layout merely to delay the first relaxation.
|
||||
// Keep telemetry truthful and use the process-start marker as a separate
|
||||
// not-before timestamp.
|
||||
layout = maintenanceStartedAt.IsZero() || now.Sub(maintenanceStartedAt) >= e.Cfg.VectorGraphLayoutRelaxInterval
|
||||
} else {
|
||||
layout = now.Sub(lastLayout) >= e.Cfg.VectorGraphLayoutRelaxInterval
|
||||
}
|
||||
}
|
||||
return reevaluate, layout
|
||||
}
|
||||
|
||||
func (e *Engine) vectorMaintenanceLoop(ctx context.Context) {
|
||||
// Give bootstrap-owned scans a short head start. If Learning is disabled the
|
||||
// loop still runs and can maintain an already persisted vector layer.
|
||||
initialDelay := 5 * time.Second
|
||||
if e.SpeedModeEnabled() {
|
||||
initialDelay = 0
|
||||
}
|
||||
initial := time.NewTimer(initialDelay)
|
||||
defer initial.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-initial.C:
|
||||
}
|
||||
if err := e.runVectorMaintenance(ctx, "startup-maintenance"); err != nil {
|
||||
slog.Warn("vector maintenance startup run failed", "error", err)
|
||||
}
|
||||
ticker := time.NewTicker(time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := e.runVectorMaintenance(ctx, "scheduled-maintenance"); err != nil {
|
||||
slog.Warn("vector maintenance run failed", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) runVectorMaintenance(ctx context.Context, trigger string) error {
|
||||
reevaluateDue, layoutDue := e.vectorMaintenanceDue(time.Now().UTC())
|
||||
if !reevaluateDue && !layoutDue {
|
||||
return nil
|
||||
}
|
||||
// Serialize with Learning Scan. Both operations may replace vector-math
|
||||
// edges/positions and must never race against each other.
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
reevaluateDue, layoutDue = e.vectorMaintenanceDue(time.Now().UTC())
|
||||
if !reevaluateDue && !layoutDue {
|
||||
return nil
|
||||
}
|
||||
if e.Graph.CountVectorsByDimension(256) != 0 {
|
||||
if e.Broker != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "vector.graph.maintenance.skipped", Source: "brain", Phase: "semantic-linking", Message: "Vector-Maintenance wartet auf echte Embeddings; 256D-Fallback-Vektoren werden nicht mit dem Produktionsraum gemischt", Strength: .28, Metadata: map[string]any{"trigger": trigger, "reason": "fallback_vectors_present"}})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
entries := e.Graph.VectorSemanticEntries(e.effectiveLearningFilter())
|
||||
if len(entries) < 2 {
|
||||
return nil
|
||||
}
|
||||
started := time.Now().UTC()
|
||||
runID := fmt.Sprintf("vector-maintenance-%d", started.UnixNano())
|
||||
if e.Broker != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "vector.graph.maintenance.started", Source: "brain", Phase: "semantic-linking", Message: "Periodische mathematische Nähepflege wurde gestartet", Strength: .4, Metadata: map[string]any{
|
||||
"run_id": runID, "trigger": trigger, "reevaluate_due": reevaluateDue, "layout_due": layoutDue, "effective_layout_mode": e.effectiveVectorLayoutMode(), "learning_enabled": e.LearningEnabled(),
|
||||
}})
|
||||
}
|
||||
execution, err := e.rebuildVectorSemanticLayer(ctx, runID, e.effectiveLearningFilter())
|
||||
if err != nil {
|
||||
if e.Broker != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "vector.graph.maintenance.failed", Source: "brain", Phase: "semantic-linking", Message: "Periodische mathematische Nähepflege ist fehlgeschlagen", Strength: .32, Metadata: map[string]any{"run_id": runID, "trigger": trigger, "error": err.Error(), "agent_required": e.Cfg.VectorGraphAgentRequired}})
|
||||
}
|
||||
return err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
e.stateMu.Lock()
|
||||
e.lastVectorGraph = now
|
||||
e.stateMu.Unlock()
|
||||
stats := execution.Stats
|
||||
if e.Broker != nil {
|
||||
meta := withRunMutations(map[string]any{
|
||||
"run_id": runID, "trigger": trigger, "algorithm": "mutual-knn-local-scaling-v1", "orphan_algorithm": "orphan-knn-local-scaling-v1", "no_model_call": true,
|
||||
"indexed": stats.Indexed, "links": stats.Links, "reciprocal_links": stats.ReciprocalLinks, "candidate_pairs": stats.CandidatePairs, "exact_comparisons": stats.ExactComparisons,
|
||||
"orphan_pass_enabled": e.Cfg.VectorGraphOrphanPass, "orphan_focus": stats.OrphanFocus, "orphan_links": stats.OrphanLinks, "orphan_exact_comparisons": stats.OrphanStats.ExactComparisons, "orphan_candidate_pairs": stats.OrphanStats.CandidatePairs,
|
||||
"position_updates": stats.PositionUpdates, "layout_mode": execution.LayoutMode, "layout_applied": execution.LayoutDue && stats.PositionUpdates > 0, "periodic_refresh": reevaluateDue, "layout_refresh": layoutDue,
|
||||
"reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(), "layout_relax_interval": e.Cfg.VectorGraphLayoutRelaxInterval.String(), "learning_enabled": e.LearningEnabled(),
|
||||
"agent_offloaded": execution.Offloaded, "agent_id": execution.AgentID, "agent_compute_ms": execution.ComputeMS, "agent_fallback_reason": execution.FallbackReason, "speed_mode": e.SpeedModeEnabled(), "cpu_workers": e.vectorPrimaryConfig(execution.LayoutDue).Workers, "duration_ms": time.Since(started).Milliseconds(),
|
||||
}, execution.Mutations)
|
||||
e.Broker.Publish(model.Activity{Type: "vector.graph.rebuilt", Source: "brain", Phase: "semantic-linking", Message: fmt.Sprintf("Periodische Vektorpflege: %d Primär- + %d Orphan-Kanten aus %d Embeddings", stats.Links, stats.OrphanLinks, stats.Indexed), Strength: .58, Metadata: meta})
|
||||
e.Broker.Publish(model.Activity{Type: "vector.graph.maintenance.completed", Source: "brain", Phase: "semantic-linking", Message: fmt.Sprintf("Mathematische Nähepflege abgeschlossen · Layoutmodus %s · %d Positionsupdates", execution.LayoutMode, stats.PositionUpdates), Strength: .5, Metadata: meta})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/config"
|
||||
)
|
||||
|
||||
func TestEffectiveVectorLayoutModeFullOverridesRelax(t *testing.T) {
|
||||
e := &Engine{Cfg: config.Config{VectorGraphLayout: true, VectorGraphRelaxLayout: true}}
|
||||
if got := e.effectiveVectorLayoutMode(); got != "full" {
|
||||
t.Fatalf("full layout must win when both flags are true, got %q", got)
|
||||
}
|
||||
e.Cfg.VectorGraphLayout = false
|
||||
if got := e.effectiveVectorLayoutMode(); got != "relax" {
|
||||
t.Fatalf("expected relax mode, got %q", got)
|
||||
}
|
||||
e.Cfg.VectorGraphRelaxLayout = false
|
||||
if got := e.effectiveVectorLayoutMode(); got != "off" {
|
||||
t.Fatalf("expected off mode, got %q", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user