+77
-99
@@ -75,6 +75,7 @@ type Engine struct {
|
||||
lastScan time.Time
|
||||
lastVectorGraph time.Time
|
||||
lastVectorLayout time.Time
|
||||
vectorMaintenanceStartedAt time.Time
|
||||
bootstrapComplete bool
|
||||
bootstrapCompletedAt time.Time
|
||||
bootstrapError string
|
||||
@@ -108,6 +109,7 @@ type Engine struct {
|
||||
interactiveInflight atomic.Int64
|
||||
autonomousWake chan struct{}
|
||||
autonomousScanRequests chan string
|
||||
sourceInboxWake chan struct{}
|
||||
autonomousRunning bool
|
||||
autonomousTaskID string
|
||||
autonomousTaskTopic string
|
||||
@@ -312,13 +314,9 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
|
||||
sharedWork := workqueue.New(cfg.ResearchOllamaMaxInflight, cfg.ResearchOllamaQueueSize)
|
||||
pool.SetSharedLimiter(sharedWork)
|
||||
persistence := persist.New(g, b, cfg.PersistInterval)
|
||||
e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: pool, Persistence: persistence, Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs, FullVerifyInterval: cfg.KnowledgeFullVerifyInterval}, bootstrapReady: make(chan struct{}), enrichRequests: make(chan string, 1), autonomousWake: make(chan struct{}, 1), autonomousScanRequests: make(chan string, 1), runtimePath: filepath.Join(cfg.DataDir, "runtime-settings.json"), researchEvidenceCache: map[string]researchEvidenceRecord{}, sharedWork: sharedWork, researchDedupe: map[string]*researchDedupeEntry{}}
|
||||
// Slow relaxation is maintenance, not a startup animation. Delaying its first
|
||||
// cycle prevents frequent process restarts from moving the cloud repeatedly.
|
||||
if cfg.VectorGraphRelaxLayout && !cfg.VectorGraphLayout {
|
||||
e.lastVectorLayout = time.Now().UTC()
|
||||
}
|
||||
e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: pool, Persistence: persistence, vectorMaintenanceStartedAt: time.Now().UTC(), Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs, FullVerifyInterval: cfg.KnowledgeFullVerifyInterval}, bootstrapReady: make(chan struct{}), enrichRequests: make(chan string, 1), autonomousWake: make(chan struct{}, 1), autonomousScanRequests: make(chan string, 1), sourceInboxWake: make(chan struct{}, 1), runtimePath: filepath.Join(cfg.DataDir, "runtime-settings.json"), researchEvidenceCache: map[string]researchEvidenceRecord{}, sharedWork: sharedWork, researchDedupe: map[string]*researchDedupeEntry{}}
|
||||
e.loadRuntimeSettings()
|
||||
e.applyRuntimePerformance(e.RuntimeSettings())
|
||||
if cfg.SearXNGURL != "" {
|
||||
e.Research = research.New(cfg.SearXNGURL)
|
||||
}
|
||||
@@ -327,7 +325,7 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
|
||||
e.GLPIKB = ingest.NewGLPIKBSyncer(ingest.GLPIKBConfig{Enabled: true, Path: cfg.GLPIKBPath, Filter: cfg.GLPIKBFilter, Limit: cfg.GLPIKBLimit, SyncInterval: cfg.GLPIKBSyncInterval, Source: cfg.GLPIKBSource, CachePath: filepath.Join(cfg.DataDir, "glpi-kb-cache.json"), ShouldSync: e.LearningEnabled}, client, g, b, persistence)
|
||||
}
|
||||
if b != nil {
|
||||
b.Publish(model.Activity{Type: "system.started", Source: "brain", Phase: "startup", Message: "Neural Brain wurde gestartet; das Analyseprotokoll zeichnet Läufe und Graphänderungen auf", Strength: .3, Metadata: map[string]any{"chat_model": cfg.ChatModel, "embedding_model": cfg.EmbeddingModel, "article_language": cfg.ArticleLanguage, "article_synthesis_model": cfg.ArticleSynthesisModel, "article_review_model": cfg.ArticleReviewModel, "article_review_repair_rounds": cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v3-quality-v2", "article_research_strategy": cfg.ArticleResearchStrategy, "cluster_article_batching": cfg.ClusterArticleBatching, "vector_graph_enabled": cfg.VectorGraphEnabled, "vector_graph_layout": cfg.VectorGraphLayout, "vector_graph_reevaluate_interval": cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_interval": cfg.VectorGraphLayoutRelaxInterval.String(), "article_cpu_quality_enabled": cfg.ArticleCPUQualityEnabled, "article_cpu_quality_agent_offload": cfg.ArticleCPUQualityAgentOffload, "research_ollama_max_inflight": cfg.ResearchOllamaMaxInflight, "research_ollama_queue_size": cfg.ResearchOllamaQueueSize, "graph_version": g.Version()}})
|
||||
b.Publish(model.Activity{Type: "system.started", Source: "brain", Phase: "startup", Message: "Neural Brain wurde gestartet; das Analyseprotokoll zeichnet Läufe und Graphänderungen auf", Strength: .3, Metadata: map[string]any{"chat_model": cfg.ChatModel, "embedding_model": cfg.EmbeddingModel, "article_language": cfg.ArticleLanguage, "article_synthesis_model": cfg.ArticleSynthesisModel, "article_review_model": cfg.ArticleReviewModel, "article_review_repair_rounds": cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v5-quality-gate-v12", "article_research_strategy": cfg.ArticleResearchStrategy, "cluster_article_batching": cfg.ClusterArticleBatching, "vector_graph_enabled": cfg.VectorGraphEnabled, "vector_graph_layout": cfg.VectorGraphLayout, "vector_graph_layout_mode": e.effectiveVectorLayoutMode(), "vector_graph_reevaluate_interval": cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_effective": cfg.VectorGraphRelaxLayout && !cfg.VectorGraphLayout, "vector_graph_layout_relax_interval": cfg.VectorGraphLayoutRelaxInterval.String(), "article_cpu_quality_enabled": cfg.ArticleCPUQualityEnabled, "article_cpu_quality_agent_offload": cfg.ArticleCPUQualityAgentOffload, "research_ollama_max_inflight": cfg.ResearchOllamaMaxInflight, "research_ollama_queue_size": cfg.ResearchOllamaQueueSize, "speed_mode": e.RuntimeSettings().SpeedMode, "speed_cpu_tasks": e.RuntimeSettings().SpeedCPUWorkers, "speed_gpu_tasks": e.RuntimeSettings().SpeedGPUInflight, "graph_version": g.Version()}})
|
||||
}
|
||||
return e
|
||||
}
|
||||
@@ -378,6 +376,12 @@ func (e *Engine) Start(ctx context.Context) {
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
if e.waitBootstrap(ctx) {
|
||||
e.vectorMaintenanceLoop(ctx)
|
||||
}
|
||||
}()
|
||||
|
||||
go e.enrichmentWorker(ctx)
|
||||
if e.Cfg.AutoEnrich {
|
||||
go e.enrichmentScheduler(ctx)
|
||||
@@ -461,6 +465,9 @@ func (e *Engine) waitBootstrap(ctx context.Context) bool {
|
||||
|
||||
func (e *Engine) enrichmentScheduler(ctx context.Context) {
|
||||
firstDelay := 12 * time.Second
|
||||
if e.SpeedModeEnabled() {
|
||||
firstDelay = 0
|
||||
}
|
||||
e.setNextEnrich(time.Now().Add(firstDelay))
|
||||
timer := time.NewTimer(firstDelay)
|
||||
defer timer.Stop()
|
||||
@@ -488,7 +495,10 @@ func (e *Engine) enrichmentWorker(ctx context.Context) {
|
||||
if !e.waitBootstrap(ctx) {
|
||||
return
|
||||
}
|
||||
e.runEnrichmentCycle(ctx, trigger)
|
||||
more := e.runEnrichmentCycle(ctx, trigger)
|
||||
if more && e.SpeedModeEnabled() && e.ThinkingEnabled() {
|
||||
e.RequestEnrich("speed-drain")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -528,7 +538,7 @@ func (e *Engine) RequestEnrich(trigger string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
|
||||
func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) bool {
|
||||
started := time.Now()
|
||||
e.stateMu.Lock()
|
||||
e.enrichRunning = true
|
||||
@@ -588,13 +598,13 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
|
||||
if outcome.Rejected {
|
||||
rejected++
|
||||
}
|
||||
if step+1 < e.Cfg.EnrichBatchSize && e.Cfg.EnrichStepDelay > 0 {
|
||||
if delay := e.effectiveEnrichStepDelay(); step+1 < e.Cfg.EnrichBatchSize && delay > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
cycleErr = ctx.Err()
|
||||
result = "cancelled"
|
||||
step = e.Cfg.EnrichBatchSize
|
||||
case <-time.After(e.Cfg.EnrichStepDelay):
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -624,13 +634,14 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
|
||||
if cycleErr != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "think.cycle.failed", Source: "brain", Phase: "autonomous", Message: "AI-THINK-Zyklus wurde mit Fehler beendet", Strength: .45, Metadata: metadata})
|
||||
slog.Warn("enrichment cycle failed", "trigger", trigger, "error", cycleErr)
|
||||
return
|
||||
return false
|
||||
}
|
||||
message := fmt.Sprintf("AI-THINK-Zyklus abgeschlossen · %d Relationen · %d Artikel · %d verworfen", relationsCreated, articlesCreated, rejected)
|
||||
if result == "no_candidate" {
|
||||
message = "AI-THINK hat im aktuell geprüften Graphbereich keinen Kandidaten oberhalb des Schwellwerts gefunden"
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "think.cycle.completed", Source: "brain", Phase: "autonomous", Message: message, Strength: .58, Metadata: metadata})
|
||||
return result == "completed" && checked > 0
|
||||
}
|
||||
|
||||
func (e *Engine) setNextEnrich(t time.Time) {
|
||||
@@ -770,11 +781,11 @@ func (e *Engine) Scan(ctx context.Context) error {
|
||||
// locally on the Brain CPU or be claimed by a compute-capable Source Agent.
|
||||
// Only 256-dimensional deterministic fallback vectors are excluded because
|
||||
// mixing them with the configured embedding space would make thresholds lie.
|
||||
e.stateMu.RLock()
|
||||
lastVectorGraph := e.lastVectorGraph
|
||||
e.stateMu.RUnlock()
|
||||
periodicVectorRefresh := lastVectorGraph.IsZero() || time.Since(lastVectorGraph) >= e.Cfg.VectorGraphReevaluateInterval
|
||||
vectorLayerNeeded := !scanResult.FastPath || needsEmbeddingWork || !e.Graph.HasEdgesByOrigin(graph.VectorMathOrigin) || periodicVectorRefresh
|
||||
// Scheduled reevaluation/layout maintenance has its own loop and therefore
|
||||
// continues even while Learning is paused. Learning scans rebuild the vector
|
||||
// layer only when their own ingest/embedding work changed the knowledge space
|
||||
// or when no vector layer exists yet.
|
||||
vectorLayerNeeded := !scanResult.FastPath || needsEmbeddingWork || !e.Graph.HasEdgesByOrigin(graph.VectorMathOrigin)
|
||||
vectorReady := e.Graph.CountVectorsByDimension(256) == 0
|
||||
if e.Cfg.VectorGraphEnabled && vectorReady && vectorLayerNeeded {
|
||||
startedVectorGraph := time.Now()
|
||||
@@ -794,8 +805,9 @@ func (e *Engine) Scan(ctx context.Context) error {
|
||||
"candidate_pairs": stats.CandidatePairs, "exact_comparisons": stats.ExactComparisons,
|
||||
"orphan_pass_enabled": e.Cfg.VectorGraphOrphanPass, "orphan_focus": stats.OrphanFocus, "orphan_links": stats.OrphanLinks,
|
||||
"orphan_exact_comparisons": stats.OrphanStats.ExactComparisons, "orphan_candidate_pairs": stats.OrphanStats.CandidatePairs,
|
||||
"position_updates": stats.PositionUpdates, "layout_enabled": e.Cfg.VectorGraphLayout, "layout_relaxation": e.Cfg.VectorGraphRelaxLayout, "periodic_refresh": periodicVectorRefresh, "reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(),
|
||||
"position_updates": stats.PositionUpdates, "layout_enabled": e.Cfg.VectorGraphLayout, "layout_relaxation_configured": e.Cfg.VectorGraphRelaxLayout, "layout_mode": execution.LayoutMode, "layout_applied": execution.LayoutDue && stats.PositionUpdates > 0, "periodic_refresh": false, "maintenance_owned": true, "reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(),
|
||||
"agent_offloaded": execution.Offloaded, "agent_id": execution.AgentID, "agent_compute_ms": execution.ComputeMS, "agent_fallback_reason": execution.FallbackReason,
|
||||
"speed_mode": e.SpeedModeEnabled(), "cpu_workers": e.vectorPrimaryConfig(execution.LayoutDue).Workers,
|
||||
"duration_ms": time.Since(startedVectorGraph).Milliseconds(),
|
||||
}, vectorMutations)})
|
||||
}
|
||||
@@ -993,7 +1005,9 @@ func (e *Engine) Query(ctx context.Context, q string) (model.QueryResponse, erro
|
||||
for i, h := range hits {
|
||||
nodeIDs = append(nodeIDs, h.NodeID)
|
||||
e.Broker.Publish(model.Activity{Type: "node.activated", Source: "brain", Phase: "retrieval", Query: q, NodeIDs: []string{h.NodeID}, Message: fmt.Sprintf("Treffer %d · %.0f%% · %s", i+1, h.Score*100, h.Label), Strength: math.Max(.25, h.Score), Metadata: queryMeta(nil)})
|
||||
time.Sleep(55 * time.Millisecond)
|
||||
if !e.SpeedModeEnabled() {
|
||||
time.Sleep(55 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
edgeIDs := e.Graph.ConnectingEdges(nodeIDs)
|
||||
if len(edgeIDs) > 0 {
|
||||
@@ -1156,7 +1170,7 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
|
||||
e.stateMu.Unlock()
|
||||
e.Broker.Publish(model.Activity{Type: "think.started", Source: "brain", Phase: "association", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Verwandtschaft wird geprüft · %.0f%% semantische Nähe", sim*100), Strength: .88, Metadata: map[string]any{"trigger": trigger, "semantic_similarity": sim, "source_label": a.Label, "target_label": b.Label, "model": e.Cfg.ChatModel, "candidate_comparisons": comparisons, "exact_comparisons": comparisons, "coarse_comparisons": coarseComparisons, "indexed_nodes": indexedNodes, "candidate_pool": candidatePool, "processing_mode": processingMode, "candidate_source": candidateSource}})
|
||||
|
||||
system := "Du führst ausschließlich eine Relationserkennung für einen Wissensgraphen durch. Analysiere zwei interne Wissenseinträge, erfinde keine Fakten und entscheide, ob eine belastbare Beziehung besteht. Schreibe keinen Artikel und keine technische Synthese. Wenn externe Fakten zur Relationsentscheidung fehlen, setze needs_research=true. Gib ausschließlich JSON nach Schema zurück."
|
||||
system := "Du führst ausschließlich eine Relationserkennung für einen Wissensgraphen durch. Analysiere zwei interne Wissenseinträge, erfinde keine Fakten und entscheide, ob eine belastbare Beziehung besteht. Schreibe keinen Artikel und keine technische Synthese. Bei sehr hoher semantischer Nähe und same_topic/related_to ist normalerweise keine Webrecherche nötig. Wenn externe Fakten für eine kausale, abhängige, widersprüchliche oder zeitkritische Relationsentscheidung fehlen, setze needs_research=true. research_query darf ausschließlich fachliche Begriffe und sichtbare Titel enthalten, niemals interne Node-IDs, Hashes oder Datenbankkennungen. Gib ausschließlich JSON nach Schema zurück."
|
||||
var decision model.RelationDecision
|
||||
if err := e.Ollama.ChatJSON(ctx, system, relationContext(a, b, sim), relationSchema(), &decision); err != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "think.failed", Source: "brain", Phase: "inference", NodeIDs: []string{a.ID, b.ID}, Message: "Qwen-Beziehungsanalyse ist fehlgeschlagen; es wurde nichts gespeichert", Strength: .35, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "model": e.Cfg.ChatModel}})
|
||||
@@ -1172,88 +1186,42 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
|
||||
decision.ResearchQuery = ""
|
||||
}
|
||||
if decision.NeedsResearch && e.ResearchEnabledForRuntime() && strings.TrimSpace(decision.ResearchQuery) != "" {
|
||||
decision.ResearchQuery = sanitizeSearchQuerySiteFilters(decision.ResearchQuery)
|
||||
if strings.TrimSpace(decision.ResearchQuery) == "" {
|
||||
decision.NeedsResearch = false
|
||||
}
|
||||
}
|
||||
if decision.NeedsResearch && e.ResearchEnabledForRuntime() && strings.TrimSpace(decision.ResearchQuery) != "" {
|
||||
researchID := newResearchRunID("relation-research", decision.ResearchQuery)
|
||||
researchID := newResearchRunID("relation-research-v10", decision.ResearchQuery)
|
||||
researchStarted := time.Now()
|
||||
startMetadata := map[string]any{"trigger": trigger, "research_id": researchID, "research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label, "animation_min_ms": 2000}
|
||||
e.Broker.Publish(model.Activity{Type: "research.started", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · SearXNG durchsucht externe Quellen", Strength: .9, Metadata: startMetadata})
|
||||
|
||||
lease, reused, dedupeErr := e.beginResearchIntent(ctx, "relation", decision.ResearchQuery)
|
||||
var results []model.ResearchResult
|
||||
var diagnostic research.Diagnostic
|
||||
var researchErr error
|
||||
if dedupeErr != nil {
|
||||
researchErr = dedupeErr
|
||||
} else if !lease.owner {
|
||||
results = cloneResearchResults(reused)
|
||||
metadata := mergeResearchMetadata(startMetadata, map[string]any{"similarity": lease.similarity, "reused_results": len(results), "dedupe_threshold": e.Cfg.ResearchDedupeThreshold})
|
||||
for key, value := range researchDedupeLeaseMetadata(lease) {
|
||||
metadata[key] = value
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "research.deduplicated", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Semantisch gleiche Relationsrecherche wurde wiederverwendet · %d Treffer", len(results)), Strength: .76, Metadata: metadata})
|
||||
} else {
|
||||
// Relation research intentionally considers more than the old four
|
||||
// snippets. The full article pipeline remains the final quality gate.
|
||||
relationResultLimit := 8
|
||||
if e.Cfg.ArticleResearchResults > relationResultLimit {
|
||||
relationResultLimit = e.Cfg.ArticleResearchResults
|
||||
}
|
||||
if relationResultLimit > 12 {
|
||||
relationResultLimit = 12
|
||||
}
|
||||
researchErr = e.withSharedResearchWork(ctx, "searxng.relation_search", func() error {
|
||||
var searchErr error
|
||||
results, diagnostic, searchErr = e.Research.SearchDetailed(ctx, decision.ResearchQuery, relationResultLimit)
|
||||
return searchErr
|
||||
})
|
||||
e.completeResearchIntent(lease, results, researchErr)
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "research.started", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · Relationsrecherche prüft externe Volltextbelege einzeln", Strength: .9, Metadata: map[string]any{"trigger": trigger, "research_id": researchID, "research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label, "evidence_gate": "relation-fulltext-v10", "animation_min_ms": 2000}})
|
||||
|
||||
acceptedEvidence, researchMetadata, researchErr := e.collectRelationResearchEvidence(ctx, trigger, a, b, decision)
|
||||
researchMetadata["research_id"] = researchID
|
||||
researchMetadata["duration_ms"] = time.Since(researchStarted).Milliseconds()
|
||||
if researchErr != nil {
|
||||
metadata := mergeResearchMetadata(startMetadata, researchDiagnosticMetadata(diagnostic))
|
||||
metadata["error"] = researchErr.Error()
|
||||
metadata["duration_ms"] = time.Since(researchStarted).Milliseconds()
|
||||
slog.Warn("research failed", "query", decision.ResearchQuery, "base_url", diagnostic.BaseURL, "kind", diagnostic.ErrorKind, "http_status", diagnostic.HTTPStatus, "duration_ms", diagnostic.DurationMS, "error", researchErr)
|
||||
e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "SearXNG-Recherche ist fehlgeschlagen", Strength: .35, Metadata: metadata})
|
||||
researchMetadata["error"] = researchErr.Error()
|
||||
decision.Related = false
|
||||
decision.Confidence = math.Min(decision.Confidence, math.Max(0, e.Cfg.RelationThreshold-.01))
|
||||
decision.Explanation = strings.TrimSpace(decision.Explanation + " · erforderliche externe Relationsrecherche fehlgeschlagen")
|
||||
e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Relationsrecherche ist fehlgeschlagen; die beweisabhängige Relation wird nicht übernommen", Strength: .35, Metadata: researchMetadata})
|
||||
} else {
|
||||
allowedResults := e.filterResearchEvidenceForThinking(results, unique(append(append([]string{}, a.Categories...), b.Categories...)))
|
||||
resultMetadata := mergeResearchMetadata(researchEventMetadata(trigger, researchID, decision.ResearchQuery, allowedResults, time.Since(researchStarted)), researchDiagnosticMetadata(diagnostic))
|
||||
resultMetadata["unfiltered_result_count"] = len(results)
|
||||
resultMetadata["source_filter_rejected_count"] = len(results) - len(allowedResults)
|
||||
resultMetadata["deduplicated"] = !lease.owner
|
||||
message := fmt.Sprintf("SearXNG hat %d durch den Thinking-Filter erlaubte Webquellen geliefert", len(allowedResults))
|
||||
if len(allowedResults) == 0 {
|
||||
message = "SearXNG-Treffer lagen außerhalb des wirksamen Thinking-Quellenfilters"
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "research.results", Source: "searxng", Phase: "research-results", NodeIDs: []string{a.ID, b.ID}, Message: message, Strength: .92, Metadata: resultMetadata})
|
||||
if len(allowedResults) > 0 {
|
||||
// Keep relation-search hits in memory until the relation itself has
|
||||
// passed the second model review. Rejected relation research must not
|
||||
// permanently inflate the graph with unused external nodes.
|
||||
researchResults = allowedResults
|
||||
researchResults = acceptedEvidence
|
||||
e.Broker.Publish(model.Activity{Type: "research.results", Source: "brain", Phase: "research-results", NodeIDs: []string{a.ID, b.ID}, Message: relationResearchResultMessage(researchMetadata), Strength: .82, Metadata: researchMetadata})
|
||||
if len(acceptedEvidence) == 0 {
|
||||
decision.Related = false
|
||||
decision.Confidence = math.Min(decision.Confidence, math.Max(0, e.Cfg.RelationThreshold-.01))
|
||||
decision.Explanation = strings.TrimSpace(decision.Explanation + " · erforderliche externe Relationsrecherche lieferte keinen belastbaren Volltextbeleg")
|
||||
e.Broker.Publish(model.Activity{Type: "think.research.insufficient", Source: "brain", Phase: "research-routing", NodeIDs: []string{a.ID, b.ID}, Message: "Die Relation benötigte externe Fakten, aber kein Suchtreffer bestand die Volltext-Evidenzprüfung", Strength: .4, Metadata: researchMetadata})
|
||||
} else {
|
||||
var reviewed model.RelationDecision
|
||||
reviewSystem := "Bewerte die Beziehung erneut anhand der zwei internen Wissenseinträge und der beigefügten Web-Suchergebnisse. Suchtreffer sind Hinweise, keine garantierten Fakten. Erfinde nichts, kennzeichne verbleibende Unsicherheit und gib ausschließlich JSON nach Schema zurück."
|
||||
if err := e.Ollama.ChatJSON(ctx, reviewSystem, relationContextWithResearch(a, b, sim, allowedResults), relationSchema(), &reviewed); err != nil {
|
||||
slog.Warn("research review failed; keeping pre-research decision", "error", err)
|
||||
reviewSystem := "Bewerte die Beziehung erneut anhand der zwei internen Wissenseinträge und ausschließlich der beigefügten, einzeln geprüften Volltextbelege. Die Belege haben bereits Topic-, Quellenqualitäts- und Volltext-Relevanz-Gates bestanden. Erfinde nichts. Interne Node-IDs sind keine Websuchbegriffe. Wenn die Relation weiterhin nicht belastbar ist, setze related=false. Gib ausschließlich JSON nach Schema zurück."
|
||||
if err := e.Ollama.ChatJSON(ctx, reviewSystem, relationContextWithResearch(a, b, sim, acceptedEvidence), relationSchema(), &reviewed); err != nil {
|
||||
slog.Warn("relation evidence review failed; rejecting evidence-dependent relation", "error", err)
|
||||
researchResults = nil
|
||||
decision.Related = false
|
||||
decision.Confidence = math.Min(decision.Confidence, math.Max(0, e.Cfg.RelationThreshold-.01))
|
||||
decision.Explanation = strings.TrimSpace(decision.Explanation + " · zweiter Relationsreview mit geprüfter Evidenz fehlgeschlagen")
|
||||
} else {
|
||||
decision = reviewed
|
||||
}
|
||||
}
|
||||
// Always close the relation-research lifecycle explicitly. Older code
|
||||
// only emitted research.ingested when at least one source passed the
|
||||
// Thinking filter, leaving zero-result and filtered-out searches stuck
|
||||
// as "running" forever in the analysis history.
|
||||
completedMetadata := mergeResearchMetadata(resultMetadata, map[string]any{
|
||||
"duration_ms": time.Since(researchStarted).Milliseconds(),
|
||||
"accepted_count": len(allowedResults),
|
||||
"result": "completed",
|
||||
})
|
||||
e.Broker.Publish(model.Activity{Type: "research.completed", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Relationsrecherche beendet · %d verwendbare Treffer", len(allowedResults)), Strength: .72, Metadata: completedMetadata})
|
||||
researchMetadata["result"] = "completed"
|
||||
e.Broker.Publish(model.Activity{Type: "research.completed", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: relationResearchResultMessage(researchMetadata), Strength: .72, Metadata: researchMetadata})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1304,11 +1272,18 @@ func (e *Engine) addResearch(ctx context.Context, a, b model.Node, results []mod
|
||||
refs := researchGraphRefs{}
|
||||
for _, r := range results {
|
||||
id := graph.ID("external", r.URL)
|
||||
n := model.Node{ID: id, Kind: "external", Label: r.Title, Summary: clamp(r.Content, 700), Status: "research", Origin: "research", ExternalID: r.URL, URI: r.URL, Categories: unique(append(append([]string{}, a.Categories...), b.Categories...)), Weight: .8, Metadata: map[string]any{"source": graph.SourceFromURL(r.URL), "query_pair": []string{a.ID, b.ID}}, UpdatedAt: time.Now().UTC()}
|
||||
metadata := map[string]any{"source": graph.SourceFromURL(r.URL), "query_pair": []string{a.ID, b.ID}, "validation_state": "relation_fulltext_gate_v10", "relevance": r.Relevance, "source_quality": r.SourceQuality, "source_quality_score": r.SourceQualityScore, "assessment_reason": r.AssessmentReason}
|
||||
if r.Fetched && r.Relevant {
|
||||
if relPath, contentHash, err := e.queueResearchEvidence(r); err == nil {
|
||||
metadata["evidence_path"] = relPath
|
||||
metadata["evidence_sha256"] = contentHash
|
||||
}
|
||||
}
|
||||
n := model.Node{ID: id, Kind: "external", Label: r.Title, Summary: clamp(r.Content, 700), Status: "research", Origin: "research", ExternalID: r.URL, URI: r.URL, Categories: unique(append(append([]string{}, a.Categories...), b.Categories...)), Weight: math.Max(.8, r.SourceQualityScore), Metadata: metadata, UpdatedAt: time.Now().UTC()}
|
||||
refs.Mutations.Add(e.Graph.UpsertNodeWithStats(n))
|
||||
refs.NodeIDs = append(refs.NodeIDs, id)
|
||||
for _, targetID := range []string{a.ID, b.ID} {
|
||||
edge := model.Edge{Source: id, Target: targetID, Type: "research_evidence", Origin: "research", Status: "staging", Confidence: .55, Weight: .4}
|
||||
edge := model.Edge{Source: id, Target: targetID, Type: "research_evidence", Origin: "research", Status: "staging", Confidence: math.Max(.65, r.Relevance), Weight: math.Max(.5, r.SourceQualityScore*.7), Explanation: r.AssessmentReason, Metadata: map[string]any{"validation_state": "relation_fulltext_gate_v10", "source_quality": r.SourceQuality, "relevance": r.Relevance}}
|
||||
refs.Mutations.Add(e.Graph.UpsertEdgeWithStats(edge))
|
||||
refs.EdgeIDs = append(refs.EdgeIDs, graph.EdgeID(edge.Source, edge.Target, edge.Type, edge.Origin))
|
||||
}
|
||||
@@ -1381,14 +1356,14 @@ func (e *Engine) Status() map[string]any {
|
||||
"article_research_min_relevance": e.Cfg.ArticleResearchMinRelevance, "article_research_min_quality": e.Cfg.ArticleResearchMinQuality,
|
||||
"article_research_page_max_bytes": e.Cfg.ArticleResearchPageMaxBytes, "article_research_page_max_chars": e.Cfg.ArticleResearchPageMaxChars,
|
||||
"article_research_fetch_timeout": e.Cfg.ArticleResearchFetchTimeout.String(), "article_research_allow_private": e.Cfg.ArticleResearchAllowPrivate,
|
||||
"article_language": e.Cfg.ArticleLanguage, "article_synthesis_model": e.Cfg.ArticleSynthesisModel, "article_review_model": e.Cfg.ArticleReviewModel, "article_review_repair_rounds": e.Cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v3-quality-v2", "article_research_strategy": e.Cfg.ArticleResearchStrategy, "article_effective_research_strategy": e.effectiveArticleResearchStrategy(), "article_adaptive_initial_queries": e.Cfg.ArticleAdaptiveInitialQueries, "article_adaptive_initial_fetch": e.Cfg.ArticleAdaptiveInitialFetch, "research_dedupe": e.researchDedupeStatus(),
|
||||
"article_language": e.Cfg.ArticleLanguage, "article_synthesis_model": e.Cfg.ArticleSynthesisModel, "article_review_model": e.Cfg.ArticleReviewModel, "article_review_repair_rounds": e.Cfg.ArticleReviewRepairRounds, "article_pipeline": "adaptive_generate_review/v5-quality-gate-v12", "article_research_strategy": e.Cfg.ArticleResearchStrategy, "article_effective_research_strategy": e.effectiveArticleResearchStrategy(), "article_adaptive_initial_queries": e.Cfg.ArticleAdaptiveInitialQueries, "article_adaptive_initial_fetch": e.Cfg.ArticleAdaptiveInitialFetch, "research_dedupe": e.researchDedupeStatus(),
|
||||
"scan_interval": e.Cfg.ScanInterval.String(), "knowledge_full_verify_interval": e.Cfg.KnowledgeFullVerifyInterval.String(),
|
||||
"enrich_interval": e.Cfg.EnrichInterval.String(),
|
||||
"enrich_batch_size": e.Cfg.EnrichBatchSize, "enrich_anchors": e.Cfg.EnrichAnchors,
|
||||
"processing_mode": e.RuntimeSettings().ProcessingMode, "cluster_hash_bits": e.Cfg.ClusterHashBits, "cluster_hash_tables": e.Cfg.ClusterHashTables,
|
||||
"processing_mode": e.RuntimeSettings().ProcessingMode, "speed_mode": e.RuntimeSettings().SpeedMode, "speed_cpu_tasks": e.RuntimeSettings().SpeedCPUWorkers, "speed_gpu_tasks": e.RuntimeSettings().SpeedGPUInflight, "speed_algorithm": speedModeVersion, "cluster_hash_bits": e.Cfg.ClusterHashBits, "cluster_hash_tables": e.Cfg.ClusterHashTables,
|
||||
"cluster_candidates_per_anchor": e.Cfg.ClusterCandidatesPerAnchor, "cluster_article_candidates": e.Cfg.ClusterArticleCandidates,
|
||||
"cluster_review_evidence": e.Cfg.ClusterReviewEvidence, "cluster_review_context_chars": e.Cfg.ClusterReviewContextChars, "cluster_article_batching": e.Cfg.ClusterArticleBatching,
|
||||
"vector_graph_enabled": e.Cfg.VectorGraphEnabled, "vector_graph_neighbors": e.Cfg.VectorGraphNeighbors, "vector_graph_candidates": e.Cfg.VectorGraphCandidates, "vector_graph_min_similarity": e.Cfg.VectorGraphMinSimilarity, "vector_graph_min_affinity": e.Cfg.VectorGraphMinAffinity, "vector_graph_layout": e.Cfg.VectorGraphLayout, "vector_graph_reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": e.Cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_interval": e.Cfg.VectorGraphLayoutRelaxInterval.String(), "vector_graph_layout_blend": e.Cfg.VectorGraphLayoutBlend, "vector_graph_layout_max_shift": e.Cfg.VectorGraphLayoutMaxShift, "last_vector_graph": e.lastVectorGraph, "last_vector_layout": e.lastVectorLayout,
|
||||
"vector_graph_enabled": e.Cfg.VectorGraphEnabled, "vector_graph_neighbors": e.Cfg.VectorGraphNeighbors, "vector_graph_candidates": e.Cfg.VectorGraphCandidates, "vector_graph_min_similarity": e.Cfg.VectorGraphMinSimilarity, "vector_graph_min_affinity": e.Cfg.VectorGraphMinAffinity, "vector_graph_layout": e.Cfg.VectorGraphLayout, "vector_graph_layout_mode": e.effectiveVectorLayoutMode(), "vector_graph_reevaluate_interval": e.Cfg.VectorGraphReevaluateInterval.String(), "vector_graph_relax_layout": e.Cfg.VectorGraphRelaxLayout, "vector_graph_layout_relax_effective": e.Cfg.VectorGraphRelaxLayout && !e.Cfg.VectorGraphLayout, "vector_graph_layout_relax_interval": e.Cfg.VectorGraphLayoutRelaxInterval.String(), "vector_graph_layout_blend": e.Cfg.VectorGraphLayoutBlend, "vector_graph_layout_max_shift": e.Cfg.VectorGraphLayoutMaxShift, "last_vector_graph": e.lastVectorGraph, "last_vector_layout": e.lastVectorLayout,
|
||||
"vector_graph_orphan_pass": e.Cfg.VectorGraphOrphanPass, "vector_graph_orphan_neighbors": e.Cfg.VectorGraphOrphanNeighbors, "vector_graph_orphan_candidates": e.Cfg.VectorGraphOrphanCandidates, "vector_graph_orphan_min_similarity": e.Cfg.VectorGraphOrphanMinSimilarity, "vector_graph_orphan_min_affinity": e.Cfg.VectorGraphOrphanMinAffinity,
|
||||
"vector_graph_agent_offload": e.Cfg.VectorGraphAgentOffload, "vector_graph_agent_required": e.Cfg.VectorGraphAgentRequired, "vector_graph_agent_wait": e.Cfg.VectorGraphAgentWait.String(), "thinking_vector_guided": e.Cfg.ThinkingVectorGuided,
|
||||
"research_enabled": e.ResearchEnabledForRuntime(), "chat_model": e.Cfg.ChatModel, "embedding_model": e.Cfg.EmbeddingModel,
|
||||
@@ -1475,7 +1450,10 @@ func relationContextWithResearch(a, b model.Node, sim float64, results []model.R
|
||||
}
|
||||
|
||||
func relationContext(a, b model.Node, sim float64) string {
|
||||
return fmt.Sprintf("SEMANTISCHE_NÄHE: %.4f\n\nA\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s\n\nB\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s", sim, a.ID, a.Label, strings.Join(a.Categories, ", "), a.Summary, b.ID, b.Label, strings.Join(b.Categories, ", "), b.Summary)
|
||||
// Internal IDs are intentionally omitted from the model-visible relation
|
||||
// context. They are routing metadata, not semantic search terms, and older
|
||||
// prompts occasionally copied them into public SearXNG queries.
|
||||
return fmt.Sprintf("SEMANTISCHE_NÄHE: %.4f\n\nA\nTitel: %s\nKategorien: %s\nInhalt: %s\n\nB\nTitel: %s\nKategorien: %s\nInhalt: %s", sim, a.Label, strings.Join(a.Categories, ", "), a.Summary, b.Label, strings.Join(b.Categories, ", "), b.Summary)
|
||||
}
|
||||
func relationSchema() map[string]any {
|
||||
return map[string]any{"type": "object", "properties": map[string]any{"related": map[string]any{"type": "boolean"}, "relation_type": map[string]any{"type": "string", "enum": []string{"related_to", "depends_on", "supports", "contradicts", "extends", "same_topic", "caused_by"}}, "confidence": map[string]any{"type": "number", "minimum": 0, "maximum": 1}, "explanation": map[string]any{"type": "string"}, "needs_research": map[string]any{"type": "boolean"}, "research_query": map[string]any{"type": "string"}, "topic_label": map[string]any{"type": "string"}, "keywords": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}}, "required": []string{"related", "relation_type", "confidence", "explanation", "needs_research", "research_query", "topic_label", "keywords"}}
|
||||
|
||||
Reference in New Issue
Block a user