jbergner
94dbd4ccab
All checks were successful
release-tag / release-image (push) Successful in 2m32s
258 lines
9.9 KiB
Go
258 lines
9.9 KiB
Go
package engine
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/local/glpi-neural-brain/internal/model"
|
|
"github.com/local/glpi-neural-brain/internal/research"
|
|
)
|
|
|
|
var internalNodeIDPattern = regexp.MustCompile(`(?i)\b[0-9a-f]{20,64}\b`)
|
|
|
|
// normalizeRelationResearchQuery prevents internal graph identifiers and generic
|
|
// bookkeeping language from leaking into public Web searches. If the planner's
|
|
// query does not contain a meaningful term from either source label/topic, the
|
|
// Brain deterministically rebuilds it from human-readable source labels.
|
|
func normalizeRelationResearchQuery(a, b model.Node, decision model.RelationDecision) (string, bool, string) {
|
|
original := strings.TrimSpace(sanitizeSearchQuerySiteFilters(decision.ResearchQuery))
|
|
if original == "" {
|
|
return "", false, "empty"
|
|
}
|
|
labels := strings.TrimSpace(strings.Join([]string{a.Label, b.Label, decision.TopicLabel}, " "))
|
|
anchorTerms := researchTerms(labels)
|
|
queryTerms := researchTerms(original)
|
|
matched := 0
|
|
for term := range anchorTerms {
|
|
if queryTerms[term] {
|
|
matched++
|
|
}
|
|
}
|
|
containsInternalID := (strings.TrimSpace(a.ID) != "" && strings.Contains(original, a.ID)) || (strings.TrimSpace(b.ID) != "" && strings.Contains(original, b.ID)) || internalNodeIDPattern.MatchString(original)
|
|
genericOnly := len(anchorTerms) > 0 && matched == 0
|
|
if !containsInternalID && !genericOnly {
|
|
return original, false, ""
|
|
}
|
|
|
|
relationIntent := map[string]string{
|
|
"same_topic": "Gemeinsamkeiten Unterschiede fachliche Einordnung",
|
|
"related_to": "fachlicher Zusammenhang Abgrenzung",
|
|
"depends_on": "Abhängigkeit Voraussetzung Zusammenhang",
|
|
"supports": "Unterstützung Zusammenhang Nachweis",
|
|
"contradicts": "Widerspruch Unterschiede Nachweis",
|
|
"extends": "Erweiterung Zusammenhang Abgrenzung",
|
|
"caused_by": "Ursache Zusammenhang Nachweis",
|
|
}[safeRelation(decision.RelationType)]
|
|
if relationIntent == "" {
|
|
relationIntent = "fachlicher Zusammenhang Nachweis"
|
|
}
|
|
parts := []string{strings.TrimSpace(a.Label), strings.TrimSpace(b.Label), strings.TrimSpace(decision.TopicLabel), relationIntent, "offizielle Dokumentation"}
|
|
rebuilt := strings.TrimSpace(strings.Join(unique(parts), " "))
|
|
reason := "missing_topic_anchor"
|
|
if containsInternalID {
|
|
reason = "internal_node_id_removed"
|
|
}
|
|
return rebuilt, true, reason
|
|
}
|
|
|
|
func relationResearchQuestion(a, b model.Node, decision model.RelationDecision, query string) model.ResearchQuestion {
|
|
question := strings.TrimSpace(strings.Join([]string{
|
|
"Prüfe den fachlichen Zusammenhang zwischen",
|
|
a.Label,
|
|
"und",
|
|
b.Label + ".",
|
|
"Zu bewertende Relation:",
|
|
safeRelation(decision.RelationType) + ".",
|
|
strings.TrimSpace(decision.Explanation),
|
|
}, " "))
|
|
return model.ResearchQuestion{
|
|
GapID: "relation-evidence",
|
|
Question: question,
|
|
Critical: true,
|
|
ExpectActionable: false,
|
|
QueriesDE: []string{query},
|
|
}
|
|
}
|
|
|
|
// collectRelationResearchEvidence performs a strict, per-source evidence path.
|
|
// Search snippets remain ephemeral. Only fetched full text that passes the
|
|
// deterministic topic guard, source filter and full-text relevance/quality gate
|
|
// is returned to the relation reviewer. Nothing is written to the graph here.
|
|
func (e *Engine) collectRelationResearchEvidence(ctx context.Context, trigger string, a, b model.Node, decision model.RelationDecision) ([]model.ResearchResult, map[string]any, error) {
|
|
query, rebuilt, rebuildReason := normalizeRelationResearchQuery(a, b, decision)
|
|
metadata := map[string]any{
|
|
"trigger": trigger,
|
|
"research_query": query,
|
|
"source_label": a.Label,
|
|
"target_label": b.Label,
|
|
"query_rebuilt": rebuilt,
|
|
"query_rebuild_reason": rebuildReason,
|
|
}
|
|
if query == "" {
|
|
return nil, metadata, nil
|
|
}
|
|
if rebuilt && e.Broker != nil {
|
|
e.Broker.Publish(model.Activity{Type: "think.research.query.rebuilt", Source: "brain", Phase: "research-routing", NodeIDs: []string{a.ID, b.ID}, Message: "Relationsrecherche wurde aus den sichtbaren Themenbegriffen neu aufgebaut; interne Node-IDs werden nicht ins Web gesendet", Strength: .56, Metadata: map[string]any{
|
|
"trigger": trigger, "original_query": decision.ResearchQuery, "research_query": query, "reason": rebuildReason, "source_label": a.Label, "target_label": b.Label,
|
|
}})
|
|
}
|
|
|
|
question := relationResearchQuestion(a, b, decision, query)
|
|
lease, reused, err := e.beginResearchIntent(ctx, "relation-v10", query)
|
|
if err != nil {
|
|
return nil, metadata, err
|
|
}
|
|
if !lease.owner {
|
|
validated, rejected := e.revalidateReusableResearchEvidence(ctx, question, reused)
|
|
metadata["deduplicated"] = true
|
|
metadata["reused_results"] = len(reused)
|
|
metadata["reused_rejected"] = rejected
|
|
metadata["accepted_count"] = len(validated)
|
|
return validated, metadata, nil
|
|
}
|
|
|
|
resultLimit := e.Cfg.ArticleResearchResults
|
|
if resultLimit < 8 {
|
|
resultLimit = 8
|
|
}
|
|
if resultLimit > 12 {
|
|
resultLimit = 12
|
|
}
|
|
var results []model.ResearchResult
|
|
var diagnostic research.Diagnostic
|
|
searchStarted := time.Now()
|
|
searchErr := e.withSharedResearchWork(ctx, "searxng.relation_search_v10", func() error {
|
|
var inner error
|
|
results, diagnostic, inner = e.Research.SearchDetailed(ctx, query, resultLimit)
|
|
return inner
|
|
})
|
|
if searchErr != nil {
|
|
e.completeResearchIntent(lease, nil, searchErr)
|
|
metadata = mergeResearchMetadata(metadata, researchDiagnosticMetadata(diagnostic))
|
|
metadata["duration_ms"] = time.Since(searchStarted).Milliseconds()
|
|
return nil, metadata, searchErr
|
|
}
|
|
for i := range results {
|
|
results[i].Query = query
|
|
results[i].Round = 1
|
|
}
|
|
metadata["unfiltered_result_count"] = len(results)
|
|
categories := unique(append(append([]string{}, a.Categories...), b.Categories...))
|
|
allowed := e.filterResearchEvidenceForThinking(results, categories)
|
|
metadata["source_filter_rejected_count"] = len(results) - len(allowed)
|
|
metadata["source_filter_allowed_count"] = len(allowed)
|
|
if len(allowed) == 0 {
|
|
e.completeResearchIntent(lease, nil, nil)
|
|
metadata["accepted_count"] = 0
|
|
return nil, metadata, nil
|
|
}
|
|
|
|
// Snippet selection is deterministic/heuristic; semantic model budget is
|
|
// spent only after full text has been fetched.
|
|
ranked := rankResearchCandidatesHeuristic(question, allowed, false)
|
|
fetchLimit := e.Cfg.ArticleResearchFetchResults
|
|
if fetchLimit < 1 {
|
|
fetchLimit = 4
|
|
}
|
|
if fetchLimit > 4 {
|
|
fetchLimit = 4
|
|
}
|
|
prefetchMin := math.Max(.35, e.Cfg.ArticleResearchPrefetchMinRelevance)
|
|
finalMin := math.Max(.60, e.Cfg.ArticleResearchMinRelevance)
|
|
minQuality := math.Max(.55, e.Cfg.ArticleResearchMinQuality)
|
|
selection := selectResearchCandidates(question, ranked, map[string]bool{}, fetchLimit, 1, prefetchMin, finalMin, minQuality)
|
|
metadata["snippet_gate_selected_count"] = len(selection.Selected)
|
|
metadata["snippet_gate_rejected_count"] = selection.GateRejected
|
|
metadata["authoritative_exploration_selected_count"] = countSelectedMode(selection.Decisions, "authoritative_exploration")
|
|
if len(selection.Selected) == 0 {
|
|
e.completeResearchIntent(lease, nil, nil)
|
|
metadata["accepted_count"] = 0
|
|
return nil, metadata, nil
|
|
}
|
|
|
|
fetched := make([]model.ResearchResult, 0, len(selection.Selected))
|
|
fetchFailures := 0
|
|
for _, candidate := range selection.Selected {
|
|
var page research.FetchedPage
|
|
fetchErr := e.withSharedResearchWork(ctx, "web.relation_fetch_v10", func() error {
|
|
var inner error
|
|
page, _, inner = e.Research.FetchPage(ctx, candidate.Result.URL, research.FetchOptions{
|
|
MaxBytes: e.Cfg.ArticleResearchPageMaxBytes, MaxChars: e.Cfg.ArticleResearchPageMaxChars,
|
|
Timeout: e.Cfg.ArticleResearchFetchTimeout, AllowPrivate: e.Cfg.ArticleResearchAllowPrivate,
|
|
})
|
|
return inner
|
|
})
|
|
if fetchErr != nil {
|
|
fetchFailures++
|
|
continue
|
|
}
|
|
item := candidate.Result
|
|
item.URL = nonempty(canonicalResearchURL(page.URL), page.URL)
|
|
if strings.TrimSpace(page.Title) != "" {
|
|
item.Title = page.Title
|
|
}
|
|
item.Content = page.Content
|
|
item.ContentType = page.ContentType
|
|
item.Fetched = true
|
|
item.Query = query
|
|
item.Round = 1
|
|
fetched = append(fetched, item)
|
|
}
|
|
metadata["fetched_count"] = len(fetched)
|
|
metadata["fetch_failed_count"] = fetchFailures
|
|
if len(fetched) == 0 {
|
|
e.completeResearchIntent(lease, nil, nil)
|
|
metadata["accepted_count"] = 0
|
|
return nil, metadata, nil
|
|
}
|
|
|
|
assessed := e.rankResearchCandidates(ctx, question, fetched, true)
|
|
accepted := make([]model.ResearchResult, 0, len(assessed))
|
|
fulltextRejected := 0
|
|
for _, candidate := range assessed {
|
|
assessment := candidate.Assessment
|
|
strict := candidate.TopicGuardPassed && assessment.Relevant && assessment.Relevance >= finalMin && assessment.SourceQualityScore >= minQuality
|
|
if !strict {
|
|
fulltextRejected++
|
|
continue
|
|
}
|
|
item := candidate.Result
|
|
item.Relevant = true
|
|
item.Relevance = assessment.Relevance
|
|
item.SourceQuality = assessment.SourceQuality
|
|
item.SourceQualityScore = assessment.SourceQualityScore
|
|
item.Actionable = assessment.Actionable
|
|
item.CoveredGapIDs = unique(append(assessment.CoveredGapIDs, question.GapID))
|
|
item.AssessmentReason = assessment.Reason
|
|
accepted = append(accepted, item)
|
|
}
|
|
// Stable ordering makes relation-review prompts and dedupe cache deterministic.
|
|
sort.SliceStable(accepted, func(i, j int) bool {
|
|
if accepted[i].Relevance != accepted[j].Relevance {
|
|
return accepted[i].Relevance > accepted[j].Relevance
|
|
}
|
|
return accepted[i].URL < accepted[j].URL
|
|
})
|
|
metadata["fulltext_rejected_count"] = fulltextRejected
|
|
metadata["accepted_count"] = len(accepted)
|
|
metadata["accepted_titles"] = researchTitles(accepted)
|
|
metadata["minimum_relevance"] = finalMin
|
|
metadata["minimum_quality"] = minQuality
|
|
metadata["duration_ms"] = time.Since(searchStarted).Milliseconds()
|
|
e.completeResearchIntent(lease, accepted, nil)
|
|
return accepted, metadata, nil
|
|
}
|
|
|
|
func relationResearchResultMessage(metadata map[string]any) string {
|
|
accepted := 0
|
|
if value, ok := metadata["accepted_count"].(int); ok {
|
|
accepted = value
|
|
}
|
|
return fmt.Sprintf("Relationsrecherche beendet · %d einzeln geprüfte Volltextbelege", accepted)
|
|
}
|