package engine import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "log/slog" "os" "path/filepath" "sort" "strings" "time" "github.com/local/glpi-neural-brain/internal/graph" "github.com/local/glpi-neural-brain/internal/model" ) type articleFreshnessDecision struct { Required bool Reason string Queries []string } func (e *Engine) effectiveArticleResearchStrategy() string { strategy := strings.ToLower(strings.TrimSpace(e.Cfg.ArticleResearchStrategy)) if strategy == "" || strategy == "auto" { if e.RuntimeSettings().ProcessingMode == "clustered" { return "adaptive" } return "always" } switch strategy { case "always", "adaptive", "review_only": return strategy default: return "adaptive" } } // detectArticleFreshnessNeed is deliberately deterministic and cheap. It only // identifies topics whose correctness commonly depends on time, version or // operational state. Ambiguous/static topics are left to the author/reviewer // path instead of paying for speculative Web research up front. func detectArticleFreshnessNeed(plan model.ArticlePlanDecision, relation model.RelationDecision, sources []articleSource) articleFreshnessDecision { parts := []string{plan.ExpectedValue, plan.Reason, plan.ResearchQuery, relation.TopicLabel} parts = append(parts, plan.MissingInformation...) parts = append(parts, relation.Keywords...) for _, source := range sources { parts = append(parts, source.Node.Label) } text := " " + strings.ToLower(strings.Join(parts, " \n ")) + " " markers := []struct { needle string reason string }{ {" aktuell", "explizit aktuelle Information"}, {" neueste", "explizit neueste Information"}, {" heute", "tagesaktuelle Information"}, {" derzeit", "gegenwärtiger Zustand"}, {" momentan", "gegenwärtiger Zustand"}, {" latest", "latest/current wording"}, {" currently", "latest/current wording"}, {" aktuelle version", "aktueller Versionsstand"}, {" neueste version", "aktueller Versionsstand"}, {" latest version", "aktueller Versionsstand"}, {" current version", "aktueller Versionsstand"}, {" unterstützte version", "Support-Matrix"}, {" unterstützten version", "Support-Matrix"}, {" supported version", "Support-Matrix"}, {" support matrix", "Support-Matrix"}, {" aktueller release", "Release-Stand"}, {" latest release", "Release-Stand"}, {" release notes", "Release-Stand"}, {" eol", "End-of-Life-/Supportstatus"}, {" end of life", "End-of-Life-/Supportstatus"}, {" end-of-life", "End-of-Life-/Supportstatus"}, {" supportstatus", "End-of-Life-/Supportstatus"}, {" cve-", "CVE-/Sicherheitslage"}, {" security advisory", "Security Advisory"}, {" aktuelles advisory", "Security Advisory"}, {" aktuelle sicherheitslücke", "Sicherheitslage"}, {" current vulnerability", "Sicherheitslage"}, {" aktueller patch", "Patchstand"}, {" patchstand", "Patchstand"}, {" security patch", "Patchstand"}, {" firmwareversion", "Firmwarestand"}, {" aktuelle firmware", "Firmwarestand"}, {" aktueller preis", "Preis-/Lizenzstand"}, {" pricing", "Preis-/Lizenzstand"}, {" lizenzkosten", "Preis-/Lizenzstand"}, {" aktuelle lizenz", "Lizenzstand"}, {" outage", "Live-Betriebsstatus"}, {" aktuelle störung", "Live-Betriebsstatus"}, {" status page", "Live-Betriebsstatus"}, {" laufender vorfall", "aktuelles Ereignis"}, {" ongoing incident", "aktuelles Ereignis"}, } for _, marker := range markers { if strings.Contains(text, marker.needle) { queries := freshnessQueries(plan, relation, sources) return articleFreshnessDecision{Required: true, Reason: marker.reason, Queries: queries} } } return articleFreshnessDecision{} } func freshnessQueries(plan model.ArticlePlanDecision, relation model.RelationDecision, sources []articleSource) []string { out := make([]string, 0, 4) if q := strings.TrimSpace(plan.ResearchQuery); q != "" { out = append(out, q) } for _, gap := range plan.MissingInformation { gap = strings.TrimSpace(gap) if gap != "" && containsFreshnessLanguage(gap) { out = append(out, gap) } } topic := strings.TrimSpace(plan.ExpectedValue) if topic == "" { topic = strings.TrimSpace(relation.TopicLabel) } if topic == "" && len(sources) > 0 { topic = strings.TrimSpace(sources[0].Node.Label) } if topic != "" { out = append(out, topic+" aktuelle offizielle Dokumentation Version Support") out = append(out, topic+" current official documentation version support") } return unique(out) } func containsFreshnessLanguage(value string) bool { lower := " " + strings.ToLower(value) + " " for _, marker := range []string{" aktuell", " neueste", " heute", " derzeit", " momentan", " latest", " currently", " aktuelle version", " neueste version", " latest version", " current version", " unterstützte version", " supported version", " support matrix", " aktueller release", " latest release", " release notes", " eol", " end of life", " end-of-life", " supportstatus", " cve-", " security advisory", " aktuelle sicherheitslücke", " current vulnerability", " aktueller patch", " patchstand", " security patch", " firmwareversion", " aktuelle firmware", " pricing", " aktueller preis", " lizenzkosten", " outage", " aktuelle störung", " status page", " laufender vorfall", " ongoing incident"} { if strings.Contains(lower, marker) { return true } } return false } func sanitizeAuthorResearchQueries(values []string, limit int) []string { if limit < 1 { return nil } out := make([]string, 0, limit) seen := map[string]bool{} for _, raw := range values { q := strings.TrimSpace(sanitizeSearchQuerySiteFilters(raw)) if q == "" { continue } key := strings.ToLower(q) if seen[key] { continue } seen[key] = true out = append(out, q) if len(out) >= limit { break } } return out } func (e *Engine) collectAdaptiveInitialResearch(ctx context.Context, trigger string, nodeIDs []string, queries []string, fetchCap int) ([]model.ResearchResult, articleResearchReport) { queries = sanitizeAuthorResearchQueries(queries, e.Cfg.ArticleAdaptiveInitialQueries) if len(queries) == 0 { return nil, articleResearchReport{} } if fetchCap < 1 { fetchCap = e.Cfg.ArticleAdaptiveInitialFetch } attemptedURLs := map[string]bool{} out := make([]model.ResearchResult, 0) report := articleResearchReport{Rounds: 1} for i, query := range queries { report.Queries++ freshnessSensitive := containsFreshnessLanguage(query) inbox := e.sourceInboxResearch(ctx, query, fetchCap, freshnessSensitive) if len(inbox) > 0 { report.InboxResults += len(inbox) report.Accepted += len(inbox) out = uniqueResearchEvidence(append(out, inbox...)) } if len(inbox) >= e.Cfg.SourceInboxMinResults || !e.ResearchEnabledForRuntime() { continue } remaining := fetchCap - len(inbox) if remaining < 1 { remaining = 1 } language := "en-US" if looksGermanResearchQuery(query) { language = "de-DE" } question := model.ResearchQuestion{GapID: fmt.Sprintf("ADAPTIVE-%d", i+1), Question: query, Critical: true, ExpectActionable: containsActionableLanguage(query)} items, stats := e.executeArticleResearchQueryForSynthesis(ctx, trigger, nodeIDs, question, query, language, 1, attemptedURLs, remaining) report.SearchResults += stats.SearchResults report.Fetched += stats.Fetched report.Accepted += stats.Accepted report.Rejected += stats.Rejected report.FetchFailed += stats.FetchFailed report.SearchFailed += stats.SearchFailed out = uniqueResearchEvidence(append(out, items...)) } return out, report } func (e *Engine) articlePipelineFingerprintIdentity() string { return fmt.Sprintf("adaptive_generate_review/v5-quality-gate-v12|topic-guard=strict-v3-relation-evidence|provenance=v2|lang=%s|author=%s|reviewer=%s|research=%s|repair=%d", articleLanguageTag(e.Cfg.ArticleLanguage), strings.TrimSpace(e.Cfg.ArticleSynthesisModel), strings.TrimSpace(e.Cfg.ArticleReviewModel), e.effectiveArticleResearchStrategy(), e.Cfg.ArticleReviewRepairRounds) } func articleWorkFingerprint(sources []articleSource, relation model.RelationDecision, research []model.ResearchResult, pipelineIdentity string) string { parts := make([]string, 0, len(sources)+4) for _, source := range sources { contentHash := sha256.Sum256([]byte(strings.TrimSpace(source.Content))) parts = append(parts, source.Node.ID+":"+hex.EncodeToString(contentHash[:])) } for _, result := range uniqueResearchEvidence(research) { content := result.Content if strings.TrimSpace(content) == "" { content = result.Snippet } contentHash := sha256.Sum256([]byte(strings.TrimSpace(content))) parts = append(parts, "research="+canonicalResearchURL(result.URL)+":"+hex.EncodeToString(contentHash[:])) } sort.Strings(parts) keywords := unique(relation.Keywords) sort.Strings(keywords) parts = append(parts, "pipeline="+strings.TrimSpace(pipelineIdentity), "relation="+safeRelation(relation.RelationType), "topic="+strings.ToLower(strings.TrimSpace(relation.TopicLabel)), "keywords="+strings.ToLower(strings.Join(keywords, ",")), ) sum := sha256.Sum256([]byte(strings.Join(parts, "\x00"))) return hex.EncodeToString(sum[:]) } func (e *Engine) articleWorkFingerprintPath(fingerprint string) string { return filepath.Join(e.Cfg.DataDir, "article-work-fingerprints", strings.ToLower(strings.TrimSpace(fingerprint))+".json") } func (e *Engine) hasArticleWorkFingerprint(fingerprint string) bool { if fingerprint == "" || strings.TrimSpace(e.Cfg.DataDir) == "" { return false } path := e.articleWorkFingerprintPath(fingerprint) if e.Persistence != nil && e.Persistence.Pending(path) { return true } _, err := os.Stat(path) return err == nil } func (e *Engine) queueArticleWorkFingerprint(fingerprint, articleID string, relation model.RelationDecision) error { if fingerprint == "" || e.Persistence == nil { return nil } payload := map[string]any{ "schema": "article-work-fingerprint/v1", "fingerprint": fingerprint, "article_id": articleID, "relation_type": safeRelation(relation.RelationType), "topic_label": relation.TopicLabel, "generated_at": time.Now().UTC(), } data, err := json.MarshalIndent(payload, "", " ") if err != nil { return err } _, err = e.Persistence.QueueFile(e.articleWorkFingerprintPath(fingerprint), append(data, '\n'), 0o640) return err } func articleSourceFingerprint(sources []articleSource, plan model.ArticlePlanDecision, research []model.ResearchResult, pipelineIdentity string) string { parts := make([]string, 0, len(sources)+4) for _, source := range sources { contentHash := sha256.Sum256([]byte(strings.TrimSpace(source.Content))) parts = append(parts, source.Node.ID+":"+hex.EncodeToString(contentHash[:])) } for _, result := range uniqueResearchEvidence(research) { content := result.Content if strings.TrimSpace(content) == "" { content = result.Snippet } contentHash := sha256.Sum256([]byte(strings.TrimSpace(content))) parts = append(parts, "research="+canonicalResearchURL(result.URL)+":"+hex.EncodeToString(contentHash[:])) } sort.Strings(parts) parts = append(parts, "pipeline="+strings.TrimSpace(pipelineIdentity), "action="+plan.Action, "target="+plan.TargetArticleID, "type="+normalizeArticleType(plan.ArticleType)) sum := sha256.Sum256([]byte(strings.Join(parts, "\x00"))) return hex.EncodeToString(sum[:]) } func (e *Engine) articleFingerprintPath(fingerprint string) string { return filepath.Join(e.Cfg.DataDir, "article-fingerprints", strings.ToLower(strings.TrimSpace(fingerprint))+".json") } func (e *Engine) hasArticleSourceFingerprint(fingerprint string) bool { if fingerprint == "" || strings.TrimSpace(e.Cfg.DataDir) == "" { return false } path := e.articleFingerprintPath(fingerprint) if e.Persistence != nil && e.Persistence.Pending(path) { return true } _, err := os.Stat(path) return err == nil } func (e *Engine) queueArticleSourceFingerprint(fingerprint, articleID string, plan model.ArticlePlanDecision) error { if fingerprint == "" || e.Persistence == nil { return nil } payload := map[string]any{ "schema": "article-source-fingerprint/v1", "fingerprint": fingerprint, "article_id": articleID, "action": plan.Action, "target_article_id": plan.TargetArticleID, "article_type": normalizeArticleType(plan.ArticleType), "generated_at": time.Now().UTC(), } data, err := json.MarshalIndent(payload, "", " ") if err != nil { return err } _, err = e.Persistence.QueueFile(e.articleFingerprintPath(fingerprint), append(data, '\n'), 0o640) return err } // persistResearchMaterial keeps fetched pages reusable/auditable without // materialising them as graph nodes. Graph materialisation happens only after // the reviewer actually cites a source for a supported claim. func (e *Engine) persistResearchMaterial(results []model.ResearchResult) []string { paths := make([]string, 0, len(results)) for _, result := range results { path, _, err := e.queueResearchEvidence(result) if err == nil && path != "" { paths = append(paths, path) } } return unique(paths) } func (e *Engine) materializeGroundedResearchEvidence(articleID string, sources []articleSource, results []model.ResearchResult) ([]string, graph.MutationStats) { var stats graph.MutationStats if len(results) == 0 { return nil, stats } categories := categoriesFromArticleSources(sources) ids := make([]string, 0, len(results)) usedURLs := make([]string, 0, len(results)) usedInboxIDs := make([]string, 0, len(results)) for _, result := range results { usedURLs = append(usedURLs, result.URL) if strings.TrimSpace(result.SourceInboxID) != "" { usedInboxIDs = append(usedInboxIDs, result.SourceInboxID) } id := graph.ID("external", result.URL) path, contentHash, err := e.queueResearchEvidence(result) if err != nil { continue } node := researchResultNode(id, result, path, contentHash, categories) if node.Metadata == nil { node.Metadata = map[string]any{} } node.Metadata["validation_state"] = "grounded" node.Metadata["grounded_article_ids"] = []string{articleID} stats.Add(e.Graph.UpsertNodeWithStats(node)) ids = append(ids, id) } if e.SourceInbox != nil { var markErr error if len(usedInboxIDs) > 0 { markErr = e.SourceInbox.MarkUsedByIDs(context.Background(), usedInboxIDs) } else if len(usedURLs) > 0 { // Legacy/fallback evidence created before source_inbox_id was propagated. markErr = e.SourceInbox.MarkUsed(context.Background(), usedURLs) } if markErr != nil { slog.Warn("source inbox grounding status update failed", "article_id", articleID, "error", markErr) } } return unique(ids), stats }