jbergner
94dbd4ccab
All checks were successful
release-tag / release-image (push) Successful in 2m32s
361 lines
14 KiB
Go
361 lines
14 KiB
Go
package engine
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/local/glpi-neural-brain/internal/graph"
|
|
"github.com/local/glpi-neural-brain/internal/model"
|
|
)
|
|
|
|
type articleFreshnessDecision struct {
|
|
Required bool
|
|
Reason string
|
|
Queries []string
|
|
}
|
|
|
|
func (e *Engine) effectiveArticleResearchStrategy() string {
|
|
strategy := strings.ToLower(strings.TrimSpace(e.Cfg.ArticleResearchStrategy))
|
|
if strategy == "" || strategy == "auto" {
|
|
if e.RuntimeSettings().ProcessingMode == "clustered" {
|
|
return "adaptive"
|
|
}
|
|
return "always"
|
|
}
|
|
switch strategy {
|
|
case "always", "adaptive", "review_only":
|
|
return strategy
|
|
default:
|
|
return "adaptive"
|
|
}
|
|
}
|
|
|
|
// detectArticleFreshnessNeed is deliberately deterministic and cheap. It only
|
|
// identifies topics whose correctness commonly depends on time, version or
|
|
// operational state. Ambiguous/static topics are left to the author/reviewer
|
|
// path instead of paying for speculative Web research up front.
|
|
func detectArticleFreshnessNeed(plan model.ArticlePlanDecision, relation model.RelationDecision, sources []articleSource) articleFreshnessDecision {
|
|
parts := []string{plan.ExpectedValue, plan.Reason, plan.ResearchQuery, relation.TopicLabel}
|
|
parts = append(parts, plan.MissingInformation...)
|
|
parts = append(parts, relation.Keywords...)
|
|
for _, source := range sources {
|
|
parts = append(parts, source.Node.Label)
|
|
}
|
|
text := " " + strings.ToLower(strings.Join(parts, " \n ")) + " "
|
|
markers := []struct {
|
|
needle string
|
|
reason string
|
|
}{
|
|
{" aktuell", "explizit aktuelle Information"}, {" neueste", "explizit neueste Information"}, {" heute", "tagesaktuelle Information"},
|
|
{" derzeit", "gegenwärtiger Zustand"}, {" momentan", "gegenwärtiger Zustand"}, {" latest", "latest/current wording"}, {" currently", "latest/current wording"},
|
|
{" aktuelle version", "aktueller Versionsstand"}, {" neueste version", "aktueller Versionsstand"}, {" latest version", "aktueller Versionsstand"}, {" current version", "aktueller Versionsstand"},
|
|
{" unterstützte version", "Support-Matrix"}, {" unterstützten version", "Support-Matrix"}, {" supported version", "Support-Matrix"}, {" support matrix", "Support-Matrix"},
|
|
{" aktueller release", "Release-Stand"}, {" latest release", "Release-Stand"}, {" release notes", "Release-Stand"},
|
|
{" eol", "End-of-Life-/Supportstatus"}, {" end of life", "End-of-Life-/Supportstatus"}, {" end-of-life", "End-of-Life-/Supportstatus"}, {" supportstatus", "End-of-Life-/Supportstatus"},
|
|
{" cve-", "CVE-/Sicherheitslage"}, {" security advisory", "Security Advisory"}, {" aktuelles advisory", "Security Advisory"}, {" aktuelle sicherheitslücke", "Sicherheitslage"}, {" current vulnerability", "Sicherheitslage"},
|
|
{" aktueller patch", "Patchstand"}, {" patchstand", "Patchstand"}, {" security patch", "Patchstand"}, {" firmwareversion", "Firmwarestand"}, {" aktuelle firmware", "Firmwarestand"},
|
|
{" aktueller preis", "Preis-/Lizenzstand"}, {" pricing", "Preis-/Lizenzstand"}, {" lizenzkosten", "Preis-/Lizenzstand"}, {" aktuelle lizenz", "Lizenzstand"},
|
|
{" outage", "Live-Betriebsstatus"}, {" aktuelle störung", "Live-Betriebsstatus"}, {" status page", "Live-Betriebsstatus"}, {" laufender vorfall", "aktuelles Ereignis"}, {" ongoing incident", "aktuelles Ereignis"},
|
|
}
|
|
for _, marker := range markers {
|
|
if strings.Contains(text, marker.needle) {
|
|
queries := freshnessQueries(plan, relation, sources)
|
|
return articleFreshnessDecision{Required: true, Reason: marker.reason, Queries: queries}
|
|
}
|
|
}
|
|
return articleFreshnessDecision{}
|
|
}
|
|
|
|
func freshnessQueries(plan model.ArticlePlanDecision, relation model.RelationDecision, sources []articleSource) []string {
|
|
out := make([]string, 0, 4)
|
|
if q := strings.TrimSpace(plan.ResearchQuery); q != "" {
|
|
out = append(out, q)
|
|
}
|
|
for _, gap := range plan.MissingInformation {
|
|
gap = strings.TrimSpace(gap)
|
|
if gap != "" && containsFreshnessLanguage(gap) {
|
|
out = append(out, gap)
|
|
}
|
|
}
|
|
topic := strings.TrimSpace(plan.ExpectedValue)
|
|
if topic == "" {
|
|
topic = strings.TrimSpace(relation.TopicLabel)
|
|
}
|
|
if topic == "" && len(sources) > 0 {
|
|
topic = strings.TrimSpace(sources[0].Node.Label)
|
|
}
|
|
if topic != "" {
|
|
out = append(out, topic+" aktuelle offizielle Dokumentation Version Support")
|
|
out = append(out, topic+" current official documentation version support")
|
|
}
|
|
return unique(out)
|
|
}
|
|
|
|
func containsFreshnessLanguage(value string) bool {
|
|
lower := " " + strings.ToLower(value) + " "
|
|
for _, marker := range []string{" aktuell", " neueste", " heute", " derzeit", " momentan", " latest", " currently", " aktuelle version", " neueste version", " latest version", " current version", " unterstützte version", " supported version", " support matrix", " aktueller release", " latest release", " release notes", " eol", " end of life", " end-of-life", " supportstatus", " cve-", " security advisory", " aktuelle sicherheitslücke", " current vulnerability", " aktueller patch", " patchstand", " security patch", " firmwareversion", " aktuelle firmware", " pricing", " aktueller preis", " lizenzkosten", " outage", " aktuelle störung", " status page", " laufender vorfall", " ongoing incident"} {
|
|
if strings.Contains(lower, marker) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func sanitizeAuthorResearchQueries(values []string, limit int) []string {
|
|
if limit < 1 {
|
|
return nil
|
|
}
|
|
out := make([]string, 0, limit)
|
|
seen := map[string]bool{}
|
|
for _, raw := range values {
|
|
q := strings.TrimSpace(sanitizeSearchQuerySiteFilters(raw))
|
|
if q == "" {
|
|
continue
|
|
}
|
|
key := strings.ToLower(q)
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
out = append(out, q)
|
|
if len(out) >= limit {
|
|
break
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (e *Engine) collectAdaptiveInitialResearch(ctx context.Context, trigger string, nodeIDs []string, queries []string, fetchCap int) ([]model.ResearchResult, articleResearchReport) {
|
|
queries = sanitizeAuthorResearchQueries(queries, e.Cfg.ArticleAdaptiveInitialQueries)
|
|
if len(queries) == 0 {
|
|
return nil, articleResearchReport{}
|
|
}
|
|
if fetchCap < 1 {
|
|
fetchCap = e.Cfg.ArticleAdaptiveInitialFetch
|
|
}
|
|
attemptedURLs := map[string]bool{}
|
|
out := make([]model.ResearchResult, 0)
|
|
report := articleResearchReport{Rounds: 1}
|
|
for i, query := range queries {
|
|
report.Queries++
|
|
freshnessSensitive := containsFreshnessLanguage(query)
|
|
inbox := e.sourceInboxResearch(ctx, query, fetchCap, freshnessSensitive)
|
|
if len(inbox) > 0 {
|
|
report.InboxResults += len(inbox)
|
|
report.Accepted += len(inbox)
|
|
out = uniqueResearchEvidence(append(out, inbox...))
|
|
}
|
|
if len(inbox) >= e.Cfg.SourceInboxMinResults || !e.ResearchEnabledForRuntime() {
|
|
continue
|
|
}
|
|
remaining := fetchCap - len(inbox)
|
|
if remaining < 1 {
|
|
remaining = 1
|
|
}
|
|
language := "en-US"
|
|
if looksGermanResearchQuery(query) {
|
|
language = "de-DE"
|
|
}
|
|
question := model.ResearchQuestion{GapID: fmt.Sprintf("ADAPTIVE-%d", i+1), Question: query, Critical: true, ExpectActionable: containsActionableLanguage(query)}
|
|
items, stats := e.executeArticleResearchQueryForSynthesis(ctx, trigger, nodeIDs, question, query, language, 1, attemptedURLs, remaining)
|
|
report.SearchResults += stats.SearchResults
|
|
report.Fetched += stats.Fetched
|
|
report.Accepted += stats.Accepted
|
|
report.Rejected += stats.Rejected
|
|
report.FetchFailed += stats.FetchFailed
|
|
report.SearchFailed += stats.SearchFailed
|
|
out = uniqueResearchEvidence(append(out, items...))
|
|
}
|
|
return out, report
|
|
}
|
|
|
|
func (e *Engine) articlePipelineFingerprintIdentity() string {
|
|
return fmt.Sprintf("adaptive_generate_review/v5-quality-gate-v12|topic-guard=strict-v3-relation-evidence|provenance=v2|lang=%s|author=%s|reviewer=%s|research=%s|repair=%d", articleLanguageTag(e.Cfg.ArticleLanguage), strings.TrimSpace(e.Cfg.ArticleSynthesisModel), strings.TrimSpace(e.Cfg.ArticleReviewModel), e.effectiveArticleResearchStrategy(), e.Cfg.ArticleReviewRepairRounds)
|
|
}
|
|
|
|
func articleWorkFingerprint(sources []articleSource, relation model.RelationDecision, research []model.ResearchResult, pipelineIdentity string) string {
|
|
parts := make([]string, 0, len(sources)+4)
|
|
for _, source := range sources {
|
|
contentHash := sha256.Sum256([]byte(strings.TrimSpace(source.Content)))
|
|
parts = append(parts, source.Node.ID+":"+hex.EncodeToString(contentHash[:]))
|
|
}
|
|
for _, result := range uniqueResearchEvidence(research) {
|
|
content := result.Content
|
|
if strings.TrimSpace(content) == "" {
|
|
content = result.Snippet
|
|
}
|
|
contentHash := sha256.Sum256([]byte(strings.TrimSpace(content)))
|
|
parts = append(parts, "research="+canonicalResearchURL(result.URL)+":"+hex.EncodeToString(contentHash[:]))
|
|
}
|
|
sort.Strings(parts)
|
|
keywords := unique(relation.Keywords)
|
|
sort.Strings(keywords)
|
|
parts = append(parts,
|
|
"pipeline="+strings.TrimSpace(pipelineIdentity),
|
|
"relation="+safeRelation(relation.RelationType),
|
|
"topic="+strings.ToLower(strings.TrimSpace(relation.TopicLabel)),
|
|
"keywords="+strings.ToLower(strings.Join(keywords, ",")),
|
|
)
|
|
sum := sha256.Sum256([]byte(strings.Join(parts, "\x00")))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func (e *Engine) articleWorkFingerprintPath(fingerprint string) string {
|
|
return filepath.Join(e.Cfg.DataDir, "article-work-fingerprints", strings.ToLower(strings.TrimSpace(fingerprint))+".json")
|
|
}
|
|
|
|
func (e *Engine) hasArticleWorkFingerprint(fingerprint string) bool {
|
|
if fingerprint == "" || strings.TrimSpace(e.Cfg.DataDir) == "" {
|
|
return false
|
|
}
|
|
path := e.articleWorkFingerprintPath(fingerprint)
|
|
if e.Persistence != nil && e.Persistence.Pending(path) {
|
|
return true
|
|
}
|
|
_, err := os.Stat(path)
|
|
return err == nil
|
|
}
|
|
|
|
func (e *Engine) queueArticleWorkFingerprint(fingerprint, articleID string, relation model.RelationDecision) error {
|
|
if fingerprint == "" || e.Persistence == nil {
|
|
return nil
|
|
}
|
|
payload := map[string]any{
|
|
"schema": "article-work-fingerprint/v1",
|
|
"fingerprint": fingerprint,
|
|
"article_id": articleID,
|
|
"relation_type": safeRelation(relation.RelationType),
|
|
"topic_label": relation.TopicLabel,
|
|
"generated_at": time.Now().UTC(),
|
|
}
|
|
data, err := json.MarshalIndent(payload, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = e.Persistence.QueueFile(e.articleWorkFingerprintPath(fingerprint), append(data, '\n'), 0o640)
|
|
return err
|
|
}
|
|
|
|
func articleSourceFingerprint(sources []articleSource, plan model.ArticlePlanDecision, research []model.ResearchResult, pipelineIdentity string) string {
|
|
parts := make([]string, 0, len(sources)+4)
|
|
for _, source := range sources {
|
|
contentHash := sha256.Sum256([]byte(strings.TrimSpace(source.Content)))
|
|
parts = append(parts, source.Node.ID+":"+hex.EncodeToString(contentHash[:]))
|
|
}
|
|
for _, result := range uniqueResearchEvidence(research) {
|
|
content := result.Content
|
|
if strings.TrimSpace(content) == "" {
|
|
content = result.Snippet
|
|
}
|
|
contentHash := sha256.Sum256([]byte(strings.TrimSpace(content)))
|
|
parts = append(parts, "research="+canonicalResearchURL(result.URL)+":"+hex.EncodeToString(contentHash[:]))
|
|
}
|
|
sort.Strings(parts)
|
|
parts = append(parts, "pipeline="+strings.TrimSpace(pipelineIdentity), "action="+plan.Action, "target="+plan.TargetArticleID, "type="+normalizeArticleType(plan.ArticleType))
|
|
sum := sha256.Sum256([]byte(strings.Join(parts, "\x00")))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func (e *Engine) articleFingerprintPath(fingerprint string) string {
|
|
return filepath.Join(e.Cfg.DataDir, "article-fingerprints", strings.ToLower(strings.TrimSpace(fingerprint))+".json")
|
|
}
|
|
|
|
func (e *Engine) hasArticleSourceFingerprint(fingerprint string) bool {
|
|
if fingerprint == "" || strings.TrimSpace(e.Cfg.DataDir) == "" {
|
|
return false
|
|
}
|
|
path := e.articleFingerprintPath(fingerprint)
|
|
if e.Persistence != nil && e.Persistence.Pending(path) {
|
|
return true
|
|
}
|
|
_, err := os.Stat(path)
|
|
return err == nil
|
|
}
|
|
|
|
func (e *Engine) queueArticleSourceFingerprint(fingerprint, articleID string, plan model.ArticlePlanDecision) error {
|
|
if fingerprint == "" || e.Persistence == nil {
|
|
return nil
|
|
}
|
|
payload := map[string]any{
|
|
"schema": "article-source-fingerprint/v1",
|
|
"fingerprint": fingerprint,
|
|
"article_id": articleID,
|
|
"action": plan.Action,
|
|
"target_article_id": plan.TargetArticleID,
|
|
"article_type": normalizeArticleType(plan.ArticleType),
|
|
"generated_at": time.Now().UTC(),
|
|
}
|
|
data, err := json.MarshalIndent(payload, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = e.Persistence.QueueFile(e.articleFingerprintPath(fingerprint), append(data, '\n'), 0o640)
|
|
return err
|
|
}
|
|
|
|
// persistResearchMaterial keeps fetched pages reusable/auditable without
|
|
// materialising them as graph nodes. Graph materialisation happens only after
|
|
// the reviewer actually cites a source for a supported claim.
|
|
func (e *Engine) persistResearchMaterial(results []model.ResearchResult) []string {
|
|
paths := make([]string, 0, len(results))
|
|
for _, result := range results {
|
|
path, _, err := e.queueResearchEvidence(result)
|
|
if err == nil && path != "" {
|
|
paths = append(paths, path)
|
|
}
|
|
}
|
|
return unique(paths)
|
|
}
|
|
|
|
func (e *Engine) materializeGroundedResearchEvidence(articleID string, sources []articleSource, results []model.ResearchResult) ([]string, graph.MutationStats) {
|
|
var stats graph.MutationStats
|
|
if len(results) == 0 {
|
|
return nil, stats
|
|
}
|
|
categories := categoriesFromArticleSources(sources)
|
|
ids := make([]string, 0, len(results))
|
|
usedURLs := make([]string, 0, len(results))
|
|
usedInboxIDs := make([]string, 0, len(results))
|
|
for _, result := range results {
|
|
usedURLs = append(usedURLs, result.URL)
|
|
if strings.TrimSpace(result.SourceInboxID) != "" {
|
|
usedInboxIDs = append(usedInboxIDs, result.SourceInboxID)
|
|
}
|
|
id := graph.ID("external", result.URL)
|
|
path, contentHash, err := e.queueResearchEvidence(result)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
node := researchResultNode(id, result, path, contentHash, categories)
|
|
if node.Metadata == nil {
|
|
node.Metadata = map[string]any{}
|
|
}
|
|
node.Metadata["validation_state"] = "grounded"
|
|
node.Metadata["grounded_article_ids"] = []string{articleID}
|
|
stats.Add(e.Graph.UpsertNodeWithStats(node))
|
|
ids = append(ids, id)
|
|
}
|
|
if e.SourceInbox != nil {
|
|
var markErr error
|
|
if len(usedInboxIDs) > 0 {
|
|
markErr = e.SourceInbox.MarkUsedByIDs(context.Background(), usedInboxIDs)
|
|
} else if len(usedURLs) > 0 {
|
|
// Legacy/fallback evidence created before source_inbox_id was propagated.
|
|
markErr = e.SourceInbox.MarkUsed(context.Background(), usedURLs)
|
|
}
|
|
if markErr != nil {
|
|
slog.Warn("source inbox grounding status update failed", "article_id", articleID, "error", markErr)
|
|
}
|
|
}
|
|
return unique(ids), stats
|
|
}
|