Ch1

internal/engine/article_research.go

@@ -93,38 +93,57 @@ func (e *Engine) researchKnowledgeGapsIterative(ctx context.Context, trigger str
 		previousCritical := len(brief.CriticalGaps) + unresolvedCriticalConflictCount(brief)
 		acceptedThisRound := 0
 		for _, question := range plan.Questions {
+			lease, reused, err := e.beginResearchIntent(ctx, "evidence", question.Question)
+			if err != nil {
+				return evidence, brief, report, fmt.Errorf("research deduplication for %q failed: %w", question.GapID, err)
+			}
 			acceptedForQuestion := 0
-			queries := researchQuestionQueries(question)
-			for _, querySpec := range queries {
-				query := strings.TrimSpace(querySpec.Query)
-				if query == "" || attemptedQueries[strings.ToLower(query)] {
-					continue
+			questionEvidence := []model.ResearchResult{}
+			if !lease.owner {
+				reused = remapResearchEvidenceToQuestion(filterUsableResearchEvidence(reused), question)
+				if len(reused) > 0 {
+					refs := e.addResearchToNodeIDs(nodeIDs, reused)
+					e.Broker.Publish(model.Activity{Type: "article.research.deduplicated", Source: "brain", Phase: "knowledge-research", NodeIDs: append(append([]string{}, nodeIDs...), refs.NodeIDs...), EdgeIDs: refs.EdgeIDs, Message: fmt.Sprintf("Semantisch gleiche Recherche wurde wiederverwendet · %d vorhandene Belege", len(reused)), Strength: .78, Metadata: map[string]any{"trigger": trigger, "gap_id": question.GapID, "research_question": question.Question, "similarity": lease.similarity, "reused_evidence": len(reused), "dedupe_threshold": e.Cfg.ResearchDedupeThreshold}})
 				}
-				attemptedQueries[strings.ToLower(query)] = true
-				report.Queries++
-				accepted, stats := e.executeArticleResearchQuery(ctx, trigger, nodeIDs, question, query, querySpec.Language, round, attemptedURLs)
-				report.SearchResults += stats.SearchResults
-				report.Fetched += stats.Fetched
-				report.Accepted += stats.Accepted
-				report.Rejected += stats.Rejected
-				report.FetchFailed += stats.FetchFailed
-				report.SearchFailed += stats.SearchFailed
-				for _, item := range accepted {
-					key := canonicalResearchURL(item.URL)
-					if key == "" || seenEvidenceURLs[key] {
+				questionEvidence = reused
+			} else {
+				queries := researchQuestionQueries(question)
+				for _, querySpec := range queries {
+					query := strings.TrimSpace(querySpec.Query)
+					if query == "" || attemptedQueries[strings.ToLower(query)] {
 						continue
 					}
-					seenEvidenceURLs[key] = true
-					evidence = append(evidence, item)
+					attemptedQueries[strings.ToLower(query)] = true
+					report.Queries++
+					accepted, stats := e.executeArticleResearchQuery(ctx, trigger, nodeIDs, question, query, querySpec.Language, round, attemptedURLs)
+					report.SearchResults += stats.SearchResults
+					report.Fetched += stats.Fetched
+					report.Accepted += stats.Accepted
+					report.Rejected += stats.Rejected
+					report.FetchFailed += stats.FetchFailed
+					report.SearchFailed += stats.SearchFailed
+					questionEvidence = uniqueResearchEvidence(append(questionEvidence, accepted...))
+				}
+				e.completeResearchIntent(lease, questionEvidence, nil)
+			}
+
+			for _, item := range questionEvidence {
+				key := canonicalResearchURL(item.URL)
+				if key == "" || seenEvidenceURLs[key] {
+					continue
+				}
+				seenEvidenceURLs[key] = true
+				evidence = append(evidence, item)
+				acceptedForQuestion++
+				if lease.owner {
 					acceptedThisRound++
-					acceptedForQuestion++
 				}
 			}
 			// Re-consolidate after each focused question instead of waiting for all
 			// round queries. This stops the round as soon as the article is grounded
 			// and avoids fetching unrelated follow-up sources for an already closed gap.
 			if acceptedForQuestion > 0 {
-				updated, err := e.buildKnowledgeBrief(ctx, sources, evidence)
+				updated, err := e.buildKnowledgeBrief(ctx, sources, evidence, articlePlan.ArticleType)
 				if err != nil {
 					return evidence, brief, report, fmt.Errorf("knowledge consolidation after research question %q in round %d failed: %w", question.GapID, round, err)
 				}
@@ -192,8 +211,9 @@ Regeln:
 - Bevorzuge offizielle Herstellerdokumentation, Standards, Behörden, Projekt-Dokumentation und andere Primärquellen.
 - Vermeide allgemeine Fragen wie "Gibt es Unterschiede" und vermeide mehrere große Themen in einer Query.
 - Bei einer Vergleichslücke mit mehreren benannten Begriffen erzeugst du zunächst je Begriff eine eigene Definitions-/Ziel-/Anwendungsfallfrage mit derselben gap_id. Die spätere Konsolidierung bildet daraus den Vergleich.
-- In späteren Runden müssen bereits versuchte Queries substanziell reformuliert werden, beispielsweise mit offiziellem Produktbegriff, Fehlercode, API-/CLI-Begriff oder site:-Einschränkung.
-- preferred_domains enthält nur fachlich begründete Domainnamen ohne Schema. Erfinde keine Herstellerzuordnung.
+- In späteren Runden müssen bereits versuchte Queries substanziell reformuliert werden, beispielsweise mit offiziellem Produktbegriff, Fehlercode oder API-/CLI-Begriff.
+- Verwende niemals site:-Filter. Die Suche muss offen bleiben, damit SearXNG mehrere Hersteller-, Standard- und Primärquellen finden kann.
+- preferred_domains muss immer eine leere Liste sein. Domainpräferenzen werden nicht als Suchfilter verwendet.
 - expect_actionable ist true, wenn konkrete Implementierungs-, Diagnose-, Validierungs- oder Wiederherstellungsschritte benötigt werden.
 Gib ausschließlich JSON nach Schema zurück.`
 }
@@ -253,17 +273,9 @@ func normalizeResearchPlan(plan model.ResearchPlan, articlePlan model.ArticlePla
 		}
 		question.QueriesDE = cleanUnattemptedQueries(question.QueriesDE, attempted)
 		question.QueriesEN = cleanUnattemptedQueries(question.QueriesEN, attempted)
-		question.PreferredDomains = cleanDomains(question.PreferredDomains)
-		if len(question.PreferredDomains) > 0 {
-			// Keep at least one unrestricted language variant. A model-suggested
-			// preferred domain is useful for primary-source discovery, but must not
-			// turn the whole round into a single-domain dead end.
-			if len(question.QueriesDE) > 0 {
-				question.QueriesDE = applyPreferredDomain(question.QueriesDE, question.PreferredDomains[0])
-			} else {
-				question.QueriesEN = applyPreferredDomain(question.QueriesEN, question.PreferredDomains[0])
-			}
-		}
+		// Domain restrictions are intentionally discarded. A technically valid
+		// hostname can still be semantically wrong for the current vendor/topic.
+		question.PreferredDomains = nil
 		if len(question.QueriesDE) == 0 && len(question.QueriesEN) == 0 {
 			continue
 		}
@@ -338,7 +350,7 @@ func expandCompositeResearchQuestions(questions []model.ResearchQuestion, queryL
 			out = append(out, question)
 			continue
 		}
-		for index, subject := range subjects {
+		for _, subject := range subjects {
 			focused := model.ResearchQuestion{
 				GapID: question.GapID, Critical: question.Critical, ExpectActionable: false,
 				Question: fmt.Sprintf("Was sind Definition, Ziel und typische Anwendungsfälle von %s?", subject),
@@ -352,11 +364,6 @@ func expandCompositeResearchQuestions(questions []model.ResearchQuestion, queryL
 					focused.Question = fmt.Sprintf("What are the definition, objective, and typical use cases of %s?", subject)
 				}
 			}
-			// Use a preferred-domain probe once, then deliberately diversify the
-			// remaining focused questions to avoid a single-domain dead end.
-			if index == 0 {
-				focused.PreferredDomains = append([]string(nil), question.PreferredDomains...)
-			}
 			out = append(out, focused)
 		}
 	}
@@ -424,7 +431,7 @@ func cleanUnattemptedQueries(values []string, attempted map[string]bool) []strin
 	values = unique(values)
 	out := values[:0]
 	for _, value := range values {
-		value = strings.TrimSpace(value)
+		value = sanitizeSearchQuerySiteFilters(value)
 		if value == "" || attempted[strings.ToLower(value)] {
 			continue
 		}
@@ -433,30 +440,22 @@ func cleanUnattemptedQueries(values []string, attempted map[string]bool) []strin
 	return out
 }
 
-func applyPreferredDomain(values []string, domain string) []string {
-	if len(values) == 0 || strings.TrimSpace(domain) == "" {
-		return values
+// sanitizeSearchQuerySiteFilters removes every site: restriction. Even a
+// syntactically valid hostname can be the wrong vendor or documentation source
+// for a generated question, so research deliberately remains domain-open.
+func sanitizeSearchQuerySiteFilters(value string) string {
+	fields := strings.Fields(strings.TrimSpace(value))
+	if len(fields) == 0 {
+		return ""
 	}
-	out := append([]string(nil), values...)
-	if !strings.Contains(strings.ToLower(out[0]), "site:") {
-		out[0] = strings.TrimSpace(out[0]) + " site:" + domain
-	}
-	return out
-}
-
-func cleanDomains(values []string) []string {
-	out := make([]string, 0, len(values))
-	for _, value := range unique(values) {
-		value = strings.ToLower(strings.TrimSpace(value))
-		value = strings.TrimPrefix(value, "https://")
-		value = strings.TrimPrefix(value, "http://")
-		value = strings.TrimPrefix(value, "www.")
-		value = strings.Trim(value, "/")
-		if value != "" && !strings.ContainsAny(value, " ?#") {
-			out = append(out, value)
+	out := make([]string, 0, len(fields))
+	for _, field := range fields {
+		if strings.HasPrefix(strings.ToLower(field), "site:") {
+			continue
 		}
+		out = append(out, field)
 	}
-	return out
+	return strings.TrimSpace(strings.Join(out, " "))
 }
 
 type queryExecutionStats struct {
@@ -470,6 +469,10 @@ type queryExecutionStats struct {
 
 func (e *Engine) executeArticleResearchQuery(ctx context.Context, trigger string, nodeIDs []string, question model.ResearchQuestion, query, language string, round int, attemptedURLs map[string]bool, fetchCaps ...int) ([]model.ResearchResult, queryExecutionStats) {
 	stats := queryExecutionStats{}
+	query = sanitizeSearchQuerySiteFilters(query)
+	if strings.TrimSpace(query) == "" {
+		return nil, stats
+	}
 	researchID := newResearchRunID("article-research", query)
 	started := time.Now()
 	startMetadata := map[string]any{"trigger": trigger, "research_id": researchID, "research_query": query, "research_round": round, "gap_id": question.GapID, "research_question": question.Question, "language": language, "animation_min_ms": 2000}
@@ -481,9 +484,15 @@ func (e *Engine) executeArticleResearchQuery(ctx context.Context, trigger string
 
 	resultLimit := e.Cfg.ArticleResearchResults
 	if resultLimit < 1 {
-		resultLimit = 8
+		resultLimit = 12
 	}
-	results, diagnostic, err := e.Research.SearchDetailedLanguage(ctx, query, resultLimit, language)
+	var results []model.ResearchResult
+	var diagnostic research.Diagnostic
+	err := e.withSharedResearchWork(ctx, "searxng.search", func() error {
+		var searchErr error
+		results, diagnostic, searchErr = e.Research.SearchDetailedLanguage(ctx, query, resultLimit, language)
+		return searchErr
+	})
 	if err != nil {
 		stats.SearchFailed = 1
 		metadata := mergeResearchMetadata(startMetadata, researchDiagnosticMetadata(diagnostic))
@@ -557,7 +566,13 @@ func (e *Engine) executeArticleResearchQuery(ctx context.Context, trigger string
 	for _, candidate := range selected {
 		fetchMetadata := mergeResearchMetadata(startMetadata, map[string]any{"result_url": candidate.Result.URL, "result_title": candidate.Result.Title, "relevance": candidate.Assessment.Relevance, "source_quality": candidate.Assessment.SourceQuality, "source_quality_score": candidate.Assessment.SourceQualityScore})
 		e.Broker.Publish(model.Activity{Type: "article.research.fetch.started", Source: "web", Phase: "knowledge-research-fetch", NodeIDs: nodeIDs, Message: "Der vollständige Inhalt einer relevanten Webquelle wird geladen", Strength: .78, Metadata: fetchMetadata})
-		page, fetchDiagnostic, err := e.Research.FetchPage(ctx, candidate.Result.URL, research.FetchOptions{MaxBytes: e.Cfg.ArticleResearchPageMaxBytes, MaxChars: e.Cfg.ArticleResearchPageMaxChars, Timeout: e.Cfg.ArticleResearchFetchTimeout, AllowPrivate: e.Cfg.ArticleResearchAllowPrivate})
+		var page research.FetchedPage
+		var fetchDiagnostic research.FetchDiagnostic
+		err := e.withSharedResearchWork(ctx, "web.fetch", func() error {
+			var fetchErr error
+			page, fetchDiagnostic, fetchErr = e.Research.FetchPage(ctx, candidate.Result.URL, research.FetchOptions{MaxBytes: e.Cfg.ArticleResearchPageMaxBytes, MaxChars: e.Cfg.ArticleResearchPageMaxChars, Timeout: e.Cfg.ArticleResearchFetchTimeout, AllowPrivate: e.Cfg.ArticleResearchAllowPrivate})
+			return fetchErr
+		})
 		if err != nil {
 			stats.FetchFailed++
 			stats.Rejected++
@@ -615,10 +630,14 @@ func (e *Engine) executeArticleResearchQuery(ctx context.Context, trigger string
 		item.Actionable = assessment.Actionable
 		item.CoveredGapIDs = unique(append(assessment.CoveredGapIDs, question.GapID))
 		item.AssessmentReason = assessment.Reason
-		acceptedByGate := assessment.Relevant && assessment.Relevance >= e.Cfg.ArticleResearchMinRelevance && assessment.SourceQualityScore >= e.Cfg.ArticleResearchMinQuality
-		if question.ExpectActionable && !assessment.Actionable {
-			acceptedByGate = false
-		}
+		// Prefer recall over premature rejection: a relevant high-quality source
+		// may still be useful evidence even if it only partially closes the gap.
+		// The later knowledge-brief gate decides whether the article is sufficiently
+		// actionable; research evidence itself is intentionally accepted more broadly.
+		relevancePass := assessment.Relevant || assessment.Relevance >= e.Cfg.ArticleResearchMinRelevance
+		strictPass := assessment.Relevance >= e.Cfg.ArticleResearchMinRelevance
+		strongSourcePartialPass := assessment.Relevance >= e.Cfg.ArticleResearchPrefetchMinRelevance && assessment.SourceQualityScore >= math.Max(.70, e.Cfg.ArticleResearchMinQuality)
+		acceptedByGate := relevancePass && assessment.SourceQualityScore >= e.Cfg.ArticleResearchMinQuality && (strictPass || strongSourcePartialPass)
 		researchNode := model.Node{Kind: "external", Origin: "research", URI: item.URL, ExternalID: item.URL, Categories: researchCategories, Metadata: map[string]any{"source": graph.SourceFromURL(item.URL)}}
 		if !thinkingFilter.Matches(researchNode) {
 			acceptedByGate = false