Cluster-Work

internal/engine/article.go

@@ -107,10 +107,17 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
 	}
 
 	e.Broker.Publish(model.Activity{Type: "article.consolidation.started", Source: "brain", Phase: "knowledge-consolidation", NodeIDs: plan.SourceNodeIDs, Message: "Interne Quellen werden als Ausgangsmaterial für Recherche und Artikelsynthese strukturiert", Strength: .84, Metadata: map[string]any{"trigger": trigger, "source_count": len(selected), "synthesis_model": e.Cfg.ArticleSynthesisModel, "review_model": e.Cfg.ArticleReviewModel}})
-	brief, err := e.buildKnowledgeBrief(ctx, selected, researchResults, plan.ArticleType)
-	if err != nil {
+	var brief model.KnowledgeBrief
+	if e.RuntimeSettings().ProcessingMode == "clustered" {
 		brief = fallbackSynthesisBrief(plan, selected)
-		e.Broker.Publish(model.Activity{Type: "article.consolidation.fallback", Source: "brain", Phase: "knowledge-consolidation", NodeIDs: plan.SourceNodeIDs, Message: "Vorstrukturierung war nicht verfügbar · die Synthese arbeitet direkt mit Originalquellen und Recherchematerial weiter", Strength: .48, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "review_model": e.Cfg.ArticleReviewModel}})
+		e.Broker.Publish(model.Activity{Type: "article.consolidation.clustered", Source: "brain", Phase: "knowledge-consolidation", NodeIDs: plan.SourceNodeIDs, Message: "Cluster/Fast überspringt die zusätzliche LLM-Vorstrukturierung und arbeitet direkt mit Plan, Originalquellen und Recherchematerial", Strength: .42, Metadata: map[string]any{"trigger": trigger, "processing_mode": "clustered", "saved_model_call": true}})
+	} else {
+		var err error
+		brief, err = e.buildKnowledgeBrief(ctx, selected, researchResults, plan.ArticleType)
+		if err != nil {
+			brief = fallbackSynthesisBrief(plan, selected)
+			e.Broker.Publish(model.Activity{Type: "article.consolidation.fallback", Source: "brain", Phase: "knowledge-consolidation", NodeIDs: plan.SourceNodeIDs, Message: "Vorstrukturierung war nicht verfügbar · die Synthese arbeitet direkt mit Originalquellen und Recherchematerial weiter", Strength: .48, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "review_model": e.Cfg.ArticleReviewModel}})
+		}
 	}
 
 	// Generate-then-review pipeline: research collects broad full-text material.
@@ -151,10 +158,11 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
 		productionCount, aiCount, productionRatio, maxDepth = articleSourceStats(selected)
 		generationDepth = maxDepth + 1
 
-		quality, err = e.reviewArticleContent(ctx, draft, plan.ArticleType, selected, researchResults)
-		if err != nil {
-			return articleSynthesisOutcome{}, fmt.Errorf("article quality review with %s failed: %w", e.Cfg.ArticleReviewModel, err)
+		reviewedQuality, reviewErr := e.reviewArticleContent(ctx, draft, plan.ArticleType, selected, researchResults)
+		if reviewErr != nil {
+			return articleSynthesisOutcome{}, fmt.Errorf("article quality review with %s failed: %w", e.Cfg.ArticleReviewModel, reviewErr)
 		}
+		quality = reviewedQuality
 		draft.Confidence = quality.Confidence
 		claimCounts := articleClaimReviewCounts(quality.ClaimReviews)
 		e.Broker.Publish(model.Activity{Type: "article.review.completed", Source: "brain", Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: fmt.Sprintf("%s hat den fertigen Entwurf Claim für Claim gegen die Quellen geprüft", e.Cfg.ArticleReviewModel), Strength: .88, Metadata: map[string]any{"trigger": trigger, "accepted": quality.Accepted, "confidence": quality.Confidence, "claim_reviews": len(quality.ClaimReviews), "supported_claims": claimCounts["supported"], "partially_supported_claims": claimCounts["partially_supported"], "unsupported_claims_count": claimCounts["unsupported"], "contradicted_claims": claimCounts["contradicted"], "missing_evidence_queries": quality.MissingEvidenceQueries, "issues": quality.Issues, "review_model": e.Cfg.ArticleReviewModel, "synthesis_model": e.Cfg.ArticleSynthesisModel, "repair_attempt": repairAttempts}})
@@ -215,6 +223,9 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
 }
 
 func (e *Engine) selectArticleSources(seeds []model.Node) []articleSource {
+	if e.RuntimeSettings().ProcessingMode == "clustered" {
+		return e.selectArticleSourcesClustered(seeds)
+	}
 	snapshot := e.Graph.Snapshot()
 	seedIDs := map[string]bool{}
 	var seedVectors [][]float64
@@ -306,6 +317,106 @@ func (e *Engine) selectArticleSources(seeds []model.Node) []articleSource {
 	return out
 }
 
+func (e *Engine) selectArticleSourcesClustered(seeds []model.Node) []articleSource {
+	seedIDs := map[string]bool{}
+	var seedVectors [][]float64
+	for _, seed := range seeds {
+		seedIDs[seed.ID] = true
+		if v, ok := e.Graph.Vector(seed.ID); ok && len(v) > 0 {
+			seedVectors = append(seedVectors, v)
+		}
+	}
+	centroid := vectorCentroid(seedVectors)
+	direct := e.Graph.NeighborScores(seedIDs)
+	candidateIDs := map[string]bool{}
+	for id := range seedIDs {
+		candidateIDs[id] = true
+	}
+	for id := range direct {
+		candidateIDs[id] = true
+	}
+	stats := graph.ClusterSearchStats{}
+	if len(centroid) > 0 {
+		limit := e.Cfg.ClusterArticleCandidates
+		if limit < e.Cfg.ArticleMaxSources*4 {
+			limit = e.Cfg.ArticleMaxSources * 4
+		}
+		hits, searchStats := e.Graph.SimilarClusteredFiltered(centroid, limit, limit, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth, e.Cfg.ClusterHashBits, e.Cfg.ClusterHashTables)
+		stats = searchStats
+		for _, hit := range hits {
+			candidateIDs[hit.NodeID] = true
+		}
+	}
+	filter := e.effectiveThinkingFilter()
+	var production, ai []articleSource
+	for id := range candidateIDs {
+		node, ok := e.Graph.GetNode(id)
+		if !ok || (node.Kind != "knowledge" && node.Kind != "ai-think") || !filter.Matches(node) {
+			continue
+		}
+		depth := nodeGenerationDepth(node)
+		if node.Kind == "ai-think" && depth >= e.Cfg.ArticleMaxGenerationDepth {
+			continue
+		}
+		isProduction := node.Kind == "knowledge" && node.Status == "production"
+		isAI := node.Kind == "ai-think"
+		if !isProduction && !isAI {
+			continue
+		}
+		score := direct[node.ID] * 2.3
+		if seedIDs[node.ID] {
+			score += 8
+		}
+		if len(centroid) > 0 {
+			if v, ok := e.Graph.Vector(node.ID); ok && len(v) == len(centroid) {
+				sim := cosineVector(centroid, v)
+				if sim < e.Cfg.SimilarityThreshold*.82 && direct[node.ID] == 0 && !seedIDs[node.ID] {
+					continue
+				}
+				score += sim * 3
+			}
+		}
+		score += categoryAffinity(node, seeds) * .45
+		content := e.sourceContent(node)
+		if strings.TrimSpace(content) == "" {
+			content = node.Summary
+		}
+		source := articleSource{Node: node, Content: content, Score: score, Depth: depth}
+		if isProduction {
+			production = append(production, source)
+		} else {
+			ai = append(ai, source)
+		}
+	}
+	sortArticleSources(production)
+	sortArticleSources(ai)
+	maxSources := e.Cfg.ArticleMaxSources
+	if maxSources < 1 {
+		maxSources = 8
+	}
+	out := make([]articleSource, 0, maxSources)
+	for _, source := range production {
+		if len(out) >= maxSources {
+			break
+		}
+		out = append(out, source)
+	}
+	for _, source := range ai {
+		if len(out) >= maxSources {
+			break
+		}
+		prod, aiCount, _, _ := articleSourceStats(out)
+		if float64(aiCount+1)/float64(prod+aiCount+1) > 1-e.Cfg.ArticleMinProductionRatio {
+			continue
+		}
+		out = append(out, source)
+	}
+	if e.Broker != nil {
+		e.Broker.Publish(model.Activity{Type: "article.sources.clustered", Source: "brain", Phase: "candidate-search", NodeIDs: nodeIDsFromArticleSources(out), Message: fmt.Sprintf("Cluster/Fast reduzierte die Artikelquellenauswahl auf %d Kandidaten und %d Quellen", len(candidateIDs), len(out)), Strength: .45, Metadata: map[string]any{"processing_mode": "clustered", "candidate_nodes": len(candidateIDs), "selected_sources": len(out), "indexed_nodes": stats.IndexedNodes, "coarse_comparisons": stats.CoarseComparisons, "exact_comparisons": stats.ExactComparisons, "hash_bits": stats.HashBits, "hash_tables": stats.HashTables}})
+	}
+	return out
+}
+
 func (e *Engine) sourceContent(node model.Node) string {
 	if node.Origin == "glpi-kb" && e.GLPIKB != nil {
 		if value, ok := e.GLPIKB.Content(node.ID); ok && strings.TrimSpace(value) != "" {
@@ -920,7 +1031,11 @@ func (e *Engine) generateArticleContent(ctx context.Context, sources []articleSo
 
 func (e *Engine) reviewArticleContent(ctx context.Context, draft model.KnowledgeArticleDraft, articleType string, sources []articleSource, researchResults []model.ResearchResult) (model.ArticleQualityDecision, error) {
 	var decision model.ArticleQualityDecision
-	if err := e.Ollama.ChatJSONModel(ctx, e.Cfg.ArticleReviewModel, articleQualitySystemPrompt(e.Cfg.ArticleLanguage), e.articleQualityContext(draft, articleType, sources, researchResults), articleQualitySchema(), &decision); err != nil {
+	reviewResearch := researchResults
+	if e.RuntimeSettings().ProcessingMode == "clustered" {
+		reviewResearch = selectReviewEvidence(researchResults, e.Cfg.ClusterReviewEvidence)
+	}
+	if err := e.Ollama.ChatJSONModel(ctx, e.Cfg.ArticleReviewModel, articleQualitySystemPrompt(e.Cfg.ArticleLanguage), e.articleQualityContext(draft, articleType, sources, reviewResearch), articleQualitySchema(), &decision); err != nil {
 		return model.ArticleQualityDecision{}, err
 	}
 	if containsDraftMetaContent(draft) {
@@ -945,6 +1060,63 @@ func (e *Engine) reviewArticleContent(ctx context.Context, draft model.Knowledge
 	return decision, nil
 }
 
+func selectReviewEvidence(results []model.ResearchResult, limit int) []model.ResearchResult {
+	if limit <= 0 || len(results) <= limit {
+		return append([]model.ResearchResult(nil), results...)
+	}
+	type scored struct {
+		result model.ResearchResult
+		score  float64
+		domain string
+	}
+	items := make([]scored, 0, len(results))
+	for _, result := range results {
+		score := result.Relevance*.48 + result.SourceQualityScore*.42
+		if result.Actionable {
+			score += .10
+		}
+		if result.Fetched {
+			score += .04
+		}
+		items = append(items, scored{result: result, score: score, domain: graph.SourceFromURL(result.URL)})
+	}
+	sort.SliceStable(items, func(i, j int) bool {
+		if items[i].score == items[j].score {
+			return items[i].result.URL < items[j].result.URL
+		}
+		return items[i].score > items[j].score
+	})
+	out := make([]model.ResearchResult, 0, limit)
+	seenDomain := map[string]bool{}
+	for _, item := range items {
+		if len(out) >= limit {
+			break
+		}
+		if item.domain != "" && seenDomain[item.domain] {
+			continue
+		}
+		out = append(out, item.result)
+		seenDomain[item.domain] = true
+	}
+	if len(out) < limit {
+		seenURL := map[string]bool{}
+		for _, result := range out {
+			seenURL[result.URL] = true
+		}
+		for _, item := range items {
+			if len(out) >= limit {
+				break
+			}
+			if seenURL[item.result.URL] {
+				continue
+			}
+			out = append(out, item.result)
+			seenURL[item.result.URL] = true
+		}
+	}
+	return out
+}
+
 func (e *Engine) articleRewriteContext(sources []articleSource, plan model.ArticlePlanDecision, brief model.KnowledgeBrief, researchResults []model.ResearchResult, rejected string) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "SCHREIBAUFTRAG: Formuliere einen vollständigen, direkt nutzbaren Helpdesk-Wissensartikel.\nARTIKELTYP: %s\n", nonempty(plan.ArticleType, "how_to"))
@@ -964,6 +1136,10 @@ func (e *Engine) articleRewriteContext(sources []articleSource, plan model.Artic
 
 func (e *Engine) articleQualityContext(draft model.KnowledgeArticleDraft, articleType string, sources []articleSource, researchResults []model.ResearchResult) string {
 	var b strings.Builder
+	contextLimit := e.Cfg.MaxContextChars
+	if e.RuntimeSettings().ProcessingMode == "clustered" && e.Cfg.ClusterReviewContextChars > 0 && (contextLimit <= 0 || e.Cfg.ClusterReviewContextChars < contextLimit) {
+		contextLimit = e.Cfg.ClusterReviewContextChars
+	}
 	fmt.Fprintf(&b, "ZU PRÜFENDER SICHTBARER KB-ARTIKEL:\nARTIKELTYP: %s\n\nTITEL:\n", nonempty(articleType, "how_to"))
 	b.WriteString(draft.Title)
 	b.WriteString("\n\nPROBLEM / BESCHREIBUNG:\n")
@@ -971,10 +1147,10 @@ func (e *Engine) articleQualityContext(draft model.KnowledgeArticleDraft, articl
 	b.WriteString("\n\nLÖSUNG / ANTWORT:\n")
 	b.WriteString(formatArticleAnswer(draft, e.Cfg.ArticleLanguage))
 	b.WriteString("\n\nINTERNE BELEGQUELLEN:\n")
-	appendArticleSources(&b, sources, e.Cfg.MaxContextChars)
+	appendArticleSources(&b, sources, contextLimit)
 	if len(researchResults) > 0 {
 		b.WriteString("\nVOLLTEXT-RECHERCHEMATERIAL:\n")
-		appendResearchEvidence(&b, researchResults, e.Cfg.MaxContextChars)
+		appendResearchEvidence(&b, researchResults, contextLimit)
 	}
 	return b.String()
 }