sendnrw/glpi-neural-brain
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Cluster-Work

Browse Source
This commit is contained in:
jbergner
2026-08-07 21:54:37 +02:00
parent f8bf8c944e
commit dc2af7ca75
761 changed files with 939 additions and 35364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
internal/engine/engine.go
View File

@@ -34,14 +34,17 @@ var (
)
type EnrichOutcome struct {
Result string
Candidate bool
Created bool
RelationCreated bool
ArticleCreated bool
ArticleSkipped bool
Rejected bool
Comparisons int
Result string
Candidate bool
Created bool
RelationCreated bool
ArticleCreated bool
ArticleSkipped bool
Rejected bool
Comparisons int
CoarseComparisons int
IndexedNodes int
CandidatePool int
}
type Engine struct {
@@ -389,9 +392,10 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
e.enrichCycles++
e.stateMu.Unlock()
e.Broker.Publish(model.Activity{Type: "think.cycle.started", Source: "brain", Phase: "autonomous", Message: fmt.Sprintf("Autonomer AI-THINK-Zyklus startet · bis zu %d sequenzielle Prüfungen", e.Cfg.EnrichBatchSize), Strength: .72, Metadata: map[string]any{"trigger": trigger, "batch_size": e.Cfg.EnrichBatchSize, "anchors": e.Cfg.EnrichAnchors}})
e.Broker.Publish(model.Activity{Type: "think.cycle.started", Source: "brain", Phase: "autonomous", Message: fmt.Sprintf("Autonomer AI-THINK-Zyklus startet · bis zu %d sequenzielle Prüfungen", e.Cfg.EnrichBatchSize), Strength: .72, Metadata: map[string]any{"trigger": trigger, "batch_size": e.Cfg.EnrichBatchSize, "anchors": e.Cfg.EnrichAnchors, "processing_mode": e.RuntimeSettings().ProcessingMode}})
created, rejected, checked := 0, 0, 0
exactComparisons, coarseComparisonsTotal, candidatePoolTotal := 0, 0, 0
relationsCreated, articlesCreated, articlesSkipped := 0, 0, 0
result := "completed"
var cycleErr error
@@ -413,6 +417,9 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
break
}
checked++
exactComparisons += outcome.Comparisons
coarseComparisonsTotal += outcome.CoarseComparisons
candidatePoolTotal += outcome.CandidatePool
if outcome.Created {
created++
}
@@ -454,7 +461,7 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
e.articlesSkipped += uint64(articlesSkipped)
e.stateMu.Unlock()
metadata := map[string]any{"trigger": trigger, "checked": checked, "created": created, "relations_created": relationsCreated, "articles_created": articlesCreated, "articles_skipped": articlesSkipped, "rejected": rejected, "duration_ms": time.Since(started).Milliseconds(), "result": result}
metadata := map[string]any{"trigger": trigger, "checked": checked, "created": created, "relations_created": relationsCreated, "articles_created": articlesCreated, "articles_skipped": articlesSkipped, "rejected": rejected, "duration_ms": time.Since(started).Milliseconds(), "result": result, "processing_mode": e.RuntimeSettings().ProcessingMode, "exact_comparisons": exactComparisons, "coarse_comparisons": coarseComparisonsTotal, "candidate_pool": candidatePoolTotal}
if cycleErr != nil {
e.Broker.Publish(model.Activity{Type: "think.cycle.failed", Source: "brain", Phase: "autonomous", Message: "AI-THINK-Zyklus wurde mit Fehler beendet", Strength: .45, Metadata: metadata})
slog.Warn("enrichment cycle failed", "trigger", trigger, "error", cycleErr)
@@ -636,6 +643,17 @@ func hashEmbedding(s string, dims int) []float64 {
return v
}
func (e *Engine) similarKnowledge(query []float64, limit int, filter graph.NodeFilter, maxAIDepth int) ([]model.Hit, graph.ClusterSearchStats) {
if e.RuntimeSettings().ProcessingMode != "clustered" {
return e.Graph.SimilarFiltered(query, limit, filter), graph.ClusterSearchStats{}
}
candidateLimit := e.Cfg.ClusterCandidatesPerAnchor
if candidateLimit < limit*12 {
candidateLimit = limit * 12
}
return e.Graph.SimilarClusteredFiltered(query, limit, candidateLimit, filter, maxAIDepth, e.Cfg.ClusterHashBits, e.Cfg.ClusterHashTables)
}
func (e *Engine) Query(ctx context.Context, q string) (model.QueryResponse, error) {
e.interactiveInflight.Add(1)
defer e.interactiveInflight.Add(-1)
@@ -649,7 +667,10 @@ func (e *Engine) Query(ctx context.Context, q string) (model.QueryResponse, erro
if err != nil || len(vecs) == 0 {
vecs = [][]float64{hashEmbedding(q, 256)}
}
hits := e.Graph.SimilarFiltered(vecs[0], e.Cfg.TopK, e.effectiveLearningFilter())
hits, retrievalStats := e.similarKnowledge(vecs[0], e.Cfg.TopK, e.effectiveLearningFilter(), 0)
if e.RuntimeSettings().ProcessingMode == "clustered" {
e.Broker.Publish(model.Activity{Type: "query.retrieval.clustered", Source: "brain", Phase: "retrieval", Query: q, Message: fmt.Sprintf("Cluster-Retrieval: %d exakte Cosine-Prüfungen nach %d Hash-Vergleichen", retrievalStats.ExactComparisons, retrievalStats.CoarseComparisons), Strength: .28, Metadata: map[string]any{"processing_mode": "clustered", "indexed_nodes": retrievalStats.IndexedNodes, "coarse_comparisons": retrievalStats.CoarseComparisons, "exact_comparisons": retrievalStats.ExactComparisons, "candidate_pool": retrievalStats.CandidatePool}})
}
nodeIDs := make([]string, 0, len(hits))
for i, h := range hits {
nodeIDs = append(nodeIDs, h.NodeID)
@@ -777,13 +798,27 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
e.setOllamaOK(true)
}
a, b, sim, ok, comparisons := e.Graph.NextPairScopedDepth(e.Cfg.SimilarityThreshold, e.Cfg.EnrichAnchors, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth)
processingMode := e.RuntimeSettings().ProcessingMode
var a, b model.Node
var sim float64
var ok bool
comparisons, coarseComparisons, indexedNodes, candidatePool := 0, 0, 0, 0
if processingMode == "clustered" {
var stats graph.ClusterSearchStats
a, b, sim, ok, stats = e.Graph.NextPairClusteredScopedDepth(e.Cfg.SimilarityThreshold, e.Cfg.EnrichAnchors, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth, e.Cfg.ClusterHashBits, e.Cfg.ClusterHashTables, e.Cfg.ClusterCandidatesPerAnchor)
comparisons = stats.ExactComparisons
coarseComparisons = stats.CoarseComparisons
indexedNodes = stats.IndexedNodes
candidatePool = stats.CandidatePool
} else {
a, b, sim, ok, comparisons = e.Graph.NextPairScopedDepth(e.Cfg.SimilarityThreshold, e.Cfg.EnrichAnchors, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth)
}
if !ok {
e.stateMu.Lock()
e.lastAttempt = time.Now().UTC()
e.stateMu.Unlock()
e.Broker.Publish(model.Activity{Type: "think.no_candidate", Source: "brain", Phase: "candidate-search", Message: "Im aktuell geprüften Graphbereich wurde keine ungeprüfte Beziehung oberhalb des Ähnlichkeitsschwellwerts gefunden", Strength: .28, Metadata: map[string]any{"trigger": trigger, "threshold": e.Cfg.SimilarityThreshold, "anchors": e.Cfg.EnrichAnchors, "comparisons": comparisons}})
return EnrichOutcome{Result: "no_candidate", Comparisons: comparisons}, nil
e.Broker.Publish(model.Activity{Type: "think.no_candidate", Source: "brain", Phase: "candidate-search", Message: "Im aktuell geprüften Graphbereich wurde keine ungeprüfte Beziehung oberhalb des Ähnlichkeitsschwellwerts gefunden", Strength: .28, Metadata: map[string]any{"trigger": trigger, "threshold": e.Cfg.SimilarityThreshold, "anchors": e.Cfg.EnrichAnchors, "comparisons": comparisons, "exact_comparisons": comparisons, "coarse_comparisons": coarseComparisons, "indexed_nodes": indexedNodes, "candidate_pool": candidatePool, "processing_mode": processingMode}})
return EnrichOutcome{Result: "no_candidate", Comparisons: comparisons, CoarseComparisons: coarseComparisons, IndexedNodes: indexedNodes, CandidatePool: candidatePool}, nil
}
now := time.Now().UTC()
@@ -791,13 +826,13 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
e.lastAttempt = now
e.lastEnrich = now
e.stateMu.Unlock()
e.Broker.Publish(model.Activity{Type: "think.started", Source: "brain", Phase: "association", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Verwandtschaft wird geprüft · %.0f%% semantische Nähe", sim*100), Strength: .88, Metadata: map[string]any{"trigger": trigger, "semantic_similarity": sim, "source_label": a.Label, "target_label": b.Label, "model": e.Cfg.ChatModel, "candidate_comparisons": comparisons}})
e.Broker.Publish(model.Activity{Type: "think.started", Source: "brain", Phase: "association", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Verwandtschaft wird geprüft · %.0f%% semantische Nähe", sim*100), Strength: .88, Metadata: map[string]any{"trigger": trigger, "semantic_similarity": sim, "source_label": a.Label, "target_label": b.Label, "model": e.Cfg.ChatModel, "candidate_comparisons": comparisons, "exact_comparisons": comparisons, "coarse_comparisons": coarseComparisons, "indexed_nodes": indexedNodes, "candidate_pool": candidatePool, "processing_mode": processingMode}})
system := "Du führst ausschließlich eine Relationserkennung für einen Wissensgraphen durch. Analysiere zwei interne Wissenseinträge, erfinde keine Fakten und entscheide, ob eine belastbare Beziehung besteht. Schreibe keinen Artikel und keine technische Synthese. Wenn externe Fakten zur Relationsentscheidung fehlen, setze needs_research=true. Gib ausschließlich JSON nach Schema zurück."
var decision model.RelationDecision
if err := e.Ollama.ChatJSON(ctx, system, relationContext(a, b, sim), relationSchema(), &decision); err != nil {
e.Broker.Publish(model.Activity{Type: "think.failed", Source: "brain", Phase: "inference", NodeIDs: []string{a.ID, b.ID}, Message: "Qwen-Beziehungsanalyse ist fehlgeschlagen; es wurde nichts gespeichert", Strength: .35, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "model": e.Cfg.ChatModel}})
return EnrichOutcome{Result: "inference_failed", Candidate: true, Comparisons: comparisons}, fmt.Errorf("relation inference failed: %w", err)
return EnrichOutcome{Result: "inference_failed", Candidate: true, Comparisons: comparisons, CoarseComparisons: coarseComparisons, IndexedNodes: indexedNodes, CandidatePool: candidatePool}, fmt.Errorf("relation inference failed: %w", err)
}
var researchResults []model.ResearchResult
@@ -899,7 +934,7 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
}
e.Graph.UpsertEdge(edge)
edge.ID = graph.EdgeID(edge.Source, edge.Target, edge.Type, edge.Origin)
outcome := EnrichOutcome{Result: status, Candidate: true, Comparisons: comparisons}
outcome := EnrichOutcome{Result: status, Candidate: true, Comparisons: comparisons, CoarseComparisons: coarseComparisons, IndexedNodes: indexedNodes, CandidatePool: candidatePool}
if status == "staging" {
outcome.Created = true
outcome.RelationCreated = true
@@ -956,6 +991,9 @@ func (e *Engine) Status() map[string]any {
"article_language": e.Cfg.ArticleLanguage, "article_synthesis_model": e.Cfg.ArticleSynthesisModel, "article_review_model": e.Cfg.ArticleReviewModel, "article_review_repair_rounds": e.Cfg.ArticleReviewRepairRounds, "article_pipeline": "research_generate_review", "research_dedupe": e.researchDedupeStatus(),
"enrich_interval": e.Cfg.EnrichInterval.String(),
"enrich_batch_size": e.Cfg.EnrichBatchSize, "enrich_anchors": e.Cfg.EnrichAnchors,
"processing_mode": e.RuntimeSettings().ProcessingMode, "cluster_hash_bits": e.Cfg.ClusterHashBits, "cluster_hash_tables": e.Cfg.ClusterHashTables,
"cluster_candidates_per_anchor": e.Cfg.ClusterCandidatesPerAnchor, "cluster_article_candidates": e.Cfg.ClusterArticleCandidates,
"cluster_review_evidence": e.Cfg.ClusterReviewEvidence, "cluster_review_context_chars": e.Cfg.ClusterReviewContextChars,
"research_enabled": e.ResearchEnabledForRuntime(), "chat_model": e.Cfg.ChatModel, "embedding_model": e.Cfg.EmbeddingModel,
"searxng": e.ResearchStatus(),
"ollama_pool": e.Ollama.PoolStatus(), "article_model_status": map[string]any{"synthesis": e.Ollama.ModelStatus(e.Cfg.ArticleSynthesisModel), "review": e.Ollama.ModelStatus(e.Cfg.ArticleReviewModel)}, "persistence": e.Persistence.Status(), "graph_storage": e.Graph.StorageStatus(),