Cluster-Work

This commit is contained in:
2026-08-07 21:54:37 +02:00
parent f8bf8c944e
commit dc2af7ca75
761 changed files with 939 additions and 35364 deletions

View File

@@ -38,6 +38,13 @@ type Config struct {
EnrichStepDelay time.Duration
EnrichBatchSize int
EnrichAnchors int
ProcessingMode string
ClusterHashBits int
ClusterHashTables int
ClusterCandidatesPerAnchor int
ClusterArticleCandidates int
ClusterReviewEvidence int
ClusterReviewContextChars int
SimilarityThreshold float64
RelationThreshold float64
ArticleSynthesisEnabled bool
@@ -155,6 +162,13 @@ func Load() (Config, error) {
EnrichStepDelay: duration("BRAIN_ENRICH_STEP_DELAY", 3*time.Second),
EnrichBatchSize: integer("BRAIN_ENRICH_BATCH_SIZE", 3),
EnrichAnchors: integer("BRAIN_ENRICH_ANCHORS", 48),
ProcessingMode: strings.ToLower(env("BRAIN_PROCESSING_MODE", "precise")),
ClusterHashBits: integer("BRAIN_CLUSTER_HASH_BITS", 24),
ClusterHashTables: integer("BRAIN_CLUSTER_HASH_TABLES", 2),
ClusterCandidatesPerAnchor: integer("BRAIN_CLUSTER_CANDIDATES_PER_ANCHOR", 96),
ClusterArticleCandidates: integer("BRAIN_CLUSTER_ARTICLE_CANDIDATES", 192),
ClusterReviewEvidence: integer("BRAIN_CLUSTER_REVIEW_EVIDENCE", 8),
ClusterReviewContextChars: integer("BRAIN_CLUSTER_REVIEW_CONTEXT_CHARS", 8000),
SimilarityThreshold: number("BRAIN_SIMILARITY_THRESHOLD", 0.68),
RelationThreshold: number("BRAIN_RELATION_THRESHOLD", 0.72),
ArticleSynthesisEnabled: boolean("BRAIN_ARTICLE_SYNTHESIS_ENABLED", true),
@@ -297,6 +311,29 @@ func Load() (Config, error) {
if cfg.ArticleResearchMinQuality < 0 || cfg.ArticleResearchMinQuality > 1 {
return Config{}, fmt.Errorf("BRAIN_ARTICLE_RESEARCH_MIN_QUALITY must be between 0 and 1")
}
switch cfg.ProcessingMode {
case "precise", "clustered":
default:
return Config{}, fmt.Errorf("BRAIN_PROCESSING_MODE must be precise or clustered")
}
if cfg.ClusterHashBits < 8 || cfg.ClusterHashBits > 63 {
return Config{}, fmt.Errorf("BRAIN_CLUSTER_HASH_BITS must be between 8 and 63")
}
if cfg.ClusterHashTables < 1 || cfg.ClusterHashTables > 4 {
return Config{}, fmt.Errorf("BRAIN_CLUSTER_HASH_TABLES must be between 1 and 4")
}
if cfg.ClusterCandidatesPerAnchor < 16 || cfg.ClusterCandidatesPerAnchor > 2048 {
return Config{}, fmt.Errorf("BRAIN_CLUSTER_CANDIDATES_PER_ANCHOR must be between 16 and 2048")
}
if cfg.ClusterArticleCandidates < 16 || cfg.ClusterArticleCandidates > 4096 {
return Config{}, fmt.Errorf("BRAIN_CLUSTER_ARTICLE_CANDIDATES must be between 16 and 4096")
}
if cfg.ClusterReviewEvidence < 2 || cfg.ClusterReviewEvidence > 64 {
return Config{}, fmt.Errorf("BRAIN_CLUSTER_REVIEW_EVIDENCE must be between 2 and 64")
}
if cfg.ClusterReviewContextChars < 4000 || cfg.ClusterReviewContextChars > 100000 {
return Config{}, fmt.Errorf("BRAIN_CLUSTER_REVIEW_CONTEXT_CHARS must be between 4000 and 100000")
}
if cfg.ArticleResearchPageMaxBytes < 65536 || cfg.ArticleResearchPageMaxBytes > 16777216 {
return Config{}, fmt.Errorf("BRAIN_ARTICLE_RESEARCH_PAGE_MAX_BYTES must be between 65536 and 16777216")
}

View File

@@ -149,3 +149,29 @@ func TestLoadRejectsAutonomousResearchWithoutSearXNG(t *testing.T) {
t.Fatal("expected SEARXNG_URL validation error")
}
}
func TestLoadClusterProcessingMode(t *testing.T) {
t.Setenv("BRAIN_DATA_DIR", t.TempDir())
t.Setenv("BRAIN_PROCESSING_MODE", "clustered")
t.Setenv("BRAIN_CLUSTER_HASH_BITS", "20")
t.Setenv("BRAIN_CLUSTER_HASH_TABLES", "3")
t.Setenv("BRAIN_CLUSTER_CANDIDATES_PER_ANCHOR", "80")
t.Setenv("BRAIN_CLUSTER_ARTICLE_CANDIDATES", "160")
t.Setenv("BRAIN_CLUSTER_REVIEW_EVIDENCE", "7")
t.Setenv("BRAIN_CLUSTER_REVIEW_CONTEXT_CHARS", "7000")
cfg, err := Load()
if err != nil {
t.Fatal(err)
}
if cfg.ProcessingMode != "clustered" || cfg.ClusterHashBits != 20 || cfg.ClusterHashTables != 3 || cfg.ClusterCandidatesPerAnchor != 80 || cfg.ClusterArticleCandidates != 160 || cfg.ClusterReviewEvidence != 7 || cfg.ClusterReviewContextChars != 7000 {
t.Fatalf("unexpected cluster config: %+v", cfg)
}
}
func TestLoadRejectsInvalidProcessingMode(t *testing.T) {
t.Setenv("BRAIN_DATA_DIR", t.TempDir())
t.Setenv("BRAIN_PROCESSING_MODE", "turbo")
if _, err := Load(); err == nil {
t.Fatal("expected invalid processing mode error")
}
}

View File

@@ -107,10 +107,17 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
}
e.Broker.Publish(model.Activity{Type: "article.consolidation.started", Source: "brain", Phase: "knowledge-consolidation", NodeIDs: plan.SourceNodeIDs, Message: "Interne Quellen werden als Ausgangsmaterial für Recherche und Artikelsynthese strukturiert", Strength: .84, Metadata: map[string]any{"trigger": trigger, "source_count": len(selected), "synthesis_model": e.Cfg.ArticleSynthesisModel, "review_model": e.Cfg.ArticleReviewModel}})
brief, err := e.buildKnowledgeBrief(ctx, selected, researchResults, plan.ArticleType)
if err != nil {
var brief model.KnowledgeBrief
if e.RuntimeSettings().ProcessingMode == "clustered" {
brief = fallbackSynthesisBrief(plan, selected)
e.Broker.Publish(model.Activity{Type: "article.consolidation.fallback", Source: "brain", Phase: "knowledge-consolidation", NodeIDs: plan.SourceNodeIDs, Message: "Vorstrukturierung war nicht verfügbar · die Synthese arbeitet direkt mit Originalquellen und Recherchematerial weiter", Strength: .48, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "review_model": e.Cfg.ArticleReviewModel}})
e.Broker.Publish(model.Activity{Type: "article.consolidation.clustered", Source: "brain", Phase: "knowledge-consolidation", NodeIDs: plan.SourceNodeIDs, Message: "Cluster/Fast überspringt die zusätzliche LLM-Vorstrukturierung und arbeitet direkt mit Plan, Originalquellen und Recherchematerial", Strength: .42, Metadata: map[string]any{"trigger": trigger, "processing_mode": "clustered", "saved_model_call": true}})
} else {
var err error
brief, err = e.buildKnowledgeBrief(ctx, selected, researchResults, plan.ArticleType)
if err != nil {
brief = fallbackSynthesisBrief(plan, selected)
e.Broker.Publish(model.Activity{Type: "article.consolidation.fallback", Source: "brain", Phase: "knowledge-consolidation", NodeIDs: plan.SourceNodeIDs, Message: "Vorstrukturierung war nicht verfügbar · die Synthese arbeitet direkt mit Originalquellen und Recherchematerial weiter", Strength: .48, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "review_model": e.Cfg.ArticleReviewModel}})
}
}
// Generate-then-review pipeline: research collects broad full-text material.
@@ -151,10 +158,11 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
productionCount, aiCount, productionRatio, maxDepth = articleSourceStats(selected)
generationDepth = maxDepth + 1
quality, err = e.reviewArticleContent(ctx, draft, plan.ArticleType, selected, researchResults)
if err != nil {
return articleSynthesisOutcome{}, fmt.Errorf("article quality review with %s failed: %w", e.Cfg.ArticleReviewModel, err)
reviewedQuality, reviewErr := e.reviewArticleContent(ctx, draft, plan.ArticleType, selected, researchResults)
if reviewErr != nil {
return articleSynthesisOutcome{}, fmt.Errorf("article quality review with %s failed: %w", e.Cfg.ArticleReviewModel, reviewErr)
}
quality = reviewedQuality
draft.Confidence = quality.Confidence
claimCounts := articleClaimReviewCounts(quality.ClaimReviews)
e.Broker.Publish(model.Activity{Type: "article.review.completed", Source: "brain", Phase: "quality-gate", NodeIDs: draft.SourceNodeIDs, Message: fmt.Sprintf("%s hat den fertigen Entwurf Claim für Claim gegen die Quellen geprüft", e.Cfg.ArticleReviewModel), Strength: .88, Metadata: map[string]any{"trigger": trigger, "accepted": quality.Accepted, "confidence": quality.Confidence, "claim_reviews": len(quality.ClaimReviews), "supported_claims": claimCounts["supported"], "partially_supported_claims": claimCounts["partially_supported"], "unsupported_claims_count": claimCounts["unsupported"], "contradicted_claims": claimCounts["contradicted"], "missing_evidence_queries": quality.MissingEvidenceQueries, "issues": quality.Issues, "review_model": e.Cfg.ArticleReviewModel, "synthesis_model": e.Cfg.ArticleSynthesisModel, "repair_attempt": repairAttempts}})
@@ -215,6 +223,9 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
}
func (e *Engine) selectArticleSources(seeds []model.Node) []articleSource {
if e.RuntimeSettings().ProcessingMode == "clustered" {
return e.selectArticleSourcesClustered(seeds)
}
snapshot := e.Graph.Snapshot()
seedIDs := map[string]bool{}
var seedVectors [][]float64
@@ -306,6 +317,106 @@ func (e *Engine) selectArticleSources(seeds []model.Node) []articleSource {
return out
}
func (e *Engine) selectArticleSourcesClustered(seeds []model.Node) []articleSource {
seedIDs := map[string]bool{}
var seedVectors [][]float64
for _, seed := range seeds {
seedIDs[seed.ID] = true
if v, ok := e.Graph.Vector(seed.ID); ok && len(v) > 0 {
seedVectors = append(seedVectors, v)
}
}
centroid := vectorCentroid(seedVectors)
direct := e.Graph.NeighborScores(seedIDs)
candidateIDs := map[string]bool{}
for id := range seedIDs {
candidateIDs[id] = true
}
for id := range direct {
candidateIDs[id] = true
}
stats := graph.ClusterSearchStats{}
if len(centroid) > 0 {
limit := e.Cfg.ClusterArticleCandidates
if limit < e.Cfg.ArticleMaxSources*4 {
limit = e.Cfg.ArticleMaxSources * 4
}
hits, searchStats := e.Graph.SimilarClusteredFiltered(centroid, limit, limit, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth, e.Cfg.ClusterHashBits, e.Cfg.ClusterHashTables)
stats = searchStats
for _, hit := range hits {
candidateIDs[hit.NodeID] = true
}
}
filter := e.effectiveThinkingFilter()
var production, ai []articleSource
for id := range candidateIDs {
node, ok := e.Graph.GetNode(id)
if !ok || (node.Kind != "knowledge" && node.Kind != "ai-think") || !filter.Matches(node) {
continue
}
depth := nodeGenerationDepth(node)
if node.Kind == "ai-think" && depth >= e.Cfg.ArticleMaxGenerationDepth {
continue
}
isProduction := node.Kind == "knowledge" && node.Status == "production"
isAI := node.Kind == "ai-think"
if !isProduction && !isAI {
continue
}
score := direct[node.ID] * 2.3
if seedIDs[node.ID] {
score += 8
}
if len(centroid) > 0 {
if v, ok := e.Graph.Vector(node.ID); ok && len(v) == len(centroid) {
sim := cosineVector(centroid, v)
if sim < e.Cfg.SimilarityThreshold*.82 && direct[node.ID] == 0 && !seedIDs[node.ID] {
continue
}
score += sim * 3
}
}
score += categoryAffinity(node, seeds) * .45
content := e.sourceContent(node)
if strings.TrimSpace(content) == "" {
content = node.Summary
}
source := articleSource{Node: node, Content: content, Score: score, Depth: depth}
if isProduction {
production = append(production, source)
} else {
ai = append(ai, source)
}
}
sortArticleSources(production)
sortArticleSources(ai)
maxSources := e.Cfg.ArticleMaxSources
if maxSources < 1 {
maxSources = 8
}
out := make([]articleSource, 0, maxSources)
for _, source := range production {
if len(out) >= maxSources {
break
}
out = append(out, source)
}
for _, source := range ai {
if len(out) >= maxSources {
break
}
prod, aiCount, _, _ := articleSourceStats(out)
if float64(aiCount+1)/float64(prod+aiCount+1) > 1-e.Cfg.ArticleMinProductionRatio {
continue
}
out = append(out, source)
}
if e.Broker != nil {
e.Broker.Publish(model.Activity{Type: "article.sources.clustered", Source: "brain", Phase: "candidate-search", NodeIDs: nodeIDsFromArticleSources(out), Message: fmt.Sprintf("Cluster/Fast reduzierte die Artikelquellenauswahl auf %d Kandidaten und %d Quellen", len(candidateIDs), len(out)), Strength: .45, Metadata: map[string]any{"processing_mode": "clustered", "candidate_nodes": len(candidateIDs), "selected_sources": len(out), "indexed_nodes": stats.IndexedNodes, "coarse_comparisons": stats.CoarseComparisons, "exact_comparisons": stats.ExactComparisons, "hash_bits": stats.HashBits, "hash_tables": stats.HashTables}})
}
return out
}
func (e *Engine) sourceContent(node model.Node) string {
if node.Origin == "glpi-kb" && e.GLPIKB != nil {
if value, ok := e.GLPIKB.Content(node.ID); ok && strings.TrimSpace(value) != "" {
@@ -920,7 +1031,11 @@ func (e *Engine) generateArticleContent(ctx context.Context, sources []articleSo
func (e *Engine) reviewArticleContent(ctx context.Context, draft model.KnowledgeArticleDraft, articleType string, sources []articleSource, researchResults []model.ResearchResult) (model.ArticleQualityDecision, error) {
var decision model.ArticleQualityDecision
if err := e.Ollama.ChatJSONModel(ctx, e.Cfg.ArticleReviewModel, articleQualitySystemPrompt(e.Cfg.ArticleLanguage), e.articleQualityContext(draft, articleType, sources, researchResults), articleQualitySchema(), &decision); err != nil {
reviewResearch := researchResults
if e.RuntimeSettings().ProcessingMode == "clustered" {
reviewResearch = selectReviewEvidence(researchResults, e.Cfg.ClusterReviewEvidence)
}
if err := e.Ollama.ChatJSONModel(ctx, e.Cfg.ArticleReviewModel, articleQualitySystemPrompt(e.Cfg.ArticleLanguage), e.articleQualityContext(draft, articleType, sources, reviewResearch), articleQualitySchema(), &decision); err != nil {
return model.ArticleQualityDecision{}, err
}
if containsDraftMetaContent(draft) {
@@ -945,6 +1060,63 @@ func (e *Engine) reviewArticleContent(ctx context.Context, draft model.Knowledge
return decision, nil
}
func selectReviewEvidence(results []model.ResearchResult, limit int) []model.ResearchResult {
if limit <= 0 || len(results) <= limit {
return append([]model.ResearchResult(nil), results...)
}
type scored struct {
result model.ResearchResult
score float64
domain string
}
items := make([]scored, 0, len(results))
for _, result := range results {
score := result.Relevance*.48 + result.SourceQualityScore*.42
if result.Actionable {
score += .10
}
if result.Fetched {
score += .04
}
items = append(items, scored{result: result, score: score, domain: graph.SourceFromURL(result.URL)})
}
sort.SliceStable(items, func(i, j int) bool {
if items[i].score == items[j].score {
return items[i].result.URL < items[j].result.URL
}
return items[i].score > items[j].score
})
out := make([]model.ResearchResult, 0, limit)
seenDomain := map[string]bool{}
for _, item := range items {
if len(out) >= limit {
break
}
if item.domain != "" && seenDomain[item.domain] {
continue
}
out = append(out, item.result)
seenDomain[item.domain] = true
}
if len(out) < limit {
seenURL := map[string]bool{}
for _, result := range out {
seenURL[result.URL] = true
}
for _, item := range items {
if len(out) >= limit {
break
}
if seenURL[item.result.URL] {
continue
}
out = append(out, item.result)
seenURL[item.result.URL] = true
}
}
return out
}
func (e *Engine) articleRewriteContext(sources []articleSource, plan model.ArticlePlanDecision, brief model.KnowledgeBrief, researchResults []model.ResearchResult, rejected string) string {
var b strings.Builder
fmt.Fprintf(&b, "SCHREIBAUFTRAG: Formuliere einen vollständigen, direkt nutzbaren Helpdesk-Wissensartikel.\nARTIKELTYP: %s\n", nonempty(plan.ArticleType, "how_to"))
@@ -964,6 +1136,10 @@ func (e *Engine) articleRewriteContext(sources []articleSource, plan model.Artic
func (e *Engine) articleQualityContext(draft model.KnowledgeArticleDraft, articleType string, sources []articleSource, researchResults []model.ResearchResult) string {
var b strings.Builder
contextLimit := e.Cfg.MaxContextChars
if e.RuntimeSettings().ProcessingMode == "clustered" && e.Cfg.ClusterReviewContextChars > 0 && (contextLimit <= 0 || e.Cfg.ClusterReviewContextChars < contextLimit) {
contextLimit = e.Cfg.ClusterReviewContextChars
}
fmt.Fprintf(&b, "ZU PRÜFENDER SICHTBARER KB-ARTIKEL:\nARTIKELTYP: %s\n\nTITEL:\n", nonempty(articleType, "how_to"))
b.WriteString(draft.Title)
b.WriteString("\n\nPROBLEM / BESCHREIBUNG:\n")
@@ -971,10 +1147,10 @@ func (e *Engine) articleQualityContext(draft model.KnowledgeArticleDraft, articl
b.WriteString("\n\nLÖSUNG / ANTWORT:\n")
b.WriteString(formatArticleAnswer(draft, e.Cfg.ArticleLanguage))
b.WriteString("\n\nINTERNE BELEGQUELLEN:\n")
appendArticleSources(&b, sources, e.Cfg.MaxContextChars)
appendArticleSources(&b, sources, contextLimit)
if len(researchResults) > 0 {
b.WriteString("\nVOLLTEXT-RECHERCHEMATERIAL:\n")
appendResearchEvidence(&b, researchResults, e.Cfg.MaxContextChars)
appendResearchEvidence(&b, researchResults, contextLimit)
}
return b.String()
}

View File

@@ -5,6 +5,7 @@ import (
"testing"
"github.com/local/glpi-neural-brain/internal/config"
"github.com/local/glpi-neural-brain/internal/graph"
"github.com/local/glpi-neural-brain/internal/model"
)
@@ -111,3 +112,23 @@ func TestArticleDraftValidationMetadataIsStructured(t *testing.T) {
t.Fatalf("unexpected validation metadata: %#v", metadata)
}
}
func TestSelectReviewEvidenceLimitsAndDiversifies(t *testing.T) {
results := []model.ResearchResult{
{Title: "A1", URL: "https://a.example/1", Relevance: .9, SourceQualityScore: .9, Fetched: true},
{Title: "A2", URL: "https://a.example/2", Relevance: .89, SourceQualityScore: .9, Fetched: true},
{Title: "B", URL: "https://b.example/1", Relevance: .8, SourceQualityScore: .95, Fetched: true},
{Title: "C", URL: "https://c.example/1", Relevance: .7, SourceQualityScore: .8, Fetched: true},
}
selected := selectReviewEvidence(results, 3)
if len(selected) != 3 {
t.Fatalf("expected 3 evidence items, got %d", len(selected))
}
domains := map[string]bool{}
for _, result := range selected {
domains[graph.SourceFromURL(result.URL)] = true
}
if len(domains) != 3 {
t.Fatalf("expected domain diversity, got %+v", selected)
}
}

View File

@@ -494,7 +494,8 @@ func (e *Engine) resolveAutonomousTaskSeeds(ctx context.Context, task model.Rese
if err != nil || len(vecs) == 0 {
return out
}
for _, hit := range e.Graph.SimilarFiltered(vecs[0], e.Cfg.ArticleMaxSources*2, e.effectiveThinkingFilter()) {
hits, _ := e.similarKnowledge(vecs[0], e.Cfg.ArticleMaxSources*2, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth)
for _, hit := range hits {
if seen[hit.NodeID] {
continue
}

View File

@@ -34,14 +34,17 @@ var (
)
type EnrichOutcome struct {
Result string
Candidate bool
Created bool
RelationCreated bool
ArticleCreated bool
ArticleSkipped bool
Rejected bool
Comparisons int
Result string
Candidate bool
Created bool
RelationCreated bool
ArticleCreated bool
ArticleSkipped bool
Rejected bool
Comparisons int
CoarseComparisons int
IndexedNodes int
CandidatePool int
}
type Engine struct {
@@ -389,9 +392,10 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
e.enrichCycles++
e.stateMu.Unlock()
e.Broker.Publish(model.Activity{Type: "think.cycle.started", Source: "brain", Phase: "autonomous", Message: fmt.Sprintf("Autonomer AI-THINK-Zyklus startet · bis zu %d sequenzielle Prüfungen", e.Cfg.EnrichBatchSize), Strength: .72, Metadata: map[string]any{"trigger": trigger, "batch_size": e.Cfg.EnrichBatchSize, "anchors": e.Cfg.EnrichAnchors}})
e.Broker.Publish(model.Activity{Type: "think.cycle.started", Source: "brain", Phase: "autonomous", Message: fmt.Sprintf("Autonomer AI-THINK-Zyklus startet · bis zu %d sequenzielle Prüfungen", e.Cfg.EnrichBatchSize), Strength: .72, Metadata: map[string]any{"trigger": trigger, "batch_size": e.Cfg.EnrichBatchSize, "anchors": e.Cfg.EnrichAnchors, "processing_mode": e.RuntimeSettings().ProcessingMode}})
created, rejected, checked := 0, 0, 0
exactComparisons, coarseComparisonsTotal, candidatePoolTotal := 0, 0, 0
relationsCreated, articlesCreated, articlesSkipped := 0, 0, 0
result := "completed"
var cycleErr error
@@ -413,6 +417,9 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
break
}
checked++
exactComparisons += outcome.Comparisons
coarseComparisonsTotal += outcome.CoarseComparisons
candidatePoolTotal += outcome.CandidatePool
if outcome.Created {
created++
}
@@ -454,7 +461,7 @@ func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
e.articlesSkipped += uint64(articlesSkipped)
e.stateMu.Unlock()
metadata := map[string]any{"trigger": trigger, "checked": checked, "created": created, "relations_created": relationsCreated, "articles_created": articlesCreated, "articles_skipped": articlesSkipped, "rejected": rejected, "duration_ms": time.Since(started).Milliseconds(), "result": result}
metadata := map[string]any{"trigger": trigger, "checked": checked, "created": created, "relations_created": relationsCreated, "articles_created": articlesCreated, "articles_skipped": articlesSkipped, "rejected": rejected, "duration_ms": time.Since(started).Milliseconds(), "result": result, "processing_mode": e.RuntimeSettings().ProcessingMode, "exact_comparisons": exactComparisons, "coarse_comparisons": coarseComparisonsTotal, "candidate_pool": candidatePoolTotal}
if cycleErr != nil {
e.Broker.Publish(model.Activity{Type: "think.cycle.failed", Source: "brain", Phase: "autonomous", Message: "AI-THINK-Zyklus wurde mit Fehler beendet", Strength: .45, Metadata: metadata})
slog.Warn("enrichment cycle failed", "trigger", trigger, "error", cycleErr)
@@ -636,6 +643,17 @@ func hashEmbedding(s string, dims int) []float64 {
return v
}
func (e *Engine) similarKnowledge(query []float64, limit int, filter graph.NodeFilter, maxAIDepth int) ([]model.Hit, graph.ClusterSearchStats) {
if e.RuntimeSettings().ProcessingMode != "clustered" {
return e.Graph.SimilarFiltered(query, limit, filter), graph.ClusterSearchStats{}
}
candidateLimit := e.Cfg.ClusterCandidatesPerAnchor
if candidateLimit < limit*12 {
candidateLimit = limit * 12
}
return e.Graph.SimilarClusteredFiltered(query, limit, candidateLimit, filter, maxAIDepth, e.Cfg.ClusterHashBits, e.Cfg.ClusterHashTables)
}
func (e *Engine) Query(ctx context.Context, q string) (model.QueryResponse, error) {
e.interactiveInflight.Add(1)
defer e.interactiveInflight.Add(-1)
@@ -649,7 +667,10 @@ func (e *Engine) Query(ctx context.Context, q string) (model.QueryResponse, erro
if err != nil || len(vecs) == 0 {
vecs = [][]float64{hashEmbedding(q, 256)}
}
hits := e.Graph.SimilarFiltered(vecs[0], e.Cfg.TopK, e.effectiveLearningFilter())
hits, retrievalStats := e.similarKnowledge(vecs[0], e.Cfg.TopK, e.effectiveLearningFilter(), 0)
if e.RuntimeSettings().ProcessingMode == "clustered" {
e.Broker.Publish(model.Activity{Type: "query.retrieval.clustered", Source: "brain", Phase: "retrieval", Query: q, Message: fmt.Sprintf("Cluster-Retrieval: %d exakte Cosine-Prüfungen nach %d Hash-Vergleichen", retrievalStats.ExactComparisons, retrievalStats.CoarseComparisons), Strength: .28, Metadata: map[string]any{"processing_mode": "clustered", "indexed_nodes": retrievalStats.IndexedNodes, "coarse_comparisons": retrievalStats.CoarseComparisons, "exact_comparisons": retrievalStats.ExactComparisons, "candidate_pool": retrievalStats.CandidatePool}})
}
nodeIDs := make([]string, 0, len(hits))
for i, h := range hits {
nodeIDs = append(nodeIDs, h.NodeID)
@@ -777,13 +798,27 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
e.setOllamaOK(true)
}
a, b, sim, ok, comparisons := e.Graph.NextPairScopedDepth(e.Cfg.SimilarityThreshold, e.Cfg.EnrichAnchors, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth)
processingMode := e.RuntimeSettings().ProcessingMode
var a, b model.Node
var sim float64
var ok bool
comparisons, coarseComparisons, indexedNodes, candidatePool := 0, 0, 0, 0
if processingMode == "clustered" {
var stats graph.ClusterSearchStats
a, b, sim, ok, stats = e.Graph.NextPairClusteredScopedDepth(e.Cfg.SimilarityThreshold, e.Cfg.EnrichAnchors, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth, e.Cfg.ClusterHashBits, e.Cfg.ClusterHashTables, e.Cfg.ClusterCandidatesPerAnchor)
comparisons = stats.ExactComparisons
coarseComparisons = stats.CoarseComparisons
indexedNodes = stats.IndexedNodes
candidatePool = stats.CandidatePool
} else {
a, b, sim, ok, comparisons = e.Graph.NextPairScopedDepth(e.Cfg.SimilarityThreshold, e.Cfg.EnrichAnchors, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth)
}
if !ok {
e.stateMu.Lock()
e.lastAttempt = time.Now().UTC()
e.stateMu.Unlock()
e.Broker.Publish(model.Activity{Type: "think.no_candidate", Source: "brain", Phase: "candidate-search", Message: "Im aktuell geprüften Graphbereich wurde keine ungeprüfte Beziehung oberhalb des Ähnlichkeitsschwellwerts gefunden", Strength: .28, Metadata: map[string]any{"trigger": trigger, "threshold": e.Cfg.SimilarityThreshold, "anchors": e.Cfg.EnrichAnchors, "comparisons": comparisons}})
return EnrichOutcome{Result: "no_candidate", Comparisons: comparisons}, nil
e.Broker.Publish(model.Activity{Type: "think.no_candidate", Source: "brain", Phase: "candidate-search", Message: "Im aktuell geprüften Graphbereich wurde keine ungeprüfte Beziehung oberhalb des Ähnlichkeitsschwellwerts gefunden", Strength: .28, Metadata: map[string]any{"trigger": trigger, "threshold": e.Cfg.SimilarityThreshold, "anchors": e.Cfg.EnrichAnchors, "comparisons": comparisons, "exact_comparisons": comparisons, "coarse_comparisons": coarseComparisons, "indexed_nodes": indexedNodes, "candidate_pool": candidatePool, "processing_mode": processingMode}})
return EnrichOutcome{Result: "no_candidate", Comparisons: comparisons, CoarseComparisons: coarseComparisons, IndexedNodes: indexedNodes, CandidatePool: candidatePool}, nil
}
now := time.Now().UTC()
@@ -791,13 +826,13 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
e.lastAttempt = now
e.lastEnrich = now
e.stateMu.Unlock()
e.Broker.Publish(model.Activity{Type: "think.started", Source: "brain", Phase: "association", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Verwandtschaft wird geprüft · %.0f%% semantische Nähe", sim*100), Strength: .88, Metadata: map[string]any{"trigger": trigger, "semantic_similarity": sim, "source_label": a.Label, "target_label": b.Label, "model": e.Cfg.ChatModel, "candidate_comparisons": comparisons}})
e.Broker.Publish(model.Activity{Type: "think.started", Source: "brain", Phase: "association", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Verwandtschaft wird geprüft · %.0f%% semantische Nähe", sim*100), Strength: .88, Metadata: map[string]any{"trigger": trigger, "semantic_similarity": sim, "source_label": a.Label, "target_label": b.Label, "model": e.Cfg.ChatModel, "candidate_comparisons": comparisons, "exact_comparisons": comparisons, "coarse_comparisons": coarseComparisons, "indexed_nodes": indexedNodes, "candidate_pool": candidatePool, "processing_mode": processingMode}})
system := "Du führst ausschließlich eine Relationserkennung für einen Wissensgraphen durch. Analysiere zwei interne Wissenseinträge, erfinde keine Fakten und entscheide, ob eine belastbare Beziehung besteht. Schreibe keinen Artikel und keine technische Synthese. Wenn externe Fakten zur Relationsentscheidung fehlen, setze needs_research=true. Gib ausschließlich JSON nach Schema zurück."
var decision model.RelationDecision
if err := e.Ollama.ChatJSON(ctx, system, relationContext(a, b, sim), relationSchema(), &decision); err != nil {
e.Broker.Publish(model.Activity{Type: "think.failed", Source: "brain", Phase: "inference", NodeIDs: []string{a.ID, b.ID}, Message: "Qwen-Beziehungsanalyse ist fehlgeschlagen; es wurde nichts gespeichert", Strength: .35, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "model": e.Cfg.ChatModel}})
return EnrichOutcome{Result: "inference_failed", Candidate: true, Comparisons: comparisons}, fmt.Errorf("relation inference failed: %w", err)
return EnrichOutcome{Result: "inference_failed", Candidate: true, Comparisons: comparisons, CoarseComparisons: coarseComparisons, IndexedNodes: indexedNodes, CandidatePool: candidatePool}, fmt.Errorf("relation inference failed: %w", err)
}
var researchResults []model.ResearchResult
@@ -899,7 +934,7 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
}
e.Graph.UpsertEdge(edge)
edge.ID = graph.EdgeID(edge.Source, edge.Target, edge.Type, edge.Origin)
outcome := EnrichOutcome{Result: status, Candidate: true, Comparisons: comparisons}
outcome := EnrichOutcome{Result: status, Candidate: true, Comparisons: comparisons, CoarseComparisons: coarseComparisons, IndexedNodes: indexedNodes, CandidatePool: candidatePool}
if status == "staging" {
outcome.Created = true
outcome.RelationCreated = true
@@ -956,6 +991,9 @@ func (e *Engine) Status() map[string]any {
"article_language": e.Cfg.ArticleLanguage, "article_synthesis_model": e.Cfg.ArticleSynthesisModel, "article_review_model": e.Cfg.ArticleReviewModel, "article_review_repair_rounds": e.Cfg.ArticleReviewRepairRounds, "article_pipeline": "research_generate_review", "research_dedupe": e.researchDedupeStatus(),
"enrich_interval": e.Cfg.EnrichInterval.String(),
"enrich_batch_size": e.Cfg.EnrichBatchSize, "enrich_anchors": e.Cfg.EnrichAnchors,
"processing_mode": e.RuntimeSettings().ProcessingMode, "cluster_hash_bits": e.Cfg.ClusterHashBits, "cluster_hash_tables": e.Cfg.ClusterHashTables,
"cluster_candidates_per_anchor": e.Cfg.ClusterCandidatesPerAnchor, "cluster_article_candidates": e.Cfg.ClusterArticleCandidates,
"cluster_review_evidence": e.Cfg.ClusterReviewEvidence, "cluster_review_context_chars": e.Cfg.ClusterReviewContextChars,
"research_enabled": e.ResearchEnabledForRuntime(), "chat_model": e.Cfg.ChatModel, "embedding_model": e.Cfg.EmbeddingModel,
"searxng": e.ResearchStatus(),
"ollama_pool": e.Ollama.PoolStatus(), "article_model_status": map[string]any{"synthesis": e.Ollama.ModelStatus(e.Cfg.ArticleSynthesisModel), "review": e.Ollama.ModelStatus(e.Cfg.ArticleReviewModel)}, "persistence": e.Persistence.Status(), "graph_storage": e.Graph.StorageStatus(),

View File

@@ -21,6 +21,7 @@ type RuntimeSettings struct {
ViewMode string `json:"view_mode"`
MaxDisplayNodes int `json:"max_display_nodes"`
LowPowerMode bool `json:"low_power_mode"`
ProcessingMode string `json:"processing_mode"`
AutonomousResearchEnabled bool `json:"autonomous_research_enabled"`
AutonomousResearchIdleOnly bool `json:"autonomous_research_idle_only"`
AutonomousResearchMinPriority float64 `json:"autonomous_research_min_priority"`
@@ -52,6 +53,7 @@ func (e *Engine) defaultRuntimeSettings() RuntimeSettings {
ViewMode: e.Cfg.DefaultView,
MaxDisplayNodes: e.Cfg.MaxDisplayNodes,
LowPowerMode: e.Cfg.LowPowerMode,
ProcessingMode: e.Cfg.ProcessingMode,
AutonomousResearchEnabled: e.Cfg.AutonomousResearchEnabled,
AutonomousResearchIdleOnly: e.Cfg.AutonomousResearchIdleOnly,
AutonomousResearchMinPriority: e.Cfg.AutonomousResearchMinPriority,
@@ -74,6 +76,10 @@ func normalizeRuntimeSettings(in RuntimeSettings) RuntimeSettings {
if in.ViewMode != "neural" && in.ViewMode != "honeycomb" && in.ViewMode != "constellation" {
in.ViewMode = "neural"
}
in.ProcessingMode = strings.ToLower(strings.TrimSpace(in.ProcessingMode))
if in.ProcessingMode != "clustered" {
in.ProcessingMode = "precise"
}
if in.MaxDisplayNodes < 0 {
in.MaxDisplayNodes = 0
}
@@ -166,6 +172,7 @@ func mergeRuntimeSettingsJSON(settings *RuntimeSettings, data []byte) {
decode("view_mode", &settings.ViewMode)
decode("max_display_nodes", &settings.MaxDisplayNodes)
decode("low_power_mode", &settings.LowPowerMode)
decode("processing_mode", &settings.ProcessingMode)
decode("autonomous_research_enabled", &settings.AutonomousResearchEnabled)
decode("autonomous_research_idle_only", &settings.AutonomousResearchIdleOnly)
decode("autonomous_research_min_priority", &settings.AutonomousResearchMinPriority)
@@ -210,6 +217,12 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
if settings.AutonomousResearchTasksPerCycle == 0 {
settings.AutonomousResearchTasksPerCycle = previous.AutonomousResearchTasksPerCycle
}
if settings.ProcessingMode == "" {
settings.ProcessingMode = previous.ProcessingMode
}
if settings.ProcessingMode != "precise" && settings.ProcessingMode != "clustered" {
return e.RuntimeSettings(), fmt.Errorf("processing_mode must be precise or clustered")
}
if settings.MaxDisplayNodes < 0 || settings.MaxDisplayNodes > 500000 {
return e.RuntimeSettings(), fmt.Errorf("max_display_nodes must be between 0 and 500000")
}
@@ -272,6 +285,7 @@ func (e *Engine) SetRuntimeSettings(settings RuntimeSettings) (RuntimeSettings,
"view_mode": settings.ViewMode,
"max_display_nodes": settings.MaxDisplayNodes,
"low_power_mode": settings.LowPowerMode,
"processing_mode": settings.ProcessingMode,
"autonomous_research_enabled": settings.AutonomousResearchEnabled,
"autonomous_research_idle_only": settings.AutonomousResearchIdleOnly,
"autonomous_research_min_priority": settings.AutonomousResearchMinPriority,

View File

@@ -89,3 +89,14 @@ func TestRuntimeJSONPatchPreservesAutonomousFields(t *testing.T) {
t.Fatalf("partial runtime patch was not applied: %+v", settings)
}
}
func TestNormalizeRuntimeSettingsProcessingMode(t *testing.T) {
settings := normalizeRuntimeSettings(RuntimeSettings{ProcessingMode: "clustered", AutonomousResearchMaxTasksPerDay: 1, AutonomousResearchTasksPerCycle: 1})
if settings.ProcessingMode != "clustered" {
t.Fatalf("clustered mode lost: %+v", settings)
}
settings = normalizeRuntimeSettings(RuntimeSettings{ProcessingMode: "unknown", AutonomousResearchMaxTasksPerDay: 1, AutonomousResearchTasksPerCycle: 1})
if settings.ProcessingMode != "precise" {
t.Fatalf("invalid mode should fall back to precise: %+v", settings)
}
}

View File

@@ -1120,7 +1120,7 @@ func mergeRunMetrics(metrics map[string]any, activity model.Activity) {
if metrics == nil {
return
}
for _, keys := range [][]string{{"comparisons", "candidate_comparisons"}, {"checked"}, {"relations_created"}, {"articles_created"}, {"articles_skipped"}, {"research_search_results", "result_count"}, {"research_fetched", "pages_fetched"}, {"research_accepted", "evidence_count"}, {"research_rejected"}, {"queries_executed"}, {"batch_count"}, {"duration_ms"}} {
for _, keys := range [][]string{{"comparisons", "candidate_comparisons"}, {"exact_comparisons"}, {"coarse_comparisons"}, {"candidate_pool"}, {"checked"}, {"relations_created"}, {"articles_created"}, {"articles_skipped"}, {"research_search_results", "result_count"}, {"research_fetched", "pages_fetched"}, {"research_accepted", "evidence_count"}, {"research_rejected"}, {"queries_executed"}, {"batch_count"}, {"duration_ms"}} {
name := keys[0]
value := metadataNumber(activity.Metadata, keys...)
if value == 0 {
@@ -1138,6 +1138,9 @@ func mergeRunMetrics(metrics map[string]any, activity model.Activity) {
if modelName := metadataString(activity.Metadata, "model"); modelName != "" {
metrics["model"] = modelName
}
if mode := metadataString(activity.Metadata, "processing_mode"); mode != "" {
metrics["processing_mode"] = mode
}
}
func metadataNumber(metadata map[string]any, keys ...string) float64 {

View File

@@ -0,0 +1,355 @@
package graph
import (
"container/heap"
"math/bits"
"sort"
"github.com/local/glpi-neural-brain/internal/model"
)
// ClusterSearchStats separates cheap semantic-hash work from expensive exact
// cosine work. CoarseComparisons are Hamming-distance operations; only
// ExactComparisons execute a full embedding dot product.
type ClusterSearchStats struct {
IndexedNodes int `json:"indexed_nodes"`
CoarseComparisons int `json:"coarse_comparisons"`
ExactComparisons int `json:"exact_comparisons"`
CandidatePool int `json:"candidate_pool"`
HashBits int `json:"hash_bits"`
HashTables int `json:"hash_tables"`
}
type semanticHashEntry struct {
node model.Node
vector []float32
signatures []uint64
}
type coarseCandidate struct {
index int
distance int
}
type coarseMaxHeap []coarseCandidate
func (h coarseMaxHeap) Len() int { return len(h) }
func (h coarseMaxHeap) Less(i, j int) bool {
if h[i].distance == h[j].distance {
return h[i].index > h[j].index
}
return h[i].distance > h[j].distance
}
func (h coarseMaxHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *coarseMaxHeap) Push(x any) { *h = append(*h, x.(coarseCandidate)) }
func (h *coarseMaxHeap) Pop() any {
old := *h
n := len(old)
x := old[n-1]
*h = old[:n-1]
return x
}
func normalizeClusterConfig(hashBits, hashTables, candidateLimit int) (int, int, int) {
if hashBits < 8 {
hashBits = 8
}
if hashBits > 63 {
hashBits = 63
}
if hashTables < 1 {
hashTables = 1
}
if hashTables > 4 {
hashTables = 4
}
if candidateLimit < 16 {
candidateLimit = 16
}
return hashBits, hashTables, candidateLimit
}
// sparseSemanticHash is a deterministic sparse random-projection hash. Each
// bit samples only six embedding dimensions, so building the coarse index is
// orders of magnitude cheaper than N full 768-dimensional cosine products.
func sparseSemanticHash(v []float32, table, hashBits int) uint64 {
if len(v) == 0 {
return 0
}
var signature uint64
seedBase := uint64(0x9e3779b97f4a7c15) ^ uint64(table+1)*0xbf58476d1ce4e5b9
for bit := 0; bit < hashBits; bit++ {
seed := mix64(seedBase ^ uint64(bit+1)*0x94d049bb133111eb)
var sum float32
for sample := 0; sample < 6; sample++ {
seed = mix64(seed + uint64(sample+1)*0x9e3779b97f4a7c15)
idx := int(seed % uint64(len(v)))
if seed&(1<<63) != 0 {
sum -= v[idx]
} else {
sum += v[idx]
}
}
if sum >= 0 {
signature |= 1 << bit
}
}
return signature
}
func mix64(x uint64) uint64 {
x ^= x >> 30
x *= 0xbf58476d1ce4e5b9
x ^= x >> 27
x *= 0x94d049bb133111eb
x ^= x >> 31
return x
}
func signaturesFor(v []float32, hashBits, hashTables int) []uint64 {
out := make([]uint64, hashTables)
for table := 0; table < hashTables; table++ {
out[table] = sparseSemanticHash(v, table, hashBits)
}
return out
}
func signatureDistance(a, b []uint64) int {
n := len(a)
if len(b) < n {
n = len(b)
}
distance := 0
for i := 0; i < n; i++ {
distance += bits.OnesCount64(a[i] ^ b[i])
}
return distance
}
func pushBestCoarse(h *coarseMaxHeap, candidate coarseCandidate, limit int) {
if h.Len() < limit {
heap.Push(h, candidate)
return
}
worst := (*h)[0]
if candidate.distance < worst.distance || (candidate.distance == worst.distance && candidate.index < worst.index) {
heap.Pop(h)
heap.Push(h, candidate)
}
}
// NextPairClusteredScopedDepth is the low-resource alternative to
// NextPairScopedDepth. It keeps the rotating anchor behaviour but ranks the
// corpus with cheap semantic hashes and computes exact cosine only on top-K.
func (s *Store) NextPairClusteredScopedDepth(min float64, anchorLimit int, filter NodeFilter, maxAIDepth, hashBits, hashTables, candidateLimit int) (model.Node, model.Node, float64, bool, ClusterSearchStats) {
hashBits, hashTables, candidateLimit = normalizeClusterConfig(hashBits, hashTables, candidateLimit)
stats := ClusterSearchStats{HashBits: hashBits, HashTables: hashTables}
s.mu.Lock()
defer s.mu.Unlock()
entries := make([]semanticHashEntry, 0, len(s.nodes))
for _, n := range s.nodes {
if n.Kind != "knowledge" && n.Kind != "ai-think" {
continue
}
if !filter.Matches(n) {
continue
}
if n.Kind == "ai-think" && maxAIDepth > 0 && graphNodeGenerationDepth(n) >= maxAIDepth {
continue
}
v, ok := s.vectors[n.ID]
if !ok || len(v) == 0 {
continue
}
entries = append(entries, semanticHashEntry{node: n, vector: v})
}
if len(entries) < 2 {
return model.Node{}, model.Node{}, 0, false, stats
}
sort.Slice(entries, func(i, j int) bool { return entries[i].node.ID < entries[j].node.ID })
for i := range entries {
entries[i].signatures = signaturesFor(entries[i].vector, hashBits, hashTables)
}
stats.IndexedNodes = len(entries)
if anchorLimit <= 0 || anchorLimit > len(entries) {
anchorLimit = len(entries)
}
start := s.pairCursor % len(entries)
anchorIDs := make(map[string]struct{}, anchorLimit)
for step := 0; step < anchorLimit; step++ {
anchorIDs[entries[(start+step)%len(entries)].node.ID] = struct{}{}
}
// Only materialise blocked neighbours for current anchors. This avoids a
// full pair-key allocation for every one of the ~180k graph edges.
blocked := make(map[string]map[string]struct{}, anchorLimit)
for _, edge := range s.edges {
if _, ok := anchorIDs[edge.Source]; ok {
if blocked[edge.Source] == nil {
blocked[edge.Source] = map[string]struct{}{}
}
blocked[edge.Source][edge.Target] = struct{}{}
}
if _, ok := anchorIDs[edge.Target]; ok {
if blocked[edge.Target] == nil {
blocked[edge.Target] = map[string]struct{}{}
}
blocked[edge.Target][edge.Source] = struct{}{}
}
}
best := -1.0
var bestA, bestB model.Node
evaluated := make(map[string]struct{}, anchorLimit*candidateLimit)
for step := 0; step < anchorLimit; step++ {
i := (start + step) % len(entries)
left := entries[i]
h := &coarseMaxHeap{}
heap.Init(h)
for j := range entries {
if i == j {
continue
}
right := entries[j]
if left.node.Kind == "ai-think" && right.node.Kind == "ai-think" {
continue
}
if neighbors := blocked[left.node.ID]; neighbors != nil {
if _, exists := neighbors[right.node.ID]; exists {
continue
}
}
stats.CoarseComparisons++
pushBestCoarse(h, coarseCandidate{index: j, distance: signatureDistance(left.signatures, right.signatures)}, candidateLimit)
}
candidates := make([]coarseCandidate, h.Len())
for k := len(candidates) - 1; k >= 0; k-- {
candidates[k] = heap.Pop(h).(coarseCandidate)
}
stats.CandidatePool += len(candidates)
for _, candidate := range candidates {
right := entries[candidate.index]
key := pairKey(left.node.ID, right.node.ID)
if _, seen := evaluated[key]; seen {
continue
}
evaluated[key] = struct{}{}
if len(left.vector) != len(right.vector) {
continue
}
stats.ExactComparisons++
score := cosine32(left.vector, right.vector)
if score >= min && score > best {
best, bestA, bestB = score, left.node, right.node
}
}
}
s.pairCursor = (start + anchorLimit) % len(entries)
return bestA, bestB, best, best >= 0, stats
}
// SimilarClusteredFiltered performs approximate nearest-neighbour retrieval by
// semantic hash followed by exact cosine on a bounded shortlist.
func (s *Store) SimilarClusteredFiltered(query []float64, limit, candidateLimit int, filter NodeFilter, maxAIDepth, hashBits, hashTables int) ([]model.Hit, ClusterSearchStats) {
hashBits, hashTables, candidateLimit = normalizeClusterConfig(hashBits, hashTables, candidateLimit)
if limit < 1 {
limit = 1
}
if candidateLimit < limit {
candidateLimit = limit
}
stats := ClusterSearchStats{HashBits: hashBits, HashTables: hashTables}
q := make([]float32, len(query))
for i, value := range query {
q[i] = float32(value)
}
qsig := signaturesFor(q, hashBits, hashTables)
s.mu.RLock()
defer s.mu.RUnlock()
entries := make([]semanticHashEntry, 0, len(s.nodes))
for _, n := range s.nodes {
if n.Kind != "knowledge" && n.Kind != "ai-think" {
continue
}
if !filter.Matches(n) {
continue
}
if n.Kind == "ai-think" && maxAIDepth > 0 && graphNodeGenerationDepth(n) >= maxAIDepth {
continue
}
v, ok := s.vectors[n.ID]
if !ok || len(v) != len(q) {
continue
}
entries = append(entries, semanticHashEntry{node: n, vector: v})
}
sort.Slice(entries, func(i, j int) bool { return entries[i].node.ID < entries[j].node.ID })
stats.IndexedNodes = len(entries)
h := &coarseMaxHeap{}
heap.Init(h)
for i := range entries {
entries[i].signatures = signaturesFor(entries[i].vector, hashBits, hashTables)
stats.CoarseComparisons++
pushBestCoarse(h, coarseCandidate{index: i, distance: signatureDistance(qsig, entries[i].signatures)}, candidateLimit)
}
candidates := make([]coarseCandidate, h.Len())
for i := len(candidates) - 1; i >= 0; i-- {
candidates[i] = heap.Pop(h).(coarseCandidate)
}
stats.CandidatePool = len(candidates)
hits := make([]model.Hit, 0, len(candidates))
for _, candidate := range candidates {
entry := entries[candidate.index]
stats.ExactComparisons++
hits = append(hits, model.Hit{NodeID: entry.node.ID, Label: entry.node.Label, Score: cosine32(q, entry.vector), Kind: entry.node.Kind, Status: entry.node.Status})
}
sort.Slice(hits, func(i, j int) bool {
if hits[i].Score == hits[j].Score {
return hits[i].NodeID < hits[j].NodeID
}
return hits[i].Score > hits[j].Score
})
if len(hits) > limit {
hits = hits[:limit]
}
return hits, stats
}
// NeighborScores returns only non-taxonomy graph links incident to the seeds,
// without allocating a full graph Snapshot.
func (s *Store) NeighborScores(seedIDs map[string]bool) map[string]float64 {
s.mu.RLock()
defer s.mu.RUnlock()
out := map[string]float64{}
for _, edge := range s.edges {
if edge.Status == "rejected" || clusterTaxonomyEdge(edge.Type) {
continue
}
weight := edge.Confidence
if edge.Weight > weight {
weight = edge.Weight
}
if weight < .2 {
weight = .2
}
if seedIDs[edge.Source] {
out[edge.Target] += weight
}
if seedIDs[edge.Target] {
out[edge.Source] += weight
}
}
return out
}
func clusterTaxonomyEdge(edgeType string) bool {
switch edgeType {
case "categorized_as", "mentions", "derived_from":
return true
default:
return false
}
}

View File

@@ -0,0 +1,54 @@
package graph
import (
"fmt"
"testing"
"github.com/local/glpi-neural-brain/internal/model"
)
func newSemanticTestStore(t *testing.T, count int) *Store {
t.Helper()
s := &Store{nodes: map[string]model.Node{}, edges: map[string]model.Edge{}, vectors: map[string][]float32{}, dirtyNodes: map[string]uint64{}, dirtyEdges: map[string]uint64{}, dirtyVectors: map[string]uint64{}, deletedNodes: map[string]uint64{}, deletedEdges: map[string]uint64{}, deletedVectors: map[string]uint64{}}
for i := 0; i < count; i++ {
id := fmt.Sprintf("n-%03d", i)
v := make([]float64, 16)
v[i%16] = 1
v[(i*7+3)%16] += float64((i%5)+1) * .03
s.UpsertNode(model.Node{ID: id, Kind: "knowledge", Status: "production", Label: id})
s.SetVector(id, v)
}
return s
}
func TestClusteredPairSearchUsesFarFewerExactCosines(t *testing.T) {
s := newSemanticTestStore(t, 240)
s.SetVector("n-000", []float64{1, .02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0})
s.SetVector("n-001", []float64{1, .021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0})
_, _, sim, ok, stats := s.NextPairClusteredScopedDepth(.95, 12, NodeFilter{}, 2, 24, 2, 24)
if !ok || sim < .95 {
t.Fatalf("expected clustered candidate, ok=%v sim=%.4f stats=%+v", ok, sim, stats)
}
if stats.ExactComparisons <= 0 || stats.ExactComparisons > 12*24 {
t.Fatalf("unexpected exact work: %+v", stats)
}
if stats.CoarseComparisons < 2000 {
t.Fatalf("expected cheap coarse scan over corpus, got %+v", stats)
}
if stats.ExactComparisons*5 >= stats.CoarseComparisons {
t.Fatalf("cluster mode did not reduce expensive comparisons enough: %+v", stats)
}
}
func TestClusteredSimilarFindsNearDuplicate(t *testing.T) {
s := newSemanticTestStore(t, 160)
target := []float64{.91, .31, .12, .04, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
s.SetVector("n-055", target)
hits, stats := s.SimilarClusteredFiltered(target, 8, 48, NodeFilter{}, 2, 24, 2)
if len(hits) == 0 || hits[0].NodeID != "n-055" || hits[0].Score < .999 {
t.Fatalf("near duplicate not ranked first: hits=%+v stats=%+v", hits, stats)
}
if stats.ExactComparisons > 48 {
t.Fatalf("too many exact comparisons: %+v", stats)
}
}

View File

@@ -159,7 +159,7 @@
$('summaryQualitySub').textContent = 'Komponenten · Wissenswaisen · Widerspruchs-Edges';
const settings = system.runtime_settings || {};
$('summaryThinking').textContent = settings.thinking_enabled ? (system.enrich_running ? 'LÄUFT' : 'AKTIV') : 'PAUSIERT';
$('summaryThinkingSub').textContent = `${num(system.relations_created)} Relationen · ${num(system.articles_created)} Artikel seit Prozessstart`;
$('summaryThinkingSub').textContent = `${num(system.relations_created)} Relationen · ${num(system.articles_created)} Artikel · ${(settings.processing_mode === 'clustered' ? 'CLUSTER/FAST' : 'PRÄZISE')}`;
const autonomous = system.autonomous_research || {};
const queue = autonomous.counts || {};
$('summaryAutonomous').textContent = settings.autonomous_research_enabled ? (autonomous.running ? 'LÄUFT' : 'AKTIV') : 'PAUSIERT';
@@ -246,7 +246,7 @@
if (!filtered.length) { body.innerHTML='<tr><td colspan="6" class="empty-state">Keine Läufe entsprechen den Filtern.</td></tr>'; return; }
body.innerHTML = filtered.map(run => {
const metrics = run.metrics || {};
const metricText = [metrics.comparisons ? `${num(metrics.comparisons)} Vergleiche` : '', metrics.research_search_results ? `${num(metrics.research_search_results)} Treffer` : '', metrics.research_accepted ? `${num(metrics.research_accepted)} Belege` : '', metrics.articles_created ? `${num(metrics.articles_created)} Artikel` : ''].filter(Boolean).join(' · ') || `${num(run.event_count)} Events`;
const metricText = [metrics.exact_comparisons ? `${num(metrics.exact_comparisons)} Cosine` : (metrics.comparisons ? `${num(metrics.comparisons)} Vergleiche` : ''), metrics.coarse_comparisons ? `${num(metrics.coarse_comparisons)} Hash` : '', metrics.research_search_results ? `${num(metrics.research_search_results)} Treffer` : '', metrics.research_accepted ? `${num(metrics.research_accepted)} Belege` : '', metrics.articles_created ? `${num(metrics.articles_created)} Artikel` : ''].filter(Boolean).join(' · ') || `${num(run.event_count)} Events`;
const open = state.openRun === run.id;
return `<tr class="run-row" data-run-id="${esc(run.id)}"><td>${clock(run.started_at)}<span class="run-meta">${new Date(run.started_at).toLocaleDateString('de-DE')}</span></td><td><span class="run-title">${esc(run.title)}</span><span class="run-meta">${esc(run.kind)}${run.trigger?` · ${esc(run.trigger)}`:''}</span></td><td><span class="status-pill ${statusClass(run.status)}">${esc(run.verdict||run.status)}</span><span class="run-meta">${esc(run.outcome||'')}</span></td><td><div class="delta-tags">${deltaTags(run.mutations)}</div></td><td>${esc(metricText)}</td><td>${run.status==='running'?'läuft':duration(run.duration_ms)}</td></tr>${open ? runDetails(run) : ''}`;
}).join('');

View File

@@ -68,3 +68,5 @@ body.low-power .glass{backdrop-filter:blur(12px)}
.filter-scope-summary{margin:2px 0 10px;line-height:1.45}.source-option.disabled{opacity:.38}.source-option.disabled span{cursor:not-allowed;border-style:dashed}.source-option span small{display:block;margin-top:2px;font-size:7px;letter-spacing:.04em;color:#8a6f79}.filter-scope-summary.warn{color:#ffb37f}.filter-scope-summary.ok{color:#7898aa}
.autonomous-research-settings{flex:0 0 auto}.autonomous-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:8px;margin:10px 0}.autonomous-grid label{display:block;padding:8px;border:1px solid rgba(133,200,255,.11);border-radius:10px;background:rgba(255,255,255,.02)}.autonomous-grid span{display:block;font-size:8px;color:#7893a6;margin-bottom:5px}.autonomous-grid input{width:100%;border:1px solid rgba(133,200,255,.14);background:rgba(2,8,17,.62);color:var(--text);border-radius:8px;padding:7px 8px;outline:0}.autonomous-actions{flex-wrap:wrap}.autonomous-actions button{flex:1 1 120px}.queue-title{margin-top:14px}.autonomous-queue{display:flex;flex-direction:column;gap:7px;max-height:260px;overflow:auto;padding-right:3px}.autonomous-task{position:relative;padding:9px 10px;border:1px solid rgba(133,200,255,.11);border-radius:11px;background:rgba(255,255,255,.024)}.autonomous-task.running{border-color:rgba(93,255,189,.3);box-shadow:0 0 18px rgba(93,255,189,.07)}.autonomous-task.failed{border-color:rgba(255,105,125,.24)}.autonomous-task.completed{opacity:.72}.autonomous-task-head{display:flex;align-items:flex-start;justify-content:space-between;gap:8px}.autonomous-task b{font-size:10px;color:#d9f3ff;line-height:1.35}.autonomous-task em{font-style:normal;font-size:8px;color:#82e7ff;border:1px solid rgba(82,231,255,.2);border-radius:999px;padding:2px 6px;white-space:nowrap}.autonomous-task p{margin:5px 0 0;color:#7893a6;font-size:8px;line-height:1.4}.autonomous-task-meta{display:flex;gap:5px;flex-wrap:wrap;margin-top:7px}.autonomous-task-meta span{font-size:7px;color:#86a3b7;background:rgba(255,255,255,.035);border-radius:999px;padding:3px 6px}.autonomous-task button{position:absolute;right:8px;bottom:8px;border:0;background:transparent;color:#ff8ba0;font-size:8px;cursor:pointer}.autonomous-task button:hover{color:#ffc0ca}@media(max-width:720px){.autonomous-grid{grid-template-columns:1fr}.autonomous-actions button{flex-basis:100%}}
.dock .analysis-dashboard-link{display:inline-flex;align-items:center;border:1px solid rgba(82,231,255,.2);background:rgba(82,231,255,.055);color:#a9dbe8;border-radius:11px;padding:9px 11px;font-weight:700;letter-spacing:.08em;font-size:10px;white-space:nowrap}.dock .analysis-dashboard-link:hover{background:rgba(82,231,255,.12);color:#e5fbff}
.settings-processing-mode{display:grid;grid-template-columns:1fr 1fr;gap:6px;margin-top:7px}.settings-processing-mode button{border:1px solid rgba(133,200,255,.14);background:rgba(255,255,255,.025);color:#7893a6;border-radius:10px;padding:8px 9px;font-size:9px;letter-spacing:.08em;cursor:pointer}.settings-processing-mode button.active{color:var(--cyan);border-color:rgba(82,231,255,.38);background:rgba(82,231,255,.1);box-shadow:inset 0 0 18px rgba(82,231,255,.035)}

View File

@@ -51,7 +51,7 @@
lodOpenUntil: new Map(), lodHotUntil: new Map(), lodDirty: true, lodLastBuild: 0, lodNextExpiry: 0, lodZoomBand: 2,
renderNodes: [], renderEdges: [], renderIdleEdges: [], renderNodeById: new Map(), renderEdgeById: new Map(), visibleForNode: new Map(),
edgeRenderMap: new Map(), renderActive: new Map(), renderEdgeActive: new Map(), renderStats: {nodes: 0, edges: 0, hiddenNodes: 0, hiddenEdges: 0},
fullSnapshot: null, fullNodeById: new Map(), runtimeSettings: {source_filter_version: 1, learning_enabled: true, thinking_enabled: true, learning_sources: [], display_sources: [], thinking_sources: [], glpi_kb_source: '', view_mode: 'neural', max_display_nodes: 0, low_power_mode: false, autonomous_research_enabled: false, autonomous_research_idle_only: true, autonomous_research_min_priority: 0.65, autonomous_research_max_tasks_per_day: 12, autonomous_research_tasks_per_cycle: 1},
fullSnapshot: null, fullNodeById: new Map(), runtimeSettings: {source_filter_version: 1, learning_enabled: true, thinking_enabled: true, learning_sources: [], display_sources: [], thinking_sources: [], glpi_kb_source: '', view_mode: 'neural', max_display_nodes: 0, low_power_mode: false, processing_mode: 'precise', autonomous_research_enabled: false, autonomous_research_idle_only: true, autonomous_research_min_priority: 0.65, autonomous_research_max_tasks_per_day: 12, autonomous_research_tasks_per_cycle: 1},
availableSources: [], viewMode: 'neural', honeycombNodes: [], honeycombSpacing: 0, honeySlotByID: new Map(), honeyPointPool: [], honeyFreeSlots: [],
constellationNodes: [], constellationLinks: [], settingsOpen: false, settingsDraft: null,
forcedDisplayUntil: new Map(), nextDisplayLimitExpiry: 0, graphVersion: null, displaySignature: '', displayLimitStats: {limit: 0, eligible: 0, shown: 0},
@@ -328,6 +328,9 @@
if ($('settingsViewHoneycomb')) $('settingsViewHoneycomb').classList.toggle('active', panelSettings.view_mode === 'honeycomb');
if ($('settingsViewConstellation')) $('settingsViewConstellation').classList.toggle('active', panelSettings.view_mode === 'constellation');
if ($('settingsLowPower')) $('settingsLowPower').checked = Boolean(panelSettings.low_power_mode);
if ($('settingsProcessingPrecise')) $('settingsProcessingPrecise').classList.toggle('active', panelSettings.processing_mode !== 'clustered');
if ($('settingsProcessingClustered')) $('settingsProcessingClustered').classList.toggle('active', panelSettings.processing_mode === 'clustered');
if ($('processingModeHint')) $('processingModeHint').textContent = panelSettings.processing_mode === 'clustered' ? 'Cluster/Fast: Semantic Hashing ersetzt den Vollscan; nur Top-K-Kandidaten erhalten exakte Cosine-Bewertungen. Der Artikelreview nutzt einen kompakten, priorisierten Evidenzsatz.' : 'Präzise: vollständige Cosine-Suche über jeden Anchor und den gesamten gefilterten Wissensraum.';
if ($('settingsMaxDisplayNodes')) $('settingsMaxDisplayNodes').value = String(Math.max(0, Number(panelSettings.max_display_nodes || 0)));
if ($('settingsAutonomousResearch')) $('settingsAutonomousResearch').checked = Boolean(panelSettings.autonomous_research_enabled);
if ($('settingsAutonomousIdleOnly')) $('settingsAutonomousIdleOnly').checked = panelSettings.autonomous_research_idle_only !== false;
@@ -423,6 +426,7 @@
view_mode: ['neural', 'honeycomb', 'constellation'].includes(settings.view_mode) ? settings.view_mode : 'neural',
max_display_nodes: Math.max(0, Math.min(500000, Math.trunc(Number(settings.max_display_nodes) || 0))),
low_power_mode: Boolean(settings.low_power_mode),
processing_mode: settings.processing_mode === 'clustered' ? 'clustered' : 'precise',
autonomous_research_enabled: Boolean(settings.autonomous_research_enabled),
autonomous_research_idle_only: settings.autonomous_research_idle_only !== false,
autonomous_research_min_priority: Math.max(0, Math.min(1, Number(settings.autonomous_research_min_priority ?? 0.65))),
@@ -3067,6 +3071,8 @@
$('sourceSearch').addEventListener('input', e => renderSourceFilters(e.currentTarget.value));
$('settingsLearning').addEventListener('change', e => { if (state.settingsDraft) state.settingsDraft.learning_enabled = e.currentTarget.checked; });
$('settingsThinking').addEventListener('change', e => { if (state.settingsDraft) state.settingsDraft.thinking_enabled = e.currentTarget.checked; });
$('settingsProcessingPrecise')?.addEventListener('click', () => { if (state.settingsDraft) { state.settingsDraft.processing_mode = 'precise'; syncRuntimeControls(); } });
$('settingsProcessingClustered')?.addEventListener('click', () => { if (state.settingsDraft) { state.settingsDraft.processing_mode = 'clustered'; syncRuntimeControls(); } });
$('settingsLowPower').addEventListener('change', e => { if (state.settingsDraft) state.settingsDraft.low_power_mode = e.currentTarget.checked; });
$('settingsAutonomousResearch')?.addEventListener('change', e => { if (state.settingsDraft) state.settingsDraft.autonomous_research_enabled = e.currentTarget.checked; });
$('settingsAutonomousIdleOnly')?.addEventListener('change', e => { if (state.settingsDraft) state.settingsDraft.autonomous_research_idle_only = e.currentTarget.checked; });
@@ -3121,6 +3127,7 @@
state.settingsDraft.learning_enabled = $('settingsLearning').checked;
state.settingsDraft.thinking_enabled = $('settingsThinking').checked;
state.settingsDraft.low_power_mode = $('settingsLowPower').checked;
state.settingsDraft.processing_mode = state.settingsDraft.processing_mode === 'clustered' ? 'clustered' : 'precise';
state.settingsDraft.max_display_nodes = Math.max(0, Math.min(500000, Math.trunc(Number($('settingsMaxDisplayNodes').value) || 0)));
state.settingsDraft.autonomous_research_enabled = Boolean($('settingsAutonomousResearch')?.checked);
state.settingsDraft.autonomous_research_idle_only = $('settingsAutonomousIdleOnly')?.checked !== false;

View File

@@ -104,6 +104,12 @@
<span><b>Eco-Modus</b><small>30 FPS, geringere Pixeldichte und günstigere Glow-Effekte. Aktivität, Rechercheanimationen und Fades bleiben sichtbar.</small></span>
<input id="settingsLowPower" type="checkbox">
</label>
<div class="settings-section-title performance-subtitle"><h2>Brain-Verarbeitung</h2><span>Relationen & Artikel</span></div>
<div class="settings-processing-mode" role="group" aria-label="Verarbeitungsmodus">
<button id="settingsProcessingPrecise" class="active" type="button">PRÄZISE</button>
<button id="settingsProcessingClustered" type="button">CLUSTER / FAST</button>
</div>
<p id="processingModeHint" class="setting-hint">Präzise: vollständige Cosine-Suche. Cluster/Fast: Semantic Hashing + exakte Top-K-Prüfung und kompakterer Artikelreview.</p>
<div class="settings-section-title performance-subtitle"><h2>GPU-Limit</h2><span>0 = unbegrenzt</span></div>
<label class="number-setting" for="settingsMaxDisplayNodes">
<span><b>Maximal angezeigte Nodes</b><small>Begrenzt Neural- und Honeycomb-Ansicht. Aktive Notes werden bei Bedarf temporär eingeblendet und ersetzen inaktive Nodes.</small></span>