Update 6 SQLite

2026-08-05 05:23:43 +02:00
parent 7e36c2f3f0
commit b4388d1ebc
40 changed files with 2514 additions and 229 deletions
@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"log/slog"
 	"math"
 	"os"
 	"path/filepath"
@@ -467,12 +468,16 @@ func (e *Engine) researchKnowledgeGaps(ctx context.Context, trigger string, node
 		if resultLimit < 1 {
 			resultLimit = 4
 		}
-		results, err := e.Research.Search(ctx, query, resultLimit)
+		results, diagnostic, err := e.Research.SearchDetailed(ctx, query, resultLimit)
 		if err != nil {
-			e.Broker.Publish(model.Activity{Type: "article.research.failed", Source: "searxng", Phase: "knowledge-research", NodeIDs: nodeIDs, Message: "Die ergänzende Artikelrecherche ist fehlgeschlagen", Strength: .35, Metadata: mergeResearchMetadata(startMetadata, map[string]any{"error": err.Error(), "duration_ms": time.Since(started).Milliseconds()})})
+			metadata := mergeResearchMetadata(startMetadata, researchDiagnosticMetadata(diagnostic))
+			metadata["error"] = err.Error()
+			metadata["duration_ms"] = time.Since(started).Milliseconds()
+			slog.Warn("article research failed", "query", query, "base_url", diagnostic.BaseURL, "kind", diagnostic.ErrorKind, "http_status", diagnostic.HTTPStatus, "duration_ms", diagnostic.DurationMS, "error", err)
+			e.Broker.Publish(model.Activity{Type: "article.research.failed", Source: "searxng", Phase: "knowledge-research", NodeIDs: nodeIDs, Message: "Die ergänzende Artikelrecherche ist fehlgeschlagen", Strength: .35, Metadata: metadata})
 			return nil, err
 		}
-		resultMetadata := researchEventMetadata(trigger, researchID, query, results, time.Since(started))
+		resultMetadata := mergeResearchMetadata(researchEventMetadata(trigger, researchID, query, results, time.Since(started)), researchDiagnosticMetadata(diagnostic))
 		message := fmt.Sprintf("SearXNG hat %d Quellen für den offenen Wissenspunkt geliefert", len(results))
 		if len(results) == 0 {
 			message = "SearXNG hat für den offenen Wissenspunkt keine verwertbare Quelle geliefert"
@@ -124,6 +124,10 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
 		}
 		nodes = append(nodes, ollama.NodeConfig{Name: name, URL: rawURL, Weight: weight})
 	}
+	clearedVectors := g.ConfigureEmbeddingModel(cfg.EmbeddingModel)
+	if clearedVectors > 0 && b != nil {
+		b.Publish(model.Activity{Type: "embedding.model_changed", Source: "brain", Phase: "learning", Message: fmt.Sprintf("Embedding-Modell geändert · %d Vektoren werden neu gelernt", clearedVectors), Strength: .65, Metadata: map[string]any{"embedding_model": cfg.EmbeddingModel, "cleared_vectors": clearedVectors}})
+	}
 	pool := ollama.NewPool(ollama.PoolConfig{
 		Nodes: nodes, RoutingMode: cfg.OllamaRoutingMode, NodeMaxInflight: cfg.OllamaNodeMaxInflight,
 		HealthInterval: cfg.OllamaHealthInterval, FailureCooldown: cfg.OllamaFailureCooldown,
@@ -361,16 +365,23 @@ func (e *Engine) idle(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			s := e.Graph.Snapshot()
-			if len(s.Nodes) == 0 {
+			n, ok := e.Graph.IdleNode(time.Now().Unix() / 7)
+			if !ok {
 				continue
 			}
-			idx := int(time.Now().Unix()/7) % len(s.Nodes)
-			n := s.Nodes[idx]
 			e.Broker.Publish(model.Activity{Type: "brain.idle", Source: "brain", Phase: "idle", Message: "Leise Hintergrundaktivität", NodeIDs: []string{n.ID}, Strength: .18})
 		}
 	}
 }
+func (e *Engine) configureEmbeddingDigest() int {
+	for _, node := range e.Ollama.NodeStatuses() {
+		if node.Healthy && node.Compatible && node.EmbeddingModel && strings.TrimSpace(node.EmbeddingDigest) != "" {
+			return e.Graph.ConfigureEmbeddingIdentity(e.Cfg.EmbeddingModel, node.EmbeddingDigest)
+		}
+	}
+	return 0
+}
+
 func (e *Engine) Scan(ctx context.Context) error {
 	if !e.LearningEnabled() {
 		return ErrLearningDisabled
@@ -394,6 +405,9 @@ func (e *Engine) Scan(ctx context.Context) error {
 		e.ensureFallbackEmbeddings()
 		e.setOllamaOK(false)
 	} else {
+		if cleared := e.configureEmbeddingDigest(); cleared > 0 {
+			e.Broker.Publish(model.Activity{Type: "embedding.identity_changed", Source: "brain", Phase: "learning", Message: fmt.Sprintf("Embedding-Digest geändert · %d Vektoren werden neu gelernt", cleared), Strength: .7, Metadata: map[string]any{"embedding_model": e.Cfg.EmbeddingModel, "cleared_vectors": cleared}})
+		}
 		// Local fallback vectors use 256 dimensions. Once Ollama becomes available,
 		// discard those placeholders and replace them with real model embeddings.
 		e.Graph.ClearVectorsByDimension(256)
@@ -409,8 +423,8 @@ func (e *Engine) Scan(ctx context.Context) error {
 	e.lastScan = time.Now().UTC()
 	e.stateMu.Unlock()
 	if e.Graph.Version() != beforeVersion {
-		s := e.Graph.Snapshot()
-		e.Broker.Publish(model.Activity{Type: "graph.updated", Source: "brain", Phase: "indexed", Message: fmt.Sprintf("%d Wissenselemente · %d Nodes · %d Edges", count, len(s.Nodes), len(s.Edges)), Strength: .55, Metadata: map[string]any{"nodes": len(s.Nodes), "edges": len(s.Edges), "knowledge_elements": count}})
+		nodes, edges, _ := e.Graph.Counts()
+		e.Broker.Publish(model.Activity{Type: "graph.updated", Source: "brain", Phase: "indexed", Message: fmt.Sprintf("%d Wissenselemente · %d Nodes · %d Edges", count, nodes, edges), Strength: .55, Metadata: map[string]any{"nodes": nodes, "edges": edges, "knowledge_elements": count}})
 	}
 	return nil
 }
@@ -631,12 +645,15 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
 		researchStarted := time.Now()
 		startMetadata := map[string]any{"trigger": trigger, "research_id": researchID, "research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label, "animation_min_ms": 2000}
 		e.Broker.Publish(model.Activity{Type: "research.started", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · SearXNG durchsucht externe Quellen", Strength: .9, Metadata: startMetadata})
-		results, err := e.Research.Search(ctx, decision.ResearchQuery, 4)
+		results, diagnostic, err := e.Research.SearchDetailed(ctx, decision.ResearchQuery, 4)
 		if err != nil {
-			slog.Warn("research failed", "error", err)
-			e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "SearXNG-Recherche ist fehlgeschlagen", Strength: .35, Metadata: mergeResearchMetadata(startMetadata, map[string]any{"error": err.Error(), "duration_ms": time.Since(researchStarted).Milliseconds()})})
+			metadata := mergeResearchMetadata(startMetadata, researchDiagnosticMetadata(diagnostic))
+			metadata["error"] = err.Error()
+			metadata["duration_ms"] = time.Since(researchStarted).Milliseconds()
+			slog.Warn("research failed", "query", decision.ResearchQuery, "base_url", diagnostic.BaseURL, "kind", diagnostic.ErrorKind, "http_status", diagnostic.HTTPStatus, "duration_ms", diagnostic.DurationMS, "error", err)
+			e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "SearXNG-Recherche ist fehlgeschlagen", Strength: .35, Metadata: metadata})
 		} else {
-			resultMetadata := researchEventMetadata(trigger, researchID, decision.ResearchQuery, results, time.Since(researchStarted))
+			resultMetadata := mergeResearchMetadata(researchEventMetadata(trigger, researchID, decision.ResearchQuery, results, time.Since(researchStarted)), researchDiagnosticMetadata(diagnostic))
 			message := fmt.Sprintf("SearXNG hat %d verwertbare Webquellen geliefert", len(results))
 			if len(results) == 0 {
 				message = "SearXNG hat keine verwertbaren Webquellen geliefert"
@@ -706,10 +723,10 @@ func (e *Engine) addResearch(a, b model.Node, results []model.ResearchResult) re
 	return uniqueResearchRefs(refs)
 }
 func (e *Engine) Status() map[string]any {
-	s := e.Graph.Snapshot()
+	nodes, edges, version := e.Graph.Counts()
 	e.stateMu.RLock()
 	status := map[string]any{
-		"ok": true, "nodes": len(s.Nodes), "edges": len(s.Edges), "version": s.Version,
+		"ok": true, "nodes": nodes, "edges": edges, "version": version,
 		"last_scan": e.lastScan, "last_enrich": e.lastEnrich, "last_enrich_attempt": e.lastAttempt,
 		"next_enrich": e.nextEnrich, "ollama_ok": e.ollamaOK, "auto_enrich": e.Cfg.AutoEnrich,
 		"enrich_running": e.enrichRunning, "enrich_trigger": e.enrichTrigger, "enrich_result": e.enrichResult,
@@ -722,7 +739,8 @@ func (e *Engine) Status() map[string]any {
 		"enrich_interval":   e.Cfg.EnrichInterval.String(),
 		"enrich_batch_size": e.Cfg.EnrichBatchSize, "enrich_anchors": e.Cfg.EnrichAnchors,
 		"research_enabled": e.Cfg.ResearchEnabled, "chat_model": e.Cfg.ChatModel, "embedding_model": e.Cfg.EmbeddingModel,
-		"ollama_pool": e.Ollama.PoolStatus(), "persistence": e.Persistence.Status(),
+		"searxng":     e.ResearchStatus(),
+		"ollama_pool": e.Ollama.PoolStatus(), "persistence": e.Persistence.Status(), "graph_storage": e.Graph.StorageStatus(),
 		"runtime_settings": e.RuntimeSettings(),
 	}
 	if e.GLPIKB != nil {
@@ -738,6 +756,13 @@ func (e *Engine) Flush(ctx context.Context) error {
 	return e.Persistence.Flush(ctx, "manual")
 }

+func (e *Engine) ExportGraph(ctx context.Context, destination string) error {
+	if err := e.Persistence.Flush(ctx, "export"); err != nil {
+		return err
+	}
+	return e.Graph.Export(ctx, destination)
+}
+
 func (e *Engine) SyncGLPIKB(ctx context.Context) error {
 	if !e.LearningEnabled() {
 		return ErrLearningDisabled
@@ -0,0 +1,103 @@
+package engine
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/local/glpi-neural-brain/internal/model"
+	"github.com/local/glpi-neural-brain/internal/research"
+)
+
+type ResearchTestResult struct {
+	OK         bool                   `json:"ok"`
+	Query      string                 `json:"query"`
+	Diagnostic research.Diagnostic    `json:"diagnostic"`
+	Results    []model.ResearchResult `json:"results"`
+}
+
+func (e *Engine) ResearchStatus() research.Diagnostic {
+	if e == nil || e.Research == nil {
+		return research.Diagnostic{Configured: false, OK: false, ErrorKind: "not_configured", Error: "SearXNG ist nicht konfiguriert"}
+	}
+	status := e.Research.Status()
+	if !e.Cfg.ResearchEnabled {
+		status.OK = false
+		status.ErrorKind = "disabled"
+		status.Error = "BRAIN_RESEARCH_ENABLED ist deaktiviert"
+	}
+	return status
+}
+
+func (e *Engine) TestResearch(ctx context.Context, query string, limit int) (ResearchTestResult, error) {
+	query = strings.TrimSpace(query)
+	if query == "" {
+		query = "Btrfs Snapshots und ZFS History Unterschiede Timeline"
+	}
+	if limit < 1 {
+		limit = 4
+	}
+	if limit > 10 {
+		limit = 10
+	}
+	if !e.Cfg.ResearchEnabled {
+		err := errors.New("SearXNG-Recherche ist deaktiviert: BRAIN_RESEARCH_ENABLED=false")
+		return ResearchTestResult{Query: query, Diagnostic: e.ResearchStatus()}, err
+	}
+	if e.Research == nil {
+		err := errors.New("SearXNG-Client ist nicht konfiguriert")
+		return ResearchTestResult{Query: query, Diagnostic: e.ResearchStatus()}, err
+	}
+
+	researchID := newResearchRunID("research-test", query)
+	started := time.Now()
+	startMetadata := map[string]any{
+		"trigger":          "diagnostic",
+		"research_id":      researchID,
+		"research_query":   query,
+		"animation_min_ms": 2000,
+		"test_only":        true,
+		"searxng_base_url": e.Research.Status().BaseURL,
+	}
+	e.Broker.Publish(model.Activity{Type: "research.test.started", Source: "searxng", Phase: "diagnostic", Message: "SearXNG-Verbindung und JSON-Suche werden direkt getestet", Strength: .92, Metadata: startMetadata})
+
+	results, diagnostic, err := e.Research.SearchDetailed(ctx, query, limit)
+	if err != nil {
+		metadata := mergeResearchMetadata(startMetadata, researchDiagnosticMetadata(diagnostic))
+		metadata["duration_ms"] = time.Since(started).Milliseconds()
+		metadata["error"] = err.Error()
+		slog.Warn("SearXNG diagnostic search failed", "base_url", diagnostic.BaseURL, "kind", diagnostic.ErrorKind, "duration_ms", diagnostic.DurationMS, "error", err)
+		e.Broker.Publish(model.Activity{Type: "research.test.failed", Source: "searxng", Phase: "diagnostic", Message: "SearXNG-Test fehlgeschlagen", Strength: .4, Metadata: metadata})
+		return ResearchTestResult{OK: false, Query: query, Diagnostic: diagnostic}, err
+	}
+
+	metadata := mergeResearchMetadata(researchEventMetadata("diagnostic", researchID, query, results, time.Since(started)), researchDiagnosticMetadata(diagnostic))
+	metadata["test_only"] = true
+	message := fmt.Sprintf("SearXNG-Test erfolgreich · %d verwertbare Treffer", len(results))
+	if len(results) == 0 {
+		message = "SearXNG-Test erfolgreich · JSON-Antwort erhalten, aber keine verwertbaren Treffer"
+	}
+	e.Broker.Publish(model.Activity{Type: "research.test.results", Source: "searxng", Phase: "diagnostic", Message: message, Strength: 1, Metadata: metadata})
+	return ResearchTestResult{OK: true, Query: query, Diagnostic: diagnostic, Results: results}, nil
+}
+
+func researchDiagnosticMetadata(diagnostic research.Diagnostic) map[string]any {
+	metadata := map[string]any{
+		"searxng_configured":   diagnostic.Configured,
+		"searxng_ok":           diagnostic.OK,
+		"searxng_base_url":     diagnostic.BaseURL,
+		"searxng_http_status":  diagnostic.HTTPStatus,
+		"searxng_content_type": diagnostic.ContentType,
+		"error_kind":           diagnostic.ErrorKind,
+	}
+	if diagnostic.DurationMS > 0 {
+		metadata["duration_ms"] = diagnostic.DurationMS
+	}
+	if diagnostic.Error != "" {
+		metadata["error"] = diagnostic.Error
+	}
+	return metadata
+}
@@ -0,0 +1,70 @@
+package engine
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/local/glpi-neural-brain/internal/activity"
+	"github.com/local/glpi-neural-brain/internal/config"
+	"github.com/local/glpi-neural-brain/internal/research"
+)
+
+func TestResearchDiagnosticPublishesVisibleEvents(t *testing.T) {
+	searx := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"results":[{"title":"ZFS history","url":"https://example.test/zfs","content":"Timeline evidence"}]}`))
+	}))
+	defer searx.Close()
+
+	broker := activity.New(20)
+	eng := &Engine{
+		Cfg:      config.Config{ResearchEnabled: true},
+		Research: research.New(searx.URL),
+		Broker:   broker,
+	}
+	result, err := eng.TestResearch(context.Background(), "zfs timeline", 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !result.OK || len(result.Results) != 1 || !result.Diagnostic.OK {
+		t.Fatalf("unexpected result: %+v", result)
+	}
+	events := broker.Recent()
+	if len(events) != 2 || events[0].Type != "research.test.started" || events[1].Type != "research.test.results" {
+		t.Fatalf("unexpected events: %+v", events)
+	}
+	if events[1].Metadata["result_count"] != 1 {
+		t.Fatalf("result count missing from event: %+v", events[1].Metadata)
+	}
+}
+
+func TestResearchDiagnosticIncludesFailureReason(t *testing.T) {
+	searx := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+		_, _ = w.Write([]byte("json disabled"))
+	}))
+	defer searx.Close()
+
+	broker := activity.New(20)
+	eng := &Engine{
+		Cfg:      config.Config{ResearchEnabled: true},
+		Research: research.New(searx.URL),
+		Broker:   broker,
+	}
+	result, err := eng.TestResearch(context.Background(), "zfs timeline", 4)
+	if err == nil {
+		t.Fatal("expected failure")
+	}
+	if result.Diagnostic.ErrorKind != "http_status" || result.Diagnostic.HTTPStatus != http.StatusForbidden {
+		t.Fatalf("unexpected diagnostic: %+v", result.Diagnostic)
+	}
+	events := broker.Recent()
+	if len(events) != 2 || events[1].Type != "research.test.failed" {
+		t.Fatalf("unexpected events: %+v", events)
+	}
+	if events[1].Metadata["error_kind"] != "http_status" || events[1].Metadata["searxng_http_status"] != http.StatusForbidden {
+		t.Fatalf("failure details missing: %+v", events[1].Metadata)
+	}
+}