Update 6 SQLite

internal/graph/sqlite_backend.go

@@ -0,0 +1,700 @@
+package graph
+
+import (
+	"context"
+	"database/sql"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/local/glpi-neural-brain/internal/model"
+	_ "modernc.org/sqlite"
+)
+
+const schemaVersion = 1
+
+const nodeUpsertSQL = `INSERT INTO nodes(id,kind,label,summary,status,origin,external_id,uri,categories_json,keywords_json,metadata_json,weight,x,y,z,updated_at_ns)
+VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+ON CONFLICT(id) DO UPDATE SET kind=excluded.kind,label=excluded.label,summary=excluded.summary,status=excluded.status,origin=excluded.origin,external_id=excluded.external_id,uri=excluded.uri,categories_json=excluded.categories_json,keywords_json=excluded.keywords_json,metadata_json=excluded.metadata_json,weight=excluded.weight,x=excluded.x,y=excluded.y,z=excluded.z,updated_at_ns=excluded.updated_at_ns`
+
+const edgeUpsertSQL = `INSERT INTO edges(id,source,target,type,origin,status,confidence,weight,explanation,evidence_json,metadata_json,created_at_ns,updated_at_ns)
+VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)
+ON CONFLICT(id) DO UPDATE SET source=excluded.source,target=excluded.target,type=excluded.type,origin=excluded.origin,status=excluded.status,confidence=excluded.confidence,weight=excluded.weight,explanation=excluded.explanation,evidence_json=excluded.evidence_json,metadata_json=excluded.metadata_json,created_at_ns=excluded.created_at_ns,updated_at_ns=excluded.updated_at_ns`
+
+const vectorUpsertSQL = `INSERT INTO vectors(node_id,dimensions,data,updated_at_ns) VALUES(?,?,?,?)
+ON CONFLICT(node_id) DO UPDATE SET dimensions=excluded.dimensions,data=excluded.data,updated_at_ns=excluded.updated_at_ns`
+
+// StorageStatus describes the persistent SQLite graph store. The live graph is
+// still held in memory; only changed rows are written during a flush.
+type StorageStatus struct {
+	Backend           string `json:"backend"`
+	Path              string `json:"path"`
+	SchemaVersion     int    `json:"schema_version"`
+	JournalMode       string `json:"journal_mode"`
+	DatabaseBytes     int64  `json:"database_bytes"`
+	WALBytes          int64  `json:"wal_bytes"`
+	NodeRows          int    `json:"node_rows"`
+	EdgeRows          int    `json:"edge_rows"`
+	VectorRows        int    `json:"vector_rows"`
+	VectorBytes       int64  `json:"vector_bytes"`
+	PersistedVersion  uint64 `json:"persisted_version"`
+	EmbeddingModel    string `json:"embedding_model,omitempty"`
+	EmbeddingDigest   string `json:"embedding_digest,omitempty"`
+	LegacyJSONPresent bool   `json:"legacy_json_present"`
+	PendingNodes      int    `json:"pending_nodes"`
+	PendingEdges      int    `json:"pending_edges"`
+	PendingVectors    int    `json:"pending_vectors"`
+	PendingDeletions  int    `json:"pending_deletions"`
+	PendingMetadata   bool   `json:"pending_metadata"`
+}
+
+func Open(dir string) (*Store, error) {
+	absoluteDir, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, fmt.Errorf("resolve graph data directory: %w", err)
+	}
+	if err := os.MkdirAll(absoluteDir, 0o750); err != nil {
+		return nil, fmt.Errorf("create graph data directory %q: %w", absoluteDir, err)
+	}
+	if err := ensureWritableDirectory(absoluteDir); err != nil {
+		return nil, err
+	}
+
+	path := filepath.Join(absoluteDir, "graph.db")
+	if info, statErr := os.Stat(path); statErr == nil && info.IsDir() {
+		return nil, fmt.Errorf("graph database path %q is a directory; remove or rename it", path)
+	} else if statErr != nil && !errors.Is(statErr, os.ErrNotExist) {
+		return nil, fmt.Errorf("inspect graph database %q: %w", path, statErr)
+	}
+
+	// Use the plain filename here instead of applying PRAGMAs through the DSN.
+	// This makes startup failures attributable to one concrete step and avoids
+	// asking the driver to reserve a large mmap/cache while the connection is
+	// being created. The application keeps the full graph in Go memory already,
+	// so an additional 64 MiB SQLite page cache and 256 MiB mmap are wasteful on
+	// small systems.
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, fmt.Errorf("create sqlite handle for %q: %w", path, err)
+	}
+	// The Store serializes persistence and reads the live graph from memory.
+	// One SQLite connection is sufficient and keeps all connection-local
+	// PRAGMAs deterministic.
+	db.SetMaxOpenConns(1)
+	db.SetMaxIdleConns(1)
+	db.SetConnMaxLifetime(0)
+	db.SetConnMaxIdleTime(0)
+
+	openCtx, openCancel := context.WithTimeout(context.Background(), 15*time.Second)
+	if err := db.PingContext(openCtx); err != nil {
+		openCancel()
+		db.Close()
+		return nil, fmt.Errorf("open sqlite database %q: %w", path, err)
+	}
+	journalMode, err := configureSQLite(openCtx, db)
+	openCancel()
+	if err != nil {
+		db.Close()
+		return nil, fmt.Errorf("configure sqlite database %q: %w", path, err)
+	}
+
+	s := &Store{
+		nodes:          map[string]model.Node{},
+		edges:          map[string]model.Edge{},
+		vectors:        map[string][]float32{},
+		db:             db,
+		dbPath:         path,
+		journalMode:    journalMode,
+		dirtyNodes:     map[string]uint64{},
+		dirtyEdges:     map[string]uint64{},
+		dirtyVectors:   map[string]uint64{},
+		deletedNodes:   map[string]uint64{},
+		deletedEdges:   map[string]uint64{},
+		deletedVectors: map[string]uint64{},
+	}
+	loadCtx, loadCancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer loadCancel()
+	if err := s.initSchema(loadCtx); err != nil {
+		db.Close()
+		return nil, fmt.Errorf("initialize sqlite graph schema: %w", err)
+	}
+	if err := s.load(loadCtx); err != nil {
+		db.Close()
+		return nil, fmt.Errorf("load sqlite graph state: %w", err)
+	}
+	return s, nil
+}
+
+func ensureWritableDirectory(dir string) error {
+	info, err := os.Stat(dir)
+	if err != nil {
+		return fmt.Errorf("inspect graph data directory %q: %w", dir, err)
+	}
+	if !info.IsDir() {
+		return fmt.Errorf("graph data path %q is not a directory", dir)
+	}
+	probe, err := os.CreateTemp(dir, ".brain-write-test-*")
+	if err != nil {
+		return fmt.Errorf("graph data directory %q is not writable by the brain process: %w", dir, err)
+	}
+	name := probe.Name()
+	closeErr := probe.Close()
+	removeErr := os.Remove(name)
+	if closeErr != nil {
+		return fmt.Errorf("close graph data write test %q: %w", name, closeErr)
+	}
+	if removeErr != nil {
+		return fmt.Errorf("remove graph data write test %q: %w", name, removeErr)
+	}
+	return nil
+}
+
+func configureSQLite(ctx context.Context, db *sql.DB) (string, error) {
+	// Conservative defaults: the live graph and vectors already reside in Go
+	// memory. SQLite therefore only needs a small page cache and no mmap window.
+	// This keeps startup predictable inside memory-limited containers.
+	statements := []struct {
+		name string
+		sql  string
+	}{
+		{name: "busy_timeout", sql: `PRAGMA busy_timeout=5000`},
+		{name: "foreign_keys", sql: `PRAGMA foreign_keys=ON`},
+		{name: "synchronous", sql: `PRAGMA synchronous=NORMAL`},
+		{name: "temp_store", sql: `PRAGMA temp_store=FILE`},
+		{name: "cache_size", sql: `PRAGMA cache_size=-8192`},
+		{name: "mmap_size", sql: `PRAGMA mmap_size=0`},
+		{name: "wal_autocheckpoint", sql: `PRAGMA wal_autocheckpoint=0`},
+		{name: "journal_size_limit", sql: `PRAGMA journal_size_limit=67108864`},
+	}
+	for _, statement := range statements {
+		if _, err := db.ExecContext(ctx, statement.sql); err != nil {
+			return "", fmt.Errorf("apply PRAGMA %s: %w", statement.name, err)
+		}
+	}
+
+	var journalMode string
+	if err := db.QueryRowContext(ctx, `PRAGMA journal_mode=WAL`).Scan(&journalMode); err != nil {
+		return "", fmt.Errorf("enable WAL journal mode: %w", err)
+	}
+	journalMode = strings.ToLower(strings.TrimSpace(journalMode))
+	if journalMode != "wal" {
+		return "", fmt.Errorf("enable WAL journal mode: SQLite selected %q", journalMode)
+	}
+	return journalMode, nil
+}
+
+func (s *Store) initSchema(ctx context.Context) error {
+	statements := []string{
+		`CREATE TABLE IF NOT EXISTS graph_meta (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+) WITHOUT ROWID`,
+		`CREATE TABLE IF NOT EXISTS nodes (
+    id TEXT PRIMARY KEY,
+    kind TEXT NOT NULL,
+    label TEXT NOT NULL,
+    summary TEXT NOT NULL DEFAULT '',
+    status TEXT NOT NULL DEFAULT '',
+    origin TEXT NOT NULL,
+    external_id TEXT NOT NULL DEFAULT '',
+    uri TEXT NOT NULL DEFAULT '',
+    categories_json TEXT NOT NULL DEFAULT '[]',
+    keywords_json TEXT NOT NULL DEFAULT '[]',
+    metadata_json TEXT NOT NULL DEFAULT '{}',
+    weight REAL NOT NULL DEFAULT 1,
+    x REAL NOT NULL DEFAULT 0,
+    y REAL NOT NULL DEFAULT 0,
+    z REAL NOT NULL DEFAULT 0,
+    updated_at_ns INTEGER NOT NULL
+) WITHOUT ROWID`,
+		`CREATE INDEX IF NOT EXISTS idx_nodes_origin ON nodes(origin)`,
+		`CREATE INDEX IF NOT EXISTS idx_nodes_kind_status ON nodes(kind, status)`,
+		`CREATE INDEX IF NOT EXISTS idx_nodes_external_id ON nodes(external_id)`,
+		`CREATE TABLE IF NOT EXISTS edges (
+    id TEXT PRIMARY KEY,
+    source TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
+    target TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
+    type TEXT NOT NULL,
+    origin TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT '',
+    confidence REAL NOT NULL DEFAULT 0,
+    weight REAL NOT NULL DEFAULT 1,
+    explanation TEXT NOT NULL DEFAULT '',
+    evidence_json TEXT NOT NULL DEFAULT '[]',
+    metadata_json TEXT NOT NULL DEFAULT '{}',
+    created_at_ns INTEGER NOT NULL,
+    updated_at_ns INTEGER NOT NULL
+) WITHOUT ROWID`,
+		`CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source)`,
+		`CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target)`,
+		`CREATE INDEX IF NOT EXISTS idx_edges_origin_status ON edges(origin, status)`,
+		`CREATE TABLE IF NOT EXISTS vectors (
+    node_id TEXT PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
+    dimensions INTEGER NOT NULL,
+    data BLOB NOT NULL,
+    updated_at_ns INTEGER NOT NULL
+) WITHOUT ROWID`,
+	}
+	for _, statement := range statements {
+		if _, err := s.db.ExecContext(ctx, statement); err != nil {
+			return fmt.Errorf("initialize graph sqlite schema: %w", err)
+		}
+	}
+	var current int
+	err := s.db.QueryRowContext(ctx, `SELECT CAST(value AS INTEGER) FROM graph_meta WHERE key='schema_version'`).Scan(&current)
+	if errors.Is(err, sql.ErrNoRows) {
+		_, err = s.db.ExecContext(ctx, `INSERT INTO graph_meta(key,value) VALUES('schema_version', ?)`, schemaVersion)
+		return err
+	}
+	if err != nil {
+		return err
+	}
+	if current != schemaVersion {
+		return fmt.Errorf("unsupported graph database schema %d (expected %d)", current, schemaVersion)
+	}
+	return nil
+}
+
+func (s *Store) load(ctx context.Context) error {
+	if err := s.loadNodes(ctx); err != nil {
+		return err
+	}
+	if err := s.loadEdges(ctx); err != nil {
+		return err
+	}
+	if err := s.loadVectors(ctx); err != nil {
+		return err
+	}
+	var version uint64
+	if err := s.db.QueryRowContext(ctx, `SELECT CAST(value AS INTEGER) FROM graph_meta WHERE key='graph_version'`).Scan(&version); err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return err
+	}
+	var embeddingModel string
+	if err := s.db.QueryRowContext(ctx, `SELECT value FROM graph_meta WHERE key='embedding_model'`).Scan(&embeddingModel); err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return err
+	}
+	var embeddingDigest string
+	if err := s.db.QueryRowContext(ctx, `SELECT value FROM graph_meta WHERE key='embedding_digest'`).Scan(&embeddingDigest); err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return err
+	}
+	s.version = version
+	s.persistedVersion = version
+	s.embeddingModel = embeddingModel
+	s.embeddingDigest = embeddingDigest
+	return nil
+}
+
+func (s *Store) loadNodes(ctx context.Context) error {
+	rows, err := s.db.QueryContext(ctx, `SELECT id,kind,label,summary,status,origin,external_id,uri,categories_json,keywords_json,metadata_json,weight,x,y,z,updated_at_ns FROM nodes`)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var n model.Node
+		var categories, keywords, metadata string
+		var updated int64
+		if err := rows.Scan(&n.ID, &n.Kind, &n.Label, &n.Summary, &n.Status, &n.Origin, &n.ExternalID, &n.URI, &categories, &keywords, &metadata, &n.Weight, &n.X, &n.Y, &n.Z, &updated); err != nil {
+			return err
+		}
+		if err := decodeJSON(categories, &n.Categories); err != nil {
+			return fmt.Errorf("decode node %s categories: %w", n.ID, err)
+		}
+		if err := decodeJSON(keywords, &n.Keywords); err != nil {
+			return fmt.Errorf("decode node %s keywords: %w", n.ID, err)
+		}
+		if err := decodeJSON(metadata, &n.Metadata); err != nil {
+			return fmt.Errorf("decode node %s metadata: %w", n.ID, err)
+		}
+		if n.Metadata == nil {
+			n.Metadata = map[string]any{}
+		}
+		n.UpdatedAt = time.Unix(0, updated).UTC()
+		s.nodes[n.ID] = n
+	}
+	return rows.Err()
+}
+
+func (s *Store) loadEdges(ctx context.Context) error {
+	rows, err := s.db.QueryContext(ctx, `SELECT id,source,target,type,origin,status,confidence,weight,explanation,evidence_json,metadata_json,created_at_ns,updated_at_ns FROM edges`)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var e model.Edge
+		var evidence, metadata string
+		var created, updated int64
+		if err := rows.Scan(&e.ID, &e.Source, &e.Target, &e.Type, &e.Origin, &e.Status, &e.Confidence, &e.Weight, &e.Explanation, &evidence, &metadata, &created, &updated); err != nil {
+			return err
+		}
+		if err := decodeJSON(evidence, &e.Evidence); err != nil {
+			return fmt.Errorf("decode edge %s evidence: %w", e.ID, err)
+		}
+		if err := decodeJSON(metadata, &e.Metadata); err != nil {
+			return fmt.Errorf("decode edge %s metadata: %w", e.ID, err)
+		}
+		if e.Metadata == nil {
+			e.Metadata = map[string]any{}
+		}
+		e.CreatedAt = time.Unix(0, created).UTC()
+		e.UpdatedAt = time.Unix(0, updated).UTC()
+		s.edges[e.ID] = e
+	}
+	return rows.Err()
+}
+
+func (s *Store) loadVectors(ctx context.Context) error {
+	rows, err := s.db.QueryContext(ctx, `SELECT node_id,dimensions,data FROM vectors`)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var id string
+		var dimensions int
+		var data []byte
+		if err := rows.Scan(&id, &dimensions, &data); err != nil {
+			return err
+		}
+		v, err := decodeVector(data, dimensions)
+		if err != nil {
+			return fmt.Errorf("decode vector %s: %w", id, err)
+		}
+		s.vectors[id] = v
+	}
+	return rows.Err()
+}
+
+func decodeJSON(raw string, target any) error {
+	if strings.TrimSpace(raw) == "" {
+		return nil
+	}
+	dec := json.NewDecoder(strings.NewReader(raw))
+	dec.UseNumber()
+	return dec.Decode(target)
+}
+
+func encodeJSON(value any, empty string) (string, error) {
+	if value == nil {
+		return empty, nil
+	}
+	b, err := json.Marshal(value)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}
+
+func encodeVector(v []float32) []byte {
+	out := make([]byte, len(v)*4)
+	for i, value := range v {
+		binary.LittleEndian.PutUint32(out[i*4:], math.Float32bits(value))
+	}
+	return out
+}
+
+func decodeVector(data []byte, dimensions int) ([]float32, error) {
+	if dimensions < 0 || len(data) != dimensions*4 {
+		return nil, fmt.Errorf("invalid float32 vector blob: dimensions=%d bytes=%d", dimensions, len(data))
+	}
+	out := make([]float32, dimensions)
+	for i := range out {
+		out[i] = math.Float32frombits(binary.LittleEndian.Uint32(data[i*4:]))
+	}
+	return out, nil
+}
+
+// PersistVersion writes only records changed since the previous successful
+// flush. Vectors are encoded as little-endian float32 BLOBs.
+func (s *Store) PersistVersion() (uint64, error) {
+	return s.PersistVersionContext(context.Background())
+}
+
+func (s *Store) PersistVersionContext(ctx context.Context) (uint64, error) {
+	type nodeChange struct {
+		gen  uint64
+		node model.Node
+	}
+	type edgeChange struct {
+		gen  uint64
+		edge model.Edge
+	}
+	type vectorChange struct {
+		gen    uint64
+		vector []float32
+	}
+
+	s.mu.RLock()
+	version := s.version
+	persistedVersion := s.persistedVersion
+	embeddingModel := s.embeddingModel
+	embeddingDigest := s.embeddingDigest
+	embeddingMetaGeneration := s.embeddingMetaGeneration
+	nodes := make(map[string]nodeChange, len(s.dirtyNodes))
+	for id, gen := range s.dirtyNodes {
+		if n, ok := s.nodes[id]; ok {
+			nodes[id] = nodeChange{gen: gen, node: n}
+		}
+	}
+	edges := make(map[string]edgeChange, len(s.dirtyEdges))
+	for id, gen := range s.dirtyEdges {
+		if e, ok := s.edges[id]; ok {
+			edges[id] = edgeChange{gen: gen, edge: e}
+		}
+	}
+	vectors := make(map[string]vectorChange, len(s.dirtyVectors))
+	for id, gen := range s.dirtyVectors {
+		if v, ok := s.vectors[id]; ok {
+			vectors[id] = vectorChange{gen: gen, vector: append([]float32(nil), v...)}
+		}
+	}
+	deletedNodes := cloneGenerations(s.deletedNodes)
+	deletedEdges := cloneGenerations(s.deletedEdges)
+	deletedVectors := cloneGenerations(s.deletedVectors)
+	s.mu.RUnlock()
+
+	if len(nodes)+len(edges)+len(vectors)+len(deletedNodes)+len(deletedEdges)+len(deletedVectors) == 0 && embeddingMetaGeneration == 0 && version == persistedVersion {
+		return version, nil
+	}
+
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return 0, err
+	}
+	defer tx.Rollback()
+
+	for id := range deletedEdges {
+		if _, err := tx.ExecContext(ctx, `DELETE FROM edges WHERE id=?`, id); err != nil {
+			return 0, err
+		}
+	}
+	for id := range deletedNodes {
+		if _, err := tx.ExecContext(ctx, `DELETE FROM nodes WHERE id=?`, id); err != nil {
+			return 0, err
+		}
+	}
+	for id := range deletedVectors {
+		if _, err := tx.ExecContext(ctx, `DELETE FROM vectors WHERE node_id=?`, id); err != nil {
+			return 0, err
+		}
+	}
+
+	if len(nodes) > 0 {
+		statement, err := tx.PrepareContext(ctx, nodeUpsertSQL)
+		if err != nil {
+			return 0, err
+		}
+		for _, change := range nodes {
+			if err := upsertNode(ctx, statement, change.node); err != nil {
+				_ = statement.Close()
+				return 0, err
+			}
+		}
+		if err := statement.Close(); err != nil {
+			return 0, err
+		}
+	}
+	if len(edges) > 0 {
+		statement, err := tx.PrepareContext(ctx, edgeUpsertSQL)
+		if err != nil {
+			return 0, err
+		}
+		for _, change := range edges {
+			if err := upsertEdge(ctx, statement, change.edge); err != nil {
+				_ = statement.Close()
+				return 0, err
+			}
+		}
+		if err := statement.Close(); err != nil {
+			return 0, err
+		}
+	}
+	if len(vectors) > 0 {
+		statement, err := tx.PrepareContext(ctx, vectorUpsertSQL)
+		if err != nil {
+			return 0, err
+		}
+		now := time.Now().UTC().UnixNano()
+		for id, change := range vectors {
+			if _, err := statement.ExecContext(ctx, id, len(change.vector), encodeVector(change.vector), now); err != nil {
+				_ = statement.Close()
+				return 0, err
+			}
+		}
+		if err := statement.Close(); err != nil {
+			return 0, err
+		}
+	}
+	if embeddingMetaGeneration > 0 {
+		if _, err := tx.ExecContext(ctx, `INSERT INTO graph_meta(key,value) VALUES('embedding_model',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, embeddingModel); err != nil {
+			return 0, err
+		}
+		if _, err := tx.ExecContext(ctx, `INSERT INTO graph_meta(key,value) VALUES('embedding_digest',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, embeddingDigest); err != nil {
+			return 0, err
+		}
+	}
+	if _, err := tx.ExecContext(ctx, `INSERT INTO graph_meta(key,value) VALUES('graph_version',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, version); err != nil {
+		return 0, err
+	}
+	if err := tx.Commit(); err != nil {
+		return 0, err
+	}
+
+	s.mu.Lock()
+	for id, change := range nodes {
+		if s.dirtyNodes[id] == change.gen {
+			delete(s.dirtyNodes, id)
+		}
+	}
+	for id, change := range edges {
+		if s.dirtyEdges[id] == change.gen {
+			delete(s.dirtyEdges, id)
+		}
+	}
+	for id, change := range vectors {
+		if s.dirtyVectors[id] == change.gen {
+			delete(s.dirtyVectors, id)
+		}
+	}
+	clearDeletedIfSame(s.deletedNodes, deletedNodes)
+	clearDeletedIfSame(s.deletedEdges, deletedEdges)
+	clearDeletedIfSame(s.deletedVectors, deletedVectors)
+	if s.embeddingMetaGeneration == embeddingMetaGeneration {
+		s.embeddingMetaGeneration = 0
+	}
+	if version > s.persistedVersion {
+		s.persistedVersion = version
+	}
+	s.mu.Unlock()
+
+	// Keep the WAL bounded. PASSIVE never blocks readers and leaves busy pages
+	// for the next scheduled flush.
+	_, _ = s.db.ExecContext(ctx, `PRAGMA wal_checkpoint(PASSIVE)`)
+	return version, nil
+}
+
+func upsertNode(ctx context.Context, statement *sql.Stmt, n model.Node) error {
+	categories, err := encodeJSON(n.Categories, "[]")
+	if err != nil {
+		return err
+	}
+	keywords, err := encodeJSON(n.Keywords, "[]")
+	if err != nil {
+		return err
+	}
+	metadata, err := encodeJSON(n.Metadata, "{}")
+	if err != nil {
+		return err
+	}
+	_, err = statement.ExecContext(ctx, n.ID, n.Kind, n.Label, n.Summary, n.Status, n.Origin, n.ExternalID, n.URI, categories, keywords, metadata, n.Weight, n.X, n.Y, n.Z, n.UpdatedAt.UnixNano())
+	return err
+}
+
+func upsertEdge(ctx context.Context, statement *sql.Stmt, e model.Edge) error {
+	evidence, err := encodeJSON(e.Evidence, "[]")
+	if err != nil {
+		return err
+	}
+	metadata, err := encodeJSON(e.Metadata, "{}")
+	if err != nil {
+		return err
+	}
+	_, err = statement.ExecContext(ctx, e.ID, e.Source, e.Target, e.Type, e.Origin, e.Status, e.Confidence, e.Weight, e.Explanation, evidence, metadata, e.CreatedAt.UnixNano(), e.UpdatedAt.UnixNano())
+	return err
+}
+
+func cloneGenerations(in map[string]uint64) map[string]uint64 {
+	out := make(map[string]uint64, len(in))
+	for id, gen := range in {
+		out[id] = gen
+	}
+	return out
+}
+
+func clearDeletedIfSame(current, snapshot map[string]uint64) {
+	for id, gen := range snapshot {
+		if current[id] == gen {
+			delete(current, id)
+		}
+	}
+}
+
+func (s *Store) Persist() error {
+	_, err := s.PersistVersion()
+	return err
+}
+
+func (s *Store) Checkpoint(ctx context.Context, truncate bool) error {
+	mode := "PASSIVE"
+	if truncate {
+		mode = "TRUNCATE"
+	}
+	_, err := s.db.ExecContext(ctx, `PRAGMA wal_checkpoint(`+mode+`)`)
+	return err
+}
+
+func (s *Store) Close() error {
+	if s.db == nil {
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	_, persistErr := s.PersistVersionContext(ctx)
+	checkpointErr := s.Checkpoint(ctx, true)
+	closeErr := s.db.Close()
+	return errors.Join(persistErr, checkpointErr, closeErr)
+}
+
+func (s *Store) StorageStatus() StorageStatus {
+	s.mu.RLock()
+	var vectorBytes int64
+	for _, vector := range s.vectors {
+		vectorBytes += int64(len(vector) * 4)
+	}
+	status := StorageStatus{
+		Backend: "sqlite", Path: s.dbPath, SchemaVersion: schemaVersion, JournalMode: s.journalMode,
+		NodeRows: len(s.nodes), EdgeRows: len(s.edges), VectorRows: len(s.vectors), VectorBytes: vectorBytes,
+		PersistedVersion: s.persistedVersion,
+		EmbeddingModel:   s.embeddingModel,
+		EmbeddingDigest:  s.embeddingDigest,
+		PendingNodes:     len(s.dirtyNodes), PendingEdges: len(s.dirtyEdges), PendingVectors: len(s.dirtyVectors),
+		PendingDeletions: len(s.deletedNodes) + len(s.deletedEdges) + len(s.deletedVectors),
+		PendingMetadata:  s.embeddingMetaGeneration > 0,
+	}
+	s.mu.RUnlock()
+	if info, err := os.Stat(s.dbPath); err == nil {
+		status.DatabaseBytes = info.Size()
+	}
+	if info, err := os.Stat(s.dbPath + "-wal"); err == nil {
+		status.WALBytes = info.Size()
+	}
+	_, err := os.Stat(filepath.Join(filepath.Dir(s.dbPath), "graph-state.json"))
+	status.LegacyJSONPresent = err == nil
+	return status
+}
+
+// Export creates a compact, transactionally consistent SQLite copy. It can be
+// moved to another installation with the same knowledge-base identifiers.
+func (s *Store) Export(ctx context.Context, destination string) error {
+	if _, err := s.PersistVersionContext(ctx); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(destination), 0o750); err != nil {
+		return err
+	}
+	if err := os.Remove(destination); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	quoted := strings.ReplaceAll(destination, "'", "''")
+	if _, err := s.db.ExecContext(ctx, `VACUUM INTO '`+quoted+`'`); err != nil {
+		return fmt.Errorf("export graph database: %w", err)
+	}
+	return os.Chmod(destination, 0o640)
+}

internal/graph/sqlite_backend_test.go

@@ -0,0 +1,203 @@
+package graph
+
+import (
+	"context"
+	"math"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/local/glpi-neural-brain/internal/model"
+)
+
+func TestSQLiteRoundTripUsesBinaryFloat32Vectors(t *testing.T) {
+	dir := t.TempDir()
+	s, err := Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s.UpsertNode(model.Node{ID: "a", Kind: "knowledge", Label: "A", Origin: "test", Categories: []string{"Netzwerk"}, Metadata: map[string]any{"rank": 2}})
+	s.UpsertNode(model.Node{ID: "b", Kind: "knowledge", Label: "B", Origin: "test"})
+	s.UpsertEdge(model.Edge{Source: "a", Target: "b", Type: "related_to", Origin: "test", Confidence: .8})
+	s.SetVector("a", []float64{.125, -.5, .875})
+	if _, err := s.PersistVersionContext(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	status := s.StorageStatus()
+	if status.Backend != "sqlite" || status.VectorBytes != 12 || status.PendingNodes != 0 || status.PendingVectors != 0 {
+		t.Fatalf("unexpected storage status: %+v", status)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "graph-state.json")); !os.IsNotExist(err) {
+		t.Fatalf("legacy JSON snapshot must not be created: %v", err)
+	}
+	if err := s.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	reopened, err := Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer reopened.Close()
+	if got := reopened.Snapshot(); len(got.Nodes) != 2 || len(got.Edges) != 1 {
+		t.Fatalf("unexpected round-trip graph: nodes=%d edges=%d", len(got.Nodes), len(got.Edges))
+	}
+	vector, ok := reopened.Vector("a")
+	if !ok || len(vector) != 3 {
+		t.Fatalf("vector missing after round trip: %v %v", ok, vector)
+	}
+	want := []float64{.125, -.5, .875}
+	for i := range want {
+		if math.Abs(vector[i]-want[i]) > 1e-6 {
+			t.Fatalf("vector[%d]=%f want %f", i, vector[i], want[i])
+		}
+	}
+}
+
+func TestSQLitePersistsOnlyDirtyRows(t *testing.T) {
+	s, err := Open(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	s.UpsertNode(model.Node{ID: "a", Kind: "knowledge", Label: "A", Origin: "test"})
+	s.UpsertNode(model.Node{ID: "b", Kind: "knowledge", Label: "B", Origin: "test"})
+	s.SetVector("a", []float64{1, 0})
+	if _, err := s.PersistVersion(); err != nil {
+		t.Fatal(err)
+	}
+	if s.Dirty() {
+		t.Fatal("store remained dirty after successful flush")
+	}
+
+	n, _ := s.GetNode("a")
+	n.Label = "A2"
+	s.UpsertNode(n)
+	status := s.StorageStatus()
+	if status.PendingNodes != 1 || status.PendingEdges != 0 || status.PendingVectors != 0 {
+		t.Fatalf("expected one dirty node, got %+v", status)
+	}
+	if _, err := s.PersistVersion(); err != nil {
+		t.Fatal(err)
+	}
+	if status = s.StorageStatus(); status.PendingNodes != 0 || s.Dirty() {
+		t.Fatalf("dirty rows were not cleared: %+v", status)
+	}
+}
+
+func TestSQLiteExportCreatesPortableDatabase(t *testing.T) {
+	s, err := Open(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	s.UpsertNode(model.Node{ID: "portable", Kind: "knowledge", Label: "Portable", Origin: "test"})
+	s.SetVector("portable", []float64{1, 2, 3, 4})
+	exportDir := t.TempDir()
+	destination := filepath.Join(exportDir, "graph.db")
+	if err := s.Export(context.Background(), destination); err != nil {
+		t.Fatal(err)
+	}
+	copyStore, err := Open(exportDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer copyStore.Close()
+	if _, ok := copyStore.GetNode("portable"); !ok {
+		t.Fatal("exported database did not contain graph node")
+	}
+	if vector, ok := copyStore.Vector("portable"); !ok || len(vector) != 4 {
+		t.Fatalf("exported database did not contain vector: %v %v", ok, vector)
+	}
+}
+
+func TestReplaceOriginsLeavesUnchangedKnowledgeClean(t *testing.T) {
+	s, err := Open(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	updated := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
+	nodes := []model.Node{
+		{ID: "a", Kind: "knowledge", Label: "A", Summary: "gleich", Origin: "kb", UpdatedAt: updated},
+		{ID: "b", Kind: "knowledge", Label: "B", Summary: "gleich", Origin: "kb", UpdatedAt: updated},
+	}
+	edges := []model.Edge{{Source: "a", Target: "b", Type: "related_to", Origin: "kb", Weight: 1}}
+	s.ReplaceOrigins([]string{"kb"}, nodes, edges)
+	s.SetVector("a", []float64{1, 0, 0})
+	if _, err := s.PersistVersion(); err != nil {
+		t.Fatal(err)
+	}
+	version := s.Version()
+
+	s.ReplaceOrigins([]string{"kb"}, nodes, edges)
+	if s.Dirty() {
+		t.Fatalf("unchanged KB scan marked SQLite rows dirty: %+v", s.StorageStatus())
+	}
+	if s.Version() != version {
+		t.Fatalf("unchanged KB scan changed graph version: got %d want %d", s.Version(), version)
+	}
+	if _, ok := s.Vector("a"); !ok {
+		t.Fatal("unchanged KB scan discarded embedding")
+	}
+}
+
+func TestReplaceOriginsInvalidatesOnlyChangedEmbedding(t *testing.T) {
+	s, err := Open(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	updated := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC)
+	nodes := []model.Node{{ID: "a", Kind: "knowledge", Label: "A", Summary: "alt", Origin: "kb", UpdatedAt: updated}}
+	s.ReplaceOrigins([]string{"kb"}, nodes, nil)
+	s.SetVector("a", []float64{1, 0, 0})
+	if _, err := s.PersistVersion(); err != nil {
+		t.Fatal(err)
+	}
+
+	nodes[0].Summary = "neu"
+	nodes[0].UpdatedAt = updated.Add(time.Minute)
+	s.ReplaceOrigins([]string{"kb"}, nodes, nil)
+	if _, ok := s.Vector("a"); ok {
+		t.Fatal("changed embedding input retained stale vector")
+	}
+	status := s.StorageStatus()
+	if status.PendingNodes != 1 || status.PendingDeletions != 1 {
+		t.Fatalf("expected changed node and vector deletion, got %+v", status)
+	}
+}
+
+func TestEmbeddingModelIdentityInvalidatesImportedVectors(t *testing.T) {
+	dir := t.TempDir()
+	s, err := Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s.ConfigureEmbeddingIdentity("embeddinggemma", "sha256:d1")
+	s.UpsertNode(model.Node{ID: "a", Kind: "knowledge", Label: "A", Origin: "test"})
+	s.SetVector("a", []float64{1, 2, 3})
+	if _, err := s.PersistVersion(); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	reopened, err := Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer reopened.Close()
+	if removed := reopened.ConfigureEmbeddingIdentity("embeddinggemma", "sha256:d2"); removed != 1 {
+		t.Fatalf("removed vectors=%d want 1", removed)
+	}
+	if _, ok := reopened.Vector("a"); ok {
+		t.Fatal("vector from a different embedding model remained available")
+	}
+	status := reopened.StorageStatus()
+	if status.EmbeddingModel != "embeddinggemma" || status.EmbeddingDigest != "sha256:d2" || !status.PendingMetadata || status.PendingDeletions != 1 {
+		t.Fatalf("unexpected model-change status: %+v", status)
+	}
+}

internal/graph/store.go

@@ -2,12 +2,10 @@ package graph
 
 import (
 	"crypto/sha256"
+	"database/sql"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"math"
-	"os"
-	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -17,47 +15,29 @@ import (
 )
 
 type Store struct {
-	mu         sync.RWMutex
-	nodes      map[string]model.Node
-	edges      map[string]model.Edge
-	vectors    map[string][]float64
-	version    uint64
-	path       string
-	pairCursor int
+	mu                      sync.RWMutex
+	nodes                   map[string]model.Node
+	edges                   map[string]model.Edge
+	vectors                 map[string][]float32
+	version                 uint64
+	persistedVersion        uint64
+	pairCursor              int
+	embeddingModel          string
+	embeddingDigest         string
+	embeddingMetaGeneration uint64
+
+	db          *sql.DB
+	dbPath      string
+	journalMode string
+
+	dirtyNodes     map[string]uint64
+	dirtyEdges     map[string]uint64
+	dirtyVectors   map[string]uint64
+	deletedNodes   map[string]uint64
+	deletedEdges   map[string]uint64
+	deletedVectors map[string]uint64
 }
 
-type diskState struct {
-	Version uint64               `json:"version"`
-	Nodes   []model.Node         `json:"nodes"`
-	Edges   []model.Edge         `json:"edges"`
-	Vectors map[string][]float64 `json:"vectors,omitempty"`
-}
-
-func Open(dir string) (*Store, error) {
-	s := &Store{nodes: map[string]model.Node{}, edges: map[string]model.Edge{}, vectors: map[string][]float64{}, path: filepath.Join(dir, "graph-state.json")}
-	b, err := os.ReadFile(s.path)
-	if errors.Is(err, os.ErrNotExist) {
-		return s, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	var d diskState
-	if err = json.Unmarshal(b, &d); err != nil {
-		return nil, err
-	}
-	s.version = d.Version
-	for _, n := range d.Nodes {
-		s.nodes[n.ID] = n
-	}
-	for _, e := range d.Edges {
-		s.edges[e.ID] = e
-	}
-	if d.Vectors != nil {
-		s.vectors = d.Vectors
-	}
-	return s, nil
-}
 func ID(parts ...string) string {
 	h := sha256.Sum256([]byte(strings.Join(parts, "\x00")))
 	return hex.EncodeToString(h[:12])
@@ -65,6 +45,84 @@ func ID(parts ...string) string {
 func EdgeID(source, target, typ, origin string) string {
 	return ID("edge", source, target, typ, origin)
 }
+
+func (s *Store) markNodeDirtyLocked(id string) {
+	delete(s.deletedNodes, id)
+	s.dirtyNodes[id] = s.version
+}
+func (s *Store) markEdgeDirtyLocked(id string) {
+	delete(s.deletedEdges, id)
+	s.dirtyEdges[id] = s.version
+}
+func (s *Store) markVectorDirtyLocked(id string) {
+	delete(s.deletedVectors, id)
+	s.dirtyVectors[id] = s.version
+}
+func (s *Store) markNodeDeletedLocked(id string) {
+	delete(s.dirtyNodes, id)
+	s.deletedNodes[id] = s.version
+}
+func (s *Store) markEdgeDeletedLocked(id string) {
+	delete(s.dirtyEdges, id)
+	s.deletedEdges[id] = s.version
+}
+func (s *Store) markVectorDeletedLocked(id string) {
+	delete(s.dirtyVectors, id)
+	s.deletedVectors[id] = s.version
+}
+
+func (s *Store) Dirty() bool {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return len(s.dirtyNodes)+len(s.dirtyEdges)+len(s.dirtyVectors)+len(s.deletedNodes)+len(s.deletedEdges)+len(s.deletedVectors) > 0 || s.embeddingMetaGeneration > 0 || s.version != s.persistedVersion
+}
+
+// ConfigureEmbeddingModel records the model name before Ollama health data is
+// available. A stored digest remains intact when the model name is unchanged.
+func (s *Store) ConfigureEmbeddingModel(modelName string) int {
+	return s.ConfigureEmbeddingIdentity(modelName, "")
+}
+
+// ConfigureEmbeddingIdentity records the model and, once known, its Ollama
+// digest. Importing a database built with another model identity automatically
+// invalidates all vectors while retaining nodes and edges for selective
+// relearning.
+func (s *Store) ConfigureEmbeddingIdentity(modelName, digest string) int {
+	modelName = strings.TrimSpace(modelName)
+	digest = strings.TrimSpace(digest)
+	if modelName == "" {
+		return 0
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	modelChanged := s.embeddingModel != "" && s.embeddingModel != modelName
+	digestChanged := digest != "" && s.embeddingDigest != "" && s.embeddingDigest != digest
+	metadataChanged := s.embeddingModel != modelName || (digest != "" && s.embeddingDigest != digest)
+	if !metadataChanged {
+		return 0
+	}
+
+	s.version++
+	removed := 0
+	if modelChanged || digestChanged {
+		for id := range s.vectors {
+			delete(s.vectors, id)
+			s.deletedVectors[id] = s.version
+			delete(s.dirtyVectors, id)
+			removed++
+		}
+	}
+	s.embeddingModel = modelName
+	if modelChanged {
+		s.embeddingDigest = digest
+	} else if digest != "" {
+		s.embeddingDigest = digest
+	}
+	s.embeddingMetaGeneration = s.version
+	return removed
+}
+
 func (s *Store) UpsertNode(n model.Node) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -79,6 +137,7 @@ func (s *Store) UpsertNode(n model.Node) {
 	}
 	s.nodes[n.ID] = n
 	s.version++
+	s.markNodeDirtyLocked(n.ID)
 }
 func (s *Store) UpsertEdge(e model.Edge) {
 	s.mu.Lock()
@@ -100,6 +159,7 @@ func (s *Store) UpsertEdge(e model.Edge) {
 	}
 	s.edges[e.ID] = e
 	s.version++
+	s.markEdgeDirtyLocked(e.ID)
 }
 func (s *Store) HasEdgeBetween(a, b string) bool {
 	s.mu.RLock()
@@ -130,18 +190,30 @@ func (s *Store) LookupExternal(id string) (model.Node, bool) {
 func (s *Store) SetVector(id string, v []float64) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
+	converted := make([]float32, len(v))
+	for i, value := range v {
+		converted[i] = float32(value)
+	}
 	old, ok := s.vectors[id]
-	if ok && floatSlicesEqual(old, v) {
+	if ok && float32SlicesEqual(old, converted) {
 		return
 	}
-	s.vectors[id] = append([]float64(nil), v...)
+	s.vectors[id] = converted
 	s.version++
+	s.markVectorDirtyLocked(id)
 }
 func (s *Store) Vector(id string) ([]float64, bool) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 	v, ok := s.vectors[id]
-	return append([]float64(nil), v...), ok
+	if !ok {
+		return nil, false
+	}
+	out := make([]float64, len(v))
+	for i, value := range v {
+		out[i] = float64(value)
+	}
+	return out, true
 }
 
 func (s *Store) ClearVectorsByDimension(dim int) int {
@@ -152,11 +224,10 @@ func (s *Store) ClearVectorsByDimension(dim int) int {
 		if len(v) == dim {
 			delete(s.vectors, id)
 			removed++
+			s.version++
+			s.markVectorDeletedLocked(id)
 		}
 	}
-	if removed > 0 {
-		s.version++
-	}
 	return removed
 }
 
@@ -194,78 +265,235 @@ func (s *Store) KnowledgeNodes() []model.Node {
 	return out
 }
 func (s *Store) ReplaceOrigins(origins []string, nodes []model.Node, edges []model.Edge) {
-	set := map[string]bool{}
-	for _, o := range origins {
-		set[o] = true
+	originSet := make(map[string]struct{}, len(origins))
+	for _, origin := range origins {
+		originSet[origin] = struct{}{}
 	}
+
+	now := time.Now().UTC()
+	incomingNodes := make(map[string]model.Node, len(nodes))
+	for _, node := range nodes {
+		if node.Weight == 0 {
+			node.Weight = 1
+		}
+		normalizeNodeCollections(&node)
+		incomingNodes[node.ID] = node
+	}
+	incomingEdges := make(map[string]model.Edge, len(edges))
+	for _, edge := range edges {
+		if edge.ID == "" {
+			edge.ID = EdgeID(edge.Source, edge.Target, edge.Type, edge.Origin)
+		}
+		if edge.Weight == 0 {
+			edge.Weight = 1
+		}
+		normalizeEdgeCollections(&edge)
+		incomingEdges[edge.ID] = edge
+	}
+
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	oldVectors := make(map[string][]float64)
-	oldFingerprints := make(map[string]string)
-	for id, n := range s.nodes {
-		if set[n.Origin] {
-			if v, ok := s.vectors[id]; ok {
-				oldVectors[id] = append([]float64(nil), v...)
-				oldFingerprints[id] = n.Label + "\x00" + n.Summary + "\x00" + strings.Join(n.Categories, "\x00") + "\x00" + strings.Join(n.Keywords, "\x00")
-			}
-			delete(s.nodes, id)
-			delete(s.vectors, id)
-		}
-	}
-	for id, e := range s.edges {
-		if set[e.Origin] {
-			delete(s.edges, id)
-		}
-	}
-	now := time.Now().UTC()
-	for _, n := range nodes {
-		if n.UpdatedAt.IsZero() {
-			n.UpdatedAt = now
-		}
-		if n.Weight == 0 {
-			n.Weight = 1
-		}
-		if n.X == 0 && n.Y == 0 && n.Z == 0 {
-			n.X, n.Y, n.Z = position(n.ID, n.Categories)
-		}
-		s.nodes[n.ID] = n
-		fingerprint := n.Label + "\x00" + n.Summary + "\x00" + strings.Join(n.Categories, "\x00") + "\x00" + strings.Join(n.Keywords, "\x00")
-		if fingerprint == oldFingerprints[n.ID] {
-			if v, ok := oldVectors[n.ID]; ok {
-				s.vectors[n.ID] = append([]float64(nil), v...)
-			}
-		}
-	}
-	for _, e := range edges {
-		if e.ID == "" {
-			e.ID = EdgeID(e.Source, e.Target, e.Type, e.Origin)
-		}
-		if e.CreatedAt.IsZero() {
-			e.CreatedAt = now
-		}
-		e.UpdatedAt = now
-		if e.Weight == 0 {
-			e.Weight = 1
-		}
-		s.edges[e.ID] = e
-	}
-	for id, e := range s.edges {
-		if _, ok := s.nodes[e.Source]; !ok {
-			delete(s.edges, id)
+
+	// Remove records from managed origins that disappeared from the source.
+	for id, old := range s.nodes {
+		if _, managed := originSet[old.Origin]; !managed {
 			continue
 		}
-		if _, ok := s.nodes[e.Target]; !ok {
+		if _, present := incomingNodes[id]; present {
+			continue
+		}
+		delete(s.nodes, id)
+		if _, hadVector := s.vectors[id]; hadVector {
+			delete(s.vectors, id)
+			s.version++
+			s.markVectorDeletedLocked(id)
+		}
+		s.version++
+		s.markNodeDeletedLocked(id)
+	}
+	for id, old := range s.edges {
+		if _, managed := originSet[old.Origin]; !managed {
+			continue
+		}
+		if _, present := incomingEdges[id]; present {
+			continue
+		}
+		delete(s.edges, id)
+		s.version++
+		s.markEdgeDeletedLocked(id)
+	}
+
+	// Reconcile nodes instead of deleting and recreating every row on each scan.
+	// This is what makes the scheduled SQLite flush truly incremental for an
+	// unchanged knowledge base.
+	for id, incoming := range incomingNodes {
+		old, existed := s.nodes[id]
+		if incoming.UpdatedAt.IsZero() {
+			if existed && !old.UpdatedAt.IsZero() {
+				incoming.UpdatedAt = old.UpdatedAt
+			} else {
+				incoming.UpdatedAt = now
+			}
+		}
+		if incoming.X == 0 && incoming.Y == 0 && incoming.Z == 0 {
+			if existed && (old.X != 0 || old.Y != 0 || old.Z != 0) {
+				incoming.X, incoming.Y, incoming.Z = old.X, old.Y, old.Z
+			} else {
+				incoming.X, incoming.Y, incoming.Z = position(incoming.ID, incoming.Categories)
+			}
+		}
+
+		oldEmbeddingFingerprint := ""
+		if existed {
+			oldEmbeddingFingerprint = embeddingFingerprint(old)
+		}
+		if existed && nodesEquivalent(old, incoming) {
+			continue
+		}
+
+		s.nodes[id] = incoming
+		s.version++
+		s.markNodeDirtyLocked(id)
+
+		// Embeddings depend on text/categories/keywords, not on display
+		// coordinates or unrelated metadata. Only invalidate a vector when its
+		// actual embedding input changed.
+		if existed && oldEmbeddingFingerprint != embeddingFingerprint(incoming) {
+			if _, hadVector := s.vectors[id]; hadVector {
+				delete(s.vectors, id)
+				s.version++
+				s.markVectorDeletedLocked(id)
+			}
+		}
+	}
+
+	// Reconcile deterministic source edges. Timestamps are retained for an
+	// unchanged edge so periodic scans do not produce needless writes.
+	for id, incoming := range incomingEdges {
+		old, existed := s.edges[id]
+		if existed && edgesEquivalentIgnoringTimestamps(old, incoming) {
+			continue
+		}
+		if incoming.CreatedAt.IsZero() {
+			if existed && !old.CreatedAt.IsZero() {
+				incoming.CreatedAt = old.CreatedAt
+			} else {
+				incoming.CreatedAt = now
+			}
+		}
+		incoming.UpdatedAt = now
+		s.edges[id] = incoming
+		s.version++
+		s.markEdgeDirtyLocked(id)
+	}
+
+	// Remove any remaining edge whose endpoint no longer exists. This includes
+	// AI-derived edges that referred to a source note removed from the KB.
+	for id, edge := range s.edges {
+		if _, ok := s.nodes[edge.Source]; !ok {
 			delete(s.edges, id)
+			s.version++
+			s.markEdgeDeletedLocked(id)
+			continue
+		}
+		if _, ok := s.nodes[edge.Target]; !ok {
+			delete(s.edges, id)
+			s.version++
+			s.markEdgeDeletedLocked(id)
 		}
 	}
-	s.version++
 }
+
+func embeddingFingerprint(node model.Node) string {
+	return node.Label + "\x00" + node.Summary + "\x00" + strings.Join(node.Categories, "\x00") + "\x00" + strings.Join(node.Keywords, "\x00")
+}
+
+func normalizeNodeCollections(node *model.Node) {
+	if node.Categories == nil {
+		node.Categories = []string{}
+	}
+	if node.Keywords == nil {
+		node.Keywords = []string{}
+	}
+	if node.Metadata == nil {
+		node.Metadata = map[string]any{}
+	}
+}
+
+func normalizeEdgeCollections(edge *model.Edge) {
+	if edge.Evidence == nil {
+		edge.Evidence = []model.Evidence{}
+	}
+	if edge.Metadata == nil {
+		edge.Metadata = map[string]any{}
+	}
+}
+
+func nodesEquivalent(a, b model.Node) bool {
+	if a.ID != b.ID || a.Kind != b.Kind || a.Label != b.Label ||
+		a.Summary != b.Summary || a.Status != b.Status || a.Origin != b.Origin ||
+		a.ExternalID != b.ExternalID || a.URI != b.URI || a.Weight != b.Weight ||
+		a.X != b.X || a.Y != b.Y || a.Z != b.Z {
+		return false
+	}
+	return JSONEquivalent(a.Categories, b.Categories) &&
+		JSONEquivalent(a.Keywords, b.Keywords) &&
+		JSONEquivalent(a.Metadata, b.Metadata)
+}
+
+func edgesEquivalentIgnoringTimestamps(a, b model.Edge) bool {
+	if a.ID != b.ID || a.Source != b.Source || a.Target != b.Target ||
+		a.Type != b.Type || a.Origin != b.Origin || a.Status != b.Status ||
+		a.Confidence != b.Confidence || a.Weight != b.Weight ||
+		a.Explanation != b.Explanation {
+		return false
+	}
+	return JSONEquivalent(a.Evidence, b.Evidence) && JSONEquivalent(a.Metadata, b.Metadata)
+}
+
+func JSONEquivalent(a, b any) bool {
+	left, leftErr := json.Marshal(a)
+	right, rightErr := json.Marshal(b)
+	return leftErr == nil && rightErr == nil && string(left) == string(right)
+}
+
 func (s *Store) Version() uint64 {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 	return s.version
 }
 
+func (s *Store) Counts() (nodes, edges int, version uint64) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	nodes = len(s.nodes)
+	for _, edge := range s.edges {
+		if edge.Status != "rejected" {
+			edges++
+		}
+	}
+	return nodes, edges, s.version
+}
+
+func (s *Store) IdleNode(seed int64) (model.Node, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	if len(s.nodes) == 0 {
+		return model.Node{}, false
+	}
+	index := int(seed % int64(len(s.nodes)))
+	if index < 0 {
+		index = -index
+	}
+	for _, node := range s.nodes {
+		if index == 0 {
+			return node, true
+		}
+		index--
+	}
+	return model.Node{}, false
+}
+
 func (s *Store) Snapshot() model.Snapshot {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
@@ -284,37 +512,6 @@ func (s *Store) Snapshot() model.Snapshot {
 	sort.Slice(e, func(i, j int) bool { return e[i].ID < e[j].ID })
 	return model.Snapshot{Version: s.version, Nodes: n, Edges: e, UpdatedAt: time.Now().UTC()}
 }
-func (s *Store) Persist() error {
-	_, err := s.PersistVersion()
-	return err
-}
-
-func (s *Store) PersistVersion() (uint64, error) {
-	s.mu.RLock()
-	d := diskState{Version: s.version, Vectors: map[string][]float64{}}
-	for _, n := range s.nodes {
-		d.Nodes = append(d.Nodes, n)
-	}
-	for _, e := range s.edges {
-		d.Edges = append(d.Edges, e)
-	}
-	for k, v := range s.vectors {
-		d.Vectors[k] = append([]float64(nil), v...)
-	}
-	s.mu.RUnlock()
-	b, err := json.Marshal(d)
-	if err != nil {
-		return 0, err
-	}
-	tmp := s.path + ".tmp"
-	if err = os.WriteFile(tmp, b, 0o640); err != nil {
-		return 0, err
-	}
-	if err := os.Rename(tmp, s.path); err != nil {
-		return 0, err
-	}
-	return d.Version, nil
-}
 func (s *Store) Similar(query []float64, limit int) []model.Hit {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
@@ -324,7 +521,7 @@ func (s *Store) Similar(query []float64, limit int) []model.Hit {
 		if !ok || (n.Kind != "knowledge" && n.Kind != "ai-think" && n.Kind != "external") {
 			continue
 		}
-		score := cosine(query, v)
+		score := cosineMixed(query, v)
 		hits = append(hits, model.Hit{NodeID: id, Label: n.Label, Score: score, Kind: n.Kind, Status: n.Status})
 	}
 	sort.Slice(hits, func(i, j int) bool { return hits[i].Score > hits[j].Score })
@@ -402,7 +599,7 @@ func (s *Store) NextPairFilteredDepth(min float64, anchorLimit int, categories [
 				continue
 			}
 			comparisons++
-			score := cosine(lv, rv)
+			score := cosine32(lv, rv)
 			if score >= min && score > best {
 				best = score
 				a, b = left, right
@@ -438,7 +635,7 @@ func (s *Store) BestPair(min float64) (model.Node, model.Node, float64, bool) {
 			if edgeBetweenLocked(s.edges, nodes[i].ID, nodes[j].ID) {
 				continue
 			}
-			score := cosine(s.vectors[nodes[i].ID], s.vectors[nodes[j].ID])
+			score := cosine32(s.vectors[nodes[i].ID], s.vectors[nodes[j].ID])
 			if score >= min && score > best {
 				best = score
 				a = nodes[i]
@@ -523,7 +720,7 @@ func edgeBetweenLocked(edges map[string]model.Edge, a, b string) bool {
 	}
 	return false
 }
-func floatSlicesEqual(a, b []float64) bool {
+func float32SlicesEqual(a, b []float32) bool {
 	if len(a) != len(b) {
 		return false
 	}
@@ -535,15 +732,33 @@ func floatSlicesEqual(a, b []float64) bool {
 	return true
 }
 
-func cosine(a, b []float64) float64 {
+func cosine32(a, b []float32) float64 {
 	if len(a) == 0 || len(a) != len(b) {
 		return 0
 	}
 	var dot, aa, bb float64
 	for i := range a {
-		dot += a[i] * b[i]
+		av, bv := float64(a[i]), float64(b[i])
+		dot += av * bv
+		aa += av * av
+		bb += bv * bv
+	}
+	if aa == 0 || bb == 0 {
+		return 0
+	}
+	return dot / (math.Sqrt(aa) * math.Sqrt(bb))
+}
+
+func cosineMixed(a []float64, b []float32) float64 {
+	if len(a) == 0 || len(a) != len(b) {
+		return 0
+	}
+	var dot, aa, bb float64
+	for i := range a {
+		bv := float64(b[i])
+		dot += a[i] * bv
 		aa += a[i] * a[i]
-		bb += b[i] * b[i]
+		bb += bv * bv
 	}
 	if aa == 0 || bb == 0 {
 		return 0

internal/graph/store_test.go

@@ -11,6 +11,7 @@ func TestReplaceOriginsPreservesUnchangedVector(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	t.Cleanup(func() { _ = s.Close() })
 	n := model.Node{ID: "n1", Kind: "knowledge", Label: "VPN", Summary: "Gateway", Origin: "knowledge-production"}
 	s.ReplaceOrigins([]string{"knowledge-production"}, []model.Node{n}, nil)
 	s.SetVector("n1", []float64{1, 2, 3})
@@ -28,6 +29,7 @@ func TestReplaceOriginsPreservesUnchangedVector(t *testing.T) {
 
 func TestRejectedEdgePreventsPairReprocessingButIsHidden(t *testing.T) {
 	s, _ := Open(t.TempDir())
+	t.Cleanup(func() { _ = s.Close() })
 	a := model.Node{ID: "a", Kind: "knowledge", Label: "A", Origin: "knowledge-production"}
 	b := model.Node{ID: "b", Kind: "knowledge", Label: "B", Origin: "knowledge-production"}
 	s.UpsertNode(a)
@@ -48,6 +50,7 @@ func TestRejectedEdgePreventsPairReprocessingButIsHidden(t *testing.T) {
 
 func TestNextPairUsesBoundedRotatingAnchors(t *testing.T) {
 	s, _ := Open(t.TempDir())
+	t.Cleanup(func() { _ = s.Close() })
 	for _, id := range []string{"a", "b", "c", "d"} {
 		s.UpsertNode(model.Node{ID: id, Kind: "knowledge", Label: id, Origin: "knowledge-production"})
 	}
@@ -75,6 +78,7 @@ func TestCategoryFiltersLimitEmbeddingAndThinkingCandidates(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	t.Cleanup(func() { _ = s.Close() })
 	nodes := []model.Node{
 		{ID: "net-a", Kind: "knowledge", Label: "Net A", Categories: []string{"Netzwerk"}, Origin: "test"},
 		{ID: "net-b", Kind: "knowledge", Label: "Net B", Categories: []string{"Netzwerk"}, Origin: "test"},