Update GLPI-Knowledge

2026-07-28 14:35:03 +02:00
parent 8ac1f97174
commit 4cfdac042d
18 changed files with 992 additions and 29 deletions
@@ -27,6 +27,7 @@ type Store struct {
 	docs           []model.KnowledgeDoc
 	files          map[string]string
 	managed        map[string]bool
+	external       map[string]string
 	staticDocs     map[string]model.KnowledgeDoc
 	vectors        map[string][]float64
 	embedder       Embedder
@@ -44,7 +45,7 @@ func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool,
 	if err := os.MkdirAll(managedDir, 0o750); err != nil {
 		return nil, fmt.Errorf("create managed knowledge directory: %w", err)
 	}
-	s := &Store{dir: dir, managedDir: managedDir, vectors: map[string][]float64{}, files: map[string]string{}, managed: map[string]bool{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}}
+	s := &Store{dir: dir, managedDir: managedDir, vectors: map[string][]float64{}, files: map[string]string{}, managed: map[string]bool{}, external: map[string]string{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}}
 	for _, source := range allowedSources {
 		s.allowedSources[strings.ToLower(strings.TrimSpace(source))] = struct{}{}
 	}
@@ -195,7 +196,11 @@ func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
 	s.mu.RLock()
 	_, exists := s.files[d.ID]
 	isManaged := s.managed[d.ID]
+	externalSource := s.external[d.ID]
 	s.mu.RUnlock()
+	if externalSource != "" {
+		return fmt.Errorf("externally synchronized knowledge entry %q from %q is read-only", d.ID, externalSource)
+	}
 	if exists && !isManaged {
 		return fmt.Errorf("static knowledge entry %q is read-only; use a new id for a managed entry", d.ID)
 	}
@@ -284,6 +289,169 @@ func (s *Store) Delete(id string) error {
 }

 func (s *Store) IsManaged(id string) bool { s.mu.RLock(); defer s.mu.RUnlock(); return s.managed[id] }
+func (s *Store) Origin(id string) string {
+	if s == nil {
+		return ""
+	}
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	if s.managed[id] {
+		return "managed"
+	}
+	if src := s.external[id]; src != "" {
+		return src
+	}
+	if _, ok := s.staticDocs[id]; ok {
+		return "static"
+	}
+	return ""
+}
+
+// ReplaceExternalSource atomically replaces all read-only documents imported
+// from one connector source. Existing vectors are reused when the normalized
+// document did not change, so periodic synchronization does not re-embed the
+// whole GLPI knowledge base on every run.
+func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs []model.KnowledgeDoc) error {
+	if s == nil {
+		return fmt.Errorf("knowledge store is not initialized")
+	}
+	source = strings.ToLower(strings.TrimSpace(source))
+	if _, ok := s.allowedSources[source]; !ok {
+		return fmt.Errorf("source %q is not allowed", source)
+	}
+
+	s.mu.RLock()
+	oldDocs := make(map[string]model.KnowledgeDoc, len(s.docs))
+	oldVectors := make(map[string][]float64, len(s.vectors))
+	for _, d := range s.docs {
+		oldDocs[d.ID] = d
+	}
+	for id, v := range s.vectors {
+		oldVectors[id] = append([]float64(nil), v...)
+	}
+	s.mu.RUnlock()
+	cached := cacheFile{Hashes: map[string]string{}, Vectors: map[string][]float64{}}
+	if b, err := os.ReadFile(s.cachePath); err == nil {
+		_ = json.Unmarshal(b, &cached)
+	}
+
+	changed := make([]model.KnowledgeDoc, 0)
+	seen := map[string]struct{}{}
+	for i := range docs {
+		d := &docs[i]
+		d.ID = strings.TrimSpace(d.ID)
+		d.Title = strings.TrimSpace(d.Title)
+		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
+		if d.Source == "" {
+			d.Source = source
+		}
+		if d.Source != source {
+			return fmt.Errorf("external document %q has source %q, expected %q", d.ID, d.Source, source)
+		}
+		if d.ID == "" || d.Title == "" || !safeID(d.ID) {
+			return fmt.Errorf("invalid external knowledge document id/title")
+		}
+		if _, dup := seen[d.ID]; dup {
+			return fmt.Errorf("duplicate external knowledge id %q", d.ID)
+		}
+		seen[d.ID] = struct{}{}
+		h := hashDoc(*d)
+		old, ok := oldDocs[d.ID]
+		same := ok && hashDoc(old) == h && len(oldVectors[d.ID]) > 0
+		if !same && cached.Hashes[d.ID] == h && len(cached.Vectors[d.ID]) > 0 {
+			oldVectors[d.ID] = append([]float64(nil), cached.Vectors[d.ID]...)
+			same = true
+		}
+		if !same {
+			changed = append(changed, *d)
+		}
+	}
+	newVectors := map[string][]float64{}
+	if s.rag && len(changed) > 0 {
+		if s.embedder == nil {
+			return fmt.Errorf("RAG is enabled but no embedding provider is configured")
+		}
+		texts := make([]string, len(changed))
+		for i, d := range changed {
+			texts[i] = d.Title + "\n" + d.Text + "\n" + strings.Join(d.Keywords, " ")
+		}
+		vv, err := s.embedder.Embed(ctx, texts)
+		if err != nil {
+			return err
+		}
+		if len(vv) != len(changed) {
+			return fmt.Errorf("embedding provider returned %d vectors for %d documents", len(vv), len(changed))
+		}
+		for i, d := range changed {
+			if len(vv[i]) == 0 {
+				return fmt.Errorf("embedding provider returned empty vector for %s", d.ID)
+			}
+			newVectors[d.ID] = vv[i]
+		}
+	}
+
+	s.mu.Lock()
+	// Reject collisions with local/static documents.
+	for _, d := range docs {
+		if src := s.external[d.ID]; src == "" {
+			if _, exists := oldDocs[d.ID]; exists {
+				s.mu.Unlock()
+				return fmt.Errorf("external knowledge id %q collides with local knowledge", d.ID)
+			}
+		} else if src != source {
+			s.mu.Unlock()
+			return fmt.Errorf("external knowledge id %q belongs to source %q", d.ID, src)
+		}
+	}
+	rebuilt := make([]model.KnowledgeDoc, 0, len(s.docs)+len(docs))
+	for _, d := range s.docs {
+		if s.external[d.ID] != source {
+			rebuilt = append(rebuilt, d)
+		}
+	}
+	for id, src := range s.external {
+		if src == source {
+			delete(s.external, id)
+			delete(s.vectors, id)
+		}
+	}
+	for _, d := range docs {
+		rebuilt = append(rebuilt, d)
+		s.external[d.ID] = source
+		if v := newVectors[d.ID]; len(v) > 0 {
+			s.vectors[d.ID] = v
+		} else if v := oldVectors[d.ID]; len(v) > 0 {
+			s.vectors[d.ID] = v
+		}
+	}
+	s.docs = rebuilt
+	s.mu.Unlock()
+	return s.persistVectorCache()
+}
+
+func (s *Store) persistVectorCache() error {
+	if s == nil || !s.rag {
+		return nil
+	}
+	s.mu.RLock()
+	cf := cacheFile{Hashes: map[string]string{}, Vectors: map[string][]float64{}}
+	for _, d := range s.docs {
+		if v := s.vectors[d.ID]; len(v) > 0 {
+			cf.Hashes[d.ID] = hashDoc(d)
+			cf.Vectors[d.ID] = append([]float64(nil), v...)
+		}
+	}
+	s.mu.RUnlock()
+	b, err := json.MarshalIndent(cf, "", "  ")
+	if err != nil {
+		return err
+	}
+	tmp := s.cachePath + ".tmp"
+	if err := os.WriteFile(tmp, b, 0o640); err != nil {
+		return err
+	}
+	return os.Rename(tmp, s.cachePath)
+}
 func (s *Store) ManagedDir() string {
 	if s == nil {
 		return ""
@@ -113,3 +113,22 @@ func TestUpsertDelete(t *testing.T) {
 		t.Fatalf("count=%d", s.Count())
 	}
 }
+
+func TestExternalKnowledgeIsReadOnly(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	s, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb", "glpi-kb"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := model.KnowledgeDoc{ID: "GLPI-KB-1", Title: "Extern", Text: "Wissen", Source: "glpi-kb", Language: "de-DE", CommunicationStyle: "formal"}
+	if err := s.ReplaceExternalSource(context.Background(), "glpi-kb", []model.KnowledgeDoc{d}); err != nil {
+		t.Fatal(err)
+	}
+	if s.Count() != 1 || s.Origin("GLPI-KB-1") != "glpi-kb" {
+		t.Fatalf("unexpected external store state")
+	}
+	if err := s.Upsert(context.Background(), d); err == nil {
+		t.Fatal("expected external document to be read-only")
+	}
+}