Update am Dashboard

2026-07-28 05:33:10 +02:00
parent 1c12e3ed60
commit 40a98e0f57
21 changed files with 1165 additions and 183 deletions
@@ -21,12 +21,18 @@ type Embedder interface {
 	Embed(context.Context, []string) ([][]float64, error)
 }
 type Store struct {
-	mu        sync.RWMutex
-	docs      []model.KnowledgeDoc
-	vectors   map[string][]float64
-	embedder  Embedder
-	rag       bool
-	cachePath string
+	mu             sync.RWMutex
+	dir            string
+	managedDir     string
+	docs           []model.KnowledgeDoc
+	files          map[string]string
+	managed        map[string]bool
+	staticDocs     map[string]model.KnowledgeDoc
+	vectors        map[string][]float64
+	embedder       Embedder
+	rag            bool
+	cachePath      string
+	allowedSources map[string]struct{}
 }
 type cacheFile struct {
 	Hashes  map[string]string    `json:"hashes"`
@@ -34,40 +40,44 @@ type cacheFile struct {
 }

 func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool, allowedSources []string) (*Store, error) {
-	s := &Store{vectors: map[string][]float64{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json")}
-	allowed := make(map[string]struct{}, len(allowedSources))
+	managedDir := filepath.Join(dataDir, "knowledge-managed")
+	if err := os.MkdirAll(managedDir, 0o750); err != nil {
+		return nil, fmt.Errorf("create managed knowledge directory: %w", err)
+	}
+	s := &Store{dir: dir, managedDir: managedDir, vectors: map[string][]float64{}, files: map[string]string{}, managed: map[string]bool{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}}
 	for _, source := range allowedSources {
-		allowed[strings.ToLower(strings.TrimSpace(source))] = struct{}{}
+		s.allowedSources[strings.ToLower(strings.TrimSpace(source))] = struct{}{}
 	}
-	entries, err := os.ReadDir(dir)
+	static, staticFiles, err := readDocs(dir, s.allowedSources)
 	if err != nil {
-		return nil, fmt.Errorf("read knowledge directory %q: %w", dir, err)
+		return nil, err
 	}
-	for _, e := range entries {
-		if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
-			continue
+	for i, d := range static {
+		s.staticDocs[d.ID] = d
+		s.files[d.ID] = staticFiles[i]
+	}
+	managed, managedFiles, err := readDocs(managedDir, s.allowedSources)
+	if err != nil {
+		return nil, err
+	}
+	merged := map[string]model.KnowledgeDoc{}
+	order := []string{}
+	for _, d := range static {
+		if _, ok := merged[d.ID]; !ok {
+			order = append(order, d.ID)
 		}
-		b, err := os.ReadFile(filepath.Join(dir, e.Name()))
-		if err != nil {
-			return nil, err
+		merged[d.ID] = d
+	}
+	for i, d := range managed {
+		if _, ok := merged[d.ID]; !ok {
+			order = append(order, d.ID)
 		}
-		var d model.KnowledgeDoc
-		if err := json.Unmarshal(b, &d); err != nil {
-			return nil, fmt.Errorf("%s: %w", e.Name(), err)
-		}
-		if d.ID == "" || d.Title == "" {
-			return nil, fmt.Errorf("%s: id/title required", e.Name())
-		}
-		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
-		if d.Source == "" {
-			return nil, fmt.Errorf("%s: source required", e.Name())
-		}
-		if _, ok := allowed[d.Source]; !ok {
-			continue
-		}
-		d.Language = strings.TrimSpace(d.Language)
-		d.CommunicationStyle = strings.ToLower(strings.TrimSpace(d.CommunicationStyle))
-		s.docs = append(s.docs, d)
+		merged[d.ID] = d
+		s.files[d.ID] = managedFiles[i]
+		s.managed[d.ID] = true
+	}
+	for _, id := range order {
+		s.docs = append(s.docs, merged[id])
 	}
 	if rag && len(s.docs) > 0 {
 		if s.embedder == nil {
@@ -79,6 +89,48 @@ func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool,
 	}
 	return s, nil
 }
+
+func readDocs(dir string, allowed map[string]struct{}) ([]model.KnowledgeDoc, []string, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read knowledge directory %q: %w", dir, err)
+	}
+	var docs []model.KnowledgeDoc
+	var files []string
+	for _, e := range entries {
+		if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
+			continue
+		}
+		path := filepath.Join(dir, e.Name())
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return nil, nil, err
+		}
+		var d model.KnowledgeDoc
+		if err := json.Unmarshal(b, &d); err != nil {
+			return nil, nil, fmt.Errorf("%s: %w", e.Name(), err)
+		}
+		if d.ID == "" || d.Title == "" {
+			return nil, nil, fmt.Errorf("%s: id/title required", e.Name())
+		}
+		if !safeID(d.ID) {
+			return nil, nil, fmt.Errorf("%s: invalid id %q", e.Name(), d.ID)
+		}
+		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
+		if d.Source == "" {
+			return nil, nil, fmt.Errorf("%s: source required", e.Name())
+		}
+		if _, ok := allowed[d.Source]; !ok {
+			continue
+		}
+		d.Language = strings.TrimSpace(d.Language)
+		d.CommunicationStyle = strings.ToLower(strings.TrimSpace(d.CommunicationStyle))
+		docs = append(docs, d)
+		files = append(files, path)
+	}
+	return docs, files, nil
+}
+
 func (s *Store) Count() int {
 	if s == nil {
 		return 0
@@ -100,6 +152,156 @@ func (s *Store) ByID(id string) (model.KnowledgeDoc, bool) {
 	}
 	return model.KnowledgeDoc{}, false
 }
+func (s *Store) List() []model.KnowledgeDoc {
+	if s == nil {
+		return nil
+	}
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	out := append([]model.KnowledgeDoc(nil), s.docs...)
+	sort.SliceStable(out, func(i, j int) bool { return strings.ToLower(out[i].Title) < strings.ToLower(out[j].Title) })
+	return out
+}
+
+func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
+	if s == nil {
+		return fmt.Errorf("knowledge store is not initialized")
+	}
+	d.ID = strings.TrimSpace(d.ID)
+	d.Title = strings.TrimSpace(d.Title)
+	d.Text = strings.TrimSpace(d.Text)
+	d.Answer = strings.TrimSpace(d.Answer)
+	d.Source = strings.ToLower(strings.TrimSpace(d.Source))
+	d.Language = strings.TrimSpace(d.Language)
+	d.CommunicationStyle = strings.ToLower(strings.TrimSpace(d.CommunicationStyle))
+	if d.ID == "" || d.Title == "" {
+		return fmt.Errorf("id/title required")
+	}
+	if !safeID(d.ID) {
+		return fmt.Errorf("knowledge id may contain only letters, digits, dot, dash and underscore")
+	}
+	if d.Source == "" {
+		return fmt.Errorf("source required")
+	}
+	if _, ok := s.allowedSources[d.Source]; !ok {
+		return fmt.Errorf("source %q is not allowed", d.Source)
+	}
+	if d.Language == "" || d.CommunicationStyle == "" {
+		return fmt.Errorf("language and communication_style required")
+	}
+	if d.MinScore < 0 || d.MinScore > 1 {
+		return fmt.Errorf("min_score must be between 0 and 1")
+	}
+	s.mu.RLock()
+	_, exists := s.files[d.ID]
+	isManaged := s.managed[d.ID]
+	s.mu.RUnlock()
+	if exists && !isManaged {
+		return fmt.Errorf("static knowledge entry %q is read-only; use a new id for a managed entry", d.ID)
+	}
+	var vector []float64
+	if s.rag {
+		if s.embedder == nil {
+			return fmt.Errorf("RAG is enabled but no embedding provider is configured")
+		}
+		vv, err := s.embedder.Embed(ctx, []string{d.Title + "\n" + d.Text + "\n" + strings.Join(d.Keywords, " ")})
+		if err != nil {
+			return err
+		}
+		if len(vv) != 1 || len(vv[0]) == 0 {
+			return fmt.Errorf("embedding provider returned no vector")
+		}
+		vector = vv[0]
+	}
+	path := filepath.Join(s.managedDir, d.ID+".json")
+	b, err := json.MarshalIndent(d, "", "  ")
+	if err != nil {
+		return err
+	}
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, b, 0o640); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		_ = os.Remove(tmp)
+		return err
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	replaced := false
+	for i := range s.docs {
+		if s.docs[i].ID == d.ID {
+			s.docs[i] = d
+			replaced = true
+			break
+		}
+	}
+	if !replaced {
+		s.docs = append(s.docs, d)
+	}
+	s.files[d.ID] = path
+	s.managed[d.ID] = true
+	if s.rag {
+		s.vectors[d.ID] = vector
+	}
+	return nil
+}
+
+func (s *Store) Delete(id string) error {
+	if s == nil {
+		return fmt.Errorf("knowledge store is not initialized")
+	}
+	id = strings.TrimSpace(id)
+	if !safeID(id) {
+		return fmt.Errorf("invalid knowledge id")
+	}
+	s.mu.RLock()
+	path := s.files[id]
+	isManaged := s.managed[id]
+	s.mu.RUnlock()
+	if path == "" {
+		return os.ErrNotExist
+	}
+	if !isManaged {
+		return fmt.Errorf("static knowledge entry %q is read-only", id)
+	}
+	if err := os.Remove(path); err != nil {
+		return err
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	out := s.docs[:0]
+	for _, d := range s.docs {
+		if d.ID != id {
+			out = append(out, d)
+		}
+	}
+	s.docs = append([]model.KnowledgeDoc(nil), out...)
+	delete(s.files, id)
+	delete(s.managed, id)
+	delete(s.vectors, id)
+	return nil
+}
+
+func (s *Store) IsManaged(id string) bool { s.mu.RLock(); defer s.mu.RUnlock(); return s.managed[id] }
+func (s *Store) ManagedDir() string {
+	if s == nil {
+		return ""
+	}
+	return s.managedDir
+}
+func safeID(v string) bool {
+	if v == "" {
+		return false
+	}
+	for _, r := range v {
+		if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.') {
+			return false
+		}
+	}
+	return !strings.Contains(v, "..")
+}
+
 func (s *Store) Search(ctx context.Context, text string, topK int) ([]model.KnowledgeHit, error) {
 	if s == nil {
 		return nil, fmt.Errorf("knowledge store is not initialized")
@@ -6,6 +6,8 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/example/glpi-ai-agent/internal/model"
 )

 func TestLoadSearchesOnlyAllowedSources(t *testing.T) {
@@ -86,3 +88,28 @@ func TestRAGRequiresEmbedderWhenDocumentsExist(t *testing.T) {
 		t.Fatal("expected RAG without embedder to fail")
 	}
 }
+
+func TestUpsertDelete(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	s, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := model.KnowledgeDoc{ID: "KB-1", Title: "Test", Text: "Wissen", Source: "internal-kb", Language: "de-DE", CommunicationStyle: "formal", MinScore: 0.8}
+	if err := s.Upsert(context.Background(), d); err != nil {
+		t.Fatal(err)
+	}
+	if s.Count() != 1 {
+		t.Fatalf("count=%d", s.Count())
+	}
+	if _, ok := s.ByID("KB-1"); !ok {
+		t.Fatal("missing")
+	}
+	if err := s.Delete("KB-1"); err != nil {
+		t.Fatal(err)
+	}
+	if s.Count() != 0 {
+		t.Fatalf("count=%d", s.Count())
+	}
+}