Kategorie-Problem bei Typ String bezüglich Daten aus dem KB-System

2026-07-29 07:59:59 +02:00
parent 12fcfe1b10
commit 6b38642ba1
10 changed files with 472 additions and 31 deletions
@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"unicode"
@@ -32,6 +33,21 @@ type ScoringConfig struct {
 	ChunkOverlap      int
 	MaxChunksPerDoc   int
 	MaxQueryChunks    int
+	CategoryMode      string
+	CategoryMapFile   string
+	IgnoreGlobs       []string
+}
+
+type LoadOptions struct {
+	CategoryMode    string
+	CategoryMapFile string
+	IgnoreGlobs     []string
+}
+
+type LoadStats struct {
+	IgnoredFiles          int      `json:"ignored_files"`
+	UnmappedCategoryFiles int      `json:"unmapped_category_files"`
+	UnmappedCategories    []string `json:"unmapped_categories,omitempty"`
 }

 type Store struct {
@@ -51,6 +67,9 @@ type Store struct {
 	cachePath      string
 	allowedSources map[string]struct{}
 	scoring        ScoringConfig
+	loadOptions    LoadOptions
+	loadStats      LoadStats
+	categoryMap    map[string][]int64
 }
 type cacheFile struct {
 	Version      int                    `json:"version,omitempty"`
@@ -108,22 +127,34 @@ func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool,
 	if len(scoring) > 0 {
 		scoreCfg = normalizeScoring(scoring[0])
 	}
-	s := &Store{dir: dir, managedDir: managedDir, titleVectors: map[string][]float64{}, chunkVectors: map[string][][]float64{}, chunks: map[string][]string{}, files: map[string]string{}, managed: map[string]bool{}, external: map[string]string{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}, scoring: scoreCfg}
-	for _, source := range allowedSources {
-		s.allowedSources[strings.ToLower(strings.TrimSpace(source))] = struct{}{}
+	loadOpts := LoadOptions{CategoryMode: scoreCfg.CategoryMode, CategoryMapFile: scoreCfg.CategoryMapFile, IgnoreGlobs: scoreCfg.IgnoreGlobs}
+	if strings.TrimSpace(loadOpts.CategoryMode) == "" {
+		loadOpts.CategoryMode = "unscoped"
 	}
-	static, staticFiles, err := readDocs(dir, s.allowedSources)
+	categoryMap, err := loadCategoryMap(loadOpts.CategoryMapFile)
 	if err != nil {
 		return nil, err
 	}
+	s := &Store{dir: dir, managedDir: managedDir, titleVectors: map[string][]float64{}, chunkVectors: map[string][][]float64{}, chunks: map[string][]string{}, files: map[string]string{}, managed: map[string]bool{}, external: map[string]string{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}, scoring: scoreCfg, loadOptions: loadOpts, categoryMap: categoryMap}
+	for _, source := range allowedSources {
+		s.allowedSources[strings.ToLower(strings.TrimSpace(source))] = struct{}{}
+	}
+	static, staticFiles, stats, err := readDocs(dir, s.allowedSources, loadOpts, categoryMap)
+	if err != nil {
+		return nil, err
+	}
+	s.loadStats = stats
 	for i, d := range static {
 		s.staticDocs[d.ID] = d
 		s.files[d.ID] = staticFiles[i]
 	}
-	managed, managedFiles, err := readDocs(managedDir, s.allowedSources)
+	managed, managedFiles, managedStats, err := readDocs(managedDir, s.allowedSources, LoadOptions{CategoryMode: "strict"}, categoryMap)
 	if err != nil {
 		return nil, err
 	}
+	s.loadStats.IgnoredFiles += managedStats.IgnoredFiles
+	s.loadStats.UnmappedCategoryFiles += managedStats.UnmappedCategoryFiles
+	s.loadStats.UnmappedCategories = mergeStrings(s.loadStats.UnmappedCategories, managedStats.UnmappedCategories)
 	merged := map[string]model.KnowledgeDoc{}
 	order := []string{}
 	for _, d := range static {
@@ -154,35 +185,48 @@ func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool,
 	return s, nil
 }

-func readDocs(dir string, allowed map[string]struct{}) ([]model.KnowledgeDoc, []string, error) {
+func readDocs(dir string, allowed map[string]struct{}, opts LoadOptions, categoryMap map[string][]int64) ([]model.KnowledgeDoc, []string, LoadStats, error) {
 	entries, err := os.ReadDir(dir)
 	if err != nil {
-		return nil, nil, fmt.Errorf("read knowledge directory %q: %w", dir, err)
+		return nil, nil, LoadStats{}, fmt.Errorf("read knowledge directory %q: %w", dir, err)
 	}
 	var docs []model.KnowledgeDoc
 	var files []string
+	stats := LoadStats{}
 	for _, e := range entries {
 		if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
 			continue
 		}
+		if matchesAnyGlob(e.Name(), opts.IgnoreGlobs) {
+			stats.IgnoredFiles++
+			continue
+		}
 		path := filepath.Join(dir, e.Name())
 		b, err := os.ReadFile(path)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, stats, err
 		}
-		var d model.KnowledgeDoc
-		if err := json.Unmarshal(b, &d); err != nil {
-			return nil, nil, fmt.Errorf("%s: %w", e.Name(), err)
+		d, unmapped, skip, err := decodeKnowledgeDoc(b, opts.CategoryMode, categoryMap)
+		if err != nil {
+			return nil, nil, stats, fmt.Errorf("%s: %w", e.Name(), err)
+		}
+		if len(unmapped) > 0 {
+			stats.UnmappedCategoryFiles++
+			stats.UnmappedCategories = mergeStrings(stats.UnmappedCategories, unmapped)
+		}
+		if skip {
+			stats.IgnoredFiles++
+			continue
 		}
 		if d.ID == "" || d.Title == "" {
-			return nil, nil, fmt.Errorf("%s: id/title required", e.Name())
+			return nil, nil, stats, fmt.Errorf("%s: id/title required", e.Name())
 		}
 		if !safeID(d.ID) {
-			return nil, nil, fmt.Errorf("%s: invalid id %q", e.Name(), d.ID)
+			return nil, nil, stats, fmt.Errorf("%s: invalid id %q", e.Name(), d.ID)
 		}
 		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
 		if d.Source == "" {
-			return nil, nil, fmt.Errorf("%s: source required", e.Name())
+			return nil, nil, stats, fmt.Errorf("%s: source required", e.Name())
 		}
 		if _, ok := allowed[d.Source]; !ok {
 			continue
@@ -192,7 +236,229 @@ func readDocs(dir string, allowed map[string]struct{}) ([]model.KnowledgeDoc, []
 		docs = append(docs, d)
 		files = append(files, path)
 	}
-	return docs, files, nil
+	return docs, files, stats, nil
+}
+
+func decodeKnowledgeDoc(b []byte, mode string, categoryMap map[string][]int64) (model.KnowledgeDoc, []string, bool, error) {
+	var raw map[string]json.RawMessage
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return model.KnowledgeDoc{}, nil, false, err
+	}
+	catRaw := raw["categories"]
+	raw["categories"] = json.RawMessage(`[]`)
+	normalized, err := json.Marshal(raw)
+	if err != nil {
+		return model.KnowledgeDoc{}, nil, false, err
+	}
+	var d model.KnowledgeDoc
+	if err := json.Unmarshal(normalized, &d); err != nil {
+		return model.KnowledgeDoc{}, nil, false, err
+	}
+	ids, labels, unmapped, err := parseKnowledgeCategories(catRaw, categoryMap)
+	if err != nil {
+		return model.KnowledgeDoc{}, nil, false, err
+	}
+	d.Categories = ids
+	d.ExternalCategories = labels
+	d.UnmappedExternalCategories = append([]string(nil), unmapped...)
+	if len(unmapped) == 0 {
+		return d, nil, false, nil
+	}
+	switch strings.ToLower(strings.TrimSpace(mode)) {
+	case "strict":
+		return model.KnowledgeDoc{}, unmapped, false, fmt.Errorf("unmapped external categories: %s", strings.Join(unmapped, ", "))
+	case "skip":
+		return d, unmapped, true, nil
+	default:
+		// Unmapped external taxonomies remain searchable, but may not trigger an automatic reply.
+		d.AutoReply = false
+		return d, unmapped, false, nil
+	}
+}
+
+func parseKnowledgeCategories(raw json.RawMessage, categoryMap map[string][]int64) ([]int64, []string, []string, error) {
+	if len(raw) == 0 || string(raw) == "null" {
+		return nil, nil, nil, nil
+	}
+	var v any
+	if err := json.Unmarshal(raw, &v); err != nil {
+		return nil, nil, nil, err
+	}
+	items, ok := v.([]any)
+	if !ok {
+		items = []any{v}
+	}
+	ids := []int64{}
+	labels := []string{}
+	unmapped := []string{}
+	for _, item := range items {
+		itemIDs, label, err := parseCategoryItem(item, categoryMap)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		ids = append(ids, itemIDs...)
+		if label != "" {
+			labels = appendUniqueString(labels, label)
+			if len(itemIDs) == 0 {
+				unmapped = appendUniqueString(unmapped, label)
+			}
+		}
+	}
+	return uniqueInt64(ids), labels, unmapped, nil
+}
+
+func parseCategoryItem(v any, categoryMap map[string][]int64) ([]int64, string, error) {
+	switch x := v.(type) {
+	case nil:
+		return nil, "", nil
+	case float64:
+		if x <= 0 || math.Trunc(x) != x {
+			return nil, "", fmt.Errorf("category id must be a positive integer")
+		}
+		return []int64{int64(x)}, "", nil
+	case string:
+		label := strings.TrimSpace(x)
+		if label == "" {
+			return nil, "", nil
+		}
+		if n, err := strconv.ParseInt(label, 10, 64); err == nil && n > 0 {
+			return []int64{n}, "", nil
+		}
+		return append([]int64(nil), categoryMap[normalizeCategoryLabel(label)]...), label, nil
+	case map[string]any:
+		if id, ok := x["id"]; ok {
+			ids, _, err := parseCategoryItem(id, categoryMap)
+			if err == nil && len(ids) > 0 {
+				return ids, "", nil
+			}
+		}
+		for _, key := range []string{"name", "label", "title"} {
+			if val, ok := x[key].(string); ok {
+				return parseCategoryItem(val, categoryMap)
+			}
+		}
+		return nil, "", fmt.Errorf("unsupported category object")
+	default:
+		return nil, "", fmt.Errorf("unsupported category value type %T", v)
+	}
+}
+
+func loadCategoryMap(path string) (map[string][]int64, error) {
+	out := map[string][]int64{}
+	path = strings.TrimSpace(path)
+	if path == "" {
+		return out, nil
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read knowledge category map %q: %w", path, err)
+	}
+	var root map[string]json.RawMessage
+	if err := json.Unmarshal(b, &root); err != nil {
+		return nil, fmt.Errorf("parse knowledge category map %q: %w", path, err)
+	}
+	if nested, ok := root["mappings"]; ok {
+		var m map[string]json.RawMessage
+		if err := json.Unmarshal(nested, &m); err != nil {
+			return nil, fmt.Errorf("parse mappings in %q: %w", path, err)
+		}
+		root = m
+	}
+	for label, rv := range root {
+		ids, err := parseMappingIDs(rv)
+		if err != nil {
+			return nil, fmt.Errorf("category mapping %q: %w", label, err)
+		}
+		if len(ids) == 0 {
+			return nil, fmt.Errorf("category mapping %q contains no positive GLPI ids", label)
+		}
+		out[normalizeCategoryLabel(label)] = uniqueInt64(ids)
+	}
+	return out, nil
+}
+
+func parseMappingIDs(raw json.RawMessage) ([]int64, error) {
+	var v any
+	if err := json.Unmarshal(raw, &v); err != nil {
+		return nil, err
+	}
+	items, ok := v.([]any)
+	if !ok {
+		items = []any{v}
+	}
+	ids := []int64{}
+	for _, item := range items {
+		switch x := item.(type) {
+		case float64:
+			if x <= 0 || math.Trunc(x) != x {
+				return nil, fmt.Errorf("id must be a positive integer")
+			}
+			ids = append(ids, int64(x))
+		case string:
+			n, err := strconv.ParseInt(strings.TrimSpace(x), 10, 64)
+			if err != nil || n <= 0 {
+				return nil, fmt.Errorf("%q is not a positive GLPI category id", x)
+			}
+			ids = append(ids, n)
+		default:
+			return nil, fmt.Errorf("unsupported mapping value type %T", item)
+		}
+	}
+	return ids, nil
+}
+
+func matchesAnyGlob(name string, patterns []string) bool {
+	for _, pattern := range patterns {
+		if ok, _ := filepath.Match(pattern, name); ok {
+			return true
+		}
+	}
+	return false
+}
+func normalizeCategoryLabel(s string) string {
+	return strings.ToLower(strings.Join(strings.Fields(s), " "))
+}
+func uniqueInt64(in []int64) []int64 {
+	seen := map[int64]struct{}{}
+	out := []int64{}
+	for _, v := range in {
+		if v <= 0 {
+			continue
+		}
+		if _, ok := seen[v]; ok {
+			continue
+		}
+		seen[v] = struct{}{}
+		out = append(out, v)
+	}
+	return out
+}
+func appendUniqueString(in []string, s string) []string {
+	for _, v := range in {
+		if strings.EqualFold(v, s) {
+			return in
+		}
+	}
+	return append(in, s)
+}
+func mergeStrings(a, b []string) []string {
+	out := append([]string(nil), a...)
+	for _, s := range b {
+		out = appendUniqueString(out, s)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func (s *Store) LoadStats() LoadStats {
+	if s == nil {
+		return LoadStats{}
+	}
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	out := s.loadStats
+	out.UnmappedCategories = append([]string(nil), s.loadStats.UnmappedCategories...)
+	return out
 }

 func (s *Store) Count() int {
@@ -1206,7 +1472,7 @@ func cosine(a, b []float64) float64 {
 }
 func lexical(text string, d model.KnowledgeDoc) float64 {
 	q := tokens(text)
-	hay := tokens(d.Title + " " + d.Text + " " + strings.Join(d.Keywords, " "))
+	hay := tokens(d.Title + " " + d.Text + " " + strings.Join(d.Keywords, " ") + " " + strings.Join(d.ExternalCategories, " "))
 	if len(q) == 0 {
 		return 0
 	}
@@ -310,3 +310,85 @@ func TestRegressionShortGermanLoginTicketGetsStrongLexicalEvidence(t *testing.T)
 		t.Fatalf("anmelden/Benutzeranmeldung should match through a German support stem, got %f", sim)
 	}
 }
+
+func TestExternalStringCategoryLoadsUnscoped(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	body := `{"id":"KB-EXT-1","title":"Docker Test","text":"Docker Fehler","answer":"Pruefen","auto_reply":false,"min_score":0.7,"categories":["Docker","Security"],"keywords":["docker"],"source":"internal-kb","language":"de-DE","communication_style":"formal"}`
+	if err := os.WriteFile(filepath.Join(dir, "ext.json"), []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	s, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"}, ScoringConfig{CategoryMode: "unscoped"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	doc, ok := s.ByID("KB-EXT-1")
+	if !ok {
+		t.Fatal("document not loaded")
+	}
+	if len(doc.Categories) != 0 {
+		t.Fatalf("expected no GLPI ids, got %v", doc.Categories)
+	}
+	if len(doc.ExternalCategories) != 2 {
+		t.Fatalf("external categories=%v", doc.ExternalCategories)
+	}
+	stats := s.LoadStats()
+	if stats.UnmappedCategoryFiles != 1 {
+		t.Fatalf("stats=%+v", stats)
+	}
+}
+
+func TestExternalStringCategoryMapsToGLPI(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	mapPath := filepath.Join(data, "category-map.json")
+	if err := os.WriteFile(mapPath, []byte(`{"Docker":[12,13],"Security":7}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	body := `{"id":"KB-EXT-2","title":"Docker Test","text":"Docker Fehler","answer":"Pruefen","auto_reply":false,"min_score":0.7,"categories":["Docker","Security"],"keywords":["docker"],"source":"internal-kb","language":"de-DE","communication_style":"formal"}`
+	if err := os.WriteFile(filepath.Join(dir, "ext.json"), []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	s, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"}, ScoringConfig{CategoryMode: "unscoped", CategoryMapFile: mapPath})
+	if err != nil {
+		t.Fatal(err)
+	}
+	doc, _ := s.ByID("KB-EXT-2")
+	want := []int64{12, 13, 7}
+	if len(doc.Categories) != len(want) {
+		t.Fatalf("categories=%v", doc.Categories)
+	}
+	for _, id := range want {
+		found := false
+		for _, got := range doc.Categories {
+			if got == id {
+				found = true
+			}
+		}
+		if !found {
+			t.Fatalf("missing id %d in %v", id, doc.Categories)
+		}
+	}
+	if s.LoadStats().UnmappedCategoryFiles != 0 {
+		t.Fatalf("unexpected unmapped stats: %+v", s.LoadStats())
+	}
+}
+
+func TestKnowledgeIgnoreGlobs(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	bad := `{"id":"KB-BAD","title":"Foreign","text":"x","answer":"x","auto_reply":false,"min_score":0.7,"categories":[{"unsupported":true}],"source":"internal-kb","language":"de-DE","communication_style":"formal"}`
+	if err := os.WriteFile(filepath.Join(dir, "KB-SEC-ATTCK-AN-0001.json"), []byte(bad), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	s, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"}, ScoringConfig{CategoryMode: "strict", IgnoreGlobs: []string{"KB-SEC-ATTCK-*.json"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Count() != 0 {
+		t.Fatalf("count=%d", s.Count())
+	}
+	if s.LoadStats().IgnoredFiles != 1 {
+		t.Fatalf("stats=%+v", s.LoadStats())
+	}
+}