Update Chunking der KBs

2026-07-28 15:16:15 +02:00
parent 4cfdac042d
commit 45b3d1b2df
13 changed files with 650 additions and 111 deletions
@@ -20,6 +20,16 @@ import (
 type Embedder interface {
 	Embed(context.Context, []string) ([][]float64, error)
 }
+type ScoringConfig struct {
+	SemanticWeight  float64
+	TitleWeight     float64
+	KeywordWeight   float64
+	CategoryWeight  float64
+	ChunkWords      int
+	ChunkOverlap    int
+	MaxChunksPerDoc int
+}
+
 type Store struct {
 	mu             sync.RWMutex
 	dir            string
@@ -29,23 +39,53 @@ type Store struct {
 	managed        map[string]bool
 	external       map[string]string
 	staticDocs     map[string]model.KnowledgeDoc
-	vectors        map[string][]float64
+	titleVectors   map[string][]float64
+	chunkVectors   map[string][][]float64
+	chunks         map[string][]string
 	embedder       Embedder
 	rag            bool
 	cachePath      string
 	allowedSources map[string]struct{}
+	scoring        ScoringConfig
 }
 type cacheFile struct {
-	Hashes  map[string]string    `json:"hashes"`
-	Vectors map[string][]float64 `json:"vectors"`
+	Version      int                    `json:"version,omitempty"`
+	Hashes       map[string]string      `json:"hashes"`
+	TitleVectors map[string][]float64   `json:"title_vectors,omitempty"`
+	ChunkVectors map[string][][]float64 `json:"chunk_vectors,omitempty"`
 }

-func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool, allowedSources []string) (*Store, error) {
+func DefaultScoringConfig() ScoringConfig {
+	return ScoringConfig{SemanticWeight: .50, TitleWeight: .25, KeywordWeight: .15, CategoryWeight: .10, ChunkWords: 160, ChunkOverlap: 30, MaxChunksPerDoc: 24}
+}
+
+func normalizeScoring(c ScoringConfig) ScoringConfig {
+	d := DefaultScoringConfig()
+	if c.SemanticWeight < 0 || c.TitleWeight < 0 || c.KeywordWeight < 0 || c.CategoryWeight < 0 || c.SemanticWeight+c.TitleWeight+c.KeywordWeight+c.CategoryWeight <= 0 {
+		c.SemanticWeight, c.TitleWeight, c.KeywordWeight, c.CategoryWeight = d.SemanticWeight, d.TitleWeight, d.KeywordWeight, d.CategoryWeight
+	}
+	if c.ChunkWords <= 0 {
+		c.ChunkWords = d.ChunkWords
+	}
+	if c.ChunkOverlap < 0 || c.ChunkOverlap >= c.ChunkWords {
+		c.ChunkOverlap = d.ChunkOverlap
+	}
+	if c.MaxChunksPerDoc <= 0 {
+		c.MaxChunksPerDoc = d.MaxChunksPerDoc
+	}
+	return c
+}
+
+func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool, allowedSources []string, scoring ...ScoringConfig) (*Store, error) {
 	managedDir := filepath.Join(dataDir, "knowledge-managed")
 	if err := os.MkdirAll(managedDir, 0o750); err != nil {
 		return nil, fmt.Errorf("create managed knowledge directory: %w", err)
 	}
-	s := &Store{dir: dir, managedDir: managedDir, vectors: map[string][]float64{}, files: map[string]string{}, managed: map[string]bool{}, external: map[string]string{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}}
+	scoreCfg := DefaultScoringConfig()
+	if len(scoring) > 0 {
+		scoreCfg = normalizeScoring(scoring[0])
+	}
+	s := &Store{dir: dir, managedDir: managedDir, titleVectors: map[string][]float64{}, chunkVectors: map[string][][]float64{}, chunks: map[string][]string{}, files: map[string]string{}, managed: map[string]bool{}, external: map[string]string{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}, scoring: scoreCfg}
 	for _, source := range allowedSources {
 		s.allowedSources[strings.ToLower(strings.TrimSpace(source))] = struct{}{}
 	}
@@ -204,20 +244,22 @@ func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
 	if exists && !isManaged {
 		return fmt.Errorf("static knowledge entry %q is read-only; use a new id for a managed entry", d.ID)
 	}
-	var vector []float64
+
+	var titleVector []float64
+	var chunkVectors [][]float64
+	chunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
 	if s.rag {
 		if s.embedder == nil {
 			return fmt.Errorf("RAG is enabled but no embedding provider is configured")
 		}
-		vv, err := s.embedder.Embed(ctx, []string{d.Title + "\n" + d.Text + "\n" + strings.Join(d.Keywords, " ")})
+		embedded, err := s.embedDocuments(ctx, []model.KnowledgeDoc{d})
 		if err != nil {
 			return err
 		}
-		if len(vv) != 1 || len(vv[0]) == 0 {
-			return fmt.Errorf("embedding provider returned no vector")
-		}
-		vector = vv[0]
+		titleVector = embedded[d.ID].title
+		chunkVectors = embedded[d.ID].chunks
 	}
+
 	path := filepath.Join(s.managedDir, d.ID+".json")
 	b, err := json.MarshalIndent(d, "", "  ")
 	if err != nil {
@@ -232,7 +274,6 @@ func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
 		return err
 	}
 	s.mu.Lock()
-	defer s.mu.Unlock()
 	replaced := false
 	for i := range s.docs {
 		if s.docs[i].ID == d.ID {
@@ -246,10 +287,13 @@ func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
 	}
 	s.files[d.ID] = path
 	s.managed[d.ID] = true
+	s.chunks[d.ID] = chunks
 	if s.rag {
-		s.vectors[d.ID] = vector
+		s.titleVectors[d.ID] = titleVector
+		s.chunkVectors[d.ID] = chunkVectors
 	}
-	return nil
+	s.mu.Unlock()
+	return s.persistVectorCache()
 }

 func (s *Store) Delete(id string) error {
@@ -274,7 +318,6 @@ func (s *Store) Delete(id string) error {
 		return err
 	}
 	s.mu.Lock()
-	defer s.mu.Unlock()
 	out := s.docs[:0]
 	for _, d := range s.docs {
 		if d.ID != id {
@@ -284,11 +327,21 @@ func (s *Store) Delete(id string) error {
 	s.docs = append([]model.KnowledgeDoc(nil), out...)
 	delete(s.files, id)
 	delete(s.managed, id)
-	delete(s.vectors, id)
-	return nil
+	delete(s.titleVectors, id)
+	delete(s.chunkVectors, id)
+	delete(s.chunks, id)
+	s.mu.Unlock()
+	return s.persistVectorCache()
 }

-func (s *Store) IsManaged(id string) bool { s.mu.RLock(); defer s.mu.RUnlock(); return s.managed[id] }
+func (s *Store) IsManaged(id string) bool {
+	if s == nil {
+		return false
+	}
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.managed[id]
+}
 func (s *Store) Origin(id string) string {
 	if s == nil {
 		return ""
@@ -308,9 +361,8 @@ func (s *Store) Origin(id string) string {
 }

 // ReplaceExternalSource atomically replaces all read-only documents imported
-// from one connector source. Existing vectors are reused when the normalized
-// document did not change, so periodic synchronization does not re-embed the
-// whole GLPI knowledge base on every run.
+// from one connector source. Embeddings are reused when the normalized article
+// did not change. Long article bodies are indexed as overlapping chunks.
 func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs []model.KnowledgeDoc) error {
 	if s == nil {
 		return fmt.Errorf("knowledge store is not initialized")
@@ -322,18 +374,13 @@ func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs [

 	s.mu.RLock()
 	oldDocs := make(map[string]model.KnowledgeDoc, len(s.docs))
-	oldVectors := make(map[string][]float64, len(s.vectors))
+	oldTitle := cloneVectorMap(s.titleVectors)
+	oldChunks := cloneChunkVectorMap(s.chunkVectors)
 	for _, d := range s.docs {
 		oldDocs[d.ID] = d
 	}
-	for id, v := range s.vectors {
-		oldVectors[id] = append([]float64(nil), v...)
-	}
 	s.mu.RUnlock()
-	cached := cacheFile{Hashes: map[string]string{}, Vectors: map[string][]float64{}}
-	if b, err := os.ReadFile(s.cachePath); err == nil {
-		_ = json.Unmarshal(b, &cached)
-	}
+	cached := loadCache(s.cachePath)

 	changed := make([]model.KnowledgeDoc, 0)
 	seen := map[string]struct{}{}
@@ -356,42 +403,33 @@ func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs [
 		}
 		seen[d.ID] = struct{}{}
 		h := hashDoc(*d)
-		old, ok := oldDocs[d.ID]
-		same := ok && hashDoc(old) == h && len(oldVectors[d.ID]) > 0
-		if !same && cached.Hashes[d.ID] == h && len(cached.Vectors[d.ID]) > 0 {
-			oldVectors[d.ID] = append([]float64(nil), cached.Vectors[d.ID]...)
+		bodyChunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
+		same := false
+		if old, ok := oldDocs[d.ID]; ok && hashDoc(old) == h && len(oldTitle[d.ID]) > 0 && len(oldChunks[d.ID]) == len(bodyChunks) {
+			same = true
+		} else if cached.Hashes[d.ID] == h && len(cached.TitleVectors[d.ID]) > 0 && len(cached.ChunkVectors[d.ID]) == len(bodyChunks) {
+			oldTitle[d.ID] = append([]float64(nil), cached.TitleVectors[d.ID]...)
+			oldChunks[d.ID] = cloneChunkVectors(cached.ChunkVectors[d.ID])
 			same = true
 		}
 		if !same {
 			changed = append(changed, *d)
 		}
 	}
-	newVectors := map[string][]float64{}
+
+	newEmbedded := map[string]embeddedDoc{}
 	if s.rag && len(changed) > 0 {
 		if s.embedder == nil {
 			return fmt.Errorf("RAG is enabled but no embedding provider is configured")
 		}
-		texts := make([]string, len(changed))
-		for i, d := range changed {
-			texts[i] = d.Title + "\n" + d.Text + "\n" + strings.Join(d.Keywords, " ")
-		}
-		vv, err := s.embedder.Embed(ctx, texts)
+		var err error
+		newEmbedded, err = s.embedDocuments(ctx, changed)
 		if err != nil {
 			return err
 		}
-		if len(vv) != len(changed) {
-			return fmt.Errorf("embedding provider returned %d vectors for %d documents", len(vv), len(changed))
-		}
-		for i, d := range changed {
-			if len(vv[i]) == 0 {
-				return fmt.Errorf("embedding provider returned empty vector for %s", d.ID)
-			}
-			newVectors[d.ID] = vv[i]
-		}
 	}

 	s.mu.Lock()
-	// Reject collisions with local/static documents.
 	for _, d := range docs {
 		if src := s.external[d.ID]; src == "" {
 			if _, exists := oldDocs[d.ID]; exists {
@@ -412,16 +450,21 @@ func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs [
 	for id, src := range s.external {
 		if src == source {
 			delete(s.external, id)
-			delete(s.vectors, id)
+			delete(s.titleVectors, id)
+			delete(s.chunkVectors, id)
+			delete(s.chunks, id)
 		}
 	}
 	for _, d := range docs {
 		rebuilt = append(rebuilt, d)
 		s.external[d.ID] = source
-		if v := newVectors[d.ID]; len(v) > 0 {
-			s.vectors[d.ID] = v
-		} else if v := oldVectors[d.ID]; len(v) > 0 {
-			s.vectors[d.ID] = v
+		s.chunks[d.ID] = chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
+		if e, ok := newEmbedded[d.ID]; ok {
+			s.titleVectors[d.ID] = e.title
+			s.chunkVectors[d.ID] = e.chunks
+		} else {
+			s.titleVectors[d.ID] = oldTitle[d.ID]
+			s.chunkVectors[d.ID] = oldChunks[d.ID]
 		}
 	}
 	s.docs = rebuilt
@@ -434,12 +477,14 @@ func (s *Store) persistVectorCache() error {
 		return nil
 	}
 	s.mu.RLock()
-	cf := cacheFile{Hashes: map[string]string{}, Vectors: map[string][]float64{}}
+	cf := cacheFile{Version: 2, Hashes: map[string]string{}, TitleVectors: map[string][]float64{}, ChunkVectors: map[string][][]float64{}}
 	for _, d := range s.docs {
-		if v := s.vectors[d.ID]; len(v) > 0 {
-			cf.Hashes[d.ID] = hashDoc(d)
-			cf.Vectors[d.ID] = append([]float64(nil), v...)
+		if len(s.titleVectors[d.ID]) == 0 {
+			continue
 		}
+		cf.Hashes[d.ID] = hashDoc(d)
+		cf.TitleVectors[d.ID] = append([]float64(nil), s.titleVectors[d.ID]...)
+		cf.ChunkVectors[d.ID] = cloneChunkVectors(s.chunkVectors[d.ID])
 	}
 	s.mu.RUnlock()
 	b, err := json.MarshalIndent(cf, "", "  ")
@@ -452,6 +497,7 @@ func (s *Store) persistVectorCache() error {
 	}
 	return os.Rename(tmp, s.cachePath)
 }
+
 func (s *Store) ManagedDir() string {
 	if s == nil {
 		return ""
@@ -470,86 +516,377 @@ func safeID(v string) bool {
 	return !strings.Contains(v, "..")
 }

-func (s *Store) Search(ctx context.Context, text string, topK int) ([]model.KnowledgeHit, error) {
+// Search calculates a transparent hybrid relevance score. Embedding similarity
+// is only one component; titles, explicit keywords and category/learning hints
+// are scored separately. Missing metadata does not lower a document's score:
+// the weights of available components are normalized dynamically.
+func (s *Store) Search(ctx context.Context, text string, topK int, categorySets ...[]model.Category) ([]model.KnowledgeHit, error) {
 	if s == nil {
 		return nil, fmt.Errorf("knowledge store is not initialized")
 	}
 	s.mu.RLock()
 	docs := append([]model.KnowledgeDoc(nil), s.docs...)
-	vecs := make(map[string][]float64, len(s.vectors))
-	for k, v := range s.vectors {
-		vecs[k] = v
-	}
+	titleVecs := cloneVectorMap(s.titleVectors)
+	chunkVecs := cloneChunkVectorMap(s.chunkVectors)
+	chunks := cloneStringSliceMap(s.chunks)
+	scoreCfg := s.scoring
 	s.mu.RUnlock()
 	if len(docs) == 0 {
 		return nil, nil
 	}
-	scores := map[string]float64{}
-	if s.rag && s.embedder != nil && len(vecs) > 0 {
+	var cats []model.Category
+	if len(categorySets) > 0 {
+		cats = categorySets[0]
+	}
+
+	var queryVector []float64
+	if s.rag && s.embedder != nil {
 		q, err := s.embedder.Embed(ctx, []string{text})
 		if err != nil {
 			return nil, err
 		}
 		if len(q) > 0 {
-			for _, d := range docs {
-				scores[d.ID] = cosine(q[0], vecs[d.ID])
-			}
-		}
-	} else {
-		for _, d := range docs {
-			scores[d.ID] = lexical(text, d)
+			queryVector = q[0]
 		}
 	}
+
 	hits := make([]model.KnowledgeHit, 0, len(docs))
 	for _, d := range docs {
-		hits = append(hits, model.KnowledgeHit{Doc: d, Score: scores[d.ID]})
+		semantic, bestChunk := 0.0, ""
+		semanticAvailable := false
+		if len(queryVector) > 0 && len(chunkVecs[d.ID]) > 0 {
+			semanticAvailable = true
+			for i, v := range chunkVecs[d.ID] {
+				score := clamp01(cosine(queryVector, v))
+				if score > semantic || bestChunk == "" {
+					semantic = score
+					if i < len(chunks[d.ID]) {
+						bestChunk = chunks[d.ID][i]
+					}
+				}
+			}
+		} else if strings.TrimSpace(d.Text) != "" {
+			semanticAvailable = true
+			semantic = tokenF1(text, d.Text)
+			bestChunk = d.Text
+		}
+
+		title := 0.0
+		titleAvailable := strings.TrimSpace(d.Title) != ""
+		if titleAvailable {
+			title = titleSimilarity(text, d.Title)
+			if len(queryVector) > 0 && len(titleVecs[d.ID]) > 0 {
+				title = math.Max(title, clamp01(cosine(queryVector, titleVecs[d.ID])))
+			}
+		}
+		keyword, keywordAvailable := keywordSimilarity(text, d.Keywords)
+		category, categoryAvailable := categorySimilarity(text, d.Categories, cats)
+		total := weightedScore(scoreCfg,
+			scorePart{semantic, scoreCfg.SemanticWeight, semanticAvailable},
+			scorePart{title, scoreCfg.TitleWeight, titleAvailable},
+			scorePart{keyword, scoreCfg.KeywordWeight, keywordAvailable},
+			scorePart{category, scoreCfg.CategoryWeight, categoryAvailable},
+		)
+		hits = append(hits, model.KnowledgeHit{Doc: d, Score: total, SemanticScore: semantic, TitleScore: title, KeywordScore: keyword, CategoryScore: category, BestChunkExcerpt: excerpt(bestChunk, 280)})
 	}
-	sort.Slice(hits, func(i, j int) bool { return hits[i].Score > hits[j].Score })
+	sort.SliceStable(hits, func(i, j int) bool {
+		if hits[i].Score == hits[j].Score {
+			return hits[i].TitleScore > hits[j].TitleScore
+		}
+		return hits[i].Score > hits[j].Score
+	})
 	if topK > 0 && len(hits) > topK {
 		hits = hits[:topK]
 	}
 	return hits, nil
 }
+
 func (s *Store) index(ctx context.Context) error {
 	_ = os.MkdirAll(filepath.Dir(s.cachePath), 0o750)
-	cf := cacheFile{Hashes: map[string]string{}, Vectors: map[string][]float64{}}
-	if b, err := os.ReadFile(s.cachePath); err == nil {
-		_ = json.Unmarshal(b, &cf)
-	}
+	cf := loadCache(s.cachePath)
 	var need []model.KnowledgeDoc
 	for _, d := range s.docs {
+		bodyChunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
+		s.chunks[d.ID] = bodyChunks
 		h := hashDoc(d)
-		if cf.Hashes[d.ID] == h && len(cf.Vectors[d.ID]) > 0 {
-			s.vectors[d.ID] = cf.Vectors[d.ID]
+		if cf.Hashes[d.ID] == h && len(cf.TitleVectors[d.ID]) > 0 && len(cf.ChunkVectors[d.ID]) == len(bodyChunks) {
+			s.titleVectors[d.ID] = append([]float64(nil), cf.TitleVectors[d.ID]...)
+			s.chunkVectors[d.ID] = cloneChunkVectors(cf.ChunkVectors[d.ID])
 		} else {
 			need = append(need, d)
 		}
 	}
 	if len(need) > 0 {
-		texts := make([]string, len(need))
-		for i, d := range need {
-			texts[i] = d.Title + "\n" + d.Text + "\n" + strings.Join(d.Keywords, " ")
-		}
-		vv, err := s.embedder.Embed(ctx, texts)
+		embedded, err := s.embedDocuments(ctx, need)
 		if err != nil {
 			return err
 		}
-		for i, d := range need {
-			s.vectors[d.ID] = vv[i]
-			cf.Hashes[d.ID] = hashDoc(d)
-			cf.Vectors[d.ID] = vv[i]
-		}
-		b, _ := json.MarshalIndent(cf, "", "  ")
-		tmp := s.cachePath + ".tmp"
-		if err := os.WriteFile(tmp, b, 0o640); err != nil {
-			return err
-		}
-		if err := os.Rename(tmp, s.cachePath); err != nil {
-			return err
+		for _, d := range need {
+			s.titleVectors[d.ID] = embedded[d.ID].title
+			s.chunkVectors[d.ID] = embedded[d.ID].chunks
 		}
 	}
-	return nil
+	return s.persistVectorCache()
 }
+
+type embeddedDoc struct {
+	title  []float64
+	chunks [][]float64
+}
+
+func (s *Store) embedDocuments(ctx context.Context, docs []model.KnowledgeDoc) (map[string]embeddedDoc, error) {
+	out := make(map[string]embeddedDoc, len(docs))
+	type ref struct {
+		id    string
+		title bool
+		chunk int
+	}
+	var texts []string
+	var refs []ref
+	for _, d := range docs {
+		texts = append(texts, d.Title)
+		refs = append(refs, ref{id: d.ID, title: true})
+		parts := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
+		for i, part := range parts {
+			texts = append(texts, part)
+			refs = append(refs, ref{id: d.ID, chunk: i})
+		}
+	}
+	vectors, err := s.embedTexts(ctx, texts, 64)
+	if err != nil {
+		return nil, err
+	}
+	if len(vectors) != len(refs) {
+		return nil, fmt.Errorf("embedding provider returned %d vectors for %d inputs", len(vectors), len(refs))
+	}
+	for i, r := range refs {
+		if len(vectors[i]) == 0 {
+			return nil, fmt.Errorf("embedding provider returned empty vector for %s", r.id)
+		}
+		e := out[r.id]
+		if r.title {
+			e.title = vectors[i]
+		} else {
+			for len(e.chunks) <= r.chunk {
+				e.chunks = append(e.chunks, nil)
+			}
+			e.chunks[r.chunk] = vectors[i]
+		}
+		out[r.id] = e
+	}
+	return out, nil
+}
+
+func (s *Store) embedTexts(ctx context.Context, texts []string, batch int) ([][]float64, error) {
+	if len(texts) == 0 {
+		return nil, nil
+	}
+	if batch <= 0 {
+		batch = 64
+	}
+	out := make([][]float64, 0, len(texts))
+	for start := 0; start < len(texts); start += batch {
+		end := start + batch
+		if end > len(texts) {
+			end = len(texts)
+		}
+		vv, err := s.embedder.Embed(ctx, texts[start:end])
+		if err != nil {
+			return nil, err
+		}
+		if len(vv) != end-start {
+			return nil, fmt.Errorf("embedding provider returned %d vectors for %d inputs", len(vv), end-start)
+		}
+		out = append(out, vv...)
+	}
+	return out, nil
+}
+
+func loadCache(path string) cacheFile {
+	cf := cacheFile{Version: 2, Hashes: map[string]string{}, TitleVectors: map[string][]float64{}, ChunkVectors: map[string][][]float64{}}
+	if b, err := os.ReadFile(path); err == nil {
+		_ = json.Unmarshal(b, &cf)
+	}
+	if cf.Hashes == nil {
+		cf.Hashes = map[string]string{}
+	}
+	if cf.TitleVectors == nil {
+		cf.TitleVectors = map[string][]float64{}
+	}
+	if cf.ChunkVectors == nil {
+		cf.ChunkVectors = map[string][][]float64{}
+	}
+	return cf
+}
+
+func chunkText(text string, words, overlap, maxChunks int) []string {
+	parts := strings.Fields(strings.TrimSpace(text))
+	if len(parts) == 0 {
+		return nil
+	}
+	if words <= 0 {
+		words = 160
+	}
+	if overlap < 0 || overlap >= words {
+		overlap = 0
+	}
+	if maxChunks <= 0 {
+		maxChunks = 24
+	}
+	step := words - overlap
+	out := make([]string, 0, minInt(maxChunks, (len(parts)+step-1)/step))
+	for start := 0; start < len(parts) && len(out) < maxChunks; start += step {
+		end := start + words
+		if end > len(parts) {
+			end = len(parts)
+		}
+		out = append(out, strings.Join(parts[start:end], " "))
+		if end == len(parts) {
+			break
+		}
+	}
+	return out
+}
+
+type scorePart struct {
+	value, weight float64
+	available     bool
+}
+
+func weightedScore(_ ScoringConfig, parts ...scorePart) float64 {
+	var sum, weights float64
+	for _, p := range parts {
+		if !p.available || p.weight <= 0 {
+			continue
+		}
+		sum += clamp01(p.value) * p.weight
+		weights += p.weight
+	}
+	if weights == 0 {
+		return 0
+	}
+	return clamp01(sum / weights)
+}
+
+func titleSimilarity(query, title string) float64 {
+	best := tokenF1(query, title)
+	q := strings.ToLower(strings.Join(strings.Fields(query), " "))
+	t := strings.ToLower(strings.Join(strings.Fields(title), " "))
+	if t != "" && strings.Contains(q, t) {
+		return 1
+	}
+	return best
+}
+
+func keywordSimilarity(query string, keywords []string) (float64, bool) {
+	if len(keywords) == 0 {
+		return 0, false
+	}
+	best := tokenF1(query, strings.Join(keywords, " "))
+	q := strings.ToLower(query)
+	for _, kw := range keywords {
+		kw = strings.ToLower(strings.TrimSpace(kw))
+		if kw != "" && strings.Contains(q, kw) {
+			best = math.Max(best, 1)
+		}
+	}
+	return clamp01(best), true
+}
+
+func categorySimilarity(query string, ids []int64, categories []model.Category) (float64, bool) {
+	if len(ids) == 0 || len(categories) == 0 {
+		return 0, false
+	}
+	wanted := make(map[int64]struct{}, len(ids))
+	for _, id := range ids {
+		wanted[id] = struct{}{}
+	}
+	best, found := 0.0, false
+	for _, c := range categories {
+		if _, ok := wanted[c.ID]; !ok {
+			continue
+		}
+		found = true
+		profileParts := []string{c.Name, c.CompleteName}
+		profileParts = append(profileParts, c.Hints...)
+		profileParts = append(profileParts, c.Examples...)
+		profile := strings.Join(profileParts, " ")
+		best = math.Max(best, tokenF1(query, profile))
+	}
+	return clamp01(best), found
+}
+
+func tokenF1(a, b string) float64 {
+	aTok, bTok := tokens(a), tokens(b)
+	if len(aTok) == 0 || len(bTok) == 0 {
+		return 0
+	}
+	common := 0
+	for t := range aTok {
+		if _, ok := bTok[t]; ok {
+			common++
+		}
+	}
+	if common == 0 {
+		return 0
+	}
+	precision := float64(common) / float64(len(aTok))
+	recall := float64(common) / float64(len(bTok))
+	return 2 * precision * recall / (precision + recall)
+}
+
+func excerpt(s string, max int) string {
+	s = strings.Join(strings.Fields(s), " ")
+	if len([]rune(s)) <= max {
+		return s
+	}
+	r := []rune(s)
+	return string(r[:max]) + "…"
+}
+func clamp01(v float64) float64 {
+	if v < 0 {
+		return 0
+	}
+	if v > 1 {
+		return 1
+	}
+	return v
+}
+func cloneVectorMap(in map[string][]float64) map[string][]float64 {
+	out := make(map[string][]float64, len(in))
+	for k, v := range in {
+		out[k] = append([]float64(nil), v...)
+	}
+	return out
+}
+func cloneChunkVectorMap(in map[string][][]float64) map[string][][]float64 {
+	out := make(map[string][][]float64, len(in))
+	for k, v := range in {
+		out[k] = cloneChunkVectors(v)
+	}
+	return out
+}
+func cloneChunkVectors(in [][]float64) [][]float64 {
+	out := make([][]float64, len(in))
+	for i, v := range in {
+		out[i] = append([]float64(nil), v...)
+	}
+	return out
+}
+func cloneStringSliceMap(in map[string][]string) map[string][]string {
+	out := make(map[string][]string, len(in))
+	for k, v := range in {
+		out[k] = append([]string(nil), v...)
+	}
+	return out
+}
+func minInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
 func hashDoc(d model.KnowledgeDoc) string {
 	b, _ := json.Marshal(d)
 	h := sha256.Sum256(b)
@@ -2,6 +2,7 @@ package knowledge

 import (
 	"context"
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"strings"
@@ -132,3 +133,71 @@ func TestExternalKnowledgeIsReadOnly(t *testing.T) {
 		t.Fatal("expected external document to be read-only")
 	}
 }
+
+type semanticTestEmbedder struct{}
+
+func (semanticTestEmbedder) Embed(_ context.Context, texts []string) ([][]float64, error) {
+	out := make([][]float64, len(texts))
+	for i, text := range texts {
+		s := strings.ToLower(text)
+		v := []float64{0, 0, 0, 0}
+		if strings.Contains(s, "benutzerkonto") || strings.Contains(s, "konto gesperrt") || strings.Contains(s, "gesperrt") {
+			v[0] = 1
+		}
+		if strings.Contains(s, "anmeld") || strings.Contains(s, "login") || strings.Contains(s, "authent") {
+			v[1] = 1
+		}
+		if strings.Contains(s, "drucker") {
+			v[2] = 1
+		}
+		if strings.Contains(s, "allgemein") || strings.Contains(s, "hinweis") {
+			v[3] = 1
+		}
+		if v[0]+v[1]+v[2]+v[3] == 0 {
+			v[3] = .1
+		}
+		out[i] = v
+	}
+	return out, nil
+}
+
+func TestHybridScoringUsesChunksTitleKeywordsAndCategoryHints(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	body := strings.Repeat("Allgemeine technische Hinweise ohne Bezug zum Benutzer. ", 80) +
+		" Wenn ein Benutzerkonto gesperrt ist und die Anmeldung nicht möglich ist, muss die Kontosperre geprüft werden. " +
+		strings.Repeat("Weitere allgemeine Hinweise. ", 80)
+	doc := model.KnowledgeDoc{ID: "KB-AD-1", Title: "Benutzerkonto gesperrt", Text: body, Answer: "x", Source: "internal-kb", Language: "de-DE", CommunicationStyle: "formal", Categories: []int64{2}, Keywords: []string{"Konto gesperrt", "Anmeldung", "Login"}}
+	b, _ := json.Marshal(doc)
+	if err := os.WriteFile(filepath.Join(dir, "ad.json"), b, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	s, err := Load(context.Background(), dir, data, semanticTestEmbedder{}, true, []string{"internal-kb"}, ScoringConfig{SemanticWeight: .5, TitleWeight: .25, KeywordWeight: .15, CategoryWeight: .10, ChunkWords: 40, ChunkOverlap: 10, MaxChunksPerDoc: 24})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cats := []model.Category{{ID: 2, Name: "Active Directory", Hints: []string{"Benutzerkonto gesperrt", "Anmeldung Login Authentifizierung"}, Examples: []string{"Mein Benutzerkonto ist gesperrt und ich kann mich nicht anmelden"}}}
+	hits, err := s.Search(context.Background(), "Benutzerkonto gesperrt, Anmeldung nicht möglich", 1, cats)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(hits) != 1 {
+		t.Fatalf("hits=%d", len(hits))
+	}
+	h := hits[0]
+	if h.Score < .75 {
+		t.Fatalf("hybrid score too low: %+v", h)
+	}
+	if h.SemanticScore < .8 {
+		t.Fatalf("expected strong best-chunk semantic score: %+v", h)
+	}
+	if h.TitleScore < .7 {
+		t.Fatalf("expected strong title score: %+v", h)
+	}
+	if h.KeywordScore <= 0 || h.CategoryScore <= 0 {
+		t.Fatalf("expected keyword/category contributions: %+v", h)
+	}
+	if !strings.Contains(strings.ToLower(h.BestChunkExcerpt), "benutzerkonto") {
+		t.Fatalf("wrong best chunk: %q", h.BestChunkExcerpt)
+	}
+}