glpi-ai-agent/internal/knowledge/store.go

package knowledge

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"unicode"

	"github.com/example/glpi-ai-agent/internal/model"
)

type Embedder interface {
	Embed(context.Context, []string) ([][]float64, error)
}
type ScoringConfig struct {
	SemanticWeight    float64
	TitleWeight       float64
	LexicalWeight     float64
	KeywordWeight     float64
	CategoryWeight    float64
	EmbeddingProfile  string
	EmbeddingIdentity string
	ChunkWords        int
	ChunkOverlap      int
	MaxChunksPerDoc   int
	MaxQueryChunks    int
}

type Store struct {
	mu             sync.RWMutex
	dir            string
	managedDir     string
	docs           []model.KnowledgeDoc
	files          map[string]string
	managed        map[string]bool
	external       map[string]string
	staticDocs     map[string]model.KnowledgeDoc
	titleVectors   map[string][]float64
	chunkVectors   map[string][][]float64
	chunks         map[string][]string
	embedder       Embedder
	rag            bool
	cachePath      string
	allowedSources map[string]struct{}
	scoring        ScoringConfig
}
type cacheFile struct {
	Version      int                    `json:"version,omitempty"`
	Hashes       map[string]string      `json:"hashes"`
	TitleVectors map[string][]float64   `json:"title_vectors,omitempty"`
	ChunkVectors map[string][][]float64 `json:"chunk_vectors,omitempty"`
}

func DefaultScoringConfig() ScoringConfig {
	return ScoringConfig{SemanticWeight: .45, TitleWeight: .20, LexicalWeight: .20, KeywordWeight: .075, CategoryWeight: .075, EmbeddingProfile: "plain", ChunkWords: 160, ChunkOverlap: 30, MaxChunksPerDoc: 24, MaxQueryChunks: 64}
}

// ResolveEmbeddingProfile selects prompt formatting for the configured embedding model.
// EmbeddingGemma benefits from distinct retrieval-query and retrieval-document prompts.
func ResolveEmbeddingProfile(profile, model string) string {
	p := strings.ToLower(strings.TrimSpace(profile))
	if p == "" || p == "auto" {
		if strings.Contains(strings.ToLower(model), "embeddinggemma") {
			return "embeddinggemma"
		}
		return "plain"
	}
	return p
}

func normalizeScoring(c ScoringConfig) ScoringConfig {
	d := DefaultScoringConfig()
	if c.SemanticWeight < 0 || c.TitleWeight < 0 || c.LexicalWeight < 0 || c.KeywordWeight < 0 || c.CategoryWeight < 0 || c.SemanticWeight+c.TitleWeight+c.LexicalWeight+c.KeywordWeight+c.CategoryWeight <= 0 {
		c.SemanticWeight, c.TitleWeight, c.LexicalWeight, c.KeywordWeight, c.CategoryWeight = d.SemanticWeight, d.TitleWeight, d.LexicalWeight, d.KeywordWeight, d.CategoryWeight
	}
	if c.EmbeddingProfile == "" {
		c.EmbeddingProfile = d.EmbeddingProfile
	}
	if c.ChunkWords <= 0 {
		c.ChunkWords = d.ChunkWords
	}
	if c.ChunkOverlap < 0 || c.ChunkOverlap >= c.ChunkWords {
		c.ChunkOverlap = d.ChunkOverlap
	}
	if c.MaxChunksPerDoc <= 0 {
		c.MaxChunksPerDoc = d.MaxChunksPerDoc
	}
	if c.MaxQueryChunks <= 0 {
		c.MaxQueryChunks = d.MaxQueryChunks
	}
	return c
}

func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool, allowedSources []string, scoring ...ScoringConfig) (*Store, error) {
	managedDir := filepath.Join(dataDir, "knowledge-managed")
	if err := os.MkdirAll(managedDir, 0o750); err != nil {
		return nil, fmt.Errorf("create managed knowledge directory: %w", err)
	}
	scoreCfg := DefaultScoringConfig()
	if len(scoring) > 0 {
		scoreCfg = normalizeScoring(scoring[0])
	}
	s := &Store{dir: dir, managedDir: managedDir, titleVectors: map[string][]float64{}, chunkVectors: map[string][][]float64{}, chunks: map[string][]string{}, files: map[string]string{}, managed: map[string]bool{}, external: map[string]string{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}, scoring: scoreCfg}
	for _, source := range allowedSources {
		s.allowedSources[strings.ToLower(strings.TrimSpace(source))] = struct{}{}
	}
	static, staticFiles, err := readDocs(dir, s.allowedSources)
	if err != nil {
		return nil, err
	}
	for i, d := range static {
		s.staticDocs[d.ID] = d
		s.files[d.ID] = staticFiles[i]
	}
	managed, managedFiles, err := readDocs(managedDir, s.allowedSources)
	if err != nil {
		return nil, err
	}
	merged := map[string]model.KnowledgeDoc{}
	order := []string{}
	for _, d := range static {
		if _, ok := merged[d.ID]; !ok {
			order = append(order, d.ID)
		}
		merged[d.ID] = d
	}
	for i, d := range managed {
		if _, ok := merged[d.ID]; !ok {
			order = append(order, d.ID)
		}
		merged[d.ID] = d
		s.files[d.ID] = managedFiles[i]
		s.managed[d.ID] = true
	}
	for _, id := range order {
		s.docs = append(s.docs, merged[id])
	}
	if rag && len(s.docs) > 0 {
		if s.embedder == nil {
			return nil, fmt.Errorf("RAG is enabled but no embedding provider is configured")
		}
		if err := s.index(ctx); err != nil {
			return s, err
		}
	}
	return s, nil
}

func readDocs(dir string, allowed map[string]struct{}) ([]model.KnowledgeDoc, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, fmt.Errorf("read knowledge directory %q: %w", dir, err)
	}
	var docs []model.KnowledgeDoc
	var files []string
	for _, e := range entries {
		if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
			continue
		}
		path := filepath.Join(dir, e.Name())
		b, err := os.ReadFile(path)
		if err != nil {
			return nil, nil, err
		}
		var d model.KnowledgeDoc
		if err := json.Unmarshal(b, &d); err != nil {
			return nil, nil, fmt.Errorf("%s: %w", e.Name(), err)
		}
		if d.ID == "" || d.Title == "" {
			return nil, nil, fmt.Errorf("%s: id/title required", e.Name())
		}
		if !safeID(d.ID) {
			return nil, nil, fmt.Errorf("%s: invalid id %q", e.Name(), d.ID)
		}
		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
		if d.Source == "" {
			return nil, nil, fmt.Errorf("%s: source required", e.Name())
		}
		if _, ok := allowed[d.Source]; !ok {
			continue
		}
		d.Language = strings.TrimSpace(d.Language)
		d.CommunicationStyle = strings.ToLower(strings.TrimSpace(d.CommunicationStyle))
		docs = append(docs, d)
		files = append(files, path)
	}
	return docs, files, nil
}

func (s *Store) Count() int {
	if s == nil {
		return 0
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	return len(s.docs)
}
func (s *Store) ByID(id string) (model.KnowledgeDoc, bool) {
	if s == nil {
		return model.KnowledgeDoc{}, false
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, d := range s.docs {
		if d.ID == id {
			return d, true
		}
	}
	return model.KnowledgeDoc{}, false
}
func (s *Store) List() []model.KnowledgeDoc {
	if s == nil {
		return nil
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	out := append([]model.KnowledgeDoc(nil), s.docs...)
	sort.SliceStable(out, func(i, j int) bool { return strings.ToLower(out[i].Title) < strings.ToLower(out[j].Title) })
	return out
}

func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
	if s == nil {
		return fmt.Errorf("knowledge store is not initialized")
	}
	d.ID = strings.TrimSpace(d.ID)
	d.Title = strings.TrimSpace(d.Title)
	d.Text = strings.TrimSpace(d.Text)
	d.Answer = strings.TrimSpace(d.Answer)
	d.Source = strings.ToLower(strings.TrimSpace(d.Source))
	d.Language = strings.TrimSpace(d.Language)
	d.CommunicationStyle = strings.ToLower(strings.TrimSpace(d.CommunicationStyle))
	if d.ID == "" || d.Title == "" {
		return fmt.Errorf("id/title required")
	}
	if !safeID(d.ID) {
		return fmt.Errorf("knowledge id may contain only letters, digits, dot, dash and underscore")
	}
	if d.Source == "" {
		return fmt.Errorf("source required")
	}
	if _, ok := s.allowedSources[d.Source]; !ok {
		return fmt.Errorf("source %q is not allowed", d.Source)
	}
	if d.Language == "" || d.CommunicationStyle == "" {
		return fmt.Errorf("language and communication_style required")
	}
	if d.MinScore < 0 || d.MinScore > 1 {
		return fmt.Errorf("min_score must be between 0 and 1")
	}
	s.mu.RLock()
	_, exists := s.files[d.ID]
	isManaged := s.managed[d.ID]
	externalSource := s.external[d.ID]
	s.mu.RUnlock()
	if externalSource != "" {
		return fmt.Errorf("externally synchronized knowledge entry %q from %q is read-only", d.ID, externalSource)
	}
	if exists && !isManaged {
		return fmt.Errorf("static knowledge entry %q is read-only; use a new id for a managed entry", d.ID)
	}

	var titleVector []float64
	var chunkVectors [][]float64
	chunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
	if s.rag {
		if s.embedder == nil {
			return fmt.Errorf("RAG is enabled but no embedding provider is configured")
		}
		embedded, err := s.embedDocuments(ctx, []model.KnowledgeDoc{d})
		if err != nil {
			return err
		}
		titleVector = embedded[d.ID].title
		chunkVectors = embedded[d.ID].chunks
	}

	path := filepath.Join(s.managedDir, d.ID+".json")
	b, err := json.MarshalIndent(d, "", "  ")
	if err != nil {
		return err
	}
	tmp := path + ".tmp"
	if err := os.WriteFile(tmp, b, 0o640); err != nil {
		return err
	}
	if err := os.Rename(tmp, path); err != nil {
		_ = os.Remove(tmp)
		return err
	}
	s.mu.Lock()
	replaced := false
	for i := range s.docs {
		if s.docs[i].ID == d.ID {
			s.docs[i] = d
			replaced = true
			break
		}
	}
	if !replaced {
		s.docs = append(s.docs, d)
	}
	s.files[d.ID] = path
	s.managed[d.ID] = true
	s.chunks[d.ID] = chunks
	if s.rag {
		s.titleVectors[d.ID] = titleVector
		s.chunkVectors[d.ID] = chunkVectors
	}
	s.mu.Unlock()
	return s.persistVectorCache()
}

func (s *Store) Delete(id string) error {
	if s == nil {
		return fmt.Errorf("knowledge store is not initialized")
	}
	id = strings.TrimSpace(id)
	if !safeID(id) {
		return fmt.Errorf("invalid knowledge id")
	}
	s.mu.RLock()
	path := s.files[id]
	isManaged := s.managed[id]
	s.mu.RUnlock()
	if path == "" {
		return os.ErrNotExist
	}
	if !isManaged {
		return fmt.Errorf("static knowledge entry %q is read-only", id)
	}
	if err := os.Remove(path); err != nil {
		return err
	}
	s.mu.Lock()
	out := s.docs[:0]
	for _, d := range s.docs {
		if d.ID != id {
			out = append(out, d)
		}
	}
	s.docs = append([]model.KnowledgeDoc(nil), out...)
	delete(s.files, id)
	delete(s.managed, id)
	delete(s.titleVectors, id)
	delete(s.chunkVectors, id)
	delete(s.chunks, id)
	s.mu.Unlock()
	return s.persistVectorCache()
}

func (s *Store) IsManaged(id string) bool {
	if s == nil {
		return false
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.managed[id]
}
func (s *Store) Origin(id string) string {
	if s == nil {
		return ""
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	if s.managed[id] {
		return "managed"
	}
	if src := s.external[id]; src != "" {
		return src
	}
	if _, ok := s.staticDocs[id]; ok {
		return "static"
	}
	return ""
}

// ReplaceExternalSource atomically replaces all read-only documents imported
// from one connector source. Embeddings are reused when the normalized article
// did not change. Long article bodies are indexed as overlapping chunks.
func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs []model.KnowledgeDoc) error {
	if s == nil {
		return fmt.Errorf("knowledge store is not initialized")
	}
	source = strings.ToLower(strings.TrimSpace(source))
	if _, ok := s.allowedSources[source]; !ok {
		return fmt.Errorf("source %q is not allowed", source)
	}

	s.mu.RLock()
	oldDocs := make(map[string]model.KnowledgeDoc, len(s.docs))
	oldTitle := cloneVectorMap(s.titleVectors)
	oldChunks := cloneChunkVectorMap(s.chunkVectors)
	for _, d := range s.docs {
		oldDocs[d.ID] = d
	}
	s.mu.RUnlock()
	cached := loadCache(s.cachePath)

	changed := make([]model.KnowledgeDoc, 0)
	seen := map[string]struct{}{}
	for i := range docs {
		d := &docs[i]
		d.ID = strings.TrimSpace(d.ID)
		d.Title = strings.TrimSpace(d.Title)
		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
		if d.Source == "" {
			d.Source = source
		}
		if d.Source != source {
			return fmt.Errorf("external document %q has source %q, expected %q", d.ID, d.Source, source)
		}
		if d.ID == "" || d.Title == "" || !safeID(d.ID) {
			return fmt.Errorf("invalid external knowledge document id/title")
		}
		if _, dup := seen[d.ID]; dup {
			return fmt.Errorf("duplicate external knowledge id %q", d.ID)
		}
		seen[d.ID] = struct{}{}
		h := hashDoc(*d, s.scoring)
		bodyChunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
		same := false
		if old, ok := oldDocs[d.ID]; ok && hashDoc(old, s.scoring) == h && len(oldTitle[d.ID]) > 0 && len(oldChunks[d.ID]) == len(bodyChunks) {
			same = true
		} else if cached.Hashes[d.ID] == h && len(cached.TitleVectors[d.ID]) > 0 && len(cached.ChunkVectors[d.ID]) == len(bodyChunks) {
			oldTitle[d.ID] = append([]float64(nil), cached.TitleVectors[d.ID]...)
			oldChunks[d.ID] = cloneChunkVectors(cached.ChunkVectors[d.ID])
			same = true
		}
		if !same {
			changed = append(changed, *d)
		}
	}

	newEmbedded := map[string]embeddedDoc{}
	if s.rag && len(changed) > 0 {
		if s.embedder == nil {
			return fmt.Errorf("RAG is enabled but no embedding provider is configured")
		}
		var err error
		newEmbedded, err = s.embedDocuments(ctx, changed)
		if err != nil {
			return err
		}
	}

	s.mu.Lock()
	for _, d := range docs {
		if src := s.external[d.ID]; src == "" {
			if _, exists := oldDocs[d.ID]; exists {
				s.mu.Unlock()
				return fmt.Errorf("external knowledge id %q collides with local knowledge", d.ID)
			}
		} else if src != source {
			s.mu.Unlock()
			return fmt.Errorf("external knowledge id %q belongs to source %q", d.ID, src)
		}
	}
	rebuilt := make([]model.KnowledgeDoc, 0, len(s.docs)+len(docs))
	for _, d := range s.docs {
		if s.external[d.ID] != source {
			rebuilt = append(rebuilt, d)
		}
	}
	for id, src := range s.external {
		if src == source {
			delete(s.external, id)
			delete(s.titleVectors, id)
			delete(s.chunkVectors, id)
			delete(s.chunks, id)
		}
	}
	for _, d := range docs {
		rebuilt = append(rebuilt, d)
		s.external[d.ID] = source
		s.chunks[d.ID] = chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
		if e, ok := newEmbedded[d.ID]; ok {
			s.titleVectors[d.ID] = e.title
			s.chunkVectors[d.ID] = e.chunks
		} else {
			s.titleVectors[d.ID] = oldTitle[d.ID]
			s.chunkVectors[d.ID] = oldChunks[d.ID]
		}
	}
	s.docs = rebuilt
	s.mu.Unlock()
	return s.persistVectorCache()
}

func (s *Store) persistVectorCache() error {
	if s == nil || !s.rag {
		return nil
	}
	s.mu.RLock()
	cf := cacheFile{Version: 3, Hashes: map[string]string{}, TitleVectors: map[string][]float64{}, ChunkVectors: map[string][][]float64{}}
	for _, d := range s.docs {
		if len(s.titleVectors[d.ID]) == 0 {
			continue
		}
		cf.Hashes[d.ID] = hashDoc(d, s.scoring)
		cf.TitleVectors[d.ID] = append([]float64(nil), s.titleVectors[d.ID]...)
		cf.ChunkVectors[d.ID] = cloneChunkVectors(s.chunkVectors[d.ID])
	}
	s.mu.RUnlock()
	b, err := json.MarshalIndent(cf, "", "  ")
	if err != nil {
		return err
	}
	tmp := s.cachePath + ".tmp"
	if err := os.WriteFile(tmp, b, 0o640); err != nil {
		return err
	}
	return os.Rename(tmp, s.cachePath)
}

func (s *Store) ManagedDir() string {
	if s == nil {
		return ""
	}
	return s.managedDir
}
func safeID(v string) bool {
	if v == "" {
		return false
	}
	for _, r := range v {
		if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.') {
			return false
		}
	}
	return !strings.Contains(v, "..")
}

// Search calculates a transparent hybrid relevance score. Embedding similarity
// is only one component; titles, explicit keywords and category/learning hints
// are scored separately. Missing metadata does not lower a document's score:
// the weights of available components are normalized dynamically.
func (s *Store) Search(ctx context.Context, text string, topK int, categorySets ...[]model.Category) ([]model.KnowledgeHit, error) {
	if s == nil {
		return nil, fmt.Errorf("knowledge store is not initialized")
	}
	s.mu.RLock()
	docs := append([]model.KnowledgeDoc(nil), s.docs...)
	titleVecs := cloneVectorMap(s.titleVectors)
	chunkVecs := cloneChunkVectorMap(s.chunkVectors)
	chunks := cloneStringSliceMap(s.chunks)
	scoreCfg := s.scoring
	s.mu.RUnlock()
	if len(docs) == 0 {
		return nil, nil
	}
	var cats []model.Category
	if len(categorySets) > 0 {
		cats = categorySets[0]
	}

	queryTitle, queryBody := splitQueryText(text)
	queryChunks := chunkText(queryBody, scoreCfg.ChunkWords, scoreCfg.ChunkOverlap, scoreCfg.MaxQueryChunks)
	if len(queryChunks) == 0 {
		queryChunks = chunkText(text, scoreCfg.ChunkWords, scoreCfg.ChunkOverlap, scoreCfg.MaxQueryChunks)
	}
	var queryVectors [][]float64
	var queryTitleVector []float64
	if s.rag && s.embedder != nil {
		if len(queryChunks) > 0 {
			q, err := s.embedTexts(ctx, formatQueryEmbeddings(queryChunks, scoreCfg.EmbeddingProfile), 64)
			if err != nil {
				return nil, err
			}
			queryVectors = q
		}
		if strings.TrimSpace(queryTitle) != "" {
			tq, err := s.embedder.Embed(ctx, formatQueryEmbeddings([]string{queryTitle}, scoreCfg.EmbeddingProfile))
			if err != nil {
				return nil, err
			}
			if len(tq) > 0 {
				queryTitleVector = tq[0]
			}
		}
	}

	hits := make([]model.KnowledgeHit, 0, len(docs))
	for _, d := range docs {
		semantic, bestChunk, bestQueryChunk := 0.0, "", ""
		semanticAvailable := false
		if len(queryVectors) > 0 && len(chunkVecs[d.ID]) > 0 {
			semanticAvailable = true
			for qi, qv := range queryVectors {
				for di, dv := range chunkVecs[d.ID] {
					score := clamp01(cosine(qv, dv))
					if score > semantic || bestChunk == "" {
						semantic = score
						if di < len(chunks[d.ID]) {
							bestChunk = chunks[d.ID][di]
						}
						if qi < len(queryChunks) {
							bestQueryChunk = queryChunks[qi]
						}
					}
				}
			}
		} else if strings.TrimSpace(d.Text) != "" {
			semanticAvailable = true
			docChunks := chunks[d.ID]
			if len(docChunks) == 0 {
				docChunks = []string{d.Text}
			}
			for _, qc := range queryChunks {
				for _, dc := range docChunks {
					score := tokenF1(qc, dc)
					if score > semantic || bestChunk == "" {
						semantic, bestChunk, bestQueryChunk = score, dc, qc
					}
				}
			}
		}

		title := 0.0
		titleAvailable := strings.TrimSpace(d.Title) != ""
		if titleAvailable {
			titleQuery := queryTitle
			if strings.TrimSpace(titleQuery) == "" {
				titleQuery = text
			}
			title = titleSimilarity(titleQuery, d.Title)
			if len(queryTitleVector) > 0 && len(titleVecs[d.ID]) > 0 {
				title = math.Max(title, clamp01(cosine(queryTitleVector, titleVecs[d.ID])))
			}
		}
		lexicalScore := lexicalSimilarity(text, d)
		keyword, keywordAvailable := keywordSimilarity(text, d.Keywords)
		category, categoryAvailable := categorySimilarity(text, d.Categories, cats)
		// Keywords and category profiles are positive evidence signals. Metadata that
		// exists but has no lexical overlap must not drag an otherwise strong
		// semantic/title match toward zero.
		keywordAvailable = keywordAvailable && keyword > 0
		categoryAvailable = categoryAvailable && category > 0
		total := weightedScore(scoreCfg,
			scorePart{semantic, scoreCfg.SemanticWeight, semanticAvailable},
			scorePart{title, scoreCfg.TitleWeight, titleAvailable},
			scorePart{lexicalScore, scoreCfg.LexicalWeight, lexicalScore > 0},
			scorePart{keyword, scoreCfg.KeywordWeight, keywordAvailable},
			scorePart{category, scoreCfg.CategoryWeight, categoryAvailable},
		)
		hits = append(hits, model.KnowledgeHit{Doc: d, Score: total, SemanticScore: semantic, TitleScore: title, LexicalScore: lexicalScore, KeywordScore: keyword, CategoryScore: category, BestChunkExcerpt: excerpt(bestChunk, 280), BestQueryExcerpt: excerpt(bestQueryChunk, 280), QueryChunkCount: len(queryChunks), DocumentChunkCount: len(chunks[d.ID])})
	}
	sort.SliceStable(hits, func(i, j int) bool {
		if hits[i].Score == hits[j].Score {
			return hits[i].TitleScore > hits[j].TitleScore
		}
		return hits[i].Score > hits[j].Score
	})
	if topK > 0 && len(hits) > topK {
		hits = hits[:topK]
	}
	return hits, nil
}

// RerankForCategory applies a deterministic post-classification boost when a
// knowledge document is explicitly mapped to the category selected by the
// classifier. This happens after the model decision, so the dashboard and
// policy can distinguish retrieval evidence from category alignment.
func (s *Store) RerankForCategory(hits []model.KnowledgeHit, categoryID int64) []model.KnowledgeHit {
	if s == nil || len(hits) == 0 || categoryID == 0 {
		return hits
	}
	s.mu.RLock()
	cfg := s.scoring
	s.mu.RUnlock()
	out := append([]model.KnowledgeHit(nil), hits...)
	for i := range out {
		match := false
		for _, id := range out[i].Doc.Categories {
			if id == categoryID {
				match = true
				break
			}
		}
		if match {
			out[i].CategoryScore = 1
		}
		out[i].Score = weightedScore(cfg,
			scorePart{out[i].SemanticScore, cfg.SemanticWeight, out[i].SemanticScore > 0},
			scorePart{out[i].TitleScore, cfg.TitleWeight, out[i].TitleScore > 0},
			scorePart{out[i].LexicalScore, cfg.LexicalWeight, out[i].LexicalScore > 0},
			scorePart{out[i].KeywordScore, cfg.KeywordWeight, out[i].KeywordScore > 0},
			scorePart{out[i].CategoryScore, cfg.CategoryWeight, out[i].CategoryScore > 0},
		)
	}
	sort.SliceStable(out, func(i, j int) bool {
		if out[i].Score == out[j].Score {
			return out[i].SemanticScore > out[j].SemanticScore
		}
		return out[i].Score > out[j].Score
	})
	return out
}

func (s *Store) index(ctx context.Context) error {
	_ = os.MkdirAll(filepath.Dir(s.cachePath), 0o750)
	cf := loadCache(s.cachePath)
	var need []model.KnowledgeDoc
	for _, d := range s.docs {
		bodyChunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
		s.chunks[d.ID] = bodyChunks
		h := hashDoc(d, s.scoring)
		if cf.Hashes[d.ID] == h && len(cf.TitleVectors[d.ID]) > 0 && len(cf.ChunkVectors[d.ID]) == len(bodyChunks) {
			s.titleVectors[d.ID] = append([]float64(nil), cf.TitleVectors[d.ID]...)
			s.chunkVectors[d.ID] = cloneChunkVectors(cf.ChunkVectors[d.ID])
		} else {
			need = append(need, d)
		}
	}
	if len(need) > 0 {
		embedded, err := s.embedDocuments(ctx, need)
		if err != nil {
			return err
		}
		for _, d := range need {
			s.titleVectors[d.ID] = embedded[d.ID].title
			s.chunkVectors[d.ID] = embedded[d.ID].chunks
		}
	}
	return s.persistVectorCache()
}

type embeddedDoc struct {
	title  []float64
	chunks [][]float64
}

func (s *Store) embedDocuments(ctx context.Context, docs []model.KnowledgeDoc) (map[string]embeddedDoc, error) {
	out := make(map[string]embeddedDoc, len(docs))
	type ref struct {
		id    string
		title bool
		chunk int
	}
	var texts []string
	var refs []ref
	for _, d := range docs {
		texts = append(texts, formatDocumentEmbedding(d.Title, d.Title, s.scoring.EmbeddingProfile))
		refs = append(refs, ref{id: d.ID, title: true})
		parts := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
		for i, part := range parts {
			texts = append(texts, formatDocumentEmbedding(d.Title, part, s.scoring.EmbeddingProfile))
			refs = append(refs, ref{id: d.ID, chunk: i})
		}
	}
	vectors, err := s.embedTexts(ctx, texts, 64)
	if err != nil {
		return nil, err
	}
	if len(vectors) != len(refs) {
		return nil, fmt.Errorf("embedding provider returned %d vectors for %d inputs", len(vectors), len(refs))
	}
	for i, r := range refs {
		if len(vectors[i]) == 0 {
			return nil, fmt.Errorf("embedding provider returned empty vector for %s", r.id)
		}
		e := out[r.id]
		if r.title {
			e.title = vectors[i]
		} else {
			for len(e.chunks) <= r.chunk {
				e.chunks = append(e.chunks, nil)
			}
			e.chunks[r.chunk] = vectors[i]
		}
		out[r.id] = e
	}
	return out, nil
}

func (s *Store) embedTexts(ctx context.Context, texts []string, batch int) ([][]float64, error) {
	if len(texts) == 0 {
		return nil, nil
	}
	if batch <= 0 {
		batch = 64
	}
	out := make([][]float64, 0, len(texts))
	for start := 0; start < len(texts); start += batch {
		end := start + batch
		if end > len(texts) {
			end = len(texts)
		}
		vv, err := s.embedder.Embed(ctx, texts[start:end])
		if err != nil {
			return nil, err
		}
		if len(vv) != end-start {
			return nil, fmt.Errorf("embedding provider returned %d vectors for %d inputs", len(vv), end-start)
		}
		out = append(out, vv...)
	}
	return out, nil
}

func loadCache(path string) cacheFile {
	cf := cacheFile{Version: 3, Hashes: map[string]string{}, TitleVectors: map[string][]float64{}, ChunkVectors: map[string][][]float64{}}
	if b, err := os.ReadFile(path); err == nil {
		_ = json.Unmarshal(b, &cf)
		if cf.Version != 3 {
			cf = cacheFile{Version: 3, Hashes: map[string]string{}, TitleVectors: map[string][]float64{}, ChunkVectors: map[string][][]float64{}}
		}
	}
	if cf.Hashes == nil {
		cf.Hashes = map[string]string{}
	}
	if cf.TitleVectors == nil {
		cf.TitleVectors = map[string][]float64{}
	}
	if cf.ChunkVectors == nil {
		cf.ChunkVectors = map[string][][]float64{}
	}
	return cf
}

func formatQueryEmbeddings(texts []string, profile string) []string {
	out := make([]string, len(texts))
	for i, text := range texts {
		if profile == "embeddinggemma" {
			out[i] = "task: search result | query: " + strings.TrimSpace(text)
		} else {
			out[i] = text
		}
	}
	return out
}

func formatDocumentEmbedding(title, text, profile string) string {
	if profile == "embeddinggemma" {
		t := strings.TrimSpace(title)
		if t == "" {
			t = "none"
		}
		return "title: " + t + " | text: " + strings.TrimSpace(text)
	}
	return text
}

func splitQueryText(text string) (title, body string) {
	text = strings.TrimSpace(text)
	if text == "" {
		return "", ""
	}
	if i := strings.IndexByte(text, '\n'); i >= 0 {
		title = strings.TrimSpace(text[:i])
		body = strings.TrimSpace(text[i+1:])
		return title, body
	}
	return text, text
}

func chunkText(text string, words, overlap, maxChunks int) []string {
	parts := strings.Fields(strings.TrimSpace(text))
	if len(parts) == 0 {
		return nil
	}
	if words <= 0 {
		words = 160
	}
	if overlap < 0 || overlap >= words {
		overlap = 0
	}
	if maxChunks <= 0 {
		maxChunks = 24
	}
	step := words - overlap
	out := make([]string, 0, minInt(maxChunks, (len(parts)+step-1)/step))
	for start := 0; start < len(parts) && len(out) < maxChunks; start += step {
		end := start + words
		if end > len(parts) {
			end = len(parts)
		}
		out = append(out, strings.Join(parts[start:end], " "))
		if end == len(parts) {
			break
		}
	}
	return out
}

type scorePart struct {
	value, weight float64
	available     bool
}

func weightedScore(_ ScoringConfig, parts ...scorePart) float64 {
	var sum, weights float64
	for _, p := range parts {
		if !p.available || p.weight <= 0 {
			continue
		}
		sum += clamp01(p.value) * p.weight
		weights += p.weight
	}
	if weights == 0 {
		return 0
	}
	return clamp01(sum / weights)
}

func titleSimilarity(query, title string) float64 {
	q := strings.TrimSpace(query)
	t := strings.TrimSpace(title)
	if q == "" || t == "" {
		return 0
	}
	qn := normalizeText(q)
	tn := normalizeText(t)
	if qn == tn || strings.Contains(tn, qn) || strings.Contains(qn, tn) {
		return 1
	}
	// Title relevance is intentionally asymmetric: if the short ticket subject
	// is fully represented by one of several concepts in a longer KB title, that
	// is a strong title match rather than a low symmetric F1 score.
	return math.Max(tokenCoverage(q, t), tokenF1(q, t))
}

func lexicalSimilarity(query string, d model.KnowledgeDoc) float64 {
	best := math.Max(tokenF1(query, d.Title+" "+d.Text), tokenCoverage(query, d.Title+" "+d.Text))
	if t, _ := splitQueryText(query); strings.TrimSpace(t) != "" {
		best = math.Max(best, tokenCoverage(t, d.Title))
	}
	for _, kw := range d.Keywords {
		best = math.Max(best, phraseCoverage(kw, query))
	}
	return clamp01(best)
}

func keywordSimilarity(query string, keywords []string) (float64, bool) {
	if len(keywords) == 0 {
		return 0, false
	}
	best := 0.0
	for _, kw := range keywords {
		kw = strings.TrimSpace(kw)
		if kw == "" {
			continue
		}
		best = math.Max(best, phraseCoverage(kw, query))
	}
	return clamp01(best), true
}

func categorySimilarity(query string, ids []int64, categories []model.Category) (float64, bool) {
	if len(ids) == 0 || len(categories) == 0 {
		return 0, false
	}
	wanted := make(map[int64]struct{}, len(ids))
	for _, id := range ids {
		wanted[id] = struct{}{}
	}
	best, found := 0.0, false
	for _, c := range categories {
		if _, ok := wanted[c.ID]; !ok {
			continue
		}
		found = true
		for _, part := range append([]string{c.Name, c.CompleteName}, append(c.Hints, c.Examples...)...) {
			if strings.TrimSpace(part) == "" {
				continue
			}
			best = math.Max(best, phraseCoverage(part, query))
		}
	}
	return clamp01(best), found
}

// phraseCoverage asks "how much of this concept phrase occurs in the query?".
// It is better suited to support terminology than symmetric F1 because a long
// user ticket may contain lots of harmless extra words.
func phraseCoverage(phrase, query string) float64 {
	pn := normalizeText(phrase)
	qn := normalizeText(query)
	if pn == "" || qn == "" {
		return 0
	}
	if strings.Contains(qn, pn) {
		return 1
	}
	return tokenCoverage(phrase, query)
}

func tokenCoverage(needle, haystack string) float64 {
	a := tokenList(needle)
	b := tokenList(haystack)
	if len(a) == 0 || len(b) == 0 {
		return 0
	}
	var sum float64
	for _, x := range a {
		best := 0.0
		for _, y := range b {
			best = math.Max(best, tokenSimilarity(x, y))
		}
		sum += best
	}
	return clamp01(sum / float64(len(a)))
}

func tokenSimilarity(a, b string) float64 {
	if a == b {
		return 1
	}
	if len(a) < 4 || len(b) < 4 {
		return 0
	}
	// Helpdesk-German contains many compounds and inflections (anmelden,
	// Anmeldung, Benutzeranmeldung, Nutzerkonto, Benutzerkonto). Exact-token
	// overlap is therefore too brittle. First reward strong substring matches,
	// then compare a deliberately small set of German support stems.
	short, long := a, b
	if len(short) > len(long) {
		short, long = long, short
	}
	if len(short) >= 5 && strings.Contains(long, short) {
		ratio := float64(len(short)) / float64(len(long))
		return clamp01(.75 + .25*ratio)
	}
	sa, sb := supportStem(a), supportStem(b)
	if sa == sb && len(sa) >= 5 {
		return .95
	}
	stemShort, stemLong := sa, sb
	if len(stemShort) > len(stemLong) {
		stemShort, stemLong = stemLong, stemShort
	}
	if len(stemShort) >= 5 && strings.Contains(stemLong, stemShort) {
		return .90
	}
	common := 0
	limit := len(a)
	if len(b) < limit {
		limit = len(b)
	}
	for common < limit && a[common] == b[common] {
		common++
	}
	minLen := len(a)
	if len(b) < minLen {
		minLen = len(b)
	}
	maxLen := len(a)
	if len(b) > maxLen {
		maxLen = len(b)
	}
	if common >= 5 && float64(common)/float64(minLen) >= .70 {
		return clamp01(float64(common) / float64(maxLen))
	}
	return 0
}

func supportStem(s string) string {
	s = strings.ToLower(strings.TrimSpace(s))
	// Long, semantically common German suffixes first. This is intentionally
	// conservative and is not meant to be a full linguistic stemmer.
	for _, suffix := range []string{"ungen", "ern", "ung", "ieren", "ischen", "ische", "isch", "enden", "ende", "en", "er", "es", "e", "n", "s"} {
		if strings.HasSuffix(s, suffix) && len(s)-len(suffix) >= 5 {
			s = strings.TrimSuffix(s, suffix)
			break
		}
	}
	return s
}

func normalizeText(s string) string {
	return strings.Join(tokenList(s), " ")
}

func tokenList(s string) []string {
	parts := strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) })
	out := make([]string, 0, len(parts))
	for _, p := range parts {
		if len([]rune(p)) < 3 || isStopword(p) {
			continue
		}
		out = append(out, p)
	}
	return out
}

func isStopword(s string) bool {
	switch s {
	case "der", "die", "das", "den", "dem", "des", "ein", "eine", "einer", "einem", "einen", "und", "oder", "aber", "mit", "ohne", "für", "fuer", "von", "vom", "zum", "zur", "ist", "sind", "war", "wird", "werden", "ich", "wir", "sie", "seit", "heute", "gestern", "bitte", "hilfe", "vielen", "dank", "nicht", "mehr", "kann", "mich", "mir", "mein", "meine", "meinen", "meinem":
		return true
	default:
		return false
	}
}

func tokenF1(a, b string) float64 {
	aTok, bTok := tokens(a), tokens(b)
	if len(aTok) == 0 || len(bTok) == 0 {
		return 0
	}
	common := 0
	for t := range aTok {
		if _, ok := bTok[t]; ok {
			common++
		}
	}
	if common == 0 {
		return 0
	}
	precision := float64(common) / float64(len(aTok))
	recall := float64(common) / float64(len(bTok))
	return 2 * precision * recall / (precision + recall)
}

func excerpt(s string, max int) string {
	s = strings.Join(strings.Fields(s), " ")
	if len([]rune(s)) <= max {
		return s
	}
	r := []rune(s)
	return string(r[:max]) + "…"
}
func clamp01(v float64) float64 {
	if v < 0 {
		return 0
	}
	if v > 1 {
		return 1
	}
	return v
}
func cloneVectorMap(in map[string][]float64) map[string][]float64 {
	out := make(map[string][]float64, len(in))
	for k, v := range in {
		out[k] = append([]float64(nil), v...)
	}
	return out
}
func cloneChunkVectorMap(in map[string][][]float64) map[string][][]float64 {
	out := make(map[string][][]float64, len(in))
	for k, v := range in {
		out[k] = cloneChunkVectors(v)
	}
	return out
}
func cloneChunkVectors(in [][]float64) [][]float64 {
	out := make([][]float64, len(in))
	for i, v := range in {
		out[i] = append([]float64(nil), v...)
	}
	return out
}
func cloneStringSliceMap(in map[string][]string) map[string][]string {
	out := make(map[string][]string, len(in))
	for k, v := range in {
		out[k] = append([]string(nil), v...)
	}
	return out
}
func minInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func hashDoc(d model.KnowledgeDoc, cfg ScoringConfig) string {
	// Only retrieval-relevant fields belong in the embedding fingerprint.
	// Formatting-only changes to AnswerHTML must not force re-embedding.
	b, _ := json.Marshal(struct {
		ID                string   `json:"id"`
		Title             string   `json:"title"`
		Text              string   `json:"text"`
		Categories        []int64  `json:"categories"`
		Keywords          []string `json:"keywords"`
		Profile           string   `json:"profile"`
		EmbeddingIdentity string   `json:"embedding_identity"`
		ChunkWords        int      `json:"chunk_words"`
		ChunkOverlap      int      `json:"chunk_overlap"`
		MaxChunks         int      `json:"max_chunks"`
	}{d.ID, d.Title, d.Text, d.Categories, d.Keywords, cfg.EmbeddingProfile, cfg.EmbeddingIdentity, cfg.ChunkWords, cfg.ChunkOverlap, cfg.MaxChunksPerDoc})
	h := sha256.Sum256(b)
	return hex.EncodeToString(h[:])
}
func cosine(a, b []float64) float64 {
	if len(a) == 0 || len(a) != len(b) {
		return 0
	}
	var dot, aa, bb float64
	for i := range a {
		dot += a[i] * b[i]
		aa += a[i] * a[i]
		bb += b[i] * b[i]
	}
	if aa == 0 || bb == 0 {
		return 0
	}
	return dot / (math.Sqrt(aa) * math.Sqrt(bb))
}
func lexical(text string, d model.KnowledgeDoc) float64 {
	q := tokens(text)
	hay := tokens(d.Title + " " + d.Text + " " + strings.Join(d.Keywords, " "))
	if len(q) == 0 {
		return 0
	}
	hits := 0
	for t := range q {
		if _, ok := hay[t]; ok {
			hits++
		}
	}
	return float64(hits) / float64(len(q))
}
func tokens(s string) map[string]struct{} {
	m := map[string]struct{}{}
	for _, p := range tokenList(s) {
		m[p] = struct{}{}
	}
	return m
}