glpi-ai-agent/internal/knowledge/store.go

package knowledge

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"unicode"

	"github.com/example/glpi-ai-agent/internal/model"
)

type Embedder interface {
	Embed(context.Context, []string) ([][]float64, error)
}
type ScoringConfig struct {
	SemanticWeight  float64
	TitleWeight     float64
	KeywordWeight   float64
	CategoryWeight  float64
	ChunkWords      int
	ChunkOverlap    int
	MaxChunksPerDoc int
	MaxQueryChunks  int
}

type Store struct {
	mu             sync.RWMutex
	dir            string
	managedDir     string
	docs           []model.KnowledgeDoc
	files          map[string]string
	managed        map[string]bool
	external       map[string]string
	staticDocs     map[string]model.KnowledgeDoc
	titleVectors   map[string][]float64
	chunkVectors   map[string][][]float64
	chunks         map[string][]string
	embedder       Embedder
	rag            bool
	cachePath      string
	allowedSources map[string]struct{}
	scoring        ScoringConfig
}
type cacheFile struct {
	Version      int                    `json:"version,omitempty"`
	Hashes       map[string]string      `json:"hashes"`
	TitleVectors map[string][]float64   `json:"title_vectors,omitempty"`
	ChunkVectors map[string][][]float64 `json:"chunk_vectors,omitempty"`
}

func DefaultScoringConfig() ScoringConfig {
	return ScoringConfig{SemanticWeight: .50, TitleWeight: .25, KeywordWeight: .15, CategoryWeight: .10, ChunkWords: 160, ChunkOverlap: 30, MaxChunksPerDoc: 24, MaxQueryChunks: 64}
}

func normalizeScoring(c ScoringConfig) ScoringConfig {
	d := DefaultScoringConfig()
	if c.SemanticWeight < 0 || c.TitleWeight < 0 || c.KeywordWeight < 0 || c.CategoryWeight < 0 || c.SemanticWeight+c.TitleWeight+c.KeywordWeight+c.CategoryWeight <= 0 {
		c.SemanticWeight, c.TitleWeight, c.KeywordWeight, c.CategoryWeight = d.SemanticWeight, d.TitleWeight, d.KeywordWeight, d.CategoryWeight
	}
	if c.ChunkWords <= 0 {
		c.ChunkWords = d.ChunkWords
	}
	if c.ChunkOverlap < 0 || c.ChunkOverlap >= c.ChunkWords {
		c.ChunkOverlap = d.ChunkOverlap
	}
	if c.MaxChunksPerDoc <= 0 {
		c.MaxChunksPerDoc = d.MaxChunksPerDoc
	}
	if c.MaxQueryChunks <= 0 {
		c.MaxQueryChunks = d.MaxQueryChunks
	}
	return c
}

func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool, allowedSources []string, scoring ...ScoringConfig) (*Store, error) {
	managedDir := filepath.Join(dataDir, "knowledge-managed")
	if err := os.MkdirAll(managedDir, 0o750); err != nil {
		return nil, fmt.Errorf("create managed knowledge directory: %w", err)
	}
	scoreCfg := DefaultScoringConfig()
	if len(scoring) > 0 {
		scoreCfg = normalizeScoring(scoring[0])
	}
	s := &Store{dir: dir, managedDir: managedDir, titleVectors: map[string][]float64{}, chunkVectors: map[string][][]float64{}, chunks: map[string][]string{}, files: map[string]string{}, managed: map[string]bool{}, external: map[string]string{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}, scoring: scoreCfg}
	for _, source := range allowedSources {
		s.allowedSources[strings.ToLower(strings.TrimSpace(source))] = struct{}{}
	}
	static, staticFiles, err := readDocs(dir, s.allowedSources)
	if err != nil {
		return nil, err
	}
	for i, d := range static {
		s.staticDocs[d.ID] = d
		s.files[d.ID] = staticFiles[i]
	}
	managed, managedFiles, err := readDocs(managedDir, s.allowedSources)
	if err != nil {
		return nil, err
	}
	merged := map[string]model.KnowledgeDoc{}
	order := []string{}
	for _, d := range static {
		if _, ok := merged[d.ID]; !ok {
			order = append(order, d.ID)
		}
		merged[d.ID] = d
	}
	for i, d := range managed {
		if _, ok := merged[d.ID]; !ok {
			order = append(order, d.ID)
		}
		merged[d.ID] = d
		s.files[d.ID] = managedFiles[i]
		s.managed[d.ID] = true
	}
	for _, id := range order {
		s.docs = append(s.docs, merged[id])
	}
	if rag && len(s.docs) > 0 {
		if s.embedder == nil {
			return nil, fmt.Errorf("RAG is enabled but no embedding provider is configured")
		}
		if err := s.index(ctx); err != nil {
			return s, err
		}
	}
	return s, nil
}

func readDocs(dir string, allowed map[string]struct{}) ([]model.KnowledgeDoc, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, fmt.Errorf("read knowledge directory %q: %w", dir, err)
	}
	var docs []model.KnowledgeDoc
	var files []string
	for _, e := range entries {
		if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
			continue
		}
		path := filepath.Join(dir, e.Name())
		b, err := os.ReadFile(path)
		if err != nil {
			return nil, nil, err
		}
		var d model.KnowledgeDoc
		if err := json.Unmarshal(b, &d); err != nil {
			return nil, nil, fmt.Errorf("%s: %w", e.Name(), err)
		}
		if d.ID == "" || d.Title == "" {
			return nil, nil, fmt.Errorf("%s: id/title required", e.Name())
		}
		if !safeID(d.ID) {
			return nil, nil, fmt.Errorf("%s: invalid id %q", e.Name(), d.ID)
		}
		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
		if d.Source == "" {
			return nil, nil, fmt.Errorf("%s: source required", e.Name())
		}
		if _, ok := allowed[d.Source]; !ok {
			continue
		}
		d.Language = strings.TrimSpace(d.Language)
		d.CommunicationStyle = strings.ToLower(strings.TrimSpace(d.CommunicationStyle))
		docs = append(docs, d)
		files = append(files, path)
	}
	return docs, files, nil
}

func (s *Store) Count() int {
	if s == nil {
		return 0
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	return len(s.docs)
}
func (s *Store) ByID(id string) (model.KnowledgeDoc, bool) {
	if s == nil {
		return model.KnowledgeDoc{}, false
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	for _, d := range s.docs {
		if d.ID == id {
			return d, true
		}
	}
	return model.KnowledgeDoc{}, false
}
func (s *Store) List() []model.KnowledgeDoc {
	if s == nil {
		return nil
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	out := append([]model.KnowledgeDoc(nil), s.docs...)
	sort.SliceStable(out, func(i, j int) bool { return strings.ToLower(out[i].Title) < strings.ToLower(out[j].Title) })
	return out
}

func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
	if s == nil {
		return fmt.Errorf("knowledge store is not initialized")
	}
	d.ID = strings.TrimSpace(d.ID)
	d.Title = strings.TrimSpace(d.Title)
	d.Text = strings.TrimSpace(d.Text)
	d.Answer = strings.TrimSpace(d.Answer)
	d.Source = strings.ToLower(strings.TrimSpace(d.Source))
	d.Language = strings.TrimSpace(d.Language)
	d.CommunicationStyle = strings.ToLower(strings.TrimSpace(d.CommunicationStyle))
	if d.ID == "" || d.Title == "" {
		return fmt.Errorf("id/title required")
	}
	if !safeID(d.ID) {
		return fmt.Errorf("knowledge id may contain only letters, digits, dot, dash and underscore")
	}
	if d.Source == "" {
		return fmt.Errorf("source required")
	}
	if _, ok := s.allowedSources[d.Source]; !ok {
		return fmt.Errorf("source %q is not allowed", d.Source)
	}
	if d.Language == "" || d.CommunicationStyle == "" {
		return fmt.Errorf("language and communication_style required")
	}
	if d.MinScore < 0 || d.MinScore > 1 {
		return fmt.Errorf("min_score must be between 0 and 1")
	}
	s.mu.RLock()
	_, exists := s.files[d.ID]
	isManaged := s.managed[d.ID]
	externalSource := s.external[d.ID]
	s.mu.RUnlock()
	if externalSource != "" {
		return fmt.Errorf("externally synchronized knowledge entry %q from %q is read-only", d.ID, externalSource)
	}
	if exists && !isManaged {
		return fmt.Errorf("static knowledge entry %q is read-only; use a new id for a managed entry", d.ID)
	}

	var titleVector []float64
	var chunkVectors [][]float64
	chunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
	if s.rag {
		if s.embedder == nil {
			return fmt.Errorf("RAG is enabled but no embedding provider is configured")
		}
		embedded, err := s.embedDocuments(ctx, []model.KnowledgeDoc{d})
		if err != nil {
			return err
		}
		titleVector = embedded[d.ID].title
		chunkVectors = embedded[d.ID].chunks
	}

	path := filepath.Join(s.managedDir, d.ID+".json")
	b, err := json.MarshalIndent(d, "", "  ")
	if err != nil {
		return err
	}
	tmp := path + ".tmp"
	if err := os.WriteFile(tmp, b, 0o640); err != nil {
		return err
	}
	if err := os.Rename(tmp, path); err != nil {
		_ = os.Remove(tmp)
		return err
	}
	s.mu.Lock()
	replaced := false
	for i := range s.docs {
		if s.docs[i].ID == d.ID {
			s.docs[i] = d
			replaced = true
			break
		}
	}
	if !replaced {
		s.docs = append(s.docs, d)
	}
	s.files[d.ID] = path
	s.managed[d.ID] = true
	s.chunks[d.ID] = chunks
	if s.rag {
		s.titleVectors[d.ID] = titleVector
		s.chunkVectors[d.ID] = chunkVectors
	}
	s.mu.Unlock()
	return s.persistVectorCache()
}

func (s *Store) Delete(id string) error {
	if s == nil {
		return fmt.Errorf("knowledge store is not initialized")
	}
	id = strings.TrimSpace(id)
	if !safeID(id) {
		return fmt.Errorf("invalid knowledge id")
	}
	s.mu.RLock()
	path := s.files[id]
	isManaged := s.managed[id]
	s.mu.RUnlock()
	if path == "" {
		return os.ErrNotExist
	}
	if !isManaged {
		return fmt.Errorf("static knowledge entry %q is read-only", id)
	}
	if err := os.Remove(path); err != nil {
		return err
	}
	s.mu.Lock()
	out := s.docs[:0]
	for _, d := range s.docs {
		if d.ID != id {
			out = append(out, d)
		}
	}
	s.docs = append([]model.KnowledgeDoc(nil), out...)
	delete(s.files, id)
	delete(s.managed, id)
	delete(s.titleVectors, id)
	delete(s.chunkVectors, id)
	delete(s.chunks, id)
	s.mu.Unlock()
	return s.persistVectorCache()
}

func (s *Store) IsManaged(id string) bool {
	if s == nil {
		return false
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.managed[id]
}
func (s *Store) Origin(id string) string {
	if s == nil {
		return ""
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	if s.managed[id] {
		return "managed"
	}
	if src := s.external[id]; src != "" {
		return src
	}
	if _, ok := s.staticDocs[id]; ok {
		return "static"
	}
	return ""
}

// ReplaceExternalSource atomically replaces all read-only documents imported
// from one connector source. Embeddings are reused when the normalized article
// did not change. Long article bodies are indexed as overlapping chunks.
func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs []model.KnowledgeDoc) error {
	if s == nil {
		return fmt.Errorf("knowledge store is not initialized")
	}
	source = strings.ToLower(strings.TrimSpace(source))
	if _, ok := s.allowedSources[source]; !ok {
		return fmt.Errorf("source %q is not allowed", source)
	}

	s.mu.RLock()
	oldDocs := make(map[string]model.KnowledgeDoc, len(s.docs))
	oldTitle := cloneVectorMap(s.titleVectors)
	oldChunks := cloneChunkVectorMap(s.chunkVectors)
	for _, d := range s.docs {
		oldDocs[d.ID] = d
	}
	s.mu.RUnlock()
	cached := loadCache(s.cachePath)

	changed := make([]model.KnowledgeDoc, 0)
	seen := map[string]struct{}{}
	for i := range docs {
		d := &docs[i]
		d.ID = strings.TrimSpace(d.ID)
		d.Title = strings.TrimSpace(d.Title)
		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
		if d.Source == "" {
			d.Source = source
		}
		if d.Source != source {
			return fmt.Errorf("external document %q has source %q, expected %q", d.ID, d.Source, source)
		}
		if d.ID == "" || d.Title == "" || !safeID(d.ID) {
			return fmt.Errorf("invalid external knowledge document id/title")
		}
		if _, dup := seen[d.ID]; dup {
			return fmt.Errorf("duplicate external knowledge id %q", d.ID)
		}
		seen[d.ID] = struct{}{}
		h := hashDoc(*d)
		bodyChunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
		same := false
		if old, ok := oldDocs[d.ID]; ok && hashDoc(old) == h && len(oldTitle[d.ID]) > 0 && len(oldChunks[d.ID]) == len(bodyChunks) {
			same = true
		} else if cached.Hashes[d.ID] == h && len(cached.TitleVectors[d.ID]) > 0 && len(cached.ChunkVectors[d.ID]) == len(bodyChunks) {
			oldTitle[d.ID] = append([]float64(nil), cached.TitleVectors[d.ID]...)
			oldChunks[d.ID] = cloneChunkVectors(cached.ChunkVectors[d.ID])
			same = true
		}
		if !same {
			changed = append(changed, *d)
		}
	}

	newEmbedded := map[string]embeddedDoc{}
	if s.rag && len(changed) > 0 {
		if s.embedder == nil {
			return fmt.Errorf("RAG is enabled but no embedding provider is configured")
		}
		var err error
		newEmbedded, err = s.embedDocuments(ctx, changed)
		if err != nil {
			return err
		}
	}

	s.mu.Lock()
	for _, d := range docs {
		if src := s.external[d.ID]; src == "" {
			if _, exists := oldDocs[d.ID]; exists {
				s.mu.Unlock()
				return fmt.Errorf("external knowledge id %q collides with local knowledge", d.ID)
			}
		} else if src != source {
			s.mu.Unlock()
			return fmt.Errorf("external knowledge id %q belongs to source %q", d.ID, src)
		}
	}
	rebuilt := make([]model.KnowledgeDoc, 0, len(s.docs)+len(docs))
	for _, d := range s.docs {
		if s.external[d.ID] != source {
			rebuilt = append(rebuilt, d)
		}
	}
	for id, src := range s.external {
		if src == source {
			delete(s.external, id)
			delete(s.titleVectors, id)
			delete(s.chunkVectors, id)
			delete(s.chunks, id)
		}
	}
	for _, d := range docs {
		rebuilt = append(rebuilt, d)
		s.external[d.ID] = source
		s.chunks[d.ID] = chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
		if e, ok := newEmbedded[d.ID]; ok {
			s.titleVectors[d.ID] = e.title
			s.chunkVectors[d.ID] = e.chunks
		} else {
			s.titleVectors[d.ID] = oldTitle[d.ID]
			s.chunkVectors[d.ID] = oldChunks[d.ID]
		}
	}
	s.docs = rebuilt
	s.mu.Unlock()
	return s.persistVectorCache()
}

func (s *Store) persistVectorCache() error {
	if s == nil || !s.rag {
		return nil
	}
	s.mu.RLock()
	cf := cacheFile{Version: 2, Hashes: map[string]string{}, TitleVectors: map[string][]float64{}, ChunkVectors: map[string][][]float64{}}
	for _, d := range s.docs {
		if len(s.titleVectors[d.ID]) == 0 {
			continue
		}
		cf.Hashes[d.ID] = hashDoc(d)
		cf.TitleVectors[d.ID] = append([]float64(nil), s.titleVectors[d.ID]...)
		cf.ChunkVectors[d.ID] = cloneChunkVectors(s.chunkVectors[d.ID])
	}
	s.mu.RUnlock()
	b, err := json.MarshalIndent(cf, "", "  ")
	if err != nil {
		return err
	}
	tmp := s.cachePath + ".tmp"
	if err := os.WriteFile(tmp, b, 0o640); err != nil {
		return err
	}
	return os.Rename(tmp, s.cachePath)
}

func (s *Store) ManagedDir() string {
	if s == nil {
		return ""
	}
	return s.managedDir
}
func safeID(v string) bool {
	if v == "" {
		return false
	}
	for _, r := range v {
		if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.') {
			return false
		}
	}
	return !strings.Contains(v, "..")
}

// Search calculates a transparent hybrid relevance score. Embedding similarity
// is only one component; titles, explicit keywords and category/learning hints
// are scored separately. Missing metadata does not lower a document's score:
// the weights of available components are normalized dynamically.
func (s *Store) Search(ctx context.Context, text string, topK int, categorySets ...[]model.Category) ([]model.KnowledgeHit, error) {
	if s == nil {
		return nil, fmt.Errorf("knowledge store is not initialized")
	}
	s.mu.RLock()
	docs := append([]model.KnowledgeDoc(nil), s.docs...)
	titleVecs := cloneVectorMap(s.titleVectors)
	chunkVecs := cloneChunkVectorMap(s.chunkVectors)
	chunks := cloneStringSliceMap(s.chunks)
	scoreCfg := s.scoring
	s.mu.RUnlock()
	if len(docs) == 0 {
		return nil, nil
	}
	var cats []model.Category
	if len(categorySets) > 0 {
		cats = categorySets[0]
	}

	queryTitle, queryBody := splitQueryText(text)
	queryChunks := chunkText(queryBody, scoreCfg.ChunkWords, scoreCfg.ChunkOverlap, scoreCfg.MaxQueryChunks)
	if len(queryChunks) == 0 {
		queryChunks = chunkText(text, scoreCfg.ChunkWords, scoreCfg.ChunkOverlap, scoreCfg.MaxQueryChunks)
	}
	var queryVectors [][]float64
	var queryTitleVector []float64
	if s.rag && s.embedder != nil {
		if len(queryChunks) > 0 {
			q, err := s.embedTexts(ctx, queryChunks, 64)
			if err != nil {
				return nil, err
			}
			queryVectors = q
		}
		if strings.TrimSpace(queryTitle) != "" {
			tq, err := s.embedder.Embed(ctx, []string{queryTitle})
			if err != nil {
				return nil, err
			}
			if len(tq) > 0 {
				queryTitleVector = tq[0]
			}
		}
	}

	hits := make([]model.KnowledgeHit, 0, len(docs))
	for _, d := range docs {
		semantic, bestChunk, bestQueryChunk := 0.0, "", ""
		semanticAvailable := false
		if len(queryVectors) > 0 && len(chunkVecs[d.ID]) > 0 {
			semanticAvailable = true
			for qi, qv := range queryVectors {
				for di, dv := range chunkVecs[d.ID] {
					score := clamp01(cosine(qv, dv))
					if score > semantic || bestChunk == "" {
						semantic = score
						if di < len(chunks[d.ID]) {
							bestChunk = chunks[d.ID][di]
						}
						if qi < len(queryChunks) {
							bestQueryChunk = queryChunks[qi]
						}
					}
				}
			}
		} else if strings.TrimSpace(d.Text) != "" {
			semanticAvailable = true
			docChunks := chunks[d.ID]
			if len(docChunks) == 0 {
				docChunks = []string{d.Text}
			}
			for _, qc := range queryChunks {
				for _, dc := range docChunks {
					score := tokenF1(qc, dc)
					if score > semantic || bestChunk == "" {
						semantic, bestChunk, bestQueryChunk = score, dc, qc
					}
				}
			}
		}

		title := 0.0
		titleAvailable := strings.TrimSpace(d.Title) != ""
		if titleAvailable {
			titleQuery := queryTitle
			if strings.TrimSpace(titleQuery) == "" {
				titleQuery = text
			}
			title = titleSimilarity(titleQuery, d.Title)
			if len(queryTitleVector) > 0 && len(titleVecs[d.ID]) > 0 {
				title = math.Max(title, clamp01(cosine(queryTitleVector, titleVecs[d.ID])))
			}
		}
		keyword, keywordAvailable := keywordSimilarity(text, d.Keywords)
		category, categoryAvailable := categorySimilarity(text, d.Categories, cats)
		total := weightedScore(scoreCfg,
			scorePart{semantic, scoreCfg.SemanticWeight, semanticAvailable},
			scorePart{title, scoreCfg.TitleWeight, titleAvailable},
			scorePart{keyword, scoreCfg.KeywordWeight, keywordAvailable},
			scorePart{category, scoreCfg.CategoryWeight, categoryAvailable},
		)
		hits = append(hits, model.KnowledgeHit{Doc: d, Score: total, SemanticScore: semantic, TitleScore: title, KeywordScore: keyword, CategoryScore: category, BestChunkExcerpt: excerpt(bestChunk, 280), BestQueryExcerpt: excerpt(bestQueryChunk, 280), QueryChunkCount: len(queryChunks), DocumentChunkCount: len(chunks[d.ID])})
	}
	sort.SliceStable(hits, func(i, j int) bool {
		if hits[i].Score == hits[j].Score {
			return hits[i].TitleScore > hits[j].TitleScore
		}
		return hits[i].Score > hits[j].Score
	})
	if topK > 0 && len(hits) > topK {
		hits = hits[:topK]
	}
	return hits, nil
}

func (s *Store) index(ctx context.Context) error {
	_ = os.MkdirAll(filepath.Dir(s.cachePath), 0o750)
	cf := loadCache(s.cachePath)
	var need []model.KnowledgeDoc
	for _, d := range s.docs {
		bodyChunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
		s.chunks[d.ID] = bodyChunks
		h := hashDoc(d)
		if cf.Hashes[d.ID] == h && len(cf.TitleVectors[d.ID]) > 0 && len(cf.ChunkVectors[d.ID]) == len(bodyChunks) {
			s.titleVectors[d.ID] = append([]float64(nil), cf.TitleVectors[d.ID]...)
			s.chunkVectors[d.ID] = cloneChunkVectors(cf.ChunkVectors[d.ID])
		} else {
			need = append(need, d)
		}
	}
	if len(need) > 0 {
		embedded, err := s.embedDocuments(ctx, need)
		if err != nil {
			return err
		}
		for _, d := range need {
			s.titleVectors[d.ID] = embedded[d.ID].title
			s.chunkVectors[d.ID] = embedded[d.ID].chunks
		}
	}
	return s.persistVectorCache()
}

type embeddedDoc struct {
	title  []float64
	chunks [][]float64
}

func (s *Store) embedDocuments(ctx context.Context, docs []model.KnowledgeDoc) (map[string]embeddedDoc, error) {
	out := make(map[string]embeddedDoc, len(docs))
	type ref struct {
		id    string
		title bool
		chunk int
	}
	var texts []string
	var refs []ref
	for _, d := range docs {
		texts = append(texts, d.Title)
		refs = append(refs, ref{id: d.ID, title: true})
		parts := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
		for i, part := range parts {
			texts = append(texts, part)
			refs = append(refs, ref{id: d.ID, chunk: i})
		}
	}
	vectors, err := s.embedTexts(ctx, texts, 64)
	if err != nil {
		return nil, err
	}
	if len(vectors) != len(refs) {
		return nil, fmt.Errorf("embedding provider returned %d vectors for %d inputs", len(vectors), len(refs))
	}
	for i, r := range refs {
		if len(vectors[i]) == 0 {
			return nil, fmt.Errorf("embedding provider returned empty vector for %s", r.id)
		}
		e := out[r.id]
		if r.title {
			e.title = vectors[i]
		} else {
			for len(e.chunks) <= r.chunk {
				e.chunks = append(e.chunks, nil)
			}
			e.chunks[r.chunk] = vectors[i]
		}
		out[r.id] = e
	}
	return out, nil
}

func (s *Store) embedTexts(ctx context.Context, texts []string, batch int) ([][]float64, error) {
	if len(texts) == 0 {
		return nil, nil
	}
	if batch <= 0 {
		batch = 64
	}
	out := make([][]float64, 0, len(texts))
	for start := 0; start < len(texts); start += batch {
		end := start + batch
		if end > len(texts) {
			end = len(texts)
		}
		vv, err := s.embedder.Embed(ctx, texts[start:end])
		if err != nil {
			return nil, err
		}
		if len(vv) != end-start {
			return nil, fmt.Errorf("embedding provider returned %d vectors for %d inputs", len(vv), end-start)
		}
		out = append(out, vv...)
	}
	return out, nil
}

func loadCache(path string) cacheFile {
	cf := cacheFile{Version: 2, Hashes: map[string]string{}, TitleVectors: map[string][]float64{}, ChunkVectors: map[string][][]float64{}}
	if b, err := os.ReadFile(path); err == nil {
		_ = json.Unmarshal(b, &cf)
	}
	if cf.Hashes == nil {
		cf.Hashes = map[string]string{}
	}
	if cf.TitleVectors == nil {
		cf.TitleVectors = map[string][]float64{}
	}
	if cf.ChunkVectors == nil {
		cf.ChunkVectors = map[string][][]float64{}
	}
	return cf
}

func splitQueryText(text string) (title, body string) {
	text = strings.TrimSpace(text)
	if text == "" {
		return "", ""
	}
	if i := strings.IndexByte(text, '\n'); i >= 0 {
		title = strings.TrimSpace(text[:i])
		body = strings.TrimSpace(text[i+1:])
		return title, body
	}
	return text, text
}

func chunkText(text string, words, overlap, maxChunks int) []string {
	parts := strings.Fields(strings.TrimSpace(text))
	if len(parts) == 0 {
		return nil
	}
	if words <= 0 {
		words = 160
	}
	if overlap < 0 || overlap >= words {
		overlap = 0
	}
	if maxChunks <= 0 {
		maxChunks = 24
	}
	step := words - overlap
	out := make([]string, 0, minInt(maxChunks, (len(parts)+step-1)/step))
	for start := 0; start < len(parts) && len(out) < maxChunks; start += step {
		end := start + words
		if end > len(parts) {
			end = len(parts)
		}
		out = append(out, strings.Join(parts[start:end], " "))
		if end == len(parts) {
			break
		}
	}
	return out
}

type scorePart struct {
	value, weight float64
	available     bool
}

func weightedScore(_ ScoringConfig, parts ...scorePart) float64 {
	var sum, weights float64
	for _, p := range parts {
		if !p.available || p.weight <= 0 {
			continue
		}
		sum += clamp01(p.value) * p.weight
		weights += p.weight
	}
	if weights == 0 {
		return 0
	}
	return clamp01(sum / weights)
}

func titleSimilarity(query, title string) float64 {
	best := tokenF1(query, title)
	q := strings.ToLower(strings.Join(strings.Fields(query), " "))
	t := strings.ToLower(strings.Join(strings.Fields(title), " "))
	if t != "" && strings.Contains(q, t) {
		return 1
	}
	return best
}

func keywordSimilarity(query string, keywords []string) (float64, bool) {
	if len(keywords) == 0 {
		return 0, false
	}
	best := tokenF1(query, strings.Join(keywords, " "))
	q := strings.ToLower(query)
	for _, kw := range keywords {
		kw = strings.ToLower(strings.TrimSpace(kw))
		if kw != "" && strings.Contains(q, kw) {
			best = math.Max(best, 1)
		}
	}
	return clamp01(best), true
}

func categorySimilarity(query string, ids []int64, categories []model.Category) (float64, bool) {
	if len(ids) == 0 || len(categories) == 0 {
		return 0, false
	}
	wanted := make(map[int64]struct{}, len(ids))
	for _, id := range ids {
		wanted[id] = struct{}{}
	}
	best, found := 0.0, false
	for _, c := range categories {
		if _, ok := wanted[c.ID]; !ok {
			continue
		}
		found = true
		profileParts := []string{c.Name, c.CompleteName}
		profileParts = append(profileParts, c.Hints...)
		profileParts = append(profileParts, c.Examples...)
		profile := strings.Join(profileParts, " ")
		best = math.Max(best, tokenF1(query, profile))
	}
	return clamp01(best), found
}

func tokenF1(a, b string) float64 {
	aTok, bTok := tokens(a), tokens(b)
	if len(aTok) == 0 || len(bTok) == 0 {
		return 0
	}
	common := 0
	for t := range aTok {
		if _, ok := bTok[t]; ok {
			common++
		}
	}
	if common == 0 {
		return 0
	}
	precision := float64(common) / float64(len(aTok))
	recall := float64(common) / float64(len(bTok))
	return 2 * precision * recall / (precision + recall)
}

func excerpt(s string, max int) string {
	s = strings.Join(strings.Fields(s), " ")
	if len([]rune(s)) <= max {
		return s
	}
	r := []rune(s)
	return string(r[:max]) + "…"
}
func clamp01(v float64) float64 {
	if v < 0 {
		return 0
	}
	if v > 1 {
		return 1
	}
	return v
}
func cloneVectorMap(in map[string][]float64) map[string][]float64 {
	out := make(map[string][]float64, len(in))
	for k, v := range in {
		out[k] = append([]float64(nil), v...)
	}
	return out
}
func cloneChunkVectorMap(in map[string][][]float64) map[string][][]float64 {
	out := make(map[string][][]float64, len(in))
	for k, v := range in {
		out[k] = cloneChunkVectors(v)
	}
	return out
}
func cloneChunkVectors(in [][]float64) [][]float64 {
	out := make([][]float64, len(in))
	for i, v := range in {
		out[i] = append([]float64(nil), v...)
	}
	return out
}
func cloneStringSliceMap(in map[string][]string) map[string][]string {
	out := make(map[string][]string, len(in))
	for k, v := range in {
		out[k] = append([]string(nil), v...)
	}
	return out
}
func minInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func hashDoc(d model.KnowledgeDoc) string {
	b, _ := json.Marshal(d)
	h := sha256.Sum256(b)
	return hex.EncodeToString(h[:])
}
func cosine(a, b []float64) float64 {
	if len(a) == 0 || len(a) != len(b) {
		return 0
	}
	var dot, aa, bb float64
	for i := range a {
		dot += a[i] * b[i]
		aa += a[i] * a[i]
		bb += b[i] * b[i]
	}
	if aa == 0 || bb == 0 {
		return 0
	}
	return dot / (math.Sqrt(aa) * math.Sqrt(bb))
}
func lexical(text string, d model.KnowledgeDoc) float64 {
	q := tokens(text)
	hay := tokens(d.Title + " " + d.Text + " " + strings.Join(d.Keywords, " "))
	if len(q) == 0 {
		return 0
	}
	hits := 0
	for t := range q {
		if _, ok := hay[t]; ok {
			hits++
		}
	}
	return float64(hits) / float64(len(q))
}
func tokens(s string) map[string]struct{} {
	m := map[string]struct{}{}
	for _, p := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) {
		if len(p) >= 3 {
			m[p] = struct{}{}
		}
	}
	return m
}