Update Chunking der KBs
All checks were successful
release-tag / release-image (push) Successful in 1m44s
All checks were successful
release-tag / release-image (push) Successful in 1m44s
This commit is contained in:
12
.env.example
12
.env.example
@@ -49,7 +49,17 @@ OLLAMA_MAX_CONCURRENT=1
|
||||
KNOWLEDGE_DIR=./knowledge
|
||||
RAG_ENABLED=true
|
||||
KNOWLEDGE_TOP_K=3
|
||||
KNOWLEDGE_MIN_SCORE=0.88
|
||||
# Hybrid relevance score (not a probability). Recommended starting point: 0.70.
|
||||
KNOWLEDGE_MIN_SCORE=0.70
|
||||
# Hybrid ranking weights. Missing metadata is not penalized; available weights are re-normalized.
|
||||
KNOWLEDGE_WEIGHT_SEMANTIC=0.50
|
||||
KNOWLEDGE_WEIGHT_TITLE=0.25
|
||||
KNOWLEDGE_WEIGHT_KEYWORDS=0.15
|
||||
KNOWLEDGE_WEIGHT_CATEGORY=0.10
|
||||
# Long KB bodies are embedded as overlapping chunks; the best matching chunk is used.
|
||||
KNOWLEDGE_CHUNK_WORDS=160
|
||||
KNOWLEDGE_CHUNK_OVERLAP_WORDS=30
|
||||
KNOWLEDGE_MAX_CHUNKS_PER_DOC=24
|
||||
CATEGORY_PROMPT_LIMIT=80
|
||||
# Fail-closed source policy. Only documents carrying one of these source labels are indexed/searched.
|
||||
KNOWLEDGE_ALLOWED_SOURCES=internal-kb,glpi-kb
|
||||
|
||||
21
README.md
21
README.md
@@ -186,6 +186,25 @@ Ein Auto-Reply ist nur erlaubt, wenn `language` und `communication_style` des fr
|
||||
|
||||
Bei aktiviertem RAG erzeugt Ollama Embeddings über `/api/embed`; der Cache landet in `data/embeddings.json`. Für Ticket und Knowledge wird dasselbe Embedding-Modell verwendet.
|
||||
|
||||
### Realistisches Hybrid-Scoring
|
||||
|
||||
Knowledge-Treffer werden nicht mehr nur über eine einzelne Cosine-Similarity bewertet. Lange Artikel werden in überlappende Abschnitte zerlegt und der beste semantische Abschnitt wird mit Titel-, Keyword- und Kategorie-/Lernsignalen kombiniert. Standardgewichte:
|
||||
|
||||
```env
|
||||
KNOWLEDGE_MIN_SCORE=0.70
|
||||
KNOWLEDGE_WEIGHT_SEMANTIC=0.50
|
||||
KNOWLEDGE_WEIGHT_TITLE=0.25
|
||||
KNOWLEDGE_WEIGHT_KEYWORDS=0.15
|
||||
KNOWLEDGE_WEIGHT_CATEGORY=0.10
|
||||
KNOWLEDGE_CHUNK_WORDS=160
|
||||
KNOWLEDGE_CHUNK_OVERLAP_WORDS=30
|
||||
KNOWLEDGE_MAX_CHUNKS_PER_DOC=24
|
||||
```
|
||||
|
||||
Der angezeigte Hybrid-Score ist **keine Wahrscheinlichkeit**. Er ist ein nachvollziehbarer Ranking-Score. Die Semantik verwendet die Ähnlichkeit des besten Body-Chunks; der Titel kombiniert Embedding- und exakten/lexikalischen Titelmatch; Keywords werden explizit gegen den Tickettext geprüft. Ist ein Knowledge-Dokument GLPI-ITIL-Kategorien zugeordnet, fließen deren Namen, semantische Hints und menschlich bestätigte Lernbeispiele als Kategorie-Signal ein. Fehlen einem Artikel Keywords oder Kategoriezuordnungen, wird er nicht pauschal abgestraft: Nur vorhandene Komponenten werden in die Gewichtung aufgenommen.
|
||||
|
||||
Im Audit-Dashboard werden `Hybrid`, `Semantik`, `Titel`, `Keywords`, `Kategorie/Lernen`, der beste gefundene Abschnitt und der tatsächlich erforderliche KB-Schwellwert getrennt angezeigt. Der effektive Schwellwert bleibt `max(KNOWLEDGE_MIN_SCORE, min_score des Artikels)`.
|
||||
|
||||
## Operativer Kontext: Changes, Major Incidents, Uptime Kuma und Geräte
|
||||
|
||||
Der Agent kann vor der LLM-Entscheidung zusätzliche **read-only** Betriebsdaten einsammeln. Diese Daten werden normalisiert und als Fakten in den Prompt aufgenommen; das Modell erhält keine direkten Zugangsdaten und keine zusätzlichen Schreibwerkzeuge.
|
||||
@@ -294,7 +313,7 @@ Das Dashboard zeigt deshalb unter anderem:
|
||||
- KI-Confidence und konfigurierten Schwellwert,
|
||||
- expliziten Entscheidungsgrund wie `category_confidence_below_threshold`, `category_already_correct` oder `category_written`,
|
||||
- KI-Empfehlung für Auto-Reply samt Confidence und Reply-Schwellwert,
|
||||
- besten Knowledge-Treffer mit Score,
|
||||
- besten Knowledge-Treffer mit Hybrid-Score, Einzelkomponenten, effektivem Schwellwert und bestem Artikelabschnitt,
|
||||
- den ersten Policy-Blocker für einen Reply, z. B. fehlendes Knowledge, vorhandenes Followup, unvollständigen Kontext oder einen relevanten Incident,
|
||||
- die fachliche KI-Begründung separat von den technischen Policy-Codes.
|
||||
|
||||
|
||||
19
UPGRADE.md
19
UPGRADE.md
@@ -58,3 +58,22 @@ GLPI_KB_AUTO_REPLY_CATEGORY_IDS=
|
||||
```
|
||||
|
||||
Der sichere Start ist `GLPI_KB_AUTO_REPLY=false`. Erst nachdem die importierten Artikel im Dashboard geprüft wurden, sollte `glpi-kb` optional in `KNOWLEDGE_AUTO_REPLY_SOURCES` aufgenommen und eine explizite Whitelist von GLPI-Knowledge-Base-Kategorie-IDs gesetzt werden.
|
||||
|
||||
## Hybrid Knowledge Scoring
|
||||
|
||||
Diese Version ersetzt den einzelnen Dokument-Cosine-Score durch ein Hybrid-Scoring mit Body-Chunks, Titel, Keywords und Kategorie-/Lernsignalen. Der bestehende `data/embeddings.json` Cache wird bei Bedarf automatisch im neuen Format aufgebaut; ein manuelles Löschen ist nicht erforderlich.
|
||||
|
||||
Für bestehende `.env`-Dateien werden folgende Werte empfohlen:
|
||||
|
||||
```env
|
||||
KNOWLEDGE_MIN_SCORE=0.70
|
||||
KNOWLEDGE_WEIGHT_SEMANTIC=0.50
|
||||
KNOWLEDGE_WEIGHT_TITLE=0.25
|
||||
KNOWLEDGE_WEIGHT_KEYWORDS=0.15
|
||||
KNOWLEDGE_WEIGHT_CATEGORY=0.10
|
||||
KNOWLEDGE_CHUNK_WORDS=160
|
||||
KNOWLEDGE_CHUNK_OVERLAP_WORDS=30
|
||||
KNOWLEDGE_MAX_CHUNKS_PER_DOC=24
|
||||
```
|
||||
|
||||
Der neue Hybrid-Score ist nicht direkt mit alten Cosine-Scores vergleichbar. Nach dem Upgrade zunächst im Dry-Run beobachten und den Mindestscore anhand realer Tickets kalibrieren.
|
||||
|
||||
@@ -68,7 +68,10 @@ func main() {
|
||||
slog.Error("state store initialization failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
k, err := knowledge.Load(ctx, cfg.KnowledgeDir, cfg.DataDir, o, cfg.RAGEnabled, cfg.KnowledgeAllowedSources)
|
||||
k, err := knowledge.Load(ctx, cfg.KnowledgeDir, cfg.DataDir, o, cfg.RAGEnabled, cfg.KnowledgeAllowedSources, knowledge.ScoringConfig{
|
||||
SemanticWeight: cfg.KnowledgeSemanticWeight, TitleWeight: cfg.KnowledgeTitleWeight, KeywordWeight: cfg.KnowledgeKeywordWeight, CategoryWeight: cfg.KnowledgeCategoryWeight,
|
||||
ChunkWords: cfg.KnowledgeChunkWords, ChunkOverlap: cfg.KnowledgeChunkOverlapWords, MaxChunksPerDoc: cfg.KnowledgeMaxChunksPerDoc,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("knowledge store initialization failed",
|
||||
"error", err,
|
||||
|
||||
@@ -186,7 +186,7 @@ func (s *Service) Process(ctx context.Context, id int64) error {
|
||||
}
|
||||
run.CategoryBeforeName = categoryName(categories, t.CategoryID)
|
||||
promptCats := shortlistCategories(t, categories, s.cfg.CategoryPromptLimit)
|
||||
hits, err := s.knowledge.Search(ctx, t.Name+"\n"+stripHTML(t.Content), s.cfg.KnowledgeTopK)
|
||||
hits, err := s.knowledge.Search(ctx, t.Name+"\n"+stripHTML(t.Content), s.cfg.KnowledgeTopK, categories)
|
||||
if err != nil {
|
||||
run.Reason = "knowledge_search_failed"
|
||||
finish(err)
|
||||
@@ -196,6 +196,15 @@ func (s *Service) Process(ctx context.Context, id int64) error {
|
||||
run.KnowledgeTopID = hits[0].Doc.ID
|
||||
run.KnowledgeTopTitle = hits[0].Doc.Title
|
||||
run.KnowledgeScore = hits[0].Score
|
||||
run.KnowledgeSemanticScore = hits[0].SemanticScore
|
||||
run.KnowledgeTitleScore = hits[0].TitleScore
|
||||
run.KnowledgeKeywordScore = hits[0].KeywordScore
|
||||
run.KnowledgeCategoryScore = hits[0].CategoryScore
|
||||
run.KnowledgeBestChunk = hits[0].BestChunkExcerpt
|
||||
run.KnowledgeThreshold = s.cfg.KnowledgeMinScore
|
||||
if hits[0].Doc.MinScore > run.KnowledgeThreshold {
|
||||
run.KnowledgeThreshold = hits[0].Doc.MinScore
|
||||
}
|
||||
}
|
||||
contextData := model.ContextSnapshot{}
|
||||
if s.context != nil && s.cfg.ContextEnabled {
|
||||
@@ -238,6 +247,9 @@ func (s *Service) Process(ctx context.Context, id int64) error {
|
||||
run.ReplyDecision = result.ReplyDecision
|
||||
run.ReplyProposed = result.Reply
|
||||
run.KnowledgeID = result.KnowledgeID
|
||||
if result.KnowledgeThreshold > 0 {
|
||||
run.KnowledgeThreshold = result.KnowledgeThreshold
|
||||
}
|
||||
run.PolicyReason = result.CategoryDecision + "; " + result.ReplyDecision
|
||||
if !canReply && result.Reply {
|
||||
run.ReplyProposed = false
|
||||
|
||||
@@ -133,6 +133,7 @@ func (p Policy) Evaluate(t model.Ticket, d model.Decision, categories []model.Ca
|
||||
if hit.Doc.MinScore > threshold {
|
||||
threshold = hit.Doc.MinScore
|
||||
}
|
||||
res.KnowledgeThreshold = threshold
|
||||
if !hit.Doc.AutoReply {
|
||||
res.ReplyDecision = "reply_knowledge_auto_reply_disabled"
|
||||
return res, nil
|
||||
|
||||
@@ -52,6 +52,13 @@ type Config struct {
|
||||
KnowledgeAllowedSources []string
|
||||
KnowledgeAutoReplySources []string
|
||||
KnowledgeWebEditEnabled bool
|
||||
KnowledgeSemanticWeight float64
|
||||
KnowledgeTitleWeight float64
|
||||
KnowledgeKeywordWeight float64
|
||||
KnowledgeCategoryWeight float64
|
||||
KnowledgeChunkWords int
|
||||
KnowledgeChunkOverlapWords int
|
||||
KnowledgeMaxChunksPerDoc int
|
||||
GLPIKBEnabled bool
|
||||
GLPIKBPath string
|
||||
GLPIKBFilter string
|
||||
@@ -147,6 +154,13 @@ func Load() (Config, error) {
|
||||
KnowledgeAllowedSources: envStringList("KNOWLEDGE_ALLOWED_SOURCES", "internal-kb"),
|
||||
KnowledgeAutoReplySources: envStringList("KNOWLEDGE_AUTO_REPLY_SOURCES", "internal-kb"),
|
||||
KnowledgeWebEditEnabled: envBool("KNOWLEDGE_WEB_EDIT_ENABLED", false),
|
||||
KnowledgeSemanticWeight: envFloat("KNOWLEDGE_WEIGHT_SEMANTIC", 0.50),
|
||||
KnowledgeTitleWeight: envFloat("KNOWLEDGE_WEIGHT_TITLE", 0.25),
|
||||
KnowledgeKeywordWeight: envFloat("KNOWLEDGE_WEIGHT_KEYWORDS", 0.15),
|
||||
KnowledgeCategoryWeight: envFloat("KNOWLEDGE_WEIGHT_CATEGORY", 0.10),
|
||||
KnowledgeChunkWords: envInt("KNOWLEDGE_CHUNK_WORDS", 160),
|
||||
KnowledgeChunkOverlapWords: envInt("KNOWLEDGE_CHUNK_OVERLAP_WORDS", 30),
|
||||
KnowledgeMaxChunksPerDoc: envInt("KNOWLEDGE_MAX_CHUNKS_PER_DOC", 24),
|
||||
GLPIKBEnabled: envBool("GLPI_KB_ENABLED", false),
|
||||
GLPIKBPath: env("GLPI_KB_PATH", "auto"),
|
||||
GLPIKBFilter: strings.TrimSpace(os.Getenv("GLPI_KB_FILTER")),
|
||||
@@ -167,7 +181,7 @@ func Load() (Config, error) {
|
||||
AutoReply: envBool("AUTO_REPLY", false),
|
||||
CategoryConfidence: envFloat("CATEGORY_CONFIDENCE", 0.90),
|
||||
ReplyConfidence: envFloat("REPLY_CONFIDENCE", 0.97),
|
||||
KnowledgeMinScore: envFloat("KNOWLEDGE_MIN_SCORE", 0.88),
|
||||
KnowledgeMinScore: envFloat("KNOWLEDGE_MIN_SCORE", 0.70),
|
||||
|
||||
ContextEnabled: envBool("CONTEXT_ENABLED", true),
|
||||
ContextTimeout: envDuration("CONTEXT_TIMEOUT", 12*time.Second),
|
||||
@@ -266,6 +280,26 @@ func (c Config) Validate() error {
|
||||
if c.KnowledgeWebEditEnabled && c.WebAllowAnonymous {
|
||||
return errors.New("KNOWLEDGE_WEB_EDIT_ENABLED requires authenticated dashboard access; WEB_ALLOW_ANONYMOUS must be false")
|
||||
}
|
||||
weights := []float64{c.KnowledgeSemanticWeight, c.KnowledgeTitleWeight, c.KnowledgeKeywordWeight, c.KnowledgeCategoryWeight}
|
||||
weightSum := 0.0
|
||||
for _, w := range weights {
|
||||
if w < 0 || w > 1 {
|
||||
return errors.New("KNOWLEDGE_WEIGHT_* values must be between 0 and 1")
|
||||
}
|
||||
weightSum += w
|
||||
}
|
||||
// All-zero values are allowed for Config values constructed directly in tests/embedders;
|
||||
// the knowledge store then applies its safe defaults. Values loaded from ENV are explicit.
|
||||
_ = weightSum
|
||||
if c.KnowledgeChunkWords != 0 && (c.KnowledgeChunkWords < 40 || c.KnowledgeChunkWords > 1000) {
|
||||
return errors.New("KNOWLEDGE_CHUNK_WORDS must be between 40 and 1000")
|
||||
}
|
||||
if c.KnowledgeChunkOverlapWords < 0 || (c.KnowledgeChunkWords > 0 && c.KnowledgeChunkOverlapWords >= c.KnowledgeChunkWords) {
|
||||
return errors.New("KNOWLEDGE_CHUNK_OVERLAP_WORDS must be >= 0 and smaller than KNOWLEDGE_CHUNK_WORDS")
|
||||
}
|
||||
if c.KnowledgeMaxChunksPerDoc != 0 && (c.KnowledgeMaxChunksPerDoc < 1 || c.KnowledgeMaxChunksPerDoc > 100) {
|
||||
return errors.New("KNOWLEDGE_MAX_CHUNKS_PER_DOC must be between 1 and 100")
|
||||
}
|
||||
if c.LearningEnabled {
|
||||
if c.LearningMaxExamples < 1 || c.LearningMaxExamples > 10000 {
|
||||
return errors.New("LEARNING_MAX_EXAMPLES must be between 1 and 10000")
|
||||
|
||||
@@ -159,3 +159,17 @@ func TestValidateGLPIKBAutoReplyRequiresCategoryWhitelist(t *testing.T) {
|
||||
t.Fatalf("expected explicit category whitelist to validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidKnowledgeScoringConfig(t *testing.T) {
|
||||
c := validConfig()
|
||||
c.KnowledgeSemanticWeight = 1.2
|
||||
if err := c.Validate(); err == nil {
|
||||
t.Fatal("expected knowledge weight > 1 to be rejected")
|
||||
}
|
||||
c = validConfig()
|
||||
c.KnowledgeChunkWords = 100
|
||||
c.KnowledgeChunkOverlapWords = 100
|
||||
if err := c.Validate(); err == nil {
|
||||
t.Fatal("expected overlap >= chunk size to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,16 @@ import (
|
||||
type Embedder interface {
|
||||
Embed(context.Context, []string) ([][]float64, error)
|
||||
}
|
||||
type ScoringConfig struct {
|
||||
SemanticWeight float64
|
||||
TitleWeight float64
|
||||
KeywordWeight float64
|
||||
CategoryWeight float64
|
||||
ChunkWords int
|
||||
ChunkOverlap int
|
||||
MaxChunksPerDoc int
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
mu sync.RWMutex
|
||||
dir string
|
||||
@@ -29,23 +39,53 @@ type Store struct {
|
||||
managed map[string]bool
|
||||
external map[string]string
|
||||
staticDocs map[string]model.KnowledgeDoc
|
||||
vectors map[string][]float64
|
||||
titleVectors map[string][]float64
|
||||
chunkVectors map[string][][]float64
|
||||
chunks map[string][]string
|
||||
embedder Embedder
|
||||
rag bool
|
||||
cachePath string
|
||||
allowedSources map[string]struct{}
|
||||
scoring ScoringConfig
|
||||
}
|
||||
type cacheFile struct {
|
||||
Hashes map[string]string `json:"hashes"`
|
||||
Vectors map[string][]float64 `json:"vectors"`
|
||||
Version int `json:"version,omitempty"`
|
||||
Hashes map[string]string `json:"hashes"`
|
||||
TitleVectors map[string][]float64 `json:"title_vectors,omitempty"`
|
||||
ChunkVectors map[string][][]float64 `json:"chunk_vectors,omitempty"`
|
||||
}
|
||||
|
||||
func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool, allowedSources []string) (*Store, error) {
|
||||
func DefaultScoringConfig() ScoringConfig {
|
||||
return ScoringConfig{SemanticWeight: .50, TitleWeight: .25, KeywordWeight: .15, CategoryWeight: .10, ChunkWords: 160, ChunkOverlap: 30, MaxChunksPerDoc: 24}
|
||||
}
|
||||
|
||||
func normalizeScoring(c ScoringConfig) ScoringConfig {
|
||||
d := DefaultScoringConfig()
|
||||
if c.SemanticWeight < 0 || c.TitleWeight < 0 || c.KeywordWeight < 0 || c.CategoryWeight < 0 || c.SemanticWeight+c.TitleWeight+c.KeywordWeight+c.CategoryWeight <= 0 {
|
||||
c.SemanticWeight, c.TitleWeight, c.KeywordWeight, c.CategoryWeight = d.SemanticWeight, d.TitleWeight, d.KeywordWeight, d.CategoryWeight
|
||||
}
|
||||
if c.ChunkWords <= 0 {
|
||||
c.ChunkWords = d.ChunkWords
|
||||
}
|
||||
if c.ChunkOverlap < 0 || c.ChunkOverlap >= c.ChunkWords {
|
||||
c.ChunkOverlap = d.ChunkOverlap
|
||||
}
|
||||
if c.MaxChunksPerDoc <= 0 {
|
||||
c.MaxChunksPerDoc = d.MaxChunksPerDoc
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func Load(ctx context.Context, dir, dataDir string, embedder Embedder, rag bool, allowedSources []string, scoring ...ScoringConfig) (*Store, error) {
|
||||
managedDir := filepath.Join(dataDir, "knowledge-managed")
|
||||
if err := os.MkdirAll(managedDir, 0o750); err != nil {
|
||||
return nil, fmt.Errorf("create managed knowledge directory: %w", err)
|
||||
}
|
||||
s := &Store{dir: dir, managedDir: managedDir, vectors: map[string][]float64{}, files: map[string]string{}, managed: map[string]bool{}, external: map[string]string{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}}
|
||||
scoreCfg := DefaultScoringConfig()
|
||||
if len(scoring) > 0 {
|
||||
scoreCfg = normalizeScoring(scoring[0])
|
||||
}
|
||||
s := &Store{dir: dir, managedDir: managedDir, titleVectors: map[string][]float64{}, chunkVectors: map[string][][]float64{}, chunks: map[string][]string{}, files: map[string]string{}, managed: map[string]bool{}, external: map[string]string{}, staticDocs: map[string]model.KnowledgeDoc{}, embedder: embedder, rag: rag, cachePath: filepath.Join(dataDir, "embeddings.json"), allowedSources: map[string]struct{}{}, scoring: scoreCfg}
|
||||
for _, source := range allowedSources {
|
||||
s.allowedSources[strings.ToLower(strings.TrimSpace(source))] = struct{}{}
|
||||
}
|
||||
@@ -204,20 +244,22 @@ func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
|
||||
if exists && !isManaged {
|
||||
return fmt.Errorf("static knowledge entry %q is read-only; use a new id for a managed entry", d.ID)
|
||||
}
|
||||
var vector []float64
|
||||
|
||||
var titleVector []float64
|
||||
var chunkVectors [][]float64
|
||||
chunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
|
||||
if s.rag {
|
||||
if s.embedder == nil {
|
||||
return fmt.Errorf("RAG is enabled but no embedding provider is configured")
|
||||
}
|
||||
vv, err := s.embedder.Embed(ctx, []string{d.Title + "\n" + d.Text + "\n" + strings.Join(d.Keywords, " ")})
|
||||
embedded, err := s.embedDocuments(ctx, []model.KnowledgeDoc{d})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(vv) != 1 || len(vv[0]) == 0 {
|
||||
return fmt.Errorf("embedding provider returned no vector")
|
||||
}
|
||||
vector = vv[0]
|
||||
titleVector = embedded[d.ID].title
|
||||
chunkVectors = embedded[d.ID].chunks
|
||||
}
|
||||
|
||||
path := filepath.Join(s.managedDir, d.ID+".json")
|
||||
b, err := json.MarshalIndent(d, "", " ")
|
||||
if err != nil {
|
||||
@@ -232,7 +274,6 @@ func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
replaced := false
|
||||
for i := range s.docs {
|
||||
if s.docs[i].ID == d.ID {
|
||||
@@ -246,10 +287,13 @@ func (s *Store) Upsert(ctx context.Context, d model.KnowledgeDoc) error {
|
||||
}
|
||||
s.files[d.ID] = path
|
||||
s.managed[d.ID] = true
|
||||
s.chunks[d.ID] = chunks
|
||||
if s.rag {
|
||||
s.vectors[d.ID] = vector
|
||||
s.titleVectors[d.ID] = titleVector
|
||||
s.chunkVectors[d.ID] = chunkVectors
|
||||
}
|
||||
return nil
|
||||
s.mu.Unlock()
|
||||
return s.persistVectorCache()
|
||||
}
|
||||
|
||||
func (s *Store) Delete(id string) error {
|
||||
@@ -274,7 +318,6 @@ func (s *Store) Delete(id string) error {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := s.docs[:0]
|
||||
for _, d := range s.docs {
|
||||
if d.ID != id {
|
||||
@@ -284,11 +327,21 @@ func (s *Store) Delete(id string) error {
|
||||
s.docs = append([]model.KnowledgeDoc(nil), out...)
|
||||
delete(s.files, id)
|
||||
delete(s.managed, id)
|
||||
delete(s.vectors, id)
|
||||
return nil
|
||||
delete(s.titleVectors, id)
|
||||
delete(s.chunkVectors, id)
|
||||
delete(s.chunks, id)
|
||||
s.mu.Unlock()
|
||||
return s.persistVectorCache()
|
||||
}
|
||||
|
||||
func (s *Store) IsManaged(id string) bool { s.mu.RLock(); defer s.mu.RUnlock(); return s.managed[id] }
|
||||
func (s *Store) IsManaged(id string) bool {
|
||||
if s == nil {
|
||||
return false
|
||||
}
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.managed[id]
|
||||
}
|
||||
func (s *Store) Origin(id string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
@@ -308,9 +361,8 @@ func (s *Store) Origin(id string) string {
|
||||
}
|
||||
|
||||
// ReplaceExternalSource atomically replaces all read-only documents imported
|
||||
// from one connector source. Existing vectors are reused when the normalized
|
||||
// document did not change, so periodic synchronization does not re-embed the
|
||||
// whole GLPI knowledge base on every run.
|
||||
// from one connector source. Embeddings are reused when the normalized article
|
||||
// did not change. Long article bodies are indexed as overlapping chunks.
|
||||
func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs []model.KnowledgeDoc) error {
|
||||
if s == nil {
|
||||
return fmt.Errorf("knowledge store is not initialized")
|
||||
@@ -322,18 +374,13 @@ func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs [
|
||||
|
||||
s.mu.RLock()
|
||||
oldDocs := make(map[string]model.KnowledgeDoc, len(s.docs))
|
||||
oldVectors := make(map[string][]float64, len(s.vectors))
|
||||
oldTitle := cloneVectorMap(s.titleVectors)
|
||||
oldChunks := cloneChunkVectorMap(s.chunkVectors)
|
||||
for _, d := range s.docs {
|
||||
oldDocs[d.ID] = d
|
||||
}
|
||||
for id, v := range s.vectors {
|
||||
oldVectors[id] = append([]float64(nil), v...)
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
cached := cacheFile{Hashes: map[string]string{}, Vectors: map[string][]float64{}}
|
||||
if b, err := os.ReadFile(s.cachePath); err == nil {
|
||||
_ = json.Unmarshal(b, &cached)
|
||||
}
|
||||
cached := loadCache(s.cachePath)
|
||||
|
||||
changed := make([]model.KnowledgeDoc, 0)
|
||||
seen := map[string]struct{}{}
|
||||
@@ -356,42 +403,33 @@ func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs [
|
||||
}
|
||||
seen[d.ID] = struct{}{}
|
||||
h := hashDoc(*d)
|
||||
old, ok := oldDocs[d.ID]
|
||||
same := ok && hashDoc(old) == h && len(oldVectors[d.ID]) > 0
|
||||
if !same && cached.Hashes[d.ID] == h && len(cached.Vectors[d.ID]) > 0 {
|
||||
oldVectors[d.ID] = append([]float64(nil), cached.Vectors[d.ID]...)
|
||||
bodyChunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
|
||||
same := false
|
||||
if old, ok := oldDocs[d.ID]; ok && hashDoc(old) == h && len(oldTitle[d.ID]) > 0 && len(oldChunks[d.ID]) == len(bodyChunks) {
|
||||
same = true
|
||||
} else if cached.Hashes[d.ID] == h && len(cached.TitleVectors[d.ID]) > 0 && len(cached.ChunkVectors[d.ID]) == len(bodyChunks) {
|
||||
oldTitle[d.ID] = append([]float64(nil), cached.TitleVectors[d.ID]...)
|
||||
oldChunks[d.ID] = cloneChunkVectors(cached.ChunkVectors[d.ID])
|
||||
same = true
|
||||
}
|
||||
if !same {
|
||||
changed = append(changed, *d)
|
||||
}
|
||||
}
|
||||
newVectors := map[string][]float64{}
|
||||
|
||||
newEmbedded := map[string]embeddedDoc{}
|
||||
if s.rag && len(changed) > 0 {
|
||||
if s.embedder == nil {
|
||||
return fmt.Errorf("RAG is enabled but no embedding provider is configured")
|
||||
}
|
||||
texts := make([]string, len(changed))
|
||||
for i, d := range changed {
|
||||
texts[i] = d.Title + "\n" + d.Text + "\n" + strings.Join(d.Keywords, " ")
|
||||
}
|
||||
vv, err := s.embedder.Embed(ctx, texts)
|
||||
var err error
|
||||
newEmbedded, err = s.embedDocuments(ctx, changed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(vv) != len(changed) {
|
||||
return fmt.Errorf("embedding provider returned %d vectors for %d documents", len(vv), len(changed))
|
||||
}
|
||||
for i, d := range changed {
|
||||
if len(vv[i]) == 0 {
|
||||
return fmt.Errorf("embedding provider returned empty vector for %s", d.ID)
|
||||
}
|
||||
newVectors[d.ID] = vv[i]
|
||||
}
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
// Reject collisions with local/static documents.
|
||||
for _, d := range docs {
|
||||
if src := s.external[d.ID]; src == "" {
|
||||
if _, exists := oldDocs[d.ID]; exists {
|
||||
@@ -412,16 +450,21 @@ func (s *Store) ReplaceExternalSource(ctx context.Context, source string, docs [
|
||||
for id, src := range s.external {
|
||||
if src == source {
|
||||
delete(s.external, id)
|
||||
delete(s.vectors, id)
|
||||
delete(s.titleVectors, id)
|
||||
delete(s.chunkVectors, id)
|
||||
delete(s.chunks, id)
|
||||
}
|
||||
}
|
||||
for _, d := range docs {
|
||||
rebuilt = append(rebuilt, d)
|
||||
s.external[d.ID] = source
|
||||
if v := newVectors[d.ID]; len(v) > 0 {
|
||||
s.vectors[d.ID] = v
|
||||
} else if v := oldVectors[d.ID]; len(v) > 0 {
|
||||
s.vectors[d.ID] = v
|
||||
s.chunks[d.ID] = chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
|
||||
if e, ok := newEmbedded[d.ID]; ok {
|
||||
s.titleVectors[d.ID] = e.title
|
||||
s.chunkVectors[d.ID] = e.chunks
|
||||
} else {
|
||||
s.titleVectors[d.ID] = oldTitle[d.ID]
|
||||
s.chunkVectors[d.ID] = oldChunks[d.ID]
|
||||
}
|
||||
}
|
||||
s.docs = rebuilt
|
||||
@@ -434,12 +477,14 @@ func (s *Store) persistVectorCache() error {
|
||||
return nil
|
||||
}
|
||||
s.mu.RLock()
|
||||
cf := cacheFile{Hashes: map[string]string{}, Vectors: map[string][]float64{}}
|
||||
cf := cacheFile{Version: 2, Hashes: map[string]string{}, TitleVectors: map[string][]float64{}, ChunkVectors: map[string][][]float64{}}
|
||||
for _, d := range s.docs {
|
||||
if v := s.vectors[d.ID]; len(v) > 0 {
|
||||
cf.Hashes[d.ID] = hashDoc(d)
|
||||
cf.Vectors[d.ID] = append([]float64(nil), v...)
|
||||
if len(s.titleVectors[d.ID]) == 0 {
|
||||
continue
|
||||
}
|
||||
cf.Hashes[d.ID] = hashDoc(d)
|
||||
cf.TitleVectors[d.ID] = append([]float64(nil), s.titleVectors[d.ID]...)
|
||||
cf.ChunkVectors[d.ID] = cloneChunkVectors(s.chunkVectors[d.ID])
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
b, err := json.MarshalIndent(cf, "", " ")
|
||||
@@ -452,6 +497,7 @@ func (s *Store) persistVectorCache() error {
|
||||
}
|
||||
return os.Rename(tmp, s.cachePath)
|
||||
}
|
||||
|
||||
func (s *Store) ManagedDir() string {
|
||||
if s == nil {
|
||||
return ""
|
||||
@@ -470,86 +516,377 @@ func safeID(v string) bool {
|
||||
return !strings.Contains(v, "..")
|
||||
}
|
||||
|
||||
func (s *Store) Search(ctx context.Context, text string, topK int) ([]model.KnowledgeHit, error) {
|
||||
// Search calculates a transparent hybrid relevance score. Embedding similarity
|
||||
// is only one component; titles, explicit keywords and category/learning hints
|
||||
// are scored separately. Missing metadata does not lower a document's score:
|
||||
// the weights of available components are normalized dynamically.
|
||||
func (s *Store) Search(ctx context.Context, text string, topK int, categorySets ...[]model.Category) ([]model.KnowledgeHit, error) {
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("knowledge store is not initialized")
|
||||
}
|
||||
s.mu.RLock()
|
||||
docs := append([]model.KnowledgeDoc(nil), s.docs...)
|
||||
vecs := make(map[string][]float64, len(s.vectors))
|
||||
for k, v := range s.vectors {
|
||||
vecs[k] = v
|
||||
}
|
||||
titleVecs := cloneVectorMap(s.titleVectors)
|
||||
chunkVecs := cloneChunkVectorMap(s.chunkVectors)
|
||||
chunks := cloneStringSliceMap(s.chunks)
|
||||
scoreCfg := s.scoring
|
||||
s.mu.RUnlock()
|
||||
if len(docs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
scores := map[string]float64{}
|
||||
if s.rag && s.embedder != nil && len(vecs) > 0 {
|
||||
var cats []model.Category
|
||||
if len(categorySets) > 0 {
|
||||
cats = categorySets[0]
|
||||
}
|
||||
|
||||
var queryVector []float64
|
||||
if s.rag && s.embedder != nil {
|
||||
q, err := s.embedder.Embed(ctx, []string{text})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(q) > 0 {
|
||||
for _, d := range docs {
|
||||
scores[d.ID] = cosine(q[0], vecs[d.ID])
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, d := range docs {
|
||||
scores[d.ID] = lexical(text, d)
|
||||
queryVector = q[0]
|
||||
}
|
||||
}
|
||||
|
||||
hits := make([]model.KnowledgeHit, 0, len(docs))
|
||||
for _, d := range docs {
|
||||
hits = append(hits, model.KnowledgeHit{Doc: d, Score: scores[d.ID]})
|
||||
semantic, bestChunk := 0.0, ""
|
||||
semanticAvailable := false
|
||||
if len(queryVector) > 0 && len(chunkVecs[d.ID]) > 0 {
|
||||
semanticAvailable = true
|
||||
for i, v := range chunkVecs[d.ID] {
|
||||
score := clamp01(cosine(queryVector, v))
|
||||
if score > semantic || bestChunk == "" {
|
||||
semantic = score
|
||||
if i < len(chunks[d.ID]) {
|
||||
bestChunk = chunks[d.ID][i]
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if strings.TrimSpace(d.Text) != "" {
|
||||
semanticAvailable = true
|
||||
semantic = tokenF1(text, d.Text)
|
||||
bestChunk = d.Text
|
||||
}
|
||||
|
||||
title := 0.0
|
||||
titleAvailable := strings.TrimSpace(d.Title) != ""
|
||||
if titleAvailable {
|
||||
title = titleSimilarity(text, d.Title)
|
||||
if len(queryVector) > 0 && len(titleVecs[d.ID]) > 0 {
|
||||
title = math.Max(title, clamp01(cosine(queryVector, titleVecs[d.ID])))
|
||||
}
|
||||
}
|
||||
keyword, keywordAvailable := keywordSimilarity(text, d.Keywords)
|
||||
category, categoryAvailable := categorySimilarity(text, d.Categories, cats)
|
||||
total := weightedScore(scoreCfg,
|
||||
scorePart{semantic, scoreCfg.SemanticWeight, semanticAvailable},
|
||||
scorePart{title, scoreCfg.TitleWeight, titleAvailable},
|
||||
scorePart{keyword, scoreCfg.KeywordWeight, keywordAvailable},
|
||||
scorePart{category, scoreCfg.CategoryWeight, categoryAvailable},
|
||||
)
|
||||
hits = append(hits, model.KnowledgeHit{Doc: d, Score: total, SemanticScore: semantic, TitleScore: title, KeywordScore: keyword, CategoryScore: category, BestChunkExcerpt: excerpt(bestChunk, 280)})
|
||||
}
|
||||
sort.Slice(hits, func(i, j int) bool { return hits[i].Score > hits[j].Score })
|
||||
sort.SliceStable(hits, func(i, j int) bool {
|
||||
if hits[i].Score == hits[j].Score {
|
||||
return hits[i].TitleScore > hits[j].TitleScore
|
||||
}
|
||||
return hits[i].Score > hits[j].Score
|
||||
})
|
||||
if topK > 0 && len(hits) > topK {
|
||||
hits = hits[:topK]
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
func (s *Store) index(ctx context.Context) error {
|
||||
_ = os.MkdirAll(filepath.Dir(s.cachePath), 0o750)
|
||||
cf := cacheFile{Hashes: map[string]string{}, Vectors: map[string][]float64{}}
|
||||
if b, err := os.ReadFile(s.cachePath); err == nil {
|
||||
_ = json.Unmarshal(b, &cf)
|
||||
}
|
||||
cf := loadCache(s.cachePath)
|
||||
var need []model.KnowledgeDoc
|
||||
for _, d := range s.docs {
|
||||
bodyChunks := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
|
||||
s.chunks[d.ID] = bodyChunks
|
||||
h := hashDoc(d)
|
||||
if cf.Hashes[d.ID] == h && len(cf.Vectors[d.ID]) > 0 {
|
||||
s.vectors[d.ID] = cf.Vectors[d.ID]
|
||||
if cf.Hashes[d.ID] == h && len(cf.TitleVectors[d.ID]) > 0 && len(cf.ChunkVectors[d.ID]) == len(bodyChunks) {
|
||||
s.titleVectors[d.ID] = append([]float64(nil), cf.TitleVectors[d.ID]...)
|
||||
s.chunkVectors[d.ID] = cloneChunkVectors(cf.ChunkVectors[d.ID])
|
||||
} else {
|
||||
need = append(need, d)
|
||||
}
|
||||
}
|
||||
if len(need) > 0 {
|
||||
texts := make([]string, len(need))
|
||||
for i, d := range need {
|
||||
texts[i] = d.Title + "\n" + d.Text + "\n" + strings.Join(d.Keywords, " ")
|
||||
}
|
||||
vv, err := s.embedder.Embed(ctx, texts)
|
||||
embedded, err := s.embedDocuments(ctx, need)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i, d := range need {
|
||||
s.vectors[d.ID] = vv[i]
|
||||
cf.Hashes[d.ID] = hashDoc(d)
|
||||
cf.Vectors[d.ID] = vv[i]
|
||||
}
|
||||
b, _ := json.MarshalIndent(cf, "", " ")
|
||||
tmp := s.cachePath + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, s.cachePath); err != nil {
|
||||
return err
|
||||
for _, d := range need {
|
||||
s.titleVectors[d.ID] = embedded[d.ID].title
|
||||
s.chunkVectors[d.ID] = embedded[d.ID].chunks
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return s.persistVectorCache()
|
||||
}
|
||||
|
||||
type embeddedDoc struct {
|
||||
title []float64
|
||||
chunks [][]float64
|
||||
}
|
||||
|
||||
func (s *Store) embedDocuments(ctx context.Context, docs []model.KnowledgeDoc) (map[string]embeddedDoc, error) {
|
||||
out := make(map[string]embeddedDoc, len(docs))
|
||||
type ref struct {
|
||||
id string
|
||||
title bool
|
||||
chunk int
|
||||
}
|
||||
var texts []string
|
||||
var refs []ref
|
||||
for _, d := range docs {
|
||||
texts = append(texts, d.Title)
|
||||
refs = append(refs, ref{id: d.ID, title: true})
|
||||
parts := chunkText(d.Text, s.scoring.ChunkWords, s.scoring.ChunkOverlap, s.scoring.MaxChunksPerDoc)
|
||||
for i, part := range parts {
|
||||
texts = append(texts, part)
|
||||
refs = append(refs, ref{id: d.ID, chunk: i})
|
||||
}
|
||||
}
|
||||
vectors, err := s.embedTexts(ctx, texts, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(vectors) != len(refs) {
|
||||
return nil, fmt.Errorf("embedding provider returned %d vectors for %d inputs", len(vectors), len(refs))
|
||||
}
|
||||
for i, r := range refs {
|
||||
if len(vectors[i]) == 0 {
|
||||
return nil, fmt.Errorf("embedding provider returned empty vector for %s", r.id)
|
||||
}
|
||||
e := out[r.id]
|
||||
if r.title {
|
||||
e.title = vectors[i]
|
||||
} else {
|
||||
for len(e.chunks) <= r.chunk {
|
||||
e.chunks = append(e.chunks, nil)
|
||||
}
|
||||
e.chunks[r.chunk] = vectors[i]
|
||||
}
|
||||
out[r.id] = e
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Store) embedTexts(ctx context.Context, texts []string, batch int) ([][]float64, error) {
|
||||
if len(texts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if batch <= 0 {
|
||||
batch = 64
|
||||
}
|
||||
out := make([][]float64, 0, len(texts))
|
||||
for start := 0; start < len(texts); start += batch {
|
||||
end := start + batch
|
||||
if end > len(texts) {
|
||||
end = len(texts)
|
||||
}
|
||||
vv, err := s.embedder.Embed(ctx, texts[start:end])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(vv) != end-start {
|
||||
return nil, fmt.Errorf("embedding provider returned %d vectors for %d inputs", len(vv), end-start)
|
||||
}
|
||||
out = append(out, vv...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func loadCache(path string) cacheFile {
|
||||
cf := cacheFile{Version: 2, Hashes: map[string]string{}, TitleVectors: map[string][]float64{}, ChunkVectors: map[string][][]float64{}}
|
||||
if b, err := os.ReadFile(path); err == nil {
|
||||
_ = json.Unmarshal(b, &cf)
|
||||
}
|
||||
if cf.Hashes == nil {
|
||||
cf.Hashes = map[string]string{}
|
||||
}
|
||||
if cf.TitleVectors == nil {
|
||||
cf.TitleVectors = map[string][]float64{}
|
||||
}
|
||||
if cf.ChunkVectors == nil {
|
||||
cf.ChunkVectors = map[string][][]float64{}
|
||||
}
|
||||
return cf
|
||||
}
|
||||
|
||||
func chunkText(text string, words, overlap, maxChunks int) []string {
|
||||
parts := strings.Fields(strings.TrimSpace(text))
|
||||
if len(parts) == 0 {
|
||||
return nil
|
||||
}
|
||||
if words <= 0 {
|
||||
words = 160
|
||||
}
|
||||
if overlap < 0 || overlap >= words {
|
||||
overlap = 0
|
||||
}
|
||||
if maxChunks <= 0 {
|
||||
maxChunks = 24
|
||||
}
|
||||
step := words - overlap
|
||||
out := make([]string, 0, minInt(maxChunks, (len(parts)+step-1)/step))
|
||||
for start := 0; start < len(parts) && len(out) < maxChunks; start += step {
|
||||
end := start + words
|
||||
if end > len(parts) {
|
||||
end = len(parts)
|
||||
}
|
||||
out = append(out, strings.Join(parts[start:end], " "))
|
||||
if end == len(parts) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type scorePart struct {
|
||||
value, weight float64
|
||||
available bool
|
||||
}
|
||||
|
||||
func weightedScore(_ ScoringConfig, parts ...scorePart) float64 {
|
||||
var sum, weights float64
|
||||
for _, p := range parts {
|
||||
if !p.available || p.weight <= 0 {
|
||||
continue
|
||||
}
|
||||
sum += clamp01(p.value) * p.weight
|
||||
weights += p.weight
|
||||
}
|
||||
if weights == 0 {
|
||||
return 0
|
||||
}
|
||||
return clamp01(sum / weights)
|
||||
}
|
||||
|
||||
func titleSimilarity(query, title string) float64 {
|
||||
best := tokenF1(query, title)
|
||||
q := strings.ToLower(strings.Join(strings.Fields(query), " "))
|
||||
t := strings.ToLower(strings.Join(strings.Fields(title), " "))
|
||||
if t != "" && strings.Contains(q, t) {
|
||||
return 1
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func keywordSimilarity(query string, keywords []string) (float64, bool) {
|
||||
if len(keywords) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
best := tokenF1(query, strings.Join(keywords, " "))
|
||||
q := strings.ToLower(query)
|
||||
for _, kw := range keywords {
|
||||
kw = strings.ToLower(strings.TrimSpace(kw))
|
||||
if kw != "" && strings.Contains(q, kw) {
|
||||
best = math.Max(best, 1)
|
||||
}
|
||||
}
|
||||
return clamp01(best), true
|
||||
}
|
||||
|
||||
func categorySimilarity(query string, ids []int64, categories []model.Category) (float64, bool) {
|
||||
if len(ids) == 0 || len(categories) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
wanted := make(map[int64]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
wanted[id] = struct{}{}
|
||||
}
|
||||
best, found := 0.0, false
|
||||
for _, c := range categories {
|
||||
if _, ok := wanted[c.ID]; !ok {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
profileParts := []string{c.Name, c.CompleteName}
|
||||
profileParts = append(profileParts, c.Hints...)
|
||||
profileParts = append(profileParts, c.Examples...)
|
||||
profile := strings.Join(profileParts, " ")
|
||||
best = math.Max(best, tokenF1(query, profile))
|
||||
}
|
||||
return clamp01(best), found
|
||||
}
|
||||
|
||||
func tokenF1(a, b string) float64 {
|
||||
aTok, bTok := tokens(a), tokens(b)
|
||||
if len(aTok) == 0 || len(bTok) == 0 {
|
||||
return 0
|
||||
}
|
||||
common := 0
|
||||
for t := range aTok {
|
||||
if _, ok := bTok[t]; ok {
|
||||
common++
|
||||
}
|
||||
}
|
||||
if common == 0 {
|
||||
return 0
|
||||
}
|
||||
precision := float64(common) / float64(len(aTok))
|
||||
recall := float64(common) / float64(len(bTok))
|
||||
return 2 * precision * recall / (precision + recall)
|
||||
}
|
||||
|
||||
func excerpt(s string, max int) string {
|
||||
s = strings.Join(strings.Fields(s), " ")
|
||||
if len([]rune(s)) <= max {
|
||||
return s
|
||||
}
|
||||
r := []rune(s)
|
||||
return string(r[:max]) + "…"
|
||||
}
|
||||
func clamp01(v float64) float64 {
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
if v > 1 {
|
||||
return 1
|
||||
}
|
||||
return v
|
||||
}
|
||||
func cloneVectorMap(in map[string][]float64) map[string][]float64 {
|
||||
out := make(map[string][]float64, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = append([]float64(nil), v...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func cloneChunkVectorMap(in map[string][][]float64) map[string][][]float64 {
|
||||
out := make(map[string][][]float64, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = cloneChunkVectors(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func cloneChunkVectors(in [][]float64) [][]float64 {
|
||||
out := make([][]float64, len(in))
|
||||
for i, v := range in {
|
||||
out[i] = append([]float64(nil), v...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func cloneStringSliceMap(in map[string][]string) map[string][]string {
|
||||
out := make(map[string][]string, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = append([]string(nil), v...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func hashDoc(d model.KnowledgeDoc) string {
|
||||
b, _ := json.Marshal(d)
|
||||
h := sha256.Sum256(b)
|
||||
|
||||
@@ -2,6 +2,7 @@ package knowledge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -132,3 +133,71 @@ func TestExternalKnowledgeIsReadOnly(t *testing.T) {
|
||||
t.Fatal("expected external document to be read-only")
|
||||
}
|
||||
}
|
||||
|
||||
type semanticTestEmbedder struct{}
|
||||
|
||||
func (semanticTestEmbedder) Embed(_ context.Context, texts []string) ([][]float64, error) {
|
||||
out := make([][]float64, len(texts))
|
||||
for i, text := range texts {
|
||||
s := strings.ToLower(text)
|
||||
v := []float64{0, 0, 0, 0}
|
||||
if strings.Contains(s, "benutzerkonto") || strings.Contains(s, "konto gesperrt") || strings.Contains(s, "gesperrt") {
|
||||
v[0] = 1
|
||||
}
|
||||
if strings.Contains(s, "anmeld") || strings.Contains(s, "login") || strings.Contains(s, "authent") {
|
||||
v[1] = 1
|
||||
}
|
||||
if strings.Contains(s, "drucker") {
|
||||
v[2] = 1
|
||||
}
|
||||
if strings.Contains(s, "allgemein") || strings.Contains(s, "hinweis") {
|
||||
v[3] = 1
|
||||
}
|
||||
if v[0]+v[1]+v[2]+v[3] == 0 {
|
||||
v[3] = .1
|
||||
}
|
||||
out[i] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func TestHybridScoringUsesChunksTitleKeywordsAndCategoryHints(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
data := t.TempDir()
|
||||
body := strings.Repeat("Allgemeine technische Hinweise ohne Bezug zum Benutzer. ", 80) +
|
||||
" Wenn ein Benutzerkonto gesperrt ist und die Anmeldung nicht möglich ist, muss die Kontosperre geprüft werden. " +
|
||||
strings.Repeat("Weitere allgemeine Hinweise. ", 80)
|
||||
doc := model.KnowledgeDoc{ID: "KB-AD-1", Title: "Benutzerkonto gesperrt", Text: body, Answer: "x", Source: "internal-kb", Language: "de-DE", CommunicationStyle: "formal", Categories: []int64{2}, Keywords: []string{"Konto gesperrt", "Anmeldung", "Login"}}
|
||||
b, _ := json.Marshal(doc)
|
||||
if err := os.WriteFile(filepath.Join(dir, "ad.json"), b, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := Load(context.Background(), dir, data, semanticTestEmbedder{}, true, []string{"internal-kb"}, ScoringConfig{SemanticWeight: .5, TitleWeight: .25, KeywordWeight: .15, CategoryWeight: .10, ChunkWords: 40, ChunkOverlap: 10, MaxChunksPerDoc: 24})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cats := []model.Category{{ID: 2, Name: "Active Directory", Hints: []string{"Benutzerkonto gesperrt", "Anmeldung Login Authentifizierung"}, Examples: []string{"Mein Benutzerkonto ist gesperrt und ich kann mich nicht anmelden"}}}
|
||||
hits, err := s.Search(context.Background(), "Benutzerkonto gesperrt, Anmeldung nicht möglich", 1, cats)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hits) != 1 {
|
||||
t.Fatalf("hits=%d", len(hits))
|
||||
}
|
||||
h := hits[0]
|
||||
if h.Score < .75 {
|
||||
t.Fatalf("hybrid score too low: %+v", h)
|
||||
}
|
||||
if h.SemanticScore < .8 {
|
||||
t.Fatalf("expected strong best-chunk semantic score: %+v", h)
|
||||
}
|
||||
if h.TitleScore < .7 {
|
||||
t.Fatalf("expected strong title score: %+v", h)
|
||||
}
|
||||
if h.KeywordScore <= 0 || h.CategoryScore <= 0 {
|
||||
t.Fatalf("expected keyword/category contributions: %+v", h)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(h.BestChunkExcerpt), "benutzerkonto") {
|
||||
t.Fatalf("wrong best chunk: %q", h.BestChunkExcerpt)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,8 +82,13 @@ type GLPIKnowledgeItem struct {
|
||||
}
|
||||
|
||||
type KnowledgeHit struct {
|
||||
Doc KnowledgeDoc `json:"doc"`
|
||||
Score float64 `json:"score"`
|
||||
Doc KnowledgeDoc `json:"doc"`
|
||||
Score float64 `json:"score"`
|
||||
SemanticScore float64 `json:"semantic_score,omitempty"`
|
||||
TitleScore float64 `json:"title_score,omitempty"`
|
||||
KeywordScore float64 `json:"keyword_score,omitempty"`
|
||||
CategoryScore float64 `json:"category_score,omitempty"`
|
||||
BestChunkExcerpt string `json:"best_chunk_excerpt,omitempty"`
|
||||
}
|
||||
|
||||
// ChangeContext is a normalized, read-only view of a GLPI Change. Only fields
|
||||
@@ -196,6 +201,7 @@ type PolicyResult struct {
|
||||
ReplyThreshold float64 `json:"reply_threshold"`
|
||||
ReplyKnowledgeID string `json:"reply_knowledge_id,omitempty"`
|
||||
ReplyDecision string `json:"reply_decision"`
|
||||
KnowledgeThreshold float64 `json:"knowledge_threshold,omitempty"`
|
||||
AIReason string `json:"ai_reason,omitempty"`
|
||||
}
|
||||
|
||||
@@ -231,6 +237,12 @@ type RunRecord struct {
|
||||
KnowledgeTopID string `json:"knowledge_top_id,omitempty"`
|
||||
KnowledgeTopTitle string `json:"knowledge_top_title,omitempty"`
|
||||
KnowledgeScore float64 `json:"knowledge_score,omitempty"`
|
||||
KnowledgeSemanticScore float64 `json:"knowledge_semantic_score,omitempty"`
|
||||
KnowledgeTitleScore float64 `json:"knowledge_title_score,omitempty"`
|
||||
KnowledgeKeywordScore float64 `json:"knowledge_keyword_score,omitempty"`
|
||||
KnowledgeCategoryScore float64 `json:"knowledge_category_score,omitempty"`
|
||||
KnowledgeThreshold float64 `json:"knowledge_threshold,omitempty"`
|
||||
KnowledgeBestChunk string `json:"knowledge_best_chunk,omitempty"`
|
||||
ContextChanges int `json:"context_changes,omitempty"`
|
||||
ContextIncidents int `json:"context_incidents,omitempty"`
|
||||
ContextIssues int `json:"context_issues,omitempty"`
|
||||
|
||||
@@ -110,6 +110,8 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request) {
|
||||
"glpi_ok": g, "ollama_ok": o, "knowledge_docs": s.metrics.KnowledgeDocs(), "last_poll": s.metrics.LastPoll(),
|
||||
"communication_language": s.cfg.CommunicationLanguage, "communication_style": s.cfg.CommunicationStyle, "knowledge_allowed_sources": s.cfg.KnowledgeAllowedSources, "knowledge_auto_reply_sources": s.cfg.KnowledgeAutoReplySources,
|
||||
"category_confidence": s.cfg.CategoryConfidence, "reply_confidence": s.cfg.ReplyConfidence, "knowledge_min_score": s.cfg.KnowledgeMinScore,
|
||||
"knowledge_weight_semantic": s.cfg.KnowledgeSemanticWeight, "knowledge_weight_title": s.cfg.KnowledgeTitleWeight, "knowledge_weight_keywords": s.cfg.KnowledgeKeywordWeight, "knowledge_weight_category": s.cfg.KnowledgeCategoryWeight,
|
||||
"knowledge_chunk_words": s.cfg.KnowledgeChunkWords, "knowledge_chunk_overlap_words": s.cfg.KnowledgeChunkOverlapWords, "knowledge_max_chunks_per_doc": s.cfg.KnowledgeMaxChunksPerDoc,
|
||||
"context_enabled": s.cfg.ContextEnabled, "context_fetches": s.metrics.ContextFetches.Load(), "context_errors": s.metrics.ContextErrors.Load(),
|
||||
"change_calendar_enabled": s.cfg.ChangeCalendarEnabled, "major_incidents_enabled": s.cfg.MajorIncidentsEnabled, "user_device_context_enabled": s.cfg.UserDeviceContextEnabled,
|
||||
"knowledge_edit_enabled": s.cfg.KnowledgeWebEditEnabled, "learning_enabled": s.cfg.LearningEnabled, "learning_examples": s.feedback.LearningCount(),
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
<div class="field"><label>Quelle</label><select id="kbSource"></select></div>
|
||||
<div class="field"><label>Sprache</label><input id="kbLanguage" value="de-DE"></div>
|
||||
<div class="field"><label>Stil</label><select id="kbStyle"><option value="formal">formal</option><option value="neutral">neutral</option><option value="informal">informal</option></select></div>
|
||||
<div class="field"><label>Min. RAG-Score</label><input id="kbScore" type="number" min="0" max="1" step="0.01" value="0.88"></div>
|
||||
<div class="field"><label>Min. RAG-Score</label><input id="kbScore" type="number" min="0" max="1" step="0.01" value="0.70"></div>
|
||||
<div class="field span2"><label>GLPI-Kategorien (leer = alle)</label><select id="kbCategories" multiple size="6"></select></div>
|
||||
<div class="field span2"><label>Keywords</label><input id="kbKeywords" placeholder="Konto gesperrt, Login, Passwort"></div>
|
||||
<div class="field span2"><label>Source URI (optional)</label><input id="kbUri" placeholder="kb://identity/account-locked"></div>
|
||||
@@ -87,16 +87,23 @@ function categoryDecision(x){
|
||||
}
|
||||
function replyDecision(x){
|
||||
const code=x.reply_decision||'';const conf=pct(x.ai_reply_confidence),threshold=pct(x.reply_threshold);let label='Keine Antwort',cls='pill-warn',detail=code||'keine Policy-Information';
|
||||
if(code==='reply_written'){label='Geschrieben';cls='pill-ok';detail=`KI ${conf} ≥ ${threshold}`} else if(code==='reply_accepted_dry_run'){label='Würde antworten';cls='pill-ok';detail=`DRY RUN · KI ${conf} ≥ ${threshold}`} else if(code==='reply_accepted'){label='Freigegeben';cls='pill-ok'} else if(code==='reply_auto_disabled'){detail='AUTO_REPLY=false'} else if(code==='reply_no_knowledge_candidates'){detail='Keine freigegebene Wissensquelle gefunden'} else if(code==='reply_model_not_recommended'){detail='KI empfiehlt keine automatische Antwort'} else if(code==='reply_confidence_below_threshold'){detail=`KI ${conf} < Schwellwert ${threshold}`} else if(code==='reply_no_knowledge_selected'){detail='Keine Knowledge-ID ausgewählt'} else if(code==='reply_context_incomplete'){detail='Kontextquelle unvollständig / nicht erreichbar'} else if(code==='reply_relevant_incident'){detail='Relevanter Major Incident oder Service-Ausfall'} else if(code==='reply_existing_followup'){detail='Ticket hatte bereits ein Followup'} else if(code==='reply_followup_appeared_before_write'){detail='Während der Analyse ist ein Followup hinzugekommen'} else if(code==='reply_knowledge_not_found'){detail='Gewählte Knowledge-ID nicht gefunden'} else if(code==='reply_source_not_allowed'){detail='Knowledge-Quelle nicht erlaubt'} else if(code==='reply_source_not_allowed_for_auto_reply'){detail='Quelle darf nicht automatisch antworten'} else if(code==='reply_language_mismatch'){detail='Knowledge-Sprache passt nicht'} else if(code==='reply_style_mismatch'){detail='Knowledge-Stil passt nicht'} else if(code==='reply_knowledge_auto_reply_disabled'){detail='KB nicht für Auto-Reply freigegeben'} else if(code==='reply_knowledge_score_below_threshold'){detail='Knowledge-Ähnlichkeit unter Schwellwert'} else if(code==='reply_knowledge_answer_empty'){detail='Kein freigegebener Antworttext'} else if(code==='reply_category_not_allowed'){detail='KB nicht für Zielkategorie freigegeben'} else if(code==='reply_ticket_changed_before_write'){detail='Ticket wurde während der Analyse verändert'} else if(code==='reply_write_failed'){label='Schreibfehler';cls='pill-bad';detail='Followup konnte nicht geschrieben werden'}
|
||||
const ai=x.ai_reply_recommended?`Ja · ${conf}`:`Nein · ${conf}`;let knowledge=x.knowledge_top_id?`<div class="knowledge">Top-KB: ${esc(x.knowledge_top_title||x.knowledge_top_id)} (${esc(x.knowledge_top_id)}) · ${esc(pct(x.knowledge_score))}</div>`:'<div class="knowledge">Knowledge: keine Treffer</div>';
|
||||
if(code==='reply_written'){label='Geschrieben';cls='pill-ok';detail=`KI ${conf} ≥ ${threshold}`} else if(code==='reply_accepted_dry_run'){label='Würde antworten';cls='pill-ok';detail=`DRY RUN · KI ${conf} ≥ ${threshold}`} else if(code==='reply_accepted'){label='Freigegeben';cls='pill-ok'} else if(code==='reply_auto_disabled'){detail='AUTO_REPLY=false'} else if(code==='reply_no_knowledge_candidates'){detail='Keine freigegebene Wissensquelle gefunden'} else if(code==='reply_model_not_recommended'){detail='KI empfiehlt keine automatische Antwort'} else if(code==='reply_confidence_below_threshold'){detail=`KI ${conf} < Schwellwert ${threshold}`} else if(code==='reply_no_knowledge_selected'){detail='Keine Knowledge-ID ausgewählt'} else if(code==='reply_context_incomplete'){detail='Kontextquelle unvollständig / nicht erreichbar'} else if(code==='reply_relevant_incident'){detail='Relevanter Major Incident oder Service-Ausfall'} else if(code==='reply_existing_followup'){detail='Ticket hatte bereits ein Followup'} else if(code==='reply_followup_appeared_before_write'){detail='Während der Analyse ist ein Followup hinzugekommen'} else if(code==='reply_knowledge_not_found'){detail='Gewählte Knowledge-ID nicht gefunden'} else if(code==='reply_source_not_allowed'){detail='Knowledge-Quelle nicht erlaubt'} else if(code==='reply_source_not_allowed_for_auto_reply'){detail='Quelle darf nicht automatisch antworten'} else if(code==='reply_language_mismatch'){detail='Knowledge-Sprache passt nicht'} else if(code==='reply_style_mismatch'){detail='Knowledge-Stil passt nicht'} else if(code==='reply_knowledge_auto_reply_disabled'){detail='KB nicht für Auto-Reply freigegeben'} else if(code==='reply_knowledge_score_below_threshold'){detail=`Hybrid-KB-Score ${pct(x.knowledge_score)} < erforderliche ${pct(x.knowledge_threshold)}`} else if(code==='reply_knowledge_answer_empty'){detail='Kein freigegebener Antworttext'} else if(code==='reply_category_not_allowed'){detail='KB nicht für Zielkategorie freigegeben'} else if(code==='reply_ticket_changed_before_write'){detail='Ticket wurde während der Analyse verändert'} else if(code==='reply_write_failed'){label='Schreibfehler';cls='pill-bad';detail='Followup konnte nicht geschrieben werden'}
|
||||
const ai=x.ai_reply_recommended?`Ja · ${conf}`:`Nein · ${conf}`;
|
||||
let knowledge='<div class="knowledge">Knowledge: keine Treffer</div>';
|
||||
if(x.knowledge_top_id){
|
||||
const parts=[`Semantik ${pct(x.knowledge_semantic_score)}`,`Titel ${pct(x.knowledge_title_score)}`];
|
||||
if(Number(x.knowledge_keyword_score||0)>0)parts.push(`Keywords ${pct(x.knowledge_keyword_score)}`);
|
||||
if(Number(x.knowledge_category_score||0)>0)parts.push(`Kategorie/Lernen ${pct(x.knowledge_category_score)}`);
|
||||
knowledge=`<div class="knowledge"><strong>Top-KB:</strong> ${esc(x.knowledge_top_title||x.knowledge_top_id)} (${esc(x.knowledge_top_id)})<br><strong>Hybrid ${esc(pct(x.knowledge_score))}</strong> · erforderlich ${esc(pct(x.knowledge_threshold))}<div class="sub">${esc(parts.join(' · '))}</div>${x.knowledge_best_chunk?`<div class="sub">Bester Abschnitt: ${esc(x.knowledge_best_chunk)}</div>`:''}</div>`;
|
||||
}
|
||||
return `<div class="decision"><div><strong>Lösungsvorschlag KI:</strong> ${esc(ai)}</div><div class="sub">Reply-Schwellwert ${esc(threshold)}${x.ai_knowledge_id?` · KB ${esc(x.ai_knowledge_id)}`:''}</div><div style="margin-top:6px"><span class="pill ${cls}">${esc(label)}</span> <span class="sub">${esc(detail)}</span></div>${knowledge}</div>`;
|
||||
}
|
||||
function renderRuns(r){runsData=r;document.querySelector('#runs').innerHTML=r.length?r.map(x=>{const aiReason=x.ai_reason||x.reason||'–';const policy=x.policy_reason||[x.category_decision,x.reply_decision].filter(Boolean).join('; ')||'–';return `<tr><td>${esc(new Date(x.finished_at).toLocaleString('de-DE'))}</td><td>#${esc(x.ticket_id)} ${esc(x.ticket_name)}</td><td><span class="pill">${esc(x.outcome)}</span>${x.dry_run?'<div class="sub">Dry Run</div>':''}</td><td>${categoryDecision(x)}</td><td>${replyDecision(x)}</td><td class="hide-md">C:${esc(x.context_changes||0)} I:${esc(x.context_incidents||0)} U:${esc(x.context_issues||0)} D:${esc(x.context_devices||0)}${(x.context_warnings||[]).length?' ⚠':''}</td><td class="hide-sm reason"><div><strong>KI:</strong> ${esc(aiReason)}</div><div class="policy"><strong>Policy:</strong> ${esc(policy)}</div>${x.error?`<div class="bad"><strong>Fehler:</strong> ${esc(x.error)}</div>`:''}</td></tr>`}).join(''):'<tr><td colspan="7">Noch keine Verarbeitung.</td></tr>'}
|
||||
function renderKB(){document.querySelector('#kbRows').innerHTML=kbDocs.length?kbDocs.map(d=>`<tr><td><div class="kb-title">${esc(d.id)}</div>${esc(d.title)}</td><td>${esc(d.source)}<div class="sub">${d.managed?'Web-verwaltet':(d.origin&&d.origin!=='static'?`${d.origin} / read-only`:'statisch / read-only')}</div></td><td>${d.auto_reply?'<span class="pill pill-ok">ja</span>':'<span class="pill">nein</span>'}</td><td>${esc((d.categories||[]).join(', ')||'alle')} ${(d.source_category_ids||[]).length?`<div class="sub">GLPI-KB-Kategorien: ${esc(d.source_category_ids.join(', '))}</div>`:''}</td><td><div class="actions">${d.managed?`<button onclick="editKB('${esc(d.id)}')">Bearbeiten</button><button class="danger" onclick="deleteKB('${esc(d.id)}')">Löschen</button>`:'<span class="sub">über Git/Datei verwalten</span>'}</div></td></tr>`).join(''):'<tr><td colspan="5">Noch keine Knowledge-Einträge.</td></tr>'}
|
||||
function renderLearning(rows){document.querySelector('#learningRows').innerHTML=rows.length?rows.map(x=>`<tr><td><strong>#${esc(x.ticket_id)} ${esc(x.subject)}</strong><div class="sub">${esc((x.text||'').slice(0,180))}</div></td><td>${esc(x.category_name)} (#${esc(x.category_id)})<div class="sub">${x.correction?'Korrektur':'Bestätigung'}${x.ai_recommended_category_id?` · KI #${esc(x.ai_recommended_category_id)} ${esc(pct(x.ai_confidence))}`:''}</div></td><td>${esc(new Date(x.created_at).toLocaleString('de-DE'))}</td><td><button class="danger" onclick="deleteLearning('${esc(x.id)}')">Löschen</button></td></tr>`).join(''):'<tr><td colspan="4">Noch keine bestätigten Beispiele.</td></tr>'}
|
||||
async function refresh(){try{const [s,r,c,k,l]=await Promise.all([api('/api/status'),api('/api/runs?limit=50'),api('/api/categories'),api('/api/knowledge'),api('/api/learning')]);categories=c;kbDocs=k;const srcSel=document.querySelector('#kbSource');const oldSource=srcSel.value;srcSel.innerHTML=(s.knowledge_allowed_sources||[]).map(x=>`<option value="${esc(x)}">${esc(x)}</option>`).join('');if(oldSource&&[...srcSel.options].some(o=>o.value===oldSource))srcSel.value=oldSource;else if([...srcSel.options].some(o=>o.value==='internal-kb'))srcSel.value='internal-kb';const kbSel=document.querySelector('#kbCategories');const selected=new Set([...kbSel.selectedOptions].map(o=>Number(o.value)));kbSel.innerHTML=categories.map(x=>`<option value="${Number(x.id)}">${esc(x.completename||x.name)} (#${Number(x.id)})</option>`).join('');[...kbSel.options].forEach(o=>o.selected=selected.has(Number(o.value)));const cards=[['GLPI',s.glpi_ok?'OK':'Fehler',s.glpi_ok],['Ollama',s.ollama_ok?'OK':'Fehler',s.ollama_ok],['Verarbeitet',s.processed,true],['Fehler',s.errors,s.errors===0],['Knowledge',s.knowledge_docs,true],['Lernbeispiele',s.learning_examples,true],['Kategorie-Schwelle',pct(s.category_confidence),true],['Reply-Schwelle',pct(s.reply_confidence),true],['Sprache',s.communication_language,true],['Stil',s.communication_style,true],['KB-Editor',s.knowledge_edit_enabled?'aktiv':'aus',s.knowledge_edit_enabled],['GLPI-KB',s.glpi_kb_enabled?(s.glpi_kb_ok?`${s.glpi_kb_documents} synchronisiert`:'Fehler'):'aus',!s.glpi_kb_enabled||s.glpi_kb_ok],['Uptime Kuma',s.uptime_kuma_enabled?'an':'aus',true]];document.querySelector('#cards').innerHTML=cards.map(c=>`<div class="card"><div class="k">${esc(c[0])}</div><div class="v ${c[2]?'ok':'bad'}">${esc(c[1])}</div></div>`).join('');document.querySelector('#kbForm').querySelectorAll('input,textarea,select,button').forEach(x=>x.disabled=!s.knowledge_edit_enabled);const kd=document.querySelector('#kbDisabled');if(!s.knowledge_edit_enabled){kd.textContent='KB-Bearbeitung ist deaktiviert. Setzen Sie KNOWLEDGE_WEB_EDIT_ENABLED=true (nur mit authentifiziertem Dashboard).';kd.className='msg show'}else{kd.className='msg'}renderRuns(r);renderKB();renderLearning(l)}catch(e){msg(e.message,'err')}}
|
||||
function resetKBForm(){document.querySelector('#kbForm').reset();if([...document.querySelector('#kbSource').options].some(o=>o.value==='internal-kb'))document.querySelector('#kbSource').value='internal-kb';document.querySelector('#kbLanguage').value='de-DE';document.querySelector('#kbStyle').value='formal';document.querySelector('#kbScore').value='0.88';document.querySelector('#kbFormTitle').textContent='Knowledge-Eintrag anlegen'}
|
||||
function editKB(id){const d=kbDocs.find(x=>x.id===id);if(!d)return;document.querySelector('#kbFormTitle').textContent=`Knowledge-Eintrag bearbeiten: ${id}`;document.querySelector('#kbId').value=d.id||'';document.querySelector('#kbTitle').value=d.title||'';document.querySelector('#kbSource').value=d.source||'internal-kb';document.querySelector('#kbLanguage').value=d.language||'de-DE';document.querySelector('#kbStyle').value=d.communication_style||'formal';document.querySelector('#kbScore').value=d.min_score??0.88;[...document.querySelector('#kbCategories').options].forEach(o=>o.selected=(d.categories||[]).includes(Number(o.value)));document.querySelector('#kbKeywords').value=(d.keywords||[]).join(', ');document.querySelector('#kbUri').value=d.source_uri||'';document.querySelector('#kbText').value=d.text||'';document.querySelector('#kbAnswer').value=d.answer||'';document.querySelector('#kbAutoReply').checked=!!d.auto_reply;document.querySelector('#kbForm').scrollIntoView({behavior:'smooth'})}
|
||||
async function refresh(){try{const [s,r,c,k,l]=await Promise.all([api('/api/status'),api('/api/runs?limit=50'),api('/api/categories'),api('/api/knowledge'),api('/api/learning')]);categories=c;kbDocs=k;const srcSel=document.querySelector('#kbSource');const oldSource=srcSel.value;srcSel.innerHTML=(s.knowledge_allowed_sources||[]).map(x=>`<option value="${esc(x)}">${esc(x)}</option>`).join('');if(oldSource&&[...srcSel.options].some(o=>o.value===oldSource))srcSel.value=oldSource;else if([...srcSel.options].some(o=>o.value==='internal-kb'))srcSel.value='internal-kb';const kbSel=document.querySelector('#kbCategories');const selected=new Set([...kbSel.selectedOptions].map(o=>Number(o.value)));kbSel.innerHTML=categories.map(x=>`<option value="${Number(x.id)}">${esc(x.completename||x.name)} (#${Number(x.id)})</option>`).join('');[...kbSel.options].forEach(o=>o.selected=selected.has(Number(o.value)));const cards=[['GLPI',s.glpi_ok?'OK':'Fehler',s.glpi_ok],['Ollama',s.ollama_ok?'OK':'Fehler',s.ollama_ok],['Verarbeitet',s.processed,true],['Fehler',s.errors,s.errors===0],['Knowledge',s.knowledge_docs,true],['KB-Mindestscore',pct(s.knowledge_min_score),true],['Lernbeispiele',s.learning_examples,true],['Kategorie-Schwelle',pct(s.category_confidence),true],['Reply-Schwelle',pct(s.reply_confidence),true],['Sprache',s.communication_language,true],['Stil',s.communication_style,true],['KB-Editor',s.knowledge_edit_enabled?'aktiv':'aus',s.knowledge_edit_enabled],['GLPI-KB',s.glpi_kb_enabled?(s.glpi_kb_ok?`${s.glpi_kb_documents} synchronisiert`:'Fehler'):'aus',!s.glpi_kb_enabled||s.glpi_kb_ok],['Uptime Kuma',s.uptime_kuma_enabled?'an':'aus',true]];document.querySelector('#cards').innerHTML=cards.map(c=>`<div class="card"><div class="k">${esc(c[0])}</div><div class="v ${c[2]?'ok':'bad'}">${esc(c[1])}</div></div>`).join('');document.querySelector('#kbForm').querySelectorAll('input,textarea,select,button').forEach(x=>x.disabled=!s.knowledge_edit_enabled);const kd=document.querySelector('#kbDisabled');if(!s.knowledge_edit_enabled){kd.textContent='KB-Bearbeitung ist deaktiviert. Setzen Sie KNOWLEDGE_WEB_EDIT_ENABLED=true (nur mit authentifiziertem Dashboard).';kd.className='msg show'}else{kd.className='msg'}renderRuns(r);renderKB();renderLearning(l)}catch(e){msg(e.message,'err')}}
|
||||
function resetKBForm(){document.querySelector('#kbForm').reset();if([...document.querySelector('#kbSource').options].some(o=>o.value==='internal-kb'))document.querySelector('#kbSource').value='internal-kb';document.querySelector('#kbLanguage').value='de-DE';document.querySelector('#kbStyle').value='formal';document.querySelector('#kbScore').value='0.70';document.querySelector('#kbFormTitle').textContent='Knowledge-Eintrag anlegen'}
|
||||
function editKB(id){const d=kbDocs.find(x=>x.id===id);if(!d)return;document.querySelector('#kbFormTitle').textContent=`Knowledge-Eintrag bearbeiten: ${id}`;document.querySelector('#kbId').value=d.id||'';document.querySelector('#kbTitle').value=d.title||'';document.querySelector('#kbSource').value=d.source||'internal-kb';document.querySelector('#kbLanguage').value=d.language||'de-DE';document.querySelector('#kbStyle').value=d.communication_style||'formal';document.querySelector('#kbScore').value=d.min_score??0.70;[...document.querySelector('#kbCategories').options].forEach(o=>o.selected=(d.categories||[]).includes(Number(o.value)));document.querySelector('#kbKeywords').value=(d.keywords||[]).join(', ');document.querySelector('#kbUri').value=d.source_uri||'';document.querySelector('#kbText').value=d.text||'';document.querySelector('#kbAnswer').value=d.answer||'';document.querySelector('#kbAutoReply').checked=!!d.auto_reply;document.querySelector('#kbForm').scrollIntoView({behavior:'smooth'})}
|
||||
document.querySelector('#kbForm').addEventListener('submit',async e=>{e.preventDefault();const strs=v=>v.split(',').map(x=>x.trim()).filter(Boolean);const selectedCats=[...document.querySelector('#kbCategories').selectedOptions].map(o=>Number(o.value));const d={id:document.querySelector('#kbId').value.trim(),title:document.querySelector('#kbTitle').value.trim(),source:document.querySelector('#kbSource').value.trim(),language:document.querySelector('#kbLanguage').value.trim(),communication_style:document.querySelector('#kbStyle').value,text:document.querySelector('#kbText').value.trim(),answer:document.querySelector('#kbAnswer').value.trim(),auto_reply:document.querySelector('#kbAutoReply').checked,min_score:Number(document.querySelector('#kbScore').value||0),categories:selectedCats,keywords:strs(document.querySelector('#kbKeywords').value),source_uri:document.querySelector('#kbUri').value.trim()};try{await api('/api/knowledge',{method:'POST',body:JSON.stringify(d)});msg('Knowledge-Eintrag gespeichert.','good');resetKBForm();await refresh()}catch(e){msg(e.message,'err')}})
|
||||
async function deleteKB(id){if(!confirm(`Knowledge-Eintrag ${id} wirklich löschen?`))return;try{await api(`/api/knowledge/${encodeURIComponent(id)}`,{method:'DELETE'});msg('Knowledge-Eintrag gelöscht.','good');await refresh()}catch(e){msg(e.message,'err')}}
|
||||
function openLearn(runID,preferred){currentLearnRun=runID;const run=runsData.find(x=>x.run_id===runID);document.querySelector('#learnTicket').textContent=run?`#${run.ticket_id} ${run.ticket_name}`:runID;const sel=document.querySelector('#learnCategory');sel.innerHTML=categories.map(c=>`<option value="${Number(c.id)}">${esc(c.completename||c.name)} (#${Number(c.id)})</option>`).join('');if(preferred>0)sel.value=String(preferred);document.querySelector('#learnModal').classList.add('show')}
|
||||
|
||||
Reference in New Issue
Block a user