Update Chunk-Config mit Teilung der KBs und Tickets

.env.example

@@ -60,6 +60,7 @@ KNOWLEDGE_WEIGHT_CATEGORY=0.10
 KNOWLEDGE_CHUNK_WORDS=160
 KNOWLEDGE_CHUNK_OVERLAP_WORDS=30
 KNOWLEDGE_MAX_CHUNKS_PER_DOC=24
+KNOWLEDGE_MAX_QUERY_CHUNKS=64
 CATEGORY_PROMPT_LIMIT=80
 # Fail-closed source policy. Only documents carrying one of these source labels are indexed/searched.
 KNOWLEDGE_ALLOWED_SOURCES=internal-kb,glpi-kb

README.md

@@ -199,6 +199,7 @@ KNOWLEDGE_WEIGHT_CATEGORY=0.10
 KNOWLEDGE_CHUNK_WORDS=160
 KNOWLEDGE_CHUNK_OVERLAP_WORDS=30
 KNOWLEDGE_MAX_CHUNKS_PER_DOC=24
+KNOWLEDGE_MAX_QUERY_CHUNKS=64
 ```
 
 Der angezeigte Hybrid-Score ist **keine Wahrscheinlichkeit**. Er ist ein nachvollziehbarer Ranking-Score. Die Semantik verwendet die Ähnlichkeit des besten Body-Chunks; der Titel kombiniert Embedding- und exakten/lexikalischen Titelmatch; Keywords werden explizit gegen den Tickettext geprüft. Ist ein Knowledge-Dokument GLPI-ITIL-Kategorien zugeordnet, fließen deren Namen, semantische Hints und menschlich bestätigte Lernbeispiele als Kategorie-Signal ein. Fehlen einem Artikel Keywords oder Kategoriezuordnungen, wird er nicht pauschal abgestraft: Nur vorhandene Komponenten werden in die Gewichtung aufgenommen.
@@ -432,3 +433,10 @@ Bei neuen Gitea-Builds genügt:
 ```bash
 docker compose -f docker-compose.registry.yml up -d --pull always
 ```
+
+### Lange Tickets und KB-Artikel
+
+Für die semantische Relevanz werden **beide Seiten** in überlappende Abschnitte zerlegt. Ticket-Abschnitte werden gegen KB-Abschnitte verglichen; der beste lokale Treffer bildet die semantische Komponente. Der Ticket-Betreff wird separat für den Titel-Score verwendet. Dadurch verwässern lange Ticketbeschreibungen einen klar passenden Lösungsabschnitt nicht mehr.
+
+Der Ollama-Embedding-Aufruf verwendet `truncate:false`. Ein Text, der trotz Chunking das Kontextfenster des Embedding-Modells überschreitet, führt damit zu einem sichtbaren Fehler statt zu stiller Kürzung.
+

internal/agent/agent.go

@@ -201,6 +201,9 @@ func (s *Service) Process(ctx context.Context, id int64) error {
 		run.KnowledgeKeywordScore = hits[0].KeywordScore
 		run.KnowledgeCategoryScore = hits[0].CategoryScore
 		run.KnowledgeBestChunk = hits[0].BestChunkExcerpt
+		run.KnowledgeBestQueryChunk = hits[0].BestQueryExcerpt
+		run.KnowledgeQueryChunks = hits[0].QueryChunkCount
+		run.KnowledgeDocumentChunks = hits[0].DocumentChunkCount
 		run.KnowledgeThreshold = s.cfg.KnowledgeMinScore
 		if hits[0].Doc.MinScore > run.KnowledgeThreshold {
 			run.KnowledgeThreshold = hits[0].Doc.MinScore

internal/config/config.go

@@ -59,6 +59,7 @@ type Config struct {
 	KnowledgeChunkWords        int
 	KnowledgeChunkOverlapWords int
 	KnowledgeMaxChunksPerDoc   int
+	KnowledgeMaxQueryChunks    int
 	GLPIKBEnabled              bool
 	GLPIKBPath                 string
 	GLPIKBFilter               string
@@ -161,6 +162,7 @@ func Load() (Config, error) {
 		KnowledgeChunkWords:         envInt("KNOWLEDGE_CHUNK_WORDS", 160),
 		KnowledgeChunkOverlapWords:  envInt("KNOWLEDGE_CHUNK_OVERLAP_WORDS", 30),
 		KnowledgeMaxChunksPerDoc:    envInt("KNOWLEDGE_MAX_CHUNKS_PER_DOC", 24),
+		KnowledgeMaxQueryChunks:     envInt("KNOWLEDGE_MAX_QUERY_CHUNKS", 64),
 		GLPIKBEnabled:               envBool("GLPI_KB_ENABLED", false),
 		GLPIKBPath:                  env("GLPI_KB_PATH", "auto"),
 		GLPIKBFilter:                strings.TrimSpace(os.Getenv("GLPI_KB_FILTER")),
@@ -300,6 +302,9 @@ func (c Config) Validate() error {
 	if c.KnowledgeMaxChunksPerDoc != 0 && (c.KnowledgeMaxChunksPerDoc < 1 || c.KnowledgeMaxChunksPerDoc > 100) {
 		return errors.New("KNOWLEDGE_MAX_CHUNKS_PER_DOC must be between 1 and 100")
 	}
+	if c.KnowledgeMaxQueryChunks != 0 && (c.KnowledgeMaxQueryChunks < 1 || c.KnowledgeMaxQueryChunks > 200) {
+		return errors.New("KNOWLEDGE_MAX_QUERY_CHUNKS must be between 1 and 200")
+	}
 	if c.LearningEnabled {
 		if c.LearningMaxExamples < 1 || c.LearningMaxExamples > 10000 {
 			return errors.New("LEARNING_MAX_EXAMPLES must be between 1 and 10000")

internal/knowledge/store.go

@@ -28,6 +28,7 @@ type ScoringConfig struct {
 	ChunkWords      int
 	ChunkOverlap    int
 	MaxChunksPerDoc int
+	MaxQueryChunks  int
 }
 
 type Store struct {
@@ -56,7 +57,7 @@ type cacheFile struct {
 }
 
 func DefaultScoringConfig() ScoringConfig {
-	return ScoringConfig{SemanticWeight: .50, TitleWeight: .25, KeywordWeight: .15, CategoryWeight: .10, ChunkWords: 160, ChunkOverlap: 30, MaxChunksPerDoc: 24}
+	return ScoringConfig{SemanticWeight: .50, TitleWeight: .25, KeywordWeight: .15, CategoryWeight: .10, ChunkWords: 160, ChunkOverlap: 30, MaxChunksPerDoc: 24, MaxQueryChunks: 64}
 }
 
 func normalizeScoring(c ScoringConfig) ScoringConfig {
@@ -73,6 +74,9 @@ func normalizeScoring(c ScoringConfig) ScoringConfig {
 	if c.MaxChunksPerDoc <= 0 {
 		c.MaxChunksPerDoc = d.MaxChunksPerDoc
 	}
+	if c.MaxQueryChunks <= 0 {
+		c.MaxQueryChunks = d.MaxQueryChunks
+	}
 	return c
 }
 
@@ -539,44 +543,78 @@ func (s *Store) Search(ctx context.Context, text string, topK int, categorySets
 		cats = categorySets[0]
 	}
 
-	var queryVector []float64
+	queryTitle, queryBody := splitQueryText(text)
+	queryChunks := chunkText(queryBody, scoreCfg.ChunkWords, scoreCfg.ChunkOverlap, scoreCfg.MaxQueryChunks)
+	if len(queryChunks) == 0 {
+		queryChunks = chunkText(text, scoreCfg.ChunkWords, scoreCfg.ChunkOverlap, scoreCfg.MaxQueryChunks)
+	}
+	var queryVectors [][]float64
+	var queryTitleVector []float64
 	if s.rag && s.embedder != nil {
-		q, err := s.embedder.Embed(ctx, []string{text})
-		if err != nil {
-			return nil, err
+		if len(queryChunks) > 0 {
+			q, err := s.embedTexts(ctx, queryChunks, 64)
+			if err != nil {
+				return nil, err
+			}
+			queryVectors = q
 		}
-		if len(q) > 0 {
-			queryVector = q[0]
+		if strings.TrimSpace(queryTitle) != "" {
+			tq, err := s.embedder.Embed(ctx, []string{queryTitle})
+			if err != nil {
+				return nil, err
+			}
+			if len(tq) > 0 {
+				queryTitleVector = tq[0]
+			}
 		}
 	}
 
 	hits := make([]model.KnowledgeHit, 0, len(docs))
 	for _, d := range docs {
-		semantic, bestChunk := 0.0, ""
+		semantic, bestChunk, bestQueryChunk := 0.0, "", ""
 		semanticAvailable := false
-		if len(queryVector) > 0 && len(chunkVecs[d.ID]) > 0 {
+		if len(queryVectors) > 0 && len(chunkVecs[d.ID]) > 0 {
 			semanticAvailable = true
-			for i, v := range chunkVecs[d.ID] {
-				score := clamp01(cosine(queryVector, v))
-				if score > semantic || bestChunk == "" {
-					semantic = score
-					if i < len(chunks[d.ID]) {
-						bestChunk = chunks[d.ID][i]
+			for qi, qv := range queryVectors {
+				for di, dv := range chunkVecs[d.ID] {
+					score := clamp01(cosine(qv, dv))
+					if score > semantic || bestChunk == "" {
+						semantic = score
+						if di < len(chunks[d.ID]) {
+							bestChunk = chunks[d.ID][di]
+						}
+						if qi < len(queryChunks) {
+							bestQueryChunk = queryChunks[qi]
+						}
 					}
 				}
 			}
 		} else if strings.TrimSpace(d.Text) != "" {
 			semanticAvailable = true
-			semantic = tokenF1(text, d.Text)
-			bestChunk = d.Text
+			docChunks := chunks[d.ID]
+			if len(docChunks) == 0 {
+				docChunks = []string{d.Text}
+			}
+			for _, qc := range queryChunks {
+				for _, dc := range docChunks {
+					score := tokenF1(qc, dc)
+					if score > semantic || bestChunk == "" {
+						semantic, bestChunk, bestQueryChunk = score, dc, qc
+					}
+				}
+			}
 		}
 
 		title := 0.0
 		titleAvailable := strings.TrimSpace(d.Title) != ""
 		if titleAvailable {
-			title = titleSimilarity(text, d.Title)
-			if len(queryVector) > 0 && len(titleVecs[d.ID]) > 0 {
-				title = math.Max(title, clamp01(cosine(queryVector, titleVecs[d.ID])))
+			titleQuery := queryTitle
+			if strings.TrimSpace(titleQuery) == "" {
+				titleQuery = text
+			}
+			title = titleSimilarity(titleQuery, d.Title)
+			if len(queryTitleVector) > 0 && len(titleVecs[d.ID]) > 0 {
+				title = math.Max(title, clamp01(cosine(queryTitleVector, titleVecs[d.ID])))
 			}
 		}
 		keyword, keywordAvailable := keywordSimilarity(text, d.Keywords)
@@ -587,7 +625,7 @@ func (s *Store) Search(ctx context.Context, text string, topK int, categorySets
 			scorePart{keyword, scoreCfg.KeywordWeight, keywordAvailable},
 			scorePart{category, scoreCfg.CategoryWeight, categoryAvailable},
 		)
-		hits = append(hits, model.KnowledgeHit{Doc: d, Score: total, SemanticScore: semantic, TitleScore: title, KeywordScore: keyword, CategoryScore: category, BestChunkExcerpt: excerpt(bestChunk, 280)})
+		hits = append(hits, model.KnowledgeHit{Doc: d, Score: total, SemanticScore: semantic, TitleScore: title, KeywordScore: keyword, CategoryScore: category, BestChunkExcerpt: excerpt(bestChunk, 280), BestQueryExcerpt: excerpt(bestQueryChunk, 280), QueryChunkCount: len(queryChunks), DocumentChunkCount: len(chunks[d.ID])})
 	}
 	sort.SliceStable(hits, func(i, j int) bool {
 		if hits[i].Score == hits[j].Score {
@@ -719,6 +757,19 @@ func loadCache(path string) cacheFile {
 	return cf
 }
 
+func splitQueryText(text string) (title, body string) {
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return "", ""
+	}
+	if i := strings.IndexByte(text, '\n'); i >= 0 {
+		title = strings.TrimSpace(text[:i])
+		body = strings.TrimSpace(text[i+1:])
+		return title, body
+	}
+	return text, text
+}
+
 func chunkText(text string, words, overlap, maxChunks int) []string {
 	parts := strings.Fields(strings.TrimSpace(text))
 	if len(parts) == 0 {

internal/knowledge/store_test.go

@@ -201,3 +201,60 @@ func TestHybridScoringUsesChunksTitleKeywordsAndCategoryHints(t *testing.T) {
 		t.Fatalf("wrong best chunk: %q", h.BestChunkExcerpt)
 	}
 }
+
+type hashTestEmbedder struct{}
+
+func (hashTestEmbedder) Embed(_ context.Context, texts []string) ([][]float64, error) {
+	out := make([][]float64, len(texts))
+	for i, text := range texts {
+		// Deterministic fixed-width vector: identical text -> identical vector;
+		// unrelated text is unlikely to point in the same direction.
+		v := make([]float64, 128)
+		for pos, r := range []byte(strings.ToLower(strings.Join(strings.Fields(text), " "))) {
+			idx := (int(r) + pos*31) % len(v)
+			if (int(r)+pos)%2 == 0 {
+				v[idx] += 1
+			} else {
+				v[idx] -= 1
+			}
+		}
+		out[i] = v
+	}
+	return out, nil
+}
+
+func TestLongQueryIsChunkedAndCanMatchIdenticalKnowledgeSection(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	parts := make([]string, 0, 400)
+	for i := 0; i < 400; i++ {
+		parts = append(parts, "Benutzerkonto Anmeldung Sperrung Active Directory Diagnose Schritt")
+	}
+	body := strings.Join(parts, " ")
+	doc := model.KnowledgeDoc{ID: "KB-LONG", Title: "Benutzerkonto gesperrt", Text: body, Source: "internal-kb", Language: "de-DE", CommunicationStyle: "formal"}
+	b, _ := json.Marshal(doc)
+	if err := os.WriteFile(filepath.Join(dir, "long.json"), b, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	s, err := Load(context.Background(), dir, data, hashTestEmbedder{}, true, []string{"internal-kb"}, ScoringConfig{SemanticWeight: 1, ChunkWords: 80, ChunkOverlap: 20, MaxChunksPerDoc: 24})
+	if err != nil {
+		t.Fatal(err)
+	}
+	hits, err := s.Search(context.Background(), "Benutzerkonto gesperrt\n"+body, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(hits) != 1 {
+		t.Fatalf("hits=%d", len(hits))
+	}
+	h := hits[0]
+	if h.QueryChunkCount <= 1 || h.DocumentChunkCount <= 1 {
+		t.Fatalf("expected both sides to be chunked: %+v", h)
+	}
+	if h.SemanticScore < 0.999999 {
+		t.Fatalf("identical long body should contain an exact chunk match, got semantic=%f", h.SemanticScore)
+	}
+	if h.Score < 0.999999 {
+		t.Fatalf("semantic-only hybrid should be ~1, got %f", h.Score)
+	}
+}

internal/model/model.go

@@ -82,13 +82,16 @@ type GLPIKnowledgeItem struct {
 }
 
 type KnowledgeHit struct {
-	Doc              KnowledgeDoc `json:"doc"`
-	Score            float64      `json:"score"`
-	SemanticScore    float64      `json:"semantic_score,omitempty"`
-	TitleScore       float64      `json:"title_score,omitempty"`
-	KeywordScore     float64      `json:"keyword_score,omitempty"`
-	CategoryScore    float64      `json:"category_score,omitempty"`
-	BestChunkExcerpt string       `json:"best_chunk_excerpt,omitempty"`
+	Doc                KnowledgeDoc `json:"doc"`
+	Score              float64      `json:"score"`
+	SemanticScore      float64      `json:"semantic_score,omitempty"`
+	TitleScore         float64      `json:"title_score,omitempty"`
+	KeywordScore       float64      `json:"keyword_score,omitempty"`
+	CategoryScore      float64      `json:"category_score,omitempty"`
+	BestChunkExcerpt   string       `json:"best_chunk_excerpt,omitempty"`
+	BestQueryExcerpt   string       `json:"best_query_excerpt,omitempty"`
+	QueryChunkCount    int          `json:"query_chunk_count,omitempty"`
+	DocumentChunkCount int          `json:"document_chunk_count,omitempty"`
 }
 
 // ChangeContext is a normalized, read-only view of a GLPI Change. Only fields
@@ -243,6 +246,9 @@ type RunRecord struct {
 	KnowledgeCategoryScore    float64   `json:"knowledge_category_score,omitempty"`
 	KnowledgeThreshold        float64   `json:"knowledge_threshold,omitempty"`
 	KnowledgeBestChunk        string    `json:"knowledge_best_chunk,omitempty"`
+	KnowledgeBestQueryChunk   string    `json:"knowledge_best_query_chunk,omitempty"`
+	KnowledgeQueryChunks      int       `json:"knowledge_query_chunks,omitempty"`
+	KnowledgeDocumentChunks   int       `json:"knowledge_document_chunks,omitempty"`
 	ContextChanges            int       `json:"context_changes,omitempty"`
 	ContextIncidents          int       `json:"context_incidents,omitempty"`
 	ContextIssues             int       `json:"context_issues,omitempty"`

internal/ollama/client.go

@@ -49,7 +49,7 @@ func (c *Client) Embed(ctx context.Context, texts []string) ([][]float64, error)
 	if len(texts) == 0 {
 		return nil, nil
 	}
-	payload := map[string]any{"model": c.embeddingModel, "input": texts}
+	payload := map[string]any{"model": c.embeddingModel, "input": texts, "truncate": false}
 	var out struct {
 		Embeddings [][]float64 `json:"embeddings"`
 	}

internal/ollama/client_test.go

@@ -69,3 +69,25 @@ func TestAnalyseRetriesInvalidJSON(t *testing.T) {
 		t.Fatalf("calls=%d decision=%+v", calls, d)
 	}
 }
+
+func TestEmbedDisablesSilentTruncation(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatal(err)
+		}
+		if body["truncate"] != false {
+			t.Fatalf("truncate=%v, want false", body["truncate"])
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float64{{1, 0}}})
+	}))
+	defer srv.Close()
+	c := New(srv.URL, "m", "e", "de-DE", "formal", time.Second, 256, time.Minute, false, 1, 0)
+	v, err := c.Embed(context.Background(), []string{"test"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(v) != 1 {
+		t.Fatalf("embeddings=%d", len(v))
+	}
+}

internal/web/templates/dashboard.html

@@ -94,7 +94,7 @@ function replyDecision(x){
    const parts=[`Semantik ${pct(x.knowledge_semantic_score)}`,`Titel ${pct(x.knowledge_title_score)}`];
    if(Number(x.knowledge_keyword_score||0)>0)parts.push(`Keywords ${pct(x.knowledge_keyword_score)}`);
    if(Number(x.knowledge_category_score||0)>0)parts.push(`Kategorie/Lernen ${pct(x.knowledge_category_score)}`);
-   knowledge=`<div class="knowledge"><strong>Top-KB:</strong> ${esc(x.knowledge_top_title||x.knowledge_top_id)} (${esc(x.knowledge_top_id)})<br><strong>Hybrid ${esc(pct(x.knowledge_score))}</strong> · erforderlich ${esc(pct(x.knowledge_threshold))}<div class="sub">${esc(parts.join(' · '))}</div>${x.knowledge_best_chunk?`<div class="sub">Bester Abschnitt: ${esc(x.knowledge_best_chunk)}</div>`:''}</div>`;
+   knowledge=`<div class="knowledge"><strong>Top-KB:</strong> ${esc(x.knowledge_top_title||x.knowledge_top_id)} (${esc(x.knowledge_top_id)})<br><strong>Hybrid ${esc(pct(x.knowledge_score))}</strong> · erforderlich ${esc(pct(x.knowledge_threshold))}<div class="sub">${esc(parts.join(' · '))}</div>${x.knowledge_best_chunk?`<div class="sub">Bester KB-Abschnitt: ${esc(x.knowledge_best_chunk)}</div>`:''}${x.knowledge_best_query_chunk?`<div class="sub">Passender Ticket-Abschnitt: ${esc(x.knowledge_best_query_chunk)}</div>`:''}${x.knowledge_query_chunks?`<div class="sub">Chunk-Vergleich: ${esc(x.knowledge_query_chunks)} Ticket × ${esc(x.knowledge_document_chunks||0)} KB</div>`:''}</div>`;
  }
  return `<div class="decision"><div><strong>Lösungsvorschlag KI:</strong> ${esc(ai)}</div><div class="sub">Reply-Schwellwert ${esc(threshold)}${x.ai_knowledge_id?` · KB ${esc(x.ai_knowledge_id)}`:''}</div><div style="margin-top:6px"><span class="pill ${cls}">${esc(label)}</span> <span class="sub">${esc(detail)}</span></div>${knowledge}</div>`;
 }