update-1.6.2b

services/agent/internal/knowledge/persistent_index.go

@@ -133,12 +133,17 @@ func (s *Store) loadPersistentSnapshot() (bool, error) {
 	// them to the current knowledge roots using the stable manifest key.
 	reboundFiles := map[string]string{}
 	for key, rec := range snap.Manifest {
-		name := filepath.Base(key)
+		rel := strings.TrimPrefix(key, "static/")
+		root := s.dir
 		if strings.HasPrefix(key, "managed/") {
-			rec.Path = filepath.Join(s.managedDir, name)
-		} else {
-			rec.Path = filepath.Join(s.dir, name)
+			rel = strings.TrimPrefix(key, "managed/")
+			root = s.managedDir
 		}
+		rel = filepath.Clean(filepath.FromSlash(rel))
+		if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+			return false, fmt.Errorf("persistent knowledge index contains invalid manifest path %q", key)
+		}
+		rec.Path = filepath.Join(root, rel)
 		snap.Manifest[key] = rec
 		if rec.Included && rec.ID != "" {
 			if rec.Managed || reboundFiles[rec.ID] == "" {
@@ -511,7 +516,7 @@ type deltaScanResult struct {
 
 func (s *Store) scanDeltaDir(ctx context.Context, dir, origin string, managed bool, opts LoadOptions, old map[string]fileRecord, oldDocs map[string]model.KnowledgeDoc) (deltaScanResult, error) {
 	res := deltaScanResult{docs: map[string]model.KnowledgeDoc{}, manifest: map[string]fileRecord{}}
-	entries, err := os.ReadDir(dir)
+	entries, err := listKnowledgeJSONFiles(dir)
 	if err != nil {
 		return res, fmt.Errorf("read knowledge directory %q: %w", dir, err)
 	}
@@ -519,16 +524,10 @@ func (s *Store) scanDeltaDir(ctx context.Context, dir, origin string, managed bo
 		if err := ctx.Err(); err != nil {
 			return res, err
 		}
-		if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
-			continue
-		}
-		key := origin + "/" + e.Name()
-		path := filepath.Join(dir, e.Name())
-		info, err := e.Info()
-		if err != nil {
-			return res, err
-		}
-		if matchesAnyGlob(e.Name(), opts.IgnoreGlobs) {
+		key := origin + "/" + e.Rel
+		path := e.Path
+		info := e.Info
+		if matchesAnyGlob(e.Rel, opts.IgnoreGlobs) {
 			res.manifest[key] = fileRecord{Key: key, Path: path, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), Managed: managed, Ignored: true}
 			res.stats.IgnoredFiles++
 			continue
@@ -572,7 +571,7 @@ func (s *Store) scanDeltaDir(ctx context.Context, dir, origin string, managed bo
 		}
 		d, unmapped, skip, err := decodeKnowledgeDoc(b, opts.CategoryMode, s.categoryMap)
 		if err != nil {
-			return res, fmt.Errorf("%s: %w", e.Name(), err)
+			return res, fmt.Errorf("%s: %w", e.Rel, err)
 		}
 		rec := fileRecord{Key: key, Path: path, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), RawHash: rawHash, Managed: managed, Unmapped: append([]string(nil), unmapped...)}
 		if len(unmapped) > 0 {
@@ -587,14 +586,14 @@ func (s *Store) scanDeltaDir(ctx context.Context, dir, origin string, managed bo
 			continue
 		}
 		if d.ID == "" || d.Title == "" {
-			return res, fmt.Errorf("%s: id/title required", e.Name())
+			return res, fmt.Errorf("%s: id/title required", e.Rel)
 		}
 		if !safeID(d.ID) {
-			return res, fmt.Errorf("%s: invalid id %q", e.Name(), d.ID)
+			return res, fmt.Errorf("%s: invalid id %q", e.Rel, d.ID)
 		}
 		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
 		if d.Source == "" {
-			return res, fmt.Errorf("%s: source required", e.Name())
+			return res, fmt.Errorf("%s: source required", e.Rel)
 		}
 		if _, allowed := s.allowedSources[d.Source]; !allowed {
 			res.manifest[key] = rec
@@ -629,7 +628,7 @@ func mergeLoadStats(a, b LoadStats) LoadStats {
 	return LoadStats{IgnoredFiles: a.IgnoredFiles + b.IgnoredFiles, UnmappedCategoryFiles: a.UnmappedCategoryFiles + b.UnmappedCategoryFiles, UnmappedCategories: mergeStrings(a.UnmappedCategories, b.UnmappedCategories)}
 }
 
-func buildManifestForDocs(docs []model.KnowledgeDoc, files []string, origin string, managed bool) (map[string]fileRecord, error) {
+func buildManifestForDocs(docs []model.KnowledgeDoc, files []string, root, origin string, managed bool) (map[string]fileRecord, error) {
 	out := make(map[string]fileRecord, len(docs))
 	for i, d := range docs {
 		if i >= len(files) {
@@ -645,7 +644,11 @@ func buildManifestForDocs(docs []model.KnowledgeDoc, files []string, origin stri
 			return nil, err
 		}
 		h := sha256.Sum256(b)
-		key := origin + "/" + filepath.Base(path)
+		rel, err := filepath.Rel(root, path)
+		if err != nil {
+			return nil, err
+		}
+		key := origin + "/" + filepath.ToSlash(rel)
 		out[key] = fileRecord{Key: key, Path: path, ID: d.ID, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), RawHash: hex.EncodeToString(h[:]), Managed: managed, Included: true, Unmapped: append([]string(nil), d.UnmappedExternalCategories...)}
 	}
 	return out, nil
@@ -688,24 +691,18 @@ func (s *Store) persistExternalVectorCache(source string) error {
 // delta scans do not repeatedly open files that are intentionally not part of
 // the active corpus.
 func augmentManifestAllFiles(dir, origin string, managed bool, opts LoadOptions, manifest map[string]fileRecord) error {
-	entries, err := os.ReadDir(dir)
+	entries, err := listKnowledgeJSONFiles(dir)
 	if err != nil {
 		return err
 	}
 	for _, e := range entries {
-		if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
-			continue
-		}
-		key := origin + "/" + e.Name()
+		key := origin + "/" + e.Rel
 		if _, ok := manifest[key]; ok {
 			continue
 		}
-		info, err := e.Info()
-		if err != nil {
-			return err
-		}
-		path := filepath.Join(dir, e.Name())
-		rec := fileRecord{Key: key, Path: path, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), Managed: managed, Included: false, Ignored: matchesAnyGlob(e.Name(), opts.IgnoreGlobs)}
+		info := e.Info
+		path := e.Path
+		rec := fileRecord{Key: key, Path: path, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), Managed: managed, Included: false, Ignored: matchesAnyGlob(e.Rel, opts.IgnoreGlobs)}
 		if !rec.Ignored {
 			b, err := os.ReadFile(path)
 			if err != nil {

services/agent/internal/knowledge/store.go

@@ -237,6 +237,15 @@ func (s *Store) Initialize(ctx context.Context) (err error) {
 			if _, err := s.syncLoadedSemanticBackend(ctx); err != nil {
 				return err
 			}
+			// A persistent snapshot is a warm start, not proof that the mounted
+			// knowledge directory is unchanged. Reconcile it before ticket workers
+			// start so files copied while the agent was stopped are available on the
+			// first ticket instead of only after the background scan eventually ends.
+			if mode == "incremental" {
+				if err := s.SyncLocal(ctx); err != nil {
+					return fmt.Errorf("reconcile persistent knowledge index: %w", err)
+				}
+			}
 			return nil
 		} else if mode == "readonly" {
 			return fmt.Errorf("KNOWLEDGE_INDEX_MODE=readonly requires a compatible persistent index at %s", s.snapshotPath)
@@ -294,11 +303,11 @@ func (s *Store) fullRebuild(ctx context.Context) (err error) {
 	stats.UnmappedCategoryFiles += managedStats.UnmappedCategoryFiles
 	stats.UnmappedCategories = mergeStrings(stats.UnmappedCategories, managedStats.UnmappedCategories)
 
-	staticManifest, err := buildManifestForDocs(static, staticFiles, "static", false)
+	staticManifest, err := buildManifestForDocs(static, staticFiles, s.dir, "static", false)
 	if err != nil {
 		return fmt.Errorf("build static knowledge manifest: %w", err)
 	}
-	managedManifest, err := buildManifestForDocs(managed, managedFiles, "managed", true)
+	managedManifest, err := buildManifestForDocs(managed, managedFiles, s.managedDir, "managed", true)
 	if err != nil {
 		return fmt.Errorf("build managed knowledge manifest: %w", err)
 	}
@@ -405,44 +414,71 @@ func (s *Store) InitStatus() InitStatus {
 
 func (s *Store) Ready() bool { return s != nil && s.InitStatus().State == "ready" }
 
+type knowledgeJSONFile struct {
+	Rel  string
+	Path string
+	Info os.FileInfo
+}
+
+// listKnowledgeJSONFiles recursively discovers JSON knowledge documents. Large
+// exports are commonly grouped into category/subcategory folders; limiting the
+// scan to the top-level directory makes such a mounted corpus appear empty.
+func listKnowledgeJSONFiles(dir string) ([]knowledgeJSONFile, error) {
+	files := make([]knowledgeJSONFile, 0)
+	err := filepath.WalkDir(dir, func(path string, entry os.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if entry.IsDir() || !strings.HasSuffix(strings.ToLower(entry.Name()), ".json") {
+			return nil
+		}
+		rel, err := filepath.Rel(dir, path)
+		if err != nil {
+			return err
+		}
+		info, err := entry.Info()
+		if err != nil {
+			return err
+		}
+		files = append(files, knowledgeJSONFile{Rel: filepath.ToSlash(rel), Path: path, Info: info})
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	sort.Slice(files, func(i, j int) bool { return files[i].Rel < files[j].Rel })
+	return files, nil
+}
+
 func readDocs(dir string, allowed map[string]struct{}, opts LoadOptions, categoryMap map[string][]int64, progress func(total, processed, loaded int)) ([]model.KnowledgeDoc, []string, LoadStats, error) {
-	entries, err := os.ReadDir(dir)
+	entries, err := listKnowledgeJSONFiles(dir)
 	if err != nil {
 		return nil, nil, LoadStats{}, fmt.Errorf("read knowledge directory %q: %w", dir, err)
 	}
 	var docs []model.KnowledgeDoc
 	var files []string
 	stats := LoadStats{}
-	total := 0
-	for _, e := range entries {
-		if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
-			total++
-		}
-	}
+	total := len(entries)
 	processed := 0
 	if progress != nil {
 		progress(total, 0, 0)
 	}
 	for _, e := range entries {
-		if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
-			continue
-		}
 		processed++
-		if matchesAnyGlob(e.Name(), opts.IgnoreGlobs) {
+		if matchesAnyGlob(e.Rel, opts.IgnoreGlobs) {
 			stats.IgnoredFiles++
 			if progress != nil {
 				progress(total, processed, len(docs))
 			}
 			continue
 		}
-		path := filepath.Join(dir, e.Name())
-		b, err := os.ReadFile(path)
+		b, err := os.ReadFile(e.Path)
 		if err != nil {
 			return nil, nil, stats, err
 		}
 		d, unmapped, skip, err := decodeKnowledgeDoc(b, opts.CategoryMode, categoryMap)
 		if err != nil {
-			return nil, nil, stats, fmt.Errorf("%s: %w", e.Name(), err)
+			return nil, nil, stats, fmt.Errorf("%s: %w", e.Rel, err)
 		}
 		if len(unmapped) > 0 {
 			stats.UnmappedCategoryFiles++
@@ -456,14 +492,14 @@ func readDocs(dir string, allowed map[string]struct{}, opts LoadOptions, categor
 			continue
 		}
 		if d.ID == "" || d.Title == "" {
-			return nil, nil, stats, fmt.Errorf("%s: id/title required", e.Name())
+			return nil, nil, stats, fmt.Errorf("%s: id/title required", e.Rel)
 		}
 		if !safeID(d.ID) {
-			return nil, nil, stats, fmt.Errorf("%s: invalid id %q", e.Name(), d.ID)
+			return nil, nil, stats, fmt.Errorf("%s: invalid id %q", e.Rel, d.ID)
 		}
 		d.Source = strings.ToLower(strings.TrimSpace(d.Source))
 		if d.Source == "" {
-			return nil, nil, stats, fmt.Errorf("%s: source required", e.Name())
+			return nil, nil, stats, fmt.Errorf("%s: source required", e.Rel)
 		}
 		if _, ok := allowed[d.Source]; !ok {
 			if progress != nil {
@@ -474,7 +510,7 @@ func readDocs(dir string, allowed map[string]struct{}, opts LoadOptions, categor
 		d.Language = strings.TrimSpace(d.Language)
 		d.CommunicationStyle = strings.ToLower(strings.TrimSpace(d.CommunicationStyle))
 		docs = append(docs, d)
-		files = append(files, path)
+		files = append(files, e.Path)
 		if progress != nil {
 			progress(total, processed, len(docs))
 		}
@@ -654,10 +690,23 @@ func parseMappingIDs(raw json.RawMessage) ([]int64, error) {
 }
 
 func matchesAnyGlob(name string, patterns []string) bool {
+	name = filepath.ToSlash(name)
+	base := filepath.Base(filepath.FromSlash(name))
 	for _, pattern := range patterns {
-		if ok, _ := filepath.Match(pattern, name); ok {
+		pattern = filepath.ToSlash(strings.TrimSpace(pattern))
+		if pattern == "" {
+			continue
+		}
+		if ok, _ := filepath.Match(filepath.FromSlash(pattern), filepath.FromSlash(name)); ok {
 			return true
 		}
+		// Backwards compatibility: filename-only patterns continue to match files
+		// inside newly supported nested knowledge directories.
+		if !strings.Contains(pattern, "/") {
+			if ok, _ := filepath.Match(pattern, base); ok {
+				return true
+			}
+		}
 	}
 	return false
 }

services/agent/internal/knowledge/store_test.go

@@ -716,3 +716,109 @@ func TestNeuroForgeSearchFailureHonorsFailOpenPolicy(t *testing.T) {
 		t.Fatal("fail-closed search must surface backend failure")
 	}
 }
+
+func TestKnowledgeLoadsAndIncrementallyTracksNestedJSONFiles(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	nested := filepath.Join(dir, "department", "network")
+	if err := os.MkdirAll(nested, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	doc1 := `{"id":"NESTED-1","title":"VPN nested","text":"gateway vpn","answer":"x","source":"internal-kb","language":"de-DE","communication_style":"formal"}`
+	if err := os.WriteFile(filepath.Join(nested, "vpn.json"), []byte(doc1), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Count() != 1 {
+		t.Fatalf("nested initial count=%d, want 1", s.Count())
+	}
+	if _, ok := s.manifest["static/department/network/vpn.json"]; !ok {
+		t.Fatalf("nested manifest key missing: %#v", s.manifest)
+	}
+
+	doc2 := `{"id":"NESTED-2","title":"DNS nested","text":"resolver dns","answer":"x","source":"internal-kb","language":"de-DE","communication_style":"formal"}`
+	deeper := filepath.Join(dir, "department", "dns", "prod")
+	if err := os.MkdirAll(deeper, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(deeper, "dns.json"), []byte(doc2), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.SyncLocal(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if s.Count() != 2 {
+		t.Fatalf("nested incremental count=%d, want 2", s.Count())
+	}
+	if _, ok := s.manifest["static/department/dns/prod/dns.json"]; !ok {
+		t.Fatalf("nested incremental manifest key missing: %#v", s.manifest)
+	}
+}
+
+func TestNestedKnowledgeSnapshotRebindsFullRelativePath(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	nested := filepath.Join(dir, "a", "b")
+	if err := os.MkdirAll(nested, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	doc := `{"id":"SNAP-NESTED","title":"Nested snapshot","text":"nested knowledge","answer":"x","source":"internal-kb","language":"de-DE","communication_style":"formal"}`
+	path := filepath.Join(nested, "item.json")
+	if err := os.WriteFile(path, []byte(doc), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	first, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first.files["SNAP-NESTED"] != path {
+		t.Fatalf("initial nested path=%q, want %q", first.files["SNAP-NESTED"], path)
+	}
+
+	second, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if second.files["SNAP-NESTED"] != path {
+		t.Fatalf("rebound nested path=%q, want %q", second.files["SNAP-NESTED"], path)
+	}
+	if !second.InitStatus().SnapshotLoaded {
+		t.Fatal("expected second load to use persistent snapshot")
+	}
+}
+
+func TestInitializeReconcilesFilesAddedAfterEmptySnapshot(t *testing.T) {
+	dir := t.TempDir()
+	data := t.TempDir()
+	first, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first.Count() != 0 {
+		t.Fatalf("initial count=%d, want 0", first.Count())
+	}
+
+	nested := filepath.Join(dir, "import", "batch-1")
+	if err := os.MkdirAll(nested, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	doc := `{"id":"AFTER-SNAPSHOT","title":"Imported after snapshot","text":"new knowledge","answer":"x","source":"internal-kb","language":"de-DE","communication_style":"formal"}`
+	if err := os.WriteFile(filepath.Join(nested, "new.json"), []byte(doc), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	second, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if second.Count() != 1 {
+		t.Fatalf("reconciled count=%d, want 1", second.Count())
+	}
+	if _, ok := second.ByID("AFTER-SNAPSHOT"); !ok {
+		t.Fatal("file added after snapshot was not reconciled during Initialize")
+	}
+}