update-1.6.2b
ci / docker-build (push) Failing after 24s
ci / test (services/knowledge) (push) Successful in 42s
ci / test (services/control) (push) Successful in 49s
ci / test (services/agent) (push) Successful in 58s
ci / test (platform/neuroforge) (push) Successful in 1m8s
release-tag / release-image (push) Successful in 6m30s
ci / docker-build (push) Failing after 24s
ci / test (services/knowledge) (push) Successful in 42s
ci / test (services/control) (push) Successful in 49s
ci / test (services/agent) (push) Successful in 58s
ci / test (platform/neuroforge) (push) Successful in 1m8s
release-tag / release-image (push) Successful in 6m30s
This commit is contained in:
@@ -655,6 +655,8 @@ GLPI_KB_FILTER=
|
||||
GLPI_KB_LIMIT=500
|
||||
# Synchronisationsintervall.
|
||||
GLPI_KB_SYNC_INTERVAL=10m
|
||||
# Gesamtzeitbudget fuer einen kompletten GLPI-KB-Refresh inklusive Embeddings.
|
||||
GLPI_KB_SYNC_TIMEOUT=30m
|
||||
# source-Wert importierter GLPI-KB-Artikel.
|
||||
GLPI_KB_SOURCE=glpi-kb
|
||||
# true:
|
||||
|
||||
+17
-17
@@ -1,5 +1,5 @@
|
||||
27dc46be5cbb1b171deff7fbd2f28bff1be802dff403797535fd8968bb98c8eb ./.cbmignore
|
||||
079e174cd38e79421469cb5fc79ff4d886015cd5ec97c538466f97d70ce5331a ./.env.example
|
||||
6b603e52548748b2e9717e53543905bc323d25ce3e541643e6bb0d4f4bc8bbce ./.env.example
|
||||
813c6a53b389380997fc71835ef72d3b593e1808c4fcc16dc0c309608a389f40 ./.env.subagent.example
|
||||
2d49d2ee955bf352cc7e76068ec842808c0bf7bffaacb6a129b641a78bbde0e5 ./.gitea/workflows/ci.yml
|
||||
ed22fda7661db8203563611dc144998161cd024e161b0471d615eaf0defeb7db ./.gitea/workflows/release-tag-gitea.yml
|
||||
@@ -61,7 +61,7 @@ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ./deployments/
|
||||
5d2ca8625ccb84cb868c3f7450a0c63fc128967aab5677363dfeab8f214f92c3 ./docker-bake.hcl
|
||||
58db2a87b703a5bbddc20b140907d187f8388e7db87b079ae9f6ffaac9c121fc ./docker-compose.dev.yml
|
||||
1360e44c867fdb9b754fe4585d57a02daf0f4820a7602a4e588c94c45eeb28e8 ./docker-compose.subagent.yml
|
||||
0b5325d5e86ff75292031316d8ea2837e8c3eecbf77dcb67db39407edc1189fe ./docker-compose.yml
|
||||
7883c941cbe0be323601661567168aebfd7ec2f441063c453cfce56fb7af1083 ./docker-compose.yml
|
||||
e4c516375e450c43b5193b1257880c02f2132b05a16e19ab1c96d57182fa070c ./docs/ARCHITECTURE.md
|
||||
34e2c7ac5992389cc51b40f740c1e206f20bffc67bdf19cb59d94b5df67a7082 ./docs/BACKUP-RESTORE-v1.5.0.md
|
||||
9467a3c0796a87bada0eab6ad191913e628b4abc05520cbb4b73fac76d334481 ./docs/CODEBASE-MEMORY-MCP.md
|
||||
@@ -413,11 +413,11 @@ a2c56eb668137e058cb78bb592ac359ef7e608cb9bedc7eaa930b3f4f66411ca ./scripts/stat
|
||||
2970d5f5049323b14a3f3983c2f4eb5392d745b4fea34742e3cf56e52d740f17 ./scripts/validate.sh
|
||||
cc0fc4433db0964ed172a82f2910e55c0f3dd6d59b1ec04fe0541b8fe5a912bb ./scripts/verify-package.sh
|
||||
17d7149607bf40ccc6f64a3c4cf3e4023a8325675918c18df0be309ea9731a36 ./services/agent/.dockerignore
|
||||
68f757ffe33c55b622f0f2f3d907530419ac649e8510c47eaa8a05e08788e602 ./services/agent/.env.example
|
||||
4db4de9cac0f05961cfdc20b8c7e741cdb64d71432f4b5162fa005cf2c19a4d3 ./services/agent/.env.example
|
||||
236713daf159ff0a8067e80a442ae3404fa28a5251ae6f24782f263bcfc17005 ./services/agent/.gitea/workflows/registry.yml
|
||||
9ee46f2156ee805c4ff40e418b004ead5cfbc3a9b6df3968a1b772fe1b313cdc ./services/agent/.gitignore
|
||||
4c893d6049499eb70ca755a333e97f0a726dd212e4ded4a9d4acabf0a25d9bd8 ./services/agent/BETRIEBSANLEITUNG_GLPI_AI_AGENT.md
|
||||
9339ce57dfc58d4731ea02c8d0ab9f2835e5966a29c071062fc6a2effd17da2c ./services/agent/BETRIEBSANLEITUNG_GLPI_AI_AGENT_OLLAMA_POOL.md
|
||||
e947cd34f7188f09581e5d04dd45ceed6fcef530d88ca4138cc4299cec08b43e ./services/agent/BETRIEBSANLEITUNG_GLPI_AI_AGENT.md
|
||||
5343fe8bde3261ac5cb8e60759a43e7ea44ca647287d1fad7dfc350672f799ac ./services/agent/BETRIEBSANLEITUNG_GLPI_AI_AGENT_OLLAMA_POOL.md
|
||||
81c91a44c5d3b7eca3051d8cee35f231ef283563da7a636c56c98b61da60218b ./services/agent/Dockerfile
|
||||
2ec93f81187ebf92665a6a15a6672b2f7dfce23bfea1546f2d4188f49de5dce0 ./services/agent/EMERGENCY-HOTFIX-TICKETVERARBEITUNG.md
|
||||
2bce6e10780004bc12042a416b8a05b417130db2d730d01a95d9d86a564e01cd ./services/agent/ESCALATION.md
|
||||
@@ -432,10 +432,10 @@ c0e13c6703cf3feda57a9eaaa718e501280fbb526f466040eb685a27700b5797 ./services/age
|
||||
0ed7ee0e4846be09994dd5f3b49ffdca6b59177b9018b1947388949f0dd49a14 ./services/agent/Makefile
|
||||
80b20d2c2c78badc0dcc47e485e46bfd647877445c35faf261fb677ad58e8359 ./services/agent/OLLAMA-POOL.md
|
||||
80b20d2c2c78badc0dcc47e485e46bfd647877445c35faf261fb677ad58e8359 ./services/agent/OLLAMA_POOL_BETRIEB.md
|
||||
a7a3f969150005ef062e7a77cf852371931bc2fa5ca4be01cb0e9eb07d35335c ./services/agent/README.md
|
||||
b0e1f907b26b31f3df4f9fae9ceaa59e468f3aa30487695fec89213912b685c8 ./services/agent/README.md
|
||||
aeac85d1adc92a8b34ffa5a95f7f19c2c6fb94742a7359a7fd452529d59ad87b ./services/agent/SECURITY.md
|
||||
edb92cdf2d1863df53ea8045657f64e415f206d503c8da3b45ea96b7f7f0960d ./services/agent/UPGRADE.md
|
||||
a499188fd29230a52949f2d5252d64d859b727e9d94a00889a025f30a1fa8981 ./services/agent/cmd/agent/main.go
|
||||
ee0b752092cd437e215c05064560b455e844c4a190780afe70900b9a275a2c2b ./services/agent/UPGRADE.md
|
||||
5f12f3841c224f7a1658ec34fcb93d7ae8d91030f80469ecba25bfeae0c91c61 ./services/agent/cmd/agent/main.go
|
||||
98be0fd63dd4fde52342c5385da113c926ac29195aedb657c0ca0f98e1593fff ./services/agent/compose_local.yml
|
||||
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ./services/agent/data/.gitkeep
|
||||
b74f901cca99eae4bed1b168f1f077a28bf1134a71c867d263c7834f6d896684 ./services/agent/deploy/glpi-ai-agent.service
|
||||
@@ -458,19 +458,19 @@ c4bf8fcb4ba6eb7962ec55cacff806a8a99561d134ab5238abd012763a2f9c74 ./services/age
|
||||
6b70a29176602e499935a85ac8831dbe36bf75fddb130a56cda376e124ecb7db ./services/agent/internal/agent/status_reply.go
|
||||
6f95bd3c85b42f1b2b098f05828c0b9e3dbb2e57f1f89b3dc4d1806453e0c4ce ./services/agent/internal/agent/status_reply_test.go
|
||||
9057a742c3fe8e0fa76efd6dd9bcdf9fcdf7ab2c37ed823822808946e8649566 ./services/agent/internal/brainactivity/client.go
|
||||
c8da5784acc4f28d632f357dadbfc61209a37b698525dae40473d27625d9c8e2 ./services/agent/internal/config/config.go
|
||||
3ce010749e2d378249e2da9ebd0471673a6c558db9273940788ddcc096d7103d ./services/agent/internal/config/config.go
|
||||
fa1717032ad536bff867a07d3fdf0ae2e0a9d5d7637a27b43274832d51322eb6 ./services/agent/internal/config/config_test.go
|
||||
6fe7335b0e66b60d5f6f27583b1ac11a0948719cfa657d51e77de0b337aab5d5 ./services/agent/internal/contextdata/collector.go
|
||||
12e9eb6fd567b5d4173a63fa2b89be913b6d1c97e17b69048f46d119e88dce26 ./services/agent/internal/contextdata/collector_test.go
|
||||
cffe6bc72ac01f0eb96599a40af43941c0719bd8cb698eddb4441fbf11703c7a ./services/agent/internal/glpi/client.go
|
||||
6d4846d6f4e327c8c55ebf7a83c9206bcc60364e739976c282873e5508f1789a ./services/agent/internal/glpi/client_test.go
|
||||
247324ba41f3cd991a3582ed270d0b0d6c4c92c36f11e7742c20d3e59b982928 ./services/agent/internal/glpikb/sync.go
|
||||
9836e2bf9153df638ead57d92c152b87f831e5a3696bcda4b1f650ff38e9ac70 ./services/agent/internal/glpikb/sync_test.go
|
||||
f48e63d605790b9b6e3307accf206034c98ec06e7b8c7900d33b6e44b4f97e6c ./services/agent/internal/glpikb/sync.go
|
||||
6411c727c1b28da1748bd2ab08da3ea4621dfd32bdf36c02c3ea4253bac50532 ./services/agent/internal/glpikb/sync_test.go
|
||||
787ab6b53623aa36ef6628f4eada11dcb4ed3011b34effd691bf07ba5d8f214d ./services/agent/internal/knowledge/category_mapping.go
|
||||
d3a9909672ea420db87be29ce669f3cd973125ff15bb46ca90a54c3aa98d9e73 ./services/agent/internal/knowledge/neuroforge_backend.go
|
||||
7bcd33bfcfb99fb06762fe75f4646308428957ed7eca26a266891ab580c3a929 ./services/agent/internal/knowledge/persistent_index.go
|
||||
df1ac214ebb88bf3ce7f68112129cf9ffc300dd725380c38661b85b04908ecdf ./services/agent/internal/knowledge/store.go
|
||||
63e47d86bcc89178fdc0f91e07ecb30bd01ee4b84bf29bfc660debbf26f8a40b ./services/agent/internal/knowledge/store_test.go
|
||||
151a88dcb7e649473acc68765058d660d7877e6cf2b111b301b41d6950d59c43 ./services/agent/internal/knowledge/persistent_index.go
|
||||
50342192dca4b4b06c560f37f6f6d139b51d8def89923a02243f5db29f41fe2a ./services/agent/internal/knowledge/store.go
|
||||
2b462c8e4eaae31420a1bc0b77d41ac9d1b2f55cab616d1e206fd622fc37b6ef ./services/agent/internal/knowledge/store_test.go
|
||||
807996228b38e1cd97ec2516cb1ecfc4bd945b638fcde465a0991051df2f1a00 ./services/agent/internal/learning/outcomes.go
|
||||
2917b3dab63783af4c92c48f269f097dc94b989008a5451f46d6c1b309ccdc9b ./services/agent/internal/learning/outcomes_test.go
|
||||
482ca79552b02a8a53bbd547318a04e01a0b41e64d4a347b00f850c0218febda ./services/agent/internal/learning/store.go
|
||||
@@ -481,8 +481,8 @@ df1ac214ebb88bf3ce7f68112129cf9ffc300dd725380c38661b85b04908ecdf ./services/age
|
||||
821850d649c7661f1665dc6be030ebeeacb60e4e007e1e84891d4b779b9a5d0d ./services/agent/internal/model/reason_codes_test.go
|
||||
c89248224adf72720289215684b916eb209387c1d38b757d64804a8d3148011c ./services/agent/internal/obsidian/export.go
|
||||
b75ffec53e9a94505ee57ecb9361bffbb6805cfd06ea05a7bf68b6b28b268c09 ./services/agent/internal/obsidian/export_test.go
|
||||
f4e32c5653b5fc3e046f1536513d376a7c0ac11b392210a107e12bd0ee63e179 ./services/agent/internal/ollama/client.go
|
||||
3522e7687e38db771dcf7e05b8dc52ad99a2fbc53d98dbdf07ded781d68b248e ./services/agent/internal/ollama/client_test.go
|
||||
73ab85693941c2af0ab6999a366e4d5baab629405233869699001123110bed32 ./services/agent/internal/ollama/client.go
|
||||
32949d24dbfef098426ed1891d27860b295a8d7f312972020beb6976360705e3 ./services/agent/internal/ollama/client_test.go
|
||||
0e4e98a3db5ad781a4f7022bd74bbc4b434e64187afee077daf9174165509ea6 ./services/agent/internal/ollama/pool.go
|
||||
4da6a7f953bfce654dbd8ab2267d1d519057fb230773e37538ce81aa049f9751 ./services/agent/internal/ollama/pool_test.go
|
||||
ab29c5808a06f6b21a0776355eeaa030f69be5eee251e7b1dbd7e00c71045c61 ./services/agent/internal/prioritysignals/signals.go
|
||||
@@ -607,7 +607,7 @@ f1eab883370e0a40ef52a6b6d785a510d8ed48a19d25e0a2bc95f4f2cc8e329e ./services/age
|
||||
895f8400aaad550ff2b96262eaa54e954d603129682e97aa4964539eb19bd7c2 ./services/agent/run.ps1
|
||||
d1d4ab8145a7131c36df51d3a9fbdcdeb0cda3de134fe24c0e8d52370862f3b2 ./services/control/Dockerfile
|
||||
f30ddf9251860d92717276483f7a2c2d7405f0516940ca62abc21d97ff3f3ad2 ./services/control/cmd/engineering-graph/main.go
|
||||
c61a014f75ceb8c2d9365d5c053d952f80cb60faa1fbc4f3049999450b49adae ./services/control/engineering-graph.json
|
||||
5b2b6ec4790c89d07834c4e52bbde9a75d67f72dd0ec1b5b4ad033958043fac3 ./services/control/engineering-graph.json
|
||||
ba44c599b9faf861eca647614c4abd18548e2708233cf462b11eb277e96449b8 ./services/control/go.mod
|
||||
da37393e58ff53847f26b1051f6d3a5270571d290b33d79360bae0dcb3513829 ./services/control/graph.go
|
||||
e6a8021a44219219fe37ab5edc45d1908fc6579f653aa80c9966810d891ad27d ./services/control/graph_test.go
|
||||
|
||||
@@ -306,6 +306,7 @@ services:
|
||||
GLPI_KB_PATH: ${GLPI_KB_PATH:-}
|
||||
GLPI_KB_SOURCE: ${GLPI_KB_SOURCE:-}
|
||||
GLPI_KB_SYNC_INTERVAL: ${GLPI_KB_SYNC_INTERVAL:-}
|
||||
GLPI_KB_SYNC_TIMEOUT: ${GLPI_KB_SYNC_TIMEOUT:-}
|
||||
GLPI_MAJOR_INCIDENT_FILTER: ${GLPI_MAJOR_INCIDENT_FILTER:-}
|
||||
GLPI_MAJOR_INCIDENT_LIMIT: ${GLPI_MAJOR_INCIDENT_LIMIT:-}
|
||||
GLPI_PASSWORD: ${GLPI_PASSWORD:-}
|
||||
|
||||
@@ -518,6 +518,8 @@ GLPI_KB_FILTER=
|
||||
GLPI_KB_LIMIT=500
|
||||
# Synchronisationsintervall.
|
||||
GLPI_KB_SYNC_INTERVAL=10m
|
||||
# Gesamtzeitbudget fuer einen kompletten GLPI-KB-Refresh inklusive Embeddings.
|
||||
GLPI_KB_SYNC_TIMEOUT=30m
|
||||
# source-Wert importierter GLPI-KB-Artikel.
|
||||
GLPI_KB_SOURCE=glpi-kb
|
||||
# true:
|
||||
|
||||
@@ -816,6 +816,7 @@ Für gemeinsam genutzte Knowledge-Verzeichnisse ist `unscoped` der kompatibelste
|
||||
| `GLPI_KB_FILTER` | Agent | Optionaler serverseitiger GLPI-Filter. alle für den Service Account sichtbaren Artikel, begrenzt durch LIMIT. | Freier Text beziehungsweise installationsspezifischer Wert. | leer | leer | Optional; Wirkung abhängig von aktivierten Funktionen. |
|
||||
| `GLPI_KB_LIMIT` | Agent | Maximale Anzahl GLPI-KB-Artikel. | 1–5000. | 500 | 500 | Optional; Wirkung abhängig von aktivierten Funktionen. |
|
||||
| `GLPI_KB_SYNC_INTERVAL` | Agent | Synchronisationsintervall. | Dauer >= 1m. | 10m | 10m | Optional; Wirkung abhängig von aktivierten Funktionen. |
|
||||
| `GLPI_KB_SYNC_TIMEOUT` | Agent | Gesamtzeitbudget eines GLPI-Wissens-Syncs inklusive Embedding/Reindexierung. `GLPI_TIMEOUT` bleibt das Timeout je GLPI-HTTP-Request. | 0 oder Dauer >= 1m. | 30m | 30m | Für große KBs ausreichend hoch wählen. |
|
||||
| `GLPI_KB_SOURCE` | Agent | source-Wert importierter GLPI-KB-Artikel. | Freier Text beziehungsweise installationsspezifischer Wert. | glpi-kb | glpi-kb | Optional; Wirkung abhängig von aktivierten Funktionen. |
|
||||
| `GLPI_KB_AUTO_REPLY` | Agent | GLPI-KB-Artikel können grundsätzlich Auto-Replies auslösen. Zusätzlich gelten weiterhin alle anderen Policy-Gates. | true \| false | false | true | Bei true: `GLPI_KB_SOURCE` muss in normalen und Auto-Reply-Quellen stehen; außerdem ist mindestens eine KB-Kategorie oder eine explizite Allowlist unkategorisierter Artikel erforderlich. |
|
||||
| `GLPI_KB_AUTO_REPLY_CATEGORY_IDS` | Agent | Whitelist der GLPI-Knowledge-Base-Kategorie-IDs. Ein kategorisierter Artikel ist nur dann grundsätzlich für Auto-Reply freigegeben, wenn mindestens eine seiner KB-Kategorien enthalten ist. Dies sind nicht die ITIL-/Ticketkategorie-IDs. | Kommagetrennte positive GLPI-KB-Kategorie-IDs; leer/none = keine. | leer | 1 | Kann allein oder zusammen mit der Allowlist unkategorisierter Artikel verwendet werden. |
|
||||
|
||||
@@ -816,6 +816,7 @@ Für gemeinsam genutzte Knowledge-Verzeichnisse ist `unscoped` der kompatibelste
|
||||
| `GLPI_KB_FILTER` | Agent | Optionaler serverseitiger GLPI-Filter. alle für den Service Account sichtbaren Artikel, begrenzt durch LIMIT. | Freier Text beziehungsweise installationsspezifischer Wert. | leer | leer | Optional; Wirkung abhängig von aktivierten Funktionen. |
|
||||
| `GLPI_KB_LIMIT` | Agent | Maximale Anzahl GLPI-KB-Artikel. | 1–5000. | 500 | 500 | Optional; Wirkung abhängig von aktivierten Funktionen. |
|
||||
| `GLPI_KB_SYNC_INTERVAL` | Agent | Synchronisationsintervall. | Dauer >= 1m. | 10m | 10m | Optional; Wirkung abhängig von aktivierten Funktionen. |
|
||||
| `GLPI_KB_SYNC_TIMEOUT` | Agent | Gesamtzeitbudget eines GLPI-Wissens-Syncs inklusive Embedding/Reindexierung. `GLPI_TIMEOUT` bleibt das Timeout je GLPI-HTTP-Request. | 0 oder Dauer >= 1m. | 30m | 30m | Für große KBs ausreichend hoch wählen. |
|
||||
| `GLPI_KB_SOURCE` | Agent | source-Wert importierter GLPI-KB-Artikel. | Freier Text beziehungsweise installationsspezifischer Wert. | glpi-kb | glpi-kb | Optional; Wirkung abhängig von aktivierten Funktionen. |
|
||||
| `GLPI_KB_AUTO_REPLY` | Agent | GLPI-KB-Artikel können grundsätzlich Auto-Replies auslösen. Zusätzlich gelten weiterhin alle anderen Policy-Gates. | true \| false | false | true | Bei true: GLPI_KB_SOURCE muss in normalen und Auto-Reply-Quellen stehen; Kategorie-ID-Whitelist darf nicht leer sein. |
|
||||
| `GLPI_KB_AUTO_REPLY_CATEGORY_IDS` | Agent | Whitelist der GLPI KNOWLEDGE-BASE-Kategorie-IDs. WICHTIG: Dies sind NICHT die ITIL-/Ticketkategorie-IDs. Mehrere Werte: 1,2,7 | Kommagetrennte positive GLPI-KB-Kategorie-IDs; leer/none = keine. | nicht ermittelt | 1 | Optional; Wirkung abhängig von aktivierten Funktionen. |
|
||||
|
||||
@@ -244,6 +244,7 @@ KNOWLEDGE_ALLOWED_SOURCES=internal-kb,glpi-kb
|
||||
GLPI_KB_ENABLED=true
|
||||
GLPI_KB_PATH=auto
|
||||
GLPI_KB_SYNC_INTERVAL=10m
|
||||
GLPI_KB_SYNC_TIMEOUT=30m
|
||||
GLPI_KB_LIMIT=500
|
||||
GLPI_KB_SOURCE=glpi-kb
|
||||
```
|
||||
|
||||
@@ -196,6 +196,7 @@ KNOWLEDGE_ALLOWED_SOURCES=internal-kb,glpi-kb
|
||||
GLPI_KB_ENABLED=true
|
||||
GLPI_KB_PATH=auto
|
||||
GLPI_KB_SYNC_INTERVAL=10m
|
||||
GLPI_KB_SYNC_TIMEOUT=30m
|
||||
GLPI_KB_LIMIT=500
|
||||
GLPI_KB_SOURCE=glpi-kb
|
||||
GLPI_KB_AUTO_REPLY=false
|
||||
|
||||
@@ -189,7 +189,7 @@ func main() {
|
||||
if err := kbSync.LoadCache(ctx); err != nil {
|
||||
slog.Warn("GLPI knowledge cache unavailable", "error", err)
|
||||
}
|
||||
syncCtx, syncCancel := context.WithTimeout(ctx, maxDuration(cfg.GLPITimeout*3, 30*time.Second))
|
||||
syncCtx, syncCancel := context.WithTimeout(ctx, glpikb.SyncTimeout(cfg))
|
||||
if err := kbSync.Sync(syncCtx); err != nil {
|
||||
slog.Error("initial GLPI knowledge base sync failed; continuing with local/cache knowledge", "error", err)
|
||||
}
|
||||
|
||||
@@ -99,6 +99,7 @@ type Config struct {
|
||||
GLPIKBFilter string
|
||||
GLPIKBLimit int
|
||||
GLPIKBSyncInterval time.Duration
|
||||
GLPIKBSyncTimeout time.Duration
|
||||
GLPIKBSource string
|
||||
GLPIKBAutoReply bool
|
||||
GLPIKBAutoReplyCategoryIDs []int64
|
||||
@@ -306,6 +307,7 @@ func Load() (Config, error) {
|
||||
GLPIKBFilter: strings.TrimSpace(os.Getenv("GLPI_KB_FILTER")),
|
||||
GLPIKBLimit: envInt("GLPI_KB_LIMIT", 500),
|
||||
GLPIKBSyncInterval: envDuration("GLPI_KB_SYNC_INTERVAL", 10*time.Minute),
|
||||
GLPIKBSyncTimeout: envDuration("GLPI_KB_SYNC_TIMEOUT", 30*time.Minute),
|
||||
GLPIKBSource: strings.ToLower(env("GLPI_KB_SOURCE", "glpi-kb")),
|
||||
GLPIKBAutoReply: envBool("GLPI_KB_AUTO_REPLY", false),
|
||||
GLPIKBAutoReplyCategoryIDs: envInt64ListAllowEmpty("GLPI_KB_AUTO_REPLY_CATEGORY_IDS"),
|
||||
@@ -738,6 +740,9 @@ func (c Config) Validate() error {
|
||||
if c.GLPIKBSyncInterval < time.Minute {
|
||||
return errors.New("GLPI_KB_SYNC_INTERVAL must be at least 1m")
|
||||
}
|
||||
if c.GLPIKBSyncTimeout < 0 || (c.GLPIKBSyncTimeout > 0 && c.GLPIKBSyncTimeout < time.Minute) {
|
||||
return errors.New("GLPI_KB_SYNC_TIMEOUT must be 0 or at least 1m")
|
||||
}
|
||||
if c.GLPIKBPath != "auto" && !validAPIPath(c.GLPIKBPath) {
|
||||
return errors.New("GLPI_KB_PATH must be 'auto' or an absolute API path")
|
||||
}
|
||||
|
||||
@@ -51,7 +51,21 @@ type Syncer struct {
|
||||
count int
|
||||
}
|
||||
|
||||
const cachePolicyVersion = 2
|
||||
const (
|
||||
cachePolicyVersion = 2
|
||||
defaultKnowledgeSyncLimit = 30 * time.Minute
|
||||
)
|
||||
|
||||
// SyncTimeout is the total budget for one complete GLPI knowledge refresh,
|
||||
// including embedding/re-indexing. GLPI_TIMEOUT remains the per-request HTTP
|
||||
// timeout; using it as the total sync budget aborts larger knowledge bases long
|
||||
// before their embeddings can finish.
|
||||
func SyncTimeout(cfg config.Config) time.Duration {
|
||||
if cfg.GLPIKBSyncTimeout > 0 {
|
||||
return cfg.GLPIKBSyncTimeout
|
||||
}
|
||||
return defaultKnowledgeSyncLimit
|
||||
}
|
||||
|
||||
type cacheFile struct {
|
||||
PolicyVersion int `json:"policy_version"`
|
||||
@@ -246,7 +260,7 @@ func (s *Syncer) Start(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
c, cancel := context.WithTimeout(ctx, maxDuration(s.cfg.GLPITimeout*3, 30*time.Second))
|
||||
c, cancel := context.WithTimeout(ctx, SyncTimeout(s.cfg))
|
||||
if err := s.Sync(c); err != nil {
|
||||
slog.Error("GLPI knowledge base sync failed", "error", err)
|
||||
}
|
||||
|
||||
@@ -269,3 +269,12 @@ func TestSyncPreservesGLPIKnowledgeLinkedItems(t *testing.T) {
|
||||
t.Fatalf("unexpected linked item: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncTimeoutUsesDedicatedKnowledgeBudget(t *testing.T) {
|
||||
if got := SyncTimeout(config.Config{}); got != 30*time.Minute {
|
||||
t.Fatalf("default sync timeout=%s, want 30m", got)
|
||||
}
|
||||
if got := SyncTimeout(config.Config{GLPIKBSyncTimeout: 7 * time.Minute}); got != 7*time.Minute {
|
||||
t.Fatalf("configured sync timeout=%s, want 7m", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,12 +133,17 @@ func (s *Store) loadPersistentSnapshot() (bool, error) {
|
||||
// them to the current knowledge roots using the stable manifest key.
|
||||
reboundFiles := map[string]string{}
|
||||
for key, rec := range snap.Manifest {
|
||||
name := filepath.Base(key)
|
||||
rel := strings.TrimPrefix(key, "static/")
|
||||
root := s.dir
|
||||
if strings.HasPrefix(key, "managed/") {
|
||||
rec.Path = filepath.Join(s.managedDir, name)
|
||||
} else {
|
||||
rec.Path = filepath.Join(s.dir, name)
|
||||
rel = strings.TrimPrefix(key, "managed/")
|
||||
root = s.managedDir
|
||||
}
|
||||
rel = filepath.Clean(filepath.FromSlash(rel))
|
||||
if rel == "." || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return false, fmt.Errorf("persistent knowledge index contains invalid manifest path %q", key)
|
||||
}
|
||||
rec.Path = filepath.Join(root, rel)
|
||||
snap.Manifest[key] = rec
|
||||
if rec.Included && rec.ID != "" {
|
||||
if rec.Managed || reboundFiles[rec.ID] == "" {
|
||||
@@ -511,7 +516,7 @@ type deltaScanResult struct {
|
||||
|
||||
func (s *Store) scanDeltaDir(ctx context.Context, dir, origin string, managed bool, opts LoadOptions, old map[string]fileRecord, oldDocs map[string]model.KnowledgeDoc) (deltaScanResult, error) {
|
||||
res := deltaScanResult{docs: map[string]model.KnowledgeDoc{}, manifest: map[string]fileRecord{}}
|
||||
entries, err := os.ReadDir(dir)
|
||||
entries, err := listKnowledgeJSONFiles(dir)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("read knowledge directory %q: %w", dir, err)
|
||||
}
|
||||
@@ -519,16 +524,10 @@ func (s *Store) scanDeltaDir(ctx context.Context, dir, origin string, managed bo
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
|
||||
continue
|
||||
}
|
||||
key := origin + "/" + e.Name()
|
||||
path := filepath.Join(dir, e.Name())
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if matchesAnyGlob(e.Name(), opts.IgnoreGlobs) {
|
||||
key := origin + "/" + e.Rel
|
||||
path := e.Path
|
||||
info := e.Info
|
||||
if matchesAnyGlob(e.Rel, opts.IgnoreGlobs) {
|
||||
res.manifest[key] = fileRecord{Key: key, Path: path, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), Managed: managed, Ignored: true}
|
||||
res.stats.IgnoredFiles++
|
||||
continue
|
||||
@@ -572,7 +571,7 @@ func (s *Store) scanDeltaDir(ctx context.Context, dir, origin string, managed bo
|
||||
}
|
||||
d, unmapped, skip, err := decodeKnowledgeDoc(b, opts.CategoryMode, s.categoryMap)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("%s: %w", e.Name(), err)
|
||||
return res, fmt.Errorf("%s: %w", e.Rel, err)
|
||||
}
|
||||
rec := fileRecord{Key: key, Path: path, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), RawHash: rawHash, Managed: managed, Unmapped: append([]string(nil), unmapped...)}
|
||||
if len(unmapped) > 0 {
|
||||
@@ -587,14 +586,14 @@ func (s *Store) scanDeltaDir(ctx context.Context, dir, origin string, managed bo
|
||||
continue
|
||||
}
|
||||
if d.ID == "" || d.Title == "" {
|
||||
return res, fmt.Errorf("%s: id/title required", e.Name())
|
||||
return res, fmt.Errorf("%s: id/title required", e.Rel)
|
||||
}
|
||||
if !safeID(d.ID) {
|
||||
return res, fmt.Errorf("%s: invalid id %q", e.Name(), d.ID)
|
||||
return res, fmt.Errorf("%s: invalid id %q", e.Rel, d.ID)
|
||||
}
|
||||
d.Source = strings.ToLower(strings.TrimSpace(d.Source))
|
||||
if d.Source == "" {
|
||||
return res, fmt.Errorf("%s: source required", e.Name())
|
||||
return res, fmt.Errorf("%s: source required", e.Rel)
|
||||
}
|
||||
if _, allowed := s.allowedSources[d.Source]; !allowed {
|
||||
res.manifest[key] = rec
|
||||
@@ -629,7 +628,7 @@ func mergeLoadStats(a, b LoadStats) LoadStats {
|
||||
return LoadStats{IgnoredFiles: a.IgnoredFiles + b.IgnoredFiles, UnmappedCategoryFiles: a.UnmappedCategoryFiles + b.UnmappedCategoryFiles, UnmappedCategories: mergeStrings(a.UnmappedCategories, b.UnmappedCategories)}
|
||||
}
|
||||
|
||||
func buildManifestForDocs(docs []model.KnowledgeDoc, files []string, origin string, managed bool) (map[string]fileRecord, error) {
|
||||
func buildManifestForDocs(docs []model.KnowledgeDoc, files []string, root, origin string, managed bool) (map[string]fileRecord, error) {
|
||||
out := make(map[string]fileRecord, len(docs))
|
||||
for i, d := range docs {
|
||||
if i >= len(files) {
|
||||
@@ -645,7 +644,11 @@ func buildManifestForDocs(docs []model.KnowledgeDoc, files []string, origin stri
|
||||
return nil, err
|
||||
}
|
||||
h := sha256.Sum256(b)
|
||||
key := origin + "/" + filepath.Base(path)
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := origin + "/" + filepath.ToSlash(rel)
|
||||
out[key] = fileRecord{Key: key, Path: path, ID: d.ID, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), RawHash: hex.EncodeToString(h[:]), Managed: managed, Included: true, Unmapped: append([]string(nil), d.UnmappedExternalCategories...)}
|
||||
}
|
||||
return out, nil
|
||||
@@ -688,24 +691,18 @@ func (s *Store) persistExternalVectorCache(source string) error {
|
||||
// delta scans do not repeatedly open files that are intentionally not part of
|
||||
// the active corpus.
|
||||
func augmentManifestAllFiles(dir, origin string, managed bool, opts LoadOptions, manifest map[string]fileRecord) error {
|
||||
entries, err := os.ReadDir(dir)
|
||||
entries, err := listKnowledgeJSONFiles(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
|
||||
continue
|
||||
}
|
||||
key := origin + "/" + e.Name()
|
||||
key := origin + "/" + e.Rel
|
||||
if _, ok := manifest[key]; ok {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path := filepath.Join(dir, e.Name())
|
||||
rec := fileRecord{Key: key, Path: path, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), Managed: managed, Included: false, Ignored: matchesAnyGlob(e.Name(), opts.IgnoreGlobs)}
|
||||
info := e.Info
|
||||
path := e.Path
|
||||
rec := fileRecord{Key: key, Path: path, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), Managed: managed, Included: false, Ignored: matchesAnyGlob(e.Rel, opts.IgnoreGlobs)}
|
||||
if !rec.Ignored {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
|
||||
@@ -237,6 +237,15 @@ func (s *Store) Initialize(ctx context.Context) (err error) {
|
||||
if _, err := s.syncLoadedSemanticBackend(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// A persistent snapshot is a warm start, not proof that the mounted
|
||||
// knowledge directory is unchanged. Reconcile it before ticket workers
|
||||
// start so files copied while the agent was stopped are available on the
|
||||
// first ticket instead of only after the background scan eventually ends.
|
||||
if mode == "incremental" {
|
||||
if err := s.SyncLocal(ctx); err != nil {
|
||||
return fmt.Errorf("reconcile persistent knowledge index: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
} else if mode == "readonly" {
|
||||
return fmt.Errorf("KNOWLEDGE_INDEX_MODE=readonly requires a compatible persistent index at %s", s.snapshotPath)
|
||||
@@ -294,11 +303,11 @@ func (s *Store) fullRebuild(ctx context.Context) (err error) {
|
||||
stats.UnmappedCategoryFiles += managedStats.UnmappedCategoryFiles
|
||||
stats.UnmappedCategories = mergeStrings(stats.UnmappedCategories, managedStats.UnmappedCategories)
|
||||
|
||||
staticManifest, err := buildManifestForDocs(static, staticFiles, "static", false)
|
||||
staticManifest, err := buildManifestForDocs(static, staticFiles, s.dir, "static", false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build static knowledge manifest: %w", err)
|
||||
}
|
||||
managedManifest, err := buildManifestForDocs(managed, managedFiles, "managed", true)
|
||||
managedManifest, err := buildManifestForDocs(managed, managedFiles, s.managedDir, "managed", true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build managed knowledge manifest: %w", err)
|
||||
}
|
||||
@@ -405,44 +414,71 @@ func (s *Store) InitStatus() InitStatus {
|
||||
|
||||
func (s *Store) Ready() bool { return s != nil && s.InitStatus().State == "ready" }
|
||||
|
||||
type knowledgeJSONFile struct {
|
||||
Rel string
|
||||
Path string
|
||||
Info os.FileInfo
|
||||
}
|
||||
|
||||
// listKnowledgeJSONFiles recursively discovers JSON knowledge documents. Large
|
||||
// exports are commonly grouped into category/subcategory folders; limiting the
|
||||
// scan to the top-level directory makes such a mounted corpus appear empty.
|
||||
func listKnowledgeJSONFiles(dir string) ([]knowledgeJSONFile, error) {
|
||||
files := make([]knowledgeJSONFile, 0)
|
||||
err := filepath.WalkDir(dir, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() || !strings.HasSuffix(strings.ToLower(entry.Name()), ".json") {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(dir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files = append(files, knowledgeJSONFile{Rel: filepath.ToSlash(rel), Path: path, Info: info})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].Rel < files[j].Rel })
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func readDocs(dir string, allowed map[string]struct{}, opts LoadOptions, categoryMap map[string][]int64, progress func(total, processed, loaded int)) ([]model.KnowledgeDoc, []string, LoadStats, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
entries, err := listKnowledgeJSONFiles(dir)
|
||||
if err != nil {
|
||||
return nil, nil, LoadStats{}, fmt.Errorf("read knowledge directory %q: %w", dir, err)
|
||||
}
|
||||
var docs []model.KnowledgeDoc
|
||||
var files []string
|
||||
stats := LoadStats{}
|
||||
total := 0
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
|
||||
total++
|
||||
}
|
||||
}
|
||||
total := len(entries)
|
||||
processed := 0
|
||||
if progress != nil {
|
||||
progress(total, 0, 0)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(strings.ToLower(e.Name()), ".json") {
|
||||
continue
|
||||
}
|
||||
processed++
|
||||
if matchesAnyGlob(e.Name(), opts.IgnoreGlobs) {
|
||||
if matchesAnyGlob(e.Rel, opts.IgnoreGlobs) {
|
||||
stats.IgnoredFiles++
|
||||
if progress != nil {
|
||||
progress(total, processed, len(docs))
|
||||
}
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(dir, e.Name())
|
||||
b, err := os.ReadFile(path)
|
||||
b, err := os.ReadFile(e.Path)
|
||||
if err != nil {
|
||||
return nil, nil, stats, err
|
||||
}
|
||||
d, unmapped, skip, err := decodeKnowledgeDoc(b, opts.CategoryMode, categoryMap)
|
||||
if err != nil {
|
||||
return nil, nil, stats, fmt.Errorf("%s: %w", e.Name(), err)
|
||||
return nil, nil, stats, fmt.Errorf("%s: %w", e.Rel, err)
|
||||
}
|
||||
if len(unmapped) > 0 {
|
||||
stats.UnmappedCategoryFiles++
|
||||
@@ -456,14 +492,14 @@ func readDocs(dir string, allowed map[string]struct{}, opts LoadOptions, categor
|
||||
continue
|
||||
}
|
||||
if d.ID == "" || d.Title == "" {
|
||||
return nil, nil, stats, fmt.Errorf("%s: id/title required", e.Name())
|
||||
return nil, nil, stats, fmt.Errorf("%s: id/title required", e.Rel)
|
||||
}
|
||||
if !safeID(d.ID) {
|
||||
return nil, nil, stats, fmt.Errorf("%s: invalid id %q", e.Name(), d.ID)
|
||||
return nil, nil, stats, fmt.Errorf("%s: invalid id %q", e.Rel, d.ID)
|
||||
}
|
||||
d.Source = strings.ToLower(strings.TrimSpace(d.Source))
|
||||
if d.Source == "" {
|
||||
return nil, nil, stats, fmt.Errorf("%s: source required", e.Name())
|
||||
return nil, nil, stats, fmt.Errorf("%s: source required", e.Rel)
|
||||
}
|
||||
if _, ok := allowed[d.Source]; !ok {
|
||||
if progress != nil {
|
||||
@@ -474,7 +510,7 @@ func readDocs(dir string, allowed map[string]struct{}, opts LoadOptions, categor
|
||||
d.Language = strings.TrimSpace(d.Language)
|
||||
d.CommunicationStyle = strings.ToLower(strings.TrimSpace(d.CommunicationStyle))
|
||||
docs = append(docs, d)
|
||||
files = append(files, path)
|
||||
files = append(files, e.Path)
|
||||
if progress != nil {
|
||||
progress(total, processed, len(docs))
|
||||
}
|
||||
@@ -654,10 +690,23 @@ func parseMappingIDs(raw json.RawMessage) ([]int64, error) {
|
||||
}
|
||||
|
||||
func matchesAnyGlob(name string, patterns []string) bool {
|
||||
name = filepath.ToSlash(name)
|
||||
base := filepath.Base(filepath.FromSlash(name))
|
||||
for _, pattern := range patterns {
|
||||
if ok, _ := filepath.Match(pattern, name); ok {
|
||||
pattern = filepath.ToSlash(strings.TrimSpace(pattern))
|
||||
if pattern == "" {
|
||||
continue
|
||||
}
|
||||
if ok, _ := filepath.Match(filepath.FromSlash(pattern), filepath.FromSlash(name)); ok {
|
||||
return true
|
||||
}
|
||||
// Backwards compatibility: filename-only patterns continue to match files
|
||||
// inside newly supported nested knowledge directories.
|
||||
if !strings.Contains(pattern, "/") {
|
||||
if ok, _ := filepath.Match(pattern, base); ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -716,3 +716,109 @@ func TestNeuroForgeSearchFailureHonorsFailOpenPolicy(t *testing.T) {
|
||||
t.Fatal("fail-closed search must surface backend failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKnowledgeLoadsAndIncrementallyTracksNestedJSONFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
data := t.TempDir()
|
||||
nested := filepath.Join(dir, "department", "network")
|
||||
if err := os.MkdirAll(nested, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doc1 := `{"id":"NESTED-1","title":"VPN nested","text":"gateway vpn","answer":"x","source":"internal-kb","language":"de-DE","communication_style":"formal"}`
|
||||
if err := os.WriteFile(filepath.Join(nested, "vpn.json"), []byte(doc1), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Count() != 1 {
|
||||
t.Fatalf("nested initial count=%d, want 1", s.Count())
|
||||
}
|
||||
if _, ok := s.manifest["static/department/network/vpn.json"]; !ok {
|
||||
t.Fatalf("nested manifest key missing: %#v", s.manifest)
|
||||
}
|
||||
|
||||
doc2 := `{"id":"NESTED-2","title":"DNS nested","text":"resolver dns","answer":"x","source":"internal-kb","language":"de-DE","communication_style":"formal"}`
|
||||
deeper := filepath.Join(dir, "department", "dns", "prod")
|
||||
if err := os.MkdirAll(deeper, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(deeper, "dns.json"), []byte(doc2), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.SyncLocal(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Count() != 2 {
|
||||
t.Fatalf("nested incremental count=%d, want 2", s.Count())
|
||||
}
|
||||
if _, ok := s.manifest["static/department/dns/prod/dns.json"]; !ok {
|
||||
t.Fatalf("nested incremental manifest key missing: %#v", s.manifest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNestedKnowledgeSnapshotRebindsFullRelativePath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
data := t.TempDir()
|
||||
nested := filepath.Join(dir, "a", "b")
|
||||
if err := os.MkdirAll(nested, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doc := `{"id":"SNAP-NESTED","title":"Nested snapshot","text":"nested knowledge","answer":"x","source":"internal-kb","language":"de-DE","communication_style":"formal"}`
|
||||
path := filepath.Join(nested, "item.json")
|
||||
if err := os.WriteFile(path, []byte(doc), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.files["SNAP-NESTED"] != path {
|
||||
t.Fatalf("initial nested path=%q, want %q", first.files["SNAP-NESTED"], path)
|
||||
}
|
||||
|
||||
second, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.files["SNAP-NESTED"] != path {
|
||||
t.Fatalf("rebound nested path=%q, want %q", second.files["SNAP-NESTED"], path)
|
||||
}
|
||||
if !second.InitStatus().SnapshotLoaded {
|
||||
t.Fatal("expected second load to use persistent snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeReconcilesFilesAddedAfterEmptySnapshot(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
data := t.TempDir()
|
||||
first, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.Count() != 0 {
|
||||
t.Fatalf("initial count=%d, want 0", first.Count())
|
||||
}
|
||||
|
||||
nested := filepath.Join(dir, "import", "batch-1")
|
||||
if err := os.MkdirAll(nested, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doc := `{"id":"AFTER-SNAPSHOT","title":"Imported after snapshot","text":"new knowledge","answer":"x","source":"internal-kb","language":"de-DE","communication_style":"formal"}`
|
||||
if err := os.WriteFile(filepath.Join(nested, "new.json"), []byte(doc), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
second, err := Load(context.Background(), dir, data, nil, false, []string{"internal-kb"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.Count() != 1 {
|
||||
t.Fatalf("reconciled count=%d, want 1", second.Count())
|
||||
}
|
||||
if _, ok := second.ByID("AFTER-SNAPSHOT"); !ok {
|
||||
t.Fatal("file added after snapshot was not reconciled during Initialize")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,9 +65,65 @@ func (c *Client) NodeStatuses() []model.OllamaNodeStatus { return c.pool.NodeSta
|
||||
func (c *Client) RoutingMode() string { return c.routingMode }
|
||||
func (c *Client) Embed(ctx context.Context, texts []string) ([][]float64, error) {
|
||||
ctx = withStage(ctx, "embedding")
|
||||
return c.embedAdaptive(ctx, texts, 0)
|
||||
}
|
||||
|
||||
const maxEmbeddingSplitDepth = 24
|
||||
|
||||
// embedAdaptive keeps truncate=false so knowledge is never silently discarded.
|
||||
// Some Ollama embedding models enforce their context window across the complete
|
||||
// input array, not only per individual string. When such a request is rejected,
|
||||
// split the batch and retry. If one pathological input still exceeds the model
|
||||
// context (for example a very long URL/base64-like token), split that text and
|
||||
// combine the partial embeddings into one length-weighted vector.
|
||||
func (c *Client) embedAdaptive(ctx context.Context, texts []string, depth int) ([][]float64, error) {
|
||||
if len(texts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
vectors, err := c.embedOnce(ctx, texts)
|
||||
if err == nil {
|
||||
return vectors, nil
|
||||
}
|
||||
if !isEmbeddingContextLengthError(err) || depth >= maxEmbeddingSplitDepth {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(texts) > 1 {
|
||||
mid := len(texts) / 2
|
||||
left, leftErr := c.embedAdaptive(ctx, texts[:mid], depth+1)
|
||||
if leftErr != nil {
|
||||
return nil, leftErr
|
||||
}
|
||||
right, rightErr := c.embedAdaptive(ctx, texts[mid:], depth+1)
|
||||
if rightErr != nil {
|
||||
return nil, rightErr
|
||||
}
|
||||
return append(left, right...), nil
|
||||
}
|
||||
|
||||
leftText, rightText, ok := splitEmbeddingText(texts[0])
|
||||
if !ok {
|
||||
return nil, err
|
||||
}
|
||||
left, leftErr := c.embedAdaptive(ctx, []string{leftText}, depth+1)
|
||||
if leftErr != nil {
|
||||
return nil, leftErr
|
||||
}
|
||||
right, rightErr := c.embedAdaptive(ctx, []string{rightText}, depth+1)
|
||||
if rightErr != nil {
|
||||
return nil, rightErr
|
||||
}
|
||||
if len(left) != 1 || len(right) != 1 {
|
||||
return nil, fmt.Errorf("adaptive embedding split returned invalid vector counts: left=%d right=%d", len(left), len(right))
|
||||
}
|
||||
combined, combineErr := weightedEmbeddingAverage(left[0], right[0], len([]rune(leftText)), len([]rune(rightText)))
|
||||
if combineErr != nil {
|
||||
return nil, combineErr
|
||||
}
|
||||
return [][]float64{combined}, nil
|
||||
}
|
||||
|
||||
func (c *Client) embedOnce(ctx context.Context, texts []string) ([][]float64, error) {
|
||||
payload := map[string]any{"model": c.embeddingModel, "input": texts, "truncate": false}
|
||||
var out struct {
|
||||
Embeddings [][]float64 `json:"embeddings"`
|
||||
@@ -80,6 +136,49 @@ func (c *Client) Embed(ctx context.Context, texts []string) ([][]float64, error)
|
||||
}
|
||||
return out.Embeddings, nil
|
||||
}
|
||||
|
||||
func isEmbeddingContextLengthError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "context length") ||
|
||||
strings.Contains(msg, "context window") ||
|
||||
(strings.Contains(msg, "context") && strings.Contains(msg, "exceed"))
|
||||
}
|
||||
|
||||
func splitEmbeddingText(text string) (string, string, bool) {
|
||||
runes := []rune(strings.TrimSpace(text))
|
||||
if len(runes) < 2 {
|
||||
return "", "", false
|
||||
}
|
||||
mid := len(runes) / 2
|
||||
left := strings.TrimSpace(string(runes[:mid]))
|
||||
right := strings.TrimSpace(string(runes[mid:]))
|
||||
if left == "" || right == "" {
|
||||
left = string(runes[:mid])
|
||||
right = string(runes[mid:])
|
||||
}
|
||||
if left == "" || right == "" || left == text || right == text {
|
||||
return "", "", false
|
||||
}
|
||||
return left, right, true
|
||||
}
|
||||
|
||||
func weightedEmbeddingAverage(a, b []float64, aWeight, bWeight int) ([]float64, error) {
|
||||
if len(a) == 0 || len(b) == 0 || len(a) != len(b) {
|
||||
return nil, fmt.Errorf("cannot combine embedding vectors with dimensions %d and %d", len(a), len(b))
|
||||
}
|
||||
if aWeight <= 0 || bWeight <= 0 {
|
||||
return nil, fmt.Errorf("cannot combine embedding vectors with non-positive weights %d and %d", aWeight, bWeight)
|
||||
}
|
||||
total := float64(aWeight + bWeight)
|
||||
out := make([]float64, len(a))
|
||||
for i := range a {
|
||||
out[i] = (a[i]*float64(aWeight) + b[i]*float64(bWeight)) / total
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (c *Client) AnalyseCategory(ctx context.Context, t model.Ticket, categories []model.Category, categoryHits []model.KnowledgeHit, contextData model.ContextSnapshot) (model.Decision, error) {
|
||||
ctx = withStage(ctx, "category")
|
||||
categoryIDs := []int64{0}
|
||||
|
||||
@@ -377,3 +377,106 @@ func TestAnalyseEscalationNormalizesNegativeDecision(t *testing.T) {
|
||||
t.Fatalf("negative escalation was not normalized: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbedSplitsBatchWhenOllamaContextWindowRejectsAggregateInput(t *testing.T) {
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
var body struct {
|
||||
Input []string `json:"input"`
|
||||
Truncate bool `json:"truncate"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Error(err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if body.Truncate {
|
||||
t.Errorf("adaptive embedding must preserve truncate=false")
|
||||
}
|
||||
if len(body.Input) > 1 {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": "estimated request size 5933 tokens exceeds model context length 2048"})
|
||||
return
|
||||
}
|
||||
value := map[string]float64{"a": 1, "b": 2, "c": 3}[body.Input[0]]
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float64{{value, 1}}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := New(srv.URL, "m", "e", "de-DE", "formal", time.Second, 256, time.Minute, false, 1, 0)
|
||||
vectors, err := c.Embed(context.Background(), []string{"a", "b", "c"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(vectors) != 3 || vectors[0][0] != 1 || vectors[1][0] != 2 || vectors[2][0] != 3 {
|
||||
t.Fatalf("unexpected vectors: %#v", vectors)
|
||||
}
|
||||
if calls != 5 {
|
||||
t.Fatalf("calls=%d, want 5 adaptive requests", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbedSplitsOversizedSingleInputWithoutSilentTruncation(t *testing.T) {
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
var body struct {
|
||||
Input []string `json:"input"`
|
||||
Truncate bool `json:"truncate"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Error(err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if body.Truncate {
|
||||
t.Errorf("adaptive embedding must preserve truncate=false")
|
||||
}
|
||||
if len(body.Input) != 1 {
|
||||
t.Errorf("input count=%d, want 1", len(body.Input))
|
||||
}
|
||||
n := len([]rune(body.Input[0]))
|
||||
if n > 8 {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": "input exceeds model context window"})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float64{{float64(n), 1}}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := New(srv.URL, "m", "e", "de-DE", "formal", time.Second, 256, time.Minute, false, 1, 0)
|
||||
vectors, err := c.Embed(context.Background(), []string{"abcdefghijklmnopqrstuvwxyz123456"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(vectors) != 1 || len(vectors[0]) != 2 {
|
||||
t.Fatalf("unexpected vectors: %#v", vectors)
|
||||
}
|
||||
if vectors[0][0] != 8 || vectors[0][1] != 1 {
|
||||
t.Fatalf("unexpected weighted aggregate vector: %#v", vectors[0])
|
||||
}
|
||||
if calls != 7 {
|
||||
t.Fatalf("calls=%d, want 7 adaptive requests", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbedDoesNotSplitUnrelatedBadRequest(t *testing.T) {
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": "model does not support embeddings"})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := New(srv.URL, "m", "e", "de-DE", "formal", time.Second, 256, time.Minute, false, 1, 0)
|
||||
_, err := c.Embed(context.Background(), []string{"a", "b"})
|
||||
if err == nil {
|
||||
t.Fatal("expected embedding error")
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("calls=%d, unrelated error must not trigger adaptive splitting", calls)
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user