Pooling
This commit is contained in:
95
.env.example
95
.env.example
@@ -225,40 +225,87 @@ GLPI_TICKET_FILTER=status.id==1
|
||||
# HTTP-Timeout für GLPI-Aufrufe.
|
||||
GLPI_TIMEOUT=20s
|
||||
###############################################################################
|
||||
# 10. GLPI AI AGENT - OLLAMA
|
||||
# 10. GLPI AI AGENT - OLLAMA-POOL
|
||||
###############################################################################
|
||||
# Ollama-Adresse aus Sicht des Agent-Containers.
|
||||
# Einzelnode-Kompatibilität. Wird nur verwendet, wenn OLLAMA_URLS leer ist.
|
||||
OLLAMA_URL=http://ollama:11434
|
||||
|
||||
# Mehrere Ollama-Instanzen, durch Komma getrennt. Alle Nodes sollten dieselbe
|
||||
# Ollama-Version, dasselbe Chat-Modell und dasselbe Embedding-Modell besitzen.
|
||||
# Beispiel für vorhandene Lenovo-Nodes:
|
||||
# OLLAMA_URLS=http://10.20.30.21:11434,http://10.20.30.22:11434,http://10.20.30.23:11434
|
||||
OLLAMA_URLS=
|
||||
|
||||
# Optionale lesbare Namen; Anzahl muss exakt zu OLLAMA_URLS passen.
|
||||
# OLLAMA_NODE_NAMES=lenovo-01,lenovo-02,lenovo-03
|
||||
OLLAMA_NODE_NAMES=
|
||||
|
||||
# Optionale Gewichte 1..100; nur für OLLAMA_ROUTING_MODE=weighted relevant.
|
||||
# OLLAMA_NODE_WEIGHTS=1,1,1
|
||||
OLLAMA_NODE_WEIGHTS=
|
||||
|
||||
# Routing-Modi:
|
||||
# least_inflight = Node mit den wenigsten laufenden Requests (empfohlen)
|
||||
# round_robin = zyklische Verteilung
|
||||
# weighted = Verteilung anhand OLLAMA_NODE_WEIGHTS und Auslastung
|
||||
# fastest_recent = bevorzugt die zuletzt schnellsten Nodes
|
||||
OLLAMA_ROUTING_MODE=least_inflight
|
||||
|
||||
# Maximale parallele Requests JE Node. Für integrierte GPUs/RAM-Sharing 1.
|
||||
OLLAMA_NODE_MAX_INFLIGHT=1
|
||||
|
||||
# Regelmäßige Prüfung von /api/tags.
|
||||
OLLAMA_NODE_HEALTH_INTERVAL=15s
|
||||
|
||||
# Nach einem retryfähigen Netzwerk-/HTTP-Fehler wird der Node so lange nicht
|
||||
# für neue Requests verwendet.
|
||||
OLLAMA_NODE_FAILURE_COOLDOWN=30s
|
||||
|
||||
# Maximalzeit für einen einzelnen Request an genau einen Node. Der übergeordnete
|
||||
# Analyse-Timeout kann kürzer sein und hat dann Vorrang.
|
||||
OLLAMA_NODE_REQUEST_TIMEOUT=10m
|
||||
|
||||
# Bei Netzwerkfehlern, HTTP 408/429/5xx oder ungültigem Response-JSON auf einen
|
||||
# anderen kompatiblen Node wechseln.
|
||||
OLLAMA_FAILOVER_ENABLED=true
|
||||
|
||||
# Maximale Anzahl verschiedener Nodes je logischem Request. 0 bedeutet:
|
||||
# automatisch alle konfigurierten Nodes. Ein positiver Wert darf höchstens der
|
||||
# Zahl der OLLAMA_URLS-Einträge entsprechen.
|
||||
OLLAMA_FAILOVER_ATTEMPTS=0
|
||||
|
||||
# Bei abweichenden Chat-/Embedding-Modelldigests wird der Pool vollständig
|
||||
# fail-closed. Für reproduzierbare Entscheidungen unbedingt true lassen.
|
||||
OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true
|
||||
|
||||
# true: Jeder Node muss auch OLLAMA_EMBEDDING_MODEL installiert haben.
|
||||
# Bei RAG empfohlen. false erlaubt Chat-only-Nodes; Embedding-Requests werden
|
||||
# trotzdem nur an Nodes mit erkanntem Embedding-Modell gesendet.
|
||||
OLLAMA_REQUIRE_EMBEDDING_MODEL=true
|
||||
|
||||
# OLLAMA_MODEL ist bereits oben im gemeinsamen Compose-/Ollama-Bereich gesetzt:
|
||||
# OLLAMA_MODEL=qwen3:8b
|
||||
# Embedding-Modell für RAG.
|
||||
OLLAMA_EMBEDDING_MODEL=embeddinggemma
|
||||
|
||||
# Modellspezifisches Retrieval-Prompting.
|
||||
#
|
||||
# Mögliche Werte:
|
||||
#
|
||||
# auto
|
||||
# Modell automatisch erkennen und passende Retrieval-Prompts verwenden.
|
||||
# Für embeddinggemma empfohlen.
|
||||
#
|
||||
# plain
|
||||
# keine modellspezifischen Retrieval-Prompts.
|
||||
# auto = Modell automatisch erkennen; für embeddinggemma empfohlen.
|
||||
# plain = keine modellspezifischen Retrieval-Prompts.
|
||||
KNOWLEDGE_EMBEDDING_PROFILE=auto
|
||||
# OLLAMA_TIMEOUT und OLLAMA_MAX_CONCURRENT sind bereits oben gesetzt.
|
||||
|
||||
# Gesamtbudget für Ollama-Aufrufe und Fallback für Node-Request-Timeouts.
|
||||
# OLLAMA_TIMEOUT ist bereits oben gesetzt.
|
||||
# OLLAMA_MAX_CONCURRENT bleibt als Legacy-Alias für
|
||||
# OLLAMA_NODE_MAX_INFLIGHT erhalten, falls der neue Wert nicht gesetzt ist.
|
||||
|
||||
# Maximale Anzahl generierter Tokens für strukturierte Antworten.
|
||||
OLLAMA_NUM_PREDICT=768
|
||||
# Wiederholungen bei fehlerhaftem / abgeschnittenem JSON.
|
||||
# Wiederholungen bei semantisch/strukturell fehlerhaftem Modell-JSON.
|
||||
# Diese Wiederholungen sind von Netzwerk-Failover getrennt.
|
||||
OLLAMA_JSON_RETRIES=1
|
||||
# Ollama-Modell nach Benutzung im Speicher halten.
|
||||
#
|
||||
# Beispiele:
|
||||
# 5m
|
||||
# 10m
|
||||
# 30m
|
||||
OLLAMA_KEEP_ALIVE=10m
|
||||
# Thinking bei unterstützten Modellen deaktivieren.
|
||||
#
|
||||
# Für strukturierte Klassifikations-/Policy-Aufgaben empfohlen.
|
||||
OLLAMA_THINK=false
|
||||
###############################################################################
|
||||
# 11. KNOWLEDGE BASE / RAG - BASIS
|
||||
@@ -726,8 +773,8 @@ GLPI_ESCALATION_LIMIT=100
|
||||
###############################################################################
|
||||
# Maximale Anzahl wartender Jobs.
|
||||
QUEUE_SIZE=256
|
||||
# Parallele Ticket-Worker.
|
||||
#
|
||||
# Darf größer als OLLAMA_MAX_CONCURRENT sein.
|
||||
# Ollama wird separat begrenzt.
|
||||
# Parallele Ticket-Worker. Der Ollama-Pool kann nur so viele unabhängige
|
||||
# Ticketpipelines gleichzeitig verteilen, wie Worker aktiv sind. Für drei
|
||||
# gleichartige Nodes ist WORKERS=3 ein sinnvoller Lasttest; jeder Node bleibt
|
||||
# zusätzlich durch OLLAMA_NODE_MAX_INFLIGHT begrenzt.
|
||||
WORKERS=2
|
||||
1104
BETRIEBSANLEITUNG_GLPI_AI_AGENT.md
Normal file
1104
BETRIEBSANLEITUNG_GLPI_AI_AGENT.md
Normal file
File diff suppressed because it is too large
Load Diff
1104
BETRIEBSANLEITUNG_GLPI_AI_AGENT_OLLAMA_POOL.md
Normal file
1104
BETRIEBSANLEITUNG_GLPI_AI_AGENT_OLLAMA_POOL.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -6,7 +6,7 @@ Diese Version erweitert die bestehende Ticketverarbeitung um ein generisches, ab
|
||||
|
||||
Die Ticketpriorisierung läuft standardmäßig im Shadow Mode. Das Modell empfiehlt eine GLPI-Priorität und kontrollierte Grundcodes; Go entscheidet anschließend deterministisch. Automatische Herabstufungen sind gesperrt, Erhöhungen je Lauf begrenzt und Live-Schreibzugriffe zusätzlich durch `AUTO_PRIORITY` und `DRY_RUN` geschützt.
|
||||
|
||||
Die Eskalation besitzt einen unabhängigen Scheduler und ist nicht an `date_mod` oder die normale FIFO-/Polling-Deduplizierung gebunden. Alte offene Tickets können dadurch erneut geprüft werden. Alter, Modellentscheidung, Confidence, Stufe, Grundcodes, Aktion, menschliche Aktivität, aktueller Ticketzustand und Idempotenz werden getrennt validiert. Als automatische Aktion ist absichtlich nur `raise_priority` implementiert.
|
||||
Die Eskalation besitzt einen unabhängigen Scheduler und ist nicht an `date_mod` oder die normale FIFO-/Polling-Deduplizierung gebunden. Alte offene Tickets können dadurch erneut geprüft werden. Alter, Modellentscheidung, Confidence, Stufe, Grundcodes, Aktion, menschliche Aktivität, aktueller Ticketzustand und Idempotenz werden getrennt validiert. Die Eskalation unterstützt die freigegebenen Aktionen `raise_priority`, `assign_second_level`, `assign_security_team`, `notify_service_owner`, `link_major_incident` und `request_manager_review`; jede Aktion besitzt eigene Policy-, Ziel- und Idempotenzprüfungen.
|
||||
|
||||
Die interne Queue ist eine priorisierte Heap-Queue. Manuelle Läufe, Webhooks, Polling und Scheduler-Läufe können unterschiedlich gewichtet werden. Dedupliziert wird je Ticket und Trigger, sodass ein normaler Ticketlauf und eine zeitgesteuerte Eskalation desselben Tickets parallel vorgemerkt werden dürfen, aber nicht doppelt je Trigger.
|
||||
|
||||
@@ -60,3 +60,8 @@ Die automatisierten Prüfungen ersetzen keinen Shadow-Mode-Test gegen die konkre
|
||||
Der Prioritätslauf erhält konservativ extrahierte, im Ticket ausdrücklich vorhandene Belege. Ollama bleibt die entscheidende Analyseinstanz; Go validiert jedoch, dass Scope, Reason Codes und Begründung den belegten Tatsachen nicht widersprechen. Die Belege und die zusätzlichen Impact-/Urgency-/Scope-Felder werden im separaten `AnalysisRun` gespeichert.
|
||||
|
||||
Zusätzlich erzeugt die Kategorieanalyse einen nicht blockierenden Diagnosehinweis, wenn eine externe Knowledge-Kategorie auf eine GLPI-Kategorie mit deutlich anderem Namen gemappt ist.
|
||||
|
||||
|
||||
## Ollama-Node-Pool
|
||||
|
||||
Der Ollama-Client unterstützt mehrere unabhängige Server mit Healthchecks, Least-In-Flight-, Round-Robin-, Weighted- und Fastest-Recent-Routing, per-Node-Parallelitätsgrenzen, Failover und optionaler Modelldigest-Gleichheit. Jeder KI-Analyselauf speichert den ausgewählten Node und sämtliche HTTP-Versuche unter `provider`. Der Pool erhöht Durchsatz und Verfügbarkeit, teilt jedoch kein einzelnes Modell über mehrere Rechner.
|
||||
|
||||
250
OLLAMA-POOL.md
Normal file
250
OLLAMA-POOL.md
Normal file
@@ -0,0 +1,250 @@
|
||||
# Betrieb mit mehreren Ollama-Instanzen
|
||||
|
||||
Der Agent kann bis zu 64 voneinander unabhängige Ollama-Server als gemeinsamen Inferenz-Pool verwenden. Jeder Node lädt das vollständige Chat- und – sofern für RAG erforderlich – Embedding-Modell lokal. Der Pool erhöht damit den **Gesamtdurchsatz und die Ausfallsicherheit**; er teilt ein einzelnes Modell nicht über mehrere Rechner auf.
|
||||
|
||||
## Architektur
|
||||
|
||||
```text
|
||||
GLPI AI Agent
|
||||
Queue / Worker / Policies
|
||||
|
|
||||
v
|
||||
Ollama Pool Router
|
||||
| | |
|
||||
Node 1 Node 2 Node 3
|
||||
```
|
||||
|
||||
Jeder logische KI-Lauf – Kategorie, Priorität, Status, Antwort oder Eskalation – wird einem verfügbaren Node zugewiesen. Bei retryfähigen Netzwerk- oder Serverfehlern kann derselbe Request auf einem anderen kompatiblen Node wiederholt werden.
|
||||
|
||||
## Voraussetzungen je Node
|
||||
|
||||
Auf allen Nodes sollten installiert sein:
|
||||
|
||||
```text
|
||||
Chat-Modell: OLLAMA_MODEL
|
||||
Embedding-Modell: OLLAMA_EMBEDDING_MODEL
|
||||
```
|
||||
|
||||
Bei `OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true` prüft der Agent über `/api/tags`, dass alle erreichbaren Nodes exakt dieselben Modelldigests melden. Schon ein abweichender Digest macht den gesamten divergierenden Pool fail-closed, damit identische Tickets nicht aufgrund verschiedener Modellstände unterschiedlich bewertet werden.
|
||||
|
||||
Für Lenovo-Systeme mit integrierter Radeon-Grafik und gemeinsamem RAM ist als Ausgangspunkt sinnvoll:
|
||||
|
||||
```env
|
||||
OLLAMA_NODE_MAX_INFLIGHT=1
|
||||
OLLAMA_ROUTING_MODE=least_inflight
|
||||
OLLAMA_KEEP_ALIVE=10m
|
||||
OLLAMA_THINK=false
|
||||
```
|
||||
|
||||
Der Gesamtdurchsatz wird zusätzlich durch `WORKERS` begrenzt. Mit drei Nodes und `WORKERS=2` können höchstens zwei Ticketpipelines gleichzeitig Inferenz anfordern. Für einen Lasttest mit drei gleichartigen Nodes ist daher beispielsweise sinnvoll:
|
||||
|
||||
```env
|
||||
WORKERS=3
|
||||
OLLAMA_NODE_MAX_INFLIGHT=1
|
||||
```
|
||||
|
||||
Die Analysestufen eines einzelnen Tickets bleiben aus fachlichen Gründen weitgehend geordnet. Der größte Poolnutzen entsteht deshalb bei mehreren gleichzeitig wartenden Tickets oder Eskalationsläufen.
|
||||
|
||||
## Minimale Pool-Konfiguration
|
||||
|
||||
```env
|
||||
OLLAMA_URLS=http://10.20.30.21:11434,http://10.20.30.22:11434,http://10.20.30.23:11434
|
||||
OLLAMA_NODE_NAMES=lenovo-01,lenovo-02,lenovo-03
|
||||
OLLAMA_ROUTING_MODE=least_inflight
|
||||
OLLAMA_NODE_MAX_INFLIGHT=1
|
||||
OLLAMA_FAILOVER_ENABLED=true
|
||||
OLLAMA_FAILOVER_ATTEMPTS=0
|
||||
OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true
|
||||
OLLAMA_REQUIRE_EMBEDDING_MODEL=true
|
||||
```
|
||||
|
||||
`OLLAMA_FAILOVER_ATTEMPTS=0` bedeutet: maximal alle konfigurierten Nodes versuchen.
|
||||
|
||||
## Routing-Modi
|
||||
|
||||
### `least_inflight`
|
||||
|
||||
Empfohlener Standard. Der Node mit den wenigsten laufenden Requests wird bevorzugt. Bei gleicher Auslastung wird zunächst der bislang seltener verwendete Node gewählt; anschließend dienen mittlere Laufzeit und Name als stabile Tie-Breaker. Dadurch verteilt sich auch serieller Verkehr über gleichartige Nodes.
|
||||
|
||||
```env
|
||||
OLLAMA_ROUTING_MODE=least_inflight
|
||||
```
|
||||
|
||||
### `round_robin`
|
||||
|
||||
Requests werden zyklisch verteilt. Dieser Modus ist einfach, berücksichtigt aber Leistungsunterschiede nur begrenzt.
|
||||
|
||||
```env
|
||||
OLLAMA_ROUTING_MODE=round_robin
|
||||
```
|
||||
|
||||
### `weighted`
|
||||
|
||||
Geeignet für gemischte Hardware. Die Gewichte stehen positionsgleich zu `OLLAMA_URLS`.
|
||||
|
||||
```env
|
||||
OLLAMA_URLS=http://lenovo-1:11434,http://lenovo-2:11434,http://gpu-server:11434
|
||||
OLLAMA_NODE_NAMES=lenovo-1,lenovo-2,gpu-server
|
||||
OLLAMA_NODE_WEIGHTS=1,1,6
|
||||
OLLAMA_ROUTING_MODE=weighted
|
||||
```
|
||||
|
||||
### `fastest_recent`
|
||||
|
||||
Bevorzugt Nodes mit der niedrigsten gleitenden mittleren Request-Laufzeit. Neue oder zurückgekehrte Nodes ohne Messwert werden zunächst einmal vermessen, damit sie nicht dauerhaft verhungern.
|
||||
|
||||
```env
|
||||
OLLAMA_ROUTING_MODE=fastest_recent
|
||||
```
|
||||
|
||||
## Startverhalten und Docker Compose
|
||||
|
||||
Der Webserver startet unabhängig vom Pool. Vor Knowledge-Initialisierung und Ticketverarbeitung wartet der Agent wiederholt auf mindestens einen gesunden, kompatiblen Ollama-Node. Ein noch bootender Node führt dadurch nicht mehr zu einem einmaligen dauerhaften Knowledge-Fehler; im Dashboard bleibt der Zustand währenddessen sichtbar.
|
||||
|
||||
Die Compose-Dateien besitzen keine harte Abhängigkeit des Agenten vom mitgelieferten `ollama`-Service mehr. Für ausschließlich externe Nodes kann gezielt nur der Agent gestartet werden:
|
||||
|
||||
```bash
|
||||
docker compose up -d agent
|
||||
```
|
||||
|
||||
`OLLAMA_URLS` hat Vorrang vor dem weiterhin aus Kompatibilitätsgründen gesetzten `OLLAMA_URL=http://ollama:11434`. Bei `docker compose up -d` ohne Servicenamen wird der gebündelte lokale Ollama-Service weiterhin mitgestartet, aber nur verwendet, wenn seine URL im effektiven Pool steht.
|
||||
|
||||
## Healthchecks und Cooldown
|
||||
|
||||
```env
|
||||
OLLAMA_NODE_HEALTH_INTERVAL=15s
|
||||
OLLAMA_NODE_FAILURE_COOLDOWN=30s
|
||||
OLLAMA_NODE_REQUEST_TIMEOUT=10m
|
||||
```
|
||||
|
||||
Der Healthcheck ruft `/api/tags` auf und prüft:
|
||||
|
||||
- HTTP-Erreichbarkeit,
|
||||
- Vorhandensein des Chat-Modells,
|
||||
- Vorhandensein des Embedding-Modells,
|
||||
- Modelldigests,
|
||||
- Kompatibilität mit den übrigen Nodes.
|
||||
|
||||
Ein retryfähiger Fehler setzt den betroffenen Node in einen Cooldown. Währenddessen erhält er keine neuen Requests. Ein späterer erfolgreicher Healthcheck macht ihn wieder sichtbar; der Cooldown läuft dennoch bis zu seinem Ende, um Flapping zu dämpfen.
|
||||
|
||||
## Failover
|
||||
|
||||
Failover wird ausgelöst bei:
|
||||
|
||||
- Verbindungsfehlern,
|
||||
- Zeitüberschreitungen,
|
||||
- HTTP 408,
|
||||
- HTTP 429,
|
||||
- HTTP 5xx,
|
||||
- ungültigem äußerem Ollama-Response-JSON.
|
||||
|
||||
```env
|
||||
OLLAMA_FAILOVER_ENABLED=true
|
||||
OLLAMA_FAILOVER_ATTEMPTS=0
|
||||
```
|
||||
|
||||
Nicht retryfähige 4xx-Fehler werden nicht auf andere Nodes gespiegelt. Die Modellaufrufe sind rein lesende Inferenzaufrufe; GLPI-Schreibaktionen erfolgen erst später durch die deterministische Go-Policy.
|
||||
|
||||
## Analyse-Diagnose
|
||||
|
||||
Jeder `AnalysisRun` speichert unter `provider`:
|
||||
|
||||
```json
|
||||
{
|
||||
"provider": "ollama-pool",
|
||||
"routing_mode": "least_inflight",
|
||||
"selected_node": "lenovo-02",
|
||||
"selected_url": "http://10.20.30.22:11434",
|
||||
"failover_used": true,
|
||||
"attempt_count": 2,
|
||||
"attempts": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"stage": "priority",
|
||||
"node_name": "lenovo-01",
|
||||
"outcome": "error",
|
||||
"retryable": true
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"stage": "priority",
|
||||
"node_name": "lenovo-02",
|
||||
"outcome": "success"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Zusätzlich werden – sofern Ollama sie liefert – Ladezeit, Prompt-Tokens, Generierungstokens und zugehörige Laufzeiten gespeichert.
|
||||
|
||||
## Dashboard und Prometheus
|
||||
|
||||
`/api/status` enthält unter anderem:
|
||||
|
||||
```text
|
||||
ollama_nodes
|
||||
ollama_node_count
|
||||
ollama_healthy_nodes
|
||||
ollama_available_nodes
|
||||
ollama_routing_mode
|
||||
```
|
||||
|
||||
Prometheus exportiert pro Node:
|
||||
|
||||
```text
|
||||
glpi_agent_ollama_node_healthy
|
||||
glpi_agent_ollama_node_available
|
||||
glpi_agent_ollama_node_inflight
|
||||
glpi_agent_ollama_node_requests_total
|
||||
glpi_agent_ollama_node_failures_total
|
||||
glpi_agent_ollama_node_average_duration_ms
|
||||
```
|
||||
|
||||
## Netzwerksicherheit
|
||||
|
||||
Ollama besitzt an seiner lokalen API üblicherweise keine eigene Mandantenauthentifizierung. Die Nodes sollten daher:
|
||||
|
||||
- in einem eigenen Server-/KI-Netz liegen,
|
||||
- Port 11434 nur vom GLPI-AI-Agent-Host akzeptieren,
|
||||
- nicht aus Benutzer-VLANs erreichbar sein,
|
||||
- niemals direkt aus dem Internet erreichbar sein,
|
||||
- bei standortübergreifender Nutzung über VPN oder einen TLS-Reverse-Proxy mit Netzwerk-/IP-Allowlist angebunden werden.
|
||||
|
||||
Beispiel auf jedem Node:
|
||||
|
||||
```env
|
||||
OLLAMA_HOST=0.0.0.0:11434
|
||||
```
|
||||
|
||||
Diese Freigabe allein ist nicht ausreichend; eine Host- oder Netzfirewall muss den Zugriff auf die Agent-IP begrenzen.
|
||||
|
||||
## Rollout-Empfehlung
|
||||
|
||||
1. Auf allen Nodes identische Ollama- und Modellstände installieren.
|
||||
2. Chat- und Embedding-Modell einmal lokal laden.
|
||||
3. Jeden Node einzeln mit `/api/tags` prüfen.
|
||||
4. Pool zunächst mit `OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true` starten.
|
||||
5. Im Dashboard kontrollieren, dass alle Nodes `healthy=true` und `compatible=true` melden.
|
||||
6. `OLLAMA_NODE_MAX_INFLIGHT=1` beibehalten und mehrere Testtickets parallel analysieren.
|
||||
7. Erst nach Messung von RAM, Temperatur und Laufzeiten höhere Parallelität testen.
|
||||
|
||||
## Modellupdates bei strikter Digest-Prüfung
|
||||
|
||||
Bei `OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true` ist ein gemischter Modellstand absichtlich nicht verfügbar. Ein Pull oder Austausch nur auf einem einzelnen aktiven Node kann den Pool deshalb beim nächsten Healthcheck fail-closed setzen. Sichere Varianten sind:
|
||||
|
||||
1. Agent in ein Wartungsfenster nehmen und das Modell auf allen Nodes aktualisieren.
|
||||
2. Einen neuen, eindeutig versionierten Modelltag zunächst auf allen Nodes bereitstellen und erst danach `OLLAMA_MODEL` zentral umstellen.
|
||||
3. Für Hardwarewartung einen Node aus `OLLAMA_URLS` entfernen, Agent neu starten und ihn erst mit passendem Digest wieder aufnehmen.
|
||||
|
||||
`OLLAMA_REQUIRE_SAME_MODEL_DIGEST=false` sollte nicht als normale Rolling-Update-Strategie verwendet werden, weil dann identische Tickets während der Übergangszeit von unterschiedlichen Modellständen bewertet werden können.
|
||||
|
||||
## Rückfall auf einen Einzelnode
|
||||
|
||||
Die bisherige Konfiguration bleibt kompatibel:
|
||||
|
||||
```env
|
||||
OLLAMA_URL=http://localhost:11434
|
||||
OLLAMA_URLS=
|
||||
```
|
||||
|
||||
Ist `OLLAMA_URLS` leer, wird automatisch `OLLAMA_URL` als einzelner Pool-Node verwendet.
|
||||
250
OLLAMA_POOL_BETRIEB.md
Normal file
250
OLLAMA_POOL_BETRIEB.md
Normal file
@@ -0,0 +1,250 @@
|
||||
# Betrieb mit mehreren Ollama-Instanzen
|
||||
|
||||
Der Agent kann bis zu 64 voneinander unabhängige Ollama-Server als gemeinsamen Inferenz-Pool verwenden. Jeder Node lädt das vollständige Chat- und – sofern für RAG erforderlich – Embedding-Modell lokal. Der Pool erhöht damit den **Gesamtdurchsatz und die Ausfallsicherheit**; er teilt ein einzelnes Modell nicht über mehrere Rechner auf.
|
||||
|
||||
## Architektur
|
||||
|
||||
```text
|
||||
GLPI AI Agent
|
||||
Queue / Worker / Policies
|
||||
|
|
||||
v
|
||||
Ollama Pool Router
|
||||
| | |
|
||||
Node 1 Node 2 Node 3
|
||||
```
|
||||
|
||||
Jeder logische KI-Lauf – Kategorie, Priorität, Status, Antwort oder Eskalation – wird einem verfügbaren Node zugewiesen. Bei retryfähigen Netzwerk- oder Serverfehlern kann derselbe Request auf einem anderen kompatiblen Node wiederholt werden.
|
||||
|
||||
## Voraussetzungen je Node
|
||||
|
||||
Auf allen Nodes sollten installiert sein:
|
||||
|
||||
```text
|
||||
Chat-Modell: OLLAMA_MODEL
|
||||
Embedding-Modell: OLLAMA_EMBEDDING_MODEL
|
||||
```
|
||||
|
||||
Bei `OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true` prüft der Agent über `/api/tags`, dass alle erreichbaren Nodes exakt dieselben Modelldigests melden. Schon ein abweichender Digest macht den gesamten divergierenden Pool fail-closed, damit identische Tickets nicht aufgrund verschiedener Modellstände unterschiedlich bewertet werden.
|
||||
|
||||
Für Lenovo-Systeme mit integrierter Radeon-Grafik und gemeinsamem RAM ist als Ausgangspunkt sinnvoll:
|
||||
|
||||
```env
|
||||
OLLAMA_NODE_MAX_INFLIGHT=1
|
||||
OLLAMA_ROUTING_MODE=least_inflight
|
||||
OLLAMA_KEEP_ALIVE=10m
|
||||
OLLAMA_THINK=false
|
||||
```
|
||||
|
||||
Der Gesamtdurchsatz wird zusätzlich durch `WORKERS` begrenzt. Mit drei Nodes und `WORKERS=2` können höchstens zwei Ticketpipelines gleichzeitig Inferenz anfordern. Für einen Lasttest mit drei gleichartigen Nodes ist daher beispielsweise sinnvoll:
|
||||
|
||||
```env
|
||||
WORKERS=3
|
||||
OLLAMA_NODE_MAX_INFLIGHT=1
|
||||
```
|
||||
|
||||
Die Analysestufen eines einzelnen Tickets bleiben aus fachlichen Gründen weitgehend geordnet. Der größte Poolnutzen entsteht deshalb bei mehreren gleichzeitig wartenden Tickets oder Eskalationsläufen.
|
||||
|
||||
## Minimale Pool-Konfiguration
|
||||
|
||||
```env
|
||||
OLLAMA_URLS=http://10.20.30.21:11434,http://10.20.30.22:11434,http://10.20.30.23:11434
|
||||
OLLAMA_NODE_NAMES=lenovo-01,lenovo-02,lenovo-03
|
||||
OLLAMA_ROUTING_MODE=least_inflight
|
||||
OLLAMA_NODE_MAX_INFLIGHT=1
|
||||
OLLAMA_FAILOVER_ENABLED=true
|
||||
OLLAMA_FAILOVER_ATTEMPTS=0
|
||||
OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true
|
||||
OLLAMA_REQUIRE_EMBEDDING_MODEL=true
|
||||
```
|
||||
|
||||
`OLLAMA_FAILOVER_ATTEMPTS=0` bedeutet: maximal alle konfigurierten Nodes versuchen.
|
||||
|
||||
## Routing-Modi
|
||||
|
||||
### `least_inflight`
|
||||
|
||||
Empfohlener Standard. Der Node mit den wenigsten laufenden Requests wird bevorzugt. Bei gleicher Auslastung wird zunächst der bislang seltener verwendete Node gewählt; anschließend dienen mittlere Laufzeit und Name als stabile Tie-Breaker. Dadurch verteilt sich auch serieller Verkehr über gleichartige Nodes.
|
||||
|
||||
```env
|
||||
OLLAMA_ROUTING_MODE=least_inflight
|
||||
```
|
||||
|
||||
### `round_robin`
|
||||
|
||||
Requests werden zyklisch verteilt. Dieser Modus ist einfach, berücksichtigt aber Leistungsunterschiede nur begrenzt.
|
||||
|
||||
```env
|
||||
OLLAMA_ROUTING_MODE=round_robin
|
||||
```
|
||||
|
||||
### `weighted`
|
||||
|
||||
Geeignet für gemischte Hardware. Die Gewichte stehen positionsgleich zu `OLLAMA_URLS`.
|
||||
|
||||
```env
|
||||
OLLAMA_URLS=http://lenovo-1:11434,http://lenovo-2:11434,http://gpu-server:11434
|
||||
OLLAMA_NODE_NAMES=lenovo-1,lenovo-2,gpu-server
|
||||
OLLAMA_NODE_WEIGHTS=1,1,6
|
||||
OLLAMA_ROUTING_MODE=weighted
|
||||
```
|
||||
|
||||
### `fastest_recent`
|
||||
|
||||
Bevorzugt Nodes mit der niedrigsten gleitenden mittleren Request-Laufzeit. Neue oder zurückgekehrte Nodes ohne Messwert werden zunächst einmal vermessen, damit sie nicht dauerhaft verhungern.
|
||||
|
||||
```env
|
||||
OLLAMA_ROUTING_MODE=fastest_recent
|
||||
```
|
||||
|
||||
## Startverhalten und Docker Compose
|
||||
|
||||
Der Webserver startet unabhängig vom Pool. Vor Knowledge-Initialisierung und Ticketverarbeitung wartet der Agent wiederholt auf mindestens einen gesunden, kompatiblen Ollama-Node. Ein noch bootender Node führt dadurch nicht mehr zu einem einmaligen dauerhaften Knowledge-Fehler; im Dashboard bleibt der Zustand währenddessen sichtbar.
|
||||
|
||||
Die Compose-Dateien besitzen keine harte Abhängigkeit des Agenten vom mitgelieferten `ollama`-Service mehr. Für ausschließlich externe Nodes kann gezielt nur der Agent gestartet werden:
|
||||
|
||||
```bash
|
||||
docker compose up -d agent
|
||||
```
|
||||
|
||||
`OLLAMA_URLS` hat Vorrang vor dem weiterhin aus Kompatibilitätsgründen gesetzten `OLLAMA_URL=http://ollama:11434`. Bei `docker compose up -d` ohne Servicenamen wird der gebündelte lokale Ollama-Service weiterhin mitgestartet, aber nur verwendet, wenn seine URL im effektiven Pool steht.
|
||||
|
||||
## Healthchecks und Cooldown
|
||||
|
||||
```env
|
||||
OLLAMA_NODE_HEALTH_INTERVAL=15s
|
||||
OLLAMA_NODE_FAILURE_COOLDOWN=30s
|
||||
OLLAMA_NODE_REQUEST_TIMEOUT=10m
|
||||
```
|
||||
|
||||
Der Healthcheck ruft `/api/tags` auf und prüft:
|
||||
|
||||
- HTTP-Erreichbarkeit,
|
||||
- Vorhandensein des Chat-Modells,
|
||||
- Vorhandensein des Embedding-Modells,
|
||||
- Modelldigests,
|
||||
- Kompatibilität mit den übrigen Nodes.
|
||||
|
||||
Ein retryfähiger Fehler setzt den betroffenen Node in einen Cooldown. Währenddessen erhält er keine neuen Requests. Ein späterer erfolgreicher Healthcheck macht ihn wieder sichtbar; der Cooldown läuft dennoch bis zu seinem Ende, um Flapping zu dämpfen.
|
||||
|
||||
## Failover
|
||||
|
||||
Failover wird ausgelöst bei:
|
||||
|
||||
- Verbindungsfehlern,
|
||||
- Zeitüberschreitungen,
|
||||
- HTTP 408,
|
||||
- HTTP 429,
|
||||
- HTTP 5xx,
|
||||
- ungültigem äußerem Ollama-Response-JSON.
|
||||
|
||||
```env
|
||||
OLLAMA_FAILOVER_ENABLED=true
|
||||
OLLAMA_FAILOVER_ATTEMPTS=0
|
||||
```
|
||||
|
||||
Nicht retryfähige 4xx-Fehler werden nicht auf andere Nodes gespiegelt. Die Modellaufrufe sind rein lesende Inferenzaufrufe; GLPI-Schreibaktionen erfolgen erst später durch die deterministische Go-Policy.
|
||||
|
||||
## Analyse-Diagnose
|
||||
|
||||
Jeder `AnalysisRun` speichert unter `provider`:
|
||||
|
||||
```json
|
||||
{
|
||||
"provider": "ollama-pool",
|
||||
"routing_mode": "least_inflight",
|
||||
"selected_node": "lenovo-02",
|
||||
"selected_url": "http://10.20.30.22:11434",
|
||||
"failover_used": true,
|
||||
"attempt_count": 2,
|
||||
"attempts": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"stage": "priority",
|
||||
"node_name": "lenovo-01",
|
||||
"outcome": "error",
|
||||
"retryable": true
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"stage": "priority",
|
||||
"node_name": "lenovo-02",
|
||||
"outcome": "success"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Zusätzlich werden – sofern Ollama sie liefert – Ladezeit, Prompt-Tokens, Generierungstokens und zugehörige Laufzeiten gespeichert.
|
||||
|
||||
## Dashboard und Prometheus
|
||||
|
||||
`/api/status` enthält unter anderem:
|
||||
|
||||
```text
|
||||
ollama_nodes
|
||||
ollama_node_count
|
||||
ollama_healthy_nodes
|
||||
ollama_available_nodes
|
||||
ollama_routing_mode
|
||||
```
|
||||
|
||||
Prometheus exportiert pro Node:
|
||||
|
||||
```text
|
||||
glpi_agent_ollama_node_healthy
|
||||
glpi_agent_ollama_node_available
|
||||
glpi_agent_ollama_node_inflight
|
||||
glpi_agent_ollama_node_requests_total
|
||||
glpi_agent_ollama_node_failures_total
|
||||
glpi_agent_ollama_node_average_duration_ms
|
||||
```
|
||||
|
||||
## Netzwerksicherheit
|
||||
|
||||
Ollama besitzt an seiner lokalen API üblicherweise keine eigene Mandantenauthentifizierung. Die Nodes sollten daher:
|
||||
|
||||
- in einem eigenen Server-/KI-Netz liegen,
|
||||
- Port 11434 nur vom GLPI-AI-Agent-Host akzeptieren,
|
||||
- nicht aus Benutzer-VLANs erreichbar sein,
|
||||
- niemals direkt aus dem Internet erreichbar sein,
|
||||
- bei standortübergreifender Nutzung über VPN oder einen TLS-Reverse-Proxy mit Netzwerk-/IP-Allowlist angebunden werden.
|
||||
|
||||
Beispiel auf jedem Node:
|
||||
|
||||
```env
|
||||
OLLAMA_HOST=0.0.0.0:11434
|
||||
```
|
||||
|
||||
Diese Freigabe allein ist nicht ausreichend; eine Host- oder Netzfirewall muss den Zugriff auf die Agent-IP begrenzen.
|
||||
|
||||
## Rollout-Empfehlung
|
||||
|
||||
1. Auf allen Nodes identische Ollama- und Modellstände installieren.
|
||||
2. Chat- und Embedding-Modell einmal lokal laden.
|
||||
3. Jeden Node einzeln mit `/api/tags` prüfen.
|
||||
4. Pool zunächst mit `OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true` starten.
|
||||
5. Im Dashboard kontrollieren, dass alle Nodes `healthy=true` und `compatible=true` melden.
|
||||
6. `OLLAMA_NODE_MAX_INFLIGHT=1` beibehalten und mehrere Testtickets parallel analysieren.
|
||||
7. Erst nach Messung von RAM, Temperatur und Laufzeiten höhere Parallelität testen.
|
||||
|
||||
## Modellupdates bei strikter Digest-Prüfung
|
||||
|
||||
Bei `OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true` ist ein gemischter Modellstand absichtlich nicht verfügbar. Ein Pull oder Austausch nur auf einem einzelnen aktiven Node kann den Pool deshalb beim nächsten Healthcheck fail-closed setzen. Sichere Varianten sind:
|
||||
|
||||
1. Agent in ein Wartungsfenster nehmen und das Modell auf allen Nodes aktualisieren.
|
||||
2. Einen neuen, eindeutig versionierten Modelltag zunächst auf allen Nodes bereitstellen und erst danach `OLLAMA_MODEL` zentral umstellen.
|
||||
3. Für Hardwarewartung einen Node aus `OLLAMA_URLS` entfernen, Agent neu starten und ihn erst mit passendem Digest wieder aufnehmen.
|
||||
|
||||
`OLLAMA_REQUIRE_SAME_MODEL_DIGEST=false` sollte nicht als normale Rolling-Update-Strategie verwendet werden, weil dann identische Tickets während der Übergangszeit von unterschiedlichen Modellständen bewertet werden können.
|
||||
|
||||
## Rückfall auf einen Einzelnode
|
||||
|
||||
Die bisherige Konfiguration bleibt kompatibel:
|
||||
|
||||
```env
|
||||
OLLAMA_URL=http://localhost:11434
|
||||
OLLAMA_URLS=
|
||||
```
|
||||
|
||||
Ist `OLLAMA_URLS` leer, wird automatisch `OLLAMA_URL` als einzelner Pool-Node verwendet.
|
||||
21
README.md
21
README.md
@@ -494,6 +494,25 @@ make build
|
||||
```
|
||||
|
||||
|
||||
## Mehrere Ollama-Nodes
|
||||
|
||||
Der Agent unterstützt einen nativen Ollama-Pool mit Least-In-Flight-Routing, Healthchecks, Failover, Modelldigest-Prüfung und Node-Diagnose pro AnalysisRun. Ein einzelnes Modell wird dabei nicht über Rechner verteilt; jeder Node führt vollständige unabhängige Inferenzrequests aus.
|
||||
|
||||
```env
|
||||
OLLAMA_URLS=http://10.20.30.21:11434,http://10.20.30.22:11434,http://10.20.30.23:11434
|
||||
OLLAMA_NODE_NAMES=lenovo-01,lenovo-02,lenovo-03
|
||||
OLLAMA_ROUTING_MODE=least_inflight
|
||||
OLLAMA_NODE_MAX_INFLIGHT=1
|
||||
OLLAMA_FAILOVER_ENABLED=true
|
||||
OLLAMA_FAILOVER_ATTEMPTS=0
|
||||
OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true
|
||||
WORKERS=3
|
||||
```
|
||||
|
||||
`WORKERS` begrenzt die Zahl gleichzeitig aktiver Ticketpipelines. Für drei gleichartige Nodes sind drei Worker ein sinnvoller Lasttest; die Ressourcen jedes einzelnen Rechners bleiben zusätzlich durch `OLLAMA_NODE_MAX_INFLIGHT=1` geschützt.
|
||||
|
||||
Die vollständige Betriebsbeschreibung steht in [OLLAMA-POOL.md](OLLAMA-POOL.md). Der Agent wartet beim Start auf einen kompatiblen Pool, während Dashboard und Node-Diagnose bereits erreichbar bleiben. Bei externen Nodes kann mit `docker compose up -d agent` nur der Agent gestartet werden.
|
||||
|
||||
## Docker troubleshooting: `/app/data` permission denied and slow Ollama
|
||||
|
||||
The Compose stack contains a one-shot `agent-data-init` service. It prepares the named `agent-data` volume for the non-root agent user before the agent starts. The agent also probes `runs.jsonl` at startup and exits immediately with a clear error if the volume is not writable.
|
||||
@@ -508,7 +527,7 @@ OLLAMA_THINK=false
|
||||
OLLAMA_MAX_CONCURRENT=1
|
||||
```
|
||||
|
||||
`OLLAMA_NUM_PREDICT` limits the maximum generated tokens for the small structured decision. `OLLAMA_KEEP_ALIVE` asks Ollama to keep the analysis model loaded between tickets. `OLLAMA_THINK=false` disables optional model thinking for this deterministic classification task. `OLLAMA_MAX_CONCURRENT=1` serializes local Ollama inference even when multiple ticket workers are active, so queued requests do not consume their HTTP timeout while waiting for the model. On very slow CPU-only hosts, use a smaller local model and/or increase `OLLAMA_TIMEOUT`.
|
||||
`OLLAMA_NUM_PREDICT` limits the maximum generated tokens for the small structured decision. `OLLAMA_KEEP_ALIVE` asks Ollama to keep the analysis model loaded between tickets. `OLLAMA_THINK=false` disables optional model thinking for this deterministic classification task. `OLLAMA_NODE_MAX_INFLIGHT=1` serializes inference on each individual pool node. `OLLAMA_MAX_CONCURRENT` remains a backwards-compatible alias when the new per-node value is not set. On very slow CPU-only hosts, use a smaller local model and/or increase `OLLAMA_TIMEOUT`.
|
||||
|
||||
After upgrading an existing Compose deployment, recreate the stack so the init service runs:
|
||||
|
||||
|
||||
12
SECURITY.md
12
SECURITY.md
@@ -107,3 +107,15 @@ Das Feld `answer_html` wird ausschließlich vom read-only GLPI-KB-Synchronisiere
|
||||
## Dynamische Begrenzung des LLM-Kontexts
|
||||
|
||||
Knowledge-Kandidaten werden nicht allein anhand einer festen Anzahl in den Modellkontext übernommen. Der Agent kombiniert einen absoluten Retrieval-Floor, einen maximalen Abstand zum besten Treffer und eine harte Obergrenze. Dadurch werden bei großen Wissensbeständen schwache, themenfremde Artikel aus dem Ollama-Prompt herausgehalten, bleiben aber optional im Audit sichtbar.
|
||||
|
||||
## Ollama-Pool und Netzwerkgrenze
|
||||
|
||||
Mehrere Ollama-Instanzen bilden eine zusätzliche interne Trust Boundary. Der Agent sendet Ticket-, Knowledge- und Kontextauszüge an jeden Node, der einen Request übernehmen kann. Deshalb dürfen ausschließlich administrierte Systeme in `OLLAMA_URLS` aufgenommen werden.
|
||||
|
||||
- Ollama-Port 11434 nur von der Agent-IP beziehungsweise dem Agent-Subnetz zulassen.
|
||||
- Nodes nicht aus Benutzer-VLANs und niemals direkt aus dem Internet erreichbar machen.
|
||||
- Bei standortübergreifender Verbindung VPN oder einen TLS-Reverse-Proxy mit Netzwerk-/IP-Allowlist verwenden.
|
||||
- Auf allen Nodes dieselben Chat- und Embedding-Modelle installieren. `OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true` lässt den Pool bei divergierenden Digests fail-closed.
|
||||
- Node-URLs, Namen und Modelldigests erscheinen in der Betriebsdiagnose. Keine Zugangsdaten in URLs einbetten.
|
||||
- Failover wiederholt ausschließlich den noch nicht akzeptierten Inferenzrequest. GLPI-Schreiboperationen erfolgen erst nach dem vollständigen KI-Lauf und den deterministischen Policies.
|
||||
- `OLLAMA_NODE_MAX_INFLIGHT=1` ist für integrierte GPUs und gemeinsam genutzten RAM der sichere Ausgangswert.
|
||||
|
||||
26
UPGRADE.md
26
UPGRADE.md
@@ -362,3 +362,29 @@ PRIORITY_ANALYSIS_TIMEOUT=45s
|
||||
```
|
||||
|
||||
Der Prioritätslauf ist nun strikt fail-open. Semantische Widersprüche werden deterministisch normalisiert und führen nicht mehr zu einem weiteren Modellaufruf. Beim Austausch des Releases `data/`, `knowledge/` und lokale Umgebungsdateien beibehalten.
|
||||
|
||||
## Upgrade auf mehrere Ollama-Nodes
|
||||
|
||||
Die bisherige Einzelnode-Konfiguration bleibt kompatibel:
|
||||
|
||||
```env
|
||||
OLLAMA_URL=http://localhost:11434
|
||||
OLLAMA_URLS=
|
||||
```
|
||||
|
||||
Für einen Pool ergänzen Sie mindestens:
|
||||
|
||||
```env
|
||||
OLLAMA_URLS=http://10.20.30.21:11434,http://10.20.30.22:11434,http://10.20.30.23:11434
|
||||
OLLAMA_NODE_NAMES=lenovo-01,lenovo-02,lenovo-03
|
||||
OLLAMA_ROUTING_MODE=least_inflight
|
||||
OLLAMA_NODE_MAX_INFLIGHT=1
|
||||
OLLAMA_FAILOVER_ENABLED=true
|
||||
OLLAMA_FAILOVER_ATTEMPTS=0
|
||||
OLLAMA_REQUIRE_SAME_MODEL_DIGEST=true
|
||||
OLLAMA_REQUIRE_EMBEDDING_MODEL=true
|
||||
```
|
||||
|
||||
Vor dem ersten Start müssen `OLLAMA_MODEL` und – bei aktivem RAG – `OLLAMA_EMBEDDING_MODEL` auf jedem Node vorhanden sein. Bei aktivierter Digest-Pflicht führt bereits ein abweichender Modellstand dazu, dass der Pool keine Requests annimmt. Das Dashboard zeigt pro Node Erreichbarkeit, Kompatibilität, Digest, Auslastung, Fehler und Laufzeit.
|
||||
|
||||
Bestehende `runs.jsonl`-Einträge bleiben lesbar. Nur neue `AnalysisRun`-Datensätze enthalten den Bereich `provider` mit Node-Auswahl und Failover-Versuchen. `state-index.json` muss beim Upgrade erhalten bleiben.
|
||||
|
||||
@@ -45,7 +45,31 @@ func main() {
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
g := glpi.New(cfg.GLPIURL, cfg.GLPIAPIVersion, cfg.GLPIClientID, cfg.GLPIClientSecret, cfg.GLPIUsername, cfg.GLPIPassword, cfg.GLPITimeout)
|
||||
o := ollama.New(cfg.OllamaURL, cfg.OllamaModel, cfg.OllamaEmbeddingModel, cfg.CommunicationLanguage, cfg.CommunicationStyle, cfg.OllamaTimeout, cfg.OllamaNumPredict, cfg.OllamaKeepAlive, cfg.OllamaThink, cfg.OllamaMaxConcurrent, cfg.OllamaJSONRetries)
|
||||
nodes := make([]ollama.NodeConfig, 0, len(cfg.OllamaURLs))
|
||||
for i, nodeURL := range cfg.OllamaURLs {
|
||||
name := ""
|
||||
if i < len(cfg.OllamaNodeNames) {
|
||||
name = cfg.OllamaNodeNames[i]
|
||||
}
|
||||
weight := 1
|
||||
if i < len(cfg.OllamaNodeWeights) {
|
||||
weight = cfg.OllamaNodeWeights[i]
|
||||
}
|
||||
nodes = append(nodes, ollama.NodeConfig{Name: name, URL: nodeURL, Weight: weight})
|
||||
}
|
||||
o, err := ollama.NewPool(ollama.PoolConfig{
|
||||
Nodes: nodes, RoutingMode: cfg.OllamaRoutingMode, NodeMaxInflight: cfg.OllamaNodeMaxInflight,
|
||||
HealthInterval: cfg.OllamaNodeHealthInterval, FailureCooldown: cfg.OllamaNodeFailureCooldown,
|
||||
NodeRequestTimeout: cfg.OllamaNodeRequestTimeout, FailoverEnabled: cfg.OllamaFailoverEnabled,
|
||||
FailoverAttempts: cfg.OllamaFailoverAttempts, RequireSameModelDigest: cfg.OllamaRequireSameDigest,
|
||||
RequireEmbeddingModel: cfg.OllamaRequireEmbeddingModel, Model: cfg.OllamaModel, EmbeddingModel: cfg.OllamaEmbeddingModel,
|
||||
}, cfg.OllamaModel, cfg.OllamaEmbeddingModel, cfg.CommunicationLanguage, cfg.CommunicationStyle, cfg.OllamaNumPredict, cfg.OllamaKeepAlive, cfg.OllamaThink, cfg.OllamaJSONRetries)
|
||||
if err != nil {
|
||||
slog.Error("Ollama pool configuration failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
o.Start(ctx)
|
||||
slog.Info("Ollama pool configured", "nodes", len(nodes), "routing", cfg.OllamaRoutingMode, "max_inflight_per_node", cfg.OllamaNodeMaxInflight, "failover", cfg.OllamaFailoverEnabled, "failover_attempts", cfg.OllamaFailoverAttempts, "require_same_digest", cfg.OllamaRequireSameDigest)
|
||||
if err := g.ValidateContract(ctx); err != nil {
|
||||
slog.Error("GLPI API contract validation failed", "error", err)
|
||||
os.Exit(1)
|
||||
@@ -92,7 +116,7 @@ func main() {
|
||||
}
|
||||
contextCollector := contextdata.New(cfg, g, kuma)
|
||||
svc := agent.New(cfg, g, o, k, l, st, q, m, contextCollector)
|
||||
web, err := webui.New(cfg, m, st, q, k, svc)
|
||||
web, err := webui.New(cfg, m, st, q, k, svc, o)
|
||||
if err != nil {
|
||||
slog.Error("web UI initialization failed", "error", err)
|
||||
os.Exit(1)
|
||||
@@ -110,6 +134,9 @@ func main() {
|
||||
// Ticket polling/workers remain paused until the local index is ready.
|
||||
go func() {
|
||||
slog.Info("knowledge initialization started in background", "knowledge_dir", cfg.KnowledgeDir, "rag_enabled", cfg.RAGEnabled)
|
||||
if err := waitForOllamaPool(ctx, o, cfg.OllamaNodeHealthInterval); err != nil {
|
||||
return
|
||||
}
|
||||
if err := k.Initialize(ctx); err != nil {
|
||||
slog.Error("knowledge store initialization failed; web UI remains available", "error", err, "knowledge_dir", cfg.KnowledgeDir, "data_dir", cfg.DataDir, "rag_enabled", cfg.RAGEnabled)
|
||||
return
|
||||
@@ -143,6 +170,39 @@ func main() {
|
||||
slog.Info("shutdown complete")
|
||||
}
|
||||
|
||||
func waitForOllamaPool(ctx context.Context, client *ollama.Client, retryInterval time.Duration) error {
|
||||
if retryInterval < 2*time.Second {
|
||||
retryInterval = 5 * time.Second
|
||||
}
|
||||
for {
|
||||
err := client.Ping(ctx)
|
||||
if err == nil {
|
||||
statuses := client.NodeStatuses()
|
||||
healthy := 0
|
||||
for _, status := range statuses {
|
||||
if status.Healthy && status.Compatible {
|
||||
healthy++
|
||||
}
|
||||
}
|
||||
slog.Info("Ollama pool ready", "healthy_nodes", healthy, "nodes", len(statuses), "routing", client.RoutingMode())
|
||||
return nil
|
||||
}
|
||||
slog.Warn("waiting for compatible Ollama pool", "retry_in", retryInterval.String(), "error", err)
|
||||
timer := time.NewTimer(retryInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func maxDuration(a, b time.Duration) time.Duration {
|
||||
if a > b {
|
||||
return a
|
||||
|
||||
@@ -20,9 +20,6 @@ services:
|
||||
volumes:
|
||||
- agent-data:/app/data
|
||||
- ./knowledge:/app/knowledge:ro
|
||||
depends_on:
|
||||
ollama:
|
||||
condition: service_started
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
|
||||
4
dist/SHA256SUMS.txt
vendored
4
dist/SHA256SUMS.txt
vendored
@@ -1,2 +1,2 @@
|
||||
f2eebac7aab8b31af1f952ca0db670e87c6a8a2516175e8fd2aeb0a0cfe610a5 glpi-ai-agent-linux-amd64
|
||||
1c9728e5282a2074a21e9263d05277144478cc529bdacb27604d00474602c8a0 glpi-ai-agent-windows-amd64.exe
|
||||
c196be36f1a99e079b13310cc266fd4ba9b8b4b3c29ccc5a6dcda808cdaace0c glpi-ai-agent-linux-amd64
|
||||
54e5f3eb9408a640a1608b46bedb2ae52e25cf175e4246a4ffd22429f1be9a9c glpi-ai-agent-windows-amd64.exe
|
||||
|
||||
BIN
dist/glpi-ai-agent-linux-amd64
vendored
BIN
dist/glpi-ai-agent-linux-amd64
vendored
Binary file not shown.
BIN
dist/glpi-ai-agent-windows-amd64.exe
vendored
BIN
dist/glpi-ai-agent-windows-amd64.exe
vendored
Binary file not shown.
@@ -7,7 +7,7 @@ services:
|
||||
environment:
|
||||
DATA_DIR: /app/data
|
||||
KNOWLEDGE_DIR: /app/knowledge
|
||||
OLLAMA_URL: http://ollama:11434
|
||||
OLLAMA_URL: ${OLLAMA_URL:-http://ollama:11434}
|
||||
OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-10m}
|
||||
OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-768}
|
||||
OLLAMA_JSON_RETRIES: ${OLLAMA_JSON_RETRIES:-1}
|
||||
@@ -21,9 +21,6 @@ services:
|
||||
# Prepare once on the Linux host: mkdir -p data knowledge && chown 65532:65532 data
|
||||
- ./data:/app/data
|
||||
- ./knowledge:/app/knowledge:ro
|
||||
depends_on:
|
||||
ollama:
|
||||
condition: service_started
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
|
||||
@@ -23,7 +23,7 @@ services:
|
||||
# Container-specific paths/hostnames override the native-friendly .env defaults.
|
||||
DATA_DIR: /app/data
|
||||
KNOWLEDGE_DIR: /app/knowledge
|
||||
OLLAMA_URL: http://ollama:11434
|
||||
OLLAMA_URL: ${OLLAMA_URL:-http://ollama:11434}
|
||||
# Local CPU inference can take several minutes on the first request.
|
||||
OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-10m}
|
||||
OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-768}
|
||||
@@ -40,8 +40,6 @@ services:
|
||||
depends_on:
|
||||
agent-data-init:
|
||||
condition: service_completed_successfully
|
||||
ollama:
|
||||
condition: service_started
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/example/glpi-ai-agent/internal/learning"
|
||||
"github.com/example/glpi-ai-agent/internal/metrics"
|
||||
"github.com/example/glpi-ai-agent/internal/model"
|
||||
"github.com/example/glpi-ai-agent/internal/ollama"
|
||||
"github.com/example/glpi-ai-agent/internal/prioritysignals"
|
||||
"github.com/example/glpi-ai-agent/internal/queue"
|
||||
"github.com/example/glpi-ai-agent/internal/state"
|
||||
@@ -282,11 +283,13 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
categoryStarted := time.Now()
|
||||
categoryAnalysis := newAnalysis(run, "category", categoryPromptVersion, map[string]any{"ticket": t, "categories": promptCats, "knowledge_candidates": categoryLLMHits, "context": contextData}, categoryStarted)
|
||||
run.CategoryAnalysisExecuted = true
|
||||
categoryDecision, err := s.ai.AnalyseCategory(ctx, t, promptCats, categoryLLMHits, contextData)
|
||||
categoryCtx, categoryTrace := ollama.WithTrace(ctx, s.cfg.OllamaRoutingMode)
|
||||
categoryDecision, err := s.ai.AnalyseCategory(categoryCtx, t, promptCats, categoryLLMHits, contextData)
|
||||
run.CategoryAnalysisDurationMS = time.Since(categoryStarted).Milliseconds()
|
||||
if err != nil {
|
||||
run.ExecutionChecks = append(run.ExecutionChecks, model.RuleCheck{Code: "execution_category_ai", Group: "execution", Label: "Kategorieanalyse konnte ausgeführt werden", Status: "fail", Blocking: true, Actual: err.Error(), Expected: "erfolgreich"})
|
||||
finishAnalysis(&categoryAnalysis, s.cfg.OllamaModel, nil, nil, "", 0, nil, model.ActionAudit{Type: "set_category", Result: "skipped: category_ai_failed"}, err)
|
||||
attachAnalysisTrace(&categoryAnalysis, categoryTrace)
|
||||
run.Analyses = append(run.Analyses, categoryAnalysis)
|
||||
run.Reason = "category_ai_failed"
|
||||
finish(err)
|
||||
@@ -295,6 +298,7 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
run.ExecutionChecks = append(run.ExecutionChecks, model.RuleCheck{Code: "execution_category_ai", Group: "execution", Label: "Kategorieanalyse konnte ausgeführt werden", Status: "pass", Actual: "erfolgreich", Expected: "erfolgreich"})
|
||||
run.CategoryAIReason = strings.TrimSpace(categoryDecision.Reason)
|
||||
finishAnalysis(&categoryAnalysis, s.cfg.OllamaModel, categoryDecision.Category, nil, categoryDecision.Reason, categoryDecision.Category.Confidence, nil, model.ActionAudit{Type: "set_category"}, nil)
|
||||
attachAnalysisTrace(&categoryAnalysis, categoryTrace)
|
||||
run.Analyses = append(run.Analyses, categoryAnalysis)
|
||||
categoryAnalysisIndex := len(run.Analyses) - 1
|
||||
|
||||
@@ -322,7 +326,8 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
run.PriorityBefore = t.Priority
|
||||
run.PriorityThreshold = s.cfg.PriorityConfidence
|
||||
if priorityClient, ok := s.ai.(priorityAI); ok {
|
||||
priorityCtx, cancelPriority := context.WithTimeout(ctx, priorityTimeout)
|
||||
priorityBaseCtx, priorityTrace := ollama.WithTrace(ctx, s.cfg.OllamaRoutingMode)
|
||||
priorityCtx, cancelPriority := context.WithTimeout(priorityBaseCtx, priorityTimeout)
|
||||
priorityDecision, priorityErr := priorityClient.AnalysePriority(priorityCtx, t, replyBasis, contextData)
|
||||
cancelPriority()
|
||||
run.PriorityAnalysisDurationMS = time.Since(priorityStarted).Milliseconds()
|
||||
@@ -352,6 +357,7 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
s.metrics.PriorityRecommendations.Add(1)
|
||||
}
|
||||
}
|
||||
attachAnalysisTrace(&priorityAnalysis, priorityTrace)
|
||||
} else {
|
||||
priorityErr := fmt.Errorf("AI client does not implement priority analysis")
|
||||
run.PriorityDecision = "priority_ai_unavailable"
|
||||
@@ -376,6 +382,7 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
var statusEval statusReplyEvaluation
|
||||
statusAnalysisStarted := time.Now()
|
||||
var statusAnalysisErr error
|
||||
var statusTrace *ollama.Trace
|
||||
switch {
|
||||
case !s.cfg.ContextStatusReplyEnabled:
|
||||
run.StatusAnalysisSkipReason = "status_reply_disabled"
|
||||
@@ -389,7 +396,9 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
run.StatusAnalysisSkipReason = "no_status_candidates"
|
||||
default:
|
||||
run.StatusAnalysisExecuted = true
|
||||
statusDecision, err = s.ai.AnalyseStatus(ctx, t, replyBasis, statusCandidates)
|
||||
statusCtx, trace := ollama.WithTrace(ctx, s.cfg.OllamaRoutingMode)
|
||||
statusTrace = trace
|
||||
statusDecision, err = s.ai.AnalyseStatus(statusCtx, t, replyBasis, statusCandidates)
|
||||
run.StatusAnalysisDurationMS = time.Since(statusAnalysisStarted).Milliseconds()
|
||||
if err != nil {
|
||||
statusAnalysisErr = err
|
||||
@@ -427,6 +436,7 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
statusActionResult = "skipped: " + run.StatusAnalysisSkipReason
|
||||
}
|
||||
finishAnalysis(&statusAnalysis, s.cfg.OllamaModel, statusEval, nil, statusDecision.Reason, statusDecision.Confidence, statusEval.Checks, model.ActionAudit{Type: "add_status_followup", Proposed: statusEval.Accepted, DryRun: s.cfg.DryRun, Result: statusActionResult}, statusAnalysisErr)
|
||||
attachAnalysisTrace(&statusAnalysis, statusTrace)
|
||||
run.Analyses = append(run.Analyses, statusAnalysis)
|
||||
statusAnalysisIndex := len(run.Analyses) - 1
|
||||
|
||||
@@ -457,6 +467,7 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
var replyDecision model.Decision
|
||||
replyAnalysisStarted := time.Now()
|
||||
var replyAnalysisErr error
|
||||
var replyTrace *ollama.Trace
|
||||
switch run.ReplyAnalysisSkipReason {
|
||||
case "status_reply_selected":
|
||||
replyDecision.Reason = "Normale Antwortanalyse nicht ausgeführt: Ein vordefiniertes Status-Template wurde freigegeben."
|
||||
@@ -472,7 +483,9 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
run.ExecutionChecks = append(run.ExecutionChecks, model.RuleCheck{Code: "execution_reply_ai", Group: "execution", Label: "Antwortanalyse wurde benötigt", Status: "info", Actual: "übersprungen: keine Kandidaten", Expected: "mindestens ein Antwortkandidat"})
|
||||
default:
|
||||
run.ReplyAnalysisExecuted = true
|
||||
replyDecision, err = s.ai.AnalyseReply(ctx, t, replyBasis, replyLLMHits, contextData)
|
||||
replyCtx, trace := ollama.WithTrace(ctx, s.cfg.OllamaRoutingMode)
|
||||
replyTrace = trace
|
||||
replyDecision, err = s.ai.AnalyseReply(replyCtx, t, replyBasis, replyLLMHits, contextData)
|
||||
run.ReplyAnalysisDurationMS = time.Since(replyAnalysisStarted).Milliseconds()
|
||||
if err != nil {
|
||||
replyAnalysisErr = err
|
||||
@@ -495,6 +508,7 @@ func (s *Service) ProcessWork(ctx context.Context, item queue.WorkItem) error {
|
||||
replyActionResult = "skipped: " + run.ReplyAnalysisSkipReason
|
||||
}
|
||||
finishAnalysis(&replyAnalysis, s.cfg.OllamaModel, replyDecision.Reply, nil, replyDecision.Reason, replyDecision.Reply.Confidence, nil, model.ActionAudit{Type: "add_followup", DryRun: s.cfg.DryRun, Result: replyActionResult}, replyAnalysisErr)
|
||||
attachAnalysisTrace(&replyAnalysis, replyTrace)
|
||||
run.Analyses = append(run.Analyses, replyAnalysis)
|
||||
replyAnalysisIndex := len(run.Analyses) - 1
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/example/glpi-ai-agent/internal/config"
|
||||
"github.com/example/glpi-ai-agent/internal/model"
|
||||
"github.com/example/glpi-ai-agent/internal/ollama"
|
||||
"github.com/example/glpi-ai-agent/internal/state"
|
||||
)
|
||||
|
||||
@@ -80,6 +81,17 @@ func finishAnalysis(a *model.AnalysisRun, modelName string, decision any, reason
|
||||
}
|
||||
}
|
||||
|
||||
func attachAnalysisTrace(a *model.AnalysisRun, trace *ollama.Trace) {
|
||||
if a == nil || trace == nil {
|
||||
return
|
||||
}
|
||||
snapshot := trace.Snapshot()
|
||||
if len(snapshot.Attempts) == 0 {
|
||||
return
|
||||
}
|
||||
a.Provider = snapshot
|
||||
}
|
||||
|
||||
func mustJSON(v any) json.RawMessage {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/example/glpi-ai-agent/internal/model"
|
||||
"github.com/example/glpi-ai-agent/internal/ollama"
|
||||
"github.com/example/glpi-ai-agent/internal/queue"
|
||||
)
|
||||
|
||||
@@ -116,15 +117,17 @@ func (s *Service) processEscalation(ctx context.Context, item queue.WorkItem) er
|
||||
finish(err)
|
||||
return err
|
||||
}
|
||||
analysisCtx := ctx
|
||||
analysisBaseCtx, escalationTrace := ollama.WithTrace(ctx, s.cfg.OllamaRoutingMode)
|
||||
analysisCtx := analysisBaseCtx
|
||||
cancel := func() {}
|
||||
if s.cfg.EscalationAnalysisTimeout > 0 {
|
||||
analysisCtx, cancel = context.WithTimeout(ctx, s.cfg.EscalationAnalysisTimeout)
|
||||
analysisCtx, cancel = context.WithTimeout(analysisBaseCtx, s.cfg.EscalationAnalysisTimeout)
|
||||
}
|
||||
decision, err := ai.AnalyseEscalation(analysisCtx, t, followups, contextData, evidence, constraints)
|
||||
cancel()
|
||||
if err != nil {
|
||||
finishAnalysis(&analysis, s.cfg.OllamaModel, nil, nil, "", 0, nil, model.ActionAudit{Type: "escalation_plan", Result: "skipped: escalation_ai_failed"}, err)
|
||||
attachAnalysisTrace(&analysis, escalationTrace)
|
||||
run.Analyses = append(run.Analyses, analysis)
|
||||
run.Reason = "escalation_ai_failed"
|
||||
finish(err)
|
||||
@@ -170,6 +173,7 @@ func (s *Service) processEscalation(ctx context.Context, item queue.WorkItem) er
|
||||
action, err = s.executeEscalationPlan(ctx, t, decision, result, contextData)
|
||||
}
|
||||
finishAnalysis(&analysis, s.cfg.OllamaModel, result, decision.ReasonCodes, decision.Reason, decision.Confidence, result.Checks, action, err)
|
||||
attachAnalysisTrace(&analysis, escalationTrace)
|
||||
run.Analyses = append(run.Analyses, analysis)
|
||||
run.Reason = decision.Reason
|
||||
run.AIReason = decision.Reason
|
||||
|
||||
@@ -37,15 +37,27 @@ type Config struct {
|
||||
GLPIAllowInsecureHTTP bool
|
||||
GLPIAllowedStatusIDs []int64
|
||||
|
||||
OllamaURL string
|
||||
OllamaModel string
|
||||
OllamaEmbeddingModel string
|
||||
OllamaTimeout time.Duration
|
||||
OllamaNumPredict int
|
||||
OllamaKeepAlive time.Duration
|
||||
OllamaThink bool
|
||||
OllamaMaxConcurrent int
|
||||
OllamaJSONRetries int
|
||||
OllamaURL string // legacy single-node value
|
||||
OllamaURLs []string
|
||||
OllamaNodeNames []string
|
||||
OllamaNodeWeights []int
|
||||
OllamaModel string
|
||||
OllamaEmbeddingModel string
|
||||
OllamaTimeout time.Duration
|
||||
OllamaNumPredict int
|
||||
OllamaKeepAlive time.Duration
|
||||
OllamaThink bool
|
||||
OllamaMaxConcurrent int // legacy alias for per-node concurrency
|
||||
OllamaNodeMaxInflight int
|
||||
OllamaRoutingMode string
|
||||
OllamaNodeHealthInterval time.Duration
|
||||
OllamaNodeFailureCooldown time.Duration
|
||||
OllamaNodeRequestTimeout time.Duration
|
||||
OllamaFailoverEnabled bool
|
||||
OllamaFailoverAttempts int
|
||||
OllamaRequireSameDigest bool
|
||||
OllamaRequireEmbeddingModel bool
|
||||
OllamaJSONRetries int
|
||||
|
||||
KnowledgeDir string
|
||||
RAGEnabled bool
|
||||
@@ -206,6 +218,9 @@ func Load() (Config, error) {
|
||||
GLPIAllowInsecureHTTP: envBool("GLPI_ALLOW_INSECURE_HTTP", false),
|
||||
GLPIAllowedStatusIDs: envInt64List("GLPI_ALLOWED_STATUS_IDS", "1"),
|
||||
OllamaURL: strings.TrimRight(env("OLLAMA_URL", "http://ollama:11434"), "/"),
|
||||
OllamaURLs: envStringListPreserveCase("OLLAMA_URLS", ""),
|
||||
OllamaNodeNames: envStringListPreserveCase("OLLAMA_NODE_NAMES", ""),
|
||||
OllamaNodeWeights: envIntListAllowEmpty("OLLAMA_NODE_WEIGHTS"),
|
||||
OllamaModel: env("OLLAMA_MODEL", "qwen3:8b"),
|
||||
OllamaEmbeddingModel: env("OLLAMA_EMBEDDING_MODEL", "embeddinggemma"),
|
||||
OllamaTimeout: envDuration("OLLAMA_TIMEOUT", 10*time.Minute),
|
||||
@@ -213,6 +228,15 @@ func Load() (Config, error) {
|
||||
OllamaKeepAlive: envDuration("OLLAMA_KEEP_ALIVE", 10*time.Minute),
|
||||
OllamaThink: envBool("OLLAMA_THINK", false),
|
||||
OllamaMaxConcurrent: envInt("OLLAMA_MAX_CONCURRENT", 1),
|
||||
OllamaNodeMaxInflight: envInt("OLLAMA_NODE_MAX_INFLIGHT", 0),
|
||||
OllamaRoutingMode: envNormalizedLower("OLLAMA_ROUTING_MODE", "least_inflight"),
|
||||
OllamaNodeHealthInterval: envDuration("OLLAMA_NODE_HEALTH_INTERVAL", 15*time.Second),
|
||||
OllamaNodeFailureCooldown: envDuration("OLLAMA_NODE_FAILURE_COOLDOWN", 30*time.Second),
|
||||
OllamaNodeRequestTimeout: envDuration("OLLAMA_NODE_REQUEST_TIMEOUT", 0),
|
||||
OllamaFailoverEnabled: envBool("OLLAMA_FAILOVER_ENABLED", true),
|
||||
OllamaFailoverAttempts: envInt("OLLAMA_FAILOVER_ATTEMPTS", 0),
|
||||
OllamaRequireSameDigest: envBool("OLLAMA_REQUIRE_SAME_MODEL_DIGEST", true),
|
||||
OllamaRequireEmbeddingModel: envBool("OLLAMA_REQUIRE_EMBEDDING_MODEL", true),
|
||||
OllamaJSONRetries: envInt("OLLAMA_JSON_RETRIES", 1),
|
||||
KnowledgeDir: env("KNOWLEDGE_DIR", "./knowledge"),
|
||||
RAGEnabled: envBool("RAG_ENABLED", true),
|
||||
@@ -344,6 +368,21 @@ func Load() (Config, error) {
|
||||
QueueSize: envInt("QUEUE_SIZE", 256),
|
||||
Workers: envInt("WORKERS", 2),
|
||||
}
|
||||
if len(c.OllamaURLs) == 0 {
|
||||
c.OllamaURLs = []string{c.OllamaURL}
|
||||
}
|
||||
for i := range c.OllamaURLs {
|
||||
c.OllamaURLs[i] = strings.TrimRight(strings.TrimSpace(c.OllamaURLs[i]), "/")
|
||||
}
|
||||
if c.OllamaNodeMaxInflight == 0 {
|
||||
c.OllamaNodeMaxInflight = c.OllamaMaxConcurrent
|
||||
}
|
||||
if c.OllamaNodeRequestTimeout == 0 {
|
||||
c.OllamaNodeRequestTimeout = c.OllamaTimeout
|
||||
}
|
||||
if c.OllamaFailoverAttempts == 0 {
|
||||
c.OllamaFailoverAttempts = len(c.OllamaURLs)
|
||||
}
|
||||
// Backwards compatibility: without an explicit category source list, the
|
||||
// same sources used for normal knowledge retrieval also inform classification.
|
||||
if _, configured := os.LookupEnv("KNOWLEDGE_CATEGORY_SOURCES"); !configured {
|
||||
@@ -410,6 +449,87 @@ func (c Config) Validate() error {
|
||||
if c.OllamaMaxConcurrent <= 0 || c.OllamaMaxConcurrent > 32 {
|
||||
return errors.New("OLLAMA_MAX_CONCURRENT must be between 1 and 32")
|
||||
}
|
||||
ollamaURLs := append([]string(nil), c.OllamaURLs...)
|
||||
if len(ollamaURLs) == 0 {
|
||||
legacyURL := strings.TrimSpace(c.OllamaURL)
|
||||
if legacyURL == "" {
|
||||
legacyURL = "http://ollama:11434"
|
||||
}
|
||||
ollamaURLs = []string{legacyURL}
|
||||
}
|
||||
if len(ollamaURLs) > 64 {
|
||||
return errors.New("OLLAMA_URLS supports at most 64 nodes")
|
||||
}
|
||||
seenOllamaURLs := map[string]struct{}{}
|
||||
for _, raw := range ollamaURLs {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
|
||||
return fmt.Errorf("invalid Ollama node URL %q", raw)
|
||||
}
|
||||
key := strings.TrimRight(u.String(), "/")
|
||||
if _, ok := seenOllamaURLs[key]; ok {
|
||||
return fmt.Errorf("duplicate Ollama node URL %q", key)
|
||||
}
|
||||
seenOllamaURLs[key] = struct{}{}
|
||||
}
|
||||
if len(c.OllamaNodeNames) > 0 && len(c.OllamaNodeNames) != len(ollamaURLs) {
|
||||
return errors.New("OLLAMA_NODE_NAMES must contain exactly one name per OLLAMA_URLS entry")
|
||||
}
|
||||
seenOllamaNames := map[string]struct{}{}
|
||||
for _, rawName := range c.OllamaNodeNames {
|
||||
name := strings.TrimSpace(rawName)
|
||||
if name == "" {
|
||||
return errors.New("OLLAMA_NODE_NAMES entries must not be empty")
|
||||
}
|
||||
if _, ok := seenOllamaNames[name]; ok {
|
||||
return fmt.Errorf("duplicate Ollama node name %q", name)
|
||||
}
|
||||
seenOllamaNames[name] = struct{}{}
|
||||
}
|
||||
if len(c.OllamaNodeWeights) > 0 && len(c.OllamaNodeWeights) != len(ollamaURLs) {
|
||||
return errors.New("OLLAMA_NODE_WEIGHTS must contain exactly one weight per OLLAMA_URLS entry")
|
||||
}
|
||||
for _, weight := range c.OllamaNodeWeights {
|
||||
if weight < 1 || weight > 100 {
|
||||
return errors.New("OLLAMA_NODE_WEIGHTS values must be between 1 and 100")
|
||||
}
|
||||
}
|
||||
nodeMaxInflight := c.OllamaNodeMaxInflight
|
||||
if nodeMaxInflight == 0 {
|
||||
nodeMaxInflight = c.OllamaMaxConcurrent
|
||||
}
|
||||
if nodeMaxInflight < 1 || nodeMaxInflight > 32 {
|
||||
return errors.New("OLLAMA_NODE_MAX_INFLIGHT must be between 1 and 32")
|
||||
}
|
||||
routingMode := c.OllamaRoutingMode
|
||||
if routingMode == "" {
|
||||
routingMode = "least_inflight"
|
||||
}
|
||||
switch routingMode {
|
||||
case "least_inflight", "round_robin", "weighted", "fastest_recent":
|
||||
default:
|
||||
return errors.New("OLLAMA_ROUTING_MODE must be one of: least_inflight, round_robin, weighted, fastest_recent")
|
||||
}
|
||||
if c.OllamaNodeHealthInterval != 0 && c.OllamaNodeHealthInterval < time.Second {
|
||||
return errors.New("OLLAMA_NODE_HEALTH_INTERVAL must be >= 1s")
|
||||
}
|
||||
if c.OllamaNodeFailureCooldown < 0 {
|
||||
return errors.New("OLLAMA_NODE_FAILURE_COOLDOWN must be >= 0")
|
||||
}
|
||||
nodeRequestTimeout := c.OllamaNodeRequestTimeout
|
||||
if nodeRequestTimeout == 0 {
|
||||
nodeRequestTimeout = c.OllamaTimeout
|
||||
}
|
||||
if nodeRequestTimeout <= 0 {
|
||||
return errors.New("OLLAMA_NODE_REQUEST_TIMEOUT must be > 0")
|
||||
}
|
||||
failoverAttempts := c.OllamaFailoverAttempts
|
||||
if failoverAttempts == 0 {
|
||||
failoverAttempts = len(ollamaURLs)
|
||||
}
|
||||
if failoverAttempts < 1 || failoverAttempts > len(ollamaURLs) {
|
||||
return errors.New("OLLAMA_FAILOVER_ATTEMPTS must be between 1 and the number of configured Ollama nodes")
|
||||
}
|
||||
if c.OllamaJSONRetries < 0 || c.OllamaJSONRetries > 3 {
|
||||
return errors.New("OLLAMA_JSON_RETRIES must be between 0 and 3")
|
||||
}
|
||||
@@ -975,6 +1095,23 @@ func envPathList(key, def string) []string {
|
||||
return vals
|
||||
}
|
||||
|
||||
func envIntListAllowEmpty(key string) []int {
|
||||
raw := strings.TrimSpace(os.Getenv(key))
|
||||
if raw == "" || strings.EqualFold(raw, "none") {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(raw, ",")
|
||||
out := make([]int, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(part))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func envBool(key string, def bool) bool {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
|
||||
@@ -440,3 +440,48 @@ func TestValidateEscalationLinkAdapterRequiresSourceAndTarget(t *testing.T) {
|
||||
t.Fatalf("valid link adapter rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateOllamaPoolConfiguration(t *testing.T) {
|
||||
c := validConfig()
|
||||
c.OllamaURLs = []string{"http://ollama-1.internal:11434", "http://ollama-2.internal:11434"}
|
||||
c.OllamaNodeNames = []string{"lenovo-1", "lenovo-2"}
|
||||
c.OllamaNodeWeights = []int{1, 2}
|
||||
c.OllamaNodeMaxInflight = 1
|
||||
c.OllamaRoutingMode = "weighted"
|
||||
c.OllamaNodeHealthInterval = 10 * time.Second
|
||||
c.OllamaNodeFailureCooldown = 30 * time.Second
|
||||
c.OllamaNodeRequestTimeout = time.Minute
|
||||
c.OllamaFailoverAttempts = 2
|
||||
if err := c.Validate(); err != nil {
|
||||
t.Fatalf("valid pool rejected: %v", err)
|
||||
}
|
||||
|
||||
c.OllamaNodeNames = []string{"only-one"}
|
||||
if err := c.Validate(); err == nil {
|
||||
t.Fatal("expected mismatched node names to be rejected")
|
||||
}
|
||||
|
||||
c.OllamaNodeNames = []string{"duplicate", "duplicate"}
|
||||
if err := c.Validate(); err == nil {
|
||||
t.Fatal("expected duplicate node names to be rejected")
|
||||
}
|
||||
|
||||
c = validConfig()
|
||||
c.OllamaURLs = []string{"http://user:secret@ollama-1.internal:11434"}
|
||||
c.OllamaNodeMaxInflight = 1
|
||||
c.OllamaNodeRequestTimeout = time.Minute
|
||||
c.OllamaFailoverAttempts = 1
|
||||
if err := c.Validate(); err == nil {
|
||||
t.Fatal("expected credentials in Ollama node URL to be rejected")
|
||||
}
|
||||
|
||||
c = validConfig()
|
||||
c.OllamaURLs = []string{"http://ollama-1.internal:11434", "http://ollama-2.internal:11434"}
|
||||
c.OllamaNodeMaxInflight = 1
|
||||
c.OllamaNodeRequestTimeout = time.Minute
|
||||
c.OllamaRoutingMode = "unknown"
|
||||
c.OllamaFailoverAttempts = 2
|
||||
if err := c.Validate(); err == nil {
|
||||
t.Fatal("expected unknown routing mode to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -337,30 +337,89 @@ type ContextAuditItem struct {
|
||||
}
|
||||
|
||||
// AnalysisRun is one independently auditable AI or deterministic analysis stage.
|
||||
// OllamaRequestAttempt captures one concrete HTTP attempt against an Ollama node.
|
||||
// It is attached to an AnalysisRun so routing and failover remain auditable.
|
||||
type OllamaRequestAttempt struct {
|
||||
Attempt int `json:"attempt"`
|
||||
Stage string `json:"stage,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
NodeName string `json:"node_name"`
|
||||
NodeURL string `json:"node_url"`
|
||||
ModelDigest string `json:"model_digest,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
InflightAtStart int64 `json:"inflight_at_start,omitempty"`
|
||||
HTTPStatus int `json:"http_status,omitempty"`
|
||||
Outcome string `json:"outcome"`
|
||||
Retryable bool `json:"retryable,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
TotalDurationNS int64 `json:"total_duration_ns,omitempty"`
|
||||
LoadDurationNS int64 `json:"load_duration_ns,omitempty"`
|
||||
PromptEvalCount int64 `json:"prompt_eval_count,omitempty"`
|
||||
PromptEvalDuration int64 `json:"prompt_eval_duration_ns,omitempty"`
|
||||
EvalCount int64 `json:"eval_count,omitempty"`
|
||||
EvalDuration int64 `json:"eval_duration_ns,omitempty"`
|
||||
}
|
||||
|
||||
// OllamaProviderTrace summarizes the routing of one logical analysis.
|
||||
type OllamaProviderTrace struct {
|
||||
Provider string `json:"provider,omitempty"`
|
||||
RoutingMode string `json:"routing_mode,omitempty"`
|
||||
SelectedNode string `json:"selected_node,omitempty"`
|
||||
SelectedURL string `json:"selected_url,omitempty"`
|
||||
FailoverUsed bool `json:"failover_used,omitempty"`
|
||||
AttemptCount int `json:"attempt_count,omitempty"`
|
||||
Attempts []OllamaRequestAttempt `json:"attempts,omitempty"`
|
||||
}
|
||||
|
||||
// OllamaNodeStatus is the read-only operational state exposed in the dashboard.
|
||||
type OllamaNodeStatus struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Weight int `json:"weight"`
|
||||
Healthy bool `json:"healthy"`
|
||||
Compatible bool `json:"compatible"`
|
||||
Available bool `json:"available"`
|
||||
InFlight int64 `json:"inflight"`
|
||||
MaxInFlight int `json:"max_inflight"`
|
||||
ChatModelDigest string `json:"chat_model_digest,omitempty"`
|
||||
EmbeddingModelDigest string `json:"embedding_model_digest,omitempty"`
|
||||
LastCheck time.Time `json:"last_check,omitempty"`
|
||||
LastSuccess time.Time `json:"last_success,omitempty"`
|
||||
CooldownUntil time.Time `json:"cooldown_until,omitempty"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
Requests uint64 `json:"requests"`
|
||||
Failures uint64 `json:"failures"`
|
||||
AverageDurationMS float64 `json:"average_duration_ms,omitempty"`
|
||||
LastRequestDurationMS int64 `json:"last_request_duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
// Decision and InputSnapshot contain the exact structured values used at execution
|
||||
// time, so later diagnostics do not depend on the current ticket or configuration.
|
||||
type AnalysisRun struct {
|
||||
AnalysisID string `json:"analysis_id"`
|
||||
ParentRunID string `json:"parent_run_id"`
|
||||
TicketID int64 `json:"ticket_id"`
|
||||
AnalysisType string `json:"analysis_type"`
|
||||
Trigger string `json:"trigger"`
|
||||
SourceVersion string `json:"source_version"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
InputHash string `json:"input_hash,omitempty"`
|
||||
InputSnapshot json.RawMessage `json:"input_snapshot,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Outcome string `json:"outcome"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Decision json.RawMessage `json:"decision,omitempty"`
|
||||
ReasonCodes []string `json:"reason_codes,omitempty"`
|
||||
Explanation string `json:"explanation,omitempty"`
|
||||
Confidence float64 `json:"confidence,omitempty"`
|
||||
Checks []RuleCheck `json:"checks,omitempty"`
|
||||
Action ActionAudit `json:"action,omitempty"`
|
||||
AnalysisID string `json:"analysis_id"`
|
||||
ParentRunID string `json:"parent_run_id"`
|
||||
TicketID int64 `json:"ticket_id"`
|
||||
AnalysisType string `json:"analysis_type"`
|
||||
Trigger string `json:"trigger"`
|
||||
SourceVersion string `json:"source_version"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
InputHash string `json:"input_hash,omitempty"`
|
||||
InputSnapshot json.RawMessage `json:"input_snapshot,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Outcome string `json:"outcome"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Decision json.RawMessage `json:"decision,omitempty"`
|
||||
ReasonCodes []string `json:"reason_codes,omitempty"`
|
||||
Explanation string `json:"explanation,omitempty"`
|
||||
Confidence float64 `json:"confidence,omitempty"`
|
||||
Checks []RuleCheck `json:"checks,omitempty"`
|
||||
Action ActionAudit `json:"action,omitempty"`
|
||||
Provider OllamaProviderTrace `json:"provider,omitempty"`
|
||||
}
|
||||
|
||||
type ActionStepAudit struct {
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
package ollama
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -16,37 +13,58 @@ import (
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
baseURL, model, embeddingModel string
|
||||
language, communicationStyle string
|
||||
numPredict int
|
||||
jsonRetries int
|
||||
keepAlive time.Duration
|
||||
think bool
|
||||
sem chan struct{}
|
||||
http *http.Client
|
||||
model, embeddingModel string
|
||||
language, communicationStyle string
|
||||
numPredict int
|
||||
jsonRetries int
|
||||
keepAlive time.Duration
|
||||
think bool
|
||||
routingMode string
|
||||
pool *Pool
|
||||
}
|
||||
|
||||
// New preserves the former single-node API and creates a one-node pool.
|
||||
func New(baseURL, model, embeddingModel, language, communicationStyle string, timeout time.Duration, numPredict int, keepAlive time.Duration, think bool, maxConcurrent, jsonRetries int) *Client {
|
||||
return &Client{
|
||||
baseURL: strings.TrimRight(baseURL, "/"), model: model, embeddingModel: embeddingModel,
|
||||
language: language, communicationStyle: communicationStyle, numPredict: numPredict, keepAlive: keepAlive, think: think, jsonRetries: jsonRetries,
|
||||
sem: make(chan struct{}, maxConcurrent),
|
||||
http: &http.Client{Timeout: timeout},
|
||||
}
|
||||
}
|
||||
func (c *Client) Ping(ctx context.Context) error {
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/api/tags", nil)
|
||||
resp, err := c.http.Do(req)
|
||||
c, err := NewPool(PoolConfig{
|
||||
Nodes: []NodeConfig{{Name: "ollama-1", URL: baseURL, Weight: 1}}, RoutingMode: "least_inflight",
|
||||
NodeMaxInflight: maxConcurrent, HealthInterval: 15 * time.Second, FailureCooldown: 30 * time.Second,
|
||||
NodeRequestTimeout: timeout, FailoverEnabled: false, FailoverAttempts: 1,
|
||||
RequireSameModelDigest: true, RequireEmbeddingModel: true, Model: model, EmbeddingModel: embeddingModel,
|
||||
}, model, embeddingModel, language, communicationStyle, numPredict, keepAlive, think, jsonRetries)
|
||||
if err != nil {
|
||||
return err
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("Ollama HTTP %d", resp.StatusCode)
|
||||
// Backwards-compatible single-node clients historically sent requests
|
||||
// without a preceding /api/tags probe. Keep that behavior for tests and
|
||||
// embedded users; the periodic health check will replace this optimistic
|
||||
// state as soon as Start or Ping is used.
|
||||
if len(c.pool.nodes) == 1 {
|
||||
n := c.pool.nodes[0]
|
||||
n.mu.Lock()
|
||||
n.healthy = true
|
||||
n.compatible = true
|
||||
n.chatDigest = "legacy-unverified"
|
||||
n.embeddingDigest = "legacy-unverified"
|
||||
n.lastCheck = time.Now()
|
||||
n.mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
return c
|
||||
}
|
||||
|
||||
func NewPool(poolCfg PoolConfig, model, embeddingModel, language, communicationStyle string, numPredict int, keepAlive time.Duration, think bool, jsonRetries int) (*Client, error) {
|
||||
p, err := newPool(poolCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Client{model: model, embeddingModel: embeddingModel, language: language, communicationStyle: communicationStyle, numPredict: numPredict, keepAlive: keepAlive, think: think, jsonRetries: jsonRetries, routingMode: p.cfg.RoutingMode, pool: p}, nil
|
||||
}
|
||||
|
||||
func (c *Client) Start(ctx context.Context) { c.pool.Start(ctx) }
|
||||
func (c *Client) Ping(ctx context.Context) error { return c.pool.Ping(ctx) }
|
||||
func (c *Client) NodeStatuses() []model.OllamaNodeStatus { return c.pool.NodeStatuses() }
|
||||
func (c *Client) RoutingMode() string { return c.routingMode }
|
||||
func (c *Client) Embed(ctx context.Context, texts []string) ([][]float64, error) {
|
||||
ctx = withStage(ctx, "embedding")
|
||||
if len(texts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -63,6 +81,7 @@ func (c *Client) Embed(ctx context.Context, texts []string) ([][]float64, error)
|
||||
return out.Embeddings, nil
|
||||
}
|
||||
func (c *Client) AnalyseCategory(ctx context.Context, t model.Ticket, categories []model.Category, categoryHits []model.KnowledgeHit, contextData model.ContextSnapshot) (model.Decision, error) {
|
||||
ctx = withStage(ctx, "category")
|
||||
categoryIDs := []int64{0}
|
||||
for _, category := range categories {
|
||||
if category.ID != 0 {
|
||||
@@ -104,6 +123,7 @@ func (c *Client) AnalyseCategory(ctx context.Context, t model.Ticket, categories
|
||||
}
|
||||
|
||||
func (c *Client) AnalyseStatus(ctx context.Context, t model.Ticket, category model.Category, candidates []model.ServiceIssueCandidate) (model.StatusDecision, error) {
|
||||
ctx = withStage(ctx, "status_match")
|
||||
if len(candidates) == 0 {
|
||||
return model.StatusDecision{Reason: "Keine aktiven Störungs- oder Wartungskandidaten verfügbar."}, nil
|
||||
}
|
||||
@@ -170,6 +190,7 @@ func (c *Client) AnalyseStatus(ctx context.Context, t model.Ticket, category mod
|
||||
}
|
||||
|
||||
func (c *Client) AnalyseReply(ctx context.Context, t model.Ticket, category model.Category, replyHits []model.KnowledgeHit, contextData model.ContextSnapshot) (model.Decision, error) {
|
||||
ctx = withStage(ctx, "reply_selection")
|
||||
if len(replyHits) == 0 {
|
||||
var d model.Decision
|
||||
d.Reason = "Keine Antwort-Knowledge-Kandidaten verfügbar."
|
||||
@@ -257,6 +278,7 @@ func (c *Client) executeDecision(ctx context.Context, payload map[string]any, va
|
||||
}
|
||||
|
||||
func (c *Client) Analyse(ctx context.Context, t model.Ticket, categories []model.Category, categoryHits, replyHits []model.KnowledgeHit, contextData model.ContextSnapshot) (model.Decision, error) {
|
||||
ctx = withStage(ctx, "combined")
|
||||
knowledgeIDs := []string{""}
|
||||
knownKnowledge := map[string]struct{}{}
|
||||
for _, h := range replyHits {
|
||||
@@ -348,44 +370,11 @@ func (c *Client) Analyse(ctx context.Context, t model.Ticket, categories []model
|
||||
return model.Decision{}, lastErr
|
||||
}
|
||||
func (c *Client) post(ctx context.Context, path string, payload any, out any) error {
|
||||
select {
|
||||
case c.sem <- struct{}{}:
|
||||
defer func() { <-c.sem }()
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
b, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("Ollama %s failed: HTTP %d: %s", path, resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return errors.New("empty Ollama response")
|
||||
}
|
||||
return json.Unmarshal(body, out)
|
||||
return c.pool.post(ctx, path, payload, out)
|
||||
}
|
||||
|
||||
func (c *Client) AnalysePriority(ctx context.Context, t model.Ticket, category model.Category, contextData model.ContextSnapshot) (model.PriorityDecision, error) {
|
||||
ctx = withStage(ctx, "priority")
|
||||
evidence := prioritysignals.Extract(t)
|
||||
reasonCodes := []string{
|
||||
"single_user_affected", "multiple_users_affected", "site_affected", "organization_affected",
|
||||
@@ -429,6 +418,7 @@ Verwende insufficient_information nur, wenn weder Auswirkung noch Dringlichkeit
|
||||
}
|
||||
|
||||
func (c *Client) AnalyseEscalation(ctx context.Context, t model.Ticket, followups []model.Followup, contextData model.ContextSnapshot, evidence model.EscalationEvidence, constraints model.EscalationConstraints) (model.EscalationDecision, error) {
|
||||
ctx = withStage(ctx, "escalation")
|
||||
actions := uniqueStrings(constraints.AllowedActions)
|
||||
if !containsString(actions, "none") {
|
||||
actions = append([]string{"none"}, actions...)
|
||||
|
||||
783
internal/ollama/pool.go
Normal file
783
internal/ollama/pool.go
Normal file
@@ -0,0 +1,783 @@
|
||||
package ollama
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/example/glpi-ai-agent/internal/model"
|
||||
)
|
||||
|
||||
const providerName = "ollama-pool"
|
||||
|
||||
type NodeConfig struct {
|
||||
Name string
|
||||
URL string
|
||||
Weight int
|
||||
}
|
||||
|
||||
type PoolConfig struct {
|
||||
Nodes []NodeConfig
|
||||
RoutingMode string
|
||||
NodeMaxInflight int
|
||||
HealthInterval time.Duration
|
||||
FailureCooldown time.Duration
|
||||
NodeRequestTimeout time.Duration
|
||||
FailoverEnabled bool
|
||||
FailoverAttempts int
|
||||
RequireSameModelDigest bool
|
||||
RequireEmbeddingModel bool
|
||||
Model string
|
||||
EmbeddingModel string
|
||||
}
|
||||
|
||||
type requestMeta struct {
|
||||
TotalDuration int64 `json:"total_duration"`
|
||||
LoadDuration int64 `json:"load_duration"`
|
||||
PromptEvalCount int64 `json:"prompt_eval_count"`
|
||||
PromptEvalDuration int64 `json:"prompt_eval_duration"`
|
||||
EvalCount int64 `json:"eval_count"`
|
||||
EvalDuration int64 `json:"eval_duration"`
|
||||
}
|
||||
|
||||
type poolNode struct {
|
||||
name string
|
||||
baseURL string
|
||||
weight int
|
||||
sem chan struct{}
|
||||
inflight atomic.Int64
|
||||
requests atomic.Uint64
|
||||
failures atomic.Uint64
|
||||
|
||||
mu sync.RWMutex
|
||||
healthy bool
|
||||
compatible bool
|
||||
chatDigest string
|
||||
embeddingDigest string
|
||||
lastCheck time.Time
|
||||
lastSuccess time.Time
|
||||
cooldownUntil time.Time
|
||||
consecutiveFailures int
|
||||
lastError string
|
||||
averageDurationMS float64
|
||||
lastRequestDurationMS int64
|
||||
}
|
||||
|
||||
func (n *poolNode) acquire() (int64, bool) {
|
||||
select {
|
||||
case n.sem <- struct{}{}:
|
||||
return n.inflight.Add(1), true
|
||||
default:
|
||||
return n.inflight.Load(), false
|
||||
}
|
||||
}
|
||||
|
||||
func (n *poolNode) release() {
|
||||
n.inflight.Add(-1)
|
||||
<-n.sem
|
||||
}
|
||||
|
||||
func (n *poolNode) isEligible(now time.Time, stage string) bool {
|
||||
n.mu.RLock()
|
||||
defer n.mu.RUnlock()
|
||||
if !n.healthy || !n.compatible || now.Before(n.cooldownUntil) {
|
||||
return false
|
||||
}
|
||||
if stage == "embedding" && n.embeddingDigest == "" {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (n *poolNode) recordRequest(duration time.Duration, ok bool, retryable bool, err error) {
|
||||
n.requests.Add(1)
|
||||
ms := duration.Milliseconds()
|
||||
if ms < 0 {
|
||||
ms = 0
|
||||
}
|
||||
n.mu.Lock()
|
||||
n.lastRequestDurationMS = ms
|
||||
if ok {
|
||||
if n.averageDurationMS == 0 {
|
||||
n.averageDurationMS = float64(ms)
|
||||
} else {
|
||||
n.averageDurationMS = n.averageDurationMS*0.8 + float64(ms)*0.2
|
||||
}
|
||||
n.lastSuccess = time.Now()
|
||||
n.consecutiveFailures = 0
|
||||
n.lastError = ""
|
||||
n.cooldownUntil = time.Time{}
|
||||
} else {
|
||||
n.failures.Add(1)
|
||||
n.consecutiveFailures++
|
||||
if err != nil {
|
||||
n.lastError = err.Error()
|
||||
}
|
||||
if retryable {
|
||||
// The caller applies the configured cooldown after releasing the lock.
|
||||
}
|
||||
}
|
||||
n.mu.Unlock()
|
||||
}
|
||||
|
||||
func (n *poolNode) status(maxInflight int) model.OllamaNodeStatus {
|
||||
n.mu.RLock()
|
||||
defer n.mu.RUnlock()
|
||||
now := time.Now()
|
||||
return model.OllamaNodeStatus{
|
||||
Name: n.name, URL: n.baseURL, Weight: n.weight,
|
||||
Healthy: n.healthy, Compatible: n.compatible,
|
||||
Available: n.healthy && n.compatible && !now.Before(n.cooldownUntil) && n.inflight.Load() < int64(maxInflight),
|
||||
InFlight: n.inflight.Load(), MaxInFlight: maxInflight,
|
||||
ChatModelDigest: n.chatDigest, EmbeddingModelDigest: n.embeddingDigest,
|
||||
LastCheck: n.lastCheck, LastSuccess: n.lastSuccess, CooldownUntil: n.cooldownUntil,
|
||||
ConsecutiveFailures: n.consecutiveFailures, LastError: n.lastError,
|
||||
Requests: n.requests.Load(), Failures: n.failures.Load(),
|
||||
AverageDurationMS: n.averageDurationMS, LastRequestDurationMS: n.lastRequestDurationMS,
|
||||
}
|
||||
}
|
||||
|
||||
type Pool struct {
|
||||
cfg PoolConfig
|
||||
nodes []*poolNode
|
||||
http *http.Client
|
||||
rr atomic.Uint64
|
||||
refresh sync.Mutex
|
||||
started atomic.Bool
|
||||
}
|
||||
|
||||
func newPool(cfg PoolConfig) (*Pool, error) {
|
||||
if len(cfg.Nodes) == 0 {
|
||||
return nil, errors.New("at least one Ollama node is required")
|
||||
}
|
||||
if len(cfg.Nodes) > 64 {
|
||||
return nil, errors.New("at most 64 Ollama nodes are supported")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Model) == "" {
|
||||
return nil, errors.New("Ollama chat model must not be empty")
|
||||
}
|
||||
if cfg.RequireEmbeddingModel && strings.TrimSpace(cfg.EmbeddingModel) == "" {
|
||||
return nil, errors.New("Ollama embedding model must not be empty when it is required")
|
||||
}
|
||||
if cfg.NodeMaxInflight <= 0 {
|
||||
cfg.NodeMaxInflight = 1
|
||||
}
|
||||
if cfg.HealthInterval <= 0 {
|
||||
cfg.HealthInterval = 15 * time.Second
|
||||
}
|
||||
if cfg.FailureCooldown < 0 {
|
||||
cfg.FailureCooldown = 0
|
||||
}
|
||||
if cfg.NodeRequestTimeout <= 0 {
|
||||
cfg.NodeRequestTimeout = 10 * time.Minute
|
||||
}
|
||||
if cfg.FailoverAttempts <= 0 {
|
||||
cfg.FailoverAttempts = len(cfg.Nodes)
|
||||
}
|
||||
if cfg.FailoverAttempts > len(cfg.Nodes) {
|
||||
cfg.FailoverAttempts = len(cfg.Nodes)
|
||||
}
|
||||
cfg.RoutingMode = strings.ToLower(strings.TrimSpace(cfg.RoutingMode))
|
||||
if cfg.RoutingMode == "" {
|
||||
cfg.RoutingMode = "least_inflight"
|
||||
}
|
||||
switch cfg.RoutingMode {
|
||||
case "least_inflight", "round_robin", "weighted", "fastest_recent":
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported Ollama routing mode %q", cfg.RoutingMode)
|
||||
}
|
||||
p := &Pool{cfg: cfg, http: &http.Client{Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 16,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
ResponseHeaderTimeout: cfg.NodeRequestTimeout,
|
||||
}}}
|
||||
seenNames := map[string]struct{}{}
|
||||
seenURLs := map[string]struct{}{}
|
||||
for i, c := range cfg.Nodes {
|
||||
base := strings.TrimRight(strings.TrimSpace(c.URL), "/")
|
||||
u, err := url.Parse(base)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
|
||||
return nil, fmt.Errorf("invalid Ollama node URL %q", c.URL)
|
||||
}
|
||||
if _, ok := seenURLs[base]; ok {
|
||||
return nil, fmt.Errorf("duplicate Ollama node URL %q", base)
|
||||
}
|
||||
seenURLs[base] = struct{}{}
|
||||
name := strings.TrimSpace(c.Name)
|
||||
if name == "" {
|
||||
name = u.Hostname()
|
||||
if name == "" {
|
||||
name = fmt.Sprintf("ollama-%d", i+1)
|
||||
}
|
||||
}
|
||||
if _, ok := seenNames[name]; ok {
|
||||
name = fmt.Sprintf("%s-%d", name, i+1)
|
||||
}
|
||||
seenNames[name] = struct{}{}
|
||||
weight := c.Weight
|
||||
if weight <= 0 {
|
||||
weight = 1
|
||||
}
|
||||
if weight > 100 {
|
||||
return nil, fmt.Errorf("Ollama node weight for %q must be between 1 and 100", name)
|
||||
}
|
||||
p.nodes = append(p.nodes, &poolNode{name: name, baseURL: base, weight: weight, sem: make(chan struct{}, cfg.NodeMaxInflight)})
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (p *Pool) Start(ctx context.Context) {
|
||||
if !p.started.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
p.refreshAll(ctx)
|
||||
ticker := time.NewTicker(p.cfg.HealthInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.refreshAll(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (p *Pool) Ping(ctx context.Context) error {
|
||||
p.refreshAll(ctx)
|
||||
for _, n := range p.nodes {
|
||||
if n.isEligible(time.Now(), "") {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return p.unavailableError()
|
||||
}
|
||||
|
||||
func (p *Pool) NodeStatuses() []model.OllamaNodeStatus {
|
||||
out := make([]model.OllamaNodeStatus, 0, len(p.nodes))
|
||||
for _, n := range p.nodes {
|
||||
out = append(out, n.status(p.cfg.NodeMaxInflight))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *Pool) refreshAll(ctx context.Context) {
|
||||
// The first request may arrive while Start is still performing the initial
|
||||
// health scan. Serialize callers instead of returning early, otherwise a
|
||||
// healthy pool can briefly look empty during startup.
|
||||
p.refresh.Lock()
|
||||
defer p.refresh.Unlock()
|
||||
|
||||
type result struct {
|
||||
node *poolNode
|
||||
healthy bool
|
||||
chatDigest string
|
||||
embeddingDigest string
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, len(p.nodes))
|
||||
for _, n := range p.nodes {
|
||||
go func(n *poolNode) {
|
||||
checkCtx := ctx
|
||||
cancel := func() {}
|
||||
timeout := p.cfg.NodeRequestTimeout
|
||||
if timeout <= 0 || timeout > 10*time.Second {
|
||||
timeout = 10 * time.Second
|
||||
}
|
||||
checkCtx, cancel = context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
chatDigest, embeddingDigest, err := p.checkNode(checkCtx, n)
|
||||
ch <- result{node: n, healthy: err == nil, chatDigest: chatDigest, embeddingDigest: embeddingDigest, err: err}
|
||||
}(n)
|
||||
}
|
||||
results := make([]result, 0, len(p.nodes))
|
||||
for range p.nodes {
|
||||
results = append(results, <-ch)
|
||||
}
|
||||
|
||||
chatDigest, chatConflict := commonDigest(results, func(r result) (string, bool) { return r.chatDigest, r.healthy })
|
||||
embedDigest, embedConflict := commonDigest(results, func(r result) (string, bool) { return r.embeddingDigest, r.healthy && r.embeddingDigest != "" })
|
||||
now := time.Now()
|
||||
for _, r := range results {
|
||||
n := r.node
|
||||
n.mu.Lock()
|
||||
previousHealthy := n.healthy
|
||||
previousCompatible := n.compatible
|
||||
previousError := n.lastError
|
||||
n.lastCheck = now
|
||||
n.healthy = r.healthy
|
||||
n.chatDigest = r.chatDigest
|
||||
n.embeddingDigest = r.embeddingDigest
|
||||
n.compatible = r.healthy
|
||||
if r.healthy && p.cfg.RequireSameModelDigest {
|
||||
switch {
|
||||
case chatConflict:
|
||||
n.compatible = false
|
||||
r.err = fmt.Errorf("chat model digests differ inside the pool; node digest=%s", r.chatDigest)
|
||||
case chatDigest != "" && r.chatDigest != chatDigest:
|
||||
n.compatible = false
|
||||
r.err = fmt.Errorf("chat model digest differs from pool digest: node=%s pool=%s", r.chatDigest, chatDigest)
|
||||
}
|
||||
// Even when chat-only nodes are allowed, all nodes that can serve
|
||||
// embeddings must expose the same embedding model digest.
|
||||
switch {
|
||||
case embedConflict:
|
||||
n.compatible = false
|
||||
r.err = fmt.Errorf("embedding model digests differ inside the pool; node digest=%s", r.embeddingDigest)
|
||||
case r.embeddingDigest != "" && embedDigest != "" && r.embeddingDigest != embedDigest:
|
||||
n.compatible = false
|
||||
r.err = fmt.Errorf("embedding model digest differs from pool digest: node=%s pool=%s", r.embeddingDigest, embedDigest)
|
||||
}
|
||||
}
|
||||
if r.err != nil {
|
||||
n.lastError = r.err.Error()
|
||||
} else {
|
||||
n.lastError = ""
|
||||
n.lastSuccess = now
|
||||
n.consecutiveFailures = 0
|
||||
}
|
||||
currentHealthy := n.healthy
|
||||
currentCompatible := n.compatible
|
||||
currentError := n.lastError
|
||||
n.mu.Unlock()
|
||||
if previousHealthy != currentHealthy || previousCompatible != currentCompatible || previousError != currentError {
|
||||
if currentHealthy && currentCompatible {
|
||||
slog.Info("Ollama node available", "node", n.name, "url", n.baseURL, "chat_digest", r.chatDigest, "embedding_digest", r.embeddingDigest)
|
||||
} else {
|
||||
slog.Warn("Ollama node unavailable", "node", n.name, "url", n.baseURL, "healthy", currentHealthy, "compatible", currentCompatible, "error", currentError)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func commonDigest[T any](results []T, getter func(T) (string, bool)) (string, bool) {
|
||||
common := ""
|
||||
for _, r := range results {
|
||||
digest, include := getter(r)
|
||||
digest = strings.TrimSpace(digest)
|
||||
if !include || digest == "" {
|
||||
continue
|
||||
}
|
||||
if common == "" {
|
||||
common = digest
|
||||
continue
|
||||
}
|
||||
if digest != common {
|
||||
return "", true
|
||||
}
|
||||
}
|
||||
return common, false
|
||||
}
|
||||
|
||||
func (p *Pool) checkNode(ctx context.Context, n *poolNode) (string, string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, n.baseURL+"/api/tags", nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
resp, err := p.http.Do(req)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return "", "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
var tags struct {
|
||||
Models []struct {
|
||||
Name string `json:"name"`
|
||||
Model string `json:"model"`
|
||||
Digest string `json:"digest"`
|
||||
} `json:"models"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &tags); err != nil {
|
||||
return "", "", fmt.Errorf("decode /api/tags: %w", err)
|
||||
}
|
||||
find := func(wanted string) string {
|
||||
wanted = strings.TrimSpace(wanted)
|
||||
for _, m := range tags.Models {
|
||||
if modelNameMatches(wanted, m.Name) || modelNameMatches(wanted, m.Model) {
|
||||
return strings.TrimSpace(m.Digest)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
chat := find(p.cfg.Model)
|
||||
if chat == "" {
|
||||
return "", "", fmt.Errorf("model %q is not installed", p.cfg.Model)
|
||||
}
|
||||
embed := ""
|
||||
if strings.TrimSpace(p.cfg.EmbeddingModel) != "" {
|
||||
embed = find(p.cfg.EmbeddingModel)
|
||||
if p.cfg.RequireEmbeddingModel && embed == "" {
|
||||
return "", "", fmt.Errorf("embedding model %q is not installed", p.cfg.EmbeddingModel)
|
||||
}
|
||||
}
|
||||
return chat, embed, nil
|
||||
}
|
||||
|
||||
func modelNameMatches(wanted, got string) bool {
|
||||
wanted = strings.TrimSpace(wanted)
|
||||
got = strings.TrimSpace(got)
|
||||
if wanted == got {
|
||||
return true
|
||||
}
|
||||
if !strings.Contains(wanted, ":") && strings.TrimSuffix(got, ":latest") == wanted {
|
||||
return true
|
||||
}
|
||||
if !strings.Contains(got, ":") && strings.TrimSuffix(wanted, ":latest") == got {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Pool) post(ctx context.Context, path string, payload any, out any) error {
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.anyKnownHealthy() {
|
||||
p.refreshAll(ctx)
|
||||
}
|
||||
attemptLimit := 1
|
||||
if p.cfg.FailoverEnabled {
|
||||
attemptLimit = p.cfg.FailoverAttempts
|
||||
if attemptLimit <= 0 || attemptLimit > len(p.nodes) {
|
||||
attemptLimit = len(p.nodes)
|
||||
}
|
||||
}
|
||||
attempted := map[string]struct{}{}
|
||||
var errs []error
|
||||
for attempt := 1; attempt <= attemptLimit; attempt++ {
|
||||
n, inflight, selectErr := p.selectNode(ctx, attempted, requestStage(ctx))
|
||||
if selectErr != nil {
|
||||
errs = append(errs, selectErr)
|
||||
break
|
||||
}
|
||||
attempted[n.name] = struct{}{}
|
||||
started := time.Now()
|
||||
stage := requestStage(ctx)
|
||||
status, raw, reqErr := p.doPost(ctx, n, path, body)
|
||||
if reqErr == nil && len(raw) > 0 {
|
||||
var envelope struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(raw, &envelope) == nil && strings.TrimSpace(envelope.Error) != "" {
|
||||
reqErr = fmt.Errorf("Ollama %s returned an error from %s: %s", path, n.name, strings.TrimSpace(envelope.Error))
|
||||
}
|
||||
}
|
||||
if reqErr == nil && out != nil {
|
||||
if len(raw) == 0 {
|
||||
reqErr = errors.New("empty Ollama response")
|
||||
} else if decodeErr := json.Unmarshal(raw, out); decodeErr != nil {
|
||||
reqErr = fmt.Errorf("decode Ollama %s response from %s: %w", path, n.name, decodeErr)
|
||||
}
|
||||
}
|
||||
duration := time.Since(started)
|
||||
n.release()
|
||||
retryable := isRetryable(reqErr, status)
|
||||
// A 2xx response that cannot be decoded is safe to fail over because no
|
||||
// application decision was accepted from this node.
|
||||
if reqErr != nil && status/100 == 2 {
|
||||
retryable = true
|
||||
}
|
||||
ok := reqErr == nil
|
||||
n.recordRequest(duration, ok, retryable, reqErr)
|
||||
if !ok && retryable && p.cfg.FailureCooldown > 0 {
|
||||
n.mu.Lock()
|
||||
n.cooldownUntil = time.Now().Add(p.cfg.FailureCooldown)
|
||||
n.mu.Unlock()
|
||||
}
|
||||
meta := requestMeta{}
|
||||
if len(raw) > 0 {
|
||||
_ = json.Unmarshal(raw, &meta)
|
||||
}
|
||||
recordTraceAttempt(ctx, model.OllamaRequestAttempt{
|
||||
Attempt: attempt, Stage: stage, Path: path, NodeName: n.name, NodeURL: n.baseURL,
|
||||
ModelDigest: n.digestForStage(stage), StartedAt: started, DurationMS: duration.Milliseconds(),
|
||||
InflightAtStart: inflight, HTTPStatus: status, Outcome: outcomeText(reqErr), Retryable: retryable,
|
||||
Error: errorText(reqErr), TotalDurationNS: meta.TotalDuration, LoadDurationNS: meta.LoadDuration,
|
||||
PromptEvalCount: meta.PromptEvalCount, PromptEvalDuration: meta.PromptEvalDuration,
|
||||
EvalCount: meta.EvalCount, EvalDuration: meta.EvalDuration,
|
||||
})
|
||||
if reqErr == nil {
|
||||
markTraceSuccess(ctx, n.name, n.baseURL)
|
||||
return nil
|
||||
}
|
||||
errs = append(errs, fmt.Errorf("%s: %w", n.name, reqErr))
|
||||
if retryable && p.cfg.FailoverEnabled && attempt < attemptLimit && ctx.Err() == nil {
|
||||
slog.Warn("Ollama request failed; trying another node", "stage", stage, "path", path, "failed_node", n.name, "attempt", attempt, "max_attempts", attemptLimit, "error", reqErr)
|
||||
}
|
||||
if !retryable || !p.cfg.FailoverEnabled || attempt == attemptLimit || ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(errs) == 0 {
|
||||
return p.unavailableError()
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (n *poolNode) digestForStage(stage string) string {
|
||||
n.mu.RLock()
|
||||
defer n.mu.RUnlock()
|
||||
if stage == "embedding" {
|
||||
return n.embeddingDigest
|
||||
}
|
||||
return n.chatDigest
|
||||
}
|
||||
|
||||
func (p *Pool) doPost(ctx context.Context, n *poolNode, path string, body []byte) (int, []byte, error) {
|
||||
reqCtx := ctx
|
||||
cancel := func() {}
|
||||
if p.cfg.NodeRequestTimeout > 0 {
|
||||
reqCtx, cancel = context.WithTimeout(ctx, p.cfg.NodeRequestTimeout)
|
||||
}
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, n.baseURL+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := p.http.Do(req)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if err != nil {
|
||||
return resp.StatusCode, nil, err
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return resp.StatusCode, raw, fmt.Errorf("Ollama %s failed: HTTP %d: %s", path, resp.StatusCode, strings.TrimSpace(string(raw)))
|
||||
}
|
||||
return resp.StatusCode, raw, nil
|
||||
}
|
||||
|
||||
func (p *Pool) selectNode(ctx context.Context, excluded map[string]struct{}, stage string) (*poolNode, int64, error) {
|
||||
for {
|
||||
nodes := p.orderedCandidates(excluded, stage)
|
||||
if len(nodes) == 0 {
|
||||
return nil, 0, p.unavailableError()
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if inflight, ok := n.acquire(); ok {
|
||||
return n, inflight, nil
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, 0, ctx.Err()
|
||||
case <-time.After(15 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pool) orderedCandidates(excluded map[string]struct{}, stage string) []*poolNode {
|
||||
now := time.Now()
|
||||
out := make([]*poolNode, 0, len(p.nodes))
|
||||
for _, n := range p.nodes {
|
||||
if _, skip := excluded[n.name]; skip {
|
||||
continue
|
||||
}
|
||||
if n.isEligible(now, stage) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
if len(out) <= 1 {
|
||||
return out
|
||||
}
|
||||
switch p.cfg.RoutingMode {
|
||||
case "round_robin":
|
||||
start := int(p.rr.Add(1)-1) % len(out)
|
||||
rotated := append([]*poolNode(nil), out[start:]...)
|
||||
rotated = append(rotated, out[:start]...)
|
||||
return rotated
|
||||
case "fastest_recent":
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
ai, aj := out[i].status(p.cfg.NodeMaxInflight), out[j].status(p.cfg.NodeMaxInflight)
|
||||
// Probe nodes without measurements before permanently preferring a
|
||||
// known node. This prevents new/recovered nodes from starving.
|
||||
if ai.AverageDurationMS == 0 && aj.AverageDurationMS != 0 {
|
||||
return true
|
||||
}
|
||||
if aj.AverageDurationMS == 0 && ai.AverageDurationMS != 0 {
|
||||
return false
|
||||
}
|
||||
if ai.AverageDurationMS == aj.AverageDurationMS {
|
||||
if ai.InFlight == aj.InFlight {
|
||||
return ai.Requests < aj.Requests
|
||||
}
|
||||
return ai.InFlight < aj.InFlight
|
||||
}
|
||||
return ai.AverageDurationMS < aj.AverageDurationMS
|
||||
})
|
||||
case "weighted":
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
// Weighted least-request scheduling also works for serial traffic;
|
||||
// using only current inflight values would permanently select the
|
||||
// highest-weight node whenever requests do not overlap.
|
||||
si := float64(out[i].requests.Load()+uint64(out[i].inflight.Load())+1) / float64(maxInt(out[i].weight, 1))
|
||||
sj := float64(out[j].requests.Load()+uint64(out[j].inflight.Load())+1) / float64(maxInt(out[j].weight, 1))
|
||||
if math.Abs(si-sj) < 1e-9 {
|
||||
return out[i].name < out[j].name
|
||||
}
|
||||
return si < sj
|
||||
})
|
||||
default: // least_inflight
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
ii, ij := out[i].inflight.Load(), out[j].inflight.Load()
|
||||
if ii == ij {
|
||||
ri, rj := out[i].requests.Load(), out[j].requests.Load()
|
||||
if ri != rj {
|
||||
return ri < rj
|
||||
}
|
||||
si, sj := out[i].status(p.cfg.NodeMaxInflight), out[j].status(p.cfg.NodeMaxInflight)
|
||||
if si.AverageDurationMS != sj.AverageDurationMS && si.AverageDurationMS > 0 && sj.AverageDurationMS > 0 {
|
||||
return si.AverageDurationMS < sj.AverageDurationMS
|
||||
}
|
||||
return out[i].name < out[j].name
|
||||
}
|
||||
return ii < ij
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *Pool) anyKnownHealthy() bool {
|
||||
for _, n := range p.nodes {
|
||||
n.mu.RLock()
|
||||
known := !n.lastCheck.IsZero()
|
||||
healthy := n.healthy && n.compatible
|
||||
n.mu.RUnlock()
|
||||
if known && healthy {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Pool) unavailableError() error {
|
||||
statuses := p.NodeStatuses()
|
||||
parts := make([]string, 0, len(statuses))
|
||||
for _, s := range statuses {
|
||||
detail := s.LastError
|
||||
if detail == "" {
|
||||
detail = "not available"
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s: %s", s.Name, detail))
|
||||
}
|
||||
return fmt.Errorf("no compatible Ollama node available (%s)", strings.Join(parts, "; "))
|
||||
}
|
||||
|
||||
func isRetryable(err error, status int) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if status == 0 || status == http.StatusRequestTimeout || status == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
return status == http.StatusBadGateway || status == http.StatusServiceUnavailable || status == http.StatusGatewayTimeout || status >= 500
|
||||
}
|
||||
|
||||
func outcomeText(err error) string {
|
||||
if err == nil {
|
||||
return "success"
|
||||
}
|
||||
return "error"
|
||||
}
|
||||
func errorText(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Trace is a concurrency-safe collector attached to one logical analysis context.
|
||||
type Trace struct {
|
||||
mu sync.Mutex
|
||||
data model.OllamaProviderTrace
|
||||
}
|
||||
|
||||
type traceKey struct{}
|
||||
type stageKey struct{}
|
||||
|
||||
func WithTrace(ctx context.Context, routingMode string) (context.Context, *Trace) {
|
||||
t := &Trace{data: model.OllamaProviderTrace{Provider: providerName, RoutingMode: routingMode}}
|
||||
return context.WithValue(ctx, traceKey{}, t), t
|
||||
}
|
||||
|
||||
func withStage(ctx context.Context, stage string) context.Context {
|
||||
return context.WithValue(ctx, stageKey{}, strings.TrimSpace(stage))
|
||||
}
|
||||
|
||||
func requestStage(ctx context.Context) string {
|
||||
v, _ := ctx.Value(stageKey{}).(string)
|
||||
return v
|
||||
}
|
||||
|
||||
func recordTraceAttempt(ctx context.Context, a model.OllamaRequestAttempt) {
|
||||
t, _ := ctx.Value(traceKey{}).(*Trace)
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
a.Attempt = len(t.data.Attempts) + 1
|
||||
if len(t.data.Attempts) > 0 {
|
||||
previous := t.data.Attempts[len(t.data.Attempts)-1]
|
||||
if previous.NodeName != a.NodeName && previous.Retryable && previous.Outcome == "error" {
|
||||
t.data.FailoverUsed = true
|
||||
}
|
||||
}
|
||||
t.data.Attempts = append(t.data.Attempts, a)
|
||||
t.data.AttemptCount = len(t.data.Attempts)
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
func markTraceSuccess(ctx context.Context, node, nodeURL string) {
|
||||
t, _ := ctx.Value(traceKey{}).(*Trace)
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.data.SelectedNode = node
|
||||
t.data.SelectedURL = nodeURL
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
func (t *Trace) Snapshot() model.OllamaProviderTrace {
|
||||
if t == nil {
|
||||
return model.OllamaProviderTrace{}
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
out := t.data
|
||||
out.Attempts = append([]model.OllamaRequestAttempt(nil), t.data.Attempts...)
|
||||
return out
|
||||
}
|
||||
390
internal/ollama/pool_test.go
Normal file
390
internal/ollama/pool_test.go
Normal file
@@ -0,0 +1,390 @@
|
||||
package ollama
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/example/glpi-ai-agent/internal/model"
|
||||
)
|
||||
|
||||
func tagsResponse(w http.ResponseWriter, chatDigest, embedDigest string) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"models": []map[string]any{
|
||||
{"name": "m:latest", "model": "m:latest", "digest": chatDigest},
|
||||
{"name": "e:latest", "model": "e:latest", "digest": embedDigest},
|
||||
}})
|
||||
}
|
||||
|
||||
func categoryResponse(w http.ResponseWriter, id int64) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"message": map[string]any{"content": `{"category":{"id":` + jsonNumber(id) + `,"confidence":0.9},"reason":"ok"}`}})
|
||||
}
|
||||
|
||||
func jsonNumber(v int64) string {
|
||||
b, _ := json.Marshal(v)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func newTestPool(t *testing.T, nodes []NodeConfig, routing string) *Client {
|
||||
t.Helper()
|
||||
c, err := NewPool(PoolConfig{
|
||||
Nodes: nodes, RoutingMode: routing, NodeMaxInflight: 1,
|
||||
HealthInterval: time.Minute, FailureCooldown: time.Second,
|
||||
NodeRequestTimeout: 2 * time.Second, FailoverEnabled: true, FailoverAttempts: len(nodes),
|
||||
RequireSameModelDigest: true, RequireEmbeddingModel: true, Model: "m", EmbeddingModel: "e",
|
||||
}, "m", "e", "de-DE", "formal", 128, time.Minute, false, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Ping(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestPoolFailsOverAndRecordsTrace(t *testing.T) {
|
||||
var badCalls atomic.Int64
|
||||
bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
badCalls.Add(1)
|
||||
http.Error(w, "temporarily unavailable", http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer bad.Close()
|
||||
var goodCalls atomic.Int64
|
||||
good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
goodCalls.Add(1)
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
defer good.Close()
|
||||
|
||||
c := newTestPool(t, []NodeConfig{{Name: "a-bad", URL: bad.URL}, {Name: "b-good", URL: good.URL}}, "least_inflight")
|
||||
ctx, trace := WithTrace(context.Background(), c.RoutingMode())
|
||||
d, err := c.AnalyseCategory(ctx, model.Ticket{ID: 1}, []model.Category{{ID: 1}}, nil, model.ContextSnapshot{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.Category.ID != 1 || badCalls.Load() != 1 || goodCalls.Load() != 1 {
|
||||
t.Fatalf("decision=%+v bad=%d good=%d", d, badCalls.Load(), goodCalls.Load())
|
||||
}
|
||||
got := trace.Snapshot()
|
||||
if !got.FailoverUsed || got.SelectedNode != "b-good" || len(got.Attempts) != 2 {
|
||||
t.Fatalf("unexpected trace: %+v", got)
|
||||
}
|
||||
if got.Attempts[0].HTTPStatus != http.StatusServiceUnavailable || !got.Attempts[0].Retryable {
|
||||
t.Fatalf("unexpected first attempt: %+v", got.Attempts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolLeastInflightUsesFreeNode(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var once sync.Once
|
||||
var callsA, callsB atomic.Int64
|
||||
a := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
callsA.Add(1)
|
||||
once.Do(func() { close(started) })
|
||||
<-release
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
defer a.Close()
|
||||
b := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
callsB.Add(1)
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
defer b.Close()
|
||||
|
||||
c := newTestPool(t, []NodeConfig{{Name: "a", URL: a.URL}, {Name: "b", URL: b.URL}}, "least_inflight")
|
||||
errCh := make(chan error, 2)
|
||||
go func() {
|
||||
_, err := c.AnalyseCategory(context.Background(), model.Ticket{ID: 1}, []model.Category{{ID: 1}}, nil, model.ContextSnapshot{})
|
||||
errCh <- err
|
||||
}()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first node did not start")
|
||||
}
|
||||
go func() {
|
||||
_, err := c.AnalyseCategory(context.Background(), model.Ticket{ID: 2}, []model.Category{{ID: 1}}, nil, model.ContextSnapshot{})
|
||||
errCh <- err
|
||||
}()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for callsB.Load() == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
close(release)
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := <-errCh; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if callsA.Load() != 1 || callsB.Load() != 1 {
|
||||
t.Fatalf("least-inflight distribution a=%d b=%d", callsA.Load(), callsB.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolRejectsMismatchedDigest(t *testing.T) {
|
||||
server := func(chatDigest string) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
tagsResponse(w, chatDigest, "embed-digest")
|
||||
}))
|
||||
}
|
||||
a := server("aaa")
|
||||
defer a.Close()
|
||||
b := server("bbb")
|
||||
defer b.Close()
|
||||
c, err := NewPool(PoolConfig{
|
||||
Nodes: []NodeConfig{{Name: "a", URL: a.URL}, {Name: "b", URL: b.URL}}, RoutingMode: "least_inflight", NodeMaxInflight: 1,
|
||||
HealthInterval: time.Minute, NodeRequestTimeout: time.Second, FailoverEnabled: true, FailoverAttempts: 2,
|
||||
RequireSameModelDigest: true, RequireEmbeddingModel: true, Model: "m", EmbeddingModel: "e",
|
||||
}, "m", "e", "de-DE", "formal", 128, time.Minute, false, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Ping(context.Background()); err == nil {
|
||||
t.Fatal("expected pool with divergent model digests to fail closed")
|
||||
}
|
||||
statuses := c.NodeStatuses()
|
||||
for _, status := range statuses {
|
||||
if status.Compatible {
|
||||
t.Fatalf("mismatched node must be incompatible: %+v", statuses)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolFirstRequestWaitsForInitialHealthScan(t *testing.T) {
|
||||
healthStarted := make(chan struct{})
|
||||
releaseHealth := make(chan struct{})
|
||||
var healthCalls atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
if healthCalls.Add(1) == 1 {
|
||||
close(healthStarted)
|
||||
<-releaseHealth
|
||||
}
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c, err := NewPool(PoolConfig{
|
||||
Nodes: []NodeConfig{{Name: "node-1", URL: server.URL}}, RoutingMode: "least_inflight", NodeMaxInflight: 1,
|
||||
HealthInterval: time.Minute, NodeRequestTimeout: 2 * time.Second, FailoverEnabled: false, FailoverAttempts: 1,
|
||||
RequireSameModelDigest: true, RequireEmbeddingModel: true, Model: "m", EmbeddingModel: "e",
|
||||
}, "m", "e", "de-DE", "formal", 128, time.Minute, false, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
c.Start(ctx)
|
||||
select {
|
||||
case <-healthStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("initial health scan did not start")
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := c.AnalyseCategory(context.Background(), model.Ticket{ID: 1}, []model.Category{{ID: 1}}, nil, model.ContextSnapshot{})
|
||||
errCh <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-errCh:
|
||||
t.Fatalf("request returned before health scan completed: %v", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
close(releaseHealth)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("request did not continue after health scan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolFailsOverOnInvalidOuterJSON(t *testing.T) {
|
||||
bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"message":`))
|
||||
}))
|
||||
defer bad.Close()
|
||||
good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
defer good.Close()
|
||||
|
||||
c := newTestPool(t, []NodeConfig{{Name: "a-bad-json", URL: bad.URL}, {Name: "b-good", URL: good.URL}}, "least_inflight")
|
||||
ctx, trace := WithTrace(context.Background(), c.RoutingMode())
|
||||
if _, err := c.AnalyseCategory(ctx, model.Ticket{ID: 1}, []model.Category{{ID: 1}}, nil, model.ContextSnapshot{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := trace.Snapshot()
|
||||
if !got.FailoverUsed || got.SelectedNode != "b-good" || len(got.Attempts) != 2 {
|
||||
t.Fatalf("unexpected trace: %+v", got)
|
||||
}
|
||||
if got.Attempts[0].Outcome != "error" || !got.Attempts[0].Retryable || got.Attempts[0].Error == "" {
|
||||
t.Fatalf("invalid JSON must be recorded as retryable error: %+v", got.Attempts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolAllowsChatOnlyNodeButRoutesEmbeddingsToCapableNode(t *testing.T) {
|
||||
var chatOnlyEmbedCalls atomic.Int64
|
||||
chatOnly := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"models": []map[string]any{{"name": "m:latest", "model": "m:latest", "digest": "chat-digest"}}})
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/api/embed" {
|
||||
chatOnlyEmbedCalls.Add(1)
|
||||
}
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
defer chatOnly.Close()
|
||||
var embeddingCalls atomic.Int64
|
||||
embeddingNode := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/api/embed" {
|
||||
embeddingCalls.Add(1)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float64{{0.1, 0.2}}})
|
||||
return
|
||||
}
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
defer embeddingNode.Close()
|
||||
|
||||
c, err := NewPool(PoolConfig{
|
||||
Nodes: []NodeConfig{{Name: "chat-only", URL: chatOnly.URL}, {Name: "embedding", URL: embeddingNode.URL}},
|
||||
RoutingMode: "least_inflight", NodeMaxInflight: 1, HealthInterval: time.Minute,
|
||||
NodeRequestTimeout: time.Second, FailoverEnabled: true, FailoverAttempts: 2,
|
||||
RequireSameModelDigest: true, RequireEmbeddingModel: false, Model: "m", EmbeddingModel: "e",
|
||||
}, "m", "e", "de-DE", "formal", 128, time.Minute, false, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Ping(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vectors, err := c.Embed(context.Background(), []string{"test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(vectors) != 1 || embeddingCalls.Load() != 1 || chatOnlyEmbedCalls.Load() != 0 {
|
||||
t.Fatalf("vectors=%v embedding_calls=%d chat_only_embed_calls=%d", vectors, embeddingCalls.Load(), chatOnlyEmbedCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolLeastInflightBalancesSerialRequests(t *testing.T) {
|
||||
var callsA, callsB atomic.Int64
|
||||
server := func(calls *atomic.Int64) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
calls.Add(1)
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
}
|
||||
a := server(&callsA)
|
||||
defer a.Close()
|
||||
b := server(&callsB)
|
||||
defer b.Close()
|
||||
c := newTestPool(t, []NodeConfig{{Name: "a", URL: a.URL}, {Name: "b", URL: b.URL}}, "least_inflight")
|
||||
for i := 0; i < 4; i++ {
|
||||
if _, err := c.AnalyseCategory(context.Background(), model.Ticket{ID: int64(i + 1)}, []model.Category{{ID: 1}}, nil, model.ContextSnapshot{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if callsA.Load() != 2 || callsB.Load() != 2 {
|
||||
t.Fatalf("serial least-inflight distribution a=%d b=%d", callsA.Load(), callsB.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolWeightedBalancesSerialRequestsByWeight(t *testing.T) {
|
||||
var callsA, callsB atomic.Int64
|
||||
server := func(calls *atomic.Int64) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
calls.Add(1)
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
}
|
||||
a := server(&callsA)
|
||||
defer a.Close()
|
||||
b := server(&callsB)
|
||||
defer b.Close()
|
||||
c := newTestPool(t, []NodeConfig{{Name: "a", URL: a.URL, Weight: 1}, {Name: "b", URL: b.URL, Weight: 3}}, "weighted")
|
||||
for i := 0; i < 8; i++ {
|
||||
if _, err := c.AnalyseCategory(context.Background(), model.Ticket{ID: int64(i + 1)}, []model.Category{{ID: 1}}, nil, model.ContextSnapshot{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if callsA.Load() == 0 || callsB.Load() <= callsA.Load() {
|
||||
t.Fatalf("weighted distribution must use both nodes and prefer weight 3: a=%d b=%d", callsA.Load(), callsB.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolFastestRecentProbesUnmeasuredNodes(t *testing.T) {
|
||||
var callsA, callsB atomic.Int64
|
||||
server := func(calls *atomic.Int64) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
tagsResponse(w, "chat-digest", "embed-digest")
|
||||
return
|
||||
}
|
||||
calls.Add(1)
|
||||
categoryResponse(w, 1)
|
||||
}))
|
||||
}
|
||||
a := server(&callsA)
|
||||
defer a.Close()
|
||||
b := server(&callsB)
|
||||
defer b.Close()
|
||||
c := newTestPool(t, []NodeConfig{{Name: "a", URL: a.URL}, {Name: "b", URL: b.URL}}, "fastest_recent")
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := c.AnalyseCategory(context.Background(), model.Ticket{ID: int64(i + 1)}, []model.Category{{ID: 1}}, nil, model.ContextSnapshot{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if callsA.Load() != 1 || callsB.Load() != 1 {
|
||||
t.Fatalf("fastest_recent must measure both nodes before preferring one: a=%d b=%d", callsA.Load(), callsB.Load())
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,10 @@ type DiagnosticsManager interface {
|
||||
DiagnoseRun(context.Context, string) (model.RunRecord, error)
|
||||
DiagnoseKnowledge(context.Context, string, string, string) (model.KnowledgeDiagnostic, error)
|
||||
}
|
||||
type OllamaNodeProvider interface {
|
||||
NodeStatuses() []model.OllamaNodeStatus
|
||||
RoutingMode() string
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
cfg config.Config
|
||||
@@ -63,15 +67,19 @@ type Server struct {
|
||||
knowledge KnowledgeManager
|
||||
feedback FeedbackManager
|
||||
diagnostics DiagnosticsManager
|
||||
ollamaNodes OllamaNodeProvider
|
||||
tpl *template.Template
|
||||
}
|
||||
|
||||
func New(cfg config.Config, m *metrics.Metrics, s *state.Store, q *queue.Queue, k KnowledgeManager, f FeedbackManager) (*Server, error) {
|
||||
func New(cfg config.Config, m *metrics.Metrics, s *state.Store, q *queue.Queue, k KnowledgeManager, f FeedbackManager, providers ...OllamaNodeProvider) (*Server, error) {
|
||||
t, err := template.ParseFS(files, "templates/*.html")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srv := &Server{cfg: cfg, metrics: m, state: s, q: q, knowledge: k, feedback: f, tpl: t}
|
||||
if len(providers) > 0 {
|
||||
srv.ollamaNodes = providers[0]
|
||||
}
|
||||
if d, ok := f.(DiagnosticsManager); ok {
|
||||
srv.diagnostics = d
|
||||
}
|
||||
@@ -123,6 +131,38 @@ func (s *Server) ready(w http.ResponseWriter, r *http.Request) {
|
||||
func (s *Server) prom(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
|
||||
s.metrics.WritePrometheus(w)
|
||||
if s.ollamaNodes == nil {
|
||||
return
|
||||
}
|
||||
io.WriteString(w, "# TYPE glpi_agent_ollama_node_healthy gauge\n")
|
||||
io.WriteString(w, "# TYPE glpi_agent_ollama_node_available gauge\n")
|
||||
io.WriteString(w, "# TYPE glpi_agent_ollama_node_inflight gauge\n")
|
||||
io.WriteString(w, "# TYPE glpi_agent_ollama_node_requests_total counter\n")
|
||||
io.WriteString(w, "# TYPE glpi_agent_ollama_node_failures_total counter\n")
|
||||
io.WriteString(w, "# TYPE glpi_agent_ollama_node_average_duration_ms gauge\n")
|
||||
for _, node := range s.ollamaNodes.NodeStatuses() {
|
||||
name := prometheusLabel(node.Name)
|
||||
fmt.Fprintf(w, "glpi_agent_ollama_node_healthy{node=\"%s\"} %d\n", name, boolMetric(node.Healthy && node.Compatible))
|
||||
fmt.Fprintf(w, "glpi_agent_ollama_node_available{node=\"%s\"} %d\n", name, boolMetric(node.Available))
|
||||
fmt.Fprintf(w, "glpi_agent_ollama_node_inflight{node=\"%s\"} %d\n", name, node.InFlight)
|
||||
fmt.Fprintf(w, "glpi_agent_ollama_node_requests_total{node=\"%s\"} %d\n", name, node.Requests)
|
||||
fmt.Fprintf(w, "glpi_agent_ollama_node_failures_total{node=\"%s\"} %d\n", name, node.Failures)
|
||||
fmt.Fprintf(w, "glpi_agent_ollama_node_average_duration_ms{node=\"%s\"} %.3f\n", name, node.AverageDurationMS)
|
||||
}
|
||||
}
|
||||
|
||||
func boolMetric(v bool) int {
|
||||
if v {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func prometheusLabel(v string) string {
|
||||
v = strings.ReplaceAll(v, `\`, `\\`)
|
||||
v = strings.ReplaceAll(v, `"`, `\"`)
|
||||
v = strings.ReplaceAll(v, "\n", `\n`)
|
||||
return v
|
||||
}
|
||||
func (s *Server) dashboard(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/" {
|
||||
@@ -334,6 +374,22 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request) {
|
||||
kbOK, kbDocs, kbLastSync, kbLastErr := s.metrics.GLPIKBStatus()
|
||||
loadStats := s.knowledge.LoadStats()
|
||||
initStatus := s.knowledge.InitStatus()
|
||||
ollamaNodes := []model.OllamaNodeStatus{}
|
||||
ollamaRoutingMode := s.cfg.OllamaRoutingMode
|
||||
if s.ollamaNodes != nil {
|
||||
ollamaNodes = s.ollamaNodes.NodeStatuses()
|
||||
ollamaRoutingMode = s.ollamaNodes.RoutingMode()
|
||||
}
|
||||
ollamaHealthyNodes := 0
|
||||
ollamaAvailableNodes := 0
|
||||
for _, node := range ollamaNodes {
|
||||
if node.Healthy && node.Compatible {
|
||||
ollamaHealthyNodes++
|
||||
}
|
||||
if node.Available {
|
||||
ollamaAvailableNodes++
|
||||
}
|
||||
}
|
||||
respondJSON(w, map[string]any{
|
||||
"uptime_seconds": int(time.Since(s.metrics.Started).Seconds()), "dry_run": s.cfg.DryRun, "auto_reply": s.cfg.AutoReply, "auto_category": s.cfg.AutoCategory,
|
||||
"priority_enabled": s.cfg.PriorityEnabled, "auto_priority": s.cfg.AutoPriority, "priority_confidence": s.cfg.PriorityConfidence, "priority_analysis_timeout": s.cfg.PriorityAnalysisTimeout.String(), "priority_max_increase": s.cfg.PriorityMaxIncrease, "priority_allowed_reason_codes": s.cfg.PriorityAllowedReasonCodes,
|
||||
@@ -354,6 +410,7 @@ func (s *Server) status(w http.ResponseWriter, r *http.Request) {
|
||||
"context_status_reply_enabled": s.cfg.ContextStatusReplyEnabled, "context_status_reply_min_relevance": s.cfg.ContextStatusReplyMinRelevance, "context_status_reply_min_ai_confidence": s.cfg.ContextStatusReplyMinAIConfidence, "context_status_reply_min_final_score": s.cfg.ContextStatusReplyMinFinalScore, "context_incident_reply_text_configured": strings.TrimSpace(s.cfg.ContextIncidentReplyText) != "", "context_maintenance_reply_text_configured": strings.TrimSpace(s.cfg.ContextMaintenanceReplyText) != "",
|
||||
"workers": s.cfg.Workers, "queue_size": s.cfg.QueueSize, "glpi_api_version": s.cfg.GLPIAPIVersion, "glpi_poll_interval": s.cfg.GLPIPollInterval.String(), "glpi_poll_limit": s.cfg.GLPIPollLimit, "glpi_allowed_status_ids": s.cfg.GLPIAllowedStatusIDs, "glpi_ticket_filter_configured": strings.TrimSpace(s.cfg.GLPITicketFilter) != "", "glpi_timeout": s.cfg.GLPITimeout.String(),
|
||||
"ollama_model": s.cfg.OllamaModel, "ollama_embedding_model": s.cfg.OllamaEmbeddingModel, "ollama_timeout": s.cfg.OllamaTimeout.String(), "ollama_num_predict": s.cfg.OllamaNumPredict, "ollama_keep_alive": s.cfg.OllamaKeepAlive.String(), "ollama_think": s.cfg.OllamaThink, "ollama_max_concurrent": s.cfg.OllamaMaxConcurrent, "ollama_json_retries": s.cfg.OllamaJSONRetries,
|
||||
"ollama_nodes": ollamaNodes, "ollama_node_count": len(ollamaNodes), "ollama_healthy_nodes": ollamaHealthyNodes, "ollama_available_nodes": ollamaAvailableNodes, "ollama_routing_mode": ollamaRoutingMode, "ollama_node_max_inflight": s.cfg.OllamaNodeMaxInflight, "ollama_node_health_interval": s.cfg.OllamaNodeHealthInterval.String(), "ollama_node_failure_cooldown": s.cfg.OllamaNodeFailureCooldown.String(), "ollama_node_request_timeout": s.cfg.OllamaNodeRequestTimeout.String(), "ollama_failover_enabled": s.cfg.OllamaFailoverEnabled, "ollama_failover_attempts": s.cfg.OllamaFailoverAttempts, "ollama_require_same_model_digest": s.cfg.OllamaRequireSameDigest, "ollama_require_embedding_model": s.cfg.OllamaRequireEmbeddingModel,
|
||||
"rag_enabled": s.cfg.RAGEnabled, "knowledge_top_k": s.cfg.KnowledgeTopK, "knowledge_audit_top_k": s.cfg.KnowledgeAuditTopK, "knowledge_candidate_max_gap": s.cfg.KnowledgeCandidateMaxGap, "category_prompt_limit": s.cfg.CategoryPromptLimit, "knowledge_max_query_chunks": s.cfg.KnowledgeMaxQueryChunks,
|
||||
"glpi_kb_path": s.cfg.GLPIKBPath, "glpi_kb_filter_configured": strings.TrimSpace(s.cfg.GLPIKBFilter) != "", "glpi_kb_limit": s.cfg.GLPIKBLimit, "glpi_kb_auto_reply": s.cfg.GLPIKBAutoReply, "glpi_kb_auto_reply_category_ids": s.cfg.GLPIKBAutoReplyCategoryIDs,
|
||||
"learning_max_examples": s.cfg.LearningMaxExamples, "learning_examples_per_category": s.cfg.LearningExamplesPerCategory,
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -314,3 +315,31 @@ func TestManualReprocessQueuesForcedWorkItem(t *testing.T) {
|
||||
t.Fatalf("unexpected response: %#v", body)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeOllamaNodeProvider struct {
|
||||
statuses []model.OllamaNodeStatus
|
||||
}
|
||||
|
||||
func (f fakeOllamaNodeProvider) NodeStatuses() []model.OllamaNodeStatus {
|
||||
return append([]model.OllamaNodeStatus(nil), f.statuses...)
|
||||
}
|
||||
func (f fakeOllamaNodeProvider) RoutingMode() string { return "least_inflight" }
|
||||
|
||||
func TestPrometheusOllamaNodeMetricTypesAreEmittedOnce(t *testing.T) {
|
||||
s := &Server{metrics: metrics.New(), ollamaNodes: fakeOllamaNodeProvider{statuses: []model.OllamaNodeStatus{
|
||||
{Name: "node-a", Healthy: true, Compatible: true, Available: true},
|
||||
{Name: "node-b", Healthy: true, Compatible: true, Available: true},
|
||||
}}}
|
||||
rr := httptest.NewRecorder()
|
||||
s.prom(rr, httptest.NewRequest(http.MethodGet, "/metrics", nil))
|
||||
body := rr.Body.String()
|
||||
for _, metric := range []string{
|
||||
"glpi_agent_ollama_node_healthy", "glpi_agent_ollama_node_available",
|
||||
"glpi_agent_ollama_node_inflight", "glpi_agent_ollama_node_requests_total",
|
||||
"glpi_agent_ollama_node_failures_total", "glpi_agent_ollama_node_average_duration_ms",
|
||||
} {
|
||||
if got := strings.Count(body, "# TYPE "+metric+" "); got != 1 {
|
||||
t.Fatalf("TYPE for %s emitted %d times:\n%s", metric, got, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -59,7 +59,7 @@ function analysisLabel(type){return({category:'Kategorie',priority:'Priorität',
|
||||
function analysisKind(a){return a.outcome==='error'?'bad':a.outcome==='skipped'?'warn':a.action?.executed?'good':'info'}
|
||||
function renderAnalyses(r){const rows=r.analyses||[];$('#analysisRuns').innerHTML=rows.length?rows.map(a=>`<div class="analysis-card" data-analysis="${esc(a.analysis_id)}"><div class="chips">${statusChip(analysisLabel(a.analysis_type),'info')}${statusChip(a.outcome||'–',analysisKind(a))}${a.action?.executed?statusChip('Aktion ausgeführt','good'):a.action?.proposed?statusChip(a.action.dry_run?'Aktion simuliert':'Aktion vorgeschlagen','warn'):''}</div><div class="name">${esc(a.analysis_id)}</div><div class="meta">${esc(a.prompt_version||'ohne Prompt-Version')} · ${Number(a.duration_ms||0).toLocaleString('de-DE')} ms<br>${esc(a.explanation||a.error||'Keine Begründung gespeichert.')}</div></div>`).join(''):'<div class="empty">Historischer Lauf ohne generische AnalysisRun-Datensätze.</div>';$('#analysisDetail').classList.add('hidden');document.querySelectorAll('[data-analysis]').forEach(x=>x.onclick=()=>showAnalysis(x.dataset.analysis))}
|
||||
function actionStepsHTML(action){const steps=action?.steps||[];if(!steps.length)return '<div class="empty">Keine einzelnen Aktionsschritte.</div>';return steps.map((x,i)=>`<div class="rule ${x.error&&x.executed?'warn':x.error?'bad':x.executed?'good':x.proposed?'warn':'info'}"><div class="rule-icon">${x.error?'×':x.executed?'✓':x.dry_run?'◌':'•'}</div><div class="rule-body"><div class="rule-label">${esc(x.step||`Schritt ${i+1}`)}${x.target?` · ${esc(x.target)}`:''}</div><div class="rule-detail">${esc(x.result||'–')}${x.error?` · ${esc(x.error)}`:''}</div></div><div class="rule-values"><strong>${x.executed?'ausgeführt':x.dry_run&&x.proposed?'simuliert':x.proposed?'vorgeschlagen':'nicht ausgeführt'}</strong><span>${esc(x.before||'–')} → ${esc(x.after||'–')}</span></div></div>`).join('')}
|
||||
async function showAnalysis(id){const box=$('#analysisDetail');box.classList.remove('hidden');box.innerHTML='<div class="empty">Lade Analyselauf …</div>';try{const a=await api(`/api/diagnostics/analysis/${encodeURIComponent(id)}`);const action=a.action||{};box.innerHTML=`<div style="display:flex;justify-content:space-between;gap:12px;align-items:flex-start"><div><div class="eyebrow">${esc(analysisLabel(a.analysis_type))}</div><h2 style="margin:4px 0">${esc(a.analysis_id)}</h2><div class="muted">Parent ${esc(a.parent_run_id)} · Trigger ${esc(a.trigger)} · ${esc(a.model||'deterministisch')} · ${esc(a.prompt_version||'–')}</div></div><div class="chips">${statusChip(a.outcome||'–',analysisKind(a))}${statusChip(`Confidence ${pct(a.confidence)}`,'info')}${action.type?statusChip(action.type,action.executed?'good':action.proposed?'warn':'info'):''}</div></div><div class="grid2" style="margin-top:14px"><div><div class="label">Begründung / Grundcodes</div><div class="raw">${esc(a.explanation||'–')}\n\n${esc((a.reason_codes||[]).join(', ')||'keine')}</div></div><div><div class="label">Aktionsplan</div>${actionStepsHTML(action)}<div class="raw" style="margin-top:10px">${esc(JSON.stringify(action,null,2))}</div></div></div><div style="margin-top:14px"><div class="panel-title">Policy-Prüfungen</div>${(a.checks||[]).length?(a.checks||[]).map(ruleHTML).join(''):'<div class="empty">Keine Regelchecks.</div>'}</div><div class="grid2" style="margin-top:14px"><div><div class="label">Historischer Input-Snapshot · SHA-256 ${esc(a.input_hash||'–')}</div><div class="raw">${esc(JSON.stringify(a.input_snapshot??{},null,2))}</div></div><div><div class="label">Strukturierte Entscheidung</div><div class="raw">${esc(JSON.stringify(a.decision??{},null,2))}</div></div></div>`}catch(e){box.innerHTML=`<div class="warning">${esc(e.message)}</div>`}}
|
||||
async function showAnalysis(id){const box=$('#analysisDetail');box.classList.remove('hidden');box.innerHTML='<div class="empty">Lade Analyselauf …</div>';try{const a=await api(`/api/diagnostics/analysis/${encodeURIComponent(id)}`);const action=a.action||{};box.innerHTML=`<div style="display:flex;justify-content:space-between;gap:12px;align-items:flex-start"><div><div class="eyebrow">${esc(analysisLabel(a.analysis_type))}</div><h2 style="margin:4px 0">${esc(a.analysis_id)}</h2><div class="muted">Parent ${esc(a.parent_run_id)} · Trigger ${esc(a.trigger)} · ${esc(a.model||'deterministisch')} · ${esc(a.prompt_version||'–')}</div></div><div class="chips">${statusChip(a.outcome||'–',analysisKind(a))}${statusChip(`Confidence ${pct(a.confidence)}`,'info')}${action.type?statusChip(action.type,action.executed?'good':action.proposed?'warn':'info'):''}</div></div><div class="grid2" style="margin-top:14px"><div><div class="label">Begründung / Grundcodes</div><div class="raw">${esc(a.explanation||'–')}\n\n${esc((a.reason_codes||[]).join(', ')||'keine')}</div></div><div><div class="label">Aktionsplan</div>${actionStepsHTML(action)}<div class="raw" style="margin-top:10px">${esc(JSON.stringify(action,null,2))}</div></div></div><div style="margin-top:14px"><div class="panel-title">Policy-Prüfungen</div>${(a.checks||[]).length?(a.checks||[]).map(ruleHTML).join(''):'<div class="empty">Keine Regelchecks.</div>'}</div><div style="margin-top:14px"><div class="panel-title">Ollama-Routing und Failover</div><div class="raw">${esc(JSON.stringify(a.provider??{provider:'keine Provider-Diagnose gespeichert'},null,2))}</div></div><div class="grid2" style="margin-top:14px"><div><div class="label">Historischer Input-Snapshot · SHA-256 ${esc(a.input_hash||'–')}</div><div class="raw">${esc(JSON.stringify(a.input_snapshot??{},null,2))}</div></div><div><div class="label">Strukturierte Entscheidung</div><div class="raw">${esc(JSON.stringify(a.decision??{},null,2))}</div></div></div>`}catch(e){box.innerHTML=`<div class="warning">${esc(e.message)}</div>`}}
|
||||
function renderRun(){
|
||||
const r=current,legacy=!r.category_analysis_executed&&!r.reply_analysis_executed&&!r.category_knowledge_candidates;
|
||||
$('#ticketTitle').textContent=`#${r.ticket_id} ${r.ticket_name||''}`;$('#runMeta').textContent=`Run ${r.run_id} · Trigger ${r.trigger||'legacy'} · ${date(r.finished_at)} · ${r.outcome}${r.caused_by_run_id?` · Ursache ${r.caused_by_run_id}`:''}`;
|
||||
|
||||
Reference in New Issue
Block a user