sendnrw/og
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

-

Browse Source
This commit is contained in:
jbergner
2026-09-11 06:14:38 +02:00
parent bf64652300
commit e581949946
161 changed files with 31126 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.production.example
+5
View File
@@ -0,0 +1,5 @@
# Copy to .env or export these values before docker compose commands.
# The safe default publishes the gateway only on the Docker host loopback.
GATEWAY_CONFIG=./gateway-config.json
GATEWAY_PUBLISH_ADDRESS=127.0.0.1
GATEWAY_PUBLISH_PORT=9080
.gitignore
+5
View File
@@ -0,0 +1,5 @@
/data/
/dist/*
.env
*.log
.DS_Store
Dockerfile
+18
View File
@@ -0,0 +1,18 @@
FROM golang:1.23-alpine AS build
WORKDIR /src
COPY . .
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/ollama-gateway ./cmd/ollama-gateway \
&& mkdir -p /out/data \
&& touch /out/data/.keep
FROM scratch
COPY --from=build /out/ollama-gateway /ollama-gateway
# Pre-create a writable state directory for the unprivileged runtime user.
# Empty named volumes mounted at /data inherit this directory's ownership.
COPY --from=build --chown=65532:65532 /out/data /data
USER 65532:65532
ENTRYPOINT ["/ollama-gateway"]
CMD ["-config", "/etc/ollama-gateway/config.json"]
# The binary contains its own HTTP probe so the scratch runtime needs no shell/curl.
HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 CMD ["/ollama-gateway", "-probe", "http://127.0.0.1:8080/healthz", "-probe-timeout", "2s"]
Makefile
+50
View File
@@ -0,0 +1,50 @@
BINARY=ollama-gateway
.PHONY: test build build-mac build-worker-telemetry worker-telemetry bench mock-ollama ha-report ha-snapshot ha-sampler ha-readiness fmt vet clean
test:
go test ./...
fmt:
gofmt -w $$(find . -name '*.go' -type f)
vet:
go vet ./...
build:
CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o dist/$(BINARY) ./cmd/ollama-gateway
build-mac:
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/$(BINARY)-darwin-arm64 ./cmd/ollama-gateway
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-bench-darwin-arm64 ./cmd/bench
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-ha-report-darwin-arm64 ./cmd/ha-report
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-ha-snapshot-darwin-arm64 ./cmd/ha-snapshot
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-ha-sampler-darwin-arm64 ./cmd/ha-sampler
build-worker-telemetry:
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-worker-telemetry-linux-amd64 ./cmd/worker-telemetry
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-worker-telemetry-linux-arm64 ./cmd/worker-telemetry
worker-telemetry:
go run ./cmd/worker-telemetry -once
bench:
go run ./cmd/bench -base-url http://127.0.0.1:8080 -concurrency 4 -requests 20
mock-ollama:
go run ./cmd/mock-ollama -listen 127.0.0.1:11435
ha-report:
go run ./cmd/ha-report -input "$${OUT_DIR:?set OUT_DIR to a completed readiness directory}"
ha-snapshot:
go run ./cmd/ha-snapshot -pid "$${GATEWAY_PID:?set GATEWAY_PID to the gateway process PID}"
ha-sampler:
go run ./cmd/ha-sampler -pid "$${GATEWAY_PID:?set GATEWAY_PID to the gateway process PID}" -stop-file "$${STOP_FILE:?set STOP_FILE to a path that will be created to stop sampling}"
ha-readiness:
./scripts/ha-readiness.sh
clean:
rm -f dist/*
README.md
+603 -1
View File
@@ -1,2 +1,604 @@
# og
# Ollama Fair Gateway
A high-performance Go gateway in front of one or more Ollama workers. The gateway exposes the native Ollama API and Ollama's OpenAI-compatible API while adding authentication, hierarchical fair scheduling, compute-credit quotas, usage metering, model-aware routing, optional durable batch execution and an embedded operations UI.
The inference coordination engine remains **100% in-memory**, while restart-worthy control-plane state is persisted to local files under `storage.data_dir`. There is no Redis client, SQL/database server, distributed lease system or external message bus. The project uses only the Go standard library and builds as a static binary with `CGO_ENABLED=0`.
## Architecture
```text
Clients / Applications
|
v
+-------------------------------+
| Ollama Fair Gateway |
| |
| OIDC / API Keys / IP bypass |
| Admission + compute credits |
| Tenant -> Actor WFQ |
| In-memory worker routing |
| Live request event bus |
| Usage aggregation |
| Durable local state |
| Embedded Web UI |
+---------------+---------------+
|
+--------+---------+
| | |
v v v
Ollama A Ollama B Ollama C
```
All fairness decisions are therefore made from one authoritative in-process state. Multiple Ollama workers are supported. If strict global fairness is required, all clients should enter through this one gateway process.
## Main features
- Native Ollama `/api/*` passthrough, including streaming and large management/blob requests.
- OpenAI-compatible `/v1/*` passthrough via Ollama's own compatibility layer.
- OIDC/JWT validation using discovery and JWKS.
- Browser OIDC Authorization Code + PKCE.
- Static configuration API keys plus persistent UI-created API-key creation/revocation from the admin UI; only SHA-256 key hashes are stored.
- CIDR-based authentication bypass for trusted IPs.
- Safe `X-Forwarded-For` handling behind explicitly trusted proxies.
- Two-level hierarchical weighted fair queueing: tenant fairness first, actor/application fairness inside each tenant.
- Per-actor and per-tenant compute-credit token buckets.
- Model-dependent cost estimation and reconciliation using actual Ollama usage data.
- Capability-aware preflight using Ollama `/api/show` (`tools`, `vision`, `thinking`, `embedding`, completion) plus context-window guard.
- Model-affinity and adaptive routing across configured Ollama workers using load, learned tok/s, GPU utilization and VRAM pressure.
- Worker circuit breaker with safe pre-stream transport retries, persistent drain/disable maintenance state and manual circuit reset in the admin UI.
- Virtual model aliases (for example `fast` or `coding`) with ordered physical-model fallback and optional capability requirements.
- Model ACLs at tenant level and per API key; denied models are removed from discovery and rejected before scheduling.
- Persistent per-worker **Model Placement** rules (allow-all/whitelist + exact/prefix allow/deny patterns) enforced before adaptive routing, with an interactive model × worker matrix in the admin UI.
- Per-model concurrency limits with exact and trailing-prefix-wildcard matching.
- Optional built-in NVIDIA telemetry through `nvidia-smi` without CGO or an exporter; native host-memory telemetry is available on Linux, macOS and Windows.
- Hard per-worker concurrency using lock-free atomic counters.
- Live request lifecycle tracking: queued -> routing -> running -> streaming -> completed/failed/cancelled.
- Optional durable batch jobs with owner-scoped APIs, admin pause/resume/cancel/output controls, restart recovery, retention and execution through the normal quota/scheduler/routing/accounting pipeline using service class `batch`.
- Embedded Canvas pulse visualization and local LLM infrastructure map.
- Worker/model inventory, pull/stop/delete operations and persistent live policy overrides.
- Prometheus-compatible metrics with durable counter/histogram snapshots.
- Persistent quota-bucket snapshots, preventing a normal restart from resetting tenant/actor burst budgets.
- Asynchronous JSONL usage journal with startup replay, so dashboard usage totals and recent history survive restarts.
- Atomically persisted UI configuration override loaded on the next process start.
- No prompt or generated response content is stored by the telemetry/usage layer. Content persistence exists only in explicit opt-in features such as encrypted Responses conversations and durable batch payload spooling.
## Runtime vs durable state
The hot-path scheduler remains in memory:
```text
Fair queue / virtual clocks
Running concurrency
Per-worker active slots
OIDC browser sessions
Live request registry
Infrastructure topology
Model operation/transient inference-job state
```
These objects are intentionally not restored because requests and sockets cannot survive a process restart. Durable control-plane/accounting state is stored locally instead:
```text
<data_dir>/gateway-config.json UI configuration override
<data_dir>/api-keys.json UI-created API-key hashes + metadata
<data_dir>/policies.json tenant policy overrides
<data_dir>/metrics.json Prometheus counter/histogram state
<data_dir>/quota.json actor/tenant credit bucket state
<data_dir>/worker-performance.json learned per-worker/model tok/s routing state
<data_dir>/model-placement.json persistent live worker/model placement overrides
<data_dir>/worker-state.json persistent worker drain/disable state
<data_dir>/conversations.enc.json optional encrypted Responses context
<data_dir>/batch-jobs.json optional durable batch metadata/state
<data_dir>/batch/input/* optional durable batch request payloads
<data_dir>/batch/output/* optional durable batch response payloads
<data_dir>/usage/usage-*.jsonl completed request accounting/history
```
State-file writes use temp-file + fsync + atomic rename. Usage remains append-only and is replayed at startup. None of these files is consulted for every inference token or proxy chunk, so durable storage is not in the inference hot path. Use a persistent filesystem and preferably an absolute `storage.data_dir` in production.
Browser OIDC sessions remain intentionally volatile; after a gateway restart a browser logs in again. Transient inference jobs that were running at restart are disconnected and are not resurrected. Durable batch definitions survive; an interrupted batch attempt is recovered to a restart-safe queued state and can execute again.
The supplied `docker-compose.yml` mounts a named volume at `/data`, so the default relative `./data` state directory is durable across container replacement. If you run the image differently, mount persistent storage at `/data` or set an absolute `storage.data_dir`. The Compose file accepts `GATEWAY_CONFIG=./your-config.json` to select the bootstrap config; if omitted it intentionally uses `config.example.json` for development. Before a production rollout, run the candidate binary/container with `-check-config` and see `docs/DEPLOYMENT-HARDENING.md`.
## Persistent storage configuration
```json
"storage": {
"data_dir": "./data",
"config_file": "gateway-config.json",
"api_keys_file": "api-keys.json",
"policies_file": "policies.json",
"metrics_file": "metrics.json",
"quota_file": "quota.json",
"worker_performance_file": "worker-performance.json",
"model_placement_file": "model-placement.json",
"worker_state_file": "worker-state.json",
"auto_tune_file": "auto-tune.json",
"warm_models_file": "warm-models.json",
"alerts_file": "alerts.json",
"conversations_file": "conversations.enc.json",
"batch_jobs_file": "batch-jobs.json",
"batch_jobs_dir": "batch",
"flush_interval": "10s"
}
```
`storage` is bootstrap-only: edit it in the startup configuration, not through the web JSON editor. Other configuration changes saved in **Admin -> Konfiguration** are validated and written to `<data_dir>/gateway-config.json`; that file becomes the effective configuration at the next start. Delete the persistent override from the UI (or remove the file offline) to fall back to the bootstrap configuration.
**Admin -> Persistenz** shows the durable files, tiered usage-retention footprint and last compaction, can force an immediate flush or retention compaction, and can download a ZIP backup containing state, detail journals and rollups.
## Reliability, maintenance, aliases and model ACLs
The P0 reliability layer is configured independently from routing scores:
```json
"reliability": {
"enabled": true,
"failure_threshold": 3,
"open_duration": "30s",
"retry_attempts": 2,
"retry_backoff": "50ms"
}
```
A retry is attempted only when the upstream transport fails **before any response is committed to the client**. Once headers/body are visible, the response is never replayed. Backend 5xx responses still contribute to the worker circuit breaker but are not retried after commitment.
Workers can be placed into `active`, `draining`, or `disabled` state from **Admin -> Worker**. Draining rejects new work while existing streams finish; the state survives restart in `worker-state.json`.
Virtual model aliases expose stable names to clients:
```json
"model_aliases": {
"fast": { "models": ["qwen3:8b", "gemma3:12b"] },
"coding": {
"models": ["qwen3.6:27b-q4_K_M", "qwen3:8b"],
"required_capabilities": ["completion"]
}
}
```
Aliases appear in `/api/tags` and `/v1/models`; the gateway resolves them to the first currently routable target. The response includes `X-Gateway-Model-Alias` and `X-Gateway-Resolved-Model`.
Model access is separate from worker placement:
```json
"model_access": {
"default": { "mode": "allow_all" },
"tenants": {
"interns": {
"mode": "whitelist",
"allowed_models": ["fast", "qwen3:8b"],
"denied_models": ["expensive-*"]
}
}
}
```
API keys created in the web UI can additionally define their own allow/deny model patterns. API-key ACLs take precedence over the tenant baseline for that identity. Tenant baseline ACLs can also be created, edited and reset at runtime from **Fairness & Quotas → Tenant Model Access**; changes are persisted before immediate publication.
- Runtime model-alias CRUD is available from the admin model page and is persisted before immediate publication to new requests.
For production rollout and rollback steps, see `docs/PRODUCTION-UPDATE.md`; the current release validation is recorded in `docs/RELEASE-VERIFICATION-CHECKPOINT-28.md`.
See `docs/ROADMAP-P0-P3.md` for the staged plan through P3. Checkpoint 28 completes the active P3 roadmap with the optional public read-only status dashboard. Multi-gateway HA has been removed from the active roadmap; the existing HA-readiness tools remain available as diagnostics if future capacity or availability requirements change.
## Build
Requires Go 1.23 or newer.
```bash
go test ./...
go vet ./...
CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" \
-o dist/ollama-gateway ./cmd/ollama-gateway
```
For Apple Silicon:
```bash
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" \
-o dist/ollama-gateway-darwin-arm64 ./cmd/ollama-gateway
```
## Quick start
Copy the example configuration:
```bash
cp config.example.json config.json
./dist/ollama-gateway-darwin-arm64 -config ./config.json
```
The default example expects Ollama on `http://127.0.0.1:11434` and exposes the gateway on port `8080`.
Native Ollama example:
```bash
curl http://127.0.0.1:8080/api/chat \
-H 'Content-Type: application/json' \
-d '{"model":"qwen3:8b","messages":[{"role":"user","content":"hello"}]}'
```
OpenAI-compatible example:
```bash
curl http://127.0.0.1:8080/v1/chat/completions \
-H 'Content-Type: application/json' \
-d '{"model":"qwen3:8b","messages":[{"role":"user","content":"hello"}]}'
```
The example configuration grants localhost an admin IP bypass. Replace that for production.
## OpenWebUI
Use the gateway as an **Ollama API connection**, not as a raw browser-side URL. Current OpenWebUI performs model discovery with `GET /api/tags` and connection verification with `GET /api/version`; when a connection key is configured it sends that key as `Authorization: Bearer <key>`. The gateway accepts both configuration API keys and keys created under **Admin -> Sicherheit -> API Keys**. UI-created keys are persisted as hashes, so OpenWebUI keeps working with the same key across gateway restarts.
A ready-to-use configuration is included as `config.openwebui.example.json`. Set a secret before starting the gateway:
```bash
export OPENWEBUI_GATEWAY_KEY='replace-with-a-long-random-secret'
./dist/ollama-gateway-darwin-arm64 -config ./config.openwebui.example.json
```
In OpenWebUI go to **Settings -> Admin -> Connections -> Manage Ollama API Connections** and use:
```text
URL: http://host.docker.internal:8080
API Key: the value of OPENWEBUI_GATEWAY_KEY
```
Do **not** append `/api` to the Ollama base URL; OpenWebUI appends `/api/version`, `/api/tags`, `/api/show`, and the inference paths itself. When OpenWebUI runs outside Docker, use the hostname/IP at which its backend can actually reach the gateway. `localhost` inside the OpenWebUI container refers to that container, not the Mac host.
Verify the path independently before configuring OpenWebUI:
```bash
curl -sS -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" \
http://127.0.0.1:8080/api/version
curl -sS -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" \
http://127.0.0.1:8080/api/tags
```
For OpenAI-compatible connections, use `http://host.docker.internal:8080/v1` with the same key. `/v1/models` is generated from the same aggregated worker inventory as `/api/tags`.
Model discovery is gateway-native rather than a control-worker passthrough: `/api/tags` queries all configured Ollama workers in parallel, de-duplicates model IDs and normalizes both `name` and `model`. `/api/ps` is also aggregated. The worker router tracks installed models from `/api/tags` in addition to loaded models from `/api/ps`, preventing a selected model from being routed to a worker that does not have it installed.
If OpenWebUI is in Docker, relying on the default localhost IP bypass will normally fail because the TCP peer seen by the gateway is the Docker/host-network address rather than `127.0.0.1`. Prefer the dedicated static API key above instead of widening the bypass CIDR.
## Scheduling and fairness
A compute request is assigned an estimated credit cost before it enters the queue. The scheduler uses a two-level weighted fair queue:
1. tenants compete according to `tenant_weight`;
2. inside a tenant, actors compete according to `actor_weight` and request cost.
This prevents one tenant from gaining extra global capacity merely by creating many users, while also preventing one heavy actor from blocking all other actors in the same tenant.
Key settings:
```json
"scheduler": {
"global_concurrency": 4,
"max_queue": 2048,
"max_queue_per_actor": 64,
"queue_timeout": "10m",
"default_tenant_weight": 1,
"default_actor_weight": 1
}
```
`global_concurrency` should normally be close to the total useful inference concurrency of the configured workers. Worker-level `max_concurrent` remains a second hard limit.
## Compute credits and quotas
Each tenant policy can limit long-term consumption:
```json
"*": {
"tenant_weight": 1,
"actor_weight": 1,
"actor_credits_per_minute": 60,
"actor_burst_credits": 180,
"tenant_credits_per_minute": 300,
"tenant_burst_credits": 900
}
```
The gateway reserves estimated credits before admission. After Ollama finishes, the reservation is reconciled with actual usage. Native Ollama responses expose prompt/completion token counts and evaluation timings, allowing the estimator to incorporate model-specific token and compute weights.
Policy overrides made in the web UI take effect immediately for new requests and are persisted in `policies.json`, so they survive restarts. The baseline `scheduler.policies` in configuration remains the fallback.
## Worker routing
Workers are polled using Ollama `/api/ps` and `/api/tags`. Installed-model ownership is used as a routing constraint when known; a worker that already has the requested model loaded receives an additional strong affinity preference, while its current active-slot ratio increases its routing score.
### Model Placement / worker whitelist
Placement is a **hard constraint before routing score calculation**. It answers "where may this model run?" while tenant policies answer "who may consume how much?". They are intentionally separate.
Static worker defaults can be declared in `config.json`:
```json
{
"name": "rtx-4090",
"url": "http://10.10.11.124:11434",
"max_concurrent": 2,
"model_placement": {
"mode": "whitelist",
"allowed_models": [
"qwen3:8b",
"gemma4:*",
"embeddinggemma:*"
],
"denied_models": [
"gemma4:e4b"
]
}
}
```
`mode: "allow_all"` allows models by default; `mode: "whitelist"` blocks models unless an allow rule matches. Patterns are exact names, `*`, or a single trailing wildcard such as `gemma4:*`. The most specific rule wins; exact rules beat prefix rules and deny wins ties. This permits useful exceptions such as denying `gemma4:*` while explicitly allowing `gemma4:latest`.
The admin UI exposes this under **Model Placement** as a model × worker matrix. A cell shows whether the model is allowed, installed and loaded. Clicking a cell creates/removes an exact persistent exception; the worker editor manages broad patterns and offers presets for **allow all**, **only installed**, and **block all**. UI overrides are written atomically to `<data_dir>/model-placement.json` and affect new requests immediately without a gateway restart. Resetting a worker restores its `config.json` baseline.
Client-facing `/api/tags` and `/v1/models` only expose models installed on at least one worker on which placement permits them, so OpenWebUI does not offer a model that the gateway itself would refuse to route. Model-routed control requests such as `/api/show` use the same placement eligibility and cannot bypass it through the control worker.
See [`docs/MODEL-PLACEMENT.md`](docs/MODEL-PLACEMENT.md) for operational examples and UI semantics.
A two-worker M5-Ultra + RTX-4090 example using the model names from the development setup is included as `config.placement.example.json`.
```json
"workers": [
{
"name": "mac-studio-m5-ultra",
"url": "http://127.0.0.1:11434",
"max_concurrent": 4,
"health_interval": "5s",
"local_system_stats": true
}
]
```
The gateway's worker concurrency is enforced entirely with atomic counters in the process. No lease renewal or external slot backend is involved.
`local_system_stats` can expose host RAM data for a **local** worker. For remote workers, leave it false and use `telemetry_url`; the shipped `cmd/worker-telemetry` agent can export host RAM plus NVIDIA or Linux AMDGPU sysfs metrics. See `docs/WORKER-TELEMETRY.md`. Ollama `size_vram` is shown as resident model footprint, not as GPU-utilization percentage.
## Authentication
Three API authentication methods can coexist:
- OIDC bearer tokens;
- API keys from configuration and persistent UI-created API keys;
- trusted-IP bypass.
Administrators can create and revoke keys in the embedded UI. The plaintext secret is returned exactly once at creation time; only its SHA-256 digest and non-secret metadata are written to the durable API-key store. Static keys loaded from the bootstrap configuration are shown as read-only and cannot be deleted from the UI.
For IP bypass, the TCP peer is authoritative by default. `auth.trusted_proxies` still controls forwarded client-IP resolution for observability, but a forwarded address cannot satisfy credential-free IP bypass unless `auth.ip_bypass_use_forwarded_ip=true` is explicitly enabled. Keep bypass and proxy networks narrow.
See `config.oidc.example.json` for a browser-OIDC configuration.
## Web control plane
Enable with:
```json
"ui": {
"enabled": true,
"path": "/admin",
"title": "Ollama Fair Gateway"
}
```
The UI is embedded in the Go binary and has no CDN, npm or runtime frontend dependency.
Important pages include:
- Overview: queue, running requests, usage and workers.
- Live Flow: animated request lifecycle visualization.
- Infrastructure: local in-memory topology Tenant -> Gateway -> Queue -> Worker -> Model.
- Workers: health, active/per-model slots, GPU/VRAM telemetry and learned per-model token throughput.
- Models: inventory, detected capabilities/context length and administrative pull/stop/delete actions. **Stop** unloads a resident Ollama model using `POST /api/generate` with `keep_alive: 0`; it is not a request-cancellation mechanism.
- Model Placement: persistent worker whitelists/deny rules, current inventory/loaded state and one-click exact routing exceptions.
- Jobs: active queued/running/streaming inference requests with administrator-triggered cancellation.
- Fairness: scheduler limits, usage and persistent policy overrides.
- Usage: recent request metadata and token/credit accounting.
- Security: OIDC, API-key creation/revocation, static keys, IP bypass and trusted proxy configuration.
- Config: validated JSON editor for a persistent configuration override. Saves are atomic and take full effect after restart; storage-path changes remain bootstrap-only.
Live metadata is delivered to the browser over authenticated Server-Sent Events. Animation is rendered with Canvas in the browser; the server sends only bounded metadata snapshots.
## Job cancellation
Administrators can cancel active compute requests from **Admin -> Jobs** or directly from the **Live Flow** request list. The control-plane endpoints are:
```text
GET /gateway/ui-api/jobs
POST /gateway/ui-api/jobs/<request-id>/cancel
```
Cancellation is implemented with the Go request context rather than a second Ollama-specific control request:
- a **queued** job is removed from the fair-scheduler heap immediately and its reserved credits are released;
- a job waiting for a worker slot is cancelled immediately;
- a **running/streaming** job cancels the upstream Ollama HTTP context, releases the worker and scheduler slots, and reconciles any usage already observable from the response;
- live/accounting metadata records cancelled jobs with internal status `499`. If Ollama has already emitted HTTP `200` for a streaming response, that wire status cannot be changed; the client instead observes the stream ending early while the gateway records `499` internally.
Model **Stop** and job **Cancel** are intentionally different operations. Model Stop only unloads a resident model. Job Cancel terminates a specific inference request.
## Browser OIDC sessions
After Authorization Code + PKCE login, the browser receives an opaque session identifier. The OIDC access token remains in the gateway's in-memory session store. Session cookies are HttpOnly and state-changing cookie-authenticated UI requests require CSRF protection.
Because sessions are in memory, a gateway restart requires browser users to log in again. This is intentional in the single-process design.
## Infrastructure API
Admin-only local infrastructure endpoints:
```text
GET /gateway/ui-api/infrastructure
GET /gateway/ui-api/infrastructure/stream
```
The snapshot contains one gateway process, all configured workers, loaded models and bounded live request metadata. It never contains prompt text or generated output.
## Usage
Gateway API endpoints:
```text
GET /gateway/v1/usage/me
GET /gateway/v1/status
GET /gateway/v1/usage/tenant?tenant=<id> # admin
```
Completed usage events are appended asynchronously to daily JSONL files. Tiered retention keeps request-level detail for a bounded window, then replaces it with dimension rollups so long-running gateways do not grow an unbounded journal:
```json
"usage": {
"journal_dir": "./data/usage",
"buffer": 16384,
"flush_interval": "1s",
"retention": {
"detail_days": 30,
"daily_days": 400,
"monthly_months": 0,
"compaction_interval": "6h"
}
}
```
With these defaults, the last 30 days remain request-addressable JSONL detail, days 31–400 remain daily aggregates, and older data is folded into monthly aggregates indefinitely (`monthly_months: 0`). Rollups preserve request/error counts, tokens, credits, queue/service time, byte counts and prompt/output evaluation time for global, tenant, actor, application, model and worker dimensions. Performance (`tok/s`) is therefore still available after detail rows are removed.
Historical UI/API queries are available through `GET /gateway/ui-api/usage/rollups?granularity=daily|monthly&dimension=...`. **Admin -> Persistenz -> Jetzt kompaktieren** triggers the same idempotent compactor manually. The supplied configurations use `./data/usage`; keep it on persistent storage if accounting history must survive container/host replacement.
## Model capability preflight
By default the gateway caches Ollama `/api/show` metadata for ten minutes and rejects known-incompatible capability requests before they consume a scheduler/worker slot. This produces a clear gateway error for cases such as a `tools` request sent to a completion-only model. Metadata lookup failure itself is fail-open.
```json
"model_capabilities": {
"mode": "enforce",
"cache_ttl": "10m",
"context_guard": "reject",
"context": {
"max_requested_tokens": 32768,
"default_worker_tokens": 4096,
"estimation_margin_percent": 15,
"vision_reserve_tokens_per_image": 2048
}
}
```
For heterogeneous workers, `default_context_tokens` describes that worker's effective Ollama default when a model is neither loaded nor configured with a Modelfile `num_ctx`. `context_limits` is a hard per-worker/model ceiling using the same exact/prefix/`*` matching style as model concurrency:
```json
{
"name": "gpu-1",
"default_context_tokens": 4096,
"context_limits": {
"qwen3:*": 16384,
"*": 32768
}
}
```
The model table exposes **Model max**, **Configured** (Modelfile `num_ctx`) and **Loaded** (`/api/ps`) separately. Requests without a native `num_ctx` are routed only to workers whose effective context can hold the estimated request.
## Adaptive routing and per-model concurrency
Worker routing can combine model affinity with live GPU/VRAM telemetry and a per-worker/per-model EWMA of observed token throughput. Per-model limits prevent a large model from consuming the same concurrency as a much smaller model.
```json
"model_concurrency": {
"qwen3:8b": 2,
"gemma4:*": 1,
"*": 1
}
```
On NVIDIA Linux or Windows hosts, `"nvidia_smi": true` enables dependency-free GPU utilization, VRAM, temperature and power telemetry. Windows host RAM uses `GlobalMemoryStatusEx`; Linux uses `/proc/meminfo`.
## Prometheus metrics
`/metrics` exposes bounded-cardinality metrics for queue/running state, worker slots, request totals, GPU/VRAM telemetry, per-model active requests, learned token throughput, dropped usage-journal events and raw/daily/monthly retention file/byte gauges plus last-compaction timestamp/reclaimed-byte gauges. Tenant/user identifiers are deliberately not emitted as Prometheus labels.
The exporter uses a **snapshot-then-render** path: registry/scheduler/worker locks are held only while copying state and are released before any HTTP response bytes are written. A slow or stalled Prometheus client therefore cannot block request completion or the inference hot path.
## Operational model
The design intentionally optimizes for one strong gateway process. Go can handle far more concurrent HTTP/streaming connections than a local LLM server can generally infer at once, so the GPU/model execution layer should become the bottleneck long before the gateway.
If the gateway process is replicated independently behind a generic load balancer, each replica has its own queue, quotas and sessions. That setup does **not** provide strict global fairness. For strict fairness, run one authoritative gateway and scale Ollama workers behind it.
## Tests
```bash
node --check internal/webui/assets/app.js
go test ./...
go vet ./...
go test -race ./...
```
See also `docs/ARCHITECTURE.md`, `docs/SECURITY.md` and `docs/IMPROVEMENTS-2026.md`. A tuned RTX 4090 example is available as `config.rtx4090.example.json`.
Multi-gateway HA is not part of the active roadmap after single-process performance proved sufficient for the current deployment. The deterministic benchmark and evidence tooling in `docs/HA-READINESS.md` is retained for future re-evaluation rather than removed.
### OpenWebUI: "Modelle konnten nicht abgerufen werden"
If OpenWebUI uses an Ollama connection with **Authentication: None**, its backend source IP must match an `auth.ip_bypass` CIDR. The default examples only bypass `127.0.0.1`/`::1`; Docker or another machine is therefore intentionally rejected. Authentication failures are logged with `client_ip=...` so an exact `/32` can be added. See `docs/OPENWEBUI.md`.
## P2/P3 operations and durable work
The current gateway includes the P2 operator/state features:
- **Warm Models**: persistent `hot`/`warm`/`cold` residency policies with placement-aware preload, idle unload, model-operation locking, and VRAM eviction suggestions. See `docs/WARM-MODELS.md`.
- **Alerts**: persistent firing/resolved events for worker health, circuit state, queue depth/wait, VRAM, OOM, storage, and quota thresholds. Generic HMAC-signed webhooks use bounded asynchronous delivery and transient retries. See `docs/ALERTS.md`.
- **Optional Responses conversations**: disabled by default; when enabled, `previous_response_id` is resolved from an AES-256-GCM encrypted, retention-bounded tenant/actor-scoped store. `store:false` remains a per-request opt-out. See `docs/CONVERSATIONS.md`.
- **Policy simulator**: evaluates identity/model/capability/placement/routing decisions without running inference.
- **Durable Batch Jobs (P3.1)**: disabled by default; accepted compute requests can be persisted, executed later under service class `batch`, paused/resumed/cancelled, recovered after restart, retained for a bounded window and inspected in the dedicated Admin -> Batch Jobs page. See `docs/BATCH-JOBS.md`.
Warm-model actions never override Model Placement or worker drain/disable state. Model unload/preload acquires a per-worker/per-model maintenance reservation so inference cannot race an unload operation.
## Hardened production Compose
For production Docker deployments, use `docker-compose.production.yml` with an explicit bootstrap config:
```sh
cp .env.production.example .env
# edit .env so GATEWAY_CONFIG points at the production JSON
./scripts/production-preflight.sh
docker compose -f docker-compose.production.yml up -d --build
```
The production Compose publishes `127.0.0.1:9080` by default, requires `GATEWAY_CONFIG`, runs the scratch image as UID/GID 65532 with a read-only root filesystem, drops all capabilities and enables `no-new-privileges`. See `docs/DEPLOYMENT-HARDENING.md` before using a non-loopback bind.
## Admin UI theme
Checkpoint 24 ships the Admin Control Plane as a light, data-dense technical dashboard. The theme uses explicit light browser controls, high-contrast status colors, compact mono-accent typography, light tables/forms/dialogs and matching light Live Flow/Infrastructure canvases. It has no external font/CDN dependency: preferred local IBM Plex/JetBrains/Cascadia faces fall back to platform fonts automatically.
The UI is embedded into the gateway binary. After upgrading from an earlier checkpoint, rebuild/redeploy the gateway image and perform one hard browser refresh if an old stylesheet is still cached. No gateway configuration or persistent-state migration is required for the theme update.
## Checkpoint 25
- Fixed sidebar navigation overflow on shorter viewport heights.
- Sidebar navigation now scrolls independently and keeps the footer visible.
- Added compact spacing rules for shorter screens.
## Checkpoint 26 — Dialog close validation fix
- Fixes **Tenant Policies → Override anlegen**: both the × button and **Abbrechen** now close the modal even while required fields are empty.
- Close controls are explicit `type="button"` actions using the shared `data-close-dialog` handler; only **Speichern** submits the policy form.
- Applies the same validation-independent close behavior to the existing **Modell pullen** dialog.
- Adds WebUI regression tests for close-target validity and required-field dialogs.
- No scheduler, proxy, auth, persistence, routing, quota, or inference behavior changed.
## Checkpoint 27 — Context / num_ctx Hardening
See `docs/CONTEXT-HARDENING.md`. The gateway now derives effective worker context from loaded `/api/ps`, Modelfile `num_ctx`, worker defaults and hard context caps; it also fixes Responses/output/instructions/suffix/vision context estimation.
## Public status dashboard
Checkpoint 28 adds an optional read-only public dashboard at `/status/`. It shows current queue/load, worker capacity, an infrastructure map and anonymized live request activity without requiring an API key. The public API is built from a separate allow-list and does not expose tenant/actor/application identities, worker URLs, policies, quotas, storage paths or error details. Worker and model names are anonymized by default and can be explicitly published or mapped to public display names. See `docs/PUBLIC-DASHBOARD.md`.
cmd/bench/main.go
+339
View File
@@ -0,0 +1,339 @@
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"net/http/httptrace"
"os"
"sort"
"sync"
"time"
)
type result struct {
Latency time.Duration
TTFB time.Duration
Prompt int64
Completion int64
Bytes int64
Status int
Err error
}
type durationStats struct {
P50 float64 `json:"p50_ms"`
P95 float64 `json:"p95_ms"`
P99 float64 `json:"p99_ms"`
Max float64 `json:"max_ms"`
}
type summary struct {
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at"`
BaseURL string `json:"base_url"`
Model string `json:"model"`
Concurrency int `json:"concurrency"`
Requests int `json:"requests"`
WarmupRequests int `json:"warmup_requests"`
Successful int `json:"successful"`
Errors int `json:"errors"`
StatusCounts map[string]int `json:"status_counts"`
WallMS float64 `json:"wall_ms"`
ThroughputRPS float64 `json:"throughput_rps"`
Latency durationStats `json:"latency"`
TTFB durationStats `json:"ttfb"`
PromptTokens int64 `json:"prompt_tokens"`
CompletionTokens int64 `json:"completion_tokens"`
CompletionTokPS float64 `json:"completion_tokens_per_second"`
BytesReceived int64 `json:"bytes_received"`
BytesPerSecond float64 `json:"bytes_per_second"`
Stream bool `json:"stream"`
KeepAlive bool `json:"keep_alive"`
ServiceClass string `json:"service_class,omitempty"`
ErrorsByMessage map[string]int `json:"errors_by_message,omitempty"`
}
type benchConfig struct {
Base string
Key string
Model string
Concurrency int
Requests int
Warmup int
MaxTokens int
Prompt string
Timeout time.Duration
Stream bool
DisableKeepAlive bool
ServiceClass string
JSONOut string
}
func main() {
base := flag.String("base-url", "http://127.0.0.1:8080", "gateway base URL")
key := flag.String("api-key", "", "API key; can also use GATEWAY_BENCH_API_KEY")
model := flag.String("model", "qwen3:8b", "model name")
conc := flag.Int("concurrency", 4, "parallel clients")
n := flag.Int("requests", 20, "total measured requests")
warmup := flag.Int("warmup", 0, "warmup requests before measurement")
maxTokens := flag.Int("max-tokens", 128, "max completion tokens")
prompt := flag.String("prompt", "Explain in three concise paragraphs why fair scheduling matters for shared LLM inference.", "prompt")
timeout := flag.Duration("timeout", 2*time.Minute, "per-request HTTP timeout")
stream := flag.Bool("stream", false, "request OpenAI streaming responses")
disableKeepAlive := flag.Bool("disable-keepalive", false, "disable HTTP connection reuse")
serviceClass := flag.String("service-class", "", "optional X-Gateway-Service-Class override")
jsonOut := flag.String("json-out", "", "optional path for machine-readable JSON summary; '-' writes JSON to stdout")
flag.Parse()
if *key == "" {
*key = os.Getenv("GATEWAY_BENCH_API_KEY")
}
cfg := benchConfig{Base: *base, Key: *key, Model: *model, Concurrency: *conc, Requests: *n, Warmup: *warmup, MaxTokens: *maxTokens, Prompt: *prompt, Timeout: *timeout, Stream: *stream, DisableKeepAlive: *disableKeepAlive, ServiceClass: *serviceClass, JSONOut: *jsonOut}
if cfg.Concurrency < 1 || cfg.Requests < 1 || cfg.Warmup < 0 {
fmt.Fprintln(os.Stderr, "concurrency and requests must be positive; warmup must be non-negative")
os.Exit(2)
}
if cfg.Timeout <= 0 {
fmt.Fprintln(os.Stderr, "timeout must be positive")
os.Exit(2)
}
client := newClient(cfg)
body, err := requestBody(cfg)
if err != nil {
fmt.Fprintln(os.Stderr, "request body:", err)
os.Exit(2)
}
ctx := context.Background()
if cfg.Warmup > 0 {
warm := run(ctx, client, cfg, body, cfg.Warmup)
for _, r := range warm {
if r.Err != nil {
fmt.Fprintln(os.Stderr, "warmup error:", r.Err)
}
}
}
started := time.Now().UTC()
results := run(ctx, client, cfg, body, cfg.Requests)
finished := time.Now().UTC()
s := summarize(cfg, started, finished, results)
printHuman(s)
if cfg.JSONOut != "" {
if err := writeJSONSummary(cfg.JSONOut, s); err != nil {
fmt.Fprintln(os.Stderr, "write JSON summary:", err)
os.Exit(1)
}
}
if s.Successful == 0 {
os.Exit(1)
}
}
func newClient(cfg benchConfig) *http.Client {
idle := max(256, cfg.Concurrency*2)
return &http.Client{
Timeout: cfg.Timeout,
Transport: &http.Transport{
MaxIdleConns: idle,
MaxIdleConnsPerHost: idle,
IdleConnTimeout: 90 * time.Second,
DisableKeepAlives: cfg.DisableKeepAlive,
},
}
}
func requestBody(cfg benchConfig) ([]byte, error) {
return json.Marshal(map[string]any{
"model": cfg.Model,
"messages": []map[string]string{{"role": "user", "content": cfg.Prompt}},
"max_tokens": cfg.MaxTokens,
"stream": cfg.Stream,
})
}
func run(ctx context.Context, client *http.Client, cfg benchConfig, body []byte, count int) []result {
jobs := make(chan struct{})
results := make(chan result, count)
var wg sync.WaitGroup
for i := 0; i < cfg.Concurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for range jobs {
results <- one(ctx, client, cfg, body)
}
}()
}
go func() {
for i := 0; i < count; i++ {
jobs <- struct{}{}
}
close(jobs)
wg.Wait()
close(results)
}()
out := make([]result, 0, count)
for r := range results {
out = append(out, r)
}
return out
}
func one(ctx context.Context, client *http.Client, cfg benchConfig, body []byte) result {
started := time.Now()
var firstByte time.Time
trace := &httptrace.ClientTrace{GotFirstResponseByte: func() { firstByte = time.Now() }}
reqCtx := httptrace.WithClientTrace(ctx, trace)
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, stringsTrimRightSlash(cfg.Base)+"/v1/chat/completions", bytes.NewReader(body))
if err != nil {
return result{Err: err}
}
req.Header.Set("Content-Type", "application/json")
if cfg.Key != "" {
req.Header.Set("Authorization", "Bearer "+cfg.Key)
}
if cfg.ServiceClass != "" {
req.Header.Set("X-Gateway-Service-Class", cfg.ServiceClass)
}
resp, err := client.Do(req)
if err != nil {
return result{Latency: time.Since(started), Err: err}
}
b, readErr := io.ReadAll(resp.Body)
closeErr := resp.Body.Close()
latency := time.Since(started)
ttfb := time.Duration(0)
if !firstByte.IsZero() {
ttfb = firstByte.Sub(started)
}
r := result{Latency: latency, TTFB: ttfb, Bytes: int64(len(b)), Status: resp.StatusCode}
if readErr != nil {
r.Err = readErr
return r
}
if closeErr != nil {
r.Err = closeErr
return r
}
if resp.StatusCode/100 != 2 {
r.Err = fmt.Errorf("HTTP %d: %s", resp.StatusCode, truncate(string(b), 240))
return r
}
if !cfg.Stream {
var doc struct {
Usage struct {
Prompt int64 `json:"prompt_tokens"`
Completion int64 `json:"completion_tokens"`
} `json:"usage"`
}
if err := json.Unmarshal(b, &doc); err == nil {
r.Prompt = doc.Usage.Prompt
r.Completion = doc.Usage.Completion
}
}
return r
}
func summarize(cfg benchConfig, started, finished time.Time, results []result) summary {
wall := finished.Sub(started)
latencies := make([]time.Duration, 0, len(results))
ttfbs := make([]time.Duration, 0, len(results))
statusCounts := map[string]int{}
errorsByMessage := map[string]int{}
var prompt, completion, received int64
errs := 0
for _, r := range results {
if r.Status != 0 {
statusCounts[fmt.Sprintf("%d", r.Status)]++
}
if r.Err != nil {
errs++
errorsByMessage[truncate(r.Err.Error(), 180)]++
continue
}
latencies = append(latencies, r.Latency)
if r.TTFB > 0 {
ttfbs = append(ttfbs, r.TTFB)
}
prompt += r.Prompt
completion += r.Completion
received += r.Bytes
}
success := len(latencies)
seconds := wall.Seconds()
if seconds <= 0 {
seconds = 1e-9
}
return summary{
StartedAt: started, FinishedAt: finished, BaseURL: cfg.Base, Model: cfg.Model,
Concurrency: cfg.Concurrency, Requests: cfg.Requests, WarmupRequests: cfg.Warmup,
Successful: success, Errors: errs, StatusCounts: statusCounts, WallMS: float64(wall) / float64(time.Millisecond),
ThroughputRPS: float64(success) / seconds, Latency: stats(latencies), TTFB: stats(ttfbs),
PromptTokens: prompt, CompletionTokens: completion, CompletionTokPS: float64(completion) / seconds,
BytesReceived: received, BytesPerSecond: float64(received) / seconds, Stream: cfg.Stream,
KeepAlive: !cfg.DisableKeepAlive, ServiceClass: cfg.ServiceClass, ErrorsByMessage: errorsByMessage,
}
}
func stats(ds []time.Duration) durationStats {
if len(ds) == 0 {
return durationStats{}
}
sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] })
pct := func(p float64) time.Duration {
idx := int(float64(len(ds)-1) * p)
return ds[idx]
}
return durationStats{P50: msFloat(pct(.50)), P95: msFloat(pct(.95)), P99: msFloat(pct(.99)), Max: msFloat(ds[len(ds)-1])}
}
func printHuman(s summary) {
fmt.Printf("successful=%d errors=%d concurrency=%d wall=%s keepalive=%t stream=%t\n", s.Successful, s.Errors, s.Concurrency, time.Duration(s.WallMS*float64(time.Millisecond)).Round(time.Millisecond), s.KeepAlive, s.Stream)
fmt.Printf("latency p50=%s p95=%s p99=%s max=%s\n", fmtMS(s.Latency.P50), fmtMS(s.Latency.P95), fmtMS(s.Latency.P99), fmtMS(s.Latency.Max))
fmt.Printf("ttfb p50=%s p95=%s p99=%s max=%s\n", fmtMS(s.TTFB.P50), fmtMS(s.TTFB.P95), fmtMS(s.TTFB.P99), fmtMS(s.TTFB.Max))
fmt.Printf("throughput=%.2f req/s bytes=%d bytes/s=%.0f prompt_tokens=%d completion_tokens=%d completion_tok/s=%.2f\n", s.ThroughputRPS, s.BytesReceived, s.BytesPerSecond, s.PromptTokens, s.CompletionTokens, s.CompletionTokPS)
if len(s.StatusCounts) > 0 {
b, _ := json.Marshal(s.StatusCounts)
fmt.Printf("status=%s\n", b)
}
if len(s.ErrorsByMessage) > 0 {
for msg, n := range s.ErrorsByMessage {
fmt.Fprintf(os.Stderr, "error x%d: %s\n", n, msg)
}
}
}
func writeJSONSummary(path string, s summary) error {
b, err := json.MarshalIndent(s, "", " ")
if err != nil {
return err
}
b = append(b, '\n')
if path == "-" {
_, err = os.Stdout.Write(b)
return err
}
return os.WriteFile(path, b, 0644)
}
func msFloat(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) }
func fmtMS(v float64) string {
return (time.Duration(v * float64(time.Millisecond))).Round(time.Microsecond).String()
}
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "…"
}
func stringsTrimRightSlash(s string) string {
for len(s) > 0 && s[len(s)-1] == '/' {
s = s[:len(s)-1]
}
return s
}
cmd/bench/main_test.go
+68
View File
@@ -0,0 +1,68 @@
package main
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestStats(t *testing.T) {
d := []time.Duration{10 * time.Millisecond, 50 * time.Millisecond, 20 * time.Millisecond, 40 * time.Millisecond, 30 * time.Millisecond}
s := stats(d)
if s.P50 != 30 || s.P95 != 40 || s.P99 != 40 || s.Max != 50 {
t.Fatalf("unexpected stats: %+v", s)
}
}
func TestOneCapturesStatusUsageBytesAndHeaders(t *testing.T) {
var gotAuth, gotClass string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
gotClass = r.Header.Get("X-Gateway-Service-Class")
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"ok"}}],"usage":{"prompt_tokens":9,"completion_tokens":4}}`)
}))
defer ts.Close()
cfg := benchConfig{Base: ts.URL + "/", Key: "secret", Model: "m", ServiceClass: "batch", Timeout: time.Second}
body, err := requestBody(cfg)
if err != nil {
t.Fatal(err)
}
r := one(context.Background(), newClient(cfg), cfg, body)
if r.Err != nil {
t.Fatal(r.Err)
}
if r.Status != http.StatusOK || r.Prompt != 9 || r.Completion != 4 || r.Bytes == 0 || r.Latency <= 0 || r.TTFB <= 0 {
t.Fatalf("unexpected result: %+v", r)
}
if gotAuth != "Bearer secret" || gotClass != "batch" {
t.Fatalf("headers auth=%q class=%q", gotAuth, gotClass)
}
}
func TestSummarizeSeparatesErrors(t *testing.T) {
cfg := benchConfig{Base: "http://gateway", Model: "m", Concurrency: 2, Requests: 3, Warmup: 1}
start := time.Unix(1, 0).UTC()
finish := start.Add(time.Second)
s := summarize(cfg, start, finish, []result{
{Latency: 10 * time.Millisecond, TTFB: 5 * time.Millisecond, Prompt: 8, Completion: 2, Bytes: 100, Status: 200},
{Latency: 20 * time.Millisecond, TTFB: 7 * time.Millisecond, Prompt: 8, Completion: 3, Bytes: 110, Status: 200},
{Latency: 2 * time.Millisecond, Status: 503, Err: fmt.Errorf("HTTP 503: unavailable")},
})
if s.Successful != 2 || s.Errors != 1 || s.StatusCounts["200"] != 2 || s.StatusCounts["503"] != 1 {
t.Fatalf("unexpected counts: %+v", s)
}
if s.ThroughputRPS != 2 || s.PromptTokens != 16 || s.CompletionTokens != 5 || s.BytesReceived != 210 {
t.Fatalf("unexpected throughput/usage: %+v", s)
}
}
func TestStringsTrimRightSlash(t *testing.T) {
if got := stringsTrimRightSlash("http://x///"); got != "http://x" {
t.Fatalf("got %q", got)
}
}
cmd/ha-report/main.go
+396
View File
@@ -0,0 +1,396 @@
package main
import (
"bufio"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/example/ollama-fair-gateway/internal/haresource"
)
type durationStats struct {
P50 float64 `json:"p50_ms"`
P95 float64 `json:"p95_ms"`
P99 float64 `json:"p99_ms"`
Max float64 `json:"max_ms"`
}
type benchSummary struct {
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at"`
BaseURL string `json:"base_url"`
Model string `json:"model"`
Concurrency int `json:"concurrency"`
Requests int `json:"requests"`
WarmupRequests int `json:"warmup_requests"`
Successful int `json:"successful"`
Errors int `json:"errors"`
StatusCounts map[string]int `json:"status_counts"`
WallMS float64 `json:"wall_ms"`
ThroughputRPS float64 `json:"throughput_rps"`
Latency durationStats `json:"latency"`
TTFB durationStats `json:"ttfb"`
ServiceClass string `json:"service_class,omitempty"`
ErrorsByMessage map[string]int `json:"errors_by_message,omitempty"`
}
type metricSample struct {
Name string
Labels string
Value float64
}
type resourceSnapshot = haresource.Snapshot
type resourceSamplingReport = haresource.SamplingReport
type levelReport struct {
Concurrency int `json:"concurrency"`
Requests int `json:"requests"`
WarmupRequests int `json:"warmup_requests"`
Successful int `json:"successful"`
Errors int `json:"errors"`
ThroughputRPS float64 `json:"throughput_rps"`
Latency durationStats `json:"latency"`
TTFB durationStats `json:"ttfb"`
GatewayRequestsDelta float64 `json:"gateway_requests_delta"`
Gateway2xxDelta float64 `json:"gateway_2xx_delta"`
GatewayErrorDelta float64 `json:"gateway_error_delta"`
RetriesDelta float64 `json:"retries_delta"`
CircuitOpensDelta float64 `json:"circuit_opens_delta"`
PromptTokensDelta float64 `json:"prompt_tokens_delta"`
CompletionTokensDelta float64 `json:"completion_tokens_delta"`
QueueObservationsDelta float64 `json:"queue_observations_delta"`
ServiceObservationsDelta float64 `json:"service_observations_delta"`
AccountingMatches bool `json:"accounting_matches"`
ResourcesBefore *resourceSnapshot `json:"resources_before,omitempty"`
ResourcesAfter *resourceSnapshot `json:"resources_after,omitempty"`
ResourceSamples *resourceSamplingReport `json:"resource_samples,omitempty"`
}
type evidenceReport struct {
GeneratedAt time.Time `json:"generated_at"`
InputDir string `json:"input_dir"`
Levels []levelReport `json:"levels"`
EvidenceComplete bool `json:"evidence_complete"`
ResourceEvidenceComplete bool `json:"resource_evidence_complete"`
SustainedResourceEvidenceComplete bool `json:"sustained_resource_evidence_complete"`
GateStatus string `json:"gate_status"`
GateReason string `json:"gate_reason"`
}
func main() {
input := flag.String("input", "", "HA-readiness result directory")
jsonOut := flag.String("json-out", "", "JSON report path; defaults to <input>/report.json")
markdownOut := flag.String("markdown-out", "", "Markdown report path; defaults to <input>/report.md")
flag.Parse()
if *input == "" {
fmt.Fprintln(os.Stderr, "-input is required")
os.Exit(2)
}
if *jsonOut == "" {
*jsonOut = filepath.Join(*input, "report.json")
}
if *markdownOut == "" {
*markdownOut = filepath.Join(*input, "report.md")
}
r, err := buildReport(*input)
if err != nil {
fmt.Fprintln(os.Stderr, "build report:", err)
os.Exit(1)
}
if err := writeJSON(*jsonOut, r); err != nil {
fmt.Fprintln(os.Stderr, "write JSON report:", err)
os.Exit(1)
}
if err := os.WriteFile(*markdownOut, []byte(renderMarkdown(r)), 0o644); err != nil {
fmt.Fprintln(os.Stderr, "write Markdown report:", err)
os.Exit(1)
}
fmt.Printf("report: %s\nreport: %s\ngate: %s — %s\n", *jsonOut, *markdownOut, r.GateStatus, r.GateReason)
}
func buildReport(dir string) (evidenceReport, error) {
matches, err := filepath.Glob(filepath.Join(dir, "concurrency-*.json"))
if err != nil {
return evidenceReport{}, err
}
if len(matches) == 0 {
return evidenceReport{}, errors.New("no concurrency-*.json files found")
}
levels := make([]levelReport, 0, len(matches))
complete := true
resourceComplete := true
resourceSeen := false
sustainedComplete := true
sustainedSeen := false
for _, p := range matches {
b, err := os.ReadFile(p)
if err != nil {
return evidenceReport{}, err
}
var s benchSummary
if err := json.Unmarshal(b, &s); err != nil {
return evidenceReport{}, fmt.Errorf("%s: %w", p, err)
}
beforePath := filepath.Join(dir, fmt.Sprintf("metrics-before-c%d.prom", s.Concurrency))
afterPath := filepath.Join(dir, fmt.Sprintf("metrics-after-c%d.prom", s.Concurrency))
before, errBefore := parsePrometheusFile(beforePath)
after, errAfter := parsePrometheusFile(afterPath)
if errBefore != nil || errAfter != nil {
complete = false
}
l := levelReport{
Concurrency: s.Concurrency, Requests: s.Requests, WarmupRequests: s.WarmupRequests,
Successful: s.Successful, Errors: s.Errors, ThroughputRPS: s.ThroughputRPS,
Latency: s.Latency, TTFB: s.TTFB,
}
resourceBefore, errResourceBefore := readResourceSnapshot(filepath.Join(dir, fmt.Sprintf("resources-before-c%d.json", s.Concurrency)))
resourceAfter, errResourceAfter := readResourceSnapshot(filepath.Join(dir, fmt.Sprintf("resources-after-c%d.json", s.Concurrency)))
if errResourceBefore == nil && errResourceAfter == nil {
l.ResourcesBefore = &resourceBefore
l.ResourcesAfter = &resourceAfter
resourceSeen = true
} else {
resourceComplete = false
}
resourceSamples, errResourceSamples := readResourceSamples(filepath.Join(dir, fmt.Sprintf("resources-samples-c%d.json", s.Concurrency)))
if errResourceSamples == nil && resourceSamples.Complete() {
l.ResourceSamples = &resourceSamples
sustainedSeen = true
} else {
sustainedComplete = false
}
if errBefore == nil && errAfter == nil {
l.GatewayRequestsDelta = deltaByName(before, after, "ollama_gateway_requests_total")
l.Gateway2xxDelta = deltaByNameLabelContains(before, after, "ollama_gateway_requests_total", `status_class="2xx"`)
l.GatewayErrorDelta = deltaByName(before, after, "ollama_gateway_errors_total")
// Older/current builds expose errors via requests_total status class rather than a dedicated counter.
if l.GatewayErrorDelta == 0 {
l.GatewayErrorDelta = l.GatewayRequestsDelta - l.Gateway2xxDelta
}
l.RetriesDelta = deltaByName(before, after, "ollama_gateway_retries_total")
l.CircuitOpensDelta = deltaByName(before, after, "ollama_gateway_circuit_opens_total")
l.PromptTokensDelta = deltaByName(before, after, "ollama_gateway_prompt_tokens_total")
l.CompletionTokensDelta = deltaByName(before, after, "ollama_gateway_completion_tokens_total")
l.QueueObservationsDelta = deltaByName(before, after, "ollama_gateway_queue_seconds_count")
l.ServiceObservationsDelta = deltaByName(before, after, "ollama_gateway_service_seconds_count")
expected := float64(s.Requests + s.WarmupRequests)
l.AccountingMatches = almostEqual(l.GatewayRequestsDelta, expected) && almostEqual(l.QueueObservationsDelta, expected) && almostEqual(l.ServiceObservationsDelta, expected)
if !l.AccountingMatches {
complete = false
}
}
levels = append(levels, l)
}
sort.Slice(levels, func(i, j int) bool { return levels[i].Concurrency < levels[j].Concurrency })
if !resourceSeen {
resourceComplete = false
}
if !sustainedSeen {
sustainedComplete = false
}
r := evidenceReport{
GeneratedAt: time.Now().UTC(), InputDir: filepath.Clean(dir), Levels: levels,
EvidenceComplete: complete, ResourceEvidenceComplete: resourceComplete,
SustainedResourceEvidenceComplete: sustainedComplete,
}
if !complete {
r.GateStatus = "incomplete"
r.GateReason = "benchmark and gateway metrics evidence are missing or do not reconcile"
} else if sustainedComplete {
r.GateStatus = "not-proven"
r.GateReason = "client/gateway accounting reconciles and sustained host/process resource sampling is complete; HA still requires an operator-demonstrated capacity, availability, or topology need"
} else if resourceComplete {
r.GateStatus = "not-proven"
r.GateReason = "client/gateway accounting reconciles and before/after host/process resource snapshots are present; sustained peak resource evidence is incomplete, and HA still requires an operator-demonstrated capacity, availability, or topology need"
} else {
r.GateStatus = "not-proven"
r.GateReason = "client and gateway counters reconcile; host/process resource evidence is incomplete, and HA still requires demonstrated capacity, availability, or topology need"
}
return r, nil
}
func readResourceSnapshot(path string) (resourceSnapshot, error) {
b, err := os.ReadFile(path)
if err != nil {
return resourceSnapshot{}, err
}
var s resourceSnapshot
if err := json.Unmarshal(b, &s); err != nil {
return resourceSnapshot{}, err
}
if s.PID <= 0 {
return resourceSnapshot{}, fmt.Errorf("%s: invalid pid", path)
}
return s, nil
}
func readResourceSamples(path string) (resourceSamplingReport, error) {
b, err := os.ReadFile(path)
if err != nil {
return resourceSamplingReport{}, err
}
var r resourceSamplingReport
if err := json.Unmarshal(b, &r); err != nil {
return resourceSamplingReport{}, err
}
if !r.Complete() {
return r, fmt.Errorf("%s: incomplete sustained resource report", path)
}
return r, nil
}
func parsePrometheusFile(path string) ([]metricSample, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
var out []metricSample
s := bufio.NewScanner(f)
buf := make([]byte, 64*1024)
s.Buffer(buf, 4*1024*1024)
for s.Scan() {
line := strings.TrimSpace(s.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
parts := strings.Fields(line)
if len(parts) < 2 {
continue
}
v, err := strconv.ParseFloat(parts[1], 64)
if err != nil {
continue
}
key := parts[0]
name, labels := key, ""
if i := strings.IndexByte(key, '{'); i >= 0 {
name = key[:i]
labels = strings.TrimSuffix(key[i+1:], "}")
}
out = append(out, metricSample{Name: name, Labels: labels, Value: v})
}
return out, s.Err()
}
func deltaByName(before, after []metricSample, name string) float64 {
return sumBy(before, name, "", false, after)
}
func deltaByNameLabelContains(before, after []metricSample, name, label string) float64 {
return sumBy(before, name, label, true, after)
}
func sumBy(before []metricSample, name, label string, filter bool, after []metricSample) float64 {
sum := func(xs []metricSample) float64 {
var n float64
for _, x := range xs {
if x.Name != name {
continue
}
if filter && !strings.Contains(x.Labels, label) {
continue
}
n += x.Value
}
return n
}
return sum(after) - sum(before)
}
func almostEqual(a, b float64) bool {
d := a - b
if d < 0 {
d = -d
}
return d < 0.000001
}
func writeJSON(path string, v any) error {
b, err := json.MarshalIndent(v, "", " ")
if err != nil {
return err
}
b = append(b, '\n')
return os.WriteFile(path, b, 0o644)
}
func renderMarkdown(r evidenceReport) string {
var b strings.Builder
fmt.Fprintln(&b, "# HA readiness evidence report")
fmt.Fprintln(&b)
fmt.Fprintf(&b, "Generated: `%s` \n", r.GeneratedAt.Format(time.RFC3339))
fmt.Fprintf(&b, "Gateway accounting complete: **%t** \n", r.EvidenceComplete)
fmt.Fprintf(&b, "Resource snapshots complete: **%t** \n", r.ResourceEvidenceComplete)
fmt.Fprintf(&b, "Sustained resource sampling complete: **%t** \n", r.SustainedResourceEvidenceComplete)
fmt.Fprintf(&b, "P3.2 gate: **%s** — %s\n\n", r.GateStatus, r.GateReason)
fmt.Fprintln(&b, "| Concurrency | Success/Error | req/s | p95 latency | p95 TTFB | Gateway requests Δ | 2xx Δ | retries Δ | circuit opens Δ | counters reconcile | after RSS | sampled peak RSS | sampled peak CPU | sampled peak FDs | samples |")
fmt.Fprintln(&b, "|---:|---:|---:|---:|---:|---:|---:|---:|---:|:---:|---:|---:|---:|---:|---:|")
for _, l := range r.Levels {
afterRSS, _, _, _, _ := resourceCells(l.ResourcesAfter)
peakRSS, peakCPU, peakFDs, sampleCount := sustainedCells(l.ResourceSamples)
fmt.Fprintf(&b, "| %d | %d/%d | %.2f | %.2f ms | %.2f ms | %.0f | %.0f | %.0f | %.0f | %t | %s | %s | %s | %s | %s |\n", l.Concurrency, l.Successful, l.Errors, l.ThroughputRPS, l.Latency.P95, l.TTFB.P95, l.GatewayRequestsDelta, l.Gateway2xxDelta, l.RetriesDelta, l.CircuitOpensDelta, l.AccountingMatches, afterRSS, peakRSS, peakCPU, peakFDs, sampleCount)
}
fmt.Fprintln(&b, "\n## Interpretation")
fmt.Fprintln(&b)
fmt.Fprintln(&b, "This report verifies that client-side benchmark counts reconcile with gateway-side Prometheus counters. Before/after resource files remain endpoint evidence. When `resources-samples-cN.json` is present and complete, the sampled peak columns come from measurements taken throughout that benchmark level; on Linux process CPU is calculated from `/proc` process/host tick deltas and may exceed 100% when multiple logical CPUs are used.")
fmt.Fprintln(&b)
fmt.Fprintln(&b, "The report intentionally does not prove that HA is required. Correlate sustained process/host evidence with worker/GPU saturation and an explicit capacity, availability, or topology requirement before changing the P3.2 gate.")
return b.String()
}
func resourceCells(s *resourceSnapshot) (rss, cpu, fds, threads, load1 string) {
if s == nil {
return "-", "-", "-", "-", "-"
}
if s.ProcessRSSBytes > 0 {
rss = fmt.Sprintf("%.1f MiB", float64(s.ProcessRSSBytes)/(1024*1024))
} else {
rss = "-"
}
if s.ProcessCPUPercent > 0 {
cpu = fmt.Sprintf("%.1f%%", s.ProcessCPUPercent)
} else {
cpu = "0.0%"
}
if s.ProcessOpenFDs > 0 {
fds = strconv.FormatInt(s.ProcessOpenFDs, 10)
} else {
fds = "-"
}
if s.ProcessThreads > 0 {
threads = strconv.FormatInt(s.ProcessThreads, 10)
} else {
threads = "-"
}
load1 = fmt.Sprintf("%.2f", s.HostLoad1)
return
}
func sustainedCells(r *resourceSamplingReport) (rss, cpu, fds, samples string) {
if r == nil {
return "-", "-", "-", "-"
}
if r.PeakProcessRSSBytes > 0 {
rss = fmt.Sprintf("%.1f MiB", float64(r.PeakProcessRSSBytes)/(1024*1024))
} else {
rss = "-"
}
cpu = fmt.Sprintf("%.1f%%", r.PeakProcessCPUPercent)
if r.PeakProcessOpenFDs > 0 {
fds = strconv.FormatInt(r.PeakProcessOpenFDs, 10)
} else {
fds = "-"
}
samples = strconv.Itoa(len(r.Samples))
return
}
cmd/ha-report/main_test.go
+207
View File
@@ -0,0 +1,207 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
func TestBuildReportReconcilesWarmupAndMeasuredRequests(t *testing.T) {
d := t.TempDir()
s := benchSummary{Concurrency: 4, Requests: 20, WarmupRequests: 3, Successful: 20, ThroughputRPS: 123.4, Latency: durationStats{P95: 8}, TTFB: durationStats{P95: 3}}
b, _ := json.Marshal(s)
if err := os.WriteFile(filepath.Join(d, "concurrency-4.json"), b, 0o600); err != nil {
t.Fatal(err)
}
before := `ollama_gateway_requests_total{api="openai",status_class="2xx"} 10
ollama_gateway_queue_seconds_count 10
ollama_gateway_service_seconds_count 10
ollama_gateway_prompt_tokens_total 100
ollama_gateway_completion_tokens_total 50
`
after := `ollama_gateway_requests_total{api="openai",status_class="2xx"} 33
ollama_gateway_queue_seconds_count 33
ollama_gateway_service_seconds_count 33
ollama_gateway_prompt_tokens_total 330
ollama_gateway_completion_tokens_total 165
ollama_gateway_retries_total{worker="w"} 0
`
if err := os.WriteFile(filepath.Join(d, "metrics-before-c4.prom"), []byte(before), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(d, "metrics-after-c4.prom"), []byte(after), 0o600); err != nil {
t.Fatal(err)
}
r, err := buildReport(d)
if err != nil {
t.Fatal(err)
}
if !r.EvidenceComplete || len(r.Levels) != 1 {
t.Fatalf("unexpected report: %+v", r)
}
l := r.Levels[0]
if l.GatewayRequestsDelta != 23 || !l.AccountingMatches {
t.Fatalf("unexpected level: %+v", l)
}
if r.GateStatus != "not-proven" {
t.Fatalf("gate=%q", r.GateStatus)
}
if got := renderMarkdown(r); !strings.Contains(got, "| 4 | 20/0 |") {
t.Fatalf("markdown missing row: %s", got)
}
}
func TestBuildReportMarksMissingMetricsIncomplete(t *testing.T) {
d := t.TempDir()
b, _ := json.Marshal(benchSummary{Concurrency: 1, Requests: 1, Successful: 1})
if err := os.WriteFile(filepath.Join(d, "concurrency-1.json"), b, 0o600); err != nil {
t.Fatal(err)
}
r, err := buildReport(d)
if err != nil {
t.Fatal(err)
}
if r.EvidenceComplete || r.GateStatus != "incomplete" {
t.Fatalf("unexpected report: %+v", r)
}
}
func TestPrometheusParserUsesFullOllamaGatewayNames(t *testing.T) {
p := filepath.Join(t.TempDir(), "m.prom")
content := "# HELP x x\nollama_gateway_requests_total{api=\"openai\",status_class=\"2xx\"} 7\n"
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
xs, err := parsePrometheusFile(p)
if err != nil {
t.Fatal(err)
}
if got := deltaByName(nil, xs, "ollama_gateway_requests_total"); got != 7 {
t.Fatalf("got %v", got)
}
if got := deltaByName(nil, xs, "gateway_requests_total"); got != 0 {
t.Fatalf("short metric name unexpectedly matched: %v", got)
}
}
func TestBuildReportIncludesCompleteResourceSnapshots(t *testing.T) {
d := t.TempDir()
s := benchSummary{Concurrency: 8, Requests: 5, WarmupRequests: 2, Successful: 5, ThroughputRPS: 50, Latency: durationStats{P95: 10}, TTFB: durationStats{P95: 4}}
b, _ := json.Marshal(s)
if err := os.WriteFile(filepath.Join(d, "concurrency-8.json"), b, 0o600); err != nil {
t.Fatal(err)
}
beforeMetrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 10\nollama_gateway_queue_seconds_count 10\nollama_gateway_service_seconds_count 10\n"
afterMetrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 17\nollama_gateway_queue_seconds_count 17\nollama_gateway_service_seconds_count 17\n"
if err := os.WriteFile(filepath.Join(d, "metrics-before-c8.prom"), []byte(beforeMetrics), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(d, "metrics-after-c8.prom"), []byte(afterMetrics), 0o600); err != nil {
t.Fatal(err)
}
before := resourceSnapshot{PID: 4242, ProcessRSSBytes: 64 * 1024 * 1024, ProcessCPUPercent: 2.5, ProcessThreads: 8, ProcessOpenFDs: 21, HostLogicalCPUs: 8, HostLoad1: 0.5}
after := resourceSnapshot{PID: 4242, ProcessRSSBytes: 96 * 1024 * 1024, ProcessCPUPercent: 33.3, ProcessThreads: 12, ProcessOpenFDs: 40, HostLogicalCPUs: 8, HostLoad1: 1.25}
for name, snap := range map[string]resourceSnapshot{"resources-before-c8.json": before, "resources-after-c8.json": after} {
data, _ := json.Marshal(snap)
if err := os.WriteFile(filepath.Join(d, name), data, 0o600); err != nil {
t.Fatal(err)
}
}
r, err := buildReport(d)
if err != nil {
t.Fatal(err)
}
if !r.EvidenceComplete || !r.ResourceEvidenceComplete {
t.Fatalf("unexpected completeness: %+v", r)
}
if r.Levels[0].ResourcesAfter == nil || r.Levels[0].ResourcesAfter.ProcessOpenFDs != 40 {
t.Fatalf("resource snapshot missing: %+v", r.Levels[0])
}
md := renderMarkdown(r)
if !strings.Contains(md, "96.0 MiB") || !strings.Contains(md, "Sustained resource sampling complete: **false**") || !strings.Contains(md, "Before/after resource files remain endpoint evidence") {
t.Fatalf("resource evidence missing from markdown: %s", md)
}
}
func TestBuildReportMarksPartialResourceSnapshotsIncompleteWithoutBreakingAccounting(t *testing.T) {
d := t.TempDir()
b, _ := json.Marshal(benchSummary{Concurrency: 2, Requests: 1, Successful: 1})
if err := os.WriteFile(filepath.Join(d, "concurrency-2.json"), b, 0o600); err != nil {
t.Fatal(err)
}
metrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 1\nollama_gateway_queue_seconds_count 1\nollama_gateway_service_seconds_count 1\n"
if err := os.WriteFile(filepath.Join(d, "metrics-before-c2.prom"), []byte("ollama_gateway_requests_total{status_class=\"2xx\"} 0\nollama_gateway_queue_seconds_count 0\nollama_gateway_service_seconds_count 0\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(d, "metrics-after-c2.prom"), []byte(metrics), 0o600); err != nil {
t.Fatal(err)
}
data, _ := json.Marshal(resourceSnapshot{PID: 99, HostLogicalCPUs: 4})
if err := os.WriteFile(filepath.Join(d, "resources-before-c2.json"), data, 0o600); err != nil {
t.Fatal(err)
}
r, err := buildReport(d)
if err != nil {
t.Fatal(err)
}
if !r.EvidenceComplete || r.ResourceEvidenceComplete || r.GateStatus != "not-proven" {
t.Fatalf("unexpected report: %+v", r)
}
if !strings.Contains(r.GateReason, "resource evidence is incomplete") {
t.Fatalf("unexpected gate reason: %s", r.GateReason)
}
}
func TestBuildReportIncludesSustainedResourceSampling(t *testing.T) {
d := t.TempDir()
s := benchSummary{Concurrency: 16, Requests: 10, WarmupRequests: 2, Successful: 10, ThroughputRPS: 75, Latency: durationStats{P95: 12}, TTFB: durationStats{P95: 5}}
b, _ := json.Marshal(s)
if err := os.WriteFile(filepath.Join(d, "concurrency-16.json"), b, 0o600); err != nil {
t.Fatal(err)
}
beforeMetrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 4\nollama_gateway_queue_seconds_count 4\nollama_gateway_service_seconds_count 4\n"
afterMetrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 16\nollama_gateway_queue_seconds_count 16\nollama_gateway_service_seconds_count 16\n"
if err := os.WriteFile(filepath.Join(d, "metrics-before-c16.prom"), []byte(beforeMetrics), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(d, "metrics-after-c16.prom"), []byte(afterMetrics), 0o600); err != nil {
t.Fatal(err)
}
for name, snap := range map[string]resourceSnapshot{
"resources-before-c16.json": {PID: 77, ProcessRSSBytes: 50 * 1024 * 1024, HostLogicalCPUs: 8},
"resources-after-c16.json": {PID: 77, ProcessRSSBytes: 60 * 1024 * 1024, HostLogicalCPUs: 8},
} {
data, _ := json.Marshal(snap)
if err := os.WriteFile(filepath.Join(d, name), data, 0o600); err != nil {
t.Fatal(err)
}
}
samples := resourceSamplingReport{
Version: 1, PID: 77, GOOS: "linux", PeakProcessRSSBytes: 80 * 1024 * 1024,
PeakProcessCPUPercent: 145.5, PeakProcessOpenFDs: 55,
Samples: []resourceSnapshot{{PID: 77}, {PID: 77}}, StopReason: "stop-file",
}
data, _ := json.Marshal(samples)
if err := os.WriteFile(filepath.Join(d, "resources-samples-c16.json"), data, 0o600); err != nil {
t.Fatal(err)
}
r, err := buildReport(d)
if err != nil {
t.Fatal(err)
}
if !r.EvidenceComplete || !r.ResourceEvidenceComplete || !r.SustainedResourceEvidenceComplete {
t.Fatalf("unexpected completeness: %+v", r)
}
if r.Levels[0].ResourceSamples == nil || r.Levels[0].ResourceSamples.PeakProcessOpenFDs != 55 {
t.Fatalf("missing sustained samples: %+v", r.Levels[0])
}
if !strings.Contains(r.GateReason, "sustained host/process resource sampling is complete") {
t.Fatalf("unexpected gate reason: %s", r.GateReason)
}
md := renderMarkdown(r)
if !strings.Contains(md, "80.0 MiB") || !strings.Contains(md, "145.5%") || !strings.Contains(md, "| 16 | 10/0 |") {
t.Fatalf("sustained evidence missing from markdown: %s", md)
}
}
cmd/ha-sampler/main.go
+50
View File
@@ -0,0 +1,50 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"os"
"os/signal"
"syscall"
"time"
"github.com/example/ollama-fair-gateway/internal/haresource"
)
func main() {
pid := flag.Int("pid", 0, "gateway process PID")
interval := flag.Duration("interval", 250*time.Millisecond, "resource sample interval (minimum 50ms)")
maxDuration := flag.Duration("max-duration", 15*time.Minute, "safety limit for one sampling run")
stopFile := flag.String("stop-file", "", "stop sampling after this file appears")
out := flag.String("json-out", "-", "output JSON path; '-' writes stdout")
flag.Parse()
if *pid <= 0 || *stopFile == "" {
fmt.Fprintln(os.Stderr, "-pid must be positive and -stop-file is required")
os.Exit(2)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
r, err := haresource.Sample(ctx, *pid, *interval, *maxDuration, *stopFile)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
b, err := json.MarshalIndent(r, "", " ")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
b = append(b, '\n')
if *out == "-" {
_, _ = os.Stdout.Write(b)
} else if err := os.WriteFile(*out, b, 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if !r.Complete() {
fmt.Fprintln(os.Stderr, "sampling completed but sustained resource evidence is incomplete")
os.Exit(3)
}
}
cmd/ha-snapshot/main.go
+40
View File
@@ -0,0 +1,40 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"os"
"path/filepath"
"github.com/example/ollama-fair-gateway/internal/haresource"
)
func main() {
pid := flag.Int("pid", 0, "gateway process PID")
out := flag.String("json-out", "-", "output JSON path; '-' writes stdout")
flag.Parse()
if *pid <= 0 {
fmt.Fprintln(os.Stderr, "-pid must be positive")
os.Exit(2)
}
s := haresource.Collect(*pid)
b, err := json.MarshalIndent(s, "", " ")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
b = append(b, '\n')
if *out == "-" {
_, _ = os.Stdout.Write(b)
return
}
if err := os.WriteFile(*out, b, 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func outputName(dir string, concurrency int, phase string) string {
return filepath.Join(dir, fmt.Sprintf("resources-%s-c%d.json", phase, concurrency))
}
cmd/ha-snapshot/main_test.go
+10
View File
@@ -0,0 +1,10 @@
package main
import "testing"
func TestOutputName(t *testing.T) {
got := outputName("out", 8, "before")
if got != "out/resources-before-c8.json" {
t.Fatalf("outputName=%q", got)
}
}
cmd/mock-ollama/main.go
+191
View File
@@ -0,0 +1,191 @@
package main
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"strings"
"sync/atomic"
"time"
)
type server struct {
model string
delay time.Duration
streamDelay time.Duration
responseBytes int
promptTokens int64
outputTokens int64
failEvery int64
requests atomic.Int64
}
func main() {
listen := flag.String("listen", "127.0.0.1:11435", "listen address")
model := flag.String("model", "qwen3:8b", "mock model name")
delay := flag.Duration("delay", 0, "delay before response headers")
streamDelay := flag.Duration("stream-delay", 0, "delay between streaming chunks")
responseBytes := flag.Int("response-bytes", 128, "approximate generated content bytes")
promptTokens := flag.Int64("prompt-tokens", 16, "reported prompt token count")
outputTokens := flag.Int64("output-tokens", 8, "reported completion token count")
failEvery := flag.Int64("fail-every", 0, "return HTTP 503 for every Nth inference request; 0 disables")
flag.Parse()
if *responseBytes < 0 || *promptTokens < 0 || *outputTokens < 0 || *failEvery < 0 {
log.Fatal("numeric flags must be non-negative")
}
s := &server{model: *model, delay: *delay, streamDelay: *streamDelay, responseBytes: *responseBytes, promptTokens: *promptTokens, outputTokens: *outputTokens, failEvery: *failEvery}
mux := http.NewServeMux()
mux.HandleFunc("/api/ps", s.ps)
mux.HandleFunc("/api/tags", s.tags)
mux.HandleFunc("/api/show", s.show)
mux.HandleFunc("/api/chat", s.nativeChat)
mux.HandleFunc("/api/generate", s.nativeGenerate)
mux.HandleFunc("/v1/chat/completions", s.openAIChat)
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) })
h := &http.Server{Addr: *listen, Handler: mux, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 2 * time.Minute}
log.Printf("mock Ollama listening on http://%s model=%s delay=%s response_bytes=%d", *listen, *model, *delay, *responseBytes)
log.Fatal(h.ListenAndServe())
}
func (s *server) ps(w http.ResponseWriter, r *http.Request) {
writeJSON(w, map[string]any{"models": []map[string]any{{"name": s.model, "model": s.model, "size": 1 << 30, "size_vram": 1 << 30, "context_length": 32768}}})
}
func (s *server) tags(w http.ResponseWriter, r *http.Request) {
writeJSON(w, map[string]any{"models": []map[string]any{{"name": s.model, "model": s.model, "size": 1 << 30, "details": map[string]any{"parameter_size": "8B", "quantization_level": "Q4_K_M"}}}})
}
func (s *server) show(w http.ResponseWriter, r *http.Request) {
writeJSON(w, map[string]any{"capabilities": []string{"completion", "tools", "thinking", "vision"}, "model_info": map[string]any{"mock.context_length": 32768}})
}
func (s *server) shouldFail() bool {
n := s.requests.Add(1)
return s.failEvery > 0 && n%s.failEvery == 0
}
func (s *server) openAIChat(w http.ResponseWriter, r *http.Request) {
if s.delay > 0 {
time.Sleep(s.delay)
}
if s.shouldFail() {
writeJSONStatus(w, http.StatusServiceUnavailable, map[string]any{"error": map[string]any{"message": "mock failure", "type": "server_error"}})
return
}
var req struct {
Stream bool `json:"stream"`
}
_ = json.NewDecoder(io.LimitReader(r.Body, 8<<20)).Decode(&req)
content := strings.Repeat("x", s.responseBytes)
if req.Stream {
w.Header().Set("Content-Type", "text/event-stream")
w.WriteHeader(http.StatusOK)
f, _ := w.(http.Flusher)
parts := split(content, 4)
for i, p := range parts {
chunk := map[string]any{"id": "chatcmpl-mock", "object": "chat.completion.chunk", "choices": []map[string]any{{"index": 0, "delta": map[string]any{"content": p}, "finish_reason": nil}}}
if i == len(parts)-1 {
chunk["choices"] = []map[string]any{{"index": 0, "delta": map[string]any{}, "finish_reason": "stop"}}
}
b, _ := json.Marshal(chunk)
fmt.Fprintf(w, "data: %s\n\n", b)
if f != nil {
f.Flush()
}
if s.streamDelay > 0 {
time.Sleep(s.streamDelay)
}
}
io.WriteString(w, "data: [DONE]\n\n")
if f != nil {
f.Flush()
}
return
}
writeJSON(w, map[string]any{
"id": "chatcmpl-mock", "object": "chat.completion", "created": time.Now().Unix(), "model": s.model,
"choices": []map[string]any{{"index": 0, "message": map[string]any{"role": "assistant", "content": content}, "finish_reason": "stop"}},
"usage": map[string]any{"prompt_tokens": s.promptTokens, "completion_tokens": s.outputTokens, "total_tokens": s.promptTokens + s.outputTokens},
})
}
func (s *server) nativeChat(w http.ResponseWriter, r *http.Request) {
s.native(w, r, "message")
}
func (s *server) nativeGenerate(w http.ResponseWriter, r *http.Request) {
s.native(w, r, "response")
}
func (s *server) native(w http.ResponseWriter, r *http.Request, field string) {
if s.delay > 0 {
time.Sleep(s.delay)
}
if s.shouldFail() {
writeJSONStatus(w, http.StatusServiceUnavailable, map[string]any{"error": "mock failure"})
return
}
var req map[string]any
_ = json.NewDecoder(io.LimitReader(r.Body, 8<<20)).Decode(&req)
stream, _ := req["stream"].(bool)
content := strings.Repeat("x", s.responseBytes)
w.Header().Set("Content-Type", "application/x-ndjson")
bw := bufio.NewWriter(w)
if !stream {
doc := map[string]any{"model": s.model, "done": true, "prompt_eval_count": s.promptTokens, "eval_count": s.outputTokens}
if field == "message" {
doc[field] = map[string]any{"role": "assistant", "content": content}
} else {
doc[field] = content
}
_ = json.NewEncoder(bw).Encode(doc)
_ = bw.Flush()
return
}
f, _ := w.(http.Flusher)
for _, p := range split(content, 4) {
doc := map[string]any{"model": s.model, "done": false}
if field == "message" {
doc[field] = map[string]any{"role": "assistant", "content": p}
} else {
doc[field] = p
}
_ = json.NewEncoder(bw).Encode(doc)
_ = bw.Flush()
if f != nil {
f.Flush()
}
if s.streamDelay > 0 {
time.Sleep(s.streamDelay)
}
}
_ = json.NewEncoder(bw).Encode(map[string]any{"model": s.model, "done": true, "prompt_eval_count": s.promptTokens, "eval_count": s.outputTokens})
_ = bw.Flush()
if f != nil {
f.Flush()
}
}
func split(s string, n int) []string {
if n <= 1 || len(s) == 0 {
return []string{s}
}
out := make([]string, 0, n)
for i := 0; i < n; i++ {
start := len(s) * i / n
end := len(s) * (i + 1) / n
out = append(out, s[start:end])
}
return out
}
func writeJSON(w http.ResponseWriter, v any) { writeJSONStatus(w, http.StatusOK, v) }
func writeJSONStatus(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(v)
}
cmd/mock-ollama/main_test.go
+108
View File
@@ -0,0 +1,108 @@
package main
import (
"bufio"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
)
func TestOpenAIChatNonStreaming(t *testing.T) {
s := &server{model: "mock:latest", responseBytes: 12, promptTokens: 7, outputTokens: 3}
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"stream":false}`))
rr := httptest.NewRecorder()
s.openAIChat(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
}
var doc struct {
Model string `json:"model"`
Choices []struct {
Message struct {
Content string `json:"content"`
} `json:"message"`
} `json:"choices"`
Usage struct {
Prompt int64 `json:"prompt_tokens"`
Output int64 `json:"completion_tokens"`
} `json:"usage"`
}
if err := json.Unmarshal(rr.Body.Bytes(), &doc); err != nil {
t.Fatal(err)
}
if doc.Model != "mock:latest" || len(doc.Choices) != 1 || len(doc.Choices[0].Message.Content) != 12 {
t.Fatalf("unexpected response: %+v", doc)
}
if doc.Usage.Prompt != 7 || doc.Usage.Output != 3 {
t.Fatalf("unexpected usage: %+v", doc.Usage)
}
}
func TestOpenAIChatStreaming(t *testing.T) {
s := &server{model: "mock:latest", responseBytes: 16}
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"stream":true}`))
rr := httptest.NewRecorder()
s.openAIChat(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
}
if ct := rr.Header().Get("Content-Type"); !strings.Contains(ct, "text/event-stream") {
t.Fatalf("content-type=%q", ct)
}
if !strings.Contains(rr.Body.String(), "data: [DONE]") {
t.Fatalf("missing done marker: %s", rr.Body.String())
}
}
func TestNativeStreamingEndsWithUsage(t *testing.T) {
s := &server{model: "mock:latest", responseBytes: 8, promptTokens: 11, outputTokens: 5}
req := httptest.NewRequest(http.MethodPost, "/api/chat", strings.NewReader(`{"stream":true}`))
rr := httptest.NewRecorder()
s.nativeChat(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
}
scanner := bufio.NewScanner(strings.NewReader(rr.Body.String()))
var last map[string]any
for scanner.Scan() {
if err := json.Unmarshal(scanner.Bytes(), &last); err != nil {
t.Fatal(err)
}
}
if err := scanner.Err(); err != nil {
t.Fatal(err)
}
if done, _ := last["done"].(bool); !done {
t.Fatalf("last chunk not done: %#v", last)
}
if last["prompt_eval_count"] != float64(11) || last["eval_count"] != float64(5) {
t.Fatalf("unexpected usage: %#v", last)
}
}
func TestShouldFailConcurrent(t *testing.T) {
const total = 1000
s := &server{failEvery: 5}
var wg sync.WaitGroup
failures := make(chan struct{}, total)
for i := 0; i < total; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if s.shouldFail() {
failures <- struct{}{}
}
}()
}
wg.Wait()
close(failures)
if got, want := len(failures), total/5; got != want {
t.Fatalf("failures=%d want=%d", got, want)
}
if got := s.requests.Load(); got != total {
t.Fatalf("requests=%d want=%d", got, total)
}
}
cmd/ollama-gateway/main.go
+255
View File
@@ -0,0 +1,255 @@
package main
import (
"context"
"flag"
"log/slog"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"github.com/example/ollama-fair-gateway/internal/alerts"
"github.com/example/ollama-fair-gateway/internal/auth"
"github.com/example/ollama-fair-gateway/internal/autotune"
"github.com/example/ollama-fair-gateway/internal/batch"
"github.com/example/ollama-fair-gateway/internal/conversation"
"github.com/example/ollama-fair-gateway/internal/cost"
"github.com/example/ollama-fair-gateway/internal/infrastructure"
"github.com/example/ollama-fair-gateway/internal/liveflow"
"github.com/example/ollama-fair-gateway/internal/metrics"
"github.com/example/ollama-fair-gateway/internal/proxy"
"github.com/example/ollama-fair-gateway/internal/quota"
"github.com/example/ollama-fair-gateway/internal/scheduler"
"github.com/example/ollama-fair-gateway/internal/server"
"github.com/example/ollama-fair-gateway/internal/session"
"github.com/example/ollama-fair-gateway/internal/state"
"github.com/example/ollama-fair-gateway/internal/telemetry"
"github.com/example/ollama-fair-gateway/internal/usage"
"github.com/example/ollama-fair-gateway/internal/warm"
"github.com/example/ollama-fair-gateway/internal/worker"
)
func main() {
configPath := flag.String("config", "config.json", "configuration file")
checkConfig := flag.Bool("check-config", false, "validate effective configuration and storage, then exit")
probeURL := flag.String("probe", "", "probe an HTTP health/readiness URL, then exit")
probeTimeout := flag.Duration("probe-timeout", 2*time.Second, "timeout for -probe")
flag.Parse()
log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
slog.SetDefault(log)
if *probeURL != "" {
if err := runProbe(*probeURL, *probeTimeout); err != nil {
log.Error("probe failed", "url", *probeURL, "error", err)
os.Exit(1)
}
return
}
if *checkConfig {
if err := runConfigCheck(*configPath, os.Stdout); err != nil {
log.Error("configuration preflight failed", "error", err)
os.Exit(2)
}
return
}
loaded, err := loadEffectiveConfig(*configPath)
if err != nil {
log.Error("configuration error", "error", err)
os.Exit(2)
}
cfg := loaded.Config
paths := loaded.Paths
configStore := loaded.Store
if loaded.Persistent {
log.Info("loaded persistent UI configuration", "path", configStore.Path())
}
rootCtx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
apiKeyStore, err := state.NewAPIKeyStore(paths.APIKeys)
if err != nil {
log.Error("API key store initialization failed", "error", err)
os.Exit(2)
}
authenticator, err := auth.NewWithRuntimeStore(rootCtx, cfg.Auth, apiKeyStore)
if err != nil {
log.Error("authentication initialization failed", "error", err)
os.Exit(2)
}
sched := scheduler.NewLocal(cfg.Scheduler.GlobalConcurrency, cfg.Scheduler.MaxQueue, cfg.Scheduler.MaxQueuePerActor)
var ledger quota.Ledger = quota.Disabled{}
var quotaMemory *quota.Memory
if cfg.Quota.Enabled {
quotaMemory = quota.NewMemory()
if err := quotaMemory.LoadPersistent(paths.Quota); err != nil {
log.Error("quota state initialization failed", "error", err)
os.Exit(2)
}
quotaMemory.StartPersistence(rootCtx, paths.Quota, cfg.Storage.FlushInterval.Value(), func(err error) { log.Error("quota persistence failed", "error", err) })
ledger = quotaMemory
}
policyStore, err := state.NewPolicyStore(paths.Policies)
if err != nil {
log.Error("policy store initialization failed", "error", err)
os.Exit(2)
}
placementStore, err := state.NewModelPlacementStore(paths.ModelPlacement)
if err != nil {
log.Error("model placement store initialization failed", "error", err)
os.Exit(2)
}
workerStateStore, err := state.NewWorkerRuntimeStore(paths.WorkerState)
if err != nil {
log.Error("worker state store initialization failed", "error", err)
os.Exit(2)
}
uiSessions := session.NewMemory()
conversationStore, err := conversation.New(cfg.Conversations, paths.Conversations)
if err != nil {
log.Error("conversation store initialization failed", "error", err)
os.Exit(2)
}
conversationStore.StartCleanup(rootCtx, func(err error) { log.Error("conversation retention cleanup failed", "error", err) })
batchManager, err := batch.New(cfg.BatchJobs, paths.BatchJobs, paths.BatchDir)
if err != nil {
log.Error("batch job manager initialization failed", "error", err)
os.Exit(2)
}
pool := worker.New(cfg.Workers, cfg.Native.ControlWorker)
pool.SetRoutingConfig(cfg.Routing)
pool.SetModelCapabilitiesConfig(cfg.ModelCapabilities)
pool.SetReliabilityConfig(cfg.Reliability)
if overrides, err := placementStore.List(rootCtx); err != nil {
log.Error("model placement overrides load failed", "error", err)
os.Exit(2)
} else {
for workerName, rule := range overrides {
if err := pool.SetPlacement(workerName, rule, true); err != nil {
// Keep orphaned rules durable when a worker is temporarily removed;
// they become active again if that worker name returns later.
log.Warn("ignoring model placement override for unknown worker", "worker", workerName, "error", err)
}
}
}
if modes, err := workerStateStore.List(rootCtx); err != nil {
log.Error("worker runtime state load failed", "error", err)
os.Exit(2)
} else {
for name, mode := range modes {
if err := pool.SetMaintenance(name, mode); err != nil {
log.Warn("ignoring worker state for unknown worker", "worker", name, "error", err)
}
}
}
if err := pool.LoadPerformance(paths.WorkerPerformance); err != nil {
log.Error("worker performance state initialization failed", "error", err)
os.Exit(2)
}
pool.StartPerformancePersistence(rootCtx, paths.WorkerPerformance, cfg.Storage.FlushInterval.Value(), func(err error) { log.Error("worker performance persistence failed", "error", err) })
tuner, err := autotune.New(cfg.AutoTuning, pool, paths.AutoTune)
if err != nil {
log.Error("auto tuning state initialization failed", "error", err)
os.Exit(2)
}
for workerName, models := range tuner.Applied() {
for model, limit := range models {
if err := pool.SetModelConcurrency(workerName, model, limit); err != nil {
log.Warn("ignoring auto-tune override", "worker", workerName, "model", model, "error", err)
}
}
}
pool.Start(rootCtx)
warmManager, err := warm.New(cfg.WarmModels, pool, paths.WarmModels)
if err != nil {
log.Error("warm model manager initialization failed", "error", err)
os.Exit(2)
}
warmManager.Start(rootCtx)
alertManager, err := alerts.New(cfg.Alerts, paths.Alerts, func() alerts.Snapshot {
st := sched.Stats(context.Background())
ws := pool.Snapshots()
aw := make([]alerts.Worker, 0, len(ws))
for _, w := range ws {
aw = append(aw, alerts.Worker{Name: w.Name, Healthy: w.Healthy, CircuitState: w.CircuitState, LastCircuitError: w.LastCircuitError, VRAMUsedBytes: w.VRAMUsedBytes, VRAMTotalBytes: w.VRAMTotalBytes})
}
return alerts.Snapshot{QueueDepth: int(st.Queued), QueueWait: st.OldestWait, Workers: aw, StorageBytes: alerts.DirSize(paths.DataDir)}
})
if err != nil {
log.Error("alerts manager initialization failed", "error", err)
os.Exit(2)
}
alertManager.Start(rootCtx)
live := liveflow.New(10*time.Second, max(512, cfg.Infrastructure.MaxRequests))
infra := infrastructure.New(cfg.Infrastructure, live, sched, pool)
infra.Start(rootCtx)
met := metrics.New()
if err := met.LoadPersistent(paths.Metrics); err != nil {
log.Error("metrics state initialization failed", "error", err)
os.Exit(2)
}
met.StartPersistence(rootCtx, paths.Metrics, cfg.Storage.FlushInterval.Value(), func(err error) { log.Error("metrics persistence failed", "error", err) })
rec, err := usage.NewWithRetention(cfg.Usage.JournalDir, cfg.Usage.Buffer, cfg.Usage.FlushInterval.Value(), usage.RetentionConfig{DetailDays: cfg.Usage.Retention.DetailDays, DailyDays: cfg.Usage.Retention.DailyDays, MonthlyMonths: cfg.Usage.Retention.MonthlyMonths, CompactionInterval: cfg.Usage.Retention.CompactionInterval.Value()}, met.DropUsage)
if err != nil {
log.Error("usage recorder initialization failed", "error", err)
os.Exit(2)
}
rec.SetRecentCapacity(cfg.UI.RecentEvents)
defer rec.Close()
otelExporter := telemetry.New(cfg.OpenTelemetry)
if otelExporter != nil {
defer func() {
cctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := otelExporter.Close(cctx); err != nil {
log.Warn("OpenTelemetry shutdown", "error", err)
}
}()
}
srvHandler := server.New(cfg, server.Dependencies{Auth: authenticator, Scheduler: sched, Quota: ledger, Estimator: cost.New(cfg.Cost), Workers: pool, Proxy: proxy.New(), Usage: rec, Metrics: met, Policies: policyStore, Sessions: uiSessions, Live: live, Infrastructure: infra, Logger: log, ConfigStore: configStore, PlacementStore: placementStore, WorkerStateStore: workerStateStore, AutoTune: tuner, OpenTelemetry: otelExporter, WarmModels: warmManager, Alerts: alertManager, Conversations: conversationStore, BatchJobs: batchManager})
batchManager.Start(rootCtx, srvHandler.ExecuteBatch)
hs := &http.Server{Addr: cfg.Server.Listen, Handler: srvHandler.Handler(), ReadHeaderTimeout: cfg.Server.ReadHeaderTimeout.Value(), IdleTimeout: cfg.Server.IdleTimeout.Value(), MaxHeaderBytes: 1 << 20}
go func() {
<-rootCtx.Done()
ctx, c := context.WithTimeout(context.Background(), 15*time.Second)
defer c()
_ = hs.Shutdown(ctx)
}()
log.Info("ollama fair gateway starting", "listen", cfg.Server.Listen, "workers", len(cfg.Workers), "coordination", "in-memory", "node", infra.NodeName(), "quota", cfg.Quota.Enabled, "storage", cfg.Storage.DataDir)
if cfg.Server.TLSCert != "" || cfg.Server.TLSKey != "" {
err = hs.ListenAndServeTLS(cfg.Server.TLSCert, cfg.Server.TLSKey)
} else {
err = hs.ListenAndServe()
}
if err != nil && err != http.ErrServerClosed {
log.Error("server failed", "error", err)
os.Exit(1)
}
if warmManager != nil {
wctx, wcancel := context.WithTimeout(context.Background(), 5*time.Second)
if err := warmManager.Wait(wctx); err != nil {
log.Warn("warm model actions still active during shutdown", "error", err)
}
wcancel()
}
if batchManager != nil {
bctx, bcancel := context.WithTimeout(context.Background(), 10*time.Second)
if err := batchManager.Wait(bctx); err != nil {
log.Warn("batch attempts still active during shutdown", "error", err)
}
bcancel()
}
rec.Close()
if quotaMemory != nil {
if err := quotaMemory.SavePersistent(paths.Quota); err != nil {
log.Error("final quota persistence failed", "error", err)
}
}
if err := met.SavePersistent(paths.Metrics); err != nil {
log.Error("final metrics persistence failed", "error", err)
}
if err := pool.SavePerformance(paths.WorkerPerformance); err != nil {
log.Error("final worker performance persistence failed", "error", err)
}
log.Info("gateway stopped")
}
cmd/ollama-gateway/preflight.go
+222
View File
@@ -0,0 +1,222 @@
package main
import (
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
"github.com/example/ollama-fair-gateway/internal/state"
)
type effectiveConfig struct {
Bootstrap *config.Config
Config *config.Config
Paths state.Paths
Store *state.ConfigStore
Persistent bool
}
func loadEffectiveConfig(configPath string) (*effectiveConfig, error) {
bootstrapCfg, err := config.Load(configPath)
if err != nil {
return nil, fmt.Errorf("configuration error: %w", err)
}
paths := state.Resolve(bootstrapCfg.Storage)
configStore := state.NewConfigStore(paths.Config)
cfg := bootstrapCfg
persistentCfg, ok, err := configStore.LoadIfExists(bootstrapCfg)
if err != nil {
return nil, fmt.Errorf("persistent configuration error (%s): %w", configStore.Path(), err)
}
if ok {
if persistentCfg.Storage != bootstrapCfg.Storage {
return nil, fmt.Errorf("persistent configuration changes bootstrap-only storage settings: bootstrap=%+v persistent=%+v", bootstrapCfg.Storage, persistentCfg.Storage)
}
cfg = persistentCfg
}
return &effectiveConfig{
Bootstrap: bootstrapCfg,
Config: cfg,
Paths: state.Resolve(cfg.Storage),
Store: configStore,
Persistent: ok,
}, nil
}
type configCheckResult struct {
Status string `json:"status"`
ConfigPath string `json:"config_path"`
PersistentOverride bool `json:"persistent_override"`
PersistentPath string `json:"persistent_path"`
Workers int `json:"workers"`
DataDir string `json:"data_dir"`
DataDirAbsolute string `json:"data_dir_absolute"`
StorageWritable bool `json:"storage_writable"`
Warnings []string `json:"warnings,omitempty"`
}
func runConfigCheck(configPath string, out io.Writer) error {
loaded, err := loadEffectiveConfig(configPath)
if err != nil {
return err
}
if err := checkWritableDir(loaded.Paths.DataDir); err != nil {
return fmt.Errorf("storage data_dir %q is not writable: %w", loaded.Paths.DataDir, err)
}
absDir, err := filepath.Abs(loaded.Paths.DataDir)
if err != nil {
absDir = loaded.Paths.DataDir
}
res := configCheckResult{
Status: "ok",
ConfigPath: configPath,
PersistentOverride: loaded.Persistent,
PersistentPath: loaded.Store.Path(),
Workers: len(loaded.Config.Workers),
DataDir: loaded.Paths.DataDir,
DataDirAbsolute: absDir,
StorageWritable: true,
Warnings: configWarnings(loaded.Config),
}
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
return enc.Encode(res)
}
func checkWritableDir(dir string) error {
if err := os.MkdirAll(dir, 0700); err != nil {
return err
}
f, err := os.CreateTemp(dir, ".gateway-preflight-*")
if err != nil {
return err
}
name := f.Name()
defer os.Remove(name)
if err := f.Chmod(0600); err != nil {
_ = f.Close()
return err
}
if _, err := f.WriteString("ok\n"); err != nil {
_ = f.Close()
return err
}
if err := f.Sync(); err != nil {
_ = f.Close()
return err
}
return f.Close()
}
func configWarnings(cfg *config.Config) []string {
var warnings []string
if control := strings.TrimSpace(cfg.Native.ControlWorker); control != "" {
found := false
for _, w := range cfg.Workers {
if w.Name == control {
found = true
break
}
}
if !found {
warnings = append(warnings, fmt.Sprintf("native.control_worker %q does not match any configured worker; management requests will fall back to another healthy worker", control))
}
}
for _, w := range cfg.Workers {
if !w.LocalSystemStats {
continue
}
u, err := url.Parse(w.URL)
if err != nil {
continue
}
host := u.Hostname()
if host == "" || isLoopbackHost(host) {
continue
}
warnings = append(warnings, fmt.Sprintf("worker %q has local_system_stats=true but URL host %q is remote; local system stats describe the gateway host, not that worker (use telemetry_url or disable local_system_stats)", w.Name, host))
}
if cfg.Auth.IPBypassUseForwardedIP && len(cfg.Auth.IPBypass) > 0 {
warnings = append(warnings, "auth.ip_bypass_use_forwarded_ip=true allows X-Forwarded-For-derived addresses to satisfy credential-free IP bypass; use only with tightly restricted trusted_proxies and network ACLs")
}
for _, raw := range cfg.Auth.TrustedProxies {
if broadTrustedProxyCIDR(raw) {
warnings = append(warnings, fmt.Sprintf("auth.trusted_proxies contains broad CIDR %q; any directly reachable peer in that range can influence X-Forwarded-For-derived client_ip, so prefer exact proxy addresses", raw))
}
}
if cfg.UI.Enabled && !cfg.UI.OIDC.Enabled && len(cfg.Auth.APIKeys) == 0 {
warnings = append(warnings, "UI is enabled without UI OIDC and without bootstrap API keys; remote UI login depends entirely on IP bypass rules")
}
if cfg.UI.Enabled && !cfg.UI.SecureCookies && cfg.UI.OIDC.Enabled {
warnings = append(warnings, "ui.secure_cookies=false while UI OIDC is enabled; enable secure cookies when the browser reaches the gateway over HTTPS")
}
if cfg.Server.MetricsPublic {
warnings = append(warnings, "server.metrics_public=true exposes gateway metrics without authentication")
}
if cfg.ModelCapabilities.Context.MaxRequestedTokens == -1 {
warnings = append(warnings, "model_capabilities.context.max_requested_tokens=-1 removes the gateway-side context cap; large native num_ctx requests can cause substantial KV-cache/VRAM pressure")
}
if cfg.ModelCapabilities.Context.DefaultWorkerTokens == -1 {
warnings = append(warnings, "model_capabilities.context.default_worker_tokens=-1 falls back to the theoretical model maximum for unloaded models without Modelfile num_ctx; configure an explicit worker default for predictable memory use")
}
if !filepath.IsAbs(cfg.Storage.DataDir) {
warnings = append(warnings, fmt.Sprintf("storage.data_dir %q is relative and therefore depends on the process working directory", cfg.Storage.DataDir))
}
return warnings
}
func broadTrustedProxyCIDR(raw string) bool {
ip, n, err := net.ParseCIDR(strings.TrimSpace(raw))
if err != nil || ip == nil || n == nil || ip.IsLoopback() {
return false
}
ones, bits := n.Mask.Size()
if bits == 32 {
return ones < 24
}
if bits == 128 {
return ones < 64
}
return false
}
func isLoopbackHost(host string) bool {
if strings.EqualFold(host, "localhost") {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
func runProbe(rawURL string, timeout time.Duration) error {
if timeout <= 0 {
return fmt.Errorf("probe timeout must be > 0")
}
u, err := url.Parse(rawURL)
if err != nil || u.Scheme == "" || u.Host == "" {
return fmt.Errorf("invalid probe URL %q", rawURL)
}
client := &http.Client{Timeout: timeout}
req, err := http.NewRequest(http.MethodGet, rawURL, nil)
if err != nil {
return err
}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("probe returned HTTP %d", resp.StatusCode)
}
return nil
}
cmd/ollama-gateway/preflight_test.go
+154
View File
@@ -0,0 +1,154 @@
package main
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
"github.com/example/ollama-fair-gateway/internal/state"
)
func writeBootstrapConfig(t *testing.T, dataDir string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "config.json")
body := `{
"auth":{"api_keys":[{"name":"admin","key":"01234567890123456789012345678901","tenant":"t","subject":"s","scopes":["gateway:admin"]}]},
"workers":[{"name":"remote","url":"http://10.2.10.48:11434","local_system_stats":true}],
"ui":{"enabled":true,"path":"/admin","title":"Bootstrap"},
"storage":{"data_dir":` + quoteJSON(dataDir) + `,"config_file":"gateway-config.json"}
}`
if err := os.WriteFile(path, []byte(body), 0600); err != nil {
t.Fatal(err)
}
return path
}
func quoteJSON(s string) string {
b, _ := json.Marshal(s)
return string(b)
}
func TestLoadEffectiveConfigUsesPersistentOverrideButBootstrapSecrets(t *testing.T) {
dataDir := t.TempDir()
path := writeBootstrapConfig(t, dataDir)
base, err := config.Load(path)
if err != nil {
t.Fatal(err)
}
persistent := *base
persistent.UI.Title = "Persistent"
store := state.NewConfigStore(state.Resolve(base.Storage).Config)
if err := store.Save(&persistent); err != nil {
t.Fatal(err)
}
loaded, err := loadEffectiveConfig(path)
if err != nil {
t.Fatal(err)
}
if !loaded.Persistent || loaded.Config.UI.Title != "Persistent" {
t.Fatalf("persistent override not loaded: %#v", loaded)
}
if got := loaded.Config.Auth.APIKeys[0].Key; got != "01234567890123456789012345678901" {
t.Fatalf("bootstrap secret was not restored, got %q", got)
}
}
func TestRunConfigCheckReportsWritableStorageAndWarning(t *testing.T) {
dataDir := filepath.Join(t.TempDir(), "state")
path := writeBootstrapConfig(t, dataDir)
var out bytes.Buffer
if err := runConfigCheck(path, &out); err != nil {
t.Fatal(err)
}
var got configCheckResult
if err := json.Unmarshal(out.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got.Status != "ok" || !got.StorageWritable || got.Workers != 1 {
t.Fatalf("unexpected result: %+v", got)
}
if len(got.Warnings) == 0 || !strings.Contains(strings.Join(got.Warnings, "\n"), "local_system_stats=true") {
t.Fatalf("expected remote local-system-stats warning, got %#v", got.Warnings)
}
}
func TestRunProbe(t *testing.T) {
ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
defer ok.Close()
if err := runProbe(ok.URL, time.Second); err != nil {
t.Fatal(err)
}
bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "not ready", http.StatusServiceUnavailable)
}))
defer bad.Close()
if err := runProbe(bad.URL, time.Second); err == nil || !strings.Contains(err.Error(), "503") {
t.Fatalf("expected HTTP 503 probe error, got %v", err)
}
}
func TestComposeDoesNotRepeatEntrypointAndHasHealthcheck(t *testing.T) {
b, err := os.ReadFile("../../docker-compose.yml")
if err != nil {
t.Fatal(err)
}
s := string(b)
if strings.Contains(s, `command: ["/ollama-gateway"`) {
t.Fatal("compose command must not repeat the image ENTRYPOINT")
}
if !strings.Contains(s, `command: ["-config"`) {
t.Fatal("compose command must pass -config as an ENTRYPOINT argument")
}
if !strings.Contains(s, "healthcheck:") || !strings.Contains(s, "/healthz") {
t.Fatal("compose must define a liveness healthcheck")
}
}
func TestConfigWarningsFlagForwardedBypassAndBroadTrustedProxy(t *testing.T) {
cfg := &config.Config{
Auth: config.AuthConfig{
IPBypassUseForwardedIP: true,
TrustedProxies: []string{"10.0.0.0/8", "127.0.0.1/8"},
IPBypass: []config.IPBypassConfig{{
CIDRs: []string{"127.0.0.1/32"},
Tenant: "local",
}},
},
Storage: config.StorageConfig{DataDir: "/var/lib/ollama-gateway"},
}
warnings := strings.Join(configWarnings(cfg), "\n")
if !strings.Contains(warnings, "ip_bypass_use_forwarded_ip=true") {
t.Fatalf("missing forwarded bypass warning: %s", warnings)
}
if !strings.Contains(warnings, `broad CIDR "10.0.0.0/8"`) {
t.Fatalf("missing broad trusted proxy warning: %s", warnings)
}
if strings.Contains(warnings, `broad CIDR "127.0.0.1/8"`) {
t.Fatalf("loopback trusted proxy should not be flagged as broad: %s", warnings)
}
}
func TestConfigWarningsFlagUnlimitedContextPolicy(t *testing.T) {
cfg := &config.Config{
ModelCapabilities: config.ModelCapabilitiesConfig{Context: config.ContextPolicyConfig{MaxRequestedTokens: -1, DefaultWorkerTokens: -1}},
Storage: config.StorageConfig{DataDir: "/var/lib/ollama-gateway"},
}
warnings := strings.Join(configWarnings(cfg), "\n")
if !strings.Contains(warnings, "max_requested_tokens=-1") {
t.Fatalf("missing unlimited context cap warning: %s", warnings)
}
if !strings.Contains(warnings, "default_worker_tokens=-1") {
t.Fatalf("missing model-max fallback warning: %s", warnings)
}
}
cmd/worker-telemetry/main.go
+180
View File
@@ -0,0 +1,180 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"net"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/example/ollama-fair-gateway/internal/hoststats"
)
type telemetry struct {
MemoryUsedBytes int64 `json:"memory_used_bytes,omitempty"`
MemoryTotalBytes int64 `json:"memory_total_bytes,omitempty"`
VRAMUsedBytes int64 `json:"vram_used_bytes,omitempty"`
VRAMTotalBytes int64 `json:"vram_total_bytes,omitempty"`
GPUUtilizationPct float64 `json:"gpu_utilization_percent,omitempty"`
GPUTemperatureC float64 `json:"gpu_temperature_c,omitempty"`
GPUPowerWatts float64 `json:"gpu_power_watts,omitempty"`
Source string `json:"source,omitempty"`
UpdatedAt time.Time `json:"updated_at,omitempty"`
Error string `json:"error,omitempty"`
}
type collector struct {
nvidia bool
nvidiaGPU string
amd bool
amdDevice string
}
func (c collector) collect(ctx context.Context) telemetry {
out := telemetry{UpdatedAt: time.Now().UTC()}
var errs []string
mctx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond)
m, err := hoststats.ReadMemory(mctx)
cancel()
if err != nil {
errs = append(errs, "memory: "+err.Error())
} else {
out.MemoryTotalBytes, out.MemoryUsedBytes = m.TotalBytes, m.UsedBytes
out.Source = appendSource(out.Source, "host-memory")
}
if c.nvidia {
gctx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond)
g, err := hoststats.ReadNVIDIA(gctx, c.nvidiaGPU)
cancel()
if err != nil {
errs = append(errs, "nvidia: "+err.Error())
} else {
out.VRAMUsedBytes, out.VRAMTotalBytes = g.MemoryUsedBytes, g.MemoryTotalBytes
out.GPUUtilizationPct, out.GPUTemperatureC, out.GPUPowerWatts = g.UtilizationPercent, g.TemperatureC, g.PowerWatts
out.Source = appendSource(out.Source, "nvidia-smi")
}
}
if c.amd {
gctx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond)
g, err := hoststats.ReadAMD(gctx, c.amdDevice)
cancel()
if err != nil {
errs = append(errs, "amd: "+err.Error())
} else {
out.VRAMUsedBytes, out.VRAMTotalBytes = g.MemoryUsedBytes, g.MemoryTotalBytes
out.GPUUtilizationPct, out.GPUTemperatureC, out.GPUPowerWatts = g.UtilizationPercent, g.TemperatureC, g.PowerWatts
out.Source = appendSource(out.Source, "amdgpu-sysfs")
}
}
out.Error = strings.Join(errs, "; ")
return out
}
func appendSource(cur, next string) string {
if cur == "" {
return next
}
return cur + "+" + next
}
type cidrAllowlist struct{ nets []*net.IPNet }
func parseCIDRs(raw string) (cidrAllowlist, error) {
var out cidrAllowlist
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
_, n, err := net.ParseCIDR(part)
if err != nil {
return out, fmt.Errorf("invalid allow CIDR %q: %w", part, err)
}
out.nets = append(out.nets, n)
}
if len(out.nets) == 0 {
return out, fmt.Errorf("at least one allow CIDR is required")
}
return out, nil
}
func (a cidrAllowlist) allowed(remote string) bool {
host, _, err := net.SplitHostPort(remote)
if err != nil {
host = remote
}
ip := net.ParseIP(strings.Trim(host, "[]"))
if ip == nil {
return false
}
for _, n := range a.nets {
if n.Contains(ip) {
return true
}
}
return false
}
func main() {
listen := flag.String("listen", "127.0.0.1:11500", "listen address")
path := flag.String("path", "/telemetry", "telemetry path")
allow := flag.String("allow-cidrs", "127.0.0.1/32,::1/128", "comma-separated client CIDRs allowed to read telemetry")
nvidia := flag.Bool("nvidia-smi", false, "collect NVIDIA telemetry with nvidia-smi")
nvidiaGPU := flag.String("nvidia-gpu", "", "optional nvidia-smi GPU selector")
amd := flag.Bool("amd-sysfs", false, "collect Linux AMDGPU telemetry from sysfs")
amdDevice := flag.String("amd-device", "", "optional AMDGPU device path such as /sys/class/drm/card0/device; empty auto-detects")
once := flag.Bool("once", false, "print one telemetry sample as JSON and exit")
flag.Parse()
if !strings.HasPrefix(*path, "/") {
log.Fatal("-path must begin with /")
}
acl, err := parseCIDRs(*allow)
if err != nil {
log.Fatal(err)
}
c := collector{nvidia: *nvidia, nvidiaGPU: *nvidiaGPU, amd: *amd, amdDevice: *amdDevice}
if *once {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
if err := enc.Encode(c.collect(context.Background())); err != nil {
log.Fatal(err)
}
return
}
mux := http.NewServeMux()
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) })
mux.HandleFunc(*path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if !acl.allowed(r.RemoteAddr) {
http.Error(w, "forbidden", http.StatusForbidden)
return
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("X-Content-Type-Options", "nosniff")
_ = json.NewEncoder(w).Encode(c.collect(r.Context()))
})
srv := &http.Server{Addr: *listen, Handler: mux, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: time.Minute}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
go func() {
<-ctx.Done()
shutdown, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
_ = srv.Shutdown(shutdown)
}()
log.Printf("worker telemetry listening on http://%s%s allowed=%s nvidia=%t amd=%t", *listen, *path, *allow, *nvidia, *amd)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatal(err)
}
}
cmd/worker-telemetry/main_test.go
+19
View File
@@ -0,0 +1,19 @@
package main
import "testing"
func TestCIDRAllowlist(t *testing.T) {
a, err := parseCIDRs("127.0.0.1/32,10.2.19.0/24")
if err != nil {
t.Fatal(err)
}
if !a.allowed("10.2.19.42:1234") || !a.allowed("127.0.0.1:1") || a.allowed("10.2.18.1:5") {
t.Fatal("unexpected allowlist result")
}
}
func TestAppendSource(t *testing.T) {
if got := appendSource("host-memory", "amdgpu-sysfs"); got != "host-memory+amdgpu-sysfs" {
t.Fatal(got)
}
}
config.example.json
+335
View File
@@ -0,0 +1,335 @@
{
"server": {
"listen": ":8080",
"read_header_timeout": "10s",
"idle_timeout": "2m",
"max_request_duration": "30m",
"max_body_bytes": 67108864,
"metrics_public": true
},
"auth": {
"oidc": {
"enabled": false
},
"api_keys": [],
"ip_bypass": [
{
"cidrs": [
"127.0.0.1/32",
"::1/128"
],
"tenant": "local",
"subject": "localhost",
"application": "local-tools",
"scopes": [
"gateway:admin"
]
}
],
"trusted_proxies": [],
"ip_bypass_use_forwarded_ip": false
},
"scheduler": {
"global_concurrency": 4,
"max_queue": 2048,
"max_queue_per_actor": 64,
"queue_timeout": "10m",
"default_tenant_weight": 1,
"default_actor_weight": 1,
"policies": {
"*": {
"tenant_weight": 1,
"actor_weight": 1,
"actor_credits_per_minute": 60,
"actor_burst_credits": 180,
"tenant_credits_per_minute": 300,
"tenant_burst_credits": 900
},
"internal": {
"tenant_weight": 2,
"actor_weight": 2,
"actor_credits_per_minute": 180,
"actor_burst_credits": 540,
"tenant_credits_per_minute": 900,
"tenant_burst_credits": 2700
}
},
"compute_paths": [
"/api/generate",
"/api/chat",
"/api/embed",
"/api/embeddings",
"/v1/chat/completions",
"/v1/completions",
"/v1/embeddings",
"/v1/responses",
"/v1/messages"
]
},
"quota": {
"enabled": true
},
"cost": {
"default": {
"input_credits_per_1k": 1,
"output_credits_per_1k": 3,
"compute_credits_per_second": 0,
"cached_input_factor": 1.0
},
"models": {
"small-*": {
"input_credits_per_1k": 0.5,
"output_credits_per_1k": 1.5
},
"large-*": {
"input_credits_per_1k": 2,
"output_credits_per_1k": 6
}
},
"default_max_output_tokens": 1024
},
"model_capabilities": {
"mode": "enforce",
"cache_ttl": "10m",
"context_guard": "reject",
"context": {
"max_requested_tokens": 32768,
"default_worker_tokens": 4096,
"estimation_margin_percent": 15,
"vision_reserve_tokens_per_image": 2048
}
},
"routing": {
"loaded_bonus": 60,
"installed_bonus": 30,
"throughput_bonus": 20,
"vram_pressure_penalty": 35,
"gpu_utilization_penalty": 10,
"avoid_vram_percent": 97
},
"workers": [
{
"name": "mac-studio-m5-ultra",
"url": "http://127.0.0.1:11434",
"max_concurrent": 4,
"health_interval": "5s",
"memory_capacity_bytes": 274877906944,
"vram_capacity_bytes": 274877906944,
"labels": {
"host": "mac-studio-m5-ultra",
"memory": "unified",
"accelerator": "apple-gpu"
},
"local_system_stats": true,
"model_concurrency": {
"*": 4
},
"model_placement": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
}
}
],
"usage": {
"journal_dir": "./data/usage",
"buffer": 16384,
"flush_interval": "1s",
"retention": {
"detail_days": 30,
"daily_days": 400,
"monthly_months": 0,
"compaction_interval": "6h"
}
},
"native": {
"management_requires_admin": true,
"control_worker": "mac-studio-m5-ultra"
},
"ui": {
"enabled": true,
"path": "/admin",
"title": "Ollama Fair Gateway",
"recent_events": 10000,
"secure_cookies": false,
"oidc": {
"enabled": false
}
},
"public_dashboard": {
"enabled": false,
"path": "/status",
"title": "Ollama Gateway Status",
"subtitle": "Live-Auslastung und Infrastruktur",
"refresh_interval": "2s",
"max_live_requests": 64,
"show_worker_names": false,
"show_model_names": false,
"show_resource_metrics": true,
"worker_display_names": {}
},
"infrastructure": {
"node_name": "mac-studio-gateway",
"refresh_interval": "250ms",
"max_requests": 256
},
"storage": {
"data_dir": "./data",
"config_file": "gateway-config.json",
"api_keys_file": "api-keys.json",
"policies_file": "policies.json",
"metrics_file": "metrics.json",
"quota_file": "quota.json",
"worker_performance_file": "worker-performance.json",
"flush_interval": "10s",
"model_placement_file": "model-placement.json",
"worker_state_file": "worker-state.json",
"auto_tune_file": "auto-tune.json",
"warm_models_file": "warm-models.json",
"alerts_file": "alerts.json",
"conversations_file": "conversations.enc.json",
"batch_jobs_file": "batch-jobs.json",
"batch_jobs_dir": "batch"
},
"reliability": {
"enabled": true,
"failure_threshold": 3,
"open_duration": "30s",
"retry_attempts": 2,
"retry_backoff": "50ms"
},
"model_access": {
"default": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
},
"tenants": {}
},
"model_aliases": {
"fast": {
"models": [
"qwen3:8b",
"gemma3:12b"
]
},
"coding": {
"models": [
"qwen3.6:27b-q4_K_M",
"qwen3:8b"
],
"required_capabilities": [
"completion"
]
}
},
"service_classes": {
"default": "interactive",
"header": "X-Gateway-Service-Class",
"override_scope": "gateway:service-class",
"classes": {
"interactive": {
"weight": 4,
"max_queue_wait": "30s",
"max_concurrent": 0
},
"system": {
"weight": 8,
"max_queue_wait": "30s",
"max_concurrent": 1
},
"background": {
"weight": 1,
"max_queue_wait": "10m",
"max_concurrent": 1
},
"batch": {
"weight": 0.5,
"max_queue_wait": "30m",
"max_concurrent": 1
}
}
},
"auto_tuning": {
"enabled": true,
"max_concurrency": 4,
"samples_per_level": 2,
"max_tokens": 96,
"timeout": "10m",
"prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.",
"ttft_weight": 0.25,
"throughput_weight": 1
},
"conversations": {
"enabled": false,
"encryption_key": "${GATEWAY_CONVERSATION_KEY}",
"retention": "24h",
"max_entries": 1000,
"max_content_bytes": 2097152
},
"batch_jobs": {
"enabled": false,
"retention": "168h",
"max_jobs": 1000,
"max_concurrent": 1,
"max_input_bytes": 16777216
},
"opentelemetry": {
"enabled": false,
"endpoint": "http://127.0.0.1:4318",
"headers": {},
"service_name": "ollama-fair-gateway",
"service_version": "",
"sample_ratio": 1,
"batch_size": 128,
"flush_interval": "2s",
"capture_content": false
},
"warm_models": {
"enabled": true,
"reconcile_interval": "30s",
"operation_timeout": "2m",
"policies": {
"qwen3:8b": {
"class": "warm",
"workers": [
"mac-studio-m5-ultra"
],
"replicas": 1,
"preload": true,
"idle_timeout": "30m"
},
"gemma4:*": {
"class": "cold",
"workers": [
"mac-studio-m5-ultra"
],
"replicas": 1,
"preload": false,
"idle_timeout": "10m"
}
}
},
"alerts": {
"enabled": true,
"evaluation_interval": "15s",
"cooldown": "5m",
"history_limit": 500,
"thresholds": {
"worker_down_for": "30s",
"circuit_open": true,
"queue_depth": 100,
"vram_percent": 95,
"storage_bytes": 0,
"quota_remaining_percent": 10,
"oom": true,
"queue_wait": "30s"
},
"webhooks": [],
"webhook_timeout": "5s",
"webhook_max_concurrent": 4,
"webhook_queue": 1024,
"webhook_retry_attempts": 3,
"webhook_retry_backoff": "500ms"
}
}
config.oidc.example.json
+340
View File
@@ -0,0 +1,340 @@
{
"server": {
"listen": ":8080",
"read_header_timeout": "10s",
"idle_timeout": "2m",
"max_request_duration": "30m",
"max_body_bytes": 67108864,
"metrics_public": false
},
"auth": {
"oidc": {
"enabled": true,
"issuer": "${OIDC_ISSUER}",
"audience": "${OIDC_AUDIENCE}",
"tenant_claim": "tenant_id",
"application_claim": "azp",
"groups_claim": "groups",
"admin_groups": [
"llm-gateway-admins"
],
"clock_skew": "60s",
"jwks_refresh_min_interval": "10s",
"allowed_algorithms": [
"RS256",
"PS256",
"ES256",
"EdDSA"
]
},
"api_keys": [
{
"name": "automation",
"key": "${GATEWAY_AUTOMATION_KEY}",
"tenant": "internal",
"subject": "automation",
"application": "automation",
"scopes": [],
"service_class": "interactive"
}
],
"ip_bypass": [
{
"cidrs": [
"10.42.0.0/24"
],
"tenant": "internal",
"subject": "trusted-lan",
"application": "lan-services",
"scopes": []
}
],
"trusted_proxies": [
"10.0.0.0/8"
]
},
"scheduler": {
"global_concurrency": 4,
"max_queue": 2048,
"max_queue_per_actor": 64,
"queue_timeout": "10m",
"default_tenant_weight": 1,
"default_actor_weight": 1,
"policies": {
"*": {
"tenant_weight": 1,
"actor_weight": 1,
"actor_credits_per_minute": 60,
"actor_burst_credits": 180,
"tenant_credits_per_minute": 300,
"tenant_burst_credits": 900
},
"internal": {
"tenant_weight": 2,
"actor_weight": 2,
"actor_credits_per_minute": 180,
"actor_burst_credits": 540,
"tenant_credits_per_minute": 900,
"tenant_burst_credits": 2700
}
},
"compute_paths": [
"/api/generate",
"/api/chat",
"/api/embed",
"/api/embeddings",
"/v1/chat/completions",
"/v1/completions",
"/v1/embeddings",
"/v1/responses",
"/v1/messages"
]
},
"quota": {
"enabled": true
},
"cost": {
"default": {
"input_credits_per_1k": 1,
"output_credits_per_1k": 3,
"compute_credits_per_second": 0,
"cached_input_factor": 1.0
},
"models": {
"small-*": {
"input_credits_per_1k": 0.5,
"output_credits_per_1k": 1.5
},
"large-*": {
"input_credits_per_1k": 2,
"output_credits_per_1k": 6
}
},
"default_max_output_tokens": 1024
},
"model_capabilities": {
"mode": "enforce",
"cache_ttl": "10m",
"context_guard": "reject",
"context": {
"max_requested_tokens": 32768,
"default_worker_tokens": 4096,
"estimation_margin_percent": 15,
"vision_reserve_tokens_per_image": 2048
}
},
"routing": {
"loaded_bonus": 60,
"installed_bonus": 30,
"throughput_bonus": 20,
"vram_pressure_penalty": 35,
"gpu_utilization_penalty": 10,
"avoid_vram_percent": 97
},
"workers": [
{
"name": "mac-studio-m5-ultra",
"url": "http://127.0.0.1:11434",
"max_concurrent": 4,
"health_interval": "5s",
"memory_capacity_bytes": 274877906944,
"vram_capacity_bytes": 274877906944,
"labels": {
"host": "mac-studio-m5-ultra",
"memory": "unified",
"accelerator": "apple-gpu"
},
"local_system_stats": true,
"model_concurrency": {
"*": 4
},
"model_placement": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
}
}
],
"usage": {
"journal_dir": "./data/usage",
"buffer": 16384,
"flush_interval": "1s",
"retention": {
"detail_days": 30,
"daily_days": 400,
"monthly_months": 0,
"compaction_interval": "6h"
}
},
"native": {
"management_requires_admin": true,
"control_worker": "mac-studio-m5-ultra"
},
"ui": {
"enabled": true,
"path": "/admin",
"title": "Ollama Fair Gateway",
"recent_events": 20000,
"session_secret": "${GATEWAY_UI_SESSION_SECRET}",
"secure_cookies": true,
"oidc": {
"enabled": true,
"client_id": "${OIDC_UI_CLIENT_ID}",
"client_secret": "${OIDC_UI_CLIENT_SECRET}",
"scopes": [
"openid",
"profile",
"email"
],
"redirect_url": "${GATEWAY_UI_REDIRECT_URL}"
}
},
"infrastructure": {
"node_name": "mac-studio-gateway",
"refresh_interval": "250ms",
"max_requests": 256
},
"storage": {
"data_dir": "./data",
"config_file": "gateway-config.json",
"api_keys_file": "api-keys.json",
"policies_file": "policies.json",
"metrics_file": "metrics.json",
"quota_file": "quota.json",
"worker_performance_file": "worker-performance.json",
"flush_interval": "10s",
"model_placement_file": "model-placement.json",
"worker_state_file": "worker-state.json",
"auto_tune_file": "auto-tune.json",
"warm_models_file": "warm-models.json",
"alerts_file": "alerts.json",
"conversations_file": "conversations.enc.json",
"batch_jobs_file": "batch-jobs.json",
"batch_jobs_dir": "batch"
},
"reliability": {
"enabled": true,
"failure_threshold": 3,
"open_duration": "30s",
"retry_attempts": 2,
"retry_backoff": "50ms"
},
"model_access": {
"default": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
},
"tenants": {}
},
"model_aliases": {},
"service_classes": {
"default": "interactive",
"header": "X-Gateway-Service-Class",
"override_scope": "gateway:service-class",
"classes": {
"interactive": {
"weight": 4,
"max_queue_wait": "30s",
"max_concurrent": 0
},
"system": {
"weight": 8,
"max_queue_wait": "30s",
"max_concurrent": 1
},
"background": {
"weight": 1,
"max_queue_wait": "10m",
"max_concurrent": 1
},
"batch": {
"weight": 0.5,
"max_queue_wait": "30m",
"max_concurrent": 1
}
}
},
"auto_tuning": {
"enabled": true,
"max_concurrency": 4,
"samples_per_level": 2,
"max_tokens": 96,
"timeout": "10m",
"prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.",
"ttft_weight": 0.25,
"throughput_weight": 1
},
"conversations": {
"enabled": false,
"encryption_key": "${GATEWAY_CONVERSATION_KEY}",
"retention": "24h",
"max_entries": 1000,
"max_content_bytes": 2097152
},
"batch_jobs": {
"enabled": false,
"retention": "168h",
"max_jobs": 1000,
"max_concurrent": 1,
"max_input_bytes": 16777216
},
"opentelemetry": {
"enabled": false,
"endpoint": "http://127.0.0.1:4318",
"headers": {},
"service_name": "ollama-fair-gateway",
"service_version": "",
"sample_ratio": 1,
"batch_size": 128,
"flush_interval": "2s",
"capture_content": false
},
"warm_models": {
"enabled": true,
"reconcile_interval": "30s",
"operation_timeout": "2m",
"policies": {
"qwen3:8b": {
"class": "warm",
"workers": [
"mac-studio-m5-ultra"
],
"replicas": 1,
"preload": true,
"idle_timeout": "30m"
},
"gemma4:*": {
"class": "cold",
"workers": [
"mac-studio-m5-ultra"
],
"replicas": 1,
"preload": false,
"idle_timeout": "10m"
}
}
},
"alerts": {
"enabled": true,
"evaluation_interval": "15s",
"cooldown": "5m",
"history_limit": 500,
"thresholds": {
"worker_down_for": "30s",
"circuit_open": true,
"queue_depth": 100,
"vram_percent": 95,
"storage_bytes": 0,
"quota_remaining_percent": 10,
"oom": true,
"queue_wait": "30s"
},
"webhooks": [],
"webhook_timeout": "5s",
"webhook_max_concurrent": 4,
"webhook_queue": 1024,
"webhook_retry_attempts": 3,
"webhook_retry_backoff": "500ms"
}
}
config.openwebui.example.json
+316
View File
@@ -0,0 +1,316 @@
{
"server": {
"listen": ":8080",
"read_header_timeout": "10s",
"idle_timeout": "2m",
"max_request_duration": "30m",
"max_body_bytes": 67108864,
"metrics_public": true
},
"auth": {
"oidc": {
"enabled": false
},
"api_keys": [
{
"name": "openwebui",
"key": "${OPENWEBUI_GATEWAY_KEY}",
"tenant": "interactive",
"subject": "openwebui",
"application": "openwebui",
"scopes": [],
"service_class": "interactive"
}
],
"ip_bypass": [
{
"cidrs": [
"127.0.0.1/32",
"::1/128"
],
"tenant": "local",
"subject": "localhost",
"application": "local-tools",
"scopes": [
"gateway:admin"
]
}
],
"trusted_proxies": []
},
"scheduler": {
"global_concurrency": 4,
"max_queue": 2048,
"max_queue_per_actor": 64,
"queue_timeout": "10m",
"default_tenant_weight": 1,
"default_actor_weight": 1,
"policies": {
"*": {
"tenant_weight": 1,
"actor_weight": 1,
"actor_credits_per_minute": 60,
"actor_burst_credits": 180,
"tenant_credits_per_minute": 300,
"tenant_burst_credits": 900
},
"internal": {
"tenant_weight": 2,
"actor_weight": 2,
"actor_credits_per_minute": 180,
"actor_burst_credits": 540,
"tenant_credits_per_minute": 900,
"tenant_burst_credits": 2700
}
},
"compute_paths": [
"/api/generate",
"/api/chat",
"/api/embed",
"/api/embeddings",
"/v1/chat/completions",
"/v1/completions",
"/v1/embeddings",
"/v1/responses",
"/v1/messages"
]
},
"quota": {
"enabled": true
},
"cost": {
"default": {
"input_credits_per_1k": 1,
"output_credits_per_1k": 3,
"compute_credits_per_second": 0,
"cached_input_factor": 1.0
},
"models": {
"small-*": {
"input_credits_per_1k": 0.5,
"output_credits_per_1k": 1.5
},
"large-*": {
"input_credits_per_1k": 2,
"output_credits_per_1k": 6
}
},
"default_max_output_tokens": 1024
},
"model_capabilities": {
"mode": "enforce",
"cache_ttl": "10m",
"context_guard": "reject",
"context": {
"max_requested_tokens": 32768,
"default_worker_tokens": 4096,
"estimation_margin_percent": 15,
"vision_reserve_tokens_per_image": 2048
}
},
"routing": {
"loaded_bonus": 60,
"installed_bonus": 30,
"throughput_bonus": 20,
"vram_pressure_penalty": 35,
"gpu_utilization_penalty": 10,
"avoid_vram_percent": 97
},
"workers": [
{
"name": "mac-studio-m5-ultra",
"url": "http://127.0.0.1:11434",
"max_concurrent": 4,
"health_interval": "5s",
"memory_capacity_bytes": 274877906944,
"vram_capacity_bytes": 274877906944,
"labels": {
"host": "mac-studio-m5-ultra",
"memory": "unified",
"accelerator": "apple-gpu"
},
"local_system_stats": true,
"model_concurrency": {
"*": 4
},
"model_placement": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
}
}
],
"usage": {
"journal_dir": "./data/usage",
"buffer": 16384,
"flush_interval": "1s",
"retention": {
"detail_days": 30,
"daily_days": 400,
"monthly_months": 0,
"compaction_interval": "6h"
}
},
"native": {
"management_requires_admin": true,
"control_worker": "mac-studio-m5-ultra"
},
"ui": {
"enabled": true,
"path": "/admin",
"title": "Ollama Fair Gateway",
"recent_events": 10000,
"secure_cookies": false,
"oidc": {
"enabled": false
}
},
"infrastructure": {
"node_name": "mac-studio-gateway",
"refresh_interval": "250ms",
"max_requests": 256
},
"storage": {
"data_dir": "./data",
"config_file": "gateway-config.json",
"api_keys_file": "api-keys.json",
"policies_file": "policies.json",
"metrics_file": "metrics.json",
"quota_file": "quota.json",
"worker_performance_file": "worker-performance.json",
"flush_interval": "10s",
"model_placement_file": "model-placement.json",
"worker_state_file": "worker-state.json",
"auto_tune_file": "auto-tune.json",
"warm_models_file": "warm-models.json",
"alerts_file": "alerts.json",
"conversations_file": "conversations.enc.json",
"batch_jobs_file": "batch-jobs.json",
"batch_jobs_dir": "batch"
},
"reliability": {
"enabled": true,
"failure_threshold": 3,
"open_duration": "30s",
"retry_attempts": 2,
"retry_backoff": "50ms"
},
"model_access": {
"default": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
},
"tenants": {}
},
"model_aliases": {},
"service_classes": {
"default": "interactive",
"header": "X-Gateway-Service-Class",
"override_scope": "gateway:service-class",
"classes": {
"interactive": {
"weight": 4,
"max_queue_wait": "30s",
"max_concurrent": 0
},
"system": {
"weight": 8,
"max_queue_wait": "30s",
"max_concurrent": 1
},
"background": {
"weight": 1,
"max_queue_wait": "10m",
"max_concurrent": 1
},
"batch": {
"weight": 0.5,
"max_queue_wait": "30m",
"max_concurrent": 1
}
}
},
"auto_tuning": {
"enabled": true,
"max_concurrency": 4,
"samples_per_level": 2,
"max_tokens": 96,
"timeout": "10m",
"prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.",
"ttft_weight": 0.25,
"throughput_weight": 1
},
"conversations": {
"enabled": false,
"encryption_key": "${GATEWAY_CONVERSATION_KEY}",
"retention": "24h",
"max_entries": 1000,
"max_content_bytes": 2097152
},
"batch_jobs": {
"enabled": false,
"retention": "168h",
"max_jobs": 1000,
"max_concurrent": 1,
"max_input_bytes": 16777216
},
"opentelemetry": {
"enabled": false,
"endpoint": "http://127.0.0.1:4318",
"headers": {},
"service_name": "ollama-fair-gateway",
"service_version": "",
"sample_ratio": 1,
"batch_size": 128,
"flush_interval": "2s",
"capture_content": false
},
"warm_models": {
"enabled": true,
"reconcile_interval": "30s",
"operation_timeout": "2m",
"policies": {
"qwen3:8b": {
"class": "warm",
"workers": [
"mac-studio-m5-ultra"
],
"replicas": 1,
"preload": true,
"idle_timeout": "30m"
},
"gemma4:*": {
"class": "cold",
"workers": [
"mac-studio-m5-ultra"
],
"replicas": 1,
"preload": false,
"idle_timeout": "10m"
}
}
},
"alerts": {
"enabled": true,
"evaluation_interval": "15s",
"cooldown": "5m",
"history_limit": 500,
"thresholds": {
"worker_down_for": "30s",
"circuit_open": true,
"queue_depth": 100,
"vram_percent": 95,
"storage_bytes": 0,
"quota_remaining_percent": 10,
"oom": true,
"queue_wait": "30s"
},
"webhooks": [],
"webhook_timeout": "5s",
"webhook_max_concurrent": 4,
"webhook_queue": 1024,
"webhook_retry_attempts": 3,
"webhook_retry_backoff": "500ms"
}
}
config.placement.example.json
+346
View File
@@ -0,0 +1,346 @@
{
"server": {
"listen": ":8080",
"read_header_timeout": "10s",
"idle_timeout": "2m",
"max_request_duration": "30m",
"max_body_bytes": 67108864,
"metrics_public": true
},
"auth": {
"oidc": {
"enabled": false
},
"api_keys": [],
"ip_bypass": [
{
"cidrs": [
"127.0.0.1/32",
"::1/128"
],
"tenant": "local",
"subject": "localhost",
"application": "local-tools",
"scopes": [
"gateway:admin"
]
}
],
"trusted_proxies": []
},
"scheduler": {
"global_concurrency": 6,
"max_queue": 2048,
"max_queue_per_actor": 64,
"queue_timeout": "10m",
"default_tenant_weight": 1,
"default_actor_weight": 1,
"policies": {
"*": {
"tenant_weight": 1,
"actor_weight": 1,
"actor_credits_per_minute": 60,
"actor_burst_credits": 180,
"tenant_credits_per_minute": 300,
"tenant_burst_credits": 900
},
"internal": {
"tenant_weight": 2,
"actor_weight": 2,
"actor_credits_per_minute": 180,
"actor_burst_credits": 540,
"tenant_credits_per_minute": 900,
"tenant_burst_credits": 2700
}
},
"compute_paths": [
"/api/generate",
"/api/chat",
"/api/embed",
"/api/embeddings",
"/v1/chat/completions",
"/v1/completions",
"/v1/embeddings",
"/v1/responses",
"/v1/messages"
]
},
"quota": {
"enabled": true
},
"cost": {
"default": {
"input_credits_per_1k": 1,
"output_credits_per_1k": 3,
"compute_credits_per_second": 0,
"cached_input_factor": 1.0
},
"models": {
"small-*": {
"input_credits_per_1k": 0.5,
"output_credits_per_1k": 1.5
},
"large-*": {
"input_credits_per_1k": 2,
"output_credits_per_1k": 6
}
},
"default_max_output_tokens": 1024
},
"model_capabilities": {
"mode": "enforce",
"cache_ttl": "10m",
"context_guard": "reject",
"context": {
"max_requested_tokens": 32768,
"default_worker_tokens": 4096,
"estimation_margin_percent": 15,
"vision_reserve_tokens_per_image": 2048
}
},
"routing": {
"loaded_bonus": 60,
"installed_bonus": 30,
"throughput_bonus": 20,
"vram_pressure_penalty": 35,
"gpu_utilization_penalty": 10,
"avoid_vram_percent": 97
},
"workers": [
{
"name": "m5-ultra",
"url": "http://10.10.11.123:11434",
"max_concurrent": 4,
"health_interval": "5s",
"memory_capacity_bytes": 274877906944,
"vram_capacity_bytes": 274877906944,
"labels": {
"host": "m5-ultra",
"memory": "unified",
"accelerator": "apple-gpu"
},
"local_system_stats": false,
"model_concurrency": {
"qwen3:8b": 4,
"gemma4:*": 2,
"qwen3.6:27b-*": 1,
"orcarouter/Qwen3.8-27B-Uncensored:*": 1,
"*": 2
},
"model_placement": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
}
},
{
"name": "rtx-4090",
"url": "http://10.10.11.124:11434",
"max_concurrent": 2,
"health_interval": "5s",
"memory_capacity_bytes": 68719476736,
"vram_capacity_bytes": 25769803776,
"local_system_stats": false,
"nvidia_smi": false,
"labels": {
"host": "rtx-4090",
"memory": "system+dedicated-vram",
"accelerator": "nvidia-rtx-4090"
},
"model_concurrency": {
"qwen3:8b": 2,
"gemma3:12b": 2,
"gemma4:*": 1,
"*": 1
},
"model_placement": {
"mode": "whitelist",
"allowed_models": [
"qwen3:8b",
"gemma3:12b",
"gemma4:*",
"embeddinggemma:*",
"qwen3-embedding:*"
],
"denied_models": [
"qwen3.6:27b-*",
"orcarouter/Qwen3.8-27B-Uncensored:*"
]
}
}
],
"usage": {
"journal_dir": "./data/usage",
"buffer": 16384,
"flush_interval": "1s",
"retention": {
"detail_days": 30,
"daily_days": 400,
"monthly_months": 0,
"compaction_interval": "6h"
}
},
"native": {
"management_requires_admin": true,
"control_worker": "m5-ultra"
},
"ui": {
"enabled": true,
"path": "/admin",
"title": "Ollama Fair Gateway",
"recent_events": 10000,
"secure_cookies": false,
"oidc": {
"enabled": false
}
},
"infrastructure": {
"node_name": "dual-worker-gateway",
"refresh_interval": "250ms",
"max_requests": 256
},
"storage": {
"data_dir": "./data",
"config_file": "gateway-config.json",
"api_keys_file": "api-keys.json",
"policies_file": "policies.json",
"metrics_file": "metrics.json",
"quota_file": "quota.json",
"worker_performance_file": "worker-performance.json",
"flush_interval": "10s",
"model_placement_file": "model-placement.json",
"worker_state_file": "worker-state.json",
"auto_tune_file": "auto-tune.json",
"warm_models_file": "warm-models.json",
"alerts_file": "alerts.json",
"conversations_file": "conversations.enc.json",
"batch_jobs_file": "batch-jobs.json",
"batch_jobs_dir": "batch"
},
"reliability": {
"enabled": true,
"failure_threshold": 3,
"open_duration": "30s",
"retry_attempts": 2,
"retry_backoff": "50ms"
},
"model_access": {
"default": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
},
"tenants": {}
},
"model_aliases": {},
"service_classes": {
"default": "interactive",
"header": "X-Gateway-Service-Class",
"override_scope": "gateway:service-class",
"classes": {
"interactive": {
"weight": 4,
"max_queue_wait": "30s",
"max_concurrent": 0
},
"system": {
"weight": 8,
"max_queue_wait": "30s",
"max_concurrent": 1
},
"background": {
"weight": 1,
"max_queue_wait": "10m",
"max_concurrent": 1
},
"batch": {
"weight": 0.5,
"max_queue_wait": "30m",
"max_concurrent": 1
}
}
},
"auto_tuning": {
"enabled": true,
"max_concurrency": 4,
"samples_per_level": 2,
"max_tokens": 96,
"timeout": "10m",
"prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.",
"ttft_weight": 0.25,
"throughput_weight": 1
},
"conversations": {
"enabled": false,
"encryption_key": "${GATEWAY_CONVERSATION_KEY}",
"retention": "24h",
"max_entries": 1000,
"max_content_bytes": 2097152
},
"batch_jobs": {
"enabled": false,
"retention": "168h",
"max_jobs": 1000,
"max_concurrent": 1,
"max_input_bytes": 16777216
},
"opentelemetry": {
"enabled": false,
"endpoint": "http://127.0.0.1:4318",
"headers": {},
"service_name": "ollama-fair-gateway",
"service_version": "",
"sample_ratio": 1,
"batch_size": 128,
"flush_interval": "2s",
"capture_content": false
},
"warm_models": {
"enabled": true,
"reconcile_interval": "30s",
"operation_timeout": "2m",
"policies": {
"qwen3:8b": {
"class": "warm",
"workers": [
"m5-ultra"
],
"replicas": 1,
"preload": true,
"idle_timeout": "30m"
},
"gemma4:*": {
"class": "cold",
"workers": [
"m5-ultra",
"rtx-4090"
],
"replicas": 1,
"preload": false,
"idle_timeout": "10m"
}
}
},
"alerts": {
"enabled": true,
"evaluation_interval": "15s",
"cooldown": "5m",
"history_limit": 500,
"thresholds": {
"worker_down_for": "30s",
"circuit_open": true,
"queue_depth": 100,
"vram_percent": 95,
"storage_bytes": 0,
"quota_remaining_percent": 10,
"oom": true,
"queue_wait": "30s"
},
"webhooks": [],
"webhook_timeout": "5s",
"webhook_max_concurrent": 4,
"webhook_queue": 1024,
"webhook_retry_attempts": 3,
"webhook_retry_backoff": "500ms"
}
}
config.rtx4090.example.json
+339
View File
@@ -0,0 +1,339 @@
{
"server": {
"listen": ":8080",
"read_header_timeout": "10s",
"idle_timeout": "2m",
"max_request_duration": "30m",
"max_body_bytes": 67108864,
"metrics_public": true
},
"auth": {
"oidc": {
"enabled": false
},
"api_keys": [
{
"name": "openwebui",
"key": "${OPENWEBUI_GATEWAY_KEY}",
"tenant": "interactive",
"subject": "openwebui",
"application": "openwebui",
"scopes": [],
"service_class": "interactive"
}
],
"ip_bypass": [
{
"cidrs": [
"127.0.0.1/32",
"::1/128"
],
"tenant": "local",
"subject": "localhost",
"application": "local-tools",
"scopes": [
"gateway:admin"
]
}
],
"trusted_proxies": []
},
"scheduler": {
"global_concurrency": 2,
"max_queue": 2048,
"max_queue_per_actor": 64,
"queue_timeout": "10m",
"default_tenant_weight": 1,
"default_actor_weight": 1,
"policies": {
"*": {
"tenant_weight": 1,
"actor_weight": 1,
"actor_credits_per_minute": 60,
"actor_burst_credits": 180,
"tenant_credits_per_minute": 300,
"tenant_burst_credits": 900
},
"internal": {
"tenant_weight": 2,
"actor_weight": 2,
"actor_credits_per_minute": 180,
"actor_burst_credits": 540,
"tenant_credits_per_minute": 900,
"tenant_burst_credits": 2700
}
},
"compute_paths": [
"/api/generate",
"/api/chat",
"/api/embed",
"/api/embeddings",
"/v1/chat/completions",
"/v1/completions",
"/v1/embeddings",
"/v1/responses",
"/v1/messages"
]
},
"quota": {
"enabled": true
},
"cost": {
"default": {
"input_credits_per_1k": 1,
"output_credits_per_1k": 3,
"compute_credits_per_second": 0,
"cached_input_factor": 1.0
},
"models": {
"small-*": {
"input_credits_per_1k": 0.5,
"output_credits_per_1k": 1.5
},
"large-*": {
"input_credits_per_1k": 2,
"output_credits_per_1k": 6
}
},
"default_max_output_tokens": 1024
},
"model_capabilities": {
"mode": "enforce",
"cache_ttl": "10m",
"context_guard": "reject",
"context": {
"max_requested_tokens": 32768,
"default_worker_tokens": 4096,
"estimation_margin_percent": 15,
"vision_reserve_tokens_per_image": 2048
}
},
"routing": {
"loaded_bonus": 60,
"installed_bonus": 30,
"throughput_bonus": 20,
"vram_pressure_penalty": 35,
"gpu_utilization_penalty": 10,
"avoid_vram_percent": 97
},
"workers": [
{
"name": "rtx-4090",
"url": "http://127.0.0.1:11434",
"max_concurrent": 2,
"model_concurrency": {
"qwen3:8b": 2,
"gemma3:12b": 2,
"gemma4:*": 1,
"qwen3.6:27b-*": 1,
"orcarouter/Qwen3.8-27B-Uncensored:*": 1,
"*": 1
},
"health_interval": "5s",
"memory_capacity_bytes": 68719476736,
"vram_capacity_bytes": 25769803776,
"local_system_stats": true,
"nvidia_smi": true,
"nvidia_gpu": "0",
"labels": {
"host": "rtx-4090-pc",
"memory": "system+dedicated-vram",
"accelerator": "nvidia-rtx-4090"
},
"model_placement": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
}
}
],
"usage": {
"journal_dir": "./data/usage",
"buffer": 16384,
"flush_interval": "1s",
"retention": {
"detail_days": 30,
"daily_days": 400,
"monthly_months": 0,
"compaction_interval": "6h"
}
},
"native": {
"management_requires_admin": true,
"control_worker": "rtx-4090"
},
"ui": {
"enabled": true,
"path": "/admin",
"title": "Ollama Fair Gateway",
"recent_events": 10000,
"secure_cookies": false,
"oidc": {
"enabled": false
}
},
"infrastructure": {
"node_name": "rtx-4090-gateway",
"refresh_interval": "250ms",
"max_requests": 256
},
"storage": {
"data_dir": "./data",
"config_file": "gateway-config.json",
"api_keys_file": "api-keys.json",
"policies_file": "policies.json",
"metrics_file": "metrics.json",
"quota_file": "quota.json",
"worker_performance_file": "worker-performance.json",
"flush_interval": "10s",
"model_placement_file": "model-placement.json",
"worker_state_file": "worker-state.json",
"auto_tune_file": "auto-tune.json",
"warm_models_file": "warm-models.json",
"alerts_file": "alerts.json",
"conversations_file": "conversations.enc.json",
"batch_jobs_file": "batch-jobs.json",
"batch_jobs_dir": "batch"
},
"reliability": {
"enabled": true,
"failure_threshold": 3,
"open_duration": "30s",
"retry_attempts": 2,
"retry_backoff": "50ms"
},
"model_access": {
"default": {
"mode": "allow_all",
"allowed_models": [],
"denied_models": []
},
"tenants": {}
},
"model_aliases": {
"fast": {
"models": [
"qwen3:8b",
"gemma3:12b"
]
},
"coding": {
"models": [
"qwen3.6:27b-q4_K_M",
"qwen3:8b"
],
"required_capabilities": [
"completion"
]
}
},
"service_classes": {
"default": "interactive",
"header": "X-Gateway-Service-Class",
"override_scope": "gateway:service-class",
"classes": {
"interactive": {
"weight": 4,
"max_queue_wait": "30s",
"max_concurrent": 0
},
"system": {
"weight": 8,
"max_queue_wait": "30s",
"max_concurrent": 1
},
"background": {
"weight": 1,
"max_queue_wait": "10m",
"max_concurrent": 1
},
"batch": {
"weight": 0.5,
"max_queue_wait": "30m",
"max_concurrent": 1
}
}
},
"auto_tuning": {
"enabled": true,
"max_concurrency": 4,
"samples_per_level": 2,
"max_tokens": 96,
"timeout": "10m",
"prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.",
"ttft_weight": 0.25,
"throughput_weight": 1
},
"conversations": {
"enabled": false,
"encryption_key": "${GATEWAY_CONVERSATION_KEY}",
"retention": "24h",
"max_entries": 1000,
"max_content_bytes": 2097152
},
"batch_jobs": {
"enabled": false,
"retention": "168h",
"max_jobs": 1000,
"max_concurrent": 1,
"max_input_bytes": 16777216
},
"opentelemetry": {
"enabled": false,
"endpoint": "http://127.0.0.1:4318",
"headers": {},
"service_name": "ollama-fair-gateway",
"service_version": "",
"sample_ratio": 1,
"batch_size": 128,
"flush_interval": "2s",
"capture_content": false
},
"warm_models": {
"enabled": true,
"reconcile_interval": "30s",
"operation_timeout": "2m",
"policies": {
"qwen3:8b": {
"class": "warm",
"workers": [
"rtx-4090"
],
"replicas": 1,
"preload": true,
"idle_timeout": "30m"
},
"gemma4:*": {
"class": "cold",
"workers": [
"rtx-4090"
],
"replicas": 1,
"preload": false,
"idle_timeout": "10m"
}
}
},
"alerts": {
"enabled": true,
"evaluation_interval": "15s",
"cooldown": "5m",
"history_limit": 500,
"thresholds": {
"worker_down_for": "30s",
"circuit_open": true,
"queue_depth": 100,
"vram_percent": 95,
"storage_bytes": 0,
"quota_remaining_percent": 10,
"oom": true,
"queue_wait": "30s"
},
"webhooks": [],
"webhook_timeout": "5s",
"webhook_max_concurrent": 4,
"webhook_queue": 1024,
"webhook_retry_attempts": 3,
"webhook_retry_backoff": "500ms"
}
}
docker-compose.production.yml
+32
View File
@@ -0,0 +1,32 @@
services:
gateway:
build: .
command: ["-config", "/etc/ollama-gateway/config.json"]
ports:
- "${GATEWAY_PUBLISH_ADDRESS:-127.0.0.1}:${GATEWAY_PUBLISH_PORT:-9080}:8080"
volumes:
- ${GATEWAY_CONFIG:?set GATEWAY_CONFIG to the production bootstrap JSON}:/etc/ollama-gateway/config.json:ro
- gateway-data:/data
user: "65532:65532"
read_only: true
tmpfs:
- /tmp:size=64m,mode=1777
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
healthcheck:
test: ["CMD", "/ollama-gateway", "-probe", "http://127.0.0.1:8080/healthz", "-probe-timeout", "2s"]
interval: 10s
timeout: 3s
retries: 3
start_period: 10s
logging:
driver: json-file
options:
max-size: "10m"
max-file: "5"
restart: unless-stopped
volumes:
gateway-data:
docker-compose.yml
+26
View File
@@ -0,0 +1,26 @@
services:
gateway:
build: .
command: ["-config", "/etc/ollama-gateway/config.json"]
ports:
- "${GATEWAY_PUBLISH_ADDRESS:-127.0.0.1}:${GATEWAY_PUBLISH_PORT:-9080}:8080"
volumes:
- ${GATEWAY_CONFIG:-./config.example.json}:/etc/ollama-gateway/config.json:ro
- gateway-data:/data
read_only: true
tmpfs:
- /tmp:size=64m,mode=1777
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
healthcheck:
test: ["CMD", "/ollama-gateway", "-probe", "http://127.0.0.1:8080/healthz", "-probe-timeout", "2s"]
interval: 10s
timeout: 3s
retries: 3
start_period: 10s
restart: unless-stopped
volumes:
gateway-data:
docs/ALERTS.md
+78
View File
@@ -0,0 +1,78 @@
# Alerts and signed webhooks
The alert manager evaluates bounded operational conditions and persists firing/resolved history. It never includes prompt, response, tool, or conversation content in webhook payloads.
## Conditions
Supported conditions include:
- worker unhealthy for `worker_down_for`;
- circuit breaker open;
- queue depth threshold;
- oldest queue wait threshold;
- VRAM pressure;
- repeated OOM indication from the worker/circuit error;
- local gateway storage size;
- actor or tenant quota remaining percentage.
Example:
```json
{
"alerts": {
"enabled": true,
"evaluation_interval": "15s",
"cooldown": "5m",
"history_limit": 500,
"webhook_timeout": "5s",
"webhook_max_concurrent": 4,
"webhook_queue": 1024,
"webhook_retry_attempts": 3,
"webhook_retry_backoff": "500ms",
"thresholds": {
"worker_down_for": "30s",
"circuit_open": true,
"queue_depth": 100,
"queue_wait": "30s",
"vram_percent": 95,
"storage_bytes": 0,
"quota_remaining_percent": 10,
"oom": true
},
"webhooks": []
}
}
```
## Webhook delivery
A webhook payload has a stable event ID and is delivered asynchronously from a bounded queue. Transient transport failures, HTTP 429, and HTTP 5xx are retried with bounded exponential backoff. Other HTTP 4xx responses are treated as permanent failures.
Headers:
- `X-Ollama-Gateway-Event-ID`
- `X-Ollama-Gateway-Timestamp`
- `X-Ollama-Gateway-Delivery-Attempt`
- `X-Ollama-Gateway-Signature` when a secret is configured
Signature format:
```text
sha256=HMAC_SHA256(secret, timestamp + "." + raw_json_body)
```
Consumers should deduplicate by event ID because retries intentionally reuse the same event ID.
## Hot-path isolation
Quota observations happen during request admission, but disk persistence and webhook I/O are not performed synchronously on that request path. State persistence is coalesced asynchronously and webhook delivery uses a bounded queue.
## Secrets
Webhook secrets are redacted from the Admin JSON configuration view. OpenTelemetry header values are redacted there as well. Saving the redacted configuration restores the existing secret values instead of persisting the literal `<redacted>` placeholder.
## Metrics
- `ollama_gateway_queue_oldest_wait_seconds`
- `ollama_gateway_alerts_active`
- `ollama_gateway_alerts_last_evaluate_timestamp_seconds`
docs/ARCHITECTURE.md
+77
View File
@@ -0,0 +1,77 @@
# Architecture
## Request path
```text
HTTP request
-> authentication / identity
-> compute request parsing + cost estimate
-> in-memory quota reservation
-> hierarchical in-memory WFQ
-> model-aware worker selection
-> streaming reverse proxy
-> actual usage reconciliation
-> in-memory usage aggregation
-> optional asynchronous JSONL journal
```
Management and blob endpoints do not enter the compute scheduler and keep their request body streaming end-to-end.
## Single authoritative engine
The gateway deliberately has no external coordination backend. All mutable control state belongs to one process. This means there is exactly one authoritative view of queue order, quota buckets and worker slots and therefore no distributed lock, lease renewal, consensus or fail-open state split.
The hot-path structures are Go-native:
- scheduler maps and heaps protected by one scheduler mutex;
- worker load using atomic counters;
- quota buckets protected by a small mutex;
- runtime policy and session maps behind RWMutex/Mutex;
- live request snapshots behind the lifecycle tracker;
- usage summaries updated in memory and journaled asynchronously.
## Fair scheduler
`internal/scheduler` implements two-level hierarchical weighted fair queueing.
A tenant has a root service score. Inside that tenant each queued request has an actor virtual-finish score derived from request cost and actor weight. When capacity becomes free, the scheduler selects the tenant with the lowest root score and then its request with the lowest actor finish score.
This yields two properties:
- tenant fairness is not defeated by creating many actors;
- actors within one tenant still share service fairly.
The scheduler is event-driven. A worker slot release signals the scheduler immediately; there is no polling backend.
## Quotas
`internal/quota` is an in-memory hierarchical token-bucket ledger whose bucket balances are periodically snapshotted for restart continuity. A request reserves estimated compute credits from actor and tenant buckets. Final usage reconciles the reservation. Token buckets refill lazily from monotonic wall-clock deltas.
## Worker routing
`internal/worker` maintains health and model residency snapshots for each Ollama endpoint. Local `active` counters are atomic and are the hard worker slot limiter. Candidate scoring combines active-slot pressure with a strong affinity bonus when the requested model is already loaded.
There are no worker leases or renewal goroutines.
## Live flow and infrastructure map
`internal/liveflow` tracks bounded prompt-free lifecycle metadata. `internal/infrastructure` turns that local state into the UI topology snapshot. The browser receives it over SSE and renders the animated pulse map with Canvas.
The infrastructure view is explicitly local to the process. It shows one gateway node plus all configured Ollama workers and models.
## Runtime policy store and sessions
Tenant policy overrides are backed by the local persistent policy store. Browser OIDC sessions remain in memory and disappear on restart by design.
## Usage
`internal/usage` maintains actor, tenant and global summaries in memory. The optional file journal runs asynchronously on a buffered channel and never participates in admission or request dispatch.
## Scaling model
Scale inference by adding Ollama workers to the one gateway. Running multiple independent gateway replicas would create independent fairness domains. If strict global fairness is required, requests must pass through the same gateway engine.
## Optional content-bearing conversation state
The Responses conversation layer is separate from browser OIDC sessions. Browser sessions remain volatile. When explicitly enabled, Responses conversation contexts are tenant/actor scoped, encrypted at rest, retention bounded, and used only to expand `previous_response_id` before normal admission/routing. The ordinary proxy path does not capture response bodies; bounded capture is enabled only for store-eligible `/v1/responses` requests.
docs/BATCH-JOBS.md
+140
View File
@@ -0,0 +1,140 @@
# Durable batch jobs
Durable batch jobs are the P3.1 background-execution layer. They are **disabled by default** because each submitted job intentionally persists its request body and, when available, its response body.
## Configuration
```json
"batch_jobs": {
"enabled": false,
"retention": "168h",
"max_jobs": 1000,
"max_concurrent": 1,
"max_input_bytes": 16777216
}
```
The service class `batch` must also exist under `service_classes.classes`. `batch_jobs.max_input_bytes` must not exceed `server.max_body_bytes`.
Storage paths are bootstrap-only:
```json
"storage": {
"batch_jobs_file": "batch-jobs.json",
"batch_jobs_dir": "batch"
}
```
## Client API
A batch job wraps one normal configured compute `POST` endpoint. The request body is the JSON body that the gateway will replay later.
```http
POST /gateway/v1/batches
Authorization: Bearer <credential>
Content-Type: application/json
{
"path": "/api/chat",
"body": {
"model": "qwen3:8b",
"messages": [{"role":"user","content":"Summarize the report"}],
"stream": false
}
}
```
A successful submission returns `202 Accepted`, a `Location: /gateway/v1/batches/<id>` header and the durable job metadata.
Owner-scoped endpoints:
```text
GET /gateway/v1/batches
GET /gateway/v1/batches/<id>
GET /gateway/v1/batches/<id>/output
POST /gateway/v1/batches/<id>/pause
POST /gateway/v1/batches/<id>/resume
POST /gateway/v1/batches/<id>/cancel
```
A caller can only see or control jobs created by the same authenticated tenant and scheduler actor. A cross-identity lookup is returned as not found.
## Admin UI and API
Administrators have a dedicated **Batch Jobs** page with all durable jobs, state, attempts, identity metadata, model/path, pause/resume/cancel controls and authenticated output download.
Admin endpoints require `gateway:admin`:
```text
GET /gateway/ui-api/batches
GET /gateway/ui-api/batches/<id>
GET /gateway/ui-api/batches/<id>/output
POST /gateway/ui-api/batches/<id>/pause
POST /gateway/ui-api/batches/<id>/resume
POST /gateway/ui-api/batches/<id>/cancel
```
## Execution semantics
Every attempt is replayed into the same gateway compute path as an interactive request, with the original identity metadata and a forced service class of `batch`. Therefore the attempt still passes through:
- tenant model ACLs and the submitted key-specific ACL snapshot;
- alias resolution, capability preflight, placement, maintenance state and circuit eligibility;
- quota reservation/reconciliation;
- weighted fair scheduling and the configured `batch` class concurrency ceiling;
- adaptive worker routing and normal safe pre-stream retry behavior;
- usage metering, request IDs, alerts and OpenTelemetry.
The submission credential itself is never stored. The durable identity snapshot contains tenant/subject/application/scopes and the key-specific model ACL needed to execute an already accepted job. Current tenant policy, worker state, placement, quota state and routing configuration are evaluated again at execution time. Revoking the submitting API key does not retroactively cancel an already accepted durable job; cancel the job explicitly if that is required operationally.
## States
```text
queued -> running -> completed
| | \
| | -> failed
| -> pausing -> paused -> queued
-> paused
-> cancelled
running/pausing -> cancelling -> cancelled
```
`attempts` increments when an execution attempt starts. A failed gateway/backend response is terminal; the durable batch manager itself does not blindly retry application failures.
## Restart and shutdown behavior
The metadata snapshot is written atomically. Input/output payloads live in separate spool files referenced from metadata.
On startup:
- `running` becomes `queued` and is eligible for retry;
- `pausing` becomes `paused`;
- `cancelling` becomes `cancelled`.
During graceful shutdown the root context cancels active attempts, and the process waits for those attempts to persist their restart-safe `queued` state before final shutdown accounting is closed. A hard kill can still leave metadata in `running`; startup recovery converts that state to `queued`.
This gives durable batches **at-least-once execution across an interruption boundary**, not exactly-once execution. If a worker completed side effects but the gateway did not durably commit the batch result before shutdown/crash, that job can run again after restart. Use idempotent batch workloads or application-level idempotency keys when duplicate execution would be harmful.
## Storage, privacy and retention
Layout:
```text
<data_dir>/batch-jobs.json
<data_dir>/batch/input/<batch-id>.json
<data_dir>/batch/output/<batch-id>.response
```
Metadata and spool payload files are created with mode `0600`; spool directories use `0750`. Input/output references are stored in `batch-jobs.json`, not the potentially large payloads themselves.
**Batch payloads are content-bearing and are not encrypted by the gateway.** This differs from the optional encrypted conversation store. Keep `batch_jobs.enabled=false` unless durable content persistence is intended, protect `storage.data_dir` with filesystem/disk encryption and access controls, and treat gateway backups as sensitive.
Terminal jobs and their input/output files are removed after `batch_jobs.retention`. `max_jobs` caps retained metadata/jobs; `max_input_bytes` caps one submitted request body.
## Accounting
Each execution attempt receives its own normal `X-Request-ID`. The durable job stores the latest execution request ID and HTTP status. The attempt appears in the ordinary usage journal with `service_class: "batch"`; prompt/completion tokens and compute credits therefore use the same accounting and rollup path as other inference traffic.
## Current scope
P3.1 deliberately implements one durable compute request per job. It is not an OpenAI Batch API clone and does not yet ingest multi-request JSONL files. Global batch coordination across multiple gateway replicas remains intentionally deferred outside the active roadmap; the current design stays single-process and durable.
docs/CONTEXT-HARDENING.md
+69
View File
@@ -0,0 +1,69 @@
# Context / `num_ctx` hardening
Checkpoint 27 treats context size as a runtime resource rather than only a model capability.
## Effective context
For requests that cannot set native Ollama `options.num_ctx` (OpenAI `/v1/*`, Responses and Anthropic-compatible routes), each worker gets an effective context window from the strongest available evidence:
1. loaded `/api/ps` `context_length`;
2. Modelfile `num_ctx` returned by `/api/show` `parameters`;
3. `workers[].default_context_tokens`;
4. `model_capabilities.context.default_worker_tokens`.
The result is clamped by the theoretical model maximum and `workers[].context_limits`. The request is routed only to workers that can hold the estimated input + output budget. If no worker can hold it and `context_guard` is `reject`, admission fails before queueing or inference.
## Native `options.num_ctx`
`options.num_ctx` is accepted only on native Ollama `/api/*` requests. It must be a positive JSON integer. The gateway checks it against:
- `model_capabilities.context.max_requested_tokens` (default 32768; `-1` removes the gateway cap);
- the model maximum discovered from `/api/show`;
- a matching `workers[].context_limits` entry.
A model already loaded with a smaller context may still receive a larger explicit `num_ctx` because Ollama can resize/reload it, but that worker loses the loaded-model routing bonus for that request.
## Conservative request estimate
The context guard now includes:
- `prompt`, messages and `system`;
- Responses `instructions` and `input`;
- native generate `suffix`;
- tool schema JSON;
- `max_tokens`, `max_completion_tokens`, `max_output_tokens` and native `num_predict`;
- a configurable reserve per image (`vision_reserve_tokens_per_image`);
- a configurable input safety margin (`estimation_margin_percent`).
The estimator intentionally remains provider-independent and is not an exact tokenizer. The margin and image reserve compensate for that uncertainty.
## Recommended configuration
```json
"model_capabilities": {
"mode": "enforce",
"cache_ttl": "10m",
"context_guard": "reject",
"context": {
"max_requested_tokens": 32768,
"default_worker_tokens": 4096,
"estimation_margin_percent": 15,
"vision_reserve_tokens_per_image": 2048
}
}
```
For a worker with a known different Ollama default or a stricter memory envelope:
```json
{
"name": "worker-a",
"default_context_tokens": 4096,
"context_limits": {
"large-*": 16384,
"*": 32768
}
}
```
Use a Modelfile `num_ctx` when OpenAI-compatible clients need a larger stable context, because those APIs do not provide native per-request `options.num_ctx` semantics.
docs/CONVERSATIONS.md
+67
View File
@@ -0,0 +1,67 @@
# Optional encrypted Responses conversations
`conversations` implements the P2.3 stateful layer for clients that use the OpenAI Responses API with `previous_response_id`.
The feature is **disabled by default**. When disabled, the gateway does not parse, capture, persist, or reinterpret response content for conversation purposes; `/v1/responses` keeps its normal passthrough behavior.
## Configuration
```json
"conversations": {
"enabled": false,
"encryption_key": "${GATEWAY_CONVERSATION_KEY}",
"retention": "24h",
"max_entries": 1000,
"max_content_bytes": 2097152
},
"storage": {
"conversations_file": "conversations.enc.json"
}
```
When enabled:
- `encryption_key` must contain at least 32 characters. Use an environment variable or another deployment secret source; do not commit it.
- `retention` is independent from usage-journal retention because this store contains prompt/output content.
- `max_entries` bounds the number of stored response contexts; oldest entries are evicted first.
- `max_content_bytes` bounds one flattened conversation context and also bounds the response capture used to construct it.
The configured key is SHA-256-derived into an AES-256 key. The state snapshot is encrypted with AES-256-GCM and written mode `0600` through the same fsync + atomic-replace mechanism used by the other local state files. The on-disk JSON envelope contains only version/algorithm metadata, nonce, ciphertext, and update time; conversation content is not plaintext on disk.
Changing or losing the encryption key makes the existing store unreadable. Startup fails closed if an enabled store cannot be decrypted.
## `previous_response_id` behavior
For an enabled store, a successful `/v1/responses` request is stored when the request does not set `"store": false`.
For a later request containing `previous_response_id`:
1. the ID is looked up only inside the authenticated tenant + actor boundary;
2. the prior flattened input/output item context is loaded;
3. the current `input` is appended;
4. `previous_response_id` is removed before forwarding upstream;
5. the resulting complete `input` is sent through the normal ACL, preflight, quota, queue, placement, routing and proxy path.
A missing ID and an ID owned by another tenant/actor are intentionally indistinguishable to the caller. Both fail as an invalid previous response reference, preventing cross-identity enumeration.
String `input` values are normalized to a user message item when history must be flattened. Existing array/object inputs are retained as items. Previous `instructions` are not copied into the stored conversation context; only input/output items are chained.
## Streaming
The proxy normally does not retain response bodies. Conversation capture is activated only for `/v1/responses` when conversations are enabled and the request is eligible for storage.
For streaming Responses, the gateway reconstructs persisted output from `response.completed` (or completed output-item events when necessary). Client streaming remains unchanged. If the capture exceeds `max_content_bytes`, the inference response still succeeds but that response is not stored for future chaining.
## Retention and deletion
Expired entries are pruned on reads/writes and by a background cleanup loop. The cleanup interval is derived from the configured retention and capped at 15 minutes, so content does not depend on future traffic to expire.
`store: false` is the per-request opt-out. It still allows a request to consume a valid prior response context, but the new response is not persisted as the next link.
The store is intentionally not exposed as a content browser in the Admin UI. The persistence page reports only operational metadata such as whether it is enabled and the number of retained entries.
## Backups and threat model
The Admin storage backup includes the encrypted conversation state file when present. Treat backups as sensitive anyway: gateway configuration backups can contain deployment secrets, and possession of both the conversation ciphertext and its encryption key allows decryption.
This feature protects content at rest from casual filesystem disclosure. It does not protect content from a compromised running gateway process, from an administrator who possesses the configured key, or from the Ollama worker that receives the expanded prompt context.
docs/DEPLOYMENT-HARDENING.md
+117
View File
@@ -0,0 +1,117 @@
# Deployment hardening and preflight
Checkpoint 19 adds deployment checks intended to catch configuration/mount mistakes before production traffic reaches the gateway. These checks do not participate in the inference hot path.
## Effective configuration preflight
Run the gateway binary with `-check-config` before starting the service:
```sh
./ollama-gateway -config ./gateway-config.json -check-config
```
The command validates the same strict bootstrap schema as normal startup, loads the persistent UI override from `storage.data_dir` when present, enforces the bootstrap-only storage rule, verifies that the effective state directory is writable, and prints a secret-free JSON summary. A non-zero exit code means the gateway should not be started with that configuration.
The summary includes:
- bootstrap config path,
- whether a persistent override was loaded,
- persistent override path,
- effective worker count,
- effective/absolute state directory,
- storage writability,
- non-fatal deployment warnings.
Warnings currently cover common production foot-guns such as `local_system_stats=true` on a remote worker URL, an unknown `native.control_worker`, public metrics, a relative state directory and UI authentication/cookie combinations that deserve review.
For Compose deployments, run the check in the exact image/user/mount context that will be used in production:
```sh
GATEWAY_CONFIG=./gateway-config.json docker compose run --rm gateway \
-config /etc/ollama-gateway/config.json -check-config
```
This is preferable to validating only on the Docker host because it also proves that the container user can read the bootstrap file and write the mounted state volume.
## Built-in HTTP probe
The scratch image contains no shell, curl or wget. The gateway binary therefore has a minimal HTTP probe mode:
```sh
/ollama-gateway -probe http://127.0.0.1:8080/healthz -probe-timeout 2s
```
It exits 0 for any 2xx response and non-zero for connection errors, timeouts or non-2xx responses. Probe mode does not load gateway configuration or state.
The image/Compose liveness healthcheck uses `/healthz`. Load balancers and rollout automation should additionally require `/readyz`; readiness checks scheduler, workers and persistent runtime stores and can return 503 while the process itself remains healthy.
## Compose entrypoint rule
The image already defines:
```text
ENTRYPOINT ["/ollama-gateway"]
```
Therefore Compose `command` must contain only arguments:
```yaml
command: ["-config", "/etc/ollama-gateway/config.json"]
```
Do **not** repeat `/ollama-gateway` in `command`. Repeating the executable turns it into the first positional argument passed to Go's flag parser and can prevent following flags from being interpreted as intended.
The supplied Compose file supports selecting a bootstrap configuration without editing the file:
```sh
GATEWAY_CONFIG=./gateway-config.json docker compose up -d --build
```
If `GATEWAY_CONFIG` is omitted, the development `config.example.json` is mounted. Do not mistake that default example for a production configuration; `-check-config` reports the effective worker count and makes that error easier to detect before rollout.
## Recommended staged update
1. Back up the bootstrap config and `storage.data_dir`.
2. Build/pull the candidate image.
3. Run `-check-config` in the candidate container with the production mounts.
4. Stop the old gateway gracefully.
5. Start the candidate and wait for the Docker liveness check.
6. Require `/readyz` before restoring traffic.
7. Verify authenticated `/gateway/ui-api/session`, one non-streaming request and one streaming request if used.
8. Keep the previous image/binary and state backup available until the observation window is complete.
## Checkpoint 23: reverse-proxy boundary and container sandbox
The supplied Compose file now publishes the gateway on host loopback by default:
```text
127.0.0.1:9080 -> container :8080
```
This is the preferred topology when the TLS/reverse proxy runs on the same Docker host. It prevents LAN clients from bypassing the proxy and reaching the gateway's published port directly. Override `GATEWAY_PUBLISH_ADDRESS` only when the real proxy is on another host. In that case bind to the specific host interface where possible and enforce a host firewall/ACL that permits only the real proxy source address.
The Compose runtime also uses a read-only root filesystem, drops all Linux capabilities and enables `no-new-privileges`. `/data` remains the only persistent writable application state mount and `/tmp` is an ephemeral tmpfs.
For the deployment observed during the checkpoint-22 rollout, browser traffic arrived at the container through Docker with TCP peer `172.30.3.1`. The hardened production configuration therefore replaces broad RFC1918 `trusted_proxies` ranges with the exact `172.30.3.1/32` peer. `ip_bypass_use_forwarded_ip` remains false. If the network path changes, re-observe the gateway's `remote_addr` and trust only the actual proxy/NAT peer that is expected to supply forwarded headers.
A host-side preflight catches the common rollout mistakes before Compose starts production traffic:
```sh
cp .env.production.example .env
GATEWAY_CONFIG=./gateway-config.json ./scripts/production-preflight.sh
```
It fails if the production config points to `config.example.json`, if the gateway is published outside loopback without explicit acknowledgement, if Compose rendering fails, or if the gateway's own `-check-config` rejects the effective configuration/state mounts.
For a remote reverse proxy, use an explicit interface and firewall policy, for example:
```sh
GATEWAY_PUBLISH_ADDRESS=10.2.10.20 \
ALLOW_NON_LOOPBACK_BIND=1 \
GATEWAY_CONFIG=./gateway-config.json \
./scripts/production-preflight.sh
```
The explicit opt-in is not a substitute for a firewall. The published port should be reachable only from the reverse proxy.
For production, prefer the stricter `docker-compose.production.yml`. Unlike the development Compose file, it requires `GATEWAY_CONFIG` to be set and never falls back to `config.example.json`.
docs/HA-READINESS.md
+131
View File
@@ -0,0 +1,131 @@
> **Status: archived diagnostic path.** Multi-gateway HA was removed from the active roadmap in checkpoint 28 because current single-process performance does not justify distributed coordination. Keep this tooling for future re-evaluation if requirements change.
# P3.2 HA readiness and single-process limit characterization
P3.2 is deliberately gated: do not add distributed consensus, global queue state or leader election until the single-process gateway is shown to be the limiting availability/capacity component rather than Ollama/GPU inference.
This repository includes a small evidence toolchain for that decision:
- `cmd/mock-ollama`: deterministic Ollama/OpenAI-compatible mock backend that removes model compute from the measurement;
- `cmd/bench`: concurrent OpenAI request driver with latency, TTFB, status, bytes, token throughput and JSON output;
- `scripts/ha-readiness.sh`: repeatable concurrency sweep that writes one JSON result per level and captures Prometheus snapshots before/after each level;
- `cmd/ha-snapshot`: optional host/process endpoint snapshotter for gateway RSS/CPU/threads/FDs and host CPU/memory/load evidence;
- `cmd/ha-sampler`: sustained resource sampler for each load level, including sampled peak RSS/CPU/threads/FDs and host-memory/load pressure;
- `cmd/ha-report`: reconciles client benchmark counts with gateway Prometheus deltas, attaches endpoint and sustained resource evidence, and writes machine-readable JSON plus a Markdown evidence summary.
## 1. Build/run the mock backend
```bash
go run ./cmd/mock-ollama \
-listen 127.0.0.1:11435 \
-model qwen3:8b \
-delay 0 \
-response-bytes 128
```
Point one gateway worker at `http://127.0.0.1:11435`. The mock implements the worker inventory/metadata endpoints plus native and OpenAI chat endpoints required by the gateway.
Useful mock controls:
```text
-delay 5ms fixed pre-response latency
-stream-delay 10ms delay between streaming chunks
-response-bytes N generated payload size
-fail-every N deterministic HTTP 503 every Nth inference request
```
The zero-delay mode approximates gateway/control-plane overhead. Delay/stream-delay modes exercise many concurrent in-flight connections without requiring a GPU.
## 2. Run a concurrency sweep
For an authenticated gateway:
```bash
export GATEWAY_BENCH_API_KEY='...'
export GATEWAY_PID="$(pgrep -n ollama-gateway)"
BASE_URL=http://127.0.0.1:8080 \
MODEL=qwen3:8b \
REQUESTS=1000 \
WARMUP=50 \
CONCURRENCIES='1 4 16 32 64 128' \
./scripts/ha-readiness.sh
```
Each level prints a human summary and writes `concurrency-<N>.json` plus `metrics-before-c<N>.prom` and `metrics-after-c<N>.prom`. When `GATEWAY_PID` is set, the sweep also writes `resources-before-c<N>.json`, `resources-after-c<N>.json`, and `resources-samples-c<N>.json`. After the sweep, `cmd/ha-report` writes `report.json` and `report.md`. The report verifies that the gateway request, queue and service-observation deltas equal **warmup + measured requests** for every level and records endpoint-resource and sustained-resource evidence completeness separately.
The sweep authenticates `/metrics` with `GATEWAY_BENCH_API_KEY` when that environment variable is set; the key is used only as an HTTP header and is never written to the result directory. Set `METRICS_URL` when metrics are exposed at a separate admin endpoint. Set `CAPTURE_METRICS=false` only for an intentionally client-only run; such a run does not produce reconciled HA evidence.
Endpoint resource capture defaults to `auto`: it is enabled when `GATEWAY_PID` is present and otherwise skipped. Set `CAPTURE_RESOURCES=true` to require it explicitly, or `CAPTURE_RESOURCES=false` to suppress it. Sustained capture separately defaults to `auto` through `CAPTURE_SUSTAINED_RESOURCES`; set it to `false` when an external profiler already supplies the sustained evidence. `GATEWAY_PID` must identify the actual gateway process, not a shell wrapper. `RESOURCE_SAMPLE_INTERVAL` defaults to `250ms`, and `RESOURCE_SAMPLE_MAX_DURATION` defaults to `15m` as a safety bound for a stuck benchmark.
The `resources-before/after` files are **endpoint snapshots immediately before and after a load level**. The `resources-samples` file is collected throughout the load level. On Linux, sampled process CPU uses deltas between `/proc/<pid>/stat` and aggregate `/proc/stat`, scaled to all logical CPUs, so a multi-core process can legitimately exceed 100%. The sampler also records sampled peak RSS, threads, open FDs and load plus minimum available host memory. On other platforms the tool falls back to repeated `ps` sampling and leaves unavailable fields absent.
Each benchmark JSON contains:
- successful/error counts and HTTP status distribution;
- requests/second;
- latency p50/p95/p99/max;
- TTFB p50/p95/p99/max;
- bytes received and bytes/second;
- prompt/completion tokens and completion tokens/second when non-streaming usage is available;
- stream/keep-alive/service-class settings.
To isolate connection setup cost, run an additional pass with `cmd/bench -disable-keepalive`. For long-lived response pressure, run with `STREAM=true` and configure a non-zero mock `-stream-delay`.
## 3. Measure three distinct regimes
### A. Gateway ceiling
Mock Ollama with `-delay 0`, small response body, non-streaming. Increase concurrency until throughput flattens or latency/error rate rises sharply. This measures HTTP/auth/admission/scheduler/routing/proxy/accounting overhead rather than LLM inference.
### B. Connection/stream pressure
Mock Ollama with streaming enabled and `-stream-delay` long enough to keep many requests open. This tests goroutine/socket/buffer behavior and cancellation/shutdown paths.
### C. Real inference
Repeat against the actual Ollama workers. This gives the practical service curve and reveals whether GPU/VRAM/model residency saturates far earlier than the gateway process.
## 4. Record gateway-side signals
The default sweep captures `/metrics` automatically. `cmd/ha-report` consumes the full `ollama_gateway_*` Prometheus names and derives per-level deltas. Important signals include:
- scheduler queued/running and queue wait;
- service-class queued/running;
- worker active/model-active slots;
- request/error/retry/circuit counters;
- usage-journal drop/storage gauges;
- process CPU/RSS/thread/file-descriptor and host load/memory endpoint snapshots from `cmd/ha-snapshot`;
- sampled peak process CPU/RSS/thread/file-descriptor and host pressure from `cmd/ha-sampler`, plus socket/runtime metrics from sustained host/container monitoring when deeper evidence is required.
The benchmark numbers are meaningful only together with CPU, memory and worker saturation. A throughput plateau while GPU/worker slots are saturated is not evidence that HA is needed for gateway capacity.
## 5. HA decision gate
Proceed from readiness work to a real P3.2 coordinator/cluster design only when at least one of these is demonstrated:
1. **Capacity:** gateway CPU/network/connection capacity saturates materially before the Ollama worker pool and cannot be resolved by tuning one process.
2. **Availability:** the required recovery objective cannot tolerate one authoritative gateway process, even with supervisor restart and durable local state.
3. **Operational topology:** multiple gateway instances are mandatory across failure domains, while strict global fairness/quota/batch semantics must still be preserved.
Do **not** deploy independent replicas behind a generic load balancer and call that P3.2. Independent replicas create separate fair queues, quota buckets, transient job registries and batch dispatchers. They improve process redundancy only by weakening global correctness.
### Evidence report gate states
- `incomplete`: one or more metric snapshots are missing, or gateway request/queue/service counters do not reconcile with the expected warmup + measured request count. Fix the measurement before drawing conclusions.
- `not-proven`: client and gateway counters reconcile, but the evidence still does not demonstrate one of the capacity/availability/topology criteria above. `resource_evidence_complete` states whether every level has before/after snapshots; `sustained_resource_evidence_complete` states whether every level has a valid sustained sampler trace. This is the normal result of a mock-backend smoke test.
The tool intentionally does **not** auto-promote the project into clustered HA based only on a throughput plateau. Sustained sampling closes the short-peak gap in endpoint snapshots, but it still must be interpreted with socket/runtime metrics and, for real inference runs, worker/GPU saturation.
## 6. P3.2 design constraints once the gate is met
The HA design must preserve the current hot-path properties:
- one authoritative global order for fair admission;
- globally consistent quota and durable batch state;
- fencing/epochs so a stale leader cannot dispatch duplicate authoritative work;
- membership and leader failover without making every streamed token depend on consensus;
- local worker proxying remains direct after a request is admitted/assigned;
- content-bearing batch/conversation storage retains its current explicit privacy boundaries;
- no mandatory Redis dependency is introduced merely as a shortcut.
A likely architecture is a small consensus-backed coordinator/control plane plus local proxy workers, not replicated copies of today’s independent in-memory scheduler.
docs/IMPROVEMENTS-2026.md
+198
View File
@@ -0,0 +1,198 @@
# Verbesserungen und Prioritäten (September 2026)
Dieses Dokument beschreibt die nach einem Abgleich mit Ollama, OpenWebUI und etablierten LLM-Gateways priorisierten Verbesserungen. Ziel bleibt ein einzelner, sehr schneller Go-Prozess ohne Redis oder Datenbank im Inference-Hot-Path.
## Priorität A – in diesem Release umgesetzt
### 1. Capability-aware Model Preflight
Ollama liefert über `POST /api/show` Modell-Capabilities und Modellinformationen. Das Gateway cached diese Metadaten pro Worker/Modell und erkennt aktuell:
- `completion`
- `tools`
- `vision`
- `thinking`
- `embedding`
Ein Request wird vor Admission gegen seine benötigten Capabilities geprüft. Das verhindert z. B., dass ein OpenWebUI-Tool-Request erst tief im Ollama-Backend mit `does not support tools` scheitert.
Konfiguration:
```json
"model_capabilities": {
"mode": "enforce",
"cache_ttl": "10m",
"context_guard": "reject",
"context": {
"max_requested_tokens": 32768,
"default_worker_tokens": 4096,
"estimation_margin_percent": 15,
"vision_reserve_tokens_per_image": 2048
}
}
```
`mode`:
- `enforce`: bekannte Capability-Konflikte mit HTTP 400 ablehnen.
- `observe`: nur Warnheader/Logs setzen.
- `off`: Metadaten-Preflight deaktivieren.
Wenn `/api/show` temporär nicht erreichbar ist, lässt das Gateway den Request bewusst durch. Verfügbarkeit hat bei unbekannten Metadaten Vorrang; nur *bekannt inkompatible* Requests werden geblockt.
### 2. Context Guard
Der Gateway trennt das **theoretische Modellmaximum** von dem Kontext, den ein konkreter Worker tatsächlich bereitstellt. Für OpenAI-/Responses-/Anthropic-kompatible Requests ohne natives `num_ctx` gilt als effektive Quelle in dieser Reihenfolge:
1. `context_length` des bereits geladenen Modells aus `/api/ps`;
2. `num_ctx` aus den Modelfile-Parametern von `/api/show`;
3. `workers[].default_context_tokens`;
4. `model_capabilities.context.default_worker_tokens`.
Anschließend greifen `workers[].context_limits` und der Gateway-Cap. Ein natives `options.num_ctx` wird nur auf `/api/*` akzeptiert, muss ein positiver Integer sein und wird gegen Gateway-Cap, Modellmaximum und Worker-Cap geprüft. Ein kleinerer aktuell geladener Kontext darf für ein explizit größeres `num_ctx` neu allokiert werden, erhält dann aber keinen Loaded-Routingbonus.
Die Admission-Schätzung zählt neben Prompt/Messages/Tool-Schemas jetzt auch `instructions`, `suffix`, `max_output_tokens`, eine konfigurierbare Vision-Reserve pro Bild und eine Sicherheitsmarge. Das ist bewusst eine konservative Schätzung und kein Tokenizer-Ersatz.
### 3. Per-model Concurrency
Ein 24-GB-Worker sollte kleine 8B-Modelle anders behandeln können als 27B-Modelle. Jeder Worker kann deshalb zusätzlich zu `max_concurrent` Modellgrenzen definieren:
```json
"model_concurrency": {
"qwen3:8b": 2,
"gemma4:*": 1,
"*": 1
}
```
Matching-Reihenfolge:
1. exakter Modellname;
2. längster passender Prefix mit abschließendem `*`;
3. `*`;
4. `max_concurrent`.
Der Slot wird im selben in-memory Worker-State geführt und bei Cancel/Fehler/Completion wieder freigegeben.
### 4. Adaptive Worker Routing
Worker-Routing berücksichtigt jetzt zusätzlich zu Health und aktiven Slots:
- bereits geladenes Modell;
- auf dem Worker installiertes Modell;
- per-model Slot-Auslastung;
- gelernte Output-Tokenrate je Worker/Modell (EWMA), persistent über Neustarts;
- tatsächlichen VRAM-Druck, wenn Telemetrie verfügbar ist;
- GPU-Auslastung;
- eine harte Zusatzstrafe für das Laden eines neuen Modells bei sehr hohem VRAM-Druck.
Die Gewichte sind konfigurierbar:
```json
"routing": {
"loaded_bonus": 60,
"installed_bonus": 30,
"throughput_bonus": 20,
"vram_pressure_penalty": 35,
"gpu_utilization_penalty": 10,
"avoid_vram_percent": 97
}
```
Durchsatzwerte werden ausschließlich aus echten abgeschlossenen Inferenzrequests gelernt und unter `storage.worker_performance_file` persistiert. Nach einem Neustart startet das Routing deshalb mit den zuletzt bekannten EWMA-Werten und lernt sie anschließend weiter.
### 5. NVIDIA-Telemetrie ohne zusätzlichen Agenten
Auf NVIDIA-Hosts kann ein Worker direkt `nvidia-smi` verwenden:
```json
"nvidia_smi": true,
"nvidia_gpu": "0"
```
Erfasst werden:
- GPU utilization
- VRAM used / total
- Temperatur
- Power Draw
Die Abfrage läuft außerhalb des Request-Hot-Paths im normalen Worker-Health-Zyklus. Das Gateway bleibt `CGO_ENABLED=0` und benötigt keine NVML-Bibliothek. Ein vorhandenes `telemetry_url` kann weiterhin verwendet werden und externe Werte überschreiben.
### 6. Operations UI und Metrics
Die Modellansicht zeigt jetzt erkannte Capabilities und Context Length. Worker-Karten zeigen – soweit vorhanden – GPU, VRAM, Temperatur, Leistung und gelernte tok/s.
Prometheus exportiert zusätzlich:
```text
ollama_gateway_worker_memory_used_bytes
ollama_gateway_worker_memory_total_bytes
ollama_gateway_worker_vram_used_bytes
ollama_gateway_worker_vram_total_bytes
ollama_gateway_worker_gpu_utilization_percent
ollama_gateway_worker_gpu_temperature_celsius
ollama_gateway_worker_gpu_power_watts
ollama_gateway_worker_model_active
ollama_gateway_worker_model_prompt_tokens_per_second
ollama_gateway_worker_model_output_tokens_per_second
ollama_gateway_worker_model_performance_samples
```
Modelle sind eine begrenzte operative Dimension. Tenant-/User-/Request-IDs werden weiterhin nicht als Prometheus-Labels ausgegeben.
### 7. Unbegrenzte Credits verständlicher im UI
`0` bedeutet weiterhin unbegrenzt. Die Policy-Oberfläche zeigt dafür jetzt `∞` und bietet explizite Checkboxen für unbegrenzte Actor- bzw. Tenant-Credits.
## RTX-4090-Startprofil
`config.rtx4090.example.json` enthält einen Startpunkt für eine RTX 4090 mit 24 GiB VRAM und beispielhaft 64 GiB System-RAM. Den RAM-Wert bitte an den tatsächlichen Host anpassen. Das Profil funktioniert unter Linux und Windows; beide Plattformen erfassen Host-RAM nativ, und `nvidia-smi` liefert die GPU-Telemetrie, sofern es im `PATH` verfügbar ist.
Für 24 GiB VRAM ist der wichtigste Unterschied zum 256-GB-Unified-Memory-Mac die Modell-spezifische Concurrency. Große ~17-GB-Modelle starten im Beispiel bei 1, kleine Modelle dürfen 2 parallele Slots erhalten.
## Priorität B – als nächste Ausbaustufe empfohlen
### Upstream Circuit Breaker + begrenzte Retries
Retries sind bei LLM-Streaming gefährlich, weil eine bereits begonnene Generierung nicht transparent wiederholt werden darf. Sinnvoll wäre deshalb:
- Retry nur vor dem ersten Response-Byte und nur bei eindeutig transienten Transportfehlern;
- Circuit Breaker pro Worker;
- Exponential Backoff bei Health-/Load-Fehlern;
- kein blindes Request Hedging, da es GPU-Arbeit dupliziert.
### API-Key Model ACLs
Pro API-Key/Tenant sollten erlaubte/verbotene Modell-Patterns möglich sein, z. B. `allowed_models: ["qwen3:*", "embeddinggemma:*"]`. Das ist besonders für teure oder unzensierte Modelle sinnvoll.
### Capability Routing / expliziter Fallback
Ein optionaler, explizit konfigurierter Fallback könnte Tool-Requests auf ein toolfähiges Modell umleiten. Default sollte `reject` bleiben, da ein stiller Modellwechsel semantisch überraschend ist.
### Persistentes Audit-Log als optionales Modul
Control-Plane-, Accounting- und Routing-Zustand ist inzwischen lokal persistent, während aktive Scheduling-/Socket-Zustände in-memory bleiben. Für Compliance wäre zusätzlich ein optionaler asynchroner Audit-Sink sinnvoll, der nur Admin-Aktionen und Metadaten schreibt und niemals Prompts/Antworten.
### OpenTelemetry
Prometheus deckt Aggregationen ab. Für verteilte Client-/Gateway-/Ollama-Latenz wäre optionales OTel Tracing sinnvoll, weiterhin ohne Prompt-Inhalte.
## Referenzen
- Ollama Show model details: https://docs.ollama.com/api-reference/show-model-details
- Ollama Tool calling: https://docs.ollama.com/capabilities/tool-calling
- Ollama Thinking: https://docs.ollama.com/capabilities/thinking
- Ollama Vision: https://docs.ollama.com/capabilities/vision
- Ollama Embeddings: https://docs.ollama.com/capabilities/embeddings
- OpenWebUI Ollama connection/context notes: https://docs.openwebui.com/getting-started/quick-start/connect-a-provider/starting-with-ollama/
- NVIDIA `nvidia-smi` selective query: https://docs.nvidia.com/deploy/nvidia-smi/
## Checkpoint 27 — Context / `num_ctx` Hardening
Der Context Guard unterscheidet jetzt zwischen dem theoretischen Modellmaximum (`/api/show model_info.*.context_length`) und dem tatsächlich nutzbaren Kontext eines Workers. Für Requests ohne explizites natives `options.num_ctx` gilt in dieser Reihenfolge: geladenes `/api/ps context_length`, Modelfile-`num_ctx`, `workers[].default_context_tokens`, dann der globale `model_capabilities.context.default_worker_tokens`. `workers[].context_limits` kann Modelle pro Worker zusätzlich begrenzen.
Native `options.num_ctx` muss ein positiver Integer sein. Es wird gegen Gateway-Cap, Modellmaximum und Worker-Cap geprüft. Ein bereits kleiner geladenes Modell bleibt für einen explizit größeren `num_ctx` grundsätzlich routbar, verliert aber den Loaded-Bonus, weil Ollama den KV-Cache neu allozieren muss.
Die Preflight-Schätzung berücksichtigt nun außerdem `max_output_tokens`, `instructions`, `suffix`, einen konfigurierbaren Vision-Reservewert pro Bild sowie eine Sicherheitsmarge. Die Modellansicht zeigt Modell-Maximum, Modelfile-Kontext und geladenen Kontext getrennt.
docs/JOBS.md
+47
View File
@@ -0,0 +1,47 @@
# Jobs and cancellation
The gateway distinguishes between **model lifecycle operations**, **transient inference jobs**, and **durable batch jobs**.
## Model Stop
The Models page uses **Stop** to unload a resident Ollama model. Ollama does not expose `/api/stop`; the gateway sends:
```http
POST /api/generate
Content-Type: application/json
{"model":"<model>","keep_alive":0,"stream":false}
```
This asks Ollama to unload the model immediately. It does not cancel a specific request.
## Inference jobs
Every compute request receives an `X-Request-ID` and is registered as an in-memory job while it is queued, routing, running, or streaming.
Admin endpoints:
```text
GET /gateway/ui-api/jobs
POST /gateway/ui-api/jobs/<request-id>/cancel
```
The web UI exposes these under **Jobs** and also places an **Abbrechen** button next to active requests in Live Flow.
### Cancellation semantics
- **queued**: remove the ticket from the WFQ heap immediately and release the credit reservation;
- **waiting for worker**: cancel worker acquisition and release the scheduler lease;
- **running/streaming**: cancel the upstream HTTP context so the Ollama request terminates, then release worker/scheduler slots;
- reconcile any usage already observable from the response;
- store terminal live state `cancelled` with internal status `499`.
For a streaming response whose upstream HTTP headers were already forwarded, the downstream client may still have HTTP status `200`; cancellation is observable as an early stream termination. Internal gateway telemetry records the request as `499`.
Transient inference-job state is process-local and disappears on gateway restart, consistent with the in-memory hot-path design.
## Durable batch jobs
P3.1 adds a separate durable job type under `/gateway/v1/batches`. Batch definitions, state, input references and output references survive restart; an active execution attempt itself is cancelled on shutdown and converted back to a restart-safe queued state.
The Admin UI exposes durable work on the dedicated **Batch Jobs** page rather than mixing it into the transient **Jobs** page. See `docs/BATCH-JOBS.md` for API, state-machine, persistence, privacy, retention and accounting semantics.
docs/MIGRATION-INMEMORY.md
+65
View File
@@ -0,0 +1,65 @@
# Migration to the in-memory engine
This release removes the external coordination backend entirely.
## Removed configuration
Delete these old top-level/settings from existing configuration files:
```text
redis
scheduler.distributed
scheduler.lease_ttl
scheduler.poll_interval
workers[].request_lease_ttl
usage.redis_aggregates
cluster
```
Replace the old local/cluster visualization settings with:
```json
"infrastructure": {
"node_name": "mac-studio-gateway",
"refresh_interval": "250ms",
"max_requests": 256
}
```
The configuration parser uses `DisallowUnknownFields`, so obsolete settings fail fast instead of being silently ignored.
## Runtime behavior
The following state now exists only inside one gateway process:
- weighted fair queue state;
- running concurrency;
- per-worker slots;
- actor and tenant quota buckets;
- runtime policy overrides;
- browser sessions;
- usage summaries;
- live/infrastructure telemetry.
Active scheduling state still resets on restart. Durable control-plane state (UI-created API-key hashes, policy overrides, quota balances, metrics snapshots and usage history) is now stored locally under `storage.data_dir`; the inference scheduler itself remains in memory.
## Multiple Ollama workers
No change is required. One gateway may still route to many Ollama workers and maintains one fair queue across all of them.
## Multiple gateway processes
Do not place multiple independent gateway processes behind a generic load balancer if you require strict global fairness. Each process is now a separate fairness/quota/session domain.
For the intended deployment, use one authoritative gateway and scale the Ollama worker pool behind it.
## API change
The former cluster visualization endpoints are now local infrastructure endpoints:
```text
GET /gateway/ui-api/infrastructure
GET /gateway/ui-api/infrastructure/stream
```
The web UI has been updated accordingly.
docs/MODEL-PLACEMENT.md
+137
View File
@@ -0,0 +1,137 @@
# Model Placement
Model Placement is the gateway's hard model-to-worker routing policy. It is evaluated **before** adaptive worker scoring, so GPU load, VRAM pressure, model affinity or learned throughput can never override a placement prohibition.
## Why it is separate from tenant policies
Tenant/Fairness policies answer:
> How much compute may a tenant or actor consume?
Model Placement answers:
> On which Ollama workers may a model execute?
Keeping those concerns separate makes both the configuration and the UI predictable.
## UI workflow
Open **Admin -> Model Placement**.
The matrix has one row per discovered model and one column per worker:
- `✓ installed` — allowed and installed on this worker.
- `○ allowed` — policy allows it, but the model is not currently installed there.
- `● loaded` — allowed, installed and currently resident according to `/api/ps`.
- `⛔ blocked` — the effective placement rule denies the model.
- `↳` — the cell is resolved by an exact model rule.
Click a cell to create an exact allow/deny exception. Clicking an exact UI exception again removes that exact rule so the broader prefix/default rule becomes effective again.
The worker cards open a ruleset editor with:
- **Allow all**: allow by default and optionally deny selected models/prefixes.
- **Whitelist**: deny by default and only allow selected models/prefixes.
- **Only installed** preset: builds a whitelist from the worker's current `/api/tags` inventory.
- **Block all** preset: an empty whitelist.
- **Reset to config default**: deletes the persistent UI override and immediately restores the worker's bootstrap/persistent-config baseline.
All UI changes are written to `storage.model_placement_file` (default `model-placement.json`) using the gateway's atomic state writer and take effect for new requests immediately.
## Pattern semantics
Patterns support:
```text
qwen3:8b exact model
gemma4:* prefix wildcard
orcarouter/* prefix wildcard
* all models
```
Wildcards are only supported as a single trailing `*`.
Resolution uses specificity:
1. exact model rule;
2. longest matching prefix rule;
3. worker mode (`allow_all` or `whitelist`).
At equal specificity, deny wins.
This means the following is valid and useful:
```json
{
"mode": "allow_all",
"allowed_models": ["gemma4:latest"],
"denied_models": ["gemma4:*"]
}
```
`gemma4:latest` is allowed because the exact rule is more specific, while other `gemma4:*` variants are blocked.
## Example: model A on node 1, model B on node 1 + 2
```json
"workers": [
{
"name": "node-1",
"url": "http://10.10.11.10:11434",
"max_concurrent": 4,
"model_placement": {
"mode": "whitelist",
"allowed_models": ["model-a:*", "model-b:*"],
"denied_models": []
}
},
{
"name": "node-2",
"url": "http://10.10.11.20:11434",
"max_concurrent": 2,
"model_placement": {
"mode": "whitelist",
"allowed_models": ["model-b:*"],
"denied_models": []
}
}
]
```
The adaptive router then chooses between node 1 and node 2 for `model-b:*`, but `model-a:*` can only run on node 1.
## Inventory behavior
Each worker's `/api/tags` inventory is tracked separately from `/api/ps` loaded state.
Routing behavior is:
1. remove unhealthy workers;
2. remove workers blocked by Model Placement;
3. if inventory is known, require the model to be installed;
4. enforce global and per-model concurrency;
5. score the remaining workers by loaded affinity, load, learned tok/s, GPU utilization and VRAM pressure.
If at least one eligible worker has a known installed copy, only those installed workers are considered. If all eligible inventories are known and none contains the model, the gateway returns a descriptive `model ... is not installed on any eligible worker` error instead of sending the request to an arbitrary node.
A worker whose inventory has never been successfully retrieved is treated as **inventory unknown**, not as "model absent". That provides a controlled fail-open path for transient `/api/tags` discovery failures while still respecting hard placement rules.
## Client model discovery
`/api/tags` and `/v1/models` are gateway-native aggregated endpoints. Models that are installed only on placement-blocked workers are excluded from client discovery. This keeps OpenWebUI and other clients aligned with the gateway's actual routing policy.
The admin model inventory remains unfiltered so operators can still see and manage models that are installed but deliberately blocked from inference.
## Persistence precedence
For a worker:
```text
persistent UI placement override
↓ (if absent)
config.json / persistent config baseline
↓
effective routing rule
```
Deleting the UI override restores the baseline immediately. The placement file is included in gateway ZIP backups and is shown on **Admin -> Persistenz**.
docs/OPENWEBUI.md
+134
View File
@@ -0,0 +1,134 @@
# OpenWebUI integration
## Recommended connection
Use OpenWebUI's Ollama provider connection with the gateway base URL (no `/api` suffix) and a dedicated gateway API key. You can either configure a durable key in `auth.api_keys` or create a persistent key under **Admin -> Sicherheit -> API Keys**.
Gateway configuration:
```json
"auth": {
"oidc": { "enabled": false },
"api_keys": [
{
"name": "openwebui",
"key": "${OPENWEBUI_GATEWAY_KEY}",
"tenant": "interactive",
"subject": "openwebui",
"application": "openwebui",
"scopes": []
}
],
"ip_bypass": [],
"trusted_proxies": []
}
```
### Create the key in the web interface
Open **Admin -> Sicherheit -> API Keys -> API-Key erstellen** and use, for example:
```text
Name: openwebui
Tenant: interactive
Application: openwebui
Scopes: (empty)
```
Copy the generated `ofg_...` secret immediately and paste it into OpenWebUI. The gateway does not retain the plaintext value. The plaintext is shown once; only its SHA-256 hash and metadata are persisted, so the same OpenWebUI credential remains valid across gateway restarts.
OpenWebUI connection:
```text
URL: http://host.docker.internal:8080
API Key: <same value as OPENWEBUI_GATEWAY_KEY>
```
OpenWebUI's backend appends native Ollama paths itself. Do not configure the URL as `...:8080/api`.
## Diagnostics
From a machine that can reach the gateway:
```bash
curl -i -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" http://GATEWAY:8080/api/version
curl -i -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" http://GATEWAY:8080/api/tags
```
Expected status for both is HTTP 200. `/api/tags` must contain a `models` array and every returned model is normalized to contain both `name` and `model`.
OpenAI-compatible discovery is also available:
```bash
curl -i -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" http://GATEWAY:8080/v1/models
```
## Docker networking
The default example IP bypass only trusts loopback. An OpenWebUI Docker container normally reaches the gateway from a Docker/host network address, so loopback bypass does not apply. Prefer a dedicated API key over widening the bypass CIDR.
On Docker Desktop for macOS, `host.docker.internal` normally resolves to the host. On Linux, use an address/service name reachable from the OpenWebUI backend or configure Docker's host-gateway mapping.
## Gateway compatibility behavior
The gateway owns model discovery instead of forwarding it to a single control worker:
- `GET /api/tags`: parallel inventory across all workers, deduplicated by model ID.
- `GET /api/ps`: aggregated loaded-model view.
- `GET /v1/models`: OpenAI model list generated from the same aggregate inventory.
- `POST /api/show`: routed to a worker that owns the requested installed model when known.
- Compute requests are constrained to workers known to own the model; a loaded copy receives additional affinity preference.
If one worker fails during discovery but another returns models, the gateway returns the available model set and sets `X-Gateway-Partial-Errors`. If no worker can provide tags, discovery returns HTTP 503 rather than an empty, misleading list.
## OpenWebUI with "Authentication: None" and IP bypass
If the OpenWebUI connection is configured with **Authentication: None**, the gateway must authenticate the OpenWebUI backend through `auth.ip_bypass`. The bypass is evaluated against the **source IP observed by the gateway**, not against the gateway URL.
For example, if the gateway log shows:
```text
authentication rejected client_ip=10.10.11.42 ... path=/api/version ...
```
add only that OpenWebUI host address when possible:
```json
"ip_bypass": [
{
"cidrs": ["10.10.11.42/32"],
"tenant": "interactive",
"subject": "openwebui",
"application": "openwebui",
"scopes": []
},
{
"cidrs": ["127.0.0.1/32", "::1/128"],
"tenant": "local",
"subject": "localhost",
"application": "local-tools",
"scopes": ["gateway:admin"]
}
]
```
Then OpenWebUI may remain configured with:
```text
URL: http://10.10.11.123:8080
Authentication: None
```
Do **not** blindly add `10.10.11.123/32` just because that is the gateway URL. That is the destination address. Use the `client_ip` value printed by the gateway when OpenWebUI performs `/api/version` or `/api/tags`.
For Docker, the observed source may be a Docker bridge/VM address rather than the LAN address of the host. A dedicated static API key is more stable than allowing a broad Docker subnet.
### Authentication diagnostics
Authentication failures now log the resolved client address and include it in the `X-Gateway-Client-IP` response header. Native Ollama endpoints also return Ollama-compatible errors such as:
```json
{"error":"authentication required"}
```
rather than an OpenAI-shaped nested object. This prevents OpenWebUI from rendering the gateway error as `[object Object]`.
docs/PERSISTENCE.md
+93
View File
@@ -0,0 +1,93 @@
# Persistent local state
The gateway deliberately separates the inference hot path from durable state. Fair queue ordering, active worker slots, live requests and browser OIDC sessions remain in memory. Restart-worthy control-plane/accounting data is stored under `storage.data_dir` using only the Go standard library.
## Files
| File | Contents | Write pattern |
|---|---|---|
| `gateway-config.json` | configuration override saved from Admin -> Konfiguration | atomic replace |
| `api-keys.json` | UI-created key metadata + SHA-256 hashes | atomic replace |
| `policies.json` | tenant policy overrides | atomic replace |
| `metrics.json` | Prometheus counters and histogram state | periodic atomic snapshot + shutdown |
| `quota.json` | actor/tenant credit bucket balances | periodic atomic snapshot + shutdown |
| `worker-performance.json` | learned per-worker/model prompt/output tok/s EWMA | periodic atomic snapshot + shutdown |
| `model-placement.json` | live per-worker model placement overrides from the admin UI | atomic replace |
| `conversations.enc.json` | optional encrypted Responses conversation contexts | synchronous encrypted atomic replace + retention cleanup |
| `batch-jobs.json` | optional durable batch definitions, identity metadata, state and spool references | synchronous atomic replace on state transitions |
| `batch/input/*.json` | durable batch request payloads | create + fsync + rename, mode 0600 |
| `batch/output/*.response` | durable batch response payloads | temp file + fsync + rename, mode 0600 |
| `usage/usage-YYYY-MM-DD.jsonl` | request-level accounting inside detail retention | append-only, buffered |
| `usage/rollups/daily/rollup-daily-YYYY-MM-DD.json` | daily dimensional usage aggregates | atomic replace during compaction |
| `usage/rollups/monthly/rollup-monthly-YYYY-MM.json` | long-term monthly dimensional aggregates | idempotent atomic replace |
By default, no prompt text or generated response content is written by these stores. There are two explicit opt-in exceptions: `conversations.enabled=true` stores Responses context encrypted at rest, while `batch_jobs.enabled=true` allows submitted batch request/response content to be written to the local spool in plaintext files protected by filesystem permissions. See `docs/CONVERSATIONS.md` and `docs/BATCH-JOBS.md`.
## Bootstrap and persistent configuration
The file passed with `-config` is the bootstrap configuration. It determines `storage.data_dir` and the state filenames. If `<data_dir>/gateway-config.json` exists and validates, it becomes the effective configuration for the process.
The web JSON editor saves a complete validated override but never exposes API-key secrets, the UI session secret or the browser OIDC client secret. Redacted secret values are preserved from the currently effective configuration. API keys themselves should be managed through the dedicated Security page.
The `storage` section is bootstrap-only. This prevents the persistent configuration from moving the file that is needed to locate itself. To move the state directory, stop the gateway, update the bootstrap config and move/copy the state directory explicitly.
## Crash and shutdown semantics
Atomic state files are written through a temporary file, `fsync`, and rename. API keys, tenant policies, model-placement overrides and durable batch state transitions are saved synchronously when correctness requires it. Metrics, quota and learned worker-performance snapshots are refreshed every `storage.flush_interval` and once again during graceful shutdown. A hard process/host crash can therefore lose at most the most recent snapshot interval for those snapshot files; a batch attempt left as `running` is repaired to `queued` during startup recovery.
Usage events are appended asynchronously and flushed according to `usage.flush_interval`. On normal shutdown the queue is drained and the current journal is flushed/synced. Before startup replay, retention compaction runs once so expired request rows are never needlessly loaded back into memory. Startup then reconstructs all-time counters from monthly rollups, daily rollups and the remaining detail journals. Only detail-journal events populate the recent-request UI.
## Tiered usage retention and aggregation
The default retention policy is:
```json
"retention": {
"detail_days": 30,
"daily_days": 400,
"monthly_months": 0,
"compaction_interval": "6h"
}
```
- **Detail**: raw request metadata for the most recent 30 calendar days.
- **Daily**: older request rows are replaced by one aggregate file per day until day 400.
- **Monthly**: older daily files are folded into monthly files. `monthly_months: 0` means keep them forever.
Each rollup stores independent aggregate dimensions for global traffic, tenants, actors, applications, models and workers. Stored measures include requests/errors, prompt/completion/cached tokens, credits, queue/service duration, prompt/eval nanoseconds and transferred bytes. Prompt/output token throughput can therefore be derived after request details have expired.
Monthly files keep their source day keyed internally. This is deliberate: if the process crashes after the monthly file is synced but before the daily source file is deleted, retrying compaction replaces that day contribution instead of adding it twice.
Compaction only touches closed historical journal days. The active day's writer is never rewritten. The background interval can be configured, and admins can trigger the same compaction with `POST /gateway/ui-api/storage/compact` or **Admin -> Persistenz -> Jetzt kompaktieren**.
Historical aggregate queries are exposed as:
```text
GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=global&limit=120
GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=tenant&name=team-a
GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=actor&name=team-a%00alice
GET /gateway/ui-api/usage/rollups?granularity=monthly&dimension=model&name=qwen3:8b
```
The Prometheus endpoint exports gauges for raw/daily/monthly file counts and byte footprints, plus the timestamp of the last compaction and the number of bytes reclaimed by that run. The cumulative Prometheus counter snapshot itself remains fixed-size and does not need retention compaction.
## Intentionally non-persistent state
- fair-queue heap and virtual clocks;
- running/streaming transient inference jobs and cancellation handles;
- active batch attempt contexts/cancellation handles (the durable batch definition and state remain persistent);
- active worker/model slots;
- live-flow/infrastructure animation state;
- browser OIDC sessions;
- transient model pull operation state.
These objects are tied to sockets, requests or process-local execution and cannot safely be resumed after a restart.
## Backup
For a consistent offline backup, stop the gateway and copy `storage.data_dir`. For normal filesystem snapshots, the atomic JSON files and append-only usage journals are safe to copy while the process is running, though the newest buffered usage records may not yet be present on disk.
## Admin storage controls
**Admin -> Persistenz** shows every durable state file plus detail/daily/monthly usage and batch-spool footprint. **Jetzt flushen** forces metrics, quota, worker performance, encrypted conversation retention state, durable batch metadata, and buffered usage to disk. **Jetzt kompaktieren** applies usage/conversation/batch retention immediately. **Backup herunterladen** flushes first and streams a ZIP containing durable state, batch spool content when present, remaining detail journals and all rollups. Treat the backup as sensitive because configuration can contain deployment secrets and enabled batch jobs can contain prompt/response content.
docs/PRODUCTION-UPDATE.md
+67
View File
@@ -0,0 +1,67 @@
# Production update checklist
This release is intended to be safe to stage as an update of an existing single-process Ollama Fair Gateway. The sustained HA-readiness work is isolated from the inference hot path: `cmd/ha-sampler`, the readiness shell wrapper and report tooling do not participate in normal request handling.
## Scope of this checkpoint
Relative to checkpoint 16, the gateway runtime behavior and persisted-state schemas are unchanged. The release adds/refines only HA-readiness tooling and documentation. The packaged gateway binaries should therefore remain byte-identical to checkpoint 16 when built with the same Go toolchain and flags; the release verification records this explicitly.
If the production system is older than checkpoint 16, treat this as a normal application upgrade because earlier checkpoints added runtime alias/tenant ACL controls, conversations and durable batch state. The configuration loader supplies defaults for omitted optional sections, but a backup is still required before replacing an older binary.
## Before updating
1. Record the currently deployed binary checksum and keep the old binary available for rollback.
2. Back up the bootstrap configuration and `storage.data_dir`. For the most consistent backup, stop the gateway before copying the data directory; see `docs/PERSISTENCE.md`.
3. Validate the intended effective configuration with `ollama-gateway -config <bootstrap.json> -check-config`. For containers, run this in the candidate image with the production config/state mounts so file permissions are tested too; see `docs/DEPLOYMENT-HARDENING.md`.
4. Confirm there is enough free space for the existing usage/batch/conversation retention settings.
5. Do not run the HA readiness sweep against production traffic unless the additional synthetic inference load is acceptable.
## Update procedure
1. Stop the supervised gateway gracefully with SIGTERM and wait for the `gateway stopped` log line.
2. Replace only the gateway executable appropriate for the host architecture. Verify its SHA-256 value against `dist/SHA256SUMS`.
3. Keep the existing bootstrap config and data directory in place.
4. Start the gateway under the same supervisor/service account.
5. Require both `/healthz` and `/readyz` to succeed before restoring external traffic.
6. Send at least one representative non-streaming and, if used in production, one streaming request through the normal authenticated client path.
7. Verify `/metrics`, the admin UI/API used operationally, and recent logs for persistence/configuration errors.
## Rollback
If startup/readiness or representative inference fails, stop the new process and restore the previous executable. When upgrading from checkpoint 16 specifically, no state-schema rollback is required by this checkpoint because the runtime/persistence code is unchanged. When upgrading from an older build, retain the pre-upgrade data-directory backup until the update has been observed successfully under normal load.
## HA readiness tooling in production
`GATEWAY_PID` must be the actual gateway PID. With it set, `scripts/ha-readiness.sh` captures before/after snapshots and starts `cmd/ha-sampler` during each load level. The sampler is read-only with respect to the gateway process; on Linux it reads `/proc` and uses `ps`, and it writes evidence only to the selected `OUT_DIR`. It does not write secrets or gateway state.
## Checkpoint 18 Admin UI hotfix
If upgrading from checkpoint 17, no configuration or persistent-state migration is required. Rebuild/redeploy the gateway executable or container because the Admin UI assets are embedded in the binary. Checkpoint 18 restores the Model Placement UI implementation and adds a regression test for missing render-dispatch functions.
## Checkpoint 19 deployment hardening
Checkpoint 19 keeps P3.2/HA gated and instead hardens the single-node production update path. The shipped Compose command no longer repeats the image entrypoint, the scratch image has a built-in liveness healthcheck via the gateway's `-probe` mode, and `-check-config` validates the effective bootstrap+persistent configuration plus state-directory writability before startup. No persisted-state schema migration is introduced by this checkpoint.
## Checkpoint 20 remote worker telemetry
Checkpoint 20 adds an optional out-of-band worker telemetry agent and hardens the existing `workers[].telemetry_url` merge path. It does **not** change any persistent-state schema and the agent is not part of inference request execution. Existing deployments can upgrade without enabling the agent; behavior remains unchanged until `telemetry_url` is configured.
For remote workers, prefer `local_system_stats: false` and point `telemetry_url` at the agent running on the actual Ollama host. Roll this out in two stages: first deploy/verify each agent with `-once` and its `/telemetry` endpoint, then change the gateway worker configuration. Keep the static `memory_capacity_bytes` and `vram_capacity_bytes` values as capacity hints even when runtime telemetry is enabled.
If telemetry becomes unavailable, the gateway continues operating; the worker telemetry snapshot records the error and routing falls back to the remaining configured/observed signals. Do not expose the agent on an untrusted network. Restrict `-allow-cidrs` to the gateway source address or protect the endpoint with a TLS-authenticated reverse proxy.
## Checkpoint 21 telemetry freshness hardening
Checkpoint 21 does not change persistent-state schemas or inference semantics. It validates timestamps from `telemetry_url` exporters, rejects stale/future samples before they affect routing pressure, preserves pre-existing local collector warnings when external telemetry fails, and makes the shipped worker agent explicitly non-cacheable. Existing exporters without `updated_at` remain compatible.
## Checkpoint 22 IP-bypass hardening
Checkpoint 22 changes only the authentication source used by credential-free `auth.ip_bypass`: the TCP peer is now authoritative by default. Forwarded client-IP resolution remains available for logs and usage attribution. Deployments that intentionally relied on an original `X-Forwarded-For` client address to satisfy IP bypass must explicitly set `auth.ip_bypass_use_forwarded_ip=true` and should restrict `auth.trusted_proxies` to exact proxy addresses plus network ACLs. API-key and OIDC authentication are unaffected. No persistent-state schema migration is introduced.
## Checkpoint 23 reverse-proxy/deployment boundary hardening
Checkpoint 23 does not change inference, scheduler, quota or persistent-state schemas. The supplied Compose topology now defaults to host-loopback publishing (`127.0.0.1:9080`), a read-only root filesystem, `no-new-privileges` and all Linux capabilities dropped. Deployments whose reverse proxy is on another host must explicitly opt into a non-loopback bind and should firewall the published port to that proxy only.
Before rollout, run `scripts/production-preflight.sh` with the production bootstrap config. The script refuses the development `config.example.json` and non-loopback publishing unless explicitly acknowledged, then runs the gateway's secret-redacted `-check-config` inside the candidate container with the production mounts.
docs/PUBLIC-DASHBOARD.md
+64
View File
@@ -0,0 +1,64 @@
# Public Status Dashboard
Checkpoint 28 adds an optional unauthenticated **read-only** dashboard for users who should be able to observe current gateway capacity without receiving admin access.
## Enable
```json
"public_dashboard": {
"enabled": true,
"path": "/status",
"title": "Ollama Gateway Status",
"subtitle": "Live-Auslastung und Infrastruktur",
"refresh_interval": "2s",
"max_live_requests": 64,
"show_worker_names": false,
"show_model_names": true,
"show_resource_metrics": true,
"worker_display_names": {
"internal-worker-name": "GPU Node A"
}
}
```
The dashboard is then available at `/status/` without a gateway credential. It is independent of the authenticated `/admin/` UI and also works when the admin UI itself is disabled.
## Privacy boundary
The public API does **not** reuse or serialize the admin snapshots. It builds a new response from an explicit allow-list. The public response excludes:
- tenant, actor, subject and application identifiers;
- API keys, scopes and authentication metadata;
- worker URLs, IP labels and arbitrary worker labels;
- prompts, responses and request bodies;
- persistent storage paths and configuration;
- policies, quota buckets, aliases and ACLs;
- internal error strings, circuit error details and telemetry errors;
- cost/credit estimates.
Request IDs are replaced by short SHA-256-derived opaque IDs. Worker names are anonymized by default and can be replaced with explicit public display names. Model names are also anonymized unless `show_model_names` is explicitly enabled.
## Public data
The dashboard exposes only operational information intended for status display:
- current queue/running/streaming counts;
- gateway uptime;
- healthy/total worker count;
- number of loaded models;
- worker slot usage and health state;
- loaded model display names when enabled;
- numeric RAM/VRAM/GPU/temperature/power values when resource metrics are enabled;
- anonymized recent/live request state, queue/service timing and token counts.
The snapshot endpoint is `GET <path>/api/snapshot`. It is cacheable for one second to reduce repeated work behind a reverse proxy. The browser refresh interval is configurable but cannot be set below one second.
## Reverse proxy
Checkpoint 23+ binds the production Compose service to loopback by default. Expose `/status/` through the same trusted reverse proxy used for the public service. Do not expose the gateway container port directly merely to publish the dashboard.
If the reverse proxy applies authentication globally, carve out only the configured public dashboard path. Keep `/admin/`, `/gateway/`, `/api/`, `/v1/` and `/metrics` under their existing gateway authentication rules.
## Security headers
The public UI sends a restrictive CSP, denies framing, disables referrer leakage and disables browser permissions such as camera, microphone, geolocation, payments and USB.
docs/RELEASE-VERIFICATION-CHECKPOINT-17.md
+47
View File
@@ -0,0 +1,47 @@
# Checkpoint 17 — production update verification
Validation date: 2026-09-08
Validation toolchain: Go 1.23.2 on Linux/amd64
## Release scope
Checkpoint 17 completes sustained HA-readiness resource sampling. It adds `cmd/ha-sampler`, shared read-only resource collection under `internal/haresource`, sweep integration, report integration, tests and operator documentation.
The normal gateway request, scheduler, routing and persistence code is unchanged relative to checkpoint 16. Rebuilding with the same Go toolchain and release flags produced byte-identical gateway executables for all four packaged gateway targets.
## Required verification results
The following checks passed on the final source tree:
- `go test ./...`
- `go test ./... -count=3` during the release cycle
- `go vet ./...`
- `go test -race ./...`
- `gofmt -l .` returned no files
- `sh -n scripts/ha-readiness.sh scripts/ollama-env.sh`
- all shipped JSON example configurations parse with `jq`
- all files in `dist/SHA256SUMS` verify successfully
## Runtime smoke validation
The packaged Linux/amd64 gateway was tested against the deterministic mock Ollama backend.
1. **Persistent restart smoke:** two complete start/readiness/request/shutdown cycles reused the same `storage.data_dir`. Each cycle served a non-streaming and a streaming OpenAI-compatible request, logged `gateway stopped`, and exited with status 0.
2. **Authenticated readiness smoke:** private `/metrics` plus a Bearer API key completed the readiness sweep. Client counts reconciled with gateway request/queue/service counters, endpoint resource snapshots were complete, and sustained resource traces were complete.
3. **Secret-leak check:** the Bearer API key used by the authenticated sweep was not present in any generated readiness result file.
4. **Final sustained-sampler E2E:** concurrency 1 and 4 both completed with zero request errors, exact counter reconciliation, and `resource_samples.stop_reason == "stop-file"`. A sampler that exits because of `max-duration`, process exit, or interruption is deliberately not accepted as complete sustained evidence.
## Gateway binary identity vs checkpoint 16
| Target | Checkpoint 17 SHA-256 | Compared with checkpoint 16 |
|---|---|---|
| Linux amd64 | `01efc1b236faa9b7b912e6bee6af8266f6a24722af419a6dd43b6f294c955f47` | byte-identical |
| Linux arm64 | `a364f03e67fe377f140936c2c0cdd2b480166e2c5d1fe519878d730268e74248` | byte-identical |
| macOS arm64 | `8f9185f561752656902ef8cf44f5f1c8f65b660f40fb5caf790c2d23773c14d8` | byte-identical |
| Windows amd64 | `5a5bbbf962f1a9fea88a76fc0ee1e28669170496800e9ea39d9ed11caf14b053` | byte-identical |
This identity is the strongest update-safety property of this checkpoint for systems already on checkpoint 16: replacing the gateway executable with the checkpoint 17 gateway executable does not change the executable bytes.
## Operator note
For systems older than checkpoint 16, use `docs/PRODUCTION-UPDATE.md` and take a configuration/data-directory backup first. Earlier checkpoints contain real gateway runtime and persistence features, so the byte-identity statement above does not apply to an upgrade directly from an older production build.
docs/RELEASE-VERIFICATION-CHECKPOINT-18.md
+67
View File
@@ -0,0 +1,67 @@
# Checkpoint 18 — Admin UI placement hotfix
Validation date: 2026-09-08
## Release scope
Checkpoint 18 is a production hotfix on top of checkpoint 17. It restores the complete Model Placement UI helper block that was accidentally removed during the runtime model-alias UI work introduced after checkpoint 13.
The regression manifested after successful admin authentication as a browser error such as:
`renderPlacement is not defined`
The missing block also contained the generic `lines()` helper used by alias, placement and tenant model-access forms, so leaving only `renderPlacement` patched would have exposed follow-on UI failures when saving those forms.
No gateway request, scheduler, routing, quota, persistence, batch, conversation, HA-sampling or worker-runtime behavior was intentionally changed by this hotfix. The gateway executables do change because the Admin UI JavaScript is embedded into the binary.
## Fix
Restored these UI helpers:
- `placementRule`
- `placementRows`
- `placementCell`
- `placementSourceLabel`
- `renderPlacement`
- `placementCellHTML`
- `placementWorkerCard`
- `loadPlacement`
- `openPlacementDialog`
- `lines`
- `installedForWorker`
- `savePlacementRule`
- `placementExactAction`
The existing placement matrix, worker-rule editor, exact allow/deny overrides, presets and reset controls now have their implementation functions again.
## Regression protection
`internal/webui/webui_test.go` now verifies that every `renderX()` function called by the main `render()` dispatcher is actually defined in the embedded Admin UI bundle. A second test verifies the complete Placement UI helper set.
This specifically prevents the original `renderPlacement is not defined` class of regression from passing `go test ./...` again.
## Required verification results
The final source tree passed:
- `go test ./...`
- `go vet ./...`
- `go test -race ./...`
- `node --check internal/webui/assets/app.js`
- release cross-builds for Linux amd64, Linux arm64, macOS arm64 and Windows amd64
- `dist/SHA256SUMS` verification
## Update guidance
For Docker/Compose deployments, rebuild or replace the gateway image/binary. Copying only the source `app.js` beside an already-built binary does not fix the problem because `internal/webui/assets/app.js` is embedded at build time.
The production configuration and persistent `/data` volume do not require a migration from checkpoint 17 to checkpoint 18.
## Gateway binary SHA-256
| Target | SHA-256 |
|---|---|
| Linux amd64 | `550f481313e5869d884280dfc3248d70b6eabf51b4b3bb44083c36f622065cf2` |
| Linux arm64 | `27f4ab0b7a1dc9a2379e7e99471e177f9da9c715445b05b089d028011b8101aa` |
| macOS arm64 | `d72dd623844050b1b7396ae364e49a30ab76d2bee21a2ced22dad308c0e839bb` |
| Windows amd64 | `dedf06c3d42fdc8c2fe3de19cd9c00fcdb6eb6fab91442df3b18ac5e8a416072` |
docs/RELEASE-VERIFICATION-CHECKPOINT-19.md
+58
View File
@@ -0,0 +1,58 @@
# Checkpoint 19 — deployment preflight and container hardening
Validation date: 2026-09-08
## Scope
Checkpoint 19 follows the Checkpoint 18 Admin UI hotfix. It does not implement P3.2/HA and does not change any persistent-state schema. Its purpose is to reduce single-node production update risk after real deployment feedback exposed configuration mount/entrypoint and file-permission mistakes that were otherwise only visible after startup.
The inference request/scheduler/quota/proxy path is unchanged by this checkpoint. Runtime changes are limited to startup configuration loading being factored through the same helper used by the new preflight, plus two explicit CLI-only modes (`-check-config` and `-probe`).
## Changes
- `-check-config`: strict effective configuration validation plus persistent override loading and state-directory write probe.
- Secret-free JSON preflight summary with worker count and deployment warnings.
- `-probe`: minimal HTTP probe implemented inside the gateway binary for scratch/container health checks.
- Dockerfile/Compose liveness healthcheck against public `/healthz`.
- Corrected Compose command: arguments only, no duplicate `/ollama-gateway` after the image `ENTRYPOINT`.
- `GATEWAY_CONFIG` Compose variable for selecting the bootstrap config explicitly.
- Regression tests for persistent-secret restoration, preflight output, HTTP probing and Compose entrypoint/healthcheck rules.
- Production deployment documentation.
## Verification completed
The final source tree passed:
- `go test ./... -count=1`
- `go vet ./...`
- Race detector over all packages, split as `go test -race ./cmd/...` and `go test -race ./internal/...` so each command completed with an observable exit status. (The single monolithic invocation exceeded the execution harness timeout; no package was omitted from the split run.)
- `node --check internal/webui/assets/app.js`
- Compose YAML parse/assertions for the corrected command, `GATEWAY_CONFIG` mount and healthcheck. The Docker CLI itself was not available in the verification environment, so `docker compose config` was not claimed.
- Real Linux-amd64 binary `-check-config` run against the migrated production configuration: `status=ok`, exactly 2 effective workers, writable state directory, expected remote `local_system_stats` warnings, and no API-key secret in the preflight output.
- Real gateway + deterministic mock Ollama smoke: `/healthz` probe, `/readyz` probe, authenticated `/gateway/ui-api/session`, OpenAI-compatible inference, SIGTERM, and gateway exit code 0.
- Cross-builds for Linux amd64, Linux arm64, macOS arm64 and Windows amd64.
- `dist/SHA256SUMS` verification for every packaged binary/helper.
## Gateway binary SHA-256
| Target | SHA-256 |
|---|---|
| Linux amd64 | `e30a7b01f8a96f6396c3582ab208bf8e4e330c6a882b724998448dc66dfa7cf7` |
| Linux arm64 | `675a72c054c21ba383c36891b9bdeb1b93cbd4690abedc22b74ae02c8bc081e4` |
| macOS arm64 | `586e489cf00c54aa5906a63f4ceada6ff22c90943a80d31fb1c864b557dde559` |
| Windows amd64 | `1d24976651a358bc80031f36eb2f59cca9233dfa89bf3189bb1bb4b6e9de96fe` |
The HA helper binaries were not modified by this checkpoint and their existing checksums remain listed and verified in `dist/SHA256SUMS`.
## Update guidance
No configuration-schema or persistent-state migration is required from Checkpoint 18. Container deployments should rebuild/recreate the image because the Dockerfile healthcheck and gateway CLI modes are part of this release.
Before starting production traffic, run the candidate container in preflight mode with the exact config/state mounts:
```sh
GATEWAY_CONFIG=./gateway-config.json docker compose run --rm gateway \
-config /etc/ollama-gateway/config.json -check-config
```
Then start the service, wait for liveness, require `/readyz`, authenticate to the admin UI/API and send representative inference traffic. See `docs/DEPLOYMENT-HARDENING.md` and `docs/PRODUCTION-UPDATE.md`.
docs/RELEASE-VERIFICATION-CHECKPOINT-20.md
+69
View File
@@ -0,0 +1,69 @@
# Checkpoint 20 — remote worker telemetry
Date: 2026-09-08
## Scope
Checkpoint 20 extends the single-node production hardening path with an optional `worker-telemetry` agent for remote Ollama hosts. It also fixes the existing external telemetry merge so omitted JSON fields no longer overwrite previously collected values with zero, while explicit zero values remain meaningful.
No persistent-state schema changes are introduced. The inference, quota and scheduler request paths are unchanged.
## New artifacts
- `cmd/worker-telemetry`
- `dist/ollama-gateway-worker-telemetry-linux-amd64`
- `dist/ollama-gateway-worker-telemetry-linux-arm64`
- `docs/WORKER-TELEMETRY.md`
- Linux AMDGPU sysfs collector in `internal/hoststats`
## Verification completed
The final source tree passed:
- `go test ./...`
- `go vet ./...`
- JavaScript syntax validation with `node --check internal/webui/assets/app.js`
- race tests across all command packages and all tested internal packages, including `internal/worker` and `internal/hoststats`
- SHA-256 verification for every file listed in `dist/SHA256SUMS`
- cross-builds for gateway Linux amd64/arm64, macOS arm64 and Windows amd64
- cross-builds for worker telemetry Linux amd64/arm64
## Remote telemetry E2E
A Linux-amd64 release gateway was started with:
- one mock Ollama worker,
- `local_system_stats: false`,
- `telemetry_url` pointing at the release worker-telemetry agent,
- a temporary simulated AMDGPU sysfs tree.
The gateway reported the worker healthy and exposed telemetry source:
`telemetry-url:host-memory+amdgpu-sysfs`
The simulated GPU values were preserved end-to-end:
- VRAM used: 4,294,967,296 bytes
- VRAM total: 17,179,869,184 bytes
- GPU utilization: 73%
- GPU temperature: 62.5 C
- GPU power: 88 W
Both the gateway and telemetry agent exited cleanly with status 0 after SIGTERM.
## Merge regression coverage
`externalTelemetry` uses pointer fields. This explicitly distinguishes an omitted JSON property from a supplied zero. Tests verify that omitted memory/GPU fields preserve existing telemetry while an explicit zero for VRAM usage or GPU utilization is applied.
## Production rollout
Enabling the agent is optional and can be staged independently from the gateway update. For a remote worker:
```json
{
"local_system_stats": false,
"telemetry_url": "http://WORKER-IP:11500/telemetry"
}
```
Validate the agent locally with `-once` first and restrict network access using `-allow-cidrs` plus the worker host firewall. See `docs/WORKER-TELEMETRY.md`.
docs/RELEASE-VERIFICATION-CHECKPOINT-21.md
+40
View File
@@ -0,0 +1,40 @@
# Checkpoint 21 — telemetry freshness hardening
Date: 2026-09-08
## Scope
Checkpoint 21 hardens the optional remote worker telemetry path introduced in checkpoint 20. No persistent-state schema changes are introduced and the inference/quota/scheduler hot paths are unchanged.
Changes:
- parse optional external `updated_at`,
- reject samples older than `max(30s, 6 x health_interval)`,
- reject timestamps more than 30 seconds in the future,
- keep backward compatibility for exporters without timestamps,
- append external HTTP/decode/freshness errors instead of overwriting an existing collector warning,
- send `Cache-Control: no-store` and `X-Content-Type-Options: nosniff` from the shipped telemetry agent.
## Safety behavior
A rejected external sample is not merged into `ResourceTelemetry`. Static worker capacity hints, Ollama health/inventory and all other routing inputs remain available. The worker remains usable; the telemetry error is surfaced for operators.
## Verification
Release verification includes unit tests for fresh/stale/future/missing timestamps, merge semantics, full repository tests/vet, race tests, release cross-builds, dist checksums and a real gateway + mock Ollama + telemetry-agent smoke path.
## E2E freshness evidence
Fresh-agent path:
- agent response included `Cache-Control: no-store`,
- source was `telemetry-url:host-memory+amdgpu-sysfs`,
- simulated VRAM/GPU/temperature/power values were merged correctly.
Stale-exporter path:
- exporter was made ready before gateway startup,
- payload timestamp was intentionally more than one hour old,
- gateway reported `telemetry sample is stale: ... exceeds 30s`,
- stale VRAM/GPU pressure values were not merged,
- gateway still shut down cleanly with exit status 0.
docs/RELEASE-VERIFICATION-CHECKPOINT-22.md
+39
View File
@@ -0,0 +1,39 @@
# Checkpoint 22 — IP-bypass / trusted-proxy hardening
Date: 2026-09-08
## Security issue addressed
Before checkpoint 22, `Authenticator.Authenticate` evaluated `auth.ip_bypass` against the fully resolved client IP. If the immediate peer was in `auth.trusted_proxies`, that address could originate from `X-Forwarded-For`. A deployment that trusted a broad reachable subnet could therefore allow a direct client in that subnet to present a bypass address such as `127.0.0.1`.
## New default
- `ClientIP()` still resolves trusted forwarding chains for observability.
- `auth.ip_bypass` is evaluated against the direct TCP peer by default.
- `Identity.ClientIP` remains the resolved client address for usage/logging.
- Legacy forwarded-IP bypass is available only with explicit `auth.ip_bypass_use_forwarded_ip=true`.
- Config preflight warns when forwarded bypass is enabled and when `auth.trusted_proxies` contains broad non-loopback CIDRs.
API-key and OIDC authentication semantics are unchanged. No persistent-state schema change is introduced.
## Regression coverage
Tests prove that:
1. a trusted peer with `X-Forwarded-For: 127.0.0.1` cannot satisfy loopback IP bypass by default,
2. forwarded client-IP resolution still reports `127.0.0.1` for observability in that test,
3. explicit compatibility mode restores forwarded-IP bypass,
4. a direct loopback peer still satisfies the normal loopback bypass,
5. preflight warns on compatibility mode and a broad `10.0.0.0/8` trusted-proxy range while not flagging loopback-only proxy ranges.
## Built-binary spoof E2E
A release Linux-amd64 gateway was started with a trusted loopback proxy peer and an IP-bypass rule for `192.0.2.123/32`.
With the new default (`ip_bypass_use_forwarded_ip=false`), a request from the trusted TCP peer carrying `X-Forwarded-For: 192.0.2.123` returned **401 Unauthorized**.
The same test with the explicit compatibility flag set to `true` returned **200 OK** with an `ip-bypass` admin identity and resolved `client_ip=192.0.2.123`. This proves both the secure default and the intentional compatibility escape hatch in the release binary.
## Production-config preflight
The user's two-worker configuration validates successfully with the checkpoint-22 candidate. Preflight reports broad trusted-proxy warnings for `10.0.0.0/8`, `172.16.0.0/12` and `192.168.0.0/16`; these no longer permit forwarded headers to satisfy IP bypass under the default, but should still be narrowed to improve client-IP attribution integrity.
docs/RELEASE-VERIFICATION-CHECKPOINT-23.md
+57
View File
@@ -0,0 +1,57 @@
# Release verification: checkpoint 23
Checkpoint 23 hardens the Docker/reverse-proxy deployment boundary. It does not change the inference hot path or persisted-state schemas.
## Scope verification
`cmd/` and `internal/` were byte-for-byte/diff checked against checkpoint 22 after the deployment changes. No runtime Go source changed. Existing release binaries therefore remain unchanged and all entries in `dist/SHA256SUMS` still verify.
## Deployment changes
- Development Compose defaults to `127.0.0.1:9080 -> :8080` instead of publishing on all host interfaces.
- `docker-compose.production.yml` requires `GATEWAY_CONFIG`; there is no production fallback to `config.example.json`.
- Container root filesystem is read-only; `/data` is the persistent writable volume and `/tmp` is tmpfs.
- Runtime explicitly uses UID/GID `65532:65532`, drops all Linux capabilities and enables `no-new-privileges`.
- JSON-file logging is bounded to five 10 MiB files in the production Compose example.
- `scripts/production-preflight.sh` rejects `config.example.json`, rejects non-loopback publishing unless explicitly acknowledged, renders Compose, and invokes the gateway's secret-redacted `-check-config` in the candidate container.
- `scripts/production-preflight_test.sh` regression-tests loopback success, example-config rejection, wildcard-bind rejection and explicit remote-proxy opt-in.
## Production config validation
The hardened two-worker production configuration was validated with the release binary:
- status: `ok`
- workers: `2`
- data directory: `/data`
- state directory writable: `true`
- broad trusted-proxy warnings: none
The configuration narrows `auth.trusted_proxies` to loopback plus the observed Docker peer `172.30.3.1/32`; `auth.ip_bypass_use_forwarded_ip` remains false.
## Test results
- `go test ./...`: PASS
- `go vet ./...`: PASS
- `go test -race ./cmd/...`: PASS
- `go test -race ./internal/...`: PASS
- `sh -n scripts/production-preflight.sh`: PASS
- `sh -n scripts/production-preflight_test.sh`: PASS
- `scripts/production-preflight_test.sh`: PASS
- Compose YAML parse check: PASS
- `dist/SHA256SUMS`: all 10 packaged binaries PASS
## Release-binary smoke test
The shipped Linux-amd64 gateway binary was started against an isolated deterministic Ollama mock and temporary state directory.
- `/healthz`: 200 / `status=ok`
- `/readyz`: 200 / `status=ready`
- authenticated `/gateway/ui-api/session`: admin identity returned
- authenticated `/v1/chat/completions`: successful response (`chatcmpl-mock`)
- SIGTERM gateway shutdown: exit code 0 and `gateway stopped` logged
The mock process is test infrastructure and is not part of the production artifact.
## Deployment recommendation
For the observed topology, keep `GATEWAY_PUBLISH_ADDRESS=127.0.0.1` so only a reverse proxy on the Docker host can reach port 9080. If the reverse proxy is remote, bind to one specific host interface, set `ALLOW_NON_LOOPBACK_BIND=1` only for the preflight, and firewall the published port to the exact reverse-proxy source IP.
docs/RELEASE-VERIFICATION-CHECKPOINT-24.md
+53
View File
@@ -0,0 +1,53 @@
# Release verification — Checkpoint 24
Checkpoint 24 is an admin-WebUI-only visual refresh based on the production-hardened Checkpoint 23 runtime. It changes the embedded HTML/CSS/JavaScript presentation layer and the four gateway binaries that embed those assets. Scheduler, quota, routing, authentication, persistence, telemetry and inference semantics are unchanged.
## Scope
- Professional light control-plane theme inspired by dense technical/mission-database UIs.
- Offline-safe font stack: preferred local IBM Plex/JetBrains/Cascadia fonts with system fallbacks; no external font or CDN dependency.
- Explicit light color scheme for browser-native controls.
- Light sidebar, panels, KPI cards, tables, forms, dialogs, callouts, code/config editor and login screen.
- Light Live Flow and Infrastructure canvases, including canvas-drawn labels/nodes/routes rather than CSS-only restyling.
- Accessibility-oriented color tokens. Primary text, muted text, accent text, primary buttons and semantic status colors meet approximately WCAG AA 4.5:1 contrast against white for normal text.
- Regression tests assert that the light theme, light metadata and light canvas colors remain embedded.
## Verification performed
```sh
node --check internal/webui/assets/app.js
python -c 'import tinycss2; ... parse internal/webui/assets/app.css ...'
go test ./...
go vet ./...
go test -race ./...
```
All commands completed successfully.
Four gateway release targets were rebuilt because WebUI assets are embedded in the gateway binary:
- linux/amd64
- linux/arm64
- darwin/arm64
- windows/amd64
`dist/SHA256SUMS` was regenerated and `sha256sum -c dist/SHA256SUMS` succeeded for all shipped binaries.
## Release-binary smoke
The rebuilt linux/amd64 binary was started with an isolated writable data directory and deterministic mock Ollama backend. The following checks passed:
- `-check-config`
- `/readyz`
- embedded `/admin/`
- embedded `/admin/app.css` contains the Checkpoint 24 light-theme marker
- embedded `/admin/app.js` passes `node --check`
- `/gateway/ui-api/session` via direct loopback admin IP bypass returns an admin identity
- `/v1/chat/completions` returns a successful mock response
- SIGTERM shutdown exits with code 0
A headless Chromium screenshot was attempted, but the container's Chromium/DBus environment did not terminate reliably. This was not counted as a passed test. Asset parsing, embedding and runtime serving were verified independently as described above.
## Deployment
No configuration or persistent-state migration is required from Checkpoint 23. Rebuild/redeploy the gateway image because the WebUI is compiled into the binary. After update, hard-refresh the browser once to discard any previously cached `app.css` or `app.js`.
docs/RELEASE-VERIFICATION-CHECKPOINT-26.md
+15
View File
@@ -0,0 +1,15 @@
# Checkpoint 26 release verification
Scope: embedded Admin WebUI only (dialog semantics + regression tests).
Verified:
- `go test ./...`
- `go vet ./...`
- `node --check internal/webui/assets/app.js`
- race tests across all packages (full run completed in two segments due execution timeout)
- rebuilt Gateway binaries for Linux amd64/arm64, macOS arm64, Windows amd64
- `dist/SHA256SUMS` verified
- real Linux-amd64 release-binary smoke confirmed embedded policy/pull close controls and clean SIGTERM shutdown
Regression covered: required HTML form controls must not prevent modal cancellation/close.
docs/RELEASE-VERIFICATION-CHECKPOINT-27.md
+84
View File
@@ -0,0 +1,84 @@
# Checkpoint 27 release verification
Checkpoint 27 hardens effective context-window handling and `num_ctx` admission/routing.
## Scope
Runtime changes are intentionally limited to context estimation, model metadata, context-aware worker eligibility/routing, the policy simulator, configuration validation, and the corresponding Admin UI context columns. Scheduler fairness, authentication, persistence formats, batch execution, conversations, and quota accounting are otherwise unchanged.
## Effective context model
The gateway now keeps these values distinct:
1. **Model maximum** from Ollama `/api/show` model metadata.
2. **Configured context** from a Modelfile `PARAMETER num_ctx`, when present.
3. **Loaded context** from Ollama `/api/ps context_length`, when the model is resident.
4. **Worker default** from `workers[].default_context_tokens` or the gateway context default.
5. **Administrative cap** from `workers[].context_limits` and `model_capabilities.context.max_requested_tokens`.
For OpenAI-compatible requests, the effective usable context is derived from the actually loaded/configured/default context and administrative limits; the theoretical model maximum is no longer treated as the runtime context by itself.
For native Ollama `/api/*` requests, a valid explicit `options.num_ctx` can request a resize/reload, but is still bounded by the model maximum, gateway cap, and worker context limit. A worker whose currently loaded context is smaller remains eligible only as a resize candidate and does not receive a misleading loaded-context routing advantage.
## Request estimation fixes
Context admission now includes:
- Responses API `max_output_tokens`;
- Responses API `instructions`;
- native `/api/generate` `suffix`;
- configurable estimation margin;
- configurable per-image vision reserve;
- the largest positive output-token budget rather than silently preferring a smaller field.
`options.num_ctx` is strictly validated as a positive integer. It is accepted only on native Ollama `/api/*` endpoints. OpenAI-compatible requests cannot inject it as a false context override.
## Validation performed
The release candidate passed:
- `go test ./...`
- `go vet ./...`
- `node --check internal/webui/assets/app.js`
- `go test -race ./...`
- strict `-check-config` against the supplied two-M75q production configuration
- Linux amd64, Linux arm64, macOS arm64, and Windows amd64 gateway cross-builds
- `dist/SHA256SUMS` verification
- a real Linux-amd64 release-binary smoke test against the deterministic mock Ollama backend:
- `/readyz` ready
- Admin API-key session HTTP 200
- `/v1/chat/completions` HTTP 200
- native string-valued `options.num_ctx` HTTP 400 before backend execution
- OpenAI `options.num_ctx` spoof attempt HTTP 400
- SIGTERM shutdown exit code 0
The final ZIP is additionally unpacked and tested again before release.
## Recommended production settings for the current two-M75q deployment
The supplied production config uses conservative starting bounds for the ~16 GiB AMD worker class:
```json
"model_capabilities": {
"mode": "enforce",
"cache_ttl": "10m",
"context_guard": "reject",
"context": {
"max_requested_tokens": 32768,
"default_worker_tokens": 4096,
"estimation_margin_percent": 15,
"vision_reserve_tokens_per_image": 2048
}
}
```
and per worker:
```json
"default_context_tokens": 4096,
"context_limits": {
"*": 16384
}
```
`16384` is an operational safety cap, not a claim that every model/workload will fit efficiently at 16k. Increase it only after measuring actual VRAM pressure, offload behavior, latency, and throughput on the target worker/model combination.
docs/RELEASE-VERIFICATION-CHECKPOINT-28.md
+61
View File
@@ -0,0 +1,61 @@
# Release verification — Checkpoint 28 public status dashboard
Checkpoint 28 replaces the active P3.2 HA roadmap item with an optional public, unauthenticated, read-only operational dashboard. Existing HA-readiness tools remain packaged but no cluster/coordinator behavior is introduced.
## Runtime scope
Changed runtime areas:
- strict configuration schema: new `public_dashboard` block;
- unauthenticated routing for the configured public dashboard path only;
- a dedicated sanitized public snapshot builder;
- embedded `internal/publicui` assets.
Unchanged semantics:
- inference/proxy protocol handling;
- scheduler/fairness/quota algorithms;
- worker selection and model placement;
- durable state schemas and storage files;
- admin authentication and admin UI authorization;
- batch/conversation persistence.
No state migration is required.
## Privacy verification
The public API is not derived by JSON-marshalling admin/infrastructure objects. Tests require an explicit allow-list and fail if known sensitive fixture values appear in the public response. Verified absent fields/data include tenant, actor, application, internal request ID, worker URL, labels, internal error strings, telemetry errors and estimated credits.
Worker names and model names default to anonymized display names. Worker aliases can be explicitly supplied in configuration. Resource metrics contain only numeric operational values.
## Automated verification
- `go test ./...` — PASS
- `go vet ./...` — PASS
- `go test -race ./...` — PASS
- `node --check internal/publicui/assets/app.js` — PASS
- `node --check internal/webui/assets/app.js` — PASS
- all four gateway release targets rebuilt — PASS
- every entry in `dist/SHA256SUMS` — PASS
- `config.example.json` strict parse test — PASS
- production-derived checkpoint-28 config through `-check-config` — PASS (2 workers, `/data` writable)
## Release-binary smoke
The packaged Linux amd64 gateway was started with a deterministic mock Ollama backend and an isolated state directory.
Verified:
- `/readyz` — 200
- `/status/` without credentials — 200
- `/status/api/snapshot` without credentials — 200
- `/gateway/ui-api/session` without credentials — 401
- `/gateway/ui-api/session` with admin API key — 200
- `/v1/chat/completions` with API key — 200
- post-request public snapshot contains the public worker alias and opaque `REQ-*` request ID
- post-request public snapshot does not contain private gateway/worker names, auth key, tenant/actor/application fields or worker URL
- SIGTERM shutdown — gateway exit code 0
## Deployment
Enable the dashboard explicitly in configuration. A recommended production-derived example is supplied separately as `gateway-config.checkpoint28.public-dashboard.json`; it publishes model names but maps the two internal M75q worker names to `GPU Node A` and `GPU Node B`.
docs/RELIABILITY.md
+27
View File
@@ -0,0 +1,27 @@
# Reliability and worker maintenance
## Circuit breaker
`reliability.enabled` activates a per-worker circuit breaker. Consecutive transport/backend failures increment the worker failure counter. At `failure_threshold` the circuit enters `open` for `open_duration`. After that period the next eligible request acts as the half-open probe; success closes the circuit, failure reopens it.
```json
"reliability": {
"enabled": true,
"failure_threshold": 3,
"open_duration": "30s",
"retry_attempts": 2,
"retry_backoff": "50ms"
}
```
Retries are deliberately conservative: only transport failures before any upstream response is committed to the client are retried. A backend HTTP 5xx contributes to the circuit breaker but is not replayed once its headers/body have begun.
## Worker maintenance
The admin Worker page exposes:
- `active`: accepts new inference jobs;
- `draining`: no new inference jobs, existing jobs continue;
- `disabled`: no new inference jobs until explicitly enabled.
Drain/disable state is persisted in `<data_dir>/worker-state.json`. Circuit state is intentionally transient and can be manually reset from the Worker page.
docs/RETENTION.md
+80
View File
@@ -0,0 +1,80 @@
# Usage retention and long-term aggregation
The gateway stores request-level usage as append-only daily JSONL journals and compacts old data into bounded rollups.
## Default policy
```json
"usage": {
"journal_dir": "./data/usage",
"buffer": 16384,
"flush_interval": "1s",
"retention": {
"detail_days": 30,
"daily_days": 400,
"monthly_months": 0,
"compaction_interval": "6h"
}
}
```
- `detail_days`: keep per-request records for this many calendar days.
- `daily_days`: retain one daily rollup until this age. Must be >= `detail_days`.
- `monthly_months`: retain monthly rollups for this many months. `0` means forever.
- `compaction_interval`: background compaction cadence; minimum one minute.
## Storage layout
```text
data/usage/
├── usage-2026-09-06.jsonl
└── rollups/
├── daily/
│ └── rollup-daily-2026-07-23.json
└── monthly/
└── rollup-monthly-2025-08.json
```
Each daily/monthly rollup retains aggregate dimensions for global traffic, tenant, actor, application, model, and worker.
Measures retained after request details expire include request/error counts, prompt/completion/cached tokens, credits, queue/service time, prompt/eval nanoseconds, bytes in/out, and last-request timestamp. Prompt and output tokens/second remain derivable from aggregate evaluation durations.
## Crash/idempotency behavior
Raw -> daily compaction writes and fsyncs the new daily file before deleting the source journal. Repeating this after a crash replaces the same daily period rather than summing it twice.
Daily -> monthly files retain a map of their source days. If a process dies after the monthly file is committed but before the source daily file is deleted, replaying compaction replaces that day entry. This makes the monthly fold idempotent.
## Startup and all-time accounting
Before replaying request journals, startup applies retention once. It then reconstructs all-time usage from retained monthly rollups, retained daily rollups, and the remaining request-level journals. Only remaining request-level journals populate the recent-request table.
If `monthly_months > 0`, data older than that window is deleted completely and removed from reconstructed all-time usage.
## Admin/API
```text
POST /gateway/ui-api/storage/compact
GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=global&limit=120
GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=tenant&name=team-a
GET /gateway/ui-api/usage/rollups?granularity=monthly&dimension=model&name=qwen3:8b
```
Allowed dimensions: `global`, `tenant`, `actor`, `application`, `model`, `worker`.
The Admin UI exposes the same controls under **Persistenz** and historical aggregates under **Nutzung**.
## Prometheus
```text
ollama_gateway_usage_raw_files
ollama_gateway_usage_daily_rollup_files
ollama_gateway_usage_monthly_rollup_files
ollama_gateway_usage_raw_bytes
ollama_gateway_usage_daily_rollup_bytes
ollama_gateway_usage_monthly_rollup_bytes
ollama_gateway_usage_last_compaction_timestamp_seconds
ollama_gateway_usage_last_reclaimed_bytes
```
No tenant/user labels are added to `/metrics`; dimensional long-term analysis belongs to the rollup API/UI, avoiding unbounded Prometheus cardinality.
docs/ROADMAP-P0-P3.md
+195
View File
@@ -0,0 +1,195 @@
# Ollama Fair Gateway — P0–P3 implementation roadmap
This roadmap turns the gateway from a local fair scheduler into a production-oriented LLM control plane while preserving the project's core properties: one Go binary, no mandatory external state service, streaming-first proxying, and no prompt/response content persistence by default.
## Delivery principles
1. **Never block the inference hot path on durable storage.** Control-plane writes may be synchronous when correctness requires it; usage/metrics remain buffered/snapshotted.
2. **Never retry after client-visible output starts.** A retry is permitted only before response headers/body are committed.
3. **Hard policies precede adaptive scoring.** Model ACL → alias resolution → capability preflight → placement → maintenance/circuit eligibility → adaptive routing.
4. **Every runtime control must be observable and persistent when operationally meaningful.** Drain/disable state, API keys, policies, placement, aliases and durable batch definitions survive restart; active sockets, transient inference jobs and in-flight batch attempt contexts do not.
5. **Protocol compatibility remains explicit.** Native Ollama, OpenAI-compatible and later Anthropic-compatible errors/streaming semantics are handled independently.
---
## P0 — Production safety and policy foundation
### P0.1 Circuit breaker + safe pre-stream retry
**Goal:** stop repeatedly routing to unhealthy/OOM/transport-failing workers.
- Worker circuit states: `closed`, `open`, `half_open`.
- Configurable consecutive-failure threshold and open duration.
- Transport failures and backend 5xx contribute to the circuit.
- Only transport failures that happen before response commitment are retried.
- Retry excludes workers that already failed the current request.
- Admin UI shows circuit state, last circuit error and manual reset.
- Prometheus counters planned for opens/retries/failures.
**Acceptance:** a connection-reset worker opens its circuit and a request succeeds on another eligible worker without duplicate client-visible output.
### P0.2 Virtual models / aliases
**Goal:** clients use stable names such as `fast`, `coding`, `vision` rather than physical Ollama tags.
- Ordered fallback list of real models.
- Optional required capability set.
- Alias participates in `/api/tags` and `/v1/models` discovery.
- Gateway rewrites the outbound model while returning diagnostic headers:
- `X-Gateway-Model-Alias`
- `X-Gateway-Resolved-Model`
- Alias target must still satisfy worker placement and inventory.
**Acceptance:** OpenWebUI can select a virtual model and the backend receives the resolved physical model.
### P0.3 Model ACLs
**Goal:** answer “who may use which model?” independently from placement (“where may it run?”).
- Tenant baseline rules.
- API-key-specific allow/deny rules.
- Exact and trailing `*` patterns; most specific match wins, deny wins ties.
- ACL is enforced for direct model requests and model discovery.
- API-key create UI exposes allow/deny lists.
**Acceptance:** a key allowed only for `coding` cannot discover or directly call a denied physical model.
### P0.4 Worker maintenance / drain
**Goal:** take workers out of rotation without killing existing streams.
- `active`: accepts new work.
- `draining`: no new work; active jobs finish.
- `disabled`: no new work until explicitly re-enabled.
- State persisted in `worker-state.json`.
- UI buttons: Drain, Disable, Activate; circuit reset beside them.
**Acceptance:** a draining worker's active job completes while all new jobs route elsewhere.
---
## P1 — Client reach, QoS and observability
### P1.1 Anthropic `/v1/messages`
- Native Anthropic-compatible request/stream passthrough to Ollama.
- Token/usage extraction and protocol-correct errors.
- Tool, vision and thinking capability preflight.
- Same ACL/quota/placement/scheduler pipeline as Ollama/OpenAI.
### P1.2 Service classes / priority
- Classes: `interactive`, `system`, `background`, `batch`.
- Weighted scheduling without starvation.
- Per-class queue wait and concurrency ceilings.
- API-key/default mapping and optional request header override with scope.
### P1.3 Auto-tuning and benchmark profiles
- Measure TTFT, prompt tok/s, output tok/s, throughput and VRAM by model/concurrency.
- Suggest `max_concurrent` / per-model concurrency.
- Optional “apply recommendation” workflow with audit entry.
- Never auto-change production settings unless explicitly enabled.
### P1.4 OpenTelemetry
- OTLP traces/metrics, content capture off by default.
- Spans: auth, admission, queue, route, upstream, first-byte/stream.
- Correlate with `X-Request-ID`.
---
## P2 — Capacity automation and operator workflows
### P2.1 Warm/preload policies — ✅ implemented
- `hot`, `warm`, `cold` model classes.
- Explicit/preferred workers.
- Idle unload using Ollama `keep_alive: 0`.
- Optional pre-warm at gateway start/worker recovery.
- Memory-pressure-aware eviction suggestions.
### P2.2 Alerts and webhooks — ✅ implemented
- Conditions: worker down, circuit open, repeated OOM, queue depth/wait, storage growth, quota near exhaustion.
- Cooldown/deduplication.
- Generic signed webhook first; provider-specific integrations later.
### P2.3 Optional stateful conversations — ✅ implemented
- Opt-in conversation store for clients that need `previous_response_id` semantics.
- Separate encryption/retention policy because this stores content.
- Disabled by default to preserve current privacy posture.
### P2.4 Policy simulator — ✅ implemented early in P0/P1
- Simulate identity/model/capabilities/context without executing inference.
- Explain ACL, alias, placement, inventory, circuit, drain, concurrency and final routing score.
- UI provides a step-by-step decision trace.
---
## P3 — Batch and public operations visibility
### P3.1 Batch jobs — ✅ implemented
- Durable job definitions and status.
- Separate background scheduling class.
- Pause/resume/cancel.
- Input/output references rather than embedding large payloads in control state.
- Retention and accounting integrated with existing usage rollups.
### P3.2 Public status dashboard — ✅ implemented
- Separate unauthenticated, strictly read-only status surface.
- Current queue/load, worker capacity, infrastructure map and anonymized live flow.
- Explicit public-data allow-list; no tenant/actor/application identities or admin/control-plane state.
- Worker/model-name privacy controls and public worker aliases.
- HA/gateway clustering is removed from the active roadmap because observed single-process performance does not justify the distributed coordination cost. The HA-readiness tooling remains available if that operational assumption changes later.
---
## Planned implementation sequence
1. **Release A — P0 reliability foundation**: circuit breaker, safe retry, drain/disable persistence, UI controls.
2. **Release B — P0 policy surface**: model aliases and model ACLs with full runtime CRUD UI and persistence.
3. **Release C — P1 protocol/QoS**: Anthropic Messages + service classes.
4. **Release D — P1 observability/tuning**: OpenTelemetry + benchmark recommendations.
5. **Release E — P2 automation**: warm model manager, alerts, policy simulator.
6. **Release F — P2 stateful optional layer**: conversations with explicit content-retention controls.
7. **Release G — P3 batch**.
8. **Release H — P3 public status dashboard**; multi-gateway HA is deferred outside the active roadmap.
## Current implementation status
- P0.1: **implemented** (circuit breaker, safe pre-stream retry and reliability metrics).
- P0.2: **implemented** (aliases, discovery, capability-qualified resolution and persistent runtime CRUD from the admin API/UI with immediate publication for new requests).
- P0.3: **implemented** (tenant/API-key ACL enforcement, API-key ACL CRUD and persistent runtime Tenant Model Access CRUD from the admin API/UI).
- P0.4: **implemented** (persistent drain/disable + UI controls).
- P1.1: **implemented** (Anthropic Messages routing, metering and protocol-aware handling).
- P1.2: **implemented** (weighted service classes with queue/concurrency controls and scoped override header).
- P1.3: **implemented** (explicit benchmark profiles and apply-recommendation workflow).
- P1.4: **implemented** (OTLP/HTTP tracing with content capture off by default).
- P2.1: **implemented** (warm/preload policies).
- P2.2: **implemented** (alerts + signed webhooks).
- P2.3: **implemented** (optional AES-256-GCM encrypted Responses conversation store with retention, `store:false`, identity scoping and `previous_response_id` expansion).
- P2.4: **implemented** (policy simulator).
- P3.1: **implemented** (durable metadata + separate content spool, batch service-class execution through the normal gateway pipeline, pause/resume/cancel, restart recovery, retention, owner API, admin UI/API and accounting integration).
- P3.2: **implemented** as the public status dashboard in checkpoint 28. Multi-gateway HA is intentionally removed from the active roadmap after observed single-process performance showed no current capacity justification. The existing HA-readiness tools remain as diagnostic evidence tooling, not as a commitment to add distributed coordination.
### Checkpoint 23 note
At checkpoint 23, HA remained gated. Checkpoint 23 further reduces single-node production risk by hardening the Docker/reverse-proxy boundary: loopback-only publishing by default, an explicit production Compose file with no example-config fallback, read-only container rootfs/capability dropping/no-new-privileges, and a host preflight that rejects accidental public binding. This does not constitute HA implementation and does not change inference or persistent-state semantics.
### Checkpoint 24 note
At checkpoint 24, HA remained gated. Checkpoint 24 does not alter runtime scheduling, routing, persistence or HA semantics; it replaces the dark admin presentation with a professional light, data-dense control-plane theme. The embedded Live Flow and Infrastructure canvases were updated together with CSS so the UI remains visually coherent. No config/state migration is required.
### Checkpoint 27 note
At checkpoint 27, HA remained gated. Checkpoint 27 hardens context-window admission and routing before any HA work: the gateway no longer equates the theoretical `/api/show` model maximum with the context actually available on a worker. Effective context now derives from loaded `/api/ps context_length`, Modelfile `num_ctx`, explicit worker defaults and per-worker/model caps. Native `options.num_ctx` is validated/capped, while OpenAI/Responses estimation now covers `max_output_tokens`, `instructions`, `suffix`, vision reserves and an admission margin. Context-suitable workers are selected before inference, and the policy simulator/model UI expose the effective context evidence.
### Checkpoint 28 note
Checkpoint 28 closes the active P3 roadmap with an optional public read-only status dashboard. `/status/` exposes current load, queue, worker capacity, an infrastructure map and anonymized live request state through a dedicated sanitized API. It never reuses the admin snapshot schema. HA remains available only as a future re-evaluation if measured availability or capacity requirements change.
docs/SECURITY.md
+76
View File
@@ -0,0 +1,76 @@
# Security notes
## OIDC
The gateway validates bearer JWTs against OIDC discovery and JWKS. It rejects `alg=none`, restricts accepted signature algorithms, validates issuer, audience, expiry and `nbf`, and refreshes JWKS when a `kid` is unknown. API bearer-token validation is also the source of truth for the optional browser Authorization Code + PKCE login flow.
## Trusted-IP bypass
IP bypass is evaluated before bearer authentication. Starting with checkpoint 22, the **TCP peer address** is the authentication source for `auth.ip_bypass` by default, even when `X-Forwarded-For` is accepted for logging and usage attribution. This prevents a trusted-proxy chain from turning a spoofed forwarded address into a credential-free identity.
Set `auth.ip_bypass_use_forwarded_ip=true` only when a deployment intentionally needs an original client address behind a reverse proxy to satisfy IP bypass. That mode requires a tightly restricted `auth.trusted_proxies` list plus network ACLs that prevent clients from reaching the gateway directly. Prefer narrow `/32` or `/128` bypass/proxy entries. Never configure a broad Internet-wide bypass for an exposed gateway.
## API keys
API keys are represented in the authenticator by their SHA-256 digest; plaintext runtime secrets are not retained. Clients may send either `X-API-Key` or `Authorization: Bearer ...`.
There are two sources:
- **Configuration keys** come from `auth.api_keys`. Put their secrets in environment variables and reference them with `${NAME}` in JSON. They survive restarts and are read-only in the web UI.
- **Runtime keys** are created by an administrator in the web UI or through `POST /gateway/ui-api/api-keys`. A cryptographically random `ofg_...` secret is returned once, then only the digest and metadata are persisted in the local key store. UI-created keys can be revoked immediately and survive process restarts until revoked.
Only identities with `gateway:admin` may list, create or revoke API keys. Cookie-authenticated mutations are covered by the same CSRF checks as the rest of the control plane. Key secrets are never included in list responses, the redacted configuration endpoint or admin logs.
## Ollama management
With `native.management_requires_admin=true`, model-management operations require `gateway:admin`. Do not expose Ollama port 11434 directly to untrusted clients; expose only the gateway.
## Web control plane
Browser OIDC uses Authorization Code + PKCE. The transient verifier/state is HMAC-protected with `ui.session_secret`. After exchange, the browser receives only an opaque session identifier; the OIDC access token remains in the gateway's in-memory session store.
A process restart invalidates all browser sessions. Cookie-authenticated state-changing requests require a same-origin CSRF token. The embedded UI uses a restrictive Content Security Policy and has no CDN, analytics or third-party JavaScript dependency.
Policy overrides are persisted locally and take effect immediately for new requests.
## In-memory security boundary
Removing external state storage reduces the network attack surface: there is no coordination service, state-store credential or message-bus endpoint to protect. Sensitive runtime material such as OIDC session tokens and quota state exists only in process memory.
A crash or restart clears browser sessions and active transient inference jobs, but API keys, policy overrides, quota state, usage history, metric counters and durable batch definitions are restored from local durable state.
## Optional content-bearing conversations
The default gateway privacy posture remains content-free: prompts and generated responses are not persisted unless `conversations.enabled=true` is explicitly configured. The optional Responses conversation store is isolated from usage retention and uses AES-256-GCM encryption at rest, a dedicated retention window, an entry cap, and a per-context byte cap.
`previous_response_id` lookup is bound to the authenticated tenant and scheduler actor. A response ID from another identity is reported exactly like a nonexistent ID to avoid enumeration. `store:false` prevents persistence of the new response. The Admin UI exposes store status but not decrypted conversation content. See `docs/CONVERSATIONS.md`.
The conversation encryption key is a deployment secret. Losing or changing it makes the existing ciphertext unreadable; exposing it together with the state file defeats at-rest confidentiality. Backups should therefore be protected as secrets.
## Optional content-bearing batch jobs
Durable batch jobs are also disabled by default. Enabling them does not persist content by itself, but every submitted job stores its request body and later its response body under `storage.batch_jobs_dir`. These spool payloads are **not encrypted by the gateway**; they rely on mode-0600 files, a mode-0750 spool directory and the deployment filesystem/disk security boundary. Use filesystem or full-disk encryption if at-rest content encryption is required.
Batch metadata stores an authenticated identity/authorization snapshot but never the bearer token or API-key secret. Owner-facing `/gateway/v1/batches` endpoints are tenant+actor scoped; cross-identity IDs are hidden as not found. Global batch inspection/control and output download are restricted to the admin UI/API and therefore require `gateway:admin`.
An accepted job can execute after the original credential has been revoked because the credential secret is intentionally not retained. Current tenant model policy, worker placement/maintenance/circuit state, quotas and scheduling are still evaluated when each attempt runs. Operationally, revoke the credential **and cancel its accepted durable jobs** when retroactive cancellation is required.
Batch input/output files and backups can contain prompts, tool arguments, retrieved data and model output. Configure a short `batch_jobs.retention`, protect `storage.data_dir`, and treat backup archives as content-bearing secrets. See `docs/BATCH-JOBS.md`.
## Live telemetry
Live telemetry contains tenant/actor identifiers, request IDs, model names, worker names, timings, token counts and credit estimates, but never prompt text or generated model output. UI telemetry endpoints require an authenticated administrator.
`workers[].telemetry_url` is administrator-controlled and fetched by the gateway. Point it only to a trusted exporter because it is an SSRF-capable configured URL, just like an Ollama worker URL. The optional shipped worker-telemetry agent should be bound to a trusted management interface and restricted with `-allow-cidrs`; it intentionally does not create a separate application-secret store.
## Model access control
Model access control is independent from worker placement. Tenant defaults are configured under `model_access`; API keys can carry a narrower or different allow/deny list. The gateway applies the identity ACL before scheduling and filters `/api/tags` and `/v1/models` accordingly.
Patterns are exact model/alias names, `*`, or one trailing wildcard such as `qwen3:*`. The most specific matching rule wins; deny wins a specificity tie. API-key ACL metadata is persisted together with the key hash and is editable at creation time in **Admin -> Sicherheit -> API Keys**.
## Reverse-proxy network boundary
Prefer publishing the gateway only on loopback when the reverse proxy is local to the Docker host. A proxy should be the only network component able to reach the gateway's published port. Avoid trusting whole RFC1918 ranges merely because the deployment uses private addressing; list only concrete proxy/NAT peer addresses in `auth.trusted_proxies`.
Checkpoint 23's Compose defaults additionally run with a read-only root filesystem, all Linux capabilities dropped and `no-new-privileges`. Persistent writes are constrained to `/data`; `/tmp` is ephemeral. These controls reduce the impact of a process compromise but do not replace API-key/OIDC authentication or host firewalling.
docs/WARM-MODELS.md
+65
View File
@@ -0,0 +1,65 @@
# Warm Model Management
The warm-model manager controls Ollama model residency without putting model-management I/O on the inference hot path.
## Classes
- `hot`: keep the model resident on the requested number of eligible workers. A hot model is proactively preloaded.
- `warm`: optionally preload the model and unload it after `idle_timeout`.
- `cold`: do not preload and unload aggressively after `idle_timeout`.
Rules support exact names and one trailing `*` wildcard. The most specific matching rule wins.
```json
{
"warm_models": {
"enabled": true,
"reconcile_interval": "30s",
"operation_timeout": "2m",
"policies": {
"qwen3:8b": {
"class": "hot",
"workers": ["rtx-4090"],
"replicas": 1,
"preload": true,
"idle_timeout": "30m"
},
"gemma4:*": {
"class": "warm",
"replicas": 1,
"preload": false,
"idle_timeout": "20m"
}
}
}
}
```
## Safety rules
Warm-model operations are subordinate to normal routing policy:
1. worker must be healthy and in `active` maintenance mode;
2. preload must be allowed by Model Placement;
3. when worker inventory is known, preload requires the model to be installed;
4. workers with an open/half-open circuit are not used for proactive preload;
5. a preload/unload reserves a per-worker/per-model maintenance slot before calling Ollama;
6. inference acquisition for the same worker/model is blocked during that short operation;
7. unload is refused while the model has active inference jobs.
The maintenance reservation closes the race where an idle unload could otherwise begin at the same moment a new request is routed to the model.
## Persistence
Runtime rules edited in **Admin → Warm Models** are stored in `warm-models.json`. The static config remains the baseline and can be restored with **Auf Config zurücksetzen**.
Action history is retained with the same state file. Last-use timestamps are deliberately runtime state; after restart, currently loaded models receive a fresh grace period before idle eviction.
## VRAM pressure
The manager does not automatically evict arbitrary models under memory pressure. It exposes eviction suggestions for inactive, non-hot models. This avoids surprise unloads and lets the operator decide whether to act.
## Metrics
- `ollama_gateway_warm_actions_running`
- `ollama_gateway_warm_eviction_suggestions`
docs/WORKER-TELEMETRY.md
+127
View File
@@ -0,0 +1,127 @@
# Remote worker telemetry
Ollama Fair Gateway can consume optional out-of-band resource telemetry through `workers[].telemetry_url`. Checkpoint 20 ships a small `worker-telemetry` agent so remote Ollama hosts can report their own RAM/GPU state instead of accidentally reporting the gateway host through `local_system_stats`.
The agent is **not** part of the inference request path. Gateway workers fetch it during the normal health/telemetry refresh cycle.
## Why this matters for remote workers
`local_system_stats: true` runs the memory collector inside the gateway process. It is correct only when the Ollama worker and gateway share the same host. For a worker such as `http://10.2.10.48:11434`, it otherwise reports the gateway machine's RAM as if it belonged to `10.2.10.48`.
For remote workers, use:
```json
{
"name": "M75q - Gen5 - 1048",
"url": "http://10.2.10.48:11434",
"local_system_stats": false,
"telemetry_url": "http://10.2.10.48:11500/telemetry"
}
```
and run the agent on that Ollama host.
## Validate locally first
Host memory only:
```sh
./ollama-gateway-worker-telemetry-linux-amd64 -once
```
AMD GPU through the Linux amdgpu sysfs interface:
```sh
./ollama-gateway-worker-telemetry-linux-amd64 -once -amd-sysfs
```
If auto-detection picks the wrong GPU, specify the device explicitly:
```sh
./ollama-gateway-worker-telemetry-linux-amd64 \
-once \
-amd-sysfs \
-amd-device /sys/class/drm/card1/device
```
NVIDIA:
```sh
./ollama-gateway-worker-telemetry-linux-amd64 -once -nvidia-smi
```
The JSON schema matches the existing gateway `telemetry_url` contract:
```json
{
"memory_used_bytes": 123,
"memory_total_bytes": 456,
"vram_used_bytes": 789,
"vram_total_bytes": 1024,
"gpu_utilization_percent": 42,
"gpu_temperature_c": 63.5,
"gpu_power_watts": 88,
"source": "host-memory+amdgpu-sysfs",
"updated_at": "2026-09-08T18:00:00Z"
}
```
A field may be absent when the operating system/driver does not expose it. The agent still returns the remaining usable metrics and puts collector failures in the `error` field.
## Run as a service
The listener defaults to loopback for safety. To expose it to the gateway, bind a worker interface and restrict clients by CIDR. Prefer an exact gateway host address when possible:
```sh
./ollama-gateway-worker-telemetry-linux-amd64 \
-listen 0.0.0.0:11500 \
-allow-cidrs 10.2.19.1/32 \
-amd-sysfs
```
Example systemd unit:
```ini
[Unit]
Description=Ollama Gateway Worker Telemetry
After=network-online.target
[Service]
ExecStart=/usr/local/bin/ollama-gateway-worker-telemetry-linux-amd64 -listen 0.0.0.0:11500 -allow-cidrs 10.2.19.1/32 -amd-sysfs
Restart=on-failure
User=nobody
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
[Install]
WantedBy=multi-user.target
```
Adjust the CIDR to the source IP the worker actually sees from the gateway. Do not use `0.0.0.0/0` merely to make the endpoint reachable.
## AMD telemetry details
On Linux the agent auto-detects the first `/sys/class/drm/card*/device` whose PCI vendor is `0x1002`. It reads standard amdgpu attributes when available:
- `mem_info_vram_total`
- `mem_info_vram_used`
- `gpu_busy_percent`
- `hwmon/*/temp1_input`
- `hwmon/*/power1_average`
No ROCm library, `rocm-smi`, NVML or CGO is required for the AMD path. Integrated/shared-memory GPUs may expose different or incomplete VRAM semantics; keep the statically configured `memory_capacity_bytes`/`vram_capacity_bytes` values as hard capacity hints and treat runtime telemetry as routing pressure information.
## Security
The agent intentionally has no application credential store. It relies on a narrow listener/firewall/CIDR allowlist so it remains dependency-free and does not introduce a second secret lifecycle. Resource telemetry can still reveal infrastructure details, so expose it only on a trusted management network, VPN or host firewall. For untrusted network segments, put the endpoint behind a TLS-authenticated reverse proxy rather than exposing it directly.
`workers[].telemetry_url` remains administrator-controlled and is an SSRF-capable URL. Point it only to trusted worker exporters.
## Freshness and caching
Checkpoint 21 validates `updated_at` when the external exporter supplies it. The shipped agent always sends the field and also returns `Cache-Control: no-store`.
A sample is rejected when it is older than `max(30s, 6 x workers[].health_interval)` or more than 30 seconds in the future. Rejected values are not merged into the routing telemetry; the reason is exposed through the worker telemetry error instead. This prevents a caching proxy or badly skewed exporter clock from silently presenting old resource pressure as current data.
For compatibility, third-party exporters that omit `updated_at` are still accepted. Their freshness cannot be verified, so new integrations should always provide an RFC3339 timestamp.
go.mod
+3
View File
@@ -0,0 +1,3 @@
module github.com/example/ollama-fair-gateway
go 1.23
internal/alerts/alerts.go
+591
View File
@@ -0,0 +1,591 @@
package alerts
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
"github.com/example/ollama-fair-gateway/internal/state"
)
type Worker struct {
Name string
Healthy bool
CircuitState string
LastCircuitError string
VRAMUsedBytes int64
VRAMTotalBytes int64
}
type Snapshot struct {
QueueDepth int
QueueWait time.Duration
Workers []Worker
StorageBytes int64
}
type Provider func() Snapshot
type Event struct {
ID string `json:"id"`
Key string `json:"key"`
Type string `json:"type"`
Severity string `json:"severity"`
State string `json:"state"` // firing | resolved
Message string `json:"message"`
Worker string `json:"worker,omitempty"`
Tenant string `json:"tenant,omitempty"`
Actor string `json:"actor,omitempty"`
Value float64 `json:"value,omitempty"`
Threshold float64 `json:"threshold,omitempty"`
StartedAt time.Time `json:"started_at"`
UpdatedAt time.Time `json:"updated_at"`
ResolvedAt time.Time `json:"resolved_at,omitempty"`
Meta map[string]any `json:"meta,omitempty"`
}
type Delivery struct {
Time time.Time `json:"time"`
EventID string `json:"event_id"`
Webhook string `json:"webhook"`
Attempt int `json:"attempt,omitempty"`
StatusCode int `json:"status_code,omitempty"`
Error string `json:"error,omitempty"`
}
type deliveryTask struct {
Event Event
Webhook config.WebhookConfig
}
type Status struct {
Enabled bool `json:"enabled"`
Active []Event `json:"active"`
History []Event `json:"history"`
Deliveries []Delivery `json:"deliveries"`
Webhooks []map[string]any `json:"webhooks"`
Thresholds config.AlertThresholds `json:"thresholds"`
LastEvaluate time.Time `json:"last_evaluate,omitempty"`
LastError string `json:"last_error,omitempty"`
}
type persistent struct {
Active map[string]Event `json:"active"`
History []Event `json:"history"`
Deliveries []Delivery `json:"deliveries"`
LastSent map[string]time.Time `json:"last_sent"`
}
type Manager struct {
mu sync.RWMutex
cfg config.AlertsConfig
file state.AtomicJSON
provider Provider
client *http.Client
active map[string]Event
history []Event
deliveries []Delivery
lastSent map[string]time.Time
downSince map[string]time.Time
lastEvaluate time.Time
lastError string
deliveryQ chan deliveryTask
persistWake chan struct{}
started atomic.Bool
}
func New(cfg config.AlertsConfig, path string, provider Provider) (*Manager, error) {
if cfg.WebhookTimeout.Value() <= 0 {
cfg.WebhookTimeout = config.Duration(5 * time.Second)
}
if cfg.WebhookMaxConcurrent <= 0 {
cfg.WebhookMaxConcurrent = 4
}
if cfg.WebhookQueue <= 0 {
cfg.WebhookQueue = 1024
}
if cfg.WebhookRetryAttempts <= 0 {
cfg.WebhookRetryAttempts = 3
}
if cfg.WebhookRetryBackoff.Value() < 0 {
cfg.WebhookRetryBackoff = 0
}
m := &Manager{cfg: cfg, file: state.AtomicJSON{Path: path, Mode: 0600}, provider: provider, client: &http.Client{Timeout: cfg.WebhookTimeout.Value()}, active: map[string]Event{}, lastSent: map[string]time.Time{}, downSince: map[string]time.Time{}, deliveryQ: make(chan deliveryTask, cfg.WebhookQueue), persistWake: make(chan struct{}, 1)}
var p persistent
if err := m.file.Load(&p); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
} else if err == nil {
if p.Active != nil {
m.active = p.Active
}
m.history = p.History
m.deliveries = p.Deliveries
if p.LastSent != nil {
m.lastSent = p.LastSent
}
m.trimLocked()
}
return m, nil
}
func (m *Manager) Start(ctx context.Context) {
if m == nil || !m.cfg.Enabled || !m.started.CompareAndSwap(false, true) {
return
}
for i := 0; i < m.cfg.WebhookMaxConcurrent; i++ {
go m.deliveryWorker(ctx)
}
go m.persistenceWorker(ctx)
go func() {
m.Evaluate()
t := time.NewTicker(m.cfg.EvaluationInterval.Value())
defer t.Stop()
for {
select {
case <-ctx.Done():
_ = m.persistNow()
return
case <-t.C:
m.Evaluate()
}
}
}()
}
func (m *Manager) Evaluate() {
if m == nil || !m.cfg.Enabled || m.provider == nil {
return
}
s := m.provider()
now := time.Now().UTC()
present := map[string]Event{}
th := m.cfg.Thresholds
if th.QueueDepth > 0 && s.QueueDepth >= th.QueueDepth {
e := newEvent("queue_depth", "warning", fmt.Sprintf("queue depth %d exceeds threshold %d", s.QueueDepth, th.QueueDepth))
e.Value = float64(s.QueueDepth)
e.Threshold = float64(th.QueueDepth)
present[e.Key] = e
}
if th.QueueWait.Value() > 0 && s.QueueWait >= th.QueueWait.Value() {
e := newEvent("queue_wait", "warning", fmt.Sprintf("oldest queued request has waited %s (threshold %s)", s.QueueWait.Round(time.Millisecond), th.QueueWait.Value()))
e.Value = s.QueueWait.Seconds()
e.Threshold = th.QueueWait.Value().Seconds()
present[e.Key] = e
}
if th.StorageBytes > 0 && s.StorageBytes >= th.StorageBytes {
e := newEvent("storage_growth", "warning", fmt.Sprintf("gateway storage %d bytes exceeds threshold %d", s.StorageBytes, th.StorageBytes))
e.Value = float64(s.StorageBytes)
e.Threshold = float64(th.StorageBytes)
present[e.Key] = e
}
m.mu.Lock()
for _, w := range s.Workers {
if !w.Healthy {
if m.downSince[w.Name].IsZero() {
m.downSince[w.Name] = now
}
if th.WorkerDownFor.Value() <= 0 || now.Sub(m.downSince[w.Name]) >= th.WorkerDownFor.Value() {
e := newEvent("worker_down:"+w.Name, "critical", fmt.Sprintf("worker %s is unhealthy", w.Name))
e.Type = "worker_down"
e.Worker = w.Name
present[e.Key] = e
}
} else {
delete(m.downSince, w.Name)
}
if th.CircuitOpen && w.CircuitState == "open" {
e := newEvent("circuit_open:"+w.Name, "warning", fmt.Sprintf("worker %s circuit breaker is open", w.Name))
e.Type = "circuit_open"
e.Worker = w.Name
present[e.Key] = e
}
if th.VRAMPercent > 0 && w.VRAMTotalBytes > 0 {
pct := 100 * float64(w.VRAMUsedBytes) / float64(w.VRAMTotalBytes)
if pct >= th.VRAMPercent {
e := newEvent("vram_pressure:"+w.Name, "warning", fmt.Sprintf("worker %s VRAM usage %.1f%% exceeds %.1f%%", w.Name, pct, th.VRAMPercent))
e.Type = "vram_pressure"
e.Worker = w.Name
e.Value = pct
e.Threshold = th.VRAMPercent
present[e.Key] = e
}
}
if th.OOM && looksOOM(w.LastCircuitError) {
e := newEvent("oom:"+w.Name, "critical", fmt.Sprintf("worker %s reported an out-of-memory failure", w.Name))
e.Type = "oom"
e.Worker = w.Name
e.Meta = map[string]any{"last_error": w.LastCircuitError}
present[e.Key] = e
}
}
toSend := m.reconcileLocked(present, now)
m.lastEvaluate = now
m.mu.Unlock()
m.schedulePersist()
for _, e := range toSend {
m.deliver(e)
}
}
func newEvent(key, severity, msg string) Event {
now := time.Now().UTC()
return Event{ID: eventID(key, now), Key: key, Type: key, Severity: severity, State: "firing", Message: msg, StartedAt: now, UpdatedAt: now}
}
func (m *Manager) reconcileLocked(present map[string]Event, now time.Time) []Event {
toSend := []Event{}
for k, next := range present {
if cur, ok := m.active[k]; ok {
next.ID = cur.ID
next.StartedAt = cur.StartedAt
next.UpdatedAt = now
m.active[k] = next
if m.cooldownReadyLocked(k, now) {
toSend = append(toSend, next)
m.lastSent[k] = now
}
continue
}
m.active[k] = next
m.history = append(m.history, next)
toSend = append(toSend, next)
m.lastSent[k] = now
}
for k, cur := range m.active {
if _, ok := present[k]; ok {
continue
}
cur.State = "resolved"
cur.UpdatedAt = now
cur.ResolvedAt = now
m.history = append(m.history, cur)
delete(m.active, k)
toSend = append(toSend, cur)
m.lastSent[k] = now
}
m.trimLocked()
return toSend
}
func (m *Manager) cooldownReadyLocked(k string, now time.Time) bool {
d := m.cfg.Cooldown.Value()
return d <= 0 || m.lastSent[k].IsZero() || now.Sub(m.lastSent[k]) >= d
}
// ObserveQuota feeds the latest token-bucket remainder into alerting without
// putting webhook delivery on the request path. Delivery always happens in a goroutine.
func (m *Manager) ObserveQuota(tenant, actor string, remainingActor, remainingTenant, actorCap, tenantCap float64) {
if m == nil || !m.cfg.Enabled || m.cfg.Thresholds.QuotaRemainingPct <= 0 {
return
}
threshold := m.cfg.Thresholds.QuotaRemainingPct
now := time.Now().UTC()
events := []Event{}
check := func(kind, name string, remaining, cap float64) {
if cap <= 0 {
return
}
pct := 100 * remaining / cap
key := "quota_" + kind + ":" + name
m.mu.Lock()
if pct <= threshold {
e := newEvent(key, "warning", fmt.Sprintf("%s quota %s has %.1f%% credits remaining", kind, name, pct))
e.Type = "quota_near_exhaustion"
e.Tenant = tenant
if kind == "actor" {
e.Actor = actor
}
e.Value = pct
e.Threshold = threshold
if cur, ok := m.active[key]; ok {
e.ID = cur.ID
e.StartedAt = cur.StartedAt
}
m.active[key] = e
if _, seen := m.lastSent[key]; !seen || m.cooldownReadyLocked(key, now) {
m.history = append(m.history, e)
m.lastSent[key] = now
events = append(events, e)
}
} else if cur, ok := m.active[key]; ok {
cur.State = "resolved"
cur.UpdatedAt = now
cur.ResolvedAt = now
m.history = append(m.history, cur)
delete(m.active, key)
events = append(events, cur)
}
m.trimLocked()
m.mu.Unlock()
m.schedulePersist()
}
check("actor", tenant+"/"+actor, remainingActor, actorCap)
check("tenant", tenant, remainingTenant, tenantCap)
for _, e := range events {
go m.deliver(e)
}
}
func (m *Manager) Status() Status {
if m == nil {
return Status{}
}
m.mu.RLock()
defer m.mu.RUnlock()
active := make([]Event, 0, len(m.active))
for _, e := range m.active {
active = append(active, e)
}
sort.Slice(active, func(i, j int) bool { return active[i].StartedAt.After(active[j].StartedAt) })
hist := append([]Event(nil), m.history...)
sort.Slice(hist, func(i, j int) bool { return hist[i].UpdatedAt.After(hist[j].UpdatedAt) })
if len(hist) > m.cfg.HistoryLimit {
hist = hist[:m.cfg.HistoryLimit]
}
ds := append([]Delivery(nil), m.deliveries...)
sort.Slice(ds, func(i, j int) bool { return ds[i].Time.After(ds[j].Time) })
if len(ds) > 100 {
ds = ds[:100]
}
wh := make([]map[string]any, 0, len(m.cfg.Webhooks))
for _, w := range m.cfg.Webhooks {
wh = append(wh, map[string]any{"name": w.Name, "url": w.URL, "enabled": w.Enabled, "signed": w.Secret != ""})
}
return Status{Enabled: m.cfg.Enabled, Active: active, History: hist, Deliveries: ds, Webhooks: wh, Thresholds: m.cfg.Thresholds, LastEvaluate: m.lastEvaluate, LastError: m.lastError}
}
func (m *Manager) TestWebhook(name string) error {
if m == nil {
return errors.New("alerts unavailable")
}
e := newEvent("webhook_test", "info", "Ollama Gateway webhook test")
e.Type = "test"
return m.deliverToNamed(e, name)
}
func (m *Manager) deliver(e Event) {
for _, w := range m.cfg.Webhooks {
if !w.Enabled {
continue
}
task := deliveryTask{Event: e, Webhook: w}
if !m.started.Load() {
// A manager that has not been Start()ed has no owned worker lifecycle
// to join at shutdown. Deliver synchronously in that uncommon/test-only
// mode so no orphan goroutine can write persistence after its caller
// has torn down the state directory. Production managers use deliveryQ.
_ = m.deliverWithRetry(context.Background(), task)
continue
}
select {
case m.deliveryQ <- task:
default:
m.recordDelivery(Delivery{Time: time.Now().UTC(), EventID: e.ID, Webhook: w.Name, Error: "webhook delivery queue full"})
}
}
}
func (m *Manager) deliverToNamed(e Event, name string) error {
for _, w := range m.cfg.Webhooks {
if w.Enabled && (name == "" || w.Name == name) {
return m.deliverWithRetry(context.Background(), deliveryTask{Event: e, Webhook: w})
}
}
return fmt.Errorf("enabled webhook %q not found", name)
}
func (m *Manager) deliveryWorker(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case task := <-m.deliveryQ:
_ = m.deliverWithRetry(ctx, task)
}
}
}
func (m *Manager) deliverWithRetry(ctx context.Context, task deliveryTask) error {
attempts := m.cfg.WebhookRetryAttempts
if attempts <= 0 {
attempts = 1
}
var last error
for attempt := 1; attempt <= attempts; attempt++ {
retry, err := m.deliverOne(ctx, task.Event, task.Webhook, attempt)
if err == nil {
return nil
}
last = err
if !retry || attempt == attempts {
break
}
d := m.cfg.WebhookRetryBackoff.Value()
if d > 0 {
// Bounded exponential backoff keeps retry storms away from downstreams.
for i := 1; i < attempt && d < 30*time.Second; i++ {
d *= 2
}
if d > 30*time.Second {
d = 30 * time.Second
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(d):
}
}
}
return last
}
// deliverOne returns retry=true only for transient failures. Event IDs remain
// stable across attempts so webhook consumers can deduplicate deliveries.
func (m *Manager) deliverOne(ctx context.Context, e Event, w config.WebhookConfig, attempt int) (retry bool, err error) {
payload := map[string]any{"version": "1", "source": "ollama-fair-gateway", "event": e}
b, _ := json.Marshal(payload)
reqCtx := ctx
var cancel context.CancelFunc
if timeout := m.cfg.WebhookTimeout.Value(); timeout > 0 {
reqCtx, cancel = context.WithTimeout(ctx, timeout)
defer cancel()
}
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, w.URL, bytes.NewReader(b))
if err != nil {
m.recordDelivery(Delivery{Time: time.Now().UTC(), EventID: e.ID, Webhook: w.Name, Attempt: attempt, Error: err.Error()})
return false, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "ollama-fair-gateway/alerts")
ts := fmt.Sprint(time.Now().Unix())
req.Header.Set("X-Ollama-Gateway-Timestamp", ts)
req.Header.Set("X-Ollama-Gateway-Event-ID", e.ID)
req.Header.Set("X-Ollama-Gateway-Delivery-Attempt", fmt.Sprint(attempt))
if w.Secret != "" {
mac := hmac.New(sha256.New, []byte(w.Secret))
_, _ = mac.Write([]byte(ts + "."))
_, _ = mac.Write(b)
req.Header.Set("X-Ollama-Gateway-Signature", "sha256="+hex.EncodeToString(mac.Sum(nil)))
}
resp, err := m.client.Do(req)
d := Delivery{Time: time.Now().UTC(), EventID: e.ID, Webhook: w.Name, Attempt: attempt}
if err != nil {
d.Error = err.Error()
m.recordDelivery(d)
return true, err
}
d.StatusCode = resp.StatusCode
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
_ = resp.Body.Close()
if resp.StatusCode/100 != 2 {
err = fmt.Errorf("webhook HTTP %d", resp.StatusCode)
d.Error = err.Error()
m.recordDelivery(d)
return resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500, err
}
m.recordDelivery(d)
return false, nil
}
func (m *Manager) recordDelivery(d Delivery) {
m.mu.Lock()
m.deliveries = append(m.deliveries, d)
if d.Error != "" {
m.lastError = d.Error
}
m.trimLocked()
m.mu.Unlock()
m.schedulePersist()
}
func (m *Manager) schedulePersist() {
if m == nil {
return
}
if !m.started.Load() {
_ = m.persistNow()
return
}
select {
case m.persistWake <- struct{}{}:
default:
}
}
func (m *Manager) persistenceWorker(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case <-m.persistWake:
if err := m.persistNow(); err != nil {
m.mu.Lock()
m.lastError = err.Error()
m.mu.Unlock()
}
}
}
}
func (m *Manager) persistNow() error {
m.mu.RLock()
active := make(map[string]Event, len(m.active))
for k, v := range m.active {
active[k] = v
}
history := append([]Event(nil), m.history...)
deliveries := append([]Delivery(nil), m.deliveries...)
lastSent := make(map[string]time.Time, len(m.lastSent))
for k, v := range m.lastSent {
lastSent[k] = v
}
m.mu.RUnlock()
return m.file.Save(persistent{Active: active, History: history, Deliveries: deliveries, LastSent: lastSent})
}
func (m *Manager) trimLocked() {
limit := m.cfg.HistoryLimit
if limit <= 0 {
limit = 500
}
if len(m.history) > limit {
m.history = append([]Event(nil), m.history[len(m.history)-limit:]...)
}
if len(m.deliveries) > 200 {
m.deliveries = append([]Delivery(nil), m.deliveries[len(m.deliveries)-200:]...)
}
}
func looksOOM(s string) bool {
s = strings.ToLower(s)
return strings.Contains(s, "out of memory") || strings.Contains(s, "oom") || strings.Contains(s, "cuda error: out of memory")
}
func eventID(k string, t time.Time) string {
h := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", k, t.UnixNano())))
return hex.EncodeToString(h[:8])
}
func DirSize(root string) int64 {
var total int64
_ = filepath.Walk(root, func(_ string, info os.FileInfo, err error) error {
if err == nil && info != nil && !info.IsDir() {
total += info.Size()
}
return nil
})
return total
}
internal/alerts/alerts_test.go
+127
View File
@@ -0,0 +1,127 @@
package alerts
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"io"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
)
func TestSignedWebhookAndResolution(t *testing.T) {
var mu sync.Mutex
healthy := false
received := make(chan string, 4)
secret := "test-secret"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := io.ReadAll(r.Body)
tsv := r.Header.Get("X-Ollama-Gateway-Timestamp")
mac := hmac.New(sha256.New, []byte(secret))
_, _ = mac.Write([]byte(tsv + "."))
_, _ = mac.Write(b)
want := "sha256=" + hex.EncodeToString(mac.Sum(nil))
if r.Header.Get("X-Ollama-Gateway-Signature") != want {
t.Errorf("bad signature")
}
received <- string(b)
w.WriteHeader(204)
}))
defer ts.Close()
cfg := config.AlertsConfig{Enabled: true, EvaluationInterval: config.Duration(time.Second), Cooldown: config.Duration(time.Hour), HistoryLimit: 20, Webhooks: []config.WebhookConfig{{Name: "test", URL: ts.URL, Secret: secret, Enabled: true}}, Thresholds: config.AlertThresholds{WorkerDownFor: config.Duration(time.Nanosecond)}}
m, err := New(cfg, t.TempDir()+"/alerts.json", func() Snapshot {
mu.Lock()
h := healthy
mu.Unlock()
return Snapshot{Workers: []Worker{{Name: "w", Healthy: h}}}
})
if err != nil {
t.Fatal(err)
}
m.Evaluate()
time.Sleep(time.Millisecond)
m.Evaluate()
select {
case <-received:
case <-time.After(2 * time.Second):
t.Fatal("firing webhook missing")
}
if len(m.Status().Active) != 1 {
t.Fatalf("active=%#v", m.Status().Active)
}
mu.Lock()
healthy = true
mu.Unlock()
m.Evaluate()
select {
case <-received:
case <-time.After(2 * time.Second):
t.Fatal("resolved webhook missing")
}
if len(m.Status().Active) != 0 {
t.Fatal("alert did not resolve")
}
}
func TestQuotaNearExhaustion(t *testing.T) {
cfg := config.AlertsConfig{Enabled: true, Cooldown: config.Duration(time.Hour), HistoryLimit: 20, Thresholds: config.AlertThresholds{QuotaRemainingPct: 10}}
m, err := New(cfg, t.TempDir()+"/alerts.json", nil)
if err != nil {
t.Fatal(err)
}
m.ObserveQuota("t", "a", 5, 50, 100, 100)
st := m.Status()
if len(st.Active) != 1 || st.Active[0].Type != "quota_near_exhaustion" {
t.Fatalf("active=%#v", st.Active)
}
m.ObserveQuota("t", "a", 50, 50, 100, 100)
if len(m.Status().Active) != 0 {
t.Fatal("quota alert did not resolve")
}
}
func TestWebhookRetriesTransientFailure(t *testing.T) {
var mu sync.Mutex
attempts := 0
seen := make(chan int, 4)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
attempts++
n := attempts
mu.Unlock()
seen <- n
if n < 3 {
http.Error(w, "temporary", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusNoContent)
}))
defer ts.Close()
cfg := config.AlertsConfig{
Enabled: true, EvaluationInterval: config.Duration(time.Hour), Cooldown: config.Duration(time.Hour), HistoryLimit: 20,
WebhookTimeout: config.Duration(time.Second), WebhookMaxConcurrent: 1, WebhookQueue: 8, WebhookRetryAttempts: 3, WebhookRetryBackoff: config.Duration(time.Millisecond),
Webhooks: []config.WebhookConfig{{Name: "retry", URL: ts.URL, Enabled: true}},
}
m, err := New(cfg, t.TempDir()+"/alerts.json", nil)
if err != nil {
t.Fatal(err)
}
if err := m.TestWebhook("retry"); err != nil {
t.Fatal(err)
}
mu.Lock()
got := attempts
mu.Unlock()
if got != 3 {
t.Fatalf("attempts=%d want 3", got)
}
st := m.Status()
if len(st.Deliveries) < 3 || st.Deliveries[0].StatusCode != http.StatusNoContent {
t.Fatalf("deliveries=%#v", st.Deliveries)
}
}
internal/auth/identity.go
+492
View File
@@ -0,0 +1,492 @@
package auth
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"net"
"net/http"
"sort"
"strings"
"sync"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
)
type Identity struct {
Tenant string `json:"tenant"`
Subject string `json:"subject"`
Application string `json:"application,omitempty"`
AuthType string `json:"auth_type"`
Scopes map[string]bool `json:"-"`
ClientIP string `json:"client_ip"`
ModelACLSet bool `json:"-"`
ModelAccess config.ModelAccessRule `json:"-"`
ServiceClass string `json:"-"`
}
func (i Identity) Actor() string {
// Interactive OIDC identities are fair-scheduled by subject, so one user
// cannot gain extra shares by using multiple OAuth clients. Static API-key
// and trusted-IP identities are normally applications and use that identity.
if i.AuthType != "oidc" && i.Application != "" {
return "app:" + i.Application
}
if i.Subject != "" {
return i.Subject
}
if i.Application != "" {
return "app:" + i.Application
}
return "anonymous"
}
func (i Identity) HasScope(s string) bool { return i.Scopes[s] || i.Scopes["*"] }
func (i Identity) IsAdmin() bool { return i.HasScope("gateway:admin") }
type ctxKey struct{}
func WithIdentity(ctx context.Context, i Identity) context.Context {
return context.WithValue(ctx, ctxKey{}, i)
}
func FromContext(ctx context.Context) (Identity, bool) {
i, ok := ctx.Value(ctxKey{}).(Identity)
return i, ok
}
// APIKeyInfo is safe to return through the admin API. It never contains the
// key secret. UI-created keys may be backed by a durable RuntimeKeyStore.
type APIKeyInfo struct {
ID string `json:"id,omitempty"`
Name string `json:"name"`
Tenant string `json:"tenant"`
Subject string `json:"subject"`
Application string `json:"application,omitempty"`
Scopes []string `json:"scopes"`
AllowedModels []string `json:"allowed_models,omitempty"`
DeniedModels []string `json:"denied_models,omitempty"`
ServiceClass string `json:"service_class,omitempty"`
Source string `json:"source"` // config | runtime | persistent
KeyHint string `json:"key_hint,omitempty"`
CreatedAt *time.Time `json:"created_at,omitempty"`
Deletable bool `json:"deletable"`
}
type APIKeyCreate struct {
Name string
Tenant string
Subject string
Application string
Scopes []string
AllowedModels []string
DeniedModels []string
ServiceClass string
}
// StoredAPIKey is the durable representation of a UI-created API key. Only
// the SHA-256 hash is persisted; the plaintext secret never leaves CreateAPIKey.
type StoredAPIKey struct {
ID string `json:"id"`
Name string `json:"name"`
Tenant string `json:"tenant"`
Subject string `json:"subject"`
Application string `json:"application,omitempty"`
Scopes []string `json:"scopes"`
AllowedModels []string `json:"allowed_models,omitempty"`
DeniedModels []string `json:"denied_models,omitempty"`
ServiceClass string `json:"service_class,omitempty"`
HashHex string `json:"hash_sha256"`
KeyHint string `json:"key_hint,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
type RuntimeKeyStore interface {
Load() ([]StoredAPIKey, error)
Put(StoredAPIKey) error
Delete(string) error
Health(context.Context) error
}
type managedKey struct {
identity Identity
info APIKeyInfo
}
type bypassRule struct {
nets []*net.IPNet
identity Identity
}
type Authenticator struct {
oidc *OIDCVerifier
mu sync.RWMutex
keys map[[32]byte]managedKey
runtime map[string][32]byte
bypass []bypassRule
trusted []*net.IPNet
bypassForwarded bool
store RuntimeKeyStore
}
func New(ctx context.Context, cfg config.AuthConfig) (*Authenticator, error) {
return NewWithRuntimeStore(ctx, cfg, nil)
}
func NewWithRuntimeStore(ctx context.Context, cfg config.AuthConfig, store RuntimeKeyStore) (*Authenticator, error) {
a := &Authenticator{keys: make(map[[32]byte]managedKey), runtime: make(map[string][32]byte), bypassForwarded: cfg.IPBypassUseForwardedIP, store: store}
for _, c := range cfg.TrustedProxies {
_, n, _ := net.ParseCIDR(c)
a.trusted = append(a.trusted, n)
}
for _, b := range cfg.IPBypass {
r := bypassRule{identity: Identity{Tenant: b.Tenant, Subject: b.Subject, Application: b.Application, AuthType: "ip-bypass", Scopes: scopeMap(b.Scopes)}}
if r.identity.Subject == "" {
r.identity.Subject = "ip-bypass"
}
for _, c := range b.CIDRs {
_, n, _ := net.ParseCIDR(c)
r.nets = append(r.nets, n)
}
a.bypass = append(a.bypass, r)
}
for _, k := range cfg.APIKeys {
if k.Key == "" {
return nil, fmt.Errorf("api key %q is empty (is its environment variable set?)", k.Name)
}
id := Identity{Tenant: k.Tenant, Subject: k.Subject, Application: k.Application, AuthType: "api-key", Scopes: scopeMap(k.Scopes), ModelACLSet: len(k.AllowedModels) > 0 || len(k.DeniedModels) > 0, ModelAccess: config.ModelAccessRule{Mode: "allow_all", AllowedModels: append([]string(nil), k.AllowedModels...), DeniedModels: append([]string(nil), k.DeniedModels...)}, ServiceClass: strings.TrimSpace(k.ServiceClass)}
if id.Tenant == "" {
return nil, fmt.Errorf("api key %q has no tenant", k.Name)
}
if id.Subject == "" {
id.Subject = "apikey:" + k.Name
}
h := sha256.Sum256([]byte(k.Key))
a.keys[h] = managedKey{identity: id, info: APIKeyInfo{Name: k.Name, Tenant: id.Tenant, Subject: id.Subject, Application: id.Application, Scopes: sortedScopes(k.Scopes), AllowedModels: append([]string(nil), k.AllowedModels...), DeniedModels: append([]string(nil), k.DeniedModels...), ServiceClass: strings.TrimSpace(k.ServiceClass), Source: "config", Deletable: false}}
}
if store != nil {
records, err := store.Load()
if err != nil {
return nil, fmt.Errorf("load persistent API keys: %w", err)
}
for _, rec := range records {
hb, err := hex.DecodeString(rec.HashHex)
if err != nil || len(hb) != sha256.Size {
return nil, fmt.Errorf("persistent API key %q has invalid hash", rec.ID)
}
var h [32]byte
copy(h[:], hb)
if rec.ID == "" || rec.Tenant == "" {
return nil, fmt.Errorf("persistent API key has missing id or tenant")
}
created := rec.CreatedAt
info := APIKeyInfo{ID: rec.ID, Name: rec.Name, Tenant: rec.Tenant, Subject: rec.Subject, Application: rec.Application, Scopes: sortedScopes(rec.Scopes), AllowedModels: append([]string(nil), rec.AllowedModels...), DeniedModels: append([]string(nil), rec.DeniedModels...), ServiceClass: rec.ServiceClass, Source: "persistent", KeyHint: rec.KeyHint, CreatedAt: &created, Deletable: true}
id := Identity{Tenant: rec.Tenant, Subject: rec.Subject, Application: rec.Application, AuthType: "api-key", Scopes: scopeMap(rec.Scopes), ModelACLSet: len(rec.AllowedModels) > 0 || len(rec.DeniedModels) > 0, ModelAccess: config.ModelAccessRule{Mode: "allow_all", AllowedModels: append([]string(nil), rec.AllowedModels...), DeniedModels: append([]string(nil), rec.DeniedModels...)}, ServiceClass: rec.ServiceClass}
if id.Subject == "" {
id.Subject = "apikey:" + rec.Name
info.Subject = id.Subject
}
if _, exists := a.keys[h]; exists {
return nil, fmt.Errorf("persistent API key hash collision for %q", rec.ID)
}
if _, exists := a.runtime[rec.ID]; exists {
return nil, fmt.Errorf("duplicate persistent API key id %q", rec.ID)
}
a.keys[h] = managedKey{identity: id, info: info}
a.runtime[rec.ID] = h
}
}
if cfg.OIDC.Enabled {
v, err := NewOIDCVerifier(ctx, cfg.OIDC)
if err != nil {
return nil, err
}
a.oidc = v
}
return a, nil
}
func scopeMap(in []string) map[string]bool {
m := map[string]bool{}
for _, s := range in {
s = strings.TrimSpace(s)
if s != "" {
m[s] = true
}
}
return m
}
func sortedScopes(in []string) []string {
m := map[string]struct{}{}
for _, s := range in {
if s = strings.TrimSpace(s); s != "" {
m[s] = struct{}{}
}
}
out := make([]string, 0, len(m))
for s := range m {
out = append(out, s)
}
sort.Strings(out)
return out
}
func (a *Authenticator) Authenticate(r *http.Request) (Identity, error) {
ip := a.ClientIP(r)
bypassIP := a.PeerIP(r)
if a.bypassForwarded {
bypassIP = ip
}
for _, rule := range a.bypass {
if containsAny(rule.nets, net.ParseIP(bypassIP)) {
id := rule.identity
// Keep the resolved client IP for observability even though the
// authentication decision defaults to the TCP peer address.
id.ClientIP = ip
return id, nil
}
}
token := ""
if x := strings.TrimSpace(r.Header.Get("X-API-Key")); x != "" {
token = x
}
if token == "" {
h := r.Header.Get("Authorization")
if len(h) > 7 && strings.EqualFold(h[:7], "Bearer ") {
token = strings.TrimSpace(h[7:])
}
}
if token != "" {
sum := sha256.Sum256([]byte(token))
a.mu.RLock()
k, ok := a.keys[sum]
a.mu.RUnlock()
if ok {
id := k.identity
id.ClientIP = ip
return id, nil
}
if a.oidc != nil {
id, err := a.oidc.Verify(r.Context(), token)
if err == nil {
id.ClientIP = ip
return id, nil
}
return Identity{}, fmt.Errorf("invalid bearer token: %w", err)
}
}
return Identity{}, ErrUnauthorized
}
var ErrUnauthorized = fmt.Errorf("authentication required")
// CreateAPIKey creates an API key. With a RuntimeKeyStore configured, only the
// key hash and metadata are persisted. The returned secret is the
// only copy of the plaintext key and must be shown to the administrator once.
func (a *Authenticator) CreateAPIKey(in APIKeyCreate) (APIKeyInfo, string, error) {
if a == nil {
return APIKeyInfo{}, "", errors.New("authenticator unavailable")
}
in.Name = strings.TrimSpace(in.Name)
in.Tenant = strings.TrimSpace(in.Tenant)
in.Subject = strings.TrimSpace(in.Subject)
in.Application = strings.TrimSpace(in.Application)
if in.Name == "" || len(in.Name) > 128 {
return APIKeyInfo{}, "", errors.New("name is required and must be at most 128 characters")
}
if in.Tenant == "" || len(in.Tenant) > 256 {
return APIKeyInfo{}, "", errors.New("tenant is required and must be at most 256 characters")
}
if len(in.Subject) > 256 || len(in.Application) > 256 {
return APIKeyInfo{}, "", errors.New("subject and application must be at most 256 characters")
}
if in.Subject == "" {
in.Subject = "apikey:" + in.Name
}
scopes := sortedScopes(in.Scopes)
for _, s := range scopes {
if len(s) > 128 {
return APIKeyInfo{}, "", errors.New("scope must be at most 128 characters")
}
}
if err := config.ValidateModelAccessRule(config.ModelAccessRule{Mode: "allow_all", AllowedModels: in.AllowedModels, DeniedModels: in.DeniedModels}); err != nil {
return APIKeyInfo{}, "", fmt.Errorf("model ACL: %w", err)
}
secretBytes := make([]byte, 32)
if _, err := rand.Read(secretBytes); err != nil {
return APIKeyInfo{}, "", fmt.Errorf("generate key: %w", err)
}
secret := "ofg_" + base64.RawURLEncoding.EncodeToString(secretBytes)
idBytes := make([]byte, 12)
if _, err := rand.Read(idBytes); err != nil {
return APIKeyInfo{}, "", fmt.Errorf("generate key id: %w", err)
}
id := base64.RawURLEncoding.EncodeToString(idBytes)
h := sha256.Sum256([]byte(secret))
createdAt := time.Now().UTC()
info := APIKeyInfo{ID: id, Name: in.Name, Tenant: in.Tenant, Subject: in.Subject, Application: in.Application, Scopes: scopes, AllowedModels: append([]string(nil), in.AllowedModels...), DeniedModels: append([]string(nil), in.DeniedModels...), ServiceClass: strings.TrimSpace(in.ServiceClass), Source: map[bool]string{true: "persistent", false: "runtime"}[a.store != nil], KeyHint: keyHint(secret), CreatedAt: &createdAt, Deletable: true}
identity := Identity{Tenant: in.Tenant, Subject: in.Subject, Application: in.Application, AuthType: "api-key", Scopes: scopeMap(scopes), ModelACLSet: len(in.AllowedModels) > 0 || len(in.DeniedModels) > 0, ModelAccess: config.ModelAccessRule{Mode: "allow_all", AllowedModels: append([]string(nil), in.AllowedModels...), DeniedModels: append([]string(nil), in.DeniedModels...)}, ServiceClass: strings.TrimSpace(in.ServiceClass)}
a.mu.Lock()
defer a.mu.Unlock()
for _, existing := range a.keys {
if existing.info.Name == in.Name && existing.info.Tenant == in.Tenant {
return APIKeyInfo{}, "", fmt.Errorf("an API key named %q already exists for tenant %q", in.Name, in.Tenant)
}
}
if _, exists := a.keys[h]; exists {
return APIKeyInfo{}, "", errors.New("generated API key collision")
}
if _, exists := a.runtime[id]; exists {
return APIKeyInfo{}, "", errors.New("generated API key id collision")
}
if a.store != nil {
rec := StoredAPIKey{ID: id, Name: in.Name, Tenant: in.Tenant, Subject: in.Subject, Application: in.Application, Scopes: scopes, AllowedModels: append([]string(nil), in.AllowedModels...), DeniedModels: append([]string(nil), in.DeniedModels...), ServiceClass: strings.TrimSpace(in.ServiceClass), HashHex: hex.EncodeToString(h[:]), KeyHint: info.KeyHint, CreatedAt: createdAt}
if err := a.store.Put(rec); err != nil {
return APIKeyInfo{}, "", fmt.Errorf("persist API key: %w", err)
}
}
a.keys[h] = managedKey{identity: identity, info: info}
a.runtime[id] = h
return info, secret, nil
}
func keyHint(secret string) string {
if len(secret) <= 12 {
return secret
}
return secret[:8] + "…" + secret[len(secret)-4:]
}
func (a *Authenticator) APIKeys() []APIKeyInfo {
if a == nil {
return nil
}
a.mu.RLock()
out := make([]APIKeyInfo, 0, len(a.keys))
for _, k := range a.keys {
i := k.info
i.Scopes = append([]string(nil), i.Scopes...)
i.AllowedModels = append([]string(nil), i.AllowedModels...)
i.DeniedModels = append([]string(nil), i.DeniedModels...)
out = append(out, i)
}
a.mu.RUnlock()
sort.Slice(out, func(i, j int) bool {
if out[i].Source != out[j].Source {
return out[i].Source < out[j].Source
}
if out[i].Tenant != out[j].Tenant {
return out[i].Tenant < out[j].Tenant
}
return out[i].Name < out[j].Name
})
return out
}
func (a *Authenticator) DeleteAPIKey(id string) (APIKeyInfo, bool, error) {
if a == nil || id == "" {
return APIKeyInfo{}, false, nil
}
a.mu.Lock()
defer a.mu.Unlock()
h, ok := a.runtime[id]
if !ok {
return APIKeyInfo{}, false, nil
}
k, ok := a.keys[h]
if !ok {
delete(a.runtime, id)
return APIKeyInfo{}, false, nil
}
if a.store != nil {
if err := a.store.Delete(id); err != nil {
return APIKeyInfo{}, true, fmt.Errorf("delete persistent API key: %w", err)
}
}
delete(a.keys, h)
delete(a.runtime, id)
return k.info, true, nil
}
func (a *Authenticator) HasPersistentRuntimeStore() bool { return a != nil && a.store != nil }
func (a *Authenticator) RuntimeStoreHealth(ctx context.Context) error {
if a == nil || a.store == nil {
return nil
}
return a.store.Health(ctx)
}
func (a *Authenticator) PeerIP(r *http.Request) string {
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host = r.RemoteAddr
}
return strings.TrimSpace(host)
}
func (a *Authenticator) ClientIP(r *http.Request) string {
host := a.PeerIP(r)
peer := net.ParseIP(host)
if peer == nil || !containsAny(a.trusted, peer) {
return host
}
parts := strings.Split(r.Header.Get("X-Forwarded-For"), ",")
chain := make([]net.IP, 0, len(parts)+1)
for _, p := range parts {
if ip := net.ParseIP(strings.TrimSpace(p)); ip != nil {
chain = append(chain, ip)
}
}
chain = append(chain, peer)
for i := len(chain) - 1; i >= 0; i-- {
if !containsAny(a.trusted, chain[i]) {
return chain[i].String()
}
}
if len(chain) > 0 {
return chain[0].String()
}
return strings.TrimSpace(host)
}
func containsAny(nets []*net.IPNet, ip net.IP) bool {
if ip == nil {
return false
}
for _, n := range nets {
if n.Contains(ip) {
return true
}
}
return false
}
func (a *Authenticator) OIDCEnabled() bool { return a != nil && a.oidc != nil }
func (a *Authenticator) OIDCBrowserEndpoints() (BrowserEndpoints, bool) {
if a == nil || a.oidc == nil {
return BrowserEndpoints{}, false
}
return a.oidc.BrowserEndpoints(), true
}
func (a *Authenticator) ExchangeOIDCCode(ctx context.Context, code, redirectURI, clientID, clientSecret, verifier string) (TokenExchange, error) {
if a == nil || a.oidc == nil {
return TokenExchange{}, errors.New("OIDC is disabled")
}
return a.oidc.ExchangeCode(ctx, code, redirectURI, clientID, clientSecret, verifier)
}
func (a *Authenticator) VerifyOIDCToken(ctx context.Context, token string) (Identity, error) {
if a == nil || a.oidc == nil {
return Identity{}, errors.New("OIDC is disabled")
}
return a.oidc.Verify(ctx, token)
}
internal/auth/identity_test.go
+103
View File
@@ -0,0 +1,103 @@
package auth
import (
"context"
"net/http/httptest"
"strings"
"testing"
"github.com/example/ollama-fair-gateway/internal/config"
)
func TestRuntimeAPIKeyCreateAuthenticateDelete(t *testing.T) {
a, err := New(context.Background(), config.AuthConfig{APIKeys: []config.APIKeyConfig{{Name: "static", Key: "static-secret", Tenant: "ops", Scopes: []string{"gateway:admin"}}}})
if err != nil {
t.Fatal(err)
}
info, secret, err := a.CreateAPIKey(APIKeyCreate{Name: "openwebui", Tenant: "interactive", Application: "openwebui", Scopes: []string{"models:read", "models:read"}})
if err != nil {
t.Fatal(err)
}
if info.ID == "" || secret == "" || !strings.HasPrefix(secret, "ofg_") || !info.Deletable || info.Source != "runtime" {
t.Fatalf("unexpected created key: %#v secret=%q", info, secret)
}
if len(info.Scopes) != 1 || info.Scopes[0] != "models:read" {
t.Fatalf("unexpected scopes: %#v", info.Scopes)
}
r := httptest.NewRequest("GET", "http://gateway/api/tags", nil)
r.Header.Set("Authorization", "Bearer "+secret)
id, err := a.Authenticate(r)
if err != nil {
t.Fatal(err)
}
if id.Tenant != "interactive" || id.Application != "openwebui" || id.Actor() != "app:openwebui" {
t.Fatalf("unexpected identity: %#v", id)
}
keys := a.APIKeys()
if len(keys) != 2 {
t.Fatalf("keys=%d want 2: %#v", len(keys), keys)
}
for _, k := range keys {
if strings.Contains(k.KeyHint, secret) {
t.Fatalf("key listing leaked secret: %#v", k)
}
}
deleted, ok, err := a.DeleteAPIKey(info.ID)
if err != nil {
t.Fatal(err)
}
if !ok || deleted.Name != "openwebui" {
t.Fatalf("delete failed: ok=%v info=%#v", ok, deleted)
}
if _, err := a.Authenticate(r); err == nil {
t.Fatal("deleted runtime key still authenticates")
}
if _, ok, err := a.DeleteAPIKey(info.ID); err != nil || ok {
t.Fatal("second delete unexpectedly succeeded")
}
}
func TestRuntimeAPIKeyRejectsDuplicateTenantName(t *testing.T) {
a, err := New(context.Background(), config.AuthConfig{})
if err != nil {
t.Fatal(err)
}
if _, _, err := a.CreateAPIKey(APIKeyCreate{Name: "client", Tenant: "team"}); err != nil {
t.Fatal(err)
}
if _, _, err := a.CreateAPIKey(APIKeyCreate{Name: "client", Tenant: "team"}); err == nil {
t.Fatal("expected duplicate error")
}
if _, _, err := a.CreateAPIKey(APIKeyCreate{Name: "client", Tenant: "other"}); err != nil {
t.Fatalf("same name in other tenant should be allowed: %v", err)
}
}
func TestRuntimeAPIKeyCarriesModelACL(t *testing.T) {
a, err := New(context.Background(), config.AuthConfig{})
if err != nil {
t.Fatal(err)
}
info, secret, err := a.CreateAPIKey(APIKeyCreate{Name: "limited", Tenant: "team", AllowedModels: []string{"coding", "qwen3:*"}, DeniedModels: []string{"qwen3:70b*"}})
if err != nil {
t.Fatal(err)
}
if len(info.AllowedModels) != 2 || len(info.DeniedModels) != 1 {
t.Fatalf("info=%#v", info)
}
r := httptest.NewRequest("GET", "http://gateway/api/tags", nil)
r.Header.Set("Authorization", "Bearer "+secret)
id, err := a.Authenticate(r)
if err != nil {
t.Fatal(err)
}
if !id.ModelACLSet {
t.Fatal("expected model ACL on identity")
}
if !config.ModelAccessAllowed(id.ModelAccess, "coding") || config.ModelAccessAllowed(id.ModelAccess, "qwen3:70b-q4") {
t.Fatalf("unexpected ACL: %#v", id.ModelAccess)
}
}
internal/auth/ip_bypass_test.go
+71
View File
@@ -0,0 +1,71 @@
package auth
import (
"context"
"net/http/httptest"
"testing"
"github.com/example/ollama-fair-gateway/internal/config"
)
func bypassTestConfig(useForwarded bool) config.AuthConfig {
return config.AuthConfig{
IPBypassUseForwardedIP: useForwarded,
TrustedProxies: []string{"10.0.0.0/8"},
IPBypass: []config.IPBypassConfig{{
CIDRs: []string{"127.0.0.1/32"},
Tenant: "local",
Subject: "localhost",
Scopes: []string{"gateway:admin"},
}},
}
}
func TestIPBypassDefaultsToDirectPeerNotForwardedHeader(t *testing.T) {
a, err := New(context.Background(), bypassTestConfig(false))
if err != nil {
t.Fatal(err)
}
r := httptest.NewRequest("GET", "http://gateway/admin", nil)
r.RemoteAddr = "10.2.3.4:12345"
r.Header.Set("X-Forwarded-For", "127.0.0.1")
if got := a.ClientIP(r); got != "127.0.0.1" {
t.Fatalf("resolved client ip changed: got %q", got)
}
if _, err := a.Authenticate(r); err == nil {
t.Fatal("spoofed forwarded loopback unexpectedly satisfied ip_bypass")
}
}
func TestIPBypassForwardedCompatibilityMustBeExplicit(t *testing.T) {
a, err := New(context.Background(), bypassTestConfig(true))
if err != nil {
t.Fatal(err)
}
r := httptest.NewRequest("GET", "http://gateway/admin", nil)
r.RemoteAddr = "10.2.3.4:12345"
r.Header.Set("X-Forwarded-For", "127.0.0.1")
id, err := a.Authenticate(r)
if err != nil {
t.Fatal(err)
}
if !id.IsAdmin() || id.AuthType != "ip-bypass" || id.ClientIP != "127.0.0.1" {
t.Fatalf("unexpected identity: %#v", id)
}
}
func TestIPBypassStillAcceptsDirectPeer(t *testing.T) {
a, err := New(context.Background(), bypassTestConfig(false))
if err != nil {
t.Fatal(err)
}
r := httptest.NewRequest("GET", "http://gateway/admin", nil)
r.RemoteAddr = "127.0.0.1:12345"
id, err := a.Authenticate(r)
if err != nil {
t.Fatal(err)
}
if !id.IsAdmin() || id.ClientIP != "127.0.0.1" {
t.Fatalf("unexpected identity: %#v", id)
}
}
internal/auth/oidc.go
+499
View File
@@ -0,0 +1,499 @@
package auth
import (
"context"
"crypto"
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rsa"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"math/big"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
)
type discoveryDoc struct {
Issuer string `json:"issuer"`
JWKSURI string `json:"jwks_uri"`
AuthorizationEndpoint string `json:"authorization_endpoint"`
TokenEndpoint string `json:"token_endpoint"`
}
type jwksDoc struct {
Keys []json.RawMessage `json:"keys"`
}
type jwkHeader struct {
Kty string `json:"kty"`
Kid string `json:"kid"`
Alg string `json:"alg"`
Use string `json:"use"`
N string `json:"n"`
E string `json:"e"`
Crv string `json:"crv"`
X string `json:"x"`
Y string `json:"y"`
}
type jwtHeader struct {
Alg string `json:"alg"`
Kid string `json:"kid"`
Typ string `json:"typ"`
}
type OIDCVerifier struct {
cfg config.OIDCConfig
issuer string
jwksURI string
authorizationEndpoint string
tokenEndpoint string
client *http.Client
mu sync.RWMutex
keys map[string]crypto.PublicKey
lastRefresh time.Time
allowed map[string]bool
adminGroups map[string]bool
}
func NewOIDCVerifier(ctx context.Context, cfg config.OIDCConfig) (*OIDCVerifier, error) {
issuer := strings.TrimSuffix(cfg.Issuer, "/")
v := &OIDCVerifier{cfg: cfg, issuer: issuer, client: &http.Client{Timeout: 10 * time.Second}, keys: map[string]crypto.PublicKey{}, allowed: map[string]bool{}, adminGroups: map[string]bool{}}
for _, a := range cfg.AllowedAlgorithms {
v.allowed[a] = true
}
for _, g := range cfg.AdminGroups {
v.adminGroups[g] = true
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, issuer+"/.well-known/openid-configuration", nil)
if err != nil {
return nil, err
}
resp, err := v.client.Do(req)
if err != nil {
return nil, fmt.Errorf("oidc discovery: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return nil, fmt.Errorf("oidc discovery: HTTP %d", resp.StatusCode)
}
var d discoveryDoc
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&d); err != nil {
return nil, err
}
if strings.TrimSuffix(d.Issuer, "/") != issuer {
return nil, fmt.Errorf("oidc discovery issuer mismatch: got %q", d.Issuer)
}
if d.JWKSURI == "" {
return nil, errors.New("oidc discovery returned no jwks_uri")
}
v.jwksURI = d.JWKSURI
v.authorizationEndpoint = d.AuthorizationEndpoint
v.tokenEndpoint = d.TokenEndpoint
if err := v.refresh(ctx, true); err != nil {
return nil, err
}
return v, nil
}
func (v *OIDCVerifier) Verify(ctx context.Context, token string) (Identity, error) {
parts := strings.Split(token, ".")
if len(parts) != 3 {
return Identity{}, errors.New("token is not a JWT")
}
hb, err := base64.RawURLEncoding.DecodeString(parts[0])
if err != nil {
return Identity{}, err
}
var h jwtHeader
if err := json.Unmarshal(hb, &h); err != nil {
return Identity{}, err
}
if h.Alg == "none" || !v.allowed[h.Alg] {
return Identity{}, fmt.Errorf("JWT algorithm %q is not allowed", h.Alg)
}
key := v.key(h.Kid)
if key == nil {
if err := v.refresh(ctx, false); err != nil {
return Identity{}, err
}
key = v.key(h.Kid)
}
if key == nil {
return Identity{}, fmt.Errorf("no signing key for kid %q", h.Kid)
}
sig, err := base64.RawURLEncoding.DecodeString(parts[2])
if err != nil {
return Identity{}, err
}
signingInput := []byte(parts[0] + "." + parts[1])
if err := verifySignature(h.Alg, key, signingInput, sig); err != nil {
// Some providers rotate key material while reusing a kid. Refresh once
// (subject to the anti-thundering-herd interval) before rejecting.
_ = v.refresh(ctx, false)
key = v.key(h.Kid)
if key == nil {
return Identity{}, err
}
if err2 := verifySignature(h.Alg, key, signingInput, sig); err2 != nil {
return Identity{}, err2
}
}
pb, err := base64.RawURLEncoding.DecodeString(parts[1])
if err != nil {
return Identity{}, err
}
dec := json.NewDecoder(strings.NewReader(string(pb)))
dec.UseNumber()
claims := map[string]any{}
if err := dec.Decode(&claims); err != nil {
return Identity{}, err
}
now := time.Now()
skew := v.cfg.ClockSkew.Value()
if iss, _ := claims["iss"].(string); strings.TrimSuffix(iss, "/") != v.issuer {
return Identity{}, errors.New("issuer mismatch")
}
if !audContains(claims["aud"], v.cfg.Audience) {
return Identity{}, errors.New("audience mismatch")
}
if exp, ok := numericTime(claims["exp"]); !ok || now.After(exp.Add(skew)) {
return Identity{}, errors.New("token expired or missing exp")
}
if nbf, ok := numericTime(claims["nbf"]); ok && now.Add(skew).Before(nbf) {
return Identity{}, errors.New("token not valid yet")
}
sub, _ := claims["sub"].(string)
if sub == "" {
return Identity{}, errors.New("token has no sub")
}
tenant := claimString(claims, v.cfg.TenantClaim)
if tenant == "" {
tenant = sub
}
app := claimString(claims, v.cfg.ApplicationClaim)
scopes := map[string]bool{}
for _, s := range claimStrings(claims["scope"]) {
for _, p := range strings.Fields(s) {
scopes[p] = true
}
}
for _, s := range claimStrings(claims["scp"]) {
for _, p := range strings.Fields(s) {
scopes[p] = true
}
}
groups := claimStrings(claimValue(claims, v.cfg.GroupsClaim))
for _, g := range groups {
if v.adminGroups[g] {
scopes["gateway:admin"] = true
}
}
return Identity{Tenant: tenant, Subject: sub, Application: app, AuthType: "oidc", Scopes: scopes}, nil
}
func (v *OIDCVerifier) key(kid string) crypto.PublicKey {
v.mu.RLock()
defer v.mu.RUnlock()
if kid != "" {
return v.keys[kid]
}
if len(v.keys) == 1 {
for _, k := range v.keys {
return k
}
}
return nil
}
func (v *OIDCVerifier) refresh(ctx context.Context, force bool) error {
v.mu.Lock()
defer v.mu.Unlock()
if !force && time.Since(v.lastRefresh) < v.cfg.JWKSRefreshMinInterval.Value() {
return nil
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, v.jwksURI, nil)
if err != nil {
return err
}
resp, err := v.client.Do(req)
if err != nil {
return fmt.Errorf("fetch jwks: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("fetch jwks: HTTP %d", resp.StatusCode)
}
var d jwksDoc
if err := json.NewDecoder(io.LimitReader(resp.Body, 2<<20)).Decode(&d); err != nil {
return err
}
keys := map[string]crypto.PublicKey{}
for _, raw := range d.Keys {
var j jwkHeader
if err := json.Unmarshal(raw, &j); err != nil {
continue
}
k, err := parseJWK(j)
if err != nil {
continue
}
keys[j.Kid] = k
}
if len(keys) == 0 {
return errors.New("jwks contained no supported signing keys")
}
v.keys = keys
v.lastRefresh = time.Now()
return nil
}
func parseJWK(j jwkHeader) (crypto.PublicKey, error) {
dec := base64.RawURLEncoding.DecodeString
switch j.Kty {
case "RSA":
nB, err := dec(j.N)
if err != nil {
return nil, err
}
eB, err := dec(j.E)
if err != nil {
return nil, err
}
e := 0
for _, b := range eB {
e = e<<8 + int(b)
}
if e == 0 {
return nil, errors.New("bad RSA exponent")
}
return &rsa.PublicKey{N: new(big.Int).SetBytes(nB), E: e}, nil
case "EC":
xb, err := dec(j.X)
if err != nil {
return nil, err
}
yb, err := dec(j.Y)
if err != nil {
return nil, err
}
var c elliptic.Curve
switch j.Crv {
case "P-256":
c = elliptic.P256()
case "P-384":
c = elliptic.P384()
case "P-521":
c = elliptic.P521()
default:
return nil, errors.New("unsupported EC curve")
}
x, y := new(big.Int).SetBytes(xb), new(big.Int).SetBytes(yb)
if !c.IsOnCurve(x, y) {
return nil, errors.New("EC key is not on curve")
}
return &ecdsa.PublicKey{Curve: c, X: x, Y: y}, nil
case "OKP":
if j.Crv != "Ed25519" {
return nil, errors.New("unsupported OKP curve")
}
b, err := dec(j.X)
if err != nil {
return nil, err
}
if len(b) != ed25519.PublicKeySize {
return nil, errors.New("bad Ed25519 key")
}
return ed25519.PublicKey(b), nil
default:
return nil, errors.New("unsupported key type")
}
}
func verifySignature(alg string, key crypto.PublicKey, msg, sig []byte) error {
var hash crypto.Hash
var digest []byte
switch alg {
case "RS256", "PS256", "ES256":
h := sha256.Sum256(msg)
hash = crypto.SHA256
digest = h[:]
case "RS384", "PS384", "ES384":
h := sha512.Sum384(msg)
hash = crypto.SHA384
digest = h[:]
case "RS512", "PS512", "ES512":
h := sha512.Sum512(msg)
hash = crypto.SHA512
digest = h[:]
case "EdDSA":
pk, ok := key.(ed25519.PublicKey)
if !ok || !ed25519.Verify(pk, msg, sig) {
return errors.New("invalid JWT signature")
}
return nil
default:
return errors.New("unsupported JWT algorithm")
}
if strings.HasPrefix(alg, "RS") {
pk, ok := key.(*rsa.PublicKey)
if !ok {
return errors.New("JWT key type mismatch")
}
if err := rsa.VerifyPKCS1v15(pk, hash, digest, sig); err != nil {
return errors.New("invalid JWT signature")
}
return nil
}
if strings.HasPrefix(alg, "PS") {
pk, ok := key.(*rsa.PublicKey)
if !ok {
return errors.New("JWT key type mismatch")
}
if err := rsa.VerifyPSS(pk, hash, digest, sig, &rsa.PSSOptions{SaltLength: rsa.PSSSaltLengthEqualsHash, Hash: hash}); err != nil {
return errors.New("invalid JWT signature")
}
return nil
}
pk, ok := key.(*ecdsa.PublicKey)
if !ok {
return errors.New("JWT key type mismatch")
}
n := (pk.Curve.Params().BitSize + 7) / 8
if len(sig) != 2*n {
return errors.New("bad ECDSA JWT signature length")
}
r, s := new(big.Int).SetBytes(sig[:n]), new(big.Int).SetBytes(sig[n:])
if !ecdsa.Verify(pk, digest, r, s) {
return errors.New("invalid JWT signature")
}
return nil
}
func audContains(v any, want string) bool {
for _, s := range claimStrings(v) {
if s == want {
return true
}
}
return false
}
func claimStrings(v any) []string {
switch x := v.(type) {
case string:
return []string{x}
case []any:
out := []string{}
for _, z := range x {
if s, ok := z.(string); ok {
out = append(out, s)
}
}
return out
case []string:
return x
}
return nil
}
func numericTime(v any) (time.Time, bool) {
var n int64
switch x := v.(type) {
case json.Number:
i, e := x.Int64()
if e != nil {
return time.Time{}, false
}
n = i
case float64:
n = int64(x)
case int64:
n = x
default:
return time.Time{}, false
}
return time.Unix(n, 0), true
}
func claimValue(m map[string]any, path string) any {
if path == "" {
return nil
}
var cur any = m
for _, p := range strings.Split(path, ".") {
mm, ok := cur.(map[string]any)
if !ok {
return nil
}
cur = mm[p]
}
return cur
}
func claimString(m map[string]any, path string) string {
if s, ok := claimValue(m, path).(string); ok {
return s
}
return ""
}
type BrowserEndpoints struct {
Authorization string `json:"authorization_endpoint"`
Token string `json:"token_endpoint"`
}
type TokenExchange struct {
AccessToken string `json:"access_token"`
TokenType string `json:"token_type"`
ExpiresIn int64 `json:"expires_in"`
IDToken string `json:"id_token,omitempty"`
Scope string `json:"scope,omitempty"`
}
func (v *OIDCVerifier) BrowserEndpoints() BrowserEndpoints {
return BrowserEndpoints{Authorization: v.authorizationEndpoint, Token: v.tokenEndpoint}
}
func (v *OIDCVerifier) ExchangeCode(ctx context.Context, code, redirectURI, clientID, clientSecret, verifier string) (TokenExchange, error) {
if v.tokenEndpoint == "" {
return TokenExchange{}, errors.New("OIDC discovery returned no token_endpoint")
}
form := url.Values{}
form.Set("grant_type", "authorization_code")
form.Set("code", code)
form.Set("redirect_uri", redirectURI)
form.Set("client_id", clientID)
if verifier != "" {
form.Set("code_verifier", verifier)
}
if clientSecret != "" {
form.Set("client_secret", clientSecret)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, v.tokenEndpoint, strings.NewReader(form.Encode()))
if err != nil {
return TokenExchange{}, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
resp, err := v.client.Do(req)
if err != nil {
return TokenExchange{}, fmt.Errorf("OIDC token exchange: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
if err != nil {
return TokenExchange{}, err
}
if resp.StatusCode/100 != 2 {
return TokenExchange{}, fmt.Errorf("OIDC token exchange: HTTP %d", resp.StatusCode)
}
var out TokenExchange
if err := json.Unmarshal(body, &out); err != nil {
return TokenExchange{}, err
}
if out.AccessToken == "" {
return TokenExchange{}, errors.New("OIDC token response contains no access_token")
}
return out, nil
}
internal/auth/oidc_test.go
+63
View File
@@ -0,0 +1,63 @@
package auth
import (
"context"
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"math/big"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
)
func TestOIDCVerifierRS256(t *testing.T) {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatal(err)
}
var issuer string
mux := http.NewServeMux()
srv := httptest.NewServer(mux)
defer srv.Close()
issuer = srv.URL
mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{"issuer": issuer, "jwks_uri": issuer + "/jwks"})
})
mux.HandleFunc("/jwks", func(w http.ResponseWriter, r *http.Request) {
e := big.NewInt(int64(key.PublicKey.E)).Bytes()
_ = json.NewEncoder(w).Encode(map[string]any{"keys": []any{map[string]any{"kty": "RSA", "kid": "k1", "alg": "RS256", "n": base64.RawURLEncoding.EncodeToString(key.PublicKey.N.Bytes()), "e": base64.RawURLEncoding.EncodeToString(e)}}})
})
v, err := NewOIDCVerifier(context.Background(), config.OIDCConfig{Enabled: true, Issuer: issuer, Audience: "gateway", TenantClaim: "tenant", ApplicationClaim: "azp", GroupsClaim: "groups", AdminGroups: []string{"admins"}, ClockSkew: config.Duration(time.Second), JWKSRefreshMinInterval: config.Duration(time.Second), AllowedAlgorithms: []string{"RS256"}})
if err != nil {
t.Fatal(err)
}
now := time.Now()
tok := signRS256(t, key, "k1", map[string]any{"iss": issuer, "aud": "gateway", "sub": "alice", "tenant": "team-a", "azp": "web", "groups": []string{"admins"}, "exp": now.Add(time.Minute).Unix(), "nbf": now.Add(-time.Second).Unix()})
id, err := v.Verify(context.Background(), tok)
if err != nil {
t.Fatal(err)
}
if id.Tenant != "team-a" || id.Subject != "alice" || id.Application != "web" || !id.IsAdmin() {
t.Fatalf("unexpected identity: %#v", id)
}
}
func signRS256(t *testing.T, key *rsa.PrivateKey, kid string, claims map[string]any) string {
t.Helper()
h, _ := json.Marshal(map[string]any{"alg": "RS256", "typ": "JWT", "kid": kid})
p, _ := json.Marshal(claims)
a := base64.RawURLEncoding.EncodeToString(h) + "." + base64.RawURLEncoding.EncodeToString(p)
sum := sha256.Sum256([]byte(a))
sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, sum[:])
if err != nil {
t.Fatal(err)
}
return fmt.Sprintf("%s.%s", a, base64.RawURLEncoding.EncodeToString(sig))
}
internal/autotune/autotune.go
+532
View File
@@ -0,0 +1,532 @@
package autotune
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"math"
"net/http"
"net/url"
"os"
"sort"
"strings"
"sync"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
"github.com/example/ollama-fair-gateway/internal/hoststats"
"github.com/example/ollama-fair-gateway/internal/state"
"github.com/example/ollama-fair-gateway/internal/worker"
)
type Sample struct {
TTFTMS float64 `json:"ttft_ms"`
ServiceMS float64 `json:"service_ms"`
PromptTokens int64 `json:"prompt_tokens"`
CompletionTokens int64 `json:"completion_tokens"`
PromptTPS float64 `json:"prompt_tps"`
OutputTPS float64 `json:"output_tps"`
Status int `json:"status"`
Error string `json:"error,omitempty"`
}
type Level struct {
Concurrency int `json:"concurrency"`
Requests int `json:"requests"`
Successful int `json:"successful"`
TTFTP50MS float64 `json:"ttft_p50_ms"`
TTFTP95MS float64 `json:"ttft_p95_ms"`
MeanServiceMS float64 `json:"mean_service_ms"`
MeanPromptTPS float64 `json:"mean_prompt_tps"`
MeanOutputTPS float64 `json:"mean_output_tps"`
AggregateOutputTPS float64 `json:"aggregate_output_tps"`
PeakVRAMBytes int64 `json:"peak_vram_bytes,omitempty"`
PeakGPUPercent float64 `json:"peak_gpu_percent,omitempty"`
Score float64 `json:"score"`
Samples []Sample `json:"samples,omitempty"`
}
type Profile struct {
ID string `json:"id"`
Worker string `json:"worker"`
Model string `json:"model"`
Status string `json:"status"` // queued|running|completed|failed|cancelled
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at,omitempty"`
Levels []Level `json:"levels,omitempty"`
RecommendedConcurrency int `json:"recommended_concurrency,omitempty"`
Applied bool `json:"applied,omitempty"`
AppliedAt time.Time `json:"applied_at,omitempty"`
Error string `json:"error,omitempty"`
}
type persistentState struct {
Version int `json:"version"`
SavedAt time.Time `json:"saved_at"`
Profiles []Profile `json:"profiles"`
Applied map[string]map[string]int `json:"applied"`
}
type StartRequest struct {
Worker string `json:"worker"`
Model string `json:"model"`
MaxConcurrency int `json:"max_concurrency,omitempty"`
SamplesPerLevel int `json:"samples_per_level,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
Prompt string `json:"prompt,omitempty"`
}
type Manager struct {
mu sync.RWMutex
cfg config.AutoTuningConfig
workers *worker.Pool
path string
client *http.Client
profiles map[string]Profile
order []string
applied map[string]map[string]int
cancel map[string]context.CancelFunc
}
func New(cfg config.AutoTuningConfig, workers *worker.Pool, path string) (*Manager, error) {
m := &Manager{cfg: cfg, workers: workers, path: path, client: &http.Client{Transport: &http.Transport{Proxy: http.ProxyFromEnvironment, MaxIdleConns: 64, MaxIdleConnsPerHost: 32, IdleConnTimeout: 30 * time.Second}}, profiles: map[string]Profile{}, applied: map[string]map[string]int{}, cancel: map[string]context.CancelFunc{}}
var ps persistentState
if path != "" {
err := (state.AtomicJSON{Path: path, Mode: 0640}).Load(&ps)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
}
for _, p := range ps.Profiles {
if p.ID == "" {
continue
}
if p.Status == "running" || p.Status == "queued" {
p.Status = "failed"
p.Error = "gateway restarted while benchmark was active"
p.FinishedAt = time.Now().UTC()
}
m.profiles[p.ID] = p
m.order = append(m.order, p.ID)
}
if ps.Applied != nil {
m.applied = ps.Applied
}
return m, nil
}
func (m *Manager) saveLocked() error {
if m.path == "" {
return nil
}
profiles := make([]Profile, 0, len(m.order))
for _, id := range m.order {
if p, ok := m.profiles[id]; ok {
profiles = append(profiles, p)
}
}
ps := persistentState{Version: 1, SavedAt: time.Now().UTC(), Profiles: profiles, Applied: m.applied}
return (state.AtomicJSON{Path: m.path, Mode: 0640}).Save(ps)
}
func (m *Manager) Applied() map[string]map[string]int {
m.mu.RLock()
defer m.mu.RUnlock()
out := map[string]map[string]int{}
for w, mm := range m.applied {
out[w] = map[string]int{}
for model, n := range mm {
out[w][model] = n
}
}
return out
}
func (m *Manager) List() []Profile {
m.mu.RLock()
defer m.mu.RUnlock()
out := make([]Profile, 0, len(m.order))
for i := len(m.order) - 1; i >= 0; i-- {
if p, ok := m.profiles[m.order[i]]; ok {
out = append(out, cloneProfile(p))
}
}
return out
}
func (m *Manager) Get(id string) (Profile, bool) {
m.mu.RLock()
defer m.mu.RUnlock()
p, ok := m.profiles[id]
return cloneProfile(p), ok
}
func cloneProfile(p Profile) Profile {
p.Levels = append([]Level(nil), p.Levels...)
for i := range p.Levels {
p.Levels[i].Samples = append([]Sample(nil), p.Levels[i].Samples...)
}
return p
}
func (m *Manager) Start(parent context.Context, in StartRequest) (Profile, error) {
if !m.cfg.Enabled {
return Profile{}, errors.New("auto tuning is disabled")
}
in.Worker = strings.TrimSpace(in.Worker)
in.Model = strings.TrimSpace(in.Model)
if in.Worker == "" || in.Model == "" {
return Profile{}, errors.New("worker and model are required")
}
wc, ok := m.workers.WorkerConfig(in.Worker)
if !ok {
return Profile{}, fmt.Errorf("unknown worker %q", in.Worker)
}
if in.MaxConcurrency <= 0 {
in.MaxConcurrency = m.cfg.MaxConcurrency
}
in.MaxConcurrency = min(in.MaxConcurrency, wc.MaxConcurrent)
if in.MaxConcurrency < 1 {
in.MaxConcurrency = 1
}
if in.SamplesPerLevel <= 0 {
in.SamplesPerLevel = m.cfg.SamplesPerLevel
}
if in.SamplesPerLevel > 20 {
in.SamplesPerLevel = 20
}
if in.MaxTokens <= 0 {
in.MaxTokens = m.cfg.MaxTokens
}
if in.MaxTokens > 4096 {
in.MaxTokens = 4096
}
if strings.TrimSpace(in.Prompt) == "" {
in.Prompt = m.cfg.Prompt
}
id := fmt.Sprintf("tune-%d", time.Now().UnixNano())
p := Profile{ID: id, Worker: in.Worker, Model: in.Model, Status: "queued", StartedAt: time.Now().UTC()}
m.mu.Lock()
m.profiles[id] = p
m.order = append(m.order, id)
_ = m.saveLocked()
m.mu.Unlock()
ctx, cancel := context.WithTimeout(context.WithoutCancel(parent), m.cfg.Timeout.Value())
m.mu.Lock()
m.cancel[id] = cancel
m.mu.Unlock()
go m.run(ctx, id, in, wc)
return p, nil
}
func (m *Manager) Cancel(id string) bool {
m.mu.RLock()
c := m.cancel[id]
m.mu.RUnlock()
if c == nil {
return false
}
c()
return true
}
func (m *Manager) Apply(id string) (Profile, error) {
m.mu.Lock()
defer m.mu.Unlock()
p, ok := m.profiles[id]
if !ok {
return Profile{}, errors.New("benchmark profile not found")
}
if p.Status != "completed" || p.RecommendedConcurrency <= 0 {
return Profile{}, errors.New("benchmark has no applicable recommendation")
}
if m.applied[p.Worker] == nil {
m.applied[p.Worker] = map[string]int{}
}
m.applied[p.Worker][p.Model] = p.RecommendedConcurrency
p.Applied = true
p.AppliedAt = time.Now().UTC()
m.profiles[id] = p
if err := m.saveLocked(); err != nil {
return Profile{}, err
}
return cloneProfile(p), nil
}
func (m *Manager) Reset(workerName, model string) error {
m.mu.Lock()
defer m.mu.Unlock()
if mm := m.applied[workerName]; mm != nil {
delete(mm, model)
if len(mm) == 0 {
delete(m.applied, workerName)
}
}
return m.saveLocked()
}
func (m *Manager) run(ctx context.Context, id string, in StartRequest, wc config.WorkerConfig) {
m.update(id, func(p *Profile) { p.Status = "running" })
levels := make([]Level, 0, in.MaxConcurrency)
for c := 1; c <= in.MaxConcurrency; c++ {
if ctx.Err() != nil {
m.finishCancelled(id, ctx.Err())
return
}
lvl := m.runLevel(ctx, wc, in.Model, in.Prompt, in.MaxTokens, c, in.SamplesPerLevel)
levels = append(levels, lvl)
m.update(id, func(p *Profile) { p.Levels = append([]Level(nil), levels...) })
}
if len(levels) == 0 {
m.finishFailed(id, "no benchmark levels completed")
return
}
maxAgg, maxP95 := 0.0, 0.0
for _, l := range levels {
if l.AggregateOutputTPS > maxAgg {
maxAgg = l.AggregateOutputTPS
}
if l.TTFTP95MS > maxP95 {
maxP95 = l.TTFTP95MS
}
}
best, bestScore := 1, -math.MaxFloat64
for i := range levels {
throughput := 0.0
if maxAgg > 0 {
throughput = levels[i].AggregateOutputTPS / maxAgg
}
latency := 0.0
if maxP95 > 0 {
latency = levels[i].TTFTP95MS / maxP95
}
failure := 1.0 - float64(levels[i].Successful)/float64(max(1, levels[i].Requests))
levels[i].Score = m.cfg.ThroughputWeight*throughput - m.cfg.TTFTWeight*latency - failure
if levels[i].Successful == levels[i].Requests && levels[i].Score > bestScore {
bestScore = levels[i].Score
best = levels[i].Concurrency
}
}
m.mu.Lock()
p := m.profiles[id]
p.Status = "completed"
p.Levels = levels
p.RecommendedConcurrency = best
p.FinishedAt = time.Now().UTC()
m.profiles[id] = p
delete(m.cancel, id)
_ = m.saveLocked()
m.mu.Unlock()
}
func (m *Manager) update(id string, fn func(*Profile)) {
m.mu.Lock()
p := m.profiles[id]
fn(&p)
m.profiles[id] = p
_ = m.saveLocked()
m.mu.Unlock()
}
func (m *Manager) finishCancelled(id string, err error) {
m.mu.Lock()
p := m.profiles[id]
p.Status = "cancelled"
if err != nil {
p.Error = err.Error()
}
p.FinishedAt = time.Now().UTC()
m.profiles[id] = p
delete(m.cancel, id)
_ = m.saveLocked()
m.mu.Unlock()
}
func (m *Manager) finishFailed(id, msg string) {
m.mu.Lock()
p := m.profiles[id]
p.Status = "failed"
p.Error = msg
p.FinishedAt = time.Now().UTC()
m.profiles[id] = p
delete(m.cancel, id)
_ = m.saveLocked()
m.mu.Unlock()
}
func (m *Manager) runLevel(ctx context.Context, wc config.WorkerConfig, model, prompt string, maxTokens, concurrency, repeats int) Level {
lvl := Level{Concurrency: concurrency, Requests: concurrency * repeats}
var samples []Sample
var totalTokens int64
var totalWall time.Duration
var peakVRAM int64
var peakGPU float64
var peakMu sync.Mutex
for rep := 0; rep < repeats; rep++ {
start := time.Now()
ch := make(chan Sample, concurrency)
var wg sync.WaitGroup
for i := 0; i < concurrency; i++ {
wg.Add(1)
go func() { defer wg.Done(); ch <- m.one(ctx, wc.URL, model, prompt, maxTokens) }()
}
done := make(chan struct{})
go func() { wg.Wait(); close(ch); close(done) }()
var telemetryWG sync.WaitGroup
if wc.NVIDIASMI {
telemetryWG.Add(1)
go func() {
defer telemetryWG.Done()
t := time.NewTicker(200 * time.Millisecond)
defer t.Stop()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case <-t.C:
c, cancel := context.WithTimeout(ctx, time.Second)
n, e := hoststats.ReadNVIDIA(c, wc.NVIDIAGPU)
cancel()
if e == nil {
peakMu.Lock()
if n.MemoryUsedBytes > peakVRAM {
peakVRAM = n.MemoryUsedBytes
}
if n.UtilizationPercent > peakGPU {
peakGPU = n.UtilizationPercent
}
peakMu.Unlock()
}
}
}
}()
}
for s := range ch {
samples = append(samples, s)
if s.Status >= 200 && s.Status < 300 && s.Error == "" {
lvl.Successful++
totalTokens += s.CompletionTokens
}
}
telemetryWG.Wait()
totalWall += time.Since(start)
}
lvl.Samples = samples
lvl.PeakVRAMBytes = peakVRAM
lvl.PeakGPUPercent = peakGPU
var ttfts, services []float64
var pTPS, oTPS float64
for _, s := range samples {
if s.Error != "" || s.Status < 200 || s.Status >= 300 {
continue
}
ttfts = append(ttfts, s.TTFTMS)
services = append(services, s.ServiceMS)
pTPS += s.PromptTPS
oTPS += s.OutputTPS
}
lvl.TTFTP50MS = percentile(ttfts, .50)
lvl.TTFTP95MS = percentile(ttfts, .95)
lvl.MeanServiceMS = mean(services)
if lvl.Successful > 0 {
lvl.MeanPromptTPS = pTPS / float64(lvl.Successful)
lvl.MeanOutputTPS = oTPS / float64(lvl.Successful)
}
if totalWall > 0 {
lvl.AggregateOutputTPS = float64(totalTokens) / totalWall.Seconds()
}
return lvl
}
func (m *Manager) one(ctx context.Context, base, model, prompt string, maxTokens int) Sample {
u, err := url.Parse(strings.TrimRight(base, "/") + "/api/generate")
if err != nil {
return Sample{Error: err.Error()}
}
b, _ := json.Marshal(map[string]any{"model": model, "prompt": prompt, "stream": true, "keep_alive": "5m", "options": map[string]any{"num_predict": maxTokens, "temperature": 0}})
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewReader(b))
if err != nil {
return Sample{Error: err.Error()}
}
req.Header.Set("Content-Type", "application/json")
start := time.Now()
resp, err := m.client.Do(req)
if err != nil {
return Sample{Error: err.Error(), ServiceMS: float64(time.Since(start).Microseconds()) / 1000}
}
defer resp.Body.Close()
out := Sample{Status: resp.StatusCode}
scan := bufio.NewScanner(resp.Body)
buf := make([]byte, 64*1024)
scan.Buffer(buf, 4<<20)
first := true
for scan.Scan() {
if first {
out.TTFTMS = float64(time.Since(start).Microseconds()) / 1000
first = false
}
var v map[string]any
if json.Unmarshal(scan.Bytes(), &v) != nil {
continue
}
if n := asI64(v["prompt_eval_count"]); n > 0 {
out.PromptTokens = n
}
if n := asI64(v["eval_count"]); n > 0 {
out.CompletionTokens = n
}
pn := asI64(v["prompt_eval_duration"])
en := asI64(v["eval_duration"])
if pn > 0 && out.PromptTokens > 0 {
out.PromptTPS = float64(out.PromptTokens) / (float64(pn) / 1e9)
}
if en > 0 && out.CompletionTokens > 0 {
out.OutputTPS = float64(out.CompletionTokens) / (float64(en) / 1e9)
}
}
if err := scan.Err(); err != nil {
out.Error = err.Error()
}
out.ServiceMS = float64(time.Since(start).Microseconds()) / 1000
return out
}
func asI64(v any) int64 {
switch x := v.(type) {
case float64:
return int64(x)
case json.Number:
n, _ := x.Int64()
return n
case int64:
return x
case int:
return int64(x)
}
return 0
}
func mean(xs []float64) float64 {
if len(xs) == 0 {
return 0
}
s := 0.0
for _, x := range xs {
s += x
}
return s / float64(len(xs))
}
func percentile(xs []float64, p float64) float64 {
if len(xs) == 0 {
return 0
}
ys := append([]float64(nil), xs...)
sort.Float64s(ys)
i := int(math.Ceil(p*float64(len(ys)))) - 1
if i < 0 {
i = 0
}
if i >= len(ys) {
i = len(ys) - 1
}
return ys[i]
}
internal/autotune/autotune_test.go
+80
View File
@@ -0,0 +1,80 @@
package autotune
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"testing"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
"github.com/example/ollama-fair-gateway/internal/worker"
)
func waitProfile(t *testing.T, m *Manager, id string) Profile {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
p, ok := m.Get(id)
if ok && p.Status != "queued" && p.Status != "running" {
return p
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("profile %s did not finish", id)
return Profile{}
}
func TestBenchmarkRecommendationApplyAndReload(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/generate" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/x-ndjson")
f, _ := w.(http.Flusher)
fmt.Fprintln(w, `{"response":"x","done":false}`)
if f != nil {
f.Flush()
}
time.Sleep(20 * time.Millisecond)
fmt.Fprintln(w, `{"done":true,"prompt_eval_count":10,"prompt_eval_duration":10000000,"eval_count":20,"eval_duration":20000000}`)
}))
defer ts.Close()
wc := config.WorkerConfig{Name: "gpu", URL: ts.URL, MaxConcurrent: 2}
pool := worker.New([]config.WorkerConfig{wc}, "gpu")
path := filepath.Join(t.TempDir(), "auto-tune.json")
cfg := config.AutoTuningConfig{Enabled: true, MaxConcurrency: 2, SamplesPerLevel: 1, MaxTokens: 32, Timeout: config.Duration(5 * time.Second), Prompt: "test", TTFTWeight: .05, ThroughputWeight: 1}
m, err := New(cfg, pool, path)
if err != nil {
t.Fatal(err)
}
p, err := m.Start(context.Background(), StartRequest{Worker: "gpu", Model: "qwen:test"})
if err != nil {
t.Fatal(err)
}
p = waitProfile(t, m, p.ID)
if p.Status != "completed" {
t.Fatalf("status=%s err=%s", p.Status, p.Error)
}
if len(p.Levels) != 2 {
t.Fatalf("levels=%d", len(p.Levels))
}
if p.RecommendedConcurrency != 2 {
t.Fatalf("recommendation=%d want 2; levels=%+v", p.RecommendedConcurrency, p.Levels)
}
if _, err := m.Apply(p.ID); err != nil {
t.Fatal(err)
}
m2, err := New(cfg, pool, path)
if err != nil {
t.Fatal(err)
}
if got := m2.Applied()["gpu"]["qwen:test"]; got != 2 {
t.Fatalf("reloaded applied=%d want 2", got)
}
}
internal/batch/manager.go
+692
View File
@@ -0,0 +1,692 @@
package batch
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
"github.com/example/ollama-fair-gateway/internal/state"
)
const (
StateQueued = "queued"
StateRunning = "running"
StatePaused = "paused"
StatePausing = "pausing"
StateCancelling = "cancelling"
StateCompleted = "completed"
StateFailed = "failed"
StateCancelled = "cancelled"
)
var (
ErrDisabled = errors.New("durable batch jobs are disabled")
ErrNotFound = errors.New("batch job not found")
ErrInvalidState = errors.New("batch job state does not allow this operation")
ErrFull = errors.New("batch job retention store is full")
ErrInputTooLarge = errors.New("batch input exceeds configured max_input_bytes")
)
// IdentitySnapshot preserves the authenticated metadata required to re-run a
// durable request through the normal gateway authorization/routing pipeline.
// It intentionally contains no bearer/API-key secret.
type IdentitySnapshot struct {
Tenant string `json:"tenant"`
Subject string `json:"subject"`
Actor string `json:"actor"`
Application string `json:"application,omitempty"`
AuthType string `json:"auth_type"`
ClientIP string `json:"client_ip,omitempty"`
Scopes []string `json:"scopes,omitempty"`
ModelACLSet bool `json:"model_acl_set,omitempty"`
ModelAccess config.ModelAccessRule `json:"model_access,omitempty"`
}
type Job struct {
ID string `json:"id"`
Identity IdentitySnapshot `json:"identity"`
Path string `json:"path"`
Model string `json:"model,omitempty"`
ServiceClass string `json:"service_class"`
InputRef string `json:"input_ref"`
OutputRef string `json:"output_ref,omitempty"`
State string `json:"state"`
Attempts int `json:"attempts"`
HTTPStatus int `json:"http_status,omitempty"`
ResponseContentType string `json:"response_content_type,omitempty"`
ExecutionRequestID string `json:"execution_request_id,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
StartedAt *time.Time `json:"started_at,omitempty"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
Error string `json:"error,omitempty"`
}
type RunResult struct {
HTTPStatus int
ResponseContentType string
RequestID string
Error string
}
type Runner func(context.Context, Job, io.Reader, io.Writer) RunResult
type snapshot struct {
Version int `json:"version"`
Jobs map[string]Job `json:"jobs"`
}
type Manager struct {
mu sync.Mutex
cfg config.BatchJobsConfig
file state.AtomicJSON
dir string
jobs map[string]Job
cancel map[string]context.CancelCauseFunc
wake chan struct{}
active int
runner Runner
ctx context.Context
now func() time.Time
wg sync.WaitGroup
}
var (
errPause = errors.New("batch pause requested")
errCancel = errors.New("batch cancel requested")
)
func New(cfg config.BatchJobsConfig, metadataPath, dir string) (*Manager, error) {
m := &Manager{cfg: cfg, file: state.AtomicJSON{Path: metadataPath, Mode: 0600}, dir: dir, jobs: map[string]Job{}, cancel: map[string]context.CancelCauseFunc{}, wake: make(chan struct{}, 1), now: time.Now}
if !cfg.Enabled {
return m, nil
}
if err := os.MkdirAll(filepath.Join(dir, "input"), 0750); err != nil {
return nil, err
}
if err := os.MkdirAll(filepath.Join(dir, "output"), 0750); err != nil {
return nil, err
}
var snap snapshot
if err := m.file.Load(&snap); err != nil {
if !errors.Is(err, os.ErrNotExist) {
return nil, err
}
} else {
if snap.Version != 1 {
return nil, fmt.Errorf("unsupported batch metadata version %d", snap.Version)
}
if snap.Jobs != nil {
m.jobs = snap.Jobs
}
}
changed := m.recoverLocked()
changed = m.pruneLocked(m.now().UTC()) || changed
if changed {
if err := m.saveLocked(); err != nil {
return nil, err
}
}
return m, nil
}
func (m *Manager) Enabled() bool { return m != nil && m.cfg.Enabled }
func (m *Manager) Start(ctx context.Context, runner Runner) {
if !m.Enabled() || runner == nil {
return
}
m.mu.Lock()
m.ctx = ctx
m.runner = runner
m.mu.Unlock()
go m.loop(ctx)
m.signal()
}
// Wait blocks until all currently running batch attempts have returned. It is
// intended for graceful shutdown after the root context has been cancelled, so
// no new attempts can be dispatched while waiting. Each finishing attempt
// persists its final restart-safe state before it releases the wait group.
func (m *Manager) Wait(ctx context.Context) error {
if !m.Enabled() {
return nil
}
done := make(chan struct{})
go func() {
m.wg.Wait()
close(done)
}()
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (m *Manager) Create(id IdentitySnapshot, path, model string, body []byte) (Job, error) {
if !m.Enabled() {
return Job{}, ErrDisabled
}
if id.Tenant == "" || id.Actor == "" {
return Job{}, errors.New("batch identity requires tenant and actor")
}
if path == "" || !strings.HasPrefix(path, "/") {
return Job{}, errors.New("batch path must be an absolute gateway path")
}
if int64(len(body)) > m.cfg.MaxInputBytes {
return Job{}, ErrInputTooLarge
}
if len(body) == 0 {
return Job{}, errors.New("batch input body is empty")
}
m.mu.Lock()
defer m.mu.Unlock()
now := m.now().UTC()
m.pruneLocked(now)
if len(m.jobs) >= m.cfg.MaxJobs {
return Job{}, ErrFull
}
jobID, err := newID()
if err != nil {
return Job{}, err
}
inputRef := filepath.ToSlash(filepath.Join("input", jobID+".json"))
if err := m.writeInputLocked(inputRef, body); err != nil {
return Job{}, err
}
j := Job{ID: jobID, Identity: id, Path: path, Model: model, ServiceClass: "batch", InputRef: inputRef, State: StateQueued, CreatedAt: now, UpdatedAt: now}
m.jobs[j.ID] = j
if err := m.saveLocked(); err != nil {
delete(m.jobs, j.ID)
_ = os.Remove(m.refPath(inputRef))
return Job{}, err
}
m.signal()
return cloneJob(j), nil
}
func (m *Manager) List(tenant, actor string, all bool) []Job {
if !m.Enabled() {
return nil
}
m.mu.Lock()
defer m.mu.Unlock()
if m.pruneLocked(m.now().UTC()) {
_ = m.saveLocked()
}
out := make([]Job, 0, len(m.jobs))
for _, j := range m.jobs {
if all || (j.Identity.Tenant == tenant && j.Identity.Actor == actor) {
out = append(out, cloneJob(j))
}
}
sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) })
return out
}
func (m *Manager) Get(id, tenant, actor string, all bool) (Job, bool) {
if !m.Enabled() {
return Job{}, false
}
m.mu.Lock()
defer m.mu.Unlock()
j, ok := m.jobs[id]
if !ok || (!all && (j.Identity.Tenant != tenant || j.Identity.Actor != actor)) {
return Job{}, false
}
return cloneJob(j), true
}
func (m *Manager) Pause(id, tenant, actor string, all bool) (Job, error) {
return m.control(id, tenant, actor, all, "pause")
}
func (m *Manager) Resume(id, tenant, actor string, all bool) (Job, error) {
return m.control(id, tenant, actor, all, "resume")
}
func (m *Manager) Cancel(id, tenant, actor string, all bool) (Job, error) {
return m.control(id, tenant, actor, all, "cancel")
}
func (m *Manager) control(id, tenant, actor string, all bool, action string) (Job, error) {
if !m.Enabled() {
return Job{}, ErrDisabled
}
m.mu.Lock()
j, ok := m.jobs[id]
if !ok || (!all && (j.Identity.Tenant != tenant || j.Identity.Actor != actor)) {
m.mu.Unlock()
return Job{}, ErrNotFound
}
now := m.now().UTC()
var cancel context.CancelCauseFunc
switch action {
case "pause":
switch j.State {
case StateQueued:
j.State = StatePaused
case StateRunning:
j.State = StatePausing
cancel = m.cancel[id]
case StatePaused, StatePausing:
// idempotent
default:
m.mu.Unlock()
return Job{}, ErrInvalidState
}
case "resume":
if j.State != StatePaused {
m.mu.Unlock()
return Job{}, ErrInvalidState
}
j.State = StateQueued
j.Error = ""
j.HTTPStatus = 0
j.ResponseContentType = ""
j.ExecutionRequestID = ""
j.OutputRef = ""
j.StartedAt = nil
j.FinishedAt = nil
case "cancel":
switch j.State {
case StateQueued, StatePaused:
j.State = StateCancelled
j.FinishedAt = ptrTime(now)
case StateRunning, StatePausing:
j.State = StateCancelling
cancel = m.cancel[id]
case StateCancelling, StateCancelled:
// idempotent
default:
m.mu.Unlock()
return Job{}, ErrInvalidState
}
default:
m.mu.Unlock()
return Job{}, errors.New("unknown batch control action")
}
j.UpdatedAt = now
m.jobs[id] = j
if err := m.saveLocked(); err != nil {
m.mu.Unlock()
return Job{}, err
}
m.mu.Unlock()
if cancel != nil {
if action == "pause" {
cancel(errPause)
} else {
cancel(errCancel)
}
}
if action == "resume" {
m.signal()
}
return cloneJob(j), nil
}
func (m *Manager) OpenOutput(id, tenant, actor string, all bool) (*os.File, Job, error) {
if !m.Enabled() {
return nil, Job{}, ErrDisabled
}
m.mu.Lock()
j, ok := m.jobs[id]
if !ok || (!all && (j.Identity.Tenant != tenant || j.Identity.Actor != actor)) {
m.mu.Unlock()
return nil, Job{}, ErrNotFound
}
ref := j.OutputRef
m.mu.Unlock()
if ref == "" {
return nil, cloneJob(j), os.ErrNotExist
}
f, err := os.Open(m.refPath(ref))
return f, cloneJob(j), err
}
func (m *Manager) Compact() error {
if !m.Enabled() {
return nil
}
m.mu.Lock()
defer m.mu.Unlock()
m.pruneLocked(m.now().UTC())
return m.saveLocked()
}
func (m *Manager) loop(ctx context.Context) {
t := time.NewTicker(time.Minute)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-m.wake:
m.dispatch(ctx)
case <-t.C:
m.mu.Lock()
changed := m.pruneLocked(m.now().UTC())
if changed {
_ = m.saveLocked()
}
m.mu.Unlock()
m.dispatch(ctx)
}
}
}
func (m *Manager) dispatch(root context.Context) {
for {
m.mu.Lock()
if m.runner == nil || m.active >= m.cfg.MaxConcurrent || root.Err() != nil {
m.mu.Unlock()
return
}
var chosen *Job
for _, j := range m.jobs {
if j.State != StateQueued {
continue
}
if chosen == nil || j.CreatedAt.Before(chosen.CreatedAt) {
jc := j
chosen = &jc
}
}
if chosen == nil {
m.mu.Unlock()
return
}
now := m.now().UTC()
j := *chosen
j.State = StateRunning
j.Attempts++
j.StartedAt = ptrTime(now)
j.FinishedAt = nil
j.UpdatedAt = now
j.Error = ""
m.jobs[j.ID] = j
ctx, cancel := context.WithCancelCause(root)
m.cancel[j.ID] = cancel
m.active++
if err := m.saveLocked(); err != nil {
m.active--
delete(m.cancel, j.ID)
j.State = StateQueued
m.jobs[j.ID] = j
m.mu.Unlock()
return
}
runner := m.runner
m.wg.Add(1)
m.mu.Unlock()
go func() {
defer m.wg.Done()
m.runOne(ctx, j, runner)
}()
}
}
func (m *Manager) runOne(ctx context.Context, j Job, runner Runner) {
input, err := os.Open(m.refPath(j.InputRef))
if err != nil {
m.finishRun(j.ID, RunResult{Error: "open input: " + err.Error()}, "", true)
return
}
defer input.Close()
outDir := filepath.Join(m.dir, "output")
tmp, err := os.CreateTemp(outDir, ".batch-output-*.tmp")
if err != nil {
m.finishRun(j.ID, RunResult{Error: "create output: " + err.Error()}, "", true)
return
}
tmpPath := tmp.Name()
if err := tmp.Chmod(0600); err != nil {
_ = tmp.Close()
_ = os.Remove(tmpPath)
m.finishRun(j.ID, RunResult{Error: "chmod output: " + err.Error()}, "", true)
return
}
res := runner(ctx, cloneJob(j), input, tmp)
if err := tmp.Sync(); err != nil && res.Error == "" {
res.Error = "sync output: " + err.Error()
}
if err := tmp.Close(); err != nil && res.Error == "" {
res.Error = "close output: " + err.Error()
}
m.finishRun(j.ID, res, tmpPath, false)
}
func (m *Manager) finishRun(id string, res RunResult, tmpPath string, setupFailure bool) {
m.mu.Lock()
defer m.mu.Unlock()
j, ok := m.jobs[id]
if !ok {
if tmpPath != "" {
_ = os.Remove(tmpPath)
}
return
}
delete(m.cancel, id)
if m.active > 0 {
m.active--
}
now := m.now().UTC()
j.UpdatedAt = now
j.HTTPStatus = res.HTTPStatus
j.ResponseContentType = res.ResponseContentType
j.ExecutionRequestID = res.RequestID
switch {
case j.State == StateCancelling:
if tmpPath != "" {
_ = os.Remove(tmpPath)
}
j.State = StateCancelled
j.FinishedAt = ptrTime(now)
j.Error = "cancelled"
case j.State == StatePausing:
if tmpPath != "" {
_ = os.Remove(tmpPath)
}
j.State = StatePaused
j.StartedAt = nil
j.Error = "paused"
case m.ctx != nil && m.ctx.Err() != nil:
if tmpPath != "" {
_ = os.Remove(tmpPath)
}
j.State = StateQueued
j.StartedAt = nil
j.Error = "interrupted by gateway shutdown; queued for retry"
case setupFailure:
j.State = StateFailed
j.FinishedAt = ptrTime(now)
j.Error = res.Error
default:
finalRef := filepath.ToSlash(filepath.Join("output", id+".response"))
finalPath := m.refPath(finalRef)
_ = os.Remove(finalPath)
if tmpPath != "" {
if err := os.Rename(tmpPath, finalPath); err != nil {
j.State = StateFailed
j.FinishedAt = ptrTime(now)
j.Error = "commit output: " + err.Error()
break
}
j.OutputRef = finalRef
}
if res.Error != "" || res.HTTPStatus < 200 || res.HTTPStatus >= 400 {
j.State = StateFailed
j.Error = res.Error
if j.Error == "" && res.HTTPStatus != 0 {
j.Error = fmt.Sprintf("gateway HTTP %d", res.HTTPStatus)
}
} else {
j.State = StateCompleted
j.Error = ""
}
j.FinishedAt = ptrTime(now)
}
m.jobs[id] = j
_ = m.saveLocked()
m.signal()
}
func (m *Manager) recoverLocked() bool {
changed := false
now := m.now().UTC()
for id, j := range m.jobs {
switch j.State {
case StateRunning:
j.State = StateQueued
j.StartedAt = nil
j.Error = "recovered after gateway restart; queued for retry"
j.UpdatedAt = now
m.jobs[id] = j
changed = true
case StatePausing:
j.State = StatePaused
j.StartedAt = nil
j.Error = "pause recovered after gateway restart"
j.UpdatedAt = now
m.jobs[id] = j
changed = true
case StateCancelling:
j.State = StateCancelled
j.FinishedAt = ptrTime(now)
j.Error = "cancel recovered after gateway restart"
j.UpdatedAt = now
m.jobs[id] = j
changed = true
}
}
return changed
}
func (m *Manager) pruneLocked(now time.Time) bool {
if m.cfg.Retention.Value() <= 0 {
return false
}
changed := false
for id, j := range m.jobs {
if !isTerminal(j.State) || j.FinishedAt == nil || now.Sub(*j.FinishedAt) < m.cfg.Retention.Value() {
continue
}
_ = os.Remove(m.refPath(j.InputRef))
if j.OutputRef != "" {
_ = os.Remove(m.refPath(j.OutputRef))
}
delete(m.jobs, id)
changed = true
}
return changed
}
func (m *Manager) saveLocked() error {
copyMap := make(map[string]Job, len(m.jobs))
for id, j := range m.jobs {
copyMap[id] = cloneJob(j)
}
return m.file.Save(snapshot{Version: 1, Jobs: copyMap})
}
func (m *Manager) writeInputLocked(ref string, body []byte) error {
path := m.refPath(ref)
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0750); err != nil {
return err
}
tmp, err := os.CreateTemp(dir, ".batch-input-*.tmp")
if err != nil {
return err
}
name := tmp.Name()
ok := false
defer func() {
_ = tmp.Close()
if !ok {
_ = os.Remove(name)
}
}()
if err := tmp.Chmod(0600); err != nil {
return err
}
if _, err := tmp.Write(body); err != nil {
return err
}
if err := tmp.Sync(); err != nil {
return err
}
if err := tmp.Close(); err != nil {
return err
}
if err := os.Rename(name, path); err != nil {
return err
}
ok = true
return nil
}
func (m *Manager) refPath(ref string) string {
ref = filepath.Clean(filepath.FromSlash(ref))
if ref == "." || filepath.IsAbs(ref) || ref == ".." || strings.HasPrefix(ref, ".."+string(filepath.Separator)) {
return filepath.Join(m.dir, "invalid-ref")
}
return filepath.Join(m.dir, ref)
}
func (m *Manager) signal() {
select {
case m.wake <- struct{}{}:
default:
}
}
func isTerminal(state string) bool {
switch state {
case StateCompleted, StateFailed, StateCancelled:
return true
default:
return false
}
}
func newID() (string, error) {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
return "", err
}
return "batch_" + hex.EncodeToString(b), nil
}
func cloneJob(j Job) Job {
j.Identity.Scopes = append([]string(nil), j.Identity.Scopes...)
j.Identity.ModelAccess.AllowedModels = append([]string(nil), j.Identity.ModelAccess.AllowedModels...)
j.Identity.ModelAccess.DeniedModels = append([]string(nil), j.Identity.ModelAccess.DeniedModels...)
if j.StartedAt != nil {
t := *j.StartedAt
j.StartedAt = &t
}
if j.FinishedAt != nil {
t := *j.FinishedAt
j.FinishedAt = &t
}
return j
}
func ptrTime(t time.Time) *time.Time { return &t }
internal/batch/manager_test.go
+216
View File
@@ -0,0 +1,216 @@
package batch
import (
"context"
"io"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
)
func batchTestConfig() config.BatchJobsConfig {
return config.BatchJobsConfig{Enabled: true, Retention: config.Duration(time.Hour), MaxJobs: 100, MaxConcurrent: 1, MaxInputBytes: 1 << 20}
}
func waitState(t *testing.T, m *Manager, id, state string) Job {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
j, ok := m.Get(id, "t", "u", false)
if ok && j.State == state {
return j
}
time.Sleep(5 * time.Millisecond)
}
j, _ := m.Get(id, "t", "u", false)
t.Fatalf("job %s did not reach %s; got %#v", id, state, j)
return Job{}
}
func TestCreateRunPersistAndOutput(t *testing.T) {
dir := t.TempDir()
meta := filepath.Join(dir, "batch-jobs.json")
spool := filepath.Join(dir, "batch")
m, err := New(batchTestConfig(), meta, spool)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
m.Start(ctx, func(ctx context.Context, j Job, in io.Reader, out io.Writer) RunResult {
b, _ := io.ReadAll(in)
if string(b) != `{"model":"m","input":"x"}` {
t.Errorf("input=%s", b)
}
io.WriteString(out, `{"id":"ok"}`)
return RunResult{HTTPStatus: 200, ResponseContentType: "application/json", RequestID: "req-1"}
})
j, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/v1/responses", "m", []byte(`{"model":"m","input":"x"}`))
if err != nil {
t.Fatal(err)
}
j = waitState(t, m, j.ID, StateCompleted)
if j.OutputRef == "" || j.ExecutionRequestID != "req-1" || j.HTTPStatus != 200 {
t.Fatalf("job=%#v", j)
}
f, _, err := m.OpenOutput(j.ID, "t", "u", false)
if err != nil {
t.Fatal(err)
}
b, _ := io.ReadAll(f)
f.Close()
if string(b) != `{"id":"ok"}` {
t.Fatalf("output=%s", b)
}
m2, err := New(batchTestConfig(), meta, spool)
if err != nil {
t.Fatal(err)
}
got, ok := m2.Get(j.ID, "t", "u", false)
if !ok || got.State != StateCompleted {
t.Fatalf("reloaded=%#v ok=%v", got, ok)
}
if _, ok := m2.Get(j.ID, "t", "other", false); ok {
t.Fatal("cross-actor job lookup must be hidden")
}
}
func TestPauseResumeAndCancel(t *testing.T) {
dir := t.TempDir()
m, err := New(batchTestConfig(), filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool"))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var attempts atomic.Int32
m.Start(ctx, func(ctx context.Context, j Job, in io.Reader, out io.Writer) RunResult {
n := attempts.Add(1)
if n == 1 {
<-ctx.Done()
return RunResult{HTTPStatus: 499, Error: context.Cause(ctx).Error()}
}
io.WriteString(out, "done")
return RunResult{HTTPStatus: 200, RequestID: "req-done"}
})
j, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/api/chat", "m", []byte(`{"model":"m"}`))
if err != nil {
t.Fatal(err)
}
waitState(t, m, j.ID, StateRunning)
if _, err := m.Pause(j.ID, "t", "u", false); err != nil {
t.Fatal(err)
}
waitState(t, m, j.ID, StatePaused)
if _, err := m.Resume(j.ID, "t", "u", false); err != nil {
t.Fatal(err)
}
waitState(t, m, j.ID, StateCompleted)
if attempts.Load() != 2 {
t.Fatalf("attempts=%d", attempts.Load())
}
j2, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/api/chat", "m", []byte(`{"model":"m"}`))
if err != nil {
t.Fatal(err)
}
// The runner completes quickly on later attempts, so cancel while queued by
// first pausing it synchronously.
if _, err := m.Pause(j2.ID, "t", "u", false); err != nil && err != ErrInvalidState {
t.Fatal(err)
}
cur, _ := m.Get(j2.ID, "t", "u", false)
if cur.State == StatePaused {
if _, err := m.Cancel(j2.ID, "t", "u", false); err != nil {
t.Fatal(err)
}
waitState(t, m, j2.ID, StateCancelled)
}
}
func TestRetentionDeletesContentFiles(t *testing.T) {
dir := t.TempDir()
cfg := batchTestConfig()
cfg.Retention = config.Duration(time.Minute)
m, err := New(cfg, filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool"))
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, 9, 8, 8, 0, 0, 0, time.UTC)
m.now = func() time.Time { return now }
j, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/api/chat", "m", []byte(`{"model":"m"}`))
if err != nil {
t.Fatal(err)
}
if _, err := m.Cancel(j.ID, "t", "u", false); err != nil {
t.Fatal(err)
}
job, _ := m.Get(j.ID, "t", "u", false)
inputPath := m.refPath(job.InputRef)
if _, err := os.Stat(inputPath); err != nil {
t.Fatal(err)
}
now = now.Add(2 * time.Minute)
if err := m.Compact(); err != nil {
t.Fatal(err)
}
if _, ok := m.Get(j.ID, "t", "u", false); ok {
t.Fatal("expired job retained")
}
if _, err := os.Stat(inputPath); !os.IsNotExist(err) {
t.Fatalf("input still exists: %v", err)
}
}
func TestInputLimit(t *testing.T) {
dir := t.TempDir()
cfg := batchTestConfig()
cfg.MaxInputBytes = 4
m, _ := New(cfg, filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool"))
_, err := m.Create(IdentitySnapshot{Tenant: "t", Actor: "u"}, "/api/chat", "m", []byte(strings.Repeat("x", 5)))
if err != ErrInputTooLarge {
t.Fatalf("err=%v", err)
}
}
func TestWaitPersistsRestartSafeStateAfterShutdown(t *testing.T) {
dir := t.TempDir()
m, err := New(batchTestConfig(), filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool"))
if err != nil {
t.Fatal(err)
}
root, cancel := context.WithCancel(context.Background())
m.Start(root, func(ctx context.Context, j Job, in io.Reader, out io.Writer) RunResult {
<-ctx.Done()
return RunResult{HTTPStatus: 499, Error: context.Cause(ctx).Error()}
})
j, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/api/chat", "m", []byte(`{"model":"m"}`))
if err != nil {
t.Fatal(err)
}
waitState(t, m, j.ID, StateRunning)
cancel()
wctx, wcancel := context.WithTimeout(context.Background(), time.Second)
defer wcancel()
if err := m.Wait(wctx); err != nil {
t.Fatal(err)
}
got, ok := m.Get(j.ID, "t", "u", false)
if !ok || got.State != StateQueued || !strings.Contains(got.Error, "shutdown") {
t.Fatalf("restart-safe state not persisted: %#v ok=%v", got, ok)
}
m2, err := New(batchTestConfig(), filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool"))
if err != nil {
t.Fatal(err)
}
reloaded, ok := m2.Get(j.ID, "t", "u", false)
if !ok || reloaded.State != StateQueued {
t.Fatalf("reloaded state=%#v ok=%v", reloaded, ok)
}
}
internal/config/config.go
+1314
View File
File diff suppressed because it is too large Load Diff
internal/config/config_test.go
+181
View File
@@ -0,0 +1,181 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestLoadRejectsRemovedRedisConfig(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "config.json")
body := `{
"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},
"workers":[{"name":"w","url":"http://127.0.0.1:11434"}],
"redis":{"url":"redis://127.0.0.1:6379"}
}`
if err := os.WriteFile(path, []byte(body), 0600); err != nil {
t.Fatal(err)
}
_, err := Load(path)
if err == nil || !strings.Contains(err.Error(), "unknown field") {
t.Fatalf("expected removed redis field to be rejected, got %v", err)
}
}
func TestLoadInMemoryExampleShape(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "config.json")
body := `{
"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},
"scheduler":{"global_concurrency":4,"max_queue":100,"max_queue_per_actor":10},
"workers":[{"name":"w","url":"http://127.0.0.1:11434","max_concurrent":4}],
"infrastructure":{"node_name":"gw","refresh_interval":"250ms","max_requests":64}
}`
if err := os.WriteFile(path, []byte(body), 0600); err != nil {
t.Fatal(err)
}
cfg, err := Load(path)
if err != nil {
t.Fatal(err)
}
if cfg.Infrastructure.NodeName != "gw" || cfg.Scheduler.GlobalConcurrency != 4 {
t.Fatalf("unexpected config: %+v", cfg)
}
}
func TestExampleConfigsStrictParse(t *testing.T) {
t.Setenv("OIDC_ISSUER", "https://issuer.example.test")
t.Setenv("OIDC_AUDIENCE", "gateway")
t.Setenv("GATEWAY_AUTOMATION_KEY", "01234567890123456789012345678901")
t.Setenv("GATEWAY_UI_SESSION_SECRET", "01234567890123456789012345678901")
t.Setenv("OIDC_UI_CLIENT_ID", "gateway-ui")
t.Setenv("OIDC_UI_CLIENT_SECRET", "01234567890123456789012345678901")
t.Setenv("GATEWAY_UI_REDIRECT_URL", "https://gateway.example.test/admin/callback")
t.Setenv("OPENWEBUI_GATEWAY_KEY", "01234567890123456789012345678901")
for _, path := range []string{
"../../config.example.json",
"../../config.oidc.example.json",
"../../config.openwebui.example.json",
"../../config.rtx4090.example.json",
"../../config.placement.example.json",
} {
if _, err := Load(path); err != nil {
t.Fatalf("%s: %v", path, err)
}
}
}
func TestModelPlacementValidation(t *testing.T) {
good := ModelPlacementRule{Mode: "whitelist", AllowedModels: []string{"qwen3:8b", "gemma4:*"}, DeniedModels: []string{"gemma4:e4b"}}
if err := ValidateModelPlacementRule(good); err != nil {
t.Fatalf("good rule rejected: %v", err)
}
for _, bad := range []ModelPlacementRule{
{Mode: "sometimes"},
{Mode: "allow_all", AllowedModels: []string{"*gemma*"}},
{Mode: "allow_all", DeniedModels: []string{"gemma*4*"}},
} {
if err := ValidateModelPlacementRule(bad); err == nil {
t.Fatalf("bad rule accepted: %#v", bad)
}
}
}
func TestConversationsRequireEncryptionKeyWhenEnabled(t *testing.T) {
body := `{
"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},
"workers":[{"name":"w","url":"http://127.0.0.1:11434"}],
"conversations":{"enabled":true,"retention":"24h","max_entries":100,"max_content_bytes":4096}
}`
_, err := ParseBytes([]byte(body))
if err == nil || !strings.Contains(err.Error(), "encryption_key") {
t.Fatalf("expected encryption key validation error, got %v", err)
}
}
func TestConversationsEnabledConfigParses(t *testing.T) {
body := `{
"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},
"workers":[{"name":"w","url":"http://127.0.0.1:11434"}],
"conversations":{"enabled":true,"encryption_key":"01234567890123456789012345678901","retention":"2h","max_entries":100,"max_content_bytes":4096}
}`
cfg, err := ParseBytes([]byte(body))
if err != nil {
t.Fatal(err)
}
if !cfg.Conversations.Enabled || cfg.Storage.ConversationsFile != "conversations.enc.json" {
t.Fatalf("unexpected conversation config: %+v storage=%+v", cfg.Conversations, cfg.Storage)
}
}
func TestContextPolicySecureDefaults(t *testing.T) {
body := `{
"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},
"workers":[{"name":"w","url":"http://127.0.0.1:11434"}]
}`
cfg, err := ParseBytes([]byte(body))
if err != nil {
t.Fatal(err)
}
got := cfg.ModelCapabilities.Context
if got.MaxRequestedTokens != 32768 || got.DefaultWorkerTokens != 4096 || got.EstimationMarginPercent != 15 || got.VisionReserveTokensPerImage != 2048 {
t.Fatalf("unexpected context defaults: %+v", got)
}
}
func TestWorkerContextLimitsValidation(t *testing.T) {
body := `{
"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},
"workers":[{"name":"w","url":"http://127.0.0.1:11434","context_limits":{"qwen:*":0}}]
}`
_, err := ParseBytes([]byte(body))
if err == nil || !strings.Contains(err.Error(), "context_limits") {
t.Fatalf("expected context limit validation error, got %v", err)
}
}
func TestPublicDashboardDefaultsAndValidation(t *testing.T) {
body := `{
"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},
"workers":[{"name":"w","url":"http://127.0.0.1:11434"}],
"public_dashboard":{"enabled":true}
}`
cfg, err := ParseBytes([]byte(body))
if err != nil {
t.Fatal(err)
}
if cfg.PublicDashboard.Path != "/status" || cfg.PublicDashboard.RefreshInterval.Value() != 2*time.Second || cfg.PublicDashboard.MaxLiveRequests != 64 {
t.Fatalf("unexpected public dashboard defaults: %+v", cfg.PublicDashboard)
}
bad := `{
"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},
"workers":[{"name":"w","url":"http://127.0.0.1:11434"}],
"ui":{"path":"/admin"},
"public_dashboard":{"enabled":true,"path":"/admin","refresh_interval":"2s"}
}`
if _, err := ParseBytes([]byte(bad)); err == nil || !strings.Contains(err.Error(), "must not overlap") {
t.Fatalf("expected public/admin path conflict, got %v", err)
}
tooFast := `{
"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},
"workers":[{"name":"w","url":"http://127.0.0.1:11434"}],
"public_dashboard":{"enabled":true,"path":"/status","refresh_interval":"250ms"}
}`
if _, err := ParseBytes([]byte(tooFast)); err == nil || !strings.Contains(err.Error(), "at least 1s") {
t.Fatalf("expected refresh interval validation, got %v", err)
}
}
func TestPublicDashboardRejectsReservedNamespaceOverlap(t *testing.T) {
for _, publicPath := range []string{"/gateway/status", "/api/status", "/v1/status", "/admin/public"} {
body := `{"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},"workers":[{"name":"w","url":"http://127.0.0.1:11434"}],"ui":{"path":"/admin"},"public_dashboard":{"enabled":true,"path":"` + publicPath + `","refresh_interval":"2s"}}`
if _, err := ParseBytes([]byte(body)); err == nil {
t.Fatalf("expected public path %q to be rejected", publicPath)
}
}
}
internal/conversation/store.go
+322
View File
@@ -0,0 +1,322 @@
package conversation
import (
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"sort"
"sync"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
"github.com/example/ollama-fair-gateway/internal/state"
)
var ErrContextTooLarge = errors.New("conversation context exceeds configured max_content_bytes")
// Entry is the minimum content-bearing state required to implement
// previous_response_id semantics for the OpenAI Responses API. Context is a
// JSON array containing the flattened input/output items through this response.
type Entry struct {
ID string `json:"id"`
Tenant string `json:"tenant"`
Actor string `json:"actor"`
Model string `json:"model,omitempty"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
Context json.RawMessage `json:"context"`
}
type Status struct {
Enabled bool `json:"enabled"`
Entries int `json:"entries"`
Oldest time.Time `json:"oldest_at,omitempty"`
Newest time.Time `json:"newest_at,omitempty"`
ExpiresAt time.Time `json:"next_expiry_at,omitempty"`
}
type Store struct {
mu sync.Mutex
cfg config.ConversationsConfig
file state.AtomicJSON
key [32]byte
now func() time.Time
data map[string]Entry
}
type diskEnvelope struct {
Version int `json:"version"`
Algorithm string `json:"algorithm"`
Nonce string `json:"nonce"`
Ciphertext string `json:"ciphertext"`
UpdatedAt time.Time `json:"updated_at"`
}
type diskPlaintext struct {
Version int `json:"version"`
Entries map[string]Entry `json:"entries"`
}
func New(cfg config.ConversationsConfig, path string) (*Store, error) {
s := &Store{cfg: cfg, file: state.AtomicJSON{Path: path, Mode: 0600}, now: time.Now, data: map[string]Entry{}}
s.key = sha256.Sum256([]byte(cfg.EncryptionKey))
if !cfg.Enabled {
return s, nil
}
if err := s.load(); err != nil {
return nil, err
}
return s, nil
}
func (s *Store) Enabled() bool { return s != nil && s.cfg.Enabled }
// StartCleanup enforces retention even when no requests touch the store.
func (s *Store) StartCleanup(ctx context.Context, onError func(error)) {
if !s.Enabled() {
return
}
interval := s.cfg.Retention.Value() / 4
if interval < time.Minute {
interval = time.Minute
}
if interval > 15*time.Minute {
interval = 15 * time.Minute
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := s.pruneAndSave(); err != nil && onError != nil {
onError(err)
}
}
}
}()
}
func (s *Store) pruneAndSave() error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.pruneLocked(s.now().UTC()) {
return nil
}
return s.saveLocked()
}
func (s *Store) Get(id, tenant, actor string) (Entry, bool, error) {
if !s.Enabled() || id == "" {
return Entry{}, false, nil
}
s.mu.Lock()
defer s.mu.Unlock()
changed := s.pruneLocked(s.now().UTC())
e, ok := s.data[id]
if !ok || e.Tenant != tenant || e.Actor != actor {
if changed {
_ = s.saveLocked()
}
return Entry{}, false, nil
}
if changed {
if err := s.saveLocked(); err != nil {
return Entry{}, false, err
}
}
e.Context = append(json.RawMessage(nil), e.Context...)
return e, true, nil
}
func (s *Store) Put(e Entry) error {
if !s.Enabled() {
return nil
}
if e.ID == "" || e.Tenant == "" || e.Actor == "" {
return errors.New("conversation entry requires id, tenant, and actor")
}
if len(e.Context) == 0 || !json.Valid(e.Context) {
return errors.New("conversation entry context must contain valid JSON")
}
if s.cfg.MaxContentBytes > 0 && int64(len(e.Context)) > s.cfg.MaxContentBytes {
return ErrContextTooLarge
}
now := s.now().UTC()
if e.CreatedAt.IsZero() {
e.CreatedAt = now
} else {
e.CreatedAt = e.CreatedAt.UTC()
}
if e.ExpiresAt.IsZero() {
e.ExpiresAt = e.CreatedAt.Add(s.cfg.Retention.Value())
} else {
e.ExpiresAt = e.ExpiresAt.UTC()
}
e.Context = append(json.RawMessage(nil), e.Context...)
s.mu.Lock()
defer s.mu.Unlock()
s.pruneLocked(now)
s.data[e.ID] = e
s.enforceLimitLocked()
return s.saveLocked()
}
func (s *Store) Delete(id, tenant, actor string) error {
if !s.Enabled() || id == "" {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
e, ok := s.data[id]
if !ok || e.Tenant != tenant || e.Actor != actor {
return nil
}
delete(s.data, id)
return s.saveLocked()
}
func (s *Store) Compact() error {
if !s.Enabled() {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
s.pruneLocked(s.now().UTC())
s.enforceLimitLocked()
return s.saveLocked()
}
func (s *Store) Status() Status {
st := Status{Enabled: s.Enabled()}
if !s.Enabled() {
return st
}
s.mu.Lock()
defer s.mu.Unlock()
now := s.now().UTC()
_ = s.pruneLocked(now)
st.Entries = len(s.data)
for _, e := range s.data {
if st.Oldest.IsZero() || e.CreatedAt.Before(st.Oldest) {
st.Oldest = e.CreatedAt
}
if st.Newest.IsZero() || e.CreatedAt.After(st.Newest) {
st.Newest = e.CreatedAt
}
if st.ExpiresAt.IsZero() || e.ExpiresAt.Before(st.ExpiresAt) {
st.ExpiresAt = e.ExpiresAt
}
}
return st
}
func (s *Store) load() error {
var env diskEnvelope
if err := s.file.Load(&env); err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
}
if env.Version != 1 || env.Algorithm != "AES-256-GCM" {
return fmt.Errorf("unsupported conversation store envelope version/algorithm")
}
nonce, err := base64.StdEncoding.DecodeString(env.Nonce)
if err != nil {
return fmt.Errorf("decode conversation nonce: %w", err)
}
ct, err := base64.StdEncoding.DecodeString(env.Ciphertext)
if err != nil {
return fmt.Errorf("decode conversation ciphertext: %w", err)
}
block, err := aes.NewCipher(s.key[:])
if err != nil {
return err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return err
}
plain, err := gcm.Open(nil, nonce, ct, []byte("ollama-fair-gateway/conversations/v1"))
if err != nil {
return errors.New("decrypt conversation store: encryption key mismatch or file corruption")
}
var snap diskPlaintext
if err := json.Unmarshal(plain, &snap); err != nil {
return fmt.Errorf("decode conversation store: %w", err)
}
if snap.Version != 1 {
return fmt.Errorf("unsupported conversation plaintext version %d", snap.Version)
}
if snap.Entries != nil {
s.data = snap.Entries
}
s.pruneLocked(s.now().UTC())
s.enforceLimitLocked()
return nil
}
func (s *Store) saveLocked() error {
plain, err := json.Marshal(diskPlaintext{Version: 1, Entries: s.data})
if err != nil {
return err
}
block, err := aes.NewCipher(s.key[:])
if err != nil {
return err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return err
}
ct := gcm.Seal(nil, nonce, plain, []byte("ollama-fair-gateway/conversations/v1"))
env := diskEnvelope{Version: 1, Algorithm: "AES-256-GCM", Nonce: base64.StdEncoding.EncodeToString(nonce), Ciphertext: base64.StdEncoding.EncodeToString(ct), UpdatedAt: s.now().UTC()}
return s.file.Save(env)
}
func (s *Store) pruneLocked(now time.Time) bool {
changed := false
for id, e := range s.data {
if !e.ExpiresAt.IsZero() && !e.ExpiresAt.After(now) {
delete(s.data, id)
changed = true
}
}
return changed
}
func (s *Store) enforceLimitLocked() {
max := s.cfg.MaxEntries
if max <= 0 || len(s.data) <= max {
return
}
type pair struct {
id string
t time.Time
}
xs := make([]pair, 0, len(s.data))
for id, e := range s.data {
xs = append(xs, pair{id: id, t: e.CreatedAt})
}
sort.Slice(xs, func(i, j int) bool { return xs[i].t.Before(xs[j].t) })
for len(s.data) > max && len(xs) > 0 {
delete(s.data, xs[0].id)
xs = xs[1:]
}
}
internal/conversation/store_test.go
+103
View File
@@ -0,0 +1,103 @@
package conversation
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
)
func testConfig() config.ConversationsConfig {
return config.ConversationsConfig{Enabled: true, EncryptionKey: strings.Repeat("k", 32), Retention: config.Duration(time.Hour), MaxEntries: 2, MaxContentBytes: 4096}
}
func TestEncryptedRoundTripAndIdentityBoundary(t *testing.T) {
path := filepath.Join(t.TempDir(), "conversations.enc")
s, err := New(testConfig(), path)
if err != nil {
t.Fatal(err)
}
ctx := json.RawMessage(`[{"role":"user","content":"secret prompt"}]`)
if err := s.Put(Entry{ID: "resp_1", Tenant: "t1", Actor: "a1", Context: ctx}); err != nil {
t.Fatal(err)
}
b, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(b), "secret prompt") {
t.Fatalf("plaintext content leaked to disk: %s", b)
}
s2, err := New(testConfig(), path)
if err != nil {
t.Fatal(err)
}
got, ok, err := s2.Get("resp_1", "t1", "a1")
if err != nil || !ok {
t.Fatalf("get: ok=%v err=%v", ok, err)
}
if string(got.Context) != string(ctx) {
t.Fatalf("context = %s", got.Context)
}
if _, ok, _ := s2.Get("resp_1", "t1", "other"); ok {
t.Fatal("cross-actor lookup must be hidden")
}
if _, ok, _ := s2.Get("resp_1", "other", "a1"); ok {
t.Fatal("cross-tenant lookup must be hidden")
}
}
func TestWrongKeyFailsClosed(t *testing.T) {
path := filepath.Join(t.TempDir(), "conversations.enc")
s, _ := New(testConfig(), path)
if err := s.Put(Entry{ID: "resp_1", Tenant: "t", Actor: "a", Context: json.RawMessage(`[]`)}); err != nil {
t.Fatal(err)
}
cfg := testConfig()
cfg.EncryptionKey = strings.Repeat("z", 32)
if _, err := New(cfg, path); err == nil {
t.Fatal("expected wrong key to fail")
}
}
func TestRetentionAndEntryLimit(t *testing.T) {
path := filepath.Join(t.TempDir(), "conversations.enc")
cfg := testConfig()
cfg.Retention = config.Duration(time.Minute)
s, err := New(cfg, path)
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, 9, 8, 8, 0, 0, 0, time.UTC)
s.now = func() time.Time { return now }
for i, id := range []string{"r1", "r2", "r3"} {
if err := s.Put(Entry{ID: id, Tenant: "t", Actor: "a", CreatedAt: now.Add(time.Duration(i) * time.Second), Context: json.RawMessage(`[]`)}); err != nil {
t.Fatal(err)
}
}
if _, ok, _ := s.Get("r1", "t", "a"); ok {
t.Fatal("oldest entry should be evicted")
}
now = now.Add(2 * time.Minute)
if _, ok, _ := s.Get("r3", "t", "a"); ok {
t.Fatal("expired entry should be pruned")
}
}
func TestContentLimit(t *testing.T) {
path := filepath.Join(t.TempDir(), "conversations.enc")
cfg := testConfig()
cfg.MaxContentBytes = 2
s, _ := New(cfg, path)
if err := s.Put(Entry{ID: "r", Tenant: "t", Actor: "a", Context: json.RawMessage(`[]`)}); err != nil {
t.Fatalf("2-byte context should fit: %v", err)
}
if err := s.Put(Entry{ID: "r2", Tenant: "t", Actor: "a", Context: json.RawMessage(`[1]`)}); err != ErrContextTooLarge {
t.Fatalf("err=%v", err)
}
}
internal/cost/estimator.go
+214
View File
@@ -0,0 +1,214 @@
package cost
import (
"bytes"
"encoding/json"
"math"
"strings"
"github.com/example/ollama-fair-gateway/internal/config"
)
type Estimate struct {
Model string
InputTokens int64
OutputTokens int64
Credits float64
}
type Usage struct {
PromptTokens int64 `json:"prompt_tokens"`
CompletionTokens int64 `json:"completion_tokens"`
CachedPromptTokens int64 `json:"cached_prompt_tokens,omitempty"`
PromptEvalNS int64 `json:"prompt_eval_ns,omitempty"`
EvalNS int64 `json:"eval_ns,omitempty"`
LoadNS int64 `json:"load_ns,omitempty"`
TotalNS int64 `json:"total_ns,omitempty"`
Approximate bool `json:"approximate,omitempty"`
}
type Estimator struct{ cfg config.CostConfig }
func New(cfg config.CostConfig) *Estimator { return &Estimator{cfg: cfg} }
func (e *Estimator) Rate(model string) config.ModelRate {
if r, ok := e.cfg.Models[model]; ok {
return fillRate(r, e.cfg.Default)
}
// permit convenient prefix entries such as "qwen3:*"
best := ""
var out config.ModelRate
for p, r := range e.cfg.Models {
if strings.HasSuffix(p, "*") && strings.HasPrefix(model, strings.TrimSuffix(p, "*")) && len(p) > len(best) {
best = p
out = r
}
}
if best != "" {
return fillRate(out, e.cfg.Default)
}
return e.cfg.Default
}
func fillRate(r, d config.ModelRate) config.ModelRate {
if r.InputCreditsPer1K <= 0 {
r.InputCreditsPer1K = d.InputCreditsPer1K
}
if r.OutputCreditsPer1K <= 0 {
r.OutputCreditsPer1K = d.OutputCreditsPer1K
}
if r.CachedInputFactor <= 0 {
r.CachedInputFactor = d.CachedInputFactor
if r.CachedInputFactor <= 0 {
r.CachedInputFactor = 1
}
}
if r.ComputeCreditsPerSecond < 0 {
r.ComputeCreditsPerSecond = 0
}
return r
}
type wireReq struct {
Model string `json:"model"`
Prompt string `json:"prompt"`
Suffix string `json:"suffix"`
System json.RawMessage `json:"system"`
Instructions json.RawMessage `json:"instructions"`
Input json.RawMessage `json:"input"`
Tools json.RawMessage `json:"tools"`
Messages []struct {
Content json.RawMessage `json:"content"`
} `json:"messages"`
MaxTokens int `json:"max_tokens"`
MaxCompletionTokens int `json:"max_completion_tokens"`
MaxOutputTokens int `json:"max_output_tokens"`
Options struct {
NumPredict int `json:"num_predict"`
NumCtx int `json:"num_ctx"`
} `json:"options"`
}
func (e *Estimator) Estimate(path string, body []byte) Estimate {
var r wireReq
_ = json.Unmarshal(body, &r)
textBytes := len(r.Prompt) + len(r.Suffix) + textualBytes(r.System)
textBytes += textualBytes(r.Instructions)
textBytes += textualBytes(r.Input)
// Tool schemas are injected into the model context. Counting their raw JSON
// bytes gives a conservative pre-run estimate without attempting to tokenize
// provider-specific JSON schema dialects.
if raw := bytes.TrimSpace(r.Tools); len(raw) > 2 && !bytes.Equal(raw, []byte("[]")) && !bytes.Equal(raw, []byte("null")) {
textBytes += len(raw)
}
for _, m := range r.Messages {
textBytes += textualBytes(m.Content)
}
in := roughTokens(textBytes)
if in < 1 { // bounded fallback for unusual request shapes
n := len(body)
if n > 64<<10 {
n = 64 << 10
}
in = roughTokens(n)
}
// Use the largest explicitly requested output budget. Different compatible
// APIs use different field names, and choosing the largest is conservative
// when a client sends more than one of them.
out := maxInt(r.MaxCompletionTokens, r.MaxOutputTokens, r.MaxTokens, r.Options.NumPredict)
if out <= 0 {
out = e.cfg.DefaultMaxOutputTokens
}
if strings.Contains(path, "embeddings") || strings.Contains(path, "/embed") {
out = 0
}
rate := e.Rate(r.Model)
credits := creditsFor(rate, in, 0, int64(out), 0)
if rate.ComputeCreditsPerSecond > 0 {
var sec float64
if rate.ExpectedPromptTokensPerSecond > 0 {
sec += float64(in) / rate.ExpectedPromptTokensPerSecond
}
if out > 0 && rate.ExpectedOutputTokensPerSecond > 0 {
sec += float64(out) / rate.ExpectedOutputTokensPerSecond
}
credits += sec * rate.ComputeCreditsPerSecond
}
if credits <= 0 {
credits = .001
}
return Estimate{Model: r.Model, InputTokens: in, OutputTokens: int64(out), Credits: credits}
}
func (e *Estimator) Actual(model string, u Usage) float64 {
r := e.Rate(model)
sec := float64(u.PromptEvalNS+u.EvalNS) / 1e9
return creditsFor(r, u.PromptTokens, u.CachedPromptTokens, u.CompletionTokens, sec)
}
func creditsFor(r config.ModelRate, in, cached, out int64, sec float64) float64 {
if cached < 0 {
cached = 0
}
if cached > in {
cached = in
}
uncached := in - cached
input := (float64(uncached) + float64(cached)*r.CachedInputFactor) / 1000 * r.InputCreditsPer1K
return input + float64(out)/1000*r.OutputCreditsPer1K + sec*r.ComputeCreditsPerSecond
}
func maxInt(values ...int) int {
m := 0
for _, v := range values {
if v > m {
m = v
}
}
return m
}
func roughTokens(n int) int64 {
if n <= 0 {
return 0
}
return int64(math.Ceil(float64(n) / 4))
}
func textualBytes(raw json.RawMessage) int {
raw = bytes.TrimSpace(raw)
if len(raw) == 0 || bytes.Equal(raw, []byte("null")) {
return 0
}
var s string
if raw[0] == '"' && json.Unmarshal(raw, &s) == nil {
return len(s)
}
var v any
dec := json.NewDecoder(bytes.NewReader(raw))
if dec.Decode(&v) != nil {
return 0
}
return walkText(v)
}
func walkText(v any) int {
switch x := v.(type) {
case string:
return len(x)
case []any:
n := 0
for _, z := range x {
n += walkText(z)
}
return n
case map[string]any:
// Follow only known textual fields. URLs/base64 image payloads are
// deliberately excluded from the pre-run token estimate.
n := 0
if v, ok := x["text"]; ok {
n += walkText(v)
}
if v, ok := x["content"]; ok {
n += walkText(v)
}
return n
}
return 0
}
internal/cost/estimator_test.go
+52
View File
@@ -0,0 +1,52 @@
package cost
import (
"github.com/example/ollama-fair-gateway/internal/config"
"strings"
"testing"
)
func TestEstimateIgnoresImagePayload(t *testing.T) {
e := New(config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 100})
b := []byte(`{"model":"m","messages":[{"content":[{"type":"text","text":"hello world"},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAAAAAAAAAAAAAAAAAAAAAA"}}]}],"max_tokens":10}`)
x := e.Estimate("/v1/chat/completions", b)
if x.InputTokens > 20 {
t.Fatalf("image data inflated estimate: %d", x.InputTokens)
}
if x.OutputTokens != 10 {
t.Fatalf("output=%d", x.OutputTokens)
}
}
func TestActualDiscountsCachedPromptTokens(t *testing.T) {
e := New(config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 2, OutputCreditsPer1K: 4, CachedInputFactor: 0.25}})
got := e.Actual("m", Usage{PromptTokens: 1000, CachedPromptTokens: 800, CompletionTokens: 100})
// input: (200 + 800*0.25)/1000*2 = 0.8; output: 0.1*4 = 0.4
if got < 1.199999 || got > 1.200001 {
t.Fatalf("credits=%f, want 1.2", got)
}
}
func TestEstimateCountsResponsesInstructionsAndMaxOutputTokens(t *testing.T) {
e := New(config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 100})
b := []byte(`{"model":"m","instructions":"` + strings.Repeat("i", 400) + `","input":"hello","max_output_tokens":8192}`)
x := e.Estimate("/v1/responses", b)
if x.InputTokens < 100 {
t.Fatalf("instructions not counted, input=%d", x.InputTokens)
}
if x.OutputTokens != 8192 {
t.Fatalf("output=%d, want 8192", x.OutputTokens)
}
}
func TestEstimateCountsGenerateSuffixAndLargestOutputBudget(t *testing.T) {
e := New(config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 100})
b := []byte(`{"model":"m","prompt":"abc","suffix":"` + strings.Repeat("s", 400) + `","max_tokens":12,"max_completion_tokens":24,"max_output_tokens":48,"options":{"num_predict":36}}`)
x := e.Estimate("/api/generate", b)
if x.InputTokens < 100 {
t.Fatalf("suffix not counted, input=%d", x.InputTokens)
}
if x.OutputTokens != 48 {
t.Fatalf("output=%d, want 48", x.OutputTokens)
}
}
internal/haresource/collect.go
+141
View File
@@ -0,0 +1,141 @@
package haresource
import (
"bufio"
"fmt"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"time"
)
// Snapshot is a point-in-time view of gateway-process and host resources.
// Fields that cannot be collected on the current platform are omitted from JSON.
type Snapshot struct {
CapturedAt time.Time `json:"captured_at"`
GOOS string `json:"goos"`
PID int `json:"pid"`
ProcessRSSBytes int64 `json:"process_rss_bytes,omitempty"`
ProcessCPUPercent float64 `json:"process_cpu_percent,omitempty"`
ProcessThreads int64 `json:"process_threads,omitempty"`
ProcessOpenFDs int64 `json:"process_open_fds,omitempty"`
HostLogicalCPUs int `json:"host_logical_cpus"`
HostMemoryTotalBytes int64 `json:"host_memory_total_bytes,omitempty"`
HostMemoryAvailBytes int64 `json:"host_memory_available_bytes,omitempty"`
HostLoad1 float64 `json:"host_load1,omitempty"`
HostLoad5 float64 `json:"host_load5,omitempty"`
HostLoad15 float64 `json:"host_load15,omitempty"`
Warnings []string `json:"warnings,omitempty"`
}
// Collect captures one resource snapshot for pid without modifying the process.
func Collect(pid int) Snapshot {
s := Snapshot{CapturedAt: time.Now().UTC(), GOOS: runtime.GOOS, PID: pid, HostLogicalCPUs: runtime.NumCPU()}
if rss, cpu, err := psProcess(pid); err == nil {
s.ProcessRSSBytes = rss
s.ProcessCPUPercent = cpu
} else {
s.Warnings = append(s.Warnings, "ps: "+err.Error())
}
if runtime.GOOS == "linux" {
if b, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid)); err == nil {
vmRSS, threads := ParseProcStatus(string(b))
if vmRSS > 0 {
s.ProcessRSSBytes = vmRSS
}
s.ProcessThreads = threads
} else {
s.Warnings = append(s.Warnings, "proc status: "+err.Error())
}
if entries, err := os.ReadDir(fmt.Sprintf("/proc/%d/fd", pid)); err == nil {
s.ProcessOpenFDs = int64(len(entries))
} else {
s.Warnings = append(s.Warnings, "proc fd: "+err.Error())
}
if b, err := os.ReadFile("/proc/meminfo"); err == nil {
s.HostMemoryTotalBytes, s.HostMemoryAvailBytes = ParseMeminfo(string(b))
}
if b, err := os.ReadFile("/proc/loadavg"); err == nil {
s.HostLoad1, s.HostLoad5, s.HostLoad15 = ParseLoadavg(string(b))
}
}
return s
}
func psProcess(pid int) (rssBytes int64, cpuPercent float64, err error) {
cmd := exec.Command("ps", "-o", "rss=", "-o", "%cpu=", "-p", strconv.Itoa(pid))
b, err := cmd.Output()
if err != nil {
return 0, 0, err
}
fields := strings.Fields(string(b))
if len(fields) < 2 {
return 0, 0, fmt.Errorf("unexpected ps output %q", strings.TrimSpace(string(b)))
}
rssKB, err := strconv.ParseInt(fields[0], 10, 64)
if err != nil {
return 0, 0, err
}
cpu, err := strconv.ParseFloat(strings.ReplaceAll(fields[1], ",", "."), 64)
if err != nil {
return 0, 0, err
}
return rssKB * 1024, cpu, nil
}
// ParseProcStatus extracts VmRSS and Threads from Linux /proc/<pid>/status text.
func ParseProcStatus(s string) (rssBytes, threads int64) {
scan := bufio.NewScanner(strings.NewReader(s))
for scan.Scan() {
line := strings.TrimSpace(scan.Text())
if strings.HasPrefix(line, "VmRSS:") {
f := strings.Fields(line)
if len(f) >= 2 {
n, _ := strconv.ParseInt(f[1], 10, 64)
rssBytes = n * 1024
}
}
if strings.HasPrefix(line, "Threads:") {
f := strings.Fields(line)
if len(f) >= 2 {
threads, _ = strconv.ParseInt(f[1], 10, 64)
}
}
}
return rssBytes, threads
}
// ParseMeminfo extracts MemTotal and MemAvailable from Linux /proc/meminfo text.
func ParseMeminfo(s string) (total, available int64) {
for _, line := range strings.Split(s, "\n") {
f := strings.Fields(line)
if len(f) < 2 {
continue
}
n, _ := strconv.ParseInt(f[1], 10, 64)
switch strings.TrimSuffix(f[0], ":") {
case "MemTotal":
total = n * 1024
case "MemAvailable":
available = n * 1024
}
}
return total, available
}
// ParseLoadavg extracts the 1, 5 and 15 minute load averages.
func ParseLoadavg(s string) (one, five, fifteen float64) {
f := strings.Fields(s)
if len(f) > 0 {
one, _ = strconv.ParseFloat(f[0], 64)
}
if len(f) > 1 {
five, _ = strconv.ParseFloat(f[1], 64)
}
if len(f) > 2 {
fifteen, _ = strconv.ParseFloat(f[2], 64)
}
return
}
internal/haresource/collect_test.go
+21
View File
@@ -0,0 +1,21 @@
package haresource
import "testing"
func TestParseProcStatus(t *testing.T) {
rss, threads := ParseProcStatus("Name:\tx\nVmRSS:\t 1234 kB\nThreads:\t7\n")
if rss != 1234*1024 || threads != 7 {
t.Fatalf("rss=%d threads=%d", rss, threads)
}
}
func TestParseMeminfoAndLoadavg(t *testing.T) {
total, avail := ParseMeminfo("MemTotal: 1000 kB\nMemAvailable: 400 kB\n")
if total != 1000*1024 || avail != 400*1024 {
t.Fatalf("total=%d avail=%d", total, avail)
}
a, b, c := ParseLoadavg("1.25 0.50 0.10 1/100 1\n")
if a != 1.25 || b != .5 || c != .1 {
t.Fatalf("load=%v %v %v", a, b, c)
}
}
internal/haresource/sample.go
+240
View File
@@ -0,0 +1,240 @@
package haresource
import (
"context"
"errors"
"fmt"
"os"
"runtime"
"strconv"
"strings"
"time"
)
const SamplingSchemaVersion = 1
// SamplingReport is a bounded sustained-resource trace around one benchmark level.
type SamplingReport struct {
Version int `json:"version"`
PID int `json:"pid"`
GOOS string `json:"goos"`
StartedAt time.Time `json:"started_at"`
FinishedAt time.Time `json:"finished_at"`
IntervalMS int64 `json:"interval_ms"`
CPUMethod string `json:"cpu_method"`
Samples []Snapshot `json:"samples"`
PeakProcessRSSBytes int64 `json:"peak_process_rss_bytes,omitempty"`
PeakProcessCPUPercent float64 `json:"peak_process_cpu_percent,omitempty"`
PeakProcessThreads int64 `json:"peak_process_threads,omitempty"`
PeakProcessOpenFDs int64 `json:"peak_process_open_fds,omitempty"`
MinHostMemoryAvailableBytes int64 `json:"min_host_memory_available_bytes,omitempty"`
PeakHostLoad1 float64 `json:"peak_host_load1,omitempty"`
ProcessExited bool `json:"process_exited,omitempty"`
StopReason string `json:"stop_reason"`
Warnings []string `json:"warnings,omitempty"`
}
// Complete reports whether the trace is suitable as sustained resource evidence.
func (r SamplingReport) Complete() bool {
return r.Version == SamplingSchemaVersion && r.PID > 0 && len(r.Samples) >= 2 && !r.ProcessExited && r.PeakProcessRSSBytes > 0 && r.StopReason == "stop-file"
}
type cpuCounters struct {
process uint64
total uint64
}
// Sample collects until stopFile appears, maxDuration expires, or ctx is canceled.
// stopFile is polled so shell wrappers do not need to forward signals reliably.
func Sample(ctx context.Context, pid int, interval, maxDuration time.Duration, stopFile string) (SamplingReport, error) {
if pid <= 0 {
return SamplingReport{}, errors.New("pid must be positive")
}
if interval < 50*time.Millisecond {
return SamplingReport{}, errors.New("interval must be at least 50ms")
}
if maxDuration <= 0 {
return SamplingReport{}, errors.New("max duration must be positive")
}
if strings.TrimSpace(stopFile) == "" {
return SamplingReport{}, errors.New("stop file is required")
}
if _, err := os.Stat(stopFile); err == nil {
return SamplingReport{}, fmt.Errorf("stop file already exists: %s", stopFile)
} else if !errors.Is(err, os.ErrNotExist) {
return SamplingReport{}, fmt.Errorf("check stop file: %w", err)
}
r := SamplingReport{
Version: SamplingSchemaVersion,
PID: pid,
GOOS: runtime.GOOS,
StartedAt: time.Now().UTC(),
IntervalMS: interval.Milliseconds(),
CPUMethod: "ps_process_percent_sample",
}
var prev *cpuCounters
if runtime.GOOS == "linux" {
if c, err := readLinuxCPUCounters(pid); err == nil {
prev = &c
r.CPUMethod = "linux_procfs_interval_all_cpus_percent"
} else {
r.Warnings = append(r.Warnings, "initial procfs cpu counters: "+err.Error())
}
}
collectOne := func() bool {
s := Collect(pid)
if runtime.GOOS == "linux" && prev != nil {
if cur, err := readLinuxCPUCounters(pid); err == nil {
if cur.total > prev.total && cur.process >= prev.process {
dProc := float64(cur.process - prev.process)
dTotal := float64(cur.total - prev.total)
s.ProcessCPUPercent = (dProc / dTotal) * float64(max(1, s.HostLogicalCPUs)) * 100
}
*prev = cur
} else {
r.Warnings = append(r.Warnings, "procfs cpu counters: "+err.Error())
if errors.Is(err, os.ErrNotExist) {
r.ProcessExited = true
}
}
}
if processMissing(s) {
r.ProcessExited = true
}
r.Samples = append(r.Samples, s)
updatePeaks(&r, s)
return !r.ProcessExited
}
if !collectOne() {
r.StopReason = "process-exited"
r.FinishedAt = time.Now().UTC()
return r, nil
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
deadline := time.NewTimer(maxDuration)
defer deadline.Stop()
for {
select {
case <-ctx.Done():
r.StopReason = "context-canceled"
r.FinishedAt = time.Now().UTC()
return r, nil
case <-deadline.C:
_ = collectOne()
r.StopReason = "max-duration"
r.FinishedAt = time.Now().UTC()
return r, nil
case <-ticker.C:
if _, err := os.Stat(stopFile); err == nil {
_ = collectOne()
r.StopReason = "stop-file"
r.FinishedAt = time.Now().UTC()
return r, nil
} else if !errors.Is(err, os.ErrNotExist) {
r.Warnings = append(r.Warnings, "check stop file: "+err.Error())
}
if !collectOne() {
r.StopReason = "process-exited"
r.FinishedAt = time.Now().UTC()
return r, nil
}
}
}
}
func updatePeaks(r *SamplingReport, s Snapshot) {
if s.ProcessRSSBytes > r.PeakProcessRSSBytes {
r.PeakProcessRSSBytes = s.ProcessRSSBytes
}
if s.ProcessCPUPercent > r.PeakProcessCPUPercent {
r.PeakProcessCPUPercent = s.ProcessCPUPercent
}
if s.ProcessThreads > r.PeakProcessThreads {
r.PeakProcessThreads = s.ProcessThreads
}
if s.ProcessOpenFDs > r.PeakProcessOpenFDs {
r.PeakProcessOpenFDs = s.ProcessOpenFDs
}
if s.HostMemoryAvailBytes > 0 && (r.MinHostMemoryAvailableBytes == 0 || s.HostMemoryAvailBytes < r.MinHostMemoryAvailableBytes) {
r.MinHostMemoryAvailableBytes = s.HostMemoryAvailBytes
}
if s.HostLoad1 > r.PeakHostLoad1 {
r.PeakHostLoad1 = s.HostLoad1
}
}
func processMissing(s Snapshot) bool {
if runtime.GOOS == "linux" {
for _, w := range s.Warnings {
if strings.HasPrefix(w, "proc status:") && (strings.Contains(w, "no such file") || strings.Contains(w, "not found")) {
return true
}
}
}
return false
}
func readLinuxCPUCounters(pid int) (cpuCounters, error) {
proc, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
if err != nil {
return cpuCounters{}, err
}
procTicks, err := ParseProcStatTicks(string(proc))
if err != nil {
return cpuCounters{}, err
}
host, err := os.ReadFile("/proc/stat")
if err != nil {
return cpuCounters{}, err
}
totalTicks, err := ParseHostCPUTicks(string(host))
if err != nil {
return cpuCounters{}, err
}
return cpuCounters{process: procTicks, total: totalTicks}, nil
}
// ParseProcStatTicks returns utime+stime from Linux /proc/<pid>/stat.
func ParseProcStatTicks(s string) (uint64, error) {
end := strings.LastIndexByte(strings.TrimSpace(s), ')')
if end < 0 || end+2 >= len(s) {
return 0, errors.New("malformed proc stat")
}
fields := strings.Fields(s[end+1:])
// fields[0] is field 3 (state); utime/stime are fields 14/15.
if len(fields) <= 12 {
return 0, errors.New("proc stat has too few fields")
}
utime, err := strconv.ParseUint(fields[11], 10, 64)
if err != nil {
return 0, fmt.Errorf("parse utime: %w", err)
}
stime, err := strconv.ParseUint(fields[12], 10, 64)
if err != nil {
return 0, fmt.Errorf("parse stime: %w", err)
}
return utime + stime, nil
}
// ParseHostCPUTicks sums the aggregate Linux /proc/stat cpu line.
func ParseHostCPUTicks(s string) (uint64, error) {
line, _, _ := strings.Cut(s, "\n")
fields := strings.Fields(line)
if len(fields) < 2 || fields[0] != "cpu" {
return 0, errors.New("missing aggregate cpu line")
}
var total uint64
for _, field := range fields[1:] {
n, err := strconv.ParseUint(field, 10, 64)
if err != nil {
return 0, fmt.Errorf("parse host cpu tick: %w", err)
}
total += n
}
return total, nil
}
internal/haresource/sample_test.go
+69
View File
@@ -0,0 +1,69 @@
package haresource
import (
"context"
"os"
"path/filepath"
"runtime"
"testing"
"time"
)
func TestParseProcStatTicksHandlesSpacesInComm(t *testing.T) {
// fields 3..15 after the closing parenthesis; utime=11 and stime=13.
line := "123 (gateway worker (test)) S 1 2 3 4 5 6 7 8 9 10 11 13 0 0 0"
got, err := ParseProcStatTicks(line)
if err != nil {
t.Fatal(err)
}
if got != 24 {
t.Fatalf("ticks=%d want=24", got)
}
}
func TestParseHostCPUTicks(t *testing.T) {
got, err := ParseHostCPUTicks("cpu 1 2 3 4 5 6 7 8 9 10\ncpu0 1 2 3\n")
if err != nil {
t.Fatal(err)
}
if got != 55 {
t.Fatalf("ticks=%d want=55", got)
}
}
func TestSamplingReportComplete(t *testing.T) {
r := SamplingReport{Version: SamplingSchemaVersion, PID: 1, Samples: []Snapshot{{}, {}}, PeakProcessRSSBytes: 1, StopReason: "stop-file"}
if !r.Complete() {
t.Fatal("expected complete report")
}
r.ProcessExited = true
if r.Complete() {
t.Fatal("process-exited report must be incomplete")
}
r.ProcessExited = false
r.StopReason = "max-duration"
if r.Complete() {
t.Fatal("max-duration report must be incomplete")
}
}
func TestSampleCurrentProcessStopsOnFile(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("ps-based collector is not portable to Windows test hosts")
}
stop := filepath.Join(t.TempDir(), "stop")
go func() {
time.Sleep(140 * time.Millisecond)
_ = os.WriteFile(stop, []byte("stop\n"), 0o600)
}()
r, err := Sample(context.Background(), os.Getpid(), 50*time.Millisecond, 2*time.Second, stop)
if err != nil {
t.Fatal(err)
}
if r.StopReason != "stop-file" || len(r.Samples) < 2 || r.PeakProcessRSSBytes <= 0 {
t.Fatalf("unexpected report: %+v", r)
}
if runtime.GOOS == "linux" && r.CPUMethod != "linux_procfs_interval_all_cpus_percent" {
t.Fatalf("cpu method=%q", r.CPUMethod)
}
}
internal/hoststats/amd.go
+15
View File
@@ -0,0 +1,15 @@
package hoststats
import "context"
// AMD contains lightweight AMDGPU telemetry. Linux reads the standard amdgpu
// sysfs counters and therefore needs no ROCm library, daemon or CGO.
type AMD struct {
UtilizationPercent float64
MemoryUsedBytes int64
MemoryTotalBytes int64
TemperatureC float64
PowerWatts float64
}
func ReadAMD(ctx context.Context, device string) (AMD, error) { return readAMD(ctx, device) }
internal/hoststats/amd_linux.go
+87
View File
@@ -0,0 +1,87 @@
//go:build linux
package hoststats
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
)
func readAMD(ctx context.Context, device string) (AMD, error) {
if err := ctx.Err(); err != nil {
return AMD{}, err
}
if strings.TrimSpace(device) == "" {
var err error
device, err = findAMDDevice("/sys/class/drm")
if err != nil {
return AMD{}, err
}
}
return readAMDDevice(ctx, device)
}
func findAMDDevice(root string) (string, error) {
cards, err := filepath.Glob(filepath.Join(root, "card*", "device"))
if err != nil {
return "", err
}
for _, device := range cards {
b, err := os.ReadFile(filepath.Join(device, "vendor"))
if err != nil {
continue
}
if strings.EqualFold(strings.TrimSpace(string(b)), "0x1002") {
return device, nil
}
}
return "", fmt.Errorf("no AMDGPU sysfs device found")
}
func readAMDDevice(ctx context.Context, device string) (AMD, error) {
if err := ctx.Err(); err != nil {
return AMD{}, err
}
var out AMD
var found bool
if v, ok := readInt64(filepath.Join(device, "mem_info_vram_total")); ok {
out.MemoryTotalBytes = v
found = true
}
if v, ok := readInt64(filepath.Join(device, "mem_info_vram_used")); ok {
out.MemoryUsedBytes = v
found = true
}
if v, ok := readInt64(filepath.Join(device, "gpu_busy_percent")); ok {
out.UtilizationPercent = float64(v)
found = true
}
hwmons, _ := filepath.Glob(filepath.Join(device, "hwmon", "hwmon*"))
for _, hw := range hwmons {
if v, ok := readInt64(filepath.Join(hw, "temp1_input")); ok && out.TemperatureC == 0 {
out.TemperatureC = float64(v) / 1000
found = true
}
if v, ok := readInt64(filepath.Join(hw, "power1_average")); ok && out.PowerWatts == 0 {
out.PowerWatts = float64(v) / 1_000_000
found = true
}
}
if !found {
return AMD{}, fmt.Errorf("no readable AMDGPU telemetry in %s", device)
}
return out, nil
}
func readInt64(path string) (int64, bool) {
b, err := os.ReadFile(path)
if err != nil {
return 0, false
}
v, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
return v, err == nil
}
internal/hoststats/amd_other.go
+12
View File
@@ -0,0 +1,12 @@
//go:build !linux
package hoststats
import (
"context"
"fmt"
)
func readAMD(context.Context, string) (AMD, error) {
return AMD{}, fmt.Errorf("AMDGPU sysfs telemetry is supported only on Linux")
}
internal/hoststats/amd_test.go
+51
View File
@@ -0,0 +1,51 @@
//go:build linux
package hoststats
import (
"context"
"os"
"path/filepath"
"testing"
)
func writeStat(t *testing.T, path, value string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(value), 0644); err != nil {
t.Fatal(err)
}
}
func TestReadAMDDevice(t *testing.T) {
device := filepath.Join(t.TempDir(), "device")
writeStat(t, filepath.Join(device, "mem_info_vram_total"), "17179869184\n")
writeStat(t, filepath.Join(device, "mem_info_vram_used"), "4294967296\n")
writeStat(t, filepath.Join(device, "gpu_busy_percent"), "73\n")
writeStat(t, filepath.Join(device, "hwmon", "hwmon0", "temp1_input"), "62500\n")
writeStat(t, filepath.Join(device, "hwmon", "hwmon0", "power1_average"), "88000000\n")
got, err := readAMDDevice(context.Background(), device)
if err != nil {
t.Fatal(err)
}
if got.MemoryTotalBytes != 17179869184 || got.MemoryUsedBytes != 4294967296 || got.UtilizationPercent != 73 || got.TemperatureC != 62.5 || got.PowerWatts != 88 {
t.Fatalf("unexpected AMD telemetry: %+v", got)
}
}
func TestFindAMDDevice(t *testing.T) {
root := t.TempDir()
writeStat(t, filepath.Join(root, "card0", "device", "vendor"), "0x10de\n")
writeStat(t, filepath.Join(root, "card1", "device", "vendor"), "0x1002\n")
got, err := findAMDDevice(root)
if err != nil {
t.Fatal(err)
}
want := filepath.Join(root, "card1", "device")
if got != want {
t.Fatalf("got %q want %q", got, want)
}
}
internal/hoststats/memory.go
+10
View File
@@ -0,0 +1,10 @@
package hoststats
import "context"
type Memory struct {
TotalBytes int64
UsedBytes int64
}
func ReadMemory(ctx context.Context) (Memory, error) { return readMemory(ctx) }
internal/hoststats/memory_darwin.go
+66
View File
@@ -0,0 +1,66 @@
//go:build darwin
package hoststats
import (
"bufio"
"context"
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
)
var pageSizeRE = regexp.MustCompile(`page size of ([0-9]+) bytes`)
func readMemory(ctx context.Context) (Memory, error) {
totalOut, err := exec.CommandContext(ctx, "/usr/sbin/sysctl", "-n", "hw.memsize").Output()
if err != nil {
return Memory{}, err
}
total, err := strconv.ParseInt(strings.TrimSpace(string(totalOut)), 10, 64)
if err != nil || total <= 0 {
return Memory{}, fmt.Errorf("invalid hw.memsize")
}
out, err := exec.CommandContext(ctx, "/usr/bin/vm_stat").Output()
if err != nil {
return Memory{}, err
}
pageSize := int64(4096)
var freePages int64
s := bufio.NewScanner(strings.NewReader(string(out)))
first := true
for s.Scan() {
line := s.Text()
if first {
first = false
if m := pageSizeRE.FindStringSubmatch(line); len(m) == 2 {
if v, e := strconv.ParseInt(m[1], 10, 64); e == nil {
pageSize = v
}
}
continue
}
parts := strings.SplitN(line, ":", 2)
if len(parts) != 2 {
continue
}
name := strings.TrimSpace(parts[0])
v := strings.TrimSpace(strings.TrimSuffix(parts[1], "."))
n, _ := strconv.ParseInt(v, 10, 64)
switch name {
case "Pages free", "Pages inactive", "Pages speculative":
freePages += n
}
}
available := freePages * pageSize
used := total - available
if used < 0 {
used = 0
}
if used > total {
used = total
}
return Memory{TotalBytes: total, UsedBytes: used}, nil
}
internal/hoststats/memory_linux.go
+46
View File
@@ -0,0 +1,46 @@
//go:build linux
package hoststats
import (
"bufio"
"context"
"fmt"
"os"
"strconv"
"strings"
)
func readMemory(context.Context) (Memory, error) {
f, err := os.Open("/proc/meminfo")
if err != nil {
return Memory{}, err
}
defer f.Close()
var total, available int64
s := bufio.NewScanner(f)
for s.Scan() {
fields := strings.Fields(s.Text())
if len(fields) < 2 {
continue
}
v, _ := strconv.ParseInt(fields[1], 10, 64)
switch strings.TrimSuffix(fields[0], ":") {
case "MemTotal":
total = v * 1024
case "MemAvailable":
available = v * 1024
}
}
if err := s.Err(); err != nil {
return Memory{}, err
}
if total <= 0 {
return Memory{}, fmt.Errorf("MemTotal unavailable")
}
used := total - available
if used < 0 {
used = 0
}
return Memory{TotalBytes: total, UsedBytes: used}, nil
}
internal/hoststats/memory_other.go
+12
View File
@@ -0,0 +1,12 @@
//go:build !linux && !darwin && !windows
package hoststats
import (
"context"
"fmt"
)
func readMemory(context.Context) (Memory, error) {
return Memory{}, fmt.Errorf("host memory telemetry unsupported on this OS")
}
internal/hoststats/memory_windows.go
+41
View File
@@ -0,0 +1,41 @@
//go:build windows
package hoststats
import (
"context"
"fmt"
"syscall"
"unsafe"
)
type memoryStatusEx struct {
Length uint32
MemoryLoad uint32
TotalPhys uint64
AvailPhys uint64
TotalPageFile uint64
AvailPageFile uint64
TotalVirtual uint64
AvailVirtual uint64
AvailExtendedVirtual uint64
}
func readMemory(ctx context.Context) (Memory, error) {
if err := ctx.Err(); err != nil {
return Memory{}, err
}
var st memoryStatusEx
st.Length = uint32(unsafe.Sizeof(st))
kernel32 := syscall.NewLazyDLL("kernel32.dll")
proc := kernel32.NewProc("GlobalMemoryStatusEx")
r1, _, callErr := proc.Call(uintptr(unsafe.Pointer(&st)))
if r1 == 0 {
if callErr != syscall.Errno(0) {
return Memory{}, fmt.Errorf("GlobalMemoryStatusEx: %w", callErr)
}
return Memory{}, fmt.Errorf("GlobalMemoryStatusEx failed")
}
used := int64(st.TotalPhys - st.AvailPhys)
return Memory{TotalBytes: int64(st.TotalPhys), UsedBytes: used}, nil
}
internal/hoststats/nvidia.go
+69
View File
@@ -0,0 +1,69 @@
package hoststats
import (
"context"
"encoding/csv"
"fmt"
"os/exec"
"strconv"
"strings"
)
// NVIDIA contains lightweight local GPU telemetry obtained from nvidia-smi.
// Values are intentionally limited to stable fields supported by the selective
// --query-gpu interface; the gateway does not require NVML or CGO.
type NVIDIA struct {
UtilizationPercent float64
MemoryUsedBytes int64
MemoryTotalBytes int64
TemperatureC float64
PowerWatts float64
}
func ReadNVIDIA(ctx context.Context, gpu string) (NVIDIA, error) {
args := []string{"--query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw", "--format=csv,noheader,nounits"}
if strings.TrimSpace(gpu) != "" {
args = append(args, "-i", strings.TrimSpace(gpu))
}
out, err := exec.CommandContext(ctx, "nvidia-smi", args...).Output()
if err != nil {
if ctx.Err() != nil {
return NVIDIA{}, ctx.Err()
}
return NVIDIA{}, fmt.Errorf("nvidia-smi: %w", err)
}
r := csv.NewReader(strings.NewReader(strings.TrimSpace(string(out))))
rec, err := r.Read()
if err != nil || len(rec) < 5 {
return NVIDIA{}, fmt.Errorf("nvidia-smi returned an unexpected row")
}
parse := func(s string) (float64, error) {
s = strings.TrimSpace(s)
if s == "" || strings.EqualFold(s, "N/A") || strings.EqualFold(s, "[Not Supported]") {
return 0, nil
}
return strconv.ParseFloat(s, 64)
}
util, err := parse(rec[0])
if err != nil {
return NVIDIA{}, fmt.Errorf("parse gpu utilization: %w", err)
}
usedMiB, err := parse(rec[1])
if err != nil {
return NVIDIA{}, fmt.Errorf("parse gpu memory used: %w", err)
}
totalMiB, err := parse(rec[2])
if err != nil {
return NVIDIA{}, fmt.Errorf("parse gpu memory total: %w", err)
}
temp, err := parse(rec[3])
if err != nil {
return NVIDIA{}, fmt.Errorf("parse gpu temperature: %w", err)
}
power, err := parse(rec[4])
if err != nil {
return NVIDIA{}, fmt.Errorf("parse gpu power: %w", err)
}
const mib = int64(1024 * 1024)
return NVIDIA{UtilizationPercent: util, MemoryUsedBytes: int64(usedMiB * float64(mib)), MemoryTotalBytes: int64(totalMiB * float64(mib)), TemperatureC: temp, PowerWatts: power}, nil
}
internal/hoststats/nvidia_test.go
+12
View File
@@ -0,0 +1,12 @@
package hoststats
import "testing"
// Parsing is exercised indirectly through the command-backed function in
// integration environments with nvidia-smi. Keep this package dependency-free.
func TestNVIDIAStruct(t *testing.T) {
n := NVIDIA{UtilizationPercent: 50, MemoryUsedBytes: 1, MemoryTotalBytes: 2}
if n.MemoryTotalBytes != 2 {
t.Fatal("unexpected struct value")
}
}
internal/infrastructure/hub.go
+201
View File
@@ -0,0 +1,201 @@
package infrastructure
import (
"context"
"crypto/rand"
"encoding/hex"
"os"
"sort"
"sync"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
"github.com/example/ollama-fair-gateway/internal/liveflow"
"github.com/example/ollama-fair-gateway/internal/scheduler"
"github.com/example/ollama-fair-gateway/internal/worker"
)
type Gateway struct {
NodeID string `json:"node_id"`
NodeName string `json:"node_name"`
Hostname string `json:"hostname"`
StartedAt time.Time `json:"started_at"`
LastSeen time.Time `json:"last_seen"`
Queued int64 `json:"queued"`
Running int64 `json:"running"`
LiveActive int `json:"live_active"`
Workers int `json:"workers"`
Healthy bool `json:"healthy"`
}
type Request struct {
liveflow.Request
GatewayID string `json:"gateway_id"`
GatewayName string `json:"gateway_name"`
}
type Worker struct {
worker.Snapshot
Gateways []string `json:"gateways"`
}
type Counts struct {
Gateways int `json:"gateways"`
Workers int `json:"workers"`
Models int `json:"models"`
Active int `json:"active"`
Queued int `json:"queued"`
Routing int `json:"routing"`
Running int `json:"running"`
Streaming int `json:"streaming"`
}
type Snapshot struct {
GeneratedAt time.Time `json:"generated_at"`
Version uint64 `json:"version"`
Mode string `json:"mode"`
Queue int64 `json:"queue"`
Running int64 `json:"running"`
Counts Counts `json:"counts"`
Gateways []Gateway `json:"gateways"`
Workers []Worker `json:"workers"`
Requests []Request `json:"requests"`
}
type Hub struct {
cfg config.InfrastructureConfig
live *liveflow.Tracker
sched scheduler.Scheduler
workers *worker.Pool
startedAt time.Time
nodeID string
nodeName string
hostname string
mu sync.RWMutex
version uint64
notify chan struct{}
}
func New(cfg config.InfrastructureConfig, live *liveflow.Tracker, sched scheduler.Scheduler, workers *worker.Pool) *Hub {
host, _ := os.Hostname()
if host == "" {
host = "gateway"
}
id := cfg.NodeID
if id == "" {
var b [6]byte
_, _ = rand.Read(b[:])
id = host + "-" + hex.EncodeToString(b[:])
}
name := cfg.NodeName
if name == "" {
name = host
}
return &Hub{cfg: cfg, live: live, sched: sched, workers: workers, startedAt: time.Now().UTC(), nodeID: id, nodeName: name, hostname: host, notify: make(chan struct{})}
}
func (h *Hub) NodeID() string { return h.nodeID }
func (h *Hub) NodeName() string { return h.nodeName }
func (h *Hub) Start(ctx context.Context) {
interval := h.cfg.RefreshInterval.Value()
if interval <= 0 {
interval = 250 * time.Millisecond
}
go func() {
t := time.NewTicker(interval)
defer t.Stop()
changed := h.live.Changed()
for {
select {
case <-ctx.Done():
return
case <-changed:
changed = h.live.Changed()
h.bump()
case <-t.C:
h.bump()
}
}
}()
}
func (h *Hub) bump() {
h.mu.Lock()
h.version++
close(h.notify)
h.notify = make(chan struct{})
h.mu.Unlock()
}
func (h *Hub) Changed() <-chan struct{} {
h.mu.RLock()
ch := h.notify
h.mu.RUnlock()
return ch
}
func (h *Hub) Snapshot() Snapshot {
now := time.Now().UTC()
live := h.live.Snapshot()
if max := h.cfg.MaxRequests; max > 0 && len(live.Requests) > max {
active := make([]liveflow.Request, 0, max)
recent := make([]liveflow.Request, 0, max)
for _, r := range live.Requests {
if r.State == liveflow.StateCompleted || r.State == liveflow.StateFailed {
recent = append(recent, r)
} else {
active = append(active, r)
}
}
if len(active) >= max {
live.Requests = active[:max]
} else {
need := max - len(active)
if need > len(recent) {
need = len(recent)
}
live.Requests = append(active, recent[len(recent)-need:]...)
}
}
st := h.sched.Stats(context.Background())
ws := h.workers.Snapshots()
h.mu.RLock()
version := h.version
h.mu.RUnlock()
out := Snapshot{GeneratedAt: now, Version: version, Mode: "in-memory", Queue: st.Queued, Running: st.Running}
out.Gateways = []Gateway{{NodeID: h.nodeID, NodeName: h.nodeName, Hostname: h.hostname, StartedAt: h.startedAt, LastSeen: now, Queued: st.Queued, Running: st.Running, LiveActive: live.Counts.Active, Workers: len(ws), Healthy: true}}
modelSet := map[string]bool{}
for _, w := range ws {
out.Workers = append(out.Workers, Worker{Snapshot: w, Gateways: []string{h.nodeName}})
for _, m := range w.LoadedModels {
if m.Name != "" {
modelSet[m.Name] = true
}
}
}
for _, r := range live.Requests {
out.Requests = append(out.Requests, Request{Request: r, GatewayID: h.nodeID, GatewayName: h.nodeName})
switch r.State {
case liveflow.StateQueued:
out.Counts.Queued++
case liveflow.StateRouting:
out.Counts.Routing++
case liveflow.StateRunning:
out.Counts.Running++
case liveflow.StateStreaming:
out.Counts.Streaming++
}
if r.State != liveflow.StateCompleted && r.State != liveflow.StateFailed {
out.Counts.Active++
}
}
sort.Slice(out.Workers, func(i, j int) bool { return out.Workers[i].Name < out.Workers[j].Name })
sort.Slice(out.Requests, func(i, j int) bool { return out.Requests[i].QueuedAt.Before(out.Requests[j].QueuedAt) })
out.Counts.Gateways = 1
out.Counts.Workers = len(out.Workers)
out.Counts.Models = len(modelSet)
return out
}
internal/infrastructure/hub_test.go
+48
View File
@@ -0,0 +1,48 @@
package infrastructure
import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/example/ollama-fair-gateway/internal/config"
"github.com/example/ollama-fair-gateway/internal/liveflow"
"github.com/example/ollama-fair-gateway/internal/scheduler"
"github.com/example/ollama-fair-gateway/internal/worker"
)
func TestSnapshotIsLocalInMemoryTopology(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/ps" {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"models":[{"name":"qwen3:8b","size":100,"size_vram":80,"context_length":32768}]}`)
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer backend.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
pool := worker.New([]config.WorkerConfig{{Name: "ollama", URL: backend.URL, MaxConcurrent: 2, HealthInterval: config.Duration(time.Hour)}}, "ollama")
pool.Start(ctx)
sched := scheduler.NewLocal(2, 16, 4)
live := liveflow.New(10*time.Second, 32)
live.Begin(liveflow.Request{ID: "r1", Tenant: "team", Actor: "app", Worker: "ollama", Model: "qwen3:8b", State: liveflow.StateStreaming})
h := New(config.InfrastructureConfig{NodeName: "gateway", RefreshInterval: config.Duration(50 * time.Millisecond), MaxRequests: 32}, live, sched, pool)
h.Start(ctx)
s := h.Snapshot()
if s.Mode != "in-memory" || len(s.Gateways) != 1 || s.Gateways[0].NodeName != "gateway" {
t.Fatalf("unexpected gateway snapshot: %+v", s)
}
if len(s.Workers) != 1 || s.Counts.Models != 1 {
t.Fatalf("unexpected worker/model snapshot: %+v", s)
}
if len(s.Requests) != 1 || s.Requests[0].GatewayName != "gateway" {
t.Fatalf("unexpected request snapshot: %+v", s.Requests)
}
}
internal/liveflow/tracker.go
+335
View File
@@ -0,0 +1,335 @@
package liveflow
import (
"sort"
"sync"
"time"
"github.com/example/ollama-fair-gateway/internal/cost"
)
const (
StateQueued = "queued"
StateRouting = "routing"
StateRunning = "running"
StateStreaming = "streaming"
StateCompleted = "completed"
StateCancelled = "cancelled"
StateFailed = "failed"
)
type Request struct {
ID string `json:"id"`
Tenant string `json:"tenant"`
Actor string `json:"actor"`
Application string `json:"application,omitempty"`
ServiceClass string `json:"service_class,omitempty"`
API string `json:"api"`
Path string `json:"path"`
Model string `json:"model,omitempty"`
Worker string `json:"worker,omitempty"`
State string `json:"state"`
EstimatedCredits float64 `json:"estimated_credits"`
ActualCredits float64 `json:"actual_credits,omitempty"`
EstimatedPromptTokens int64 `json:"estimated_prompt_tokens,omitempty"`
PromptTokens int64 `json:"prompt_tokens,omitempty"`
CompletionTokens int64 `json:"completion_tokens,omitempty"`
BytesOut int64 `json:"bytes_out,omitempty"`
Status int `json:"status,omitempty"`
QueuedAt time.Time `json:"queued_at"`
StartedAt *time.Time `json:"started_at,omitempty"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
QueueMS int64 `json:"queue_ms,omitempty"`
ServiceMS int64 `json:"service_ms,omitempty"`
}
type Counts struct {
Total int `json:"total"`
Active int `json:"active"`
Queued int `json:"queued"`
Routing int `json:"routing"`
Running int `json:"running"`
Streaming int `json:"streaming"`
Completed int `json:"completed"`
Cancelled int `json:"cancelled"`
Failed int `json:"failed"`
}
type Snapshot struct {
GeneratedAt time.Time `json:"generated_at"`
Version uint64 `json:"version"`
Counts Counts `json:"counts"`
Truncated bool `json:"truncated,omitempty"`
Requests []Request `json:"requests"`
}
type Tracker struct {
mu sync.RWMutex
active map[string]Request
recent []Request
recentTTL time.Duration
maxRecent int
version uint64
notify chan struct{}
}
func New(recentTTL time.Duration, maxRecent int) *Tracker {
if recentTTL <= 0 {
recentTTL = 8 * time.Second
}
if maxRecent <= 0 {
maxRecent = 256
}
return &Tracker{active: make(map[string]Request), recentTTL: recentTTL, maxRecent: maxRecent, notify: make(chan struct{})}
}
func (t *Tracker) Begin(r Request) {
now := time.Now().UTC()
r.State = StateQueued
r.QueuedAt = now
r.UpdatedAt = now
t.mu.Lock()
t.active[r.ID] = r
t.bumpLocked()
t.mu.Unlock()
}
func (t *Tracker) MarkRouting(id, worker string, queue time.Duration) {
t.update(id, func(r *Request, now time.Time) {
r.State = StateRouting
r.Worker = worker
r.QueueMS = queue.Milliseconds()
r.UpdatedAt = now
})
}
func (t *Tracker) MarkRunning(id string) {
t.update(id, func(r *Request, now time.Time) {
r.State = StateRunning
if r.StartedAt == nil {
x := now
r.StartedAt = &x
}
r.UpdatedAt = now
})
}
func (t *Tracker) Progress(id string, bytesOut int64, u cost.Usage) {
t.update(id, func(r *Request, now time.Time) {
if bytesOut > 0 {
r.BytesOut = bytesOut
if r.State == StateRunning || r.State == StateRouting {
r.State = StateStreaming
}
}
if u.PromptTokens > 0 {
r.PromptTokens = u.PromptTokens
}
if u.CompletionTokens > 0 {
r.CompletionTokens = u.CompletionTokens
} else if bytesOut > 0 {
// Purely for live visualization. Final accounting still comes from
// Ollama/OpenAI usage metadata in the normal request recorder.
approx := (bytesOut + 15) / 16
if approx > r.CompletionTokens {
r.CompletionTokens = approx
}
}
r.UpdatedAt = now
})
}
func (t *Tracker) Finish(id string, status int, actualCredits float64, u cost.Usage, service time.Duration) {
now := time.Now().UTC()
t.mu.Lock()
r, ok := t.active[id]
if !ok {
t.mu.Unlock()
return
}
delete(t.active, id)
if status >= 200 && status < 400 {
r.State = StateCompleted
} else {
r.State = StateFailed
}
r.Status = status
r.ActualCredits = actualCredits
r.ServiceMS = service.Milliseconds()
if u.PromptTokens > 0 {
r.PromptTokens = u.PromptTokens
}
if u.CompletionTokens > 0 {
r.CompletionTokens = u.CompletionTokens
}
x := now
r.FinishedAt = &x
r.UpdatedAt = now
t.pruneLocked(now)
t.recent = append(t.recent, r)
if len(t.recent) > t.maxRecent {
t.recent = append([]Request(nil), t.recent[len(t.recent)-t.maxRecent:]...)
}
t.bumpLocked()
t.mu.Unlock()
}
func (t *Tracker) Cancel(id string, status int, actualCredits float64, u cost.Usage, service time.Duration) {
now := time.Now().UTC()
t.mu.Lock()
r, ok := t.active[id]
if !ok {
t.mu.Unlock()
return
}
delete(t.active, id)
r.State = StateCancelled
r.Status = status
r.ActualCredits = actualCredits
r.ServiceMS = service.Milliseconds()
if u.PromptTokens > 0 {
r.PromptTokens = u.PromptTokens
}
if u.CompletionTokens > 0 {
r.CompletionTokens = u.CompletionTokens
}
x := now
r.FinishedAt = &x
r.UpdatedAt = now
t.pruneLocked(now)
t.recent = append(t.recent, r)
if len(t.recent) > t.maxRecent {
t.recent = append([]Request(nil), t.recent[len(t.recent)-t.maxRecent:]...)
}
t.bumpLocked()
t.mu.Unlock()
}
func (t *Tracker) Drop(id string, status int) {
t.Finish(id, status, 0, cost.Usage{}, 0)
}
func (t *Tracker) Snapshot() Snapshot {
now := time.Now().UTC()
t.mu.Lock()
t.pruneLocked(now)
out := make([]Request, 0, len(t.active)+len(t.recent))
counts := Counts{}
for _, r := range t.active {
out = append(out, r)
addCount(&counts, r.State)
}
for _, r := range t.recent {
out = append(out, r)
addCount(&counts, r.State)
}
counts.Total = len(t.active) + len(t.recent)
counts.Active = len(t.active)
version := t.version
limit := t.maxRecent
t.mu.Unlock()
truncated := limit > 0 && len(out) > limit
if truncated {
sort.SliceStable(out, func(i, j int) bool {
pi, pj := statePriority(out[i].State), statePriority(out[j].State)
if pi != pj {
return pi < pj
}
if out[i].State == StateCompleted || out[i].State == StateCancelled || out[i].State == StateFailed {
return out[i].UpdatedAt.After(out[j].UpdatedAt)
}
return out[i].QueuedAt.Before(out[j].QueuedAt)
})
out = out[:limit]
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].QueuedAt.Equal(out[j].QueuedAt) {
return out[i].ID < out[j].ID
}
return out[i].QueuedAt.Before(out[j].QueuedAt)
})
return Snapshot{GeneratedAt: now, Version: version, Counts: counts, Truncated: truncated, Requests: out}
}
func addCount(c *Counts, state string) {
switch state {
case StateQueued:
c.Queued++
case StateRouting:
c.Routing++
case StateRunning:
c.Running++
case StateStreaming:
c.Streaming++
case StateCompleted:
c.Completed++
case StateCancelled:
c.Cancelled++
case StateFailed:
c.Failed++
}
}
func statePriority(state string) int {
switch state {
case StateStreaming:
return 0
case StateRunning:
return 1
case StateRouting:
return 2
case StateQueued:
return 3
case StateCompleted:
return 4
case StateCancelled:
return 5
default:
return 5
}
}
func (t *Tracker) Changed() <-chan struct{} {
t.mu.RLock()
ch := t.notify
t.mu.RUnlock()
return ch
}
func (t *Tracker) update(id string, fn func(*Request, time.Time)) {
now := time.Now().UTC()
t.mu.Lock()
r, ok := t.active[id]
if ok {
fn(&r, now)
t.active[id] = r
t.bumpLocked()
}
t.mu.Unlock()
}
func (t *Tracker) pruneLocked(now time.Time) {
if len(t.recent) == 0 {
return
}
cutoff := now.Add(-t.recentTTL)
first := 0
for first < len(t.recent) {
r := t.recent[first]
if r.FinishedAt == nil || r.FinishedAt.After(cutoff) {
break
}
first++
}
if first > 0 {
t.recent = append([]Request(nil), t.recent[first:]...)
}
}
func (t *Tracker) bumpLocked() {
t.version++
close(t.notify)
t.notify = make(chan struct{})
}
internal/liveflow/tracker_test.go
+51
View File
@@ -0,0 +1,51 @@
package liveflow
import (
"testing"
"time"
"github.com/example/ollama-fair-gateway/internal/cost"
)
func TestLifecycle(t *testing.T) {
tr := New(time.Second, 10)
tr.Begin(Request{ID: "r1", Tenant: "a", Actor: "u", Model: "m", EstimatedCredits: 2})
snap := tr.Snapshot()
if len(snap.Requests) != 1 || snap.Requests[0].State != StateQueued {
t.Fatalf("unexpected initial snapshot: %#v", snap)
}
tr.MarkRouting("r1", "w1", 25*time.Millisecond)
tr.MarkRunning("r1")
tr.Progress("r1", 160, cost.Usage{PromptTokens: 12})
snap = tr.Snapshot()
if got := snap.Requests[0]; got.State != StateStreaming || got.Worker != "w1" || got.PromptTokens != 12 || got.CompletionTokens == 0 {
t.Fatalf("unexpected progress: %#v", got)
}
tr.Finish("r1", 200, 1.25, cost.Usage{PromptTokens: 12, CompletionTokens: 8}, 50*time.Millisecond)
snap = tr.Snapshot()
if got := snap.Requests[0]; got.State != StateCompleted || got.Status != 200 || got.ActualCredits != 1.25 || got.CompletionTokens != 8 {
t.Fatalf("unexpected final: %#v", got)
}
}
func TestChangedNotifies(t *testing.T) {
tr := New(time.Second, 10)
ch := tr.Changed()
tr.Begin(Request{ID: "r"})
select {
case <-ch:
case <-time.After(time.Second):
t.Fatal("tracker did not notify")
}
}
func TestSnapshotIsBoundedButCountsRemainExact(t *testing.T) {
tr := New(time.Second, 2)
tr.Begin(Request{ID: "a"})
tr.Begin(Request{ID: "b"})
tr.Begin(Request{ID: "c"})
s := tr.Snapshot()
if !s.Truncated || len(s.Requests) != 2 || s.Counts.Active != 3 || s.Counts.Queued != 3 {
t.Fatalf("unexpected bounded snapshot: %#v", s)
}
}
internal/metrics/metrics.go
+387
View File
@@ -0,0 +1,387 @@
package metrics
import (
"fmt"
"io"
"math"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
)
type histogram struct {
buckets []float64
counts []atomic.Uint64
sum atomic.Uint64
total atomic.Uint64
}
func newHistogram(b []float64) *histogram {
return &histogram{buckets: b, counts: make([]atomic.Uint64, len(b))}
}
func (h *histogram) observe(v float64) {
h.total.Add(1)
atomicAddFloat(&h.sum, v)
for i, b := range h.buckets {
if v <= b {
h.counts[i].Add(1)
}
}
}
type Registry struct {
mu sync.RWMutex
dynamicMu sync.RWMutex
requests map[string]*atomic.Uint64
errors map[string]*atomic.Uint64
queue *histogram
service *histogram
prompt atomic.Uint64
completion atomic.Uint64
creditsBits atomic.Uint64
bytesIn atomic.Uint64
bytesOut atomic.Uint64
usageDropped atomic.Uint64
upstreamFailures map[string]*atomic.Uint64
retries map[string]*atomic.Uint64
circuitOpens map[string]*atomic.Uint64
circuitResets map[string]*atomic.Uint64
dynamic func() Dynamic
}
type Dynamic struct {
Queued, Running int64
OldestQueueWaitSeconds float64
Workers []WorkerMetric
UsageRawFiles int
UsageDailyFiles int
UsageMonthlyFiles int
UsageRawBytes int64
UsageDailyBytes int64
UsageMonthlyBytes int64
UsageLastCompactionUnix int64
UsageLastReclaimedBytes int64
ServiceClasses map[string]ServiceClassMetric
OTelExportedSpans uint64
OTelFailedSpans uint64
OTelDroppedSpans uint64
WarmActionsRunning int
WarmEvictionSuggestions int
AlertsActive int
AlertsLastEvaluateUnix int64
}
type ServiceClassMetric struct{ Queued, Running int64 }
type WorkerMetric struct {
Name string
Healthy bool
Active int64
Max int
MemoryUsedBytes int64
MemoryTotalBytes int64
VRAMUsedBytes int64
VRAMTotalBytes int64
GPUUtilizationPct float64
GPUTemperatureC float64
GPUPowerWatts float64
ModelActive map[string]int
Performance []ModelPerformanceMetric
CircuitState string
Maintenance string
}
type ModelPerformanceMetric struct {
Model string
PromptTPS float64
OutputTPS float64
Samples int64
}
func New() *Registry {
return &Registry{requests: map[string]*atomic.Uint64{}, errors: map[string]*atomic.Uint64{}, upstreamFailures: map[string]*atomic.Uint64{}, retries: map[string]*atomic.Uint64{}, circuitOpens: map[string]*atomic.Uint64{}, circuitResets: map[string]*atomic.Uint64{}, queue: newHistogram([]float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60, 120, 300}), service: newHistogram([]float64{.1, .25, .5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600, 1200})}
}
func (r *Registry) SetDynamic(f func() Dynamic) {
r.dynamicMu.Lock()
r.dynamic = f
r.dynamicMu.Unlock()
}
func (r *Registry) Record(api string, status int, queue, service time.Duration, prompt, completion int64, credits float64, in, out int64) {
class := fmt.Sprintf("%dxx", status/100)
key := api + "|" + class
r.mu.Lock()
c := r.requests[key]
if c == nil {
c = &atomic.Uint64{}
r.requests[key] = c
}
c.Add(1)
if status >= 400 {
e := r.errors[key]
if e == nil {
e = &atomic.Uint64{}
r.errors[key] = e
}
e.Add(1)
}
r.mu.Unlock()
r.queue.observe(queue.Seconds())
r.service.observe(service.Seconds())
if prompt > 0 {
r.prompt.Add(uint64(prompt))
}
if completion > 0 {
r.completion.Add(uint64(completion))
}
atomicAddFloat(&r.creditsBits, credits)
if in > 0 {
r.bytesIn.Add(uint64(in))
}
if out > 0 {
r.bytesOut.Add(uint64(out))
}
}
func (r *Registry) DropUsage() { r.usageDropped.Add(1) }
func (r *Registry) incBounded(m map[string]*atomic.Uint64, key string) {
r.mu.Lock()
c := m[key]
if c == nil {
c = &atomic.Uint64{}
m[key] = c
}
c.Add(1)
r.mu.Unlock()
}
func (r *Registry) RecordUpstreamFailure(worker, class string) {
r.incBounded(r.upstreamFailures, worker+"|"+class)
}
func (r *Registry) RecordRetry(worker string) { r.incBounded(r.retries, worker) }
func (r *Registry) RecordCircuitOpen(worker string) { r.incBounded(r.circuitOpens, worker) }
func (r *Registry) RecordCircuitReset(worker string) { r.incBounded(r.circuitResets, worker) }
type requestMetric struct {
API string
StatusClass string
Value uint64
}
type namedCounter struct {
Key string
Value uint64
}
type prometheusSnapshot struct {
Requests []requestMetric
UpstreamFailures []namedCounter
Retries []namedCounter
CircuitOpens []namedCounter
CircuitResets []namedCounter
Queue histogramState
Service histogramState
Prompt uint64
Completion uint64
Credits float64
BytesIn uint64
BytesOut uint64
UsageDropped uint64
Dynamic Dynamic
}
// snapshotPrometheus copies all state needed by the exporter before any bytes
// are written to the client. In particular, no registry lock is ever held
// across network I/O. This matters because a slow or stalled Prometheus client
// must not be able to block Record() calls in the inference hot path.
func (r *Registry) snapshotPrometheus() prometheusSnapshot {
s := prometheusSnapshot{
Queue: histogramSnapshot(r.queue),
Service: histogramSnapshot(r.service),
Prompt: r.prompt.Load(),
Completion: r.completion.Load(),
Credits: atomicLoadFloat(&r.creditsBits),
BytesIn: r.bytesIn.Load(),
BytesOut: r.bytesOut.Load(),
UsageDropped: r.usageDropped.Load(),
}
// Only the map shape requires a lock. Counter values themselves are atomic.
r.mu.RLock()
keys := make([]string, 0, len(r.requests))
for k := range r.requests {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
p := strings.SplitN(k, "|", 2)
if len(p) != 2 {
continue
}
s.Requests = append(s.Requests, requestMetric{API: p[0], StatusClass: p[1], Value: r.requests[k].Load()})
}
copyCounters := func(m map[string]*atomic.Uint64) []namedCounter {
ks := make([]string, 0, len(m))
for k := range m {
ks = append(ks, k)
}
sort.Strings(ks)
out := make([]namedCounter, 0, len(ks))
for _, k := range ks {
out = append(out, namedCounter{Key: k, Value: m[k].Load()})
}
return out
}
s.UpstreamFailures = copyCounters(r.upstreamFailures)
s.Retries = copyCounters(r.retries)
s.CircuitOpens = copyCounters(r.circuitOpens)
s.CircuitResets = copyCounters(r.circuitResets)
r.mu.RUnlock()
// The dynamic callback snapshots scheduler/worker state. It is deliberately
// invoked after releasing the metrics registry lock, so it cannot form a
// lock-order cycle with request completion, worker telemetry, or persistence.
r.dynamicMu.RLock()
dynamic := r.dynamic
r.dynamicMu.RUnlock()
if dynamic != nil {
s.Dynamic = dynamic()
}
return s
}
func (r *Registry) WritePrometheus(w io.Writer) {
s := r.snapshotPrometheus()
io.WriteString(w, "# HELP ollama_gateway_requests_total Requests handled by API and status class.\n# TYPE ollama_gateway_requests_total counter\n")
for _, x := range s.Requests {
fmt.Fprintf(w, "ollama_gateway_requests_total{api=%q,status_class=%q} %d\n", x.API, x.StatusClass, x.Value)
}
writeHistState(w, "ollama_gateway_queue_seconds", "Queue wait time.", s.Queue)
writeHistState(w, "ollama_gateway_service_seconds", "Backend service time.", s.Service)
fmt.Fprintf(w, "# TYPE ollama_gateway_prompt_tokens_total counter\nollama_gateway_prompt_tokens_total %d\n", s.Prompt)
fmt.Fprintf(w, "# TYPE ollama_gateway_completion_tokens_total counter\nollama_gateway_completion_tokens_total %d\n", s.Completion)
fmt.Fprintf(w, "# TYPE ollama_gateway_credits_total counter\nollama_gateway_credits_total %.6f\n", s.Credits)
fmt.Fprintf(w, "# TYPE ollama_gateway_bytes_in_total counter\nollama_gateway_bytes_in_total %d\n", s.BytesIn)
fmt.Fprintf(w, "# TYPE ollama_gateway_bytes_out_total counter\nollama_gateway_bytes_out_total %d\n", s.BytesOut)
fmt.Fprintf(w, "# TYPE ollama_gateway_usage_events_dropped_total counter\nollama_gateway_usage_events_dropped_total %d\n", s.UsageDropped)
fmt.Fprintln(w, "# TYPE ollama_gateway_upstream_failures_total counter")
for _, x := range s.UpstreamFailures {
p := strings.SplitN(x.Key, "|", 2)
class := "other"
if len(p) > 1 {
class = p[1]
}
fmt.Fprintf(w, "ollama_gateway_upstream_failures_total{worker=%q,class=%q} %d\n", p[0], class, x.Value)
}
fmt.Fprintln(w, "# TYPE ollama_gateway_retries_total counter")
for _, x := range s.Retries {
fmt.Fprintf(w, "ollama_gateway_retries_total{worker=%q} %d\n", x.Key, x.Value)
}
fmt.Fprintln(w, "# TYPE ollama_gateway_circuit_opens_total counter")
for _, x := range s.CircuitOpens {
fmt.Fprintf(w, "ollama_gateway_circuit_opens_total{worker=%q} %d\n", x.Key, x.Value)
}
fmt.Fprintln(w, "# TYPE ollama_gateway_circuit_resets_total counter")
for _, x := range s.CircuitResets {
fmt.Fprintf(w, "ollama_gateway_circuit_resets_total{worker=%q} %d\n", x.Key, x.Value)
}
d := s.Dynamic
fmt.Fprintf(w, "# TYPE ollama_gateway_queue_depth gauge\nollama_gateway_queue_depth %d\n# TYPE ollama_gateway_running gauge\nollama_gateway_running %d\n", d.Queued, d.Running)
classNames := make([]string, 0, len(d.ServiceClasses))
for n := range d.ServiceClasses {
classNames = append(classNames, n)
}
sort.Strings(classNames)
for _, n := range classNames {
x := d.ServiceClasses[n]
fmt.Fprintf(w, "ollama_gateway_service_class_queued{name=%q} %d\nollama_gateway_service_class_running{name=%q} %d\n", n, x.Queued, n, x.Running)
}
fmt.Fprintf(w, "# TYPE ollama_gateway_usage_raw_files gauge\nollama_gateway_usage_raw_files %d\n", d.UsageRawFiles)
fmt.Fprintf(w, "# TYPE ollama_gateway_usage_daily_rollup_files gauge\nollama_gateway_usage_daily_rollup_files %d\n", d.UsageDailyFiles)
fmt.Fprintf(w, "# TYPE ollama_gateway_usage_monthly_rollup_files gauge\nollama_gateway_usage_monthly_rollup_files %d\n", d.UsageMonthlyFiles)
fmt.Fprintf(w, "# TYPE ollama_gateway_usage_raw_bytes gauge\nollama_gateway_usage_raw_bytes %d\n", d.UsageRawBytes)
fmt.Fprintf(w, "# TYPE ollama_gateway_usage_daily_rollup_bytes gauge\nollama_gateway_usage_daily_rollup_bytes %d\n", d.UsageDailyBytes)
fmt.Fprintf(w, "# TYPE ollama_gateway_usage_monthly_rollup_bytes gauge\nollama_gateway_usage_monthly_rollup_bytes %d\n", d.UsageMonthlyBytes)
fmt.Fprintf(w, "# TYPE ollama_gateway_usage_last_compaction_timestamp_seconds gauge\nollama_gateway_usage_last_compaction_timestamp_seconds %d\n", d.UsageLastCompactionUnix)
fmt.Fprintf(w, "# TYPE ollama_gateway_usage_last_reclaimed_bytes gauge\nollama_gateway_usage_last_reclaimed_bytes %d\n", d.UsageLastReclaimedBytes)
fmt.Fprintf(w, "# TYPE ollama_gateway_otel_exported_spans_total counter\nollama_gateway_otel_exported_spans_total %d\n", d.OTelExportedSpans)
fmt.Fprintf(w, "# TYPE ollama_gateway_otel_failed_spans_total counter\nollama_gateway_otel_failed_spans_total %d\n", d.OTelFailedSpans)
fmt.Fprintf(w, "# TYPE ollama_gateway_otel_dropped_spans_total counter\nollama_gateway_otel_dropped_spans_total %d\n", d.OTelDroppedSpans)
for _, x := range d.Workers {
h := 0
if x.Healthy {
h = 1
}
fmt.Fprintf(w, "ollama_gateway_worker_healthy{name=%q} %d\nollama_gateway_worker_active{name=%q} %d\nollama_gateway_worker_capacity{name=%q} %d\n", x.Name, h, x.Name, x.Active, x.Name, x.Max)
if x.MemoryTotalBytes > 0 {
fmt.Fprintf(w, "ollama_gateway_worker_memory_used_bytes{name=%q} %d\nollama_gateway_worker_memory_total_bytes{name=%q} %d\n", x.Name, x.MemoryUsedBytes, x.Name, x.MemoryTotalBytes)
}
if x.VRAMTotalBytes > 0 {
fmt.Fprintf(w, "ollama_gateway_worker_vram_used_bytes{name=%q} %d\nollama_gateway_worker_vram_total_bytes{name=%q} %d\n", x.Name, x.VRAMUsedBytes, x.Name, x.VRAMTotalBytes)
}
if x.GPUUtilizationPct > 0 {
fmt.Fprintf(w, "ollama_gateway_worker_gpu_utilization_percent{name=%q} %.3f\n", x.Name, x.GPUUtilizationPct)
}
if x.GPUTemperatureC > 0 {
fmt.Fprintf(w, "ollama_gateway_worker_gpu_temperature_celsius{name=%q} %.3f\n", x.Name, x.GPUTemperatureC)
}
if x.GPUPowerWatts > 0 {
fmt.Fprintf(w, "ollama_gateway_worker_gpu_power_watts{name=%q} %.3f\n", x.Name, x.GPUPowerWatts)
}
open, half, accepting := 0, 0, 1
if x.CircuitState == "open" {
open = 1
accepting = 0
}
if x.CircuitState == "half_open" {
half = 1
}
if x.Maintenance != "active" {
accepting = 0
}
fmt.Fprintf(w, "ollama_gateway_worker_circuit_open{name=%q} %d\nollama_gateway_worker_circuit_half_open{name=%q} %d\nollama_gateway_worker_accepting_new{name=%q} %d\n", x.Name, open, x.Name, half, x.Name, accepting)
models := make([]string, 0, len(x.ModelActive))
for model := range x.ModelActive {
models = append(models, model)
}
sort.Strings(models)
for _, model := range models {
fmt.Fprintf(w, "ollama_gateway_worker_model_active{name=%q,model=%q} %d\n", x.Name, model, x.ModelActive[model])
}
for _, perf := range x.Performance {
if perf.PromptTPS > 0 {
fmt.Fprintf(w, "ollama_gateway_worker_model_prompt_tokens_per_second{name=%q,model=%q} %.6f\n", x.Name, perf.Model, perf.PromptTPS)
}
if perf.OutputTPS > 0 {
fmt.Fprintf(w, "ollama_gateway_worker_model_output_tokens_per_second{name=%q,model=%q} %.6f\n", x.Name, perf.Model, perf.OutputTPS)
}
fmt.Fprintf(w, "ollama_gateway_worker_model_performance_samples{name=%q,model=%q} %d\n", x.Name, perf.Model, perf.Samples)
}
}
}
func writeHistState(w io.Writer, name, help string, h histogramState) {
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s histogram\n", name, help, name)
for i, b := range h.Buckets {
var count uint64
if i < len(h.Counts) {
count = h.Counts[i]
}
fmt.Fprintf(w, "%s_bucket{le=\"%g\"} %d\n", name, b, count)
}
fmt.Fprintf(w, "%s_bucket{le=\"+Inf\"} %d\n%s_sum %.9f\n%s_count %d\n", name, h.Total, name, h.Sum, name, h.Total)
}
// Atomic float helpers use CAS so simultaneous observations do not lose increments.
func atomicAddFloat(a *atomic.Uint64, v float64) {
for {
old := a.Load()
nv := mathFloat64frombits(old) + v
if a.CompareAndSwap(old, mathFloat64bits(nv)) {
return
}
}
}
func atomicLoadFloat(a *atomic.Uint64) float64 { return mathFloat64frombits(a.Load()) }
// Tiny wrappers keep the package dependency-free while using math's bit representation.
func mathFloat64bits(f float64) uint64 { return math.Float64bits(f) }
func mathFloat64frombits(b uint64) float64 { return math.Float64frombits(b) }
internal/metrics/metrics_test.go
+96
View File
@@ -0,0 +1,96 @@
package metrics
import (
"bytes"
"strings"
"sync"
"testing"
"time"
)
func TestPrometheusIncludesWorkerTelemetryAndModelThroughput(t *testing.T) {
r := New()
r.SetDynamic(func() Dynamic {
return Dynamic{UsageRawFiles: 3, UsageDailyFiles: 12, UsageMonthlyFiles: 7, UsageRawBytes: 100, UsageDailyBytes: 200, UsageMonthlyBytes: 300, UsageLastCompactionUnix: 123456, UsageLastReclaimedBytes: 789, Workers: []WorkerMetric{{
Name: "gpu-1", Healthy: true, Active: 1, Max: 2,
VRAMUsedBytes: 12, VRAMTotalBytes: 24, GPUUtilizationPct: 87.5,
GPUTemperatureC: 63, GPUPowerWatts: 310,
ModelActive: map[string]int{"qwen3:8b": 1},
Performance: []ModelPerformanceMetric{{Model: "qwen3:8b", OutputTPS: 101.25, Samples: 3}},
}}}
})
var b bytes.Buffer
r.WritePrometheus(&b)
out := b.String()
for _, want := range []string{
`ollama_gateway_worker_vram_used_bytes{name="gpu-1"} 12`,
`ollama_gateway_worker_gpu_utilization_percent{name="gpu-1"} 87.500`,
`ollama_gateway_worker_model_active{name="gpu-1",model="qwen3:8b"} 1`,
`ollama_gateway_worker_model_output_tokens_per_second{name="gpu-1",model="qwen3:8b"} 101.250000`,
`ollama_gateway_usage_raw_files 3`,
`ollama_gateway_usage_daily_rollup_files 12`,
`ollama_gateway_usage_monthly_rollup_bytes 300`,
`ollama_gateway_usage_last_compaction_timestamp_seconds 123456`,
`ollama_gateway_usage_last_reclaimed_bytes 789`,
} {
if !strings.Contains(out, want) {
t.Fatalf("missing %q in metrics:\n%s", want, out)
}
}
}
// blockingWriter simulates a Prometheus client that stops reading after the
// exporter has started writing request metrics. WritePrometheus must not hold
// the registry mutex while this writer is blocked, otherwise all request
// completions stall in Record().
type blockingWriter struct {
started chan struct{}
release chan struct{}
once sync.Once
}
func (w *blockingWriter) Write(p []byte) (int, error) {
if strings.Contains(string(p), "ollama_gateway_requests_total{") {
w.once.Do(func() { close(w.started) })
<-w.release
}
return len(p), nil
}
func TestPrometheusSlowClientDoesNotBlockRecord(t *testing.T) {
r := New()
r.Record("ollama", 200, 0, time.Millisecond, 1, 1, 1, 1, 1)
bw := &blockingWriter{started: make(chan struct{}), release: make(chan struct{})}
done := make(chan struct{})
go func() {
r.WritePrometheus(bw)
close(done)
}()
select {
case <-bw.started:
case <-time.After(time.Second):
t.Fatal("metrics writer did not reach blocking write")
}
recorded := make(chan struct{})
go func() {
r.Record("ollama", 200, 0, time.Millisecond, 1, 1, 1, 1, 1)
close(recorded)
}()
select {
case <-recorded:
// success: no registry lock is held by the stalled exporter
case <-time.After(250 * time.Millisecond):
close(bw.release)
t.Fatal("Record blocked behind a stalled /metrics writer")
}
close(bw.release)
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("metrics writer did not finish after release")
}
}
internal/metrics/persistence.go
+160
View File
@@ -0,0 +1,160 @@
package metrics
import (
"context"
"errors"
"os"
"sync/atomic"
"time"
"github.com/example/ollama-fair-gateway/internal/state"
)
type histogramState struct {
Buckets []float64 `json:"buckets"`
Counts []uint64 `json:"counts"`
Sum float64 `json:"sum"`
Total uint64 `json:"total"`
}
type PersistentState struct {
Version int `json:"version"`
SavedAt time.Time `json:"saved_at"`
Requests map[string]uint64 `json:"requests"`
Errors map[string]uint64 `json:"errors"`
Queue histogramState `json:"queue"`
Service histogramState `json:"service"`
Prompt uint64 `json:"prompt_tokens"`
Completion uint64 `json:"completion_tokens"`
Credits float64 `json:"credits"`
BytesIn uint64 `json:"bytes_in"`
BytesOut uint64 `json:"bytes_out"`
UsageDropped uint64 `json:"usage_dropped"`
UpstreamFailures map[string]uint64 `json:"upstream_failures,omitempty"`
Retries map[string]uint64 `json:"retries,omitempty"`
CircuitOpens map[string]uint64 `json:"circuit_opens,omitempty"`
CircuitResets map[string]uint64 `json:"circuit_resets,omitempty"`
}
func histogramSnapshot(h *histogram) histogramState {
x := histogramState{Buckets: append([]float64(nil), h.buckets...), Counts: make([]uint64, len(h.counts)), Sum: atomicLoadFloat(&h.sum), Total: h.total.Load()}
for i := range h.counts {
x.Counts[i] = h.counts[i].Load()
}
return x
}
func (r *Registry) SnapshotPersistent() PersistentState {
r.mu.Lock()
defer r.mu.Unlock()
s := PersistentState{Version: 1, SavedAt: time.Now().UTC(), Requests: map[string]uint64{}, Errors: map[string]uint64{}, Queue: histogramSnapshot(r.queue), Service: histogramSnapshot(r.service), Prompt: r.prompt.Load(), Completion: r.completion.Load(), Credits: atomicLoadFloat(&r.creditsBits), BytesIn: r.bytesIn.Load(), BytesOut: r.bytesOut.Load(), UsageDropped: r.usageDropped.Load(), UpstreamFailures: map[string]uint64{}, Retries: map[string]uint64{}, CircuitOpens: map[string]uint64{}, CircuitResets: map[string]uint64{}}
for k, v := range r.requests {
s.Requests[k] = v.Load()
}
for k, v := range r.errors {
s.Errors[k] = v.Load()
}
for k, v := range r.upstreamFailures {
s.UpstreamFailures[k] = v.Load()
}
for k, v := range r.retries {
s.Retries[k] = v.Load()
}
for k, v := range r.circuitOpens {
s.CircuitOpens[k] = v.Load()
}
for k, v := range r.circuitResets {
s.CircuitResets[k] = v.Load()
}
return s
}
func restoreHistogram(dst *histogram, src histogramState) {
if len(src.Counts) != len(dst.counts) {
return
}
for i := range dst.counts {
dst.counts[i].Store(src.Counts[i])
}
dst.total.Store(src.Total)
dst.sum.Store(mathFloat64bits(src.Sum))
}
func (r *Registry) RestorePersistent(s PersistentState) {
if s.Version != 1 {
return
}
r.mu.Lock()
defer r.mu.Unlock()
r.requests = map[string]*atomic.Uint64{}
for k, v := range s.Requests {
x := &atomic.Uint64{}
x.Store(v)
r.requests[k] = x
}
r.errors = map[string]*atomic.Uint64{}
for k, v := range s.Errors {
x := &atomic.Uint64{}
x.Store(v)
r.errors[k] = x
}
restoreMap := func(src map[string]uint64) map[string]*atomic.Uint64 {
out := map[string]*atomic.Uint64{}
for k, v := range src {
x := &atomic.Uint64{}
x.Store(v)
out[k] = x
}
return out
}
r.upstreamFailures = restoreMap(s.UpstreamFailures)
r.retries = restoreMap(s.Retries)
r.circuitOpens = restoreMap(s.CircuitOpens)
r.circuitResets = restoreMap(s.CircuitResets)
restoreHistogram(r.queue, s.Queue)
restoreHistogram(r.service, s.Service)
r.prompt.Store(s.Prompt)
r.completion.Store(s.Completion)
r.creditsBits.Store(mathFloat64bits(s.Credits))
r.bytesIn.Store(s.BytesIn)
r.bytesOut.Store(s.BytesOut)
r.usageDropped.Store(s.UsageDropped)
}
func (r *Registry) LoadPersistent(path string) error {
var s PersistentState
err := (state.AtomicJSON{Path: path, Mode: 0640}).Load(&s)
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return err
}
r.RestorePersistent(s)
return nil
}
func (r *Registry) SavePersistent(path string) error {
return (state.AtomicJSON{Path: path, Mode: 0640}).Save(r.SnapshotPersistent())
}
func (r *Registry) StartPersistence(ctx context.Context, path string, interval time.Duration, onError func(error)) {
if interval < time.Second {
interval = 10 * time.Second
}
go func() {
t := time.NewTicker(interval)
defer t.Stop()
defer func() {
if err := r.SavePersistent(path); err != nil && onError != nil {
onError(err)
}
}()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := r.SavePersistent(path); err != nil && onError != nil {
onError(err)
}
}
}
}()
}
internal/metrics/persistence_test.go
+29
View File
@@ -0,0 +1,29 @@
package metrics
import (
"path/filepath"
"testing"
"time"
)
func TestPersistentMetricsRoundTrip(t *testing.T) {
path := filepath.Join(t.TempDir(), "metrics.json")
a := New()
a.Record("ollama", 200, 25*time.Millisecond, 2*time.Second, 123, 45, 7.25, 1000, 2000)
a.Record("openai", 500, time.Second, 3*time.Second, 5, 1, 1.5, 10, 20)
a.DropUsage()
if err := a.SavePersistent(path); err != nil {
t.Fatal(err)
}
b := New()
if err := b.LoadPersistent(path); err != nil {
t.Fatal(err)
}
x := b.SnapshotPersistent()
if x.Prompt != 128 || x.Completion != 46 || x.Credits != 8.75 || x.BytesIn != 1010 || x.BytesOut != 2020 || x.UsageDropped != 1 {
t.Fatalf("restored=%#v", x)
}
if x.Requests["ollama|2xx"] != 1 || x.Requests["openai|5xx"] != 1 || x.Errors["openai|5xx"] != 1 {
t.Fatalf("counters=%#v errors=%#v", x.Requests, x.Errors)
}
}
internal/policy/store.go
+64
View File
@@ -0,0 +1,64 @@
package policy
import (
"context"
"sort"
"sync"
"github.com/example/ollama-fair-gateway/internal/config"
)
type Store interface {
Get(context.Context, string) (config.TenantPolicy, bool, error)
Put(context.Context, string, config.TenantPolicy) error
Delete(context.Context, string) error
List(context.Context) (map[string]config.TenantPolicy, error)
Health(context.Context) error
}
type Memory struct {
mu sync.RWMutex
m map[string]config.TenantPolicy
}
func NewMemory() *Memory { return &Memory{m: map[string]config.TenantPolicy{}} }
func (m *Memory) Get(_ context.Context, tenant string) (config.TenantPolicy, bool, error) {
m.mu.RLock()
defer m.mu.RUnlock()
p, ok := m.m[tenant]
return p, ok, nil
}
func (m *Memory) Put(_ context.Context, tenant string, p config.TenantPolicy) error {
m.mu.Lock()
m.m[tenant] = p
m.mu.Unlock()
return nil
}
func (m *Memory) Delete(_ context.Context, tenant string) error {
m.mu.Lock()
delete(m.m, tenant)
m.mu.Unlock()
return nil
}
func (m *Memory) List(context.Context) (map[string]config.TenantPolicy, error) {
m.mu.RLock()
defer m.mu.RUnlock()
out := make(map[string]config.TenantPolicy, len(m.m))
for k, v := range m.m {
out[k] = v
}
return out, nil
}
func (m *Memory) Health(context.Context) error { return nil }
// Names returns the stable sorted tenant names in a policy map. Kept here so
// both API and UI can present deterministic policy tables.
func Names(m map[string]config.TenantPolicy) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}

Some files were not shown because too many files have changed in this diff Show More