From e581949946b62326c80de9999c4c069e4620e9f4 Mon Sep 17 00:00:00 2001 From: jbergner Date: Fri, 11 Sep 2026 06:14:38 +0200 Subject: [PATCH] - --- .env.production.example | 5 + .gitignore | 5 + Dockerfile | 18 + Makefile | 50 + README.md | 604 +++++- cmd/bench/main.go | 339 ++++ cmd/bench/main_test.go | 68 + cmd/ha-report/main.go | 396 ++++ cmd/ha-report/main_test.go | 207 ++ cmd/ha-sampler/main.go | 50 + cmd/ha-snapshot/main.go | 40 + cmd/ha-snapshot/main_test.go | 10 + cmd/mock-ollama/main.go | 191 ++ cmd/mock-ollama/main_test.go | 108 + cmd/ollama-gateway/main.go | 255 +++ cmd/ollama-gateway/preflight.go | 222 +++ cmd/ollama-gateway/preflight_test.go | 154 ++ cmd/worker-telemetry/main.go | 180 ++ cmd/worker-telemetry/main_test.go | 19 + config.example.json | 335 ++++ config.oidc.example.json | 340 ++++ config.openwebui.example.json | 316 +++ config.placement.example.json | 346 ++++ config.rtx4090.example.json | 339 ++++ docker-compose.production.yml | 32 + docker-compose.yml | 26 + docs/ALERTS.md | 78 + docs/ARCHITECTURE.md | 77 + docs/BATCH-JOBS.md | 140 ++ docs/CONTEXT-HARDENING.md | 69 + docs/CONVERSATIONS.md | 67 + docs/DEPLOYMENT-HARDENING.md | 117 ++ docs/HA-READINESS.md | 131 ++ docs/IMPROVEMENTS-2026.md | 198 ++ docs/JOBS.md | 47 + docs/MIGRATION-INMEMORY.md | 65 + docs/MODEL-PLACEMENT.md | 137 ++ docs/OPENWEBUI.md | 134 ++ docs/PERSISTENCE.md | 93 + docs/PRODUCTION-UPDATE.md | 67 + docs/PUBLIC-DASHBOARD.md | 64 + docs/RELEASE-VERIFICATION-CHECKPOINT-17.md | 47 + docs/RELEASE-VERIFICATION-CHECKPOINT-18.md | 67 + docs/RELEASE-VERIFICATION-CHECKPOINT-19.md | 58 + docs/RELEASE-VERIFICATION-CHECKPOINT-20.md | 69 + docs/RELEASE-VERIFICATION-CHECKPOINT-21.md | 40 + docs/RELEASE-VERIFICATION-CHECKPOINT-22.md | 39 + docs/RELEASE-VERIFICATION-CHECKPOINT-23.md | 57 + docs/RELEASE-VERIFICATION-CHECKPOINT-24.md | 53 + docs/RELEASE-VERIFICATION-CHECKPOINT-26.md | 15 + docs/RELEASE-VERIFICATION-CHECKPOINT-27.md | 84 + docs/RELEASE-VERIFICATION-CHECKPOINT-28.md | 61 + docs/RELIABILITY.md | 27 + docs/RETENTION.md | 80 + docs/ROADMAP-P0-P3.md | 195 ++ docs/SECURITY.md | 76 + docs/WARM-MODELS.md | 65 + docs/WORKER-TELEMETRY.md | 127 ++ go.mod | 3 + internal/alerts/alerts.go | 591 ++++++ internal/alerts/alerts_test.go | 127 ++ internal/auth/identity.go | 492 +++++ internal/auth/identity_test.go | 103 + internal/auth/ip_bypass_test.go | 71 + internal/auth/oidc.go | 499 +++++ internal/auth/oidc_test.go | 63 + internal/autotune/autotune.go | 532 +++++ internal/autotune/autotune_test.go | 80 + internal/batch/manager.go | 692 +++++++ internal/batch/manager_test.go | 216 ++ internal/config/config.go | 1314 +++++++++++++ internal/config/config_test.go | 181 ++ internal/conversation/store.go | 322 +++ internal/conversation/store_test.go | 103 + internal/cost/estimator.go | 214 ++ internal/cost/estimator_test.go | 52 + internal/haresource/collect.go | 141 ++ internal/haresource/collect_test.go | 21 + internal/haresource/sample.go | 240 +++ internal/haresource/sample_test.go | 69 + internal/hoststats/amd.go | 15 + internal/hoststats/amd_linux.go | 87 + internal/hoststats/amd_other.go | 12 + internal/hoststats/amd_test.go | 51 + internal/hoststats/memory.go | 10 + internal/hoststats/memory_darwin.go | 66 + internal/hoststats/memory_linux.go | 46 + internal/hoststats/memory_other.go | 12 + internal/hoststats/memory_windows.go | 41 + internal/hoststats/nvidia.go | 69 + internal/hoststats/nvidia_test.go | 12 + internal/infrastructure/hub.go | 201 ++ internal/infrastructure/hub_test.go | 48 + internal/liveflow/tracker.go | 335 ++++ internal/liveflow/tracker_test.go | 51 + internal/metrics/metrics.go | 387 ++++ internal/metrics/metrics_test.go | 96 + internal/metrics/persistence.go | 160 ++ internal/metrics/persistence_test.go | 29 + internal/policy/store.go | 64 + internal/proxy/proxy.go | 428 ++++ internal/proxy/proxy_test.go | 65 + internal/publicui/assets/app.css | 3 + internal/publicui/assets/app.js | 17 + internal/publicui/assets/index.html | 51 + internal/publicui/publicui.go | 15 + internal/quota/persistence.go | 85 + internal/quota/persistence_test.go | 37 + internal/quota/quota.go | 135 ++ internal/scheduler/scheduler.go | 360 ++++ internal/scheduler/scheduler_test.go | 194 ++ internal/server/aliases.go | 183 ++ internal/server/batch_api.go | 298 +++ internal/server/batch_e2e_test.go | 165 ++ internal/server/conversations.go | 185 ++ internal/server/conversations_test.go | 222 +++ internal/server/jobs.go | 103 + internal/server/model_access.go | 179 ++ internal/server/models.go | 107 + internal/server/openwebui_test.go | 164 ++ internal/server/operations.go | 229 +++ internal/server/operations_test.go | 43 + internal/server/policy_simulator.go | 284 +++ internal/server/preflight.go | 343 ++++ internal/server/preflight_test.go | 344 ++++ internal/server/public_dashboard.go | 344 ++++ internal/server/server.go | 883 +++++++++ internal/server/server_test.go | 297 +++ internal/server/storage.go | 288 +++ internal/server/ui.go | 1423 ++++++++++++++ internal/server/ui_test.go | 1099 +++++++++++ internal/session/store.go | 68 + internal/state/apikeys.go | 93 + internal/state/atomic.go | 110 ++ internal/state/config.go | 77 + internal/state/modelplacement.go | 103 + internal/state/paths.go | 46 + internal/state/policies.go | 83 + internal/state/state_test.go | 180 ++ internal/state/workerstate.go | 95 + internal/telemetry/otel.go | 376 ++++ internal/telemetry/otel_test.go | 66 + internal/usage/recorder.go | 420 ++++ internal/usage/recorder_test.go | 240 +++ internal/usage/retention.go | 693 +++++++ internal/warm/manager.go | 608 ++++++ internal/warm/manager_test.go | 179 ++ internal/webui/assets/app.css | 101 + internal/webui/assets/app.js | 214 ++ internal/webui/assets/index.html | 422 ++++ internal/webui/webui.go | 15 + internal/webui/webui_test.go | 191 ++ internal/worker/persistence.go | 94 + internal/worker/persistence_test.go | 32 + internal/worker/pool.go | 2060 ++++++++++++++++++++ internal/worker/pool_test.go | 387 ++++ internal/worker/telemetry_test.go | 64 + scripts/ha-readiness.sh | 187 ++ scripts/ollama-env.sh | 9 + scripts/production-preflight.sh | 54 + scripts/production-preflight_test.sh | 70 + 161 files changed, 31126 insertions(+), 1 deletion(-) create mode 100644 .env.production.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 cmd/bench/main.go create mode 100644 cmd/bench/main_test.go create mode 100644 cmd/ha-report/main.go create mode 100644 cmd/ha-report/main_test.go create mode 100644 cmd/ha-sampler/main.go create mode 100644 cmd/ha-snapshot/main.go create mode 100644 cmd/ha-snapshot/main_test.go create mode 100644 cmd/mock-ollama/main.go create mode 100644 cmd/mock-ollama/main_test.go create mode 100644 cmd/ollama-gateway/main.go create mode 100644 cmd/ollama-gateway/preflight.go create mode 100644 cmd/ollama-gateway/preflight_test.go create mode 100644 cmd/worker-telemetry/main.go create mode 100644 cmd/worker-telemetry/main_test.go create mode 100644 config.example.json create mode 100644 config.oidc.example.json create mode 100644 config.openwebui.example.json create mode 100644 config.placement.example.json create mode 100644 config.rtx4090.example.json create mode 100644 docker-compose.production.yml create mode 100644 docker-compose.yml create mode 100644 docs/ALERTS.md create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/BATCH-JOBS.md create mode 100644 docs/CONTEXT-HARDENING.md create mode 100644 docs/CONVERSATIONS.md create mode 100644 docs/DEPLOYMENT-HARDENING.md create mode 100644 docs/HA-READINESS.md create mode 100644 docs/IMPROVEMENTS-2026.md create mode 100644 docs/JOBS.md create mode 100644 docs/MIGRATION-INMEMORY.md create mode 100644 docs/MODEL-PLACEMENT.md create mode 100644 docs/OPENWEBUI.md create mode 100644 docs/PERSISTENCE.md create mode 100644 docs/PRODUCTION-UPDATE.md create mode 100644 docs/PUBLIC-DASHBOARD.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-17.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-18.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-19.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-20.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-21.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-22.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-23.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-24.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-26.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-27.md create mode 100644 docs/RELEASE-VERIFICATION-CHECKPOINT-28.md create mode 100644 docs/RELIABILITY.md create mode 100644 docs/RETENTION.md create mode 100644 docs/ROADMAP-P0-P3.md create mode 100644 docs/SECURITY.md create mode 100644 docs/WARM-MODELS.md create mode 100644 docs/WORKER-TELEMETRY.md create mode 100644 go.mod create mode 100644 internal/alerts/alerts.go create mode 100644 internal/alerts/alerts_test.go create mode 100644 internal/auth/identity.go create mode 100644 internal/auth/identity_test.go create mode 100644 internal/auth/ip_bypass_test.go create mode 100644 internal/auth/oidc.go create mode 100644 internal/auth/oidc_test.go create mode 100644 internal/autotune/autotune.go create mode 100644 internal/autotune/autotune_test.go create mode 100644 internal/batch/manager.go create mode 100644 internal/batch/manager_test.go create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/conversation/store.go create mode 100644 internal/conversation/store_test.go create mode 100644 internal/cost/estimator.go create mode 100644 internal/cost/estimator_test.go create mode 100644 internal/haresource/collect.go create mode 100644 internal/haresource/collect_test.go create mode 100644 internal/haresource/sample.go create mode 100644 internal/haresource/sample_test.go create mode 100644 internal/hoststats/amd.go create mode 100644 internal/hoststats/amd_linux.go create mode 100644 internal/hoststats/amd_other.go create mode 100644 internal/hoststats/amd_test.go create mode 100644 internal/hoststats/memory.go create mode 100644 internal/hoststats/memory_darwin.go create mode 100644 internal/hoststats/memory_linux.go create mode 100644 internal/hoststats/memory_other.go create mode 100644 internal/hoststats/memory_windows.go create mode 100644 internal/hoststats/nvidia.go create mode 100644 internal/hoststats/nvidia_test.go create mode 100644 internal/infrastructure/hub.go create mode 100644 internal/infrastructure/hub_test.go create mode 100644 internal/liveflow/tracker.go create mode 100644 internal/liveflow/tracker_test.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/metrics/metrics_test.go create mode 100644 internal/metrics/persistence.go create mode 100644 internal/metrics/persistence_test.go create mode 100644 internal/policy/store.go create mode 100644 internal/proxy/proxy.go create mode 100644 internal/proxy/proxy_test.go create mode 100644 internal/publicui/assets/app.css create mode 100644 internal/publicui/assets/app.js create mode 100644 internal/publicui/assets/index.html create mode 100644 internal/publicui/publicui.go create mode 100644 internal/quota/persistence.go create mode 100644 internal/quota/persistence_test.go create mode 100644 internal/quota/quota.go create mode 100644 internal/scheduler/scheduler.go create mode 100644 internal/scheduler/scheduler_test.go create mode 100644 internal/server/aliases.go create mode 100644 internal/server/batch_api.go create mode 100644 internal/server/batch_e2e_test.go create mode 100644 internal/server/conversations.go create mode 100644 internal/server/conversations_test.go create mode 100644 internal/server/jobs.go create mode 100644 internal/server/model_access.go create mode 100644 internal/server/models.go create mode 100644 internal/server/openwebui_test.go create mode 100644 internal/server/operations.go create mode 100644 internal/server/operations_test.go create mode 100644 internal/server/policy_simulator.go create mode 100644 internal/server/preflight.go create mode 100644 internal/server/preflight_test.go create mode 100644 internal/server/public_dashboard.go create mode 100644 internal/server/server.go create mode 100644 internal/server/server_test.go create mode 100644 internal/server/storage.go create mode 100644 internal/server/ui.go create mode 100644 internal/server/ui_test.go create mode 100644 internal/session/store.go create mode 100644 internal/state/apikeys.go create mode 100644 internal/state/atomic.go create mode 100644 internal/state/config.go create mode 100644 internal/state/modelplacement.go create mode 100644 internal/state/paths.go create mode 100644 internal/state/policies.go create mode 100644 internal/state/state_test.go create mode 100644 internal/state/workerstate.go create mode 100644 internal/telemetry/otel.go create mode 100644 internal/telemetry/otel_test.go create mode 100644 internal/usage/recorder.go create mode 100644 internal/usage/recorder_test.go create mode 100644 internal/usage/retention.go create mode 100644 internal/warm/manager.go create mode 100644 internal/warm/manager_test.go create mode 100644 internal/webui/assets/app.css create mode 100644 internal/webui/assets/app.js create mode 100644 internal/webui/assets/index.html create mode 100644 internal/webui/webui.go create mode 100644 internal/webui/webui_test.go create mode 100644 internal/worker/persistence.go create mode 100644 internal/worker/persistence_test.go create mode 100644 internal/worker/pool.go create mode 100644 internal/worker/pool_test.go create mode 100644 internal/worker/telemetry_test.go create mode 100644 scripts/ha-readiness.sh create mode 100644 scripts/ollama-env.sh create mode 100644 scripts/production-preflight.sh create mode 100644 scripts/production-preflight_test.sh diff --git a/.env.production.example b/.env.production.example new file mode 100644 index 0000000..02bde31 --- /dev/null +++ b/.env.production.example @@ -0,0 +1,5 @@ +# Copy to .env or export these values before docker compose commands. +# The safe default publishes the gateway only on the Docker host loopback. +GATEWAY_CONFIG=./gateway-config.json +GATEWAY_PUBLISH_ADDRESS=127.0.0.1 +GATEWAY_PUBLISH_PORT=9080 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..81f9f29 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/data/ +/dist/* +.env +*.log +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4d571dc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.23-alpine AS build +WORKDIR /src +COPY . . +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/ollama-gateway ./cmd/ollama-gateway \ + && mkdir -p /out/data \ + && touch /out/data/.keep + +FROM scratch +COPY --from=build /out/ollama-gateway /ollama-gateway +# Pre-create a writable state directory for the unprivileged runtime user. +# Empty named volumes mounted at /data inherit this directory's ownership. +COPY --from=build --chown=65532:65532 /out/data /data +USER 65532:65532 +ENTRYPOINT ["/ollama-gateway"] +CMD ["-config", "/etc/ollama-gateway/config.json"] + +# The binary contains its own HTTP probe so the scratch runtime needs no shell/curl. +HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 CMD ["/ollama-gateway", "-probe", "http://127.0.0.1:8080/healthz", "-probe-timeout", "2s"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..35e9e78 --- /dev/null +++ b/Makefile @@ -0,0 +1,50 @@ +BINARY=ollama-gateway + +.PHONY: test build build-mac build-worker-telemetry worker-telemetry bench mock-ollama ha-report ha-snapshot ha-sampler ha-readiness fmt vet clean + +test: + go test ./... + +fmt: + gofmt -w $$(find . -name '*.go' -type f) + +vet: + go vet ./... + +build: + CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o dist/$(BINARY) ./cmd/ollama-gateway + +build-mac: + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/$(BINARY)-darwin-arm64 ./cmd/ollama-gateway + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-bench-darwin-arm64 ./cmd/bench + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-ha-report-darwin-arm64 ./cmd/ha-report + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-ha-snapshot-darwin-arm64 ./cmd/ha-snapshot + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-ha-sampler-darwin-arm64 ./cmd/ha-sampler + +build-worker-telemetry: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-worker-telemetry-linux-amd64 ./cmd/worker-telemetry + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o dist/ollama-gateway-worker-telemetry-linux-arm64 ./cmd/worker-telemetry + +worker-telemetry: + go run ./cmd/worker-telemetry -once + +bench: + go run ./cmd/bench -base-url http://127.0.0.1:8080 -concurrency 4 -requests 20 + +mock-ollama: + go run ./cmd/mock-ollama -listen 127.0.0.1:11435 + +ha-report: + go run ./cmd/ha-report -input "$${OUT_DIR:?set OUT_DIR to a completed readiness directory}" + +ha-snapshot: + go run ./cmd/ha-snapshot -pid "$${GATEWAY_PID:?set GATEWAY_PID to the gateway process PID}" + +ha-sampler: + go run ./cmd/ha-sampler -pid "$${GATEWAY_PID:?set GATEWAY_PID to the gateway process PID}" -stop-file "$${STOP_FILE:?set STOP_FILE to a path that will be created to stop sampling}" + +ha-readiness: + ./scripts/ha-readiness.sh + +clean: + rm -f dist/* diff --git a/README.md b/README.md index b1eaae6..85ee405 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,604 @@ -# og +# Ollama Fair Gateway +A high-performance Go gateway in front of one or more Ollama workers. The gateway exposes the native Ollama API and Ollama's OpenAI-compatible API while adding authentication, hierarchical fair scheduling, compute-credit quotas, usage metering, model-aware routing, optional durable batch execution and an embedded operations UI. + +The inference coordination engine remains **100% in-memory**, while restart-worthy control-plane state is persisted to local files under `storage.data_dir`. There is no Redis client, SQL/database server, distributed lease system or external message bus. The project uses only the Go standard library and builds as a static binary with `CGO_ENABLED=0`. + +## Architecture + +```text +Clients / Applications + | + v ++-------------------------------+ +| Ollama Fair Gateway | +| | +| OIDC / API Keys / IP bypass | +| Admission + compute credits | +| Tenant -> Actor WFQ | +| In-memory worker routing | +| Live request event bus | +| Usage aggregation | +| Durable local state | +| Embedded Web UI | ++---------------+---------------+ + | + +--------+---------+ + | | | + v v v + Ollama A Ollama B Ollama C +``` + +All fairness decisions are therefore made from one authoritative in-process state. Multiple Ollama workers are supported. If strict global fairness is required, all clients should enter through this one gateway process. + +## Main features + +- Native Ollama `/api/*` passthrough, including streaming and large management/blob requests. +- OpenAI-compatible `/v1/*` passthrough via Ollama's own compatibility layer. +- OIDC/JWT validation using discovery and JWKS. +- Browser OIDC Authorization Code + PKCE. +- Static configuration API keys plus persistent UI-created API-key creation/revocation from the admin UI; only SHA-256 key hashes are stored. +- CIDR-based authentication bypass for trusted IPs. +- Safe `X-Forwarded-For` handling behind explicitly trusted proxies. +- Two-level hierarchical weighted fair queueing: tenant fairness first, actor/application fairness inside each tenant. +- Per-actor and per-tenant compute-credit token buckets. +- Model-dependent cost estimation and reconciliation using actual Ollama usage data. +- Capability-aware preflight using Ollama `/api/show` (`tools`, `vision`, `thinking`, `embedding`, completion) plus context-window guard. +- Model-affinity and adaptive routing across configured Ollama workers using load, learned tok/s, GPU utilization and VRAM pressure. +- Worker circuit breaker with safe pre-stream transport retries, persistent drain/disable maintenance state and manual circuit reset in the admin UI. +- Virtual model aliases (for example `fast` or `coding`) with ordered physical-model fallback and optional capability requirements. +- Model ACLs at tenant level and per API key; denied models are removed from discovery and rejected before scheduling. +- Persistent per-worker **Model Placement** rules (allow-all/whitelist + exact/prefix allow/deny patterns) enforced before adaptive routing, with an interactive model × worker matrix in the admin UI. +- Per-model concurrency limits with exact and trailing-prefix-wildcard matching. +- Optional built-in NVIDIA telemetry through `nvidia-smi` without CGO or an exporter; native host-memory telemetry is available on Linux, macOS and Windows. +- Hard per-worker concurrency using lock-free atomic counters. +- Live request lifecycle tracking: queued -> routing -> running -> streaming -> completed/failed/cancelled. +- Optional durable batch jobs with owner-scoped APIs, admin pause/resume/cancel/output controls, restart recovery, retention and execution through the normal quota/scheduler/routing/accounting pipeline using service class `batch`. +- Embedded Canvas pulse visualization and local LLM infrastructure map. +- Worker/model inventory, pull/stop/delete operations and persistent live policy overrides. +- Prometheus-compatible metrics with durable counter/histogram snapshots. +- Persistent quota-bucket snapshots, preventing a normal restart from resetting tenant/actor burst budgets. +- Asynchronous JSONL usage journal with startup replay, so dashboard usage totals and recent history survive restarts. +- Atomically persisted UI configuration override loaded on the next process start. +- No prompt or generated response content is stored by the telemetry/usage layer. Content persistence exists only in explicit opt-in features such as encrypted Responses conversations and durable batch payload spooling. + +## Runtime vs durable state + +The hot-path scheduler remains in memory: + +```text +Fair queue / virtual clocks +Running concurrency +Per-worker active slots +OIDC browser sessions +Live request registry +Infrastructure topology +Model operation/transient inference-job state +``` + +These objects are intentionally not restored because requests and sockets cannot survive a process restart. Durable control-plane/accounting state is stored locally instead: + +```text +/gateway-config.json UI configuration override +/api-keys.json UI-created API-key hashes + metadata +/policies.json tenant policy overrides +/metrics.json Prometheus counter/histogram state +/quota.json actor/tenant credit bucket state +/worker-performance.json learned per-worker/model tok/s routing state +/model-placement.json persistent live worker/model placement overrides +/worker-state.json persistent worker drain/disable state +/conversations.enc.json optional encrypted Responses context +/batch-jobs.json optional durable batch metadata/state +/batch/input/* optional durable batch request payloads +/batch/output/* optional durable batch response payloads +/usage/usage-*.jsonl completed request accounting/history +``` + +State-file writes use temp-file + fsync + atomic rename. Usage remains append-only and is replayed at startup. None of these files is consulted for every inference token or proxy chunk, so durable storage is not in the inference hot path. Use a persistent filesystem and preferably an absolute `storage.data_dir` in production. + +Browser OIDC sessions remain intentionally volatile; after a gateway restart a browser logs in again. Transient inference jobs that were running at restart are disconnected and are not resurrected. Durable batch definitions survive; an interrupted batch attempt is recovered to a restart-safe queued state and can execute again. + +The supplied `docker-compose.yml` mounts a named volume at `/data`, so the default relative `./data` state directory is durable across container replacement. If you run the image differently, mount persistent storage at `/data` or set an absolute `storage.data_dir`. The Compose file accepts `GATEWAY_CONFIG=./your-config.json` to select the bootstrap config; if omitted it intentionally uses `config.example.json` for development. Before a production rollout, run the candidate binary/container with `-check-config` and see `docs/DEPLOYMENT-HARDENING.md`. + +## Persistent storage configuration + +```json +"storage": { + "data_dir": "./data", + "config_file": "gateway-config.json", + "api_keys_file": "api-keys.json", + "policies_file": "policies.json", + "metrics_file": "metrics.json", + "quota_file": "quota.json", + "worker_performance_file": "worker-performance.json", + "model_placement_file": "model-placement.json", + "worker_state_file": "worker-state.json", + "auto_tune_file": "auto-tune.json", + "warm_models_file": "warm-models.json", + "alerts_file": "alerts.json", + "conversations_file": "conversations.enc.json", + "batch_jobs_file": "batch-jobs.json", + "batch_jobs_dir": "batch", + "flush_interval": "10s" +} +``` + +`storage` is bootstrap-only: edit it in the startup configuration, not through the web JSON editor. Other configuration changes saved in **Admin -> Konfiguration** are validated and written to `/gateway-config.json`; that file becomes the effective configuration at the next start. Delete the persistent override from the UI (or remove the file offline) to fall back to the bootstrap configuration. + +**Admin -> Persistenz** shows the durable files, tiered usage-retention footprint and last compaction, can force an immediate flush or retention compaction, and can download a ZIP backup containing state, detail journals and rollups. + +## Reliability, maintenance, aliases and model ACLs + +The P0 reliability layer is configured independently from routing scores: + +```json +"reliability": { + "enabled": true, + "failure_threshold": 3, + "open_duration": "30s", + "retry_attempts": 2, + "retry_backoff": "50ms" +} +``` + +A retry is attempted only when the upstream transport fails **before any response is committed to the client**. Once headers/body are visible, the response is never replayed. Backend 5xx responses still contribute to the worker circuit breaker but are not retried after commitment. + +Workers can be placed into `active`, `draining`, or `disabled` state from **Admin -> Worker**. Draining rejects new work while existing streams finish; the state survives restart in `worker-state.json`. + +Virtual model aliases expose stable names to clients: + +```json +"model_aliases": { + "fast": { "models": ["qwen3:8b", "gemma3:12b"] }, + "coding": { + "models": ["qwen3.6:27b-q4_K_M", "qwen3:8b"], + "required_capabilities": ["completion"] + } +} +``` + +Aliases appear in `/api/tags` and `/v1/models`; the gateway resolves them to the first currently routable target. The response includes `X-Gateway-Model-Alias` and `X-Gateway-Resolved-Model`. + +Model access is separate from worker placement: + +```json +"model_access": { + "default": { "mode": "allow_all" }, + "tenants": { + "interns": { + "mode": "whitelist", + "allowed_models": ["fast", "qwen3:8b"], + "denied_models": ["expensive-*"] + } + } +} +``` + +API keys created in the web UI can additionally define their own allow/deny model patterns. API-key ACLs take precedence over the tenant baseline for that identity. Tenant baseline ACLs can also be created, edited and reset at runtime from **Fairness & Quotas → Tenant Model Access**; changes are persisted before immediate publication. + +- Runtime model-alias CRUD is available from the admin model page and is persisted before immediate publication to new requests. + +For production rollout and rollback steps, see `docs/PRODUCTION-UPDATE.md`; the current release validation is recorded in `docs/RELEASE-VERIFICATION-CHECKPOINT-28.md`. + +See `docs/ROADMAP-P0-P3.md` for the staged plan through P3. Checkpoint 28 completes the active P3 roadmap with the optional public read-only status dashboard. Multi-gateway HA has been removed from the active roadmap; the existing HA-readiness tools remain available as diagnostics if future capacity or availability requirements change. + +## Build + +Requires Go 1.23 or newer. + +```bash +go test ./... +go vet ./... +CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" \ + -o dist/ollama-gateway ./cmd/ollama-gateway +``` + +For Apple Silicon: + +```bash +CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" \ + -o dist/ollama-gateway-darwin-arm64 ./cmd/ollama-gateway +``` + +## Quick start + +Copy the example configuration: + +```bash +cp config.example.json config.json +./dist/ollama-gateway-darwin-arm64 -config ./config.json +``` + +The default example expects Ollama on `http://127.0.0.1:11434` and exposes the gateway on port `8080`. + +Native Ollama example: + +```bash +curl http://127.0.0.1:8080/api/chat \ + -H 'Content-Type: application/json' \ + -d '{"model":"qwen3:8b","messages":[{"role":"user","content":"hello"}]}' +``` + +OpenAI-compatible example: + +```bash +curl http://127.0.0.1:8080/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"qwen3:8b","messages":[{"role":"user","content":"hello"}]}' +``` + +The example configuration grants localhost an admin IP bypass. Replace that for production. + +## OpenWebUI + +Use the gateway as an **Ollama API connection**, not as a raw browser-side URL. Current OpenWebUI performs model discovery with `GET /api/tags` and connection verification with `GET /api/version`; when a connection key is configured it sends that key as `Authorization: Bearer `. The gateway accepts both configuration API keys and keys created under **Admin -> Sicherheit -> API Keys**. UI-created keys are persisted as hashes, so OpenWebUI keeps working with the same key across gateway restarts. + +A ready-to-use configuration is included as `config.openwebui.example.json`. Set a secret before starting the gateway: + +```bash +export OPENWEBUI_GATEWAY_KEY='replace-with-a-long-random-secret' +./dist/ollama-gateway-darwin-arm64 -config ./config.openwebui.example.json +``` + +In OpenWebUI go to **Settings -> Admin -> Connections -> Manage Ollama API Connections** and use: + +```text +URL: http://host.docker.internal:8080 +API Key: the value of OPENWEBUI_GATEWAY_KEY +``` + +Do **not** append `/api` to the Ollama base URL; OpenWebUI appends `/api/version`, `/api/tags`, `/api/show`, and the inference paths itself. When OpenWebUI runs outside Docker, use the hostname/IP at which its backend can actually reach the gateway. `localhost` inside the OpenWebUI container refers to that container, not the Mac host. + +Verify the path independently before configuring OpenWebUI: + +```bash +curl -sS -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" \ + http://127.0.0.1:8080/api/version + +curl -sS -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" \ + http://127.0.0.1:8080/api/tags +``` + +For OpenAI-compatible connections, use `http://host.docker.internal:8080/v1` with the same key. `/v1/models` is generated from the same aggregated worker inventory as `/api/tags`. + +Model discovery is gateway-native rather than a control-worker passthrough: `/api/tags` queries all configured Ollama workers in parallel, de-duplicates model IDs and normalizes both `name` and `model`. `/api/ps` is also aggregated. The worker router tracks installed models from `/api/tags` in addition to loaded models from `/api/ps`, preventing a selected model from being routed to a worker that does not have it installed. + +If OpenWebUI is in Docker, relying on the default localhost IP bypass will normally fail because the TCP peer seen by the gateway is the Docker/host-network address rather than `127.0.0.1`. Prefer the dedicated static API key above instead of widening the bypass CIDR. + +## Scheduling and fairness + +A compute request is assigned an estimated credit cost before it enters the queue. The scheduler uses a two-level weighted fair queue: + +1. tenants compete according to `tenant_weight`; +2. inside a tenant, actors compete according to `actor_weight` and request cost. + +This prevents one tenant from gaining extra global capacity merely by creating many users, while also preventing one heavy actor from blocking all other actors in the same tenant. + +Key settings: + +```json +"scheduler": { + "global_concurrency": 4, + "max_queue": 2048, + "max_queue_per_actor": 64, + "queue_timeout": "10m", + "default_tenant_weight": 1, + "default_actor_weight": 1 +} +``` + +`global_concurrency` should normally be close to the total useful inference concurrency of the configured workers. Worker-level `max_concurrent` remains a second hard limit. + +## Compute credits and quotas + +Each tenant policy can limit long-term consumption: + +```json +"*": { + "tenant_weight": 1, + "actor_weight": 1, + "actor_credits_per_minute": 60, + "actor_burst_credits": 180, + "tenant_credits_per_minute": 300, + "tenant_burst_credits": 900 +} +``` + +The gateway reserves estimated credits before admission. After Ollama finishes, the reservation is reconciled with actual usage. Native Ollama responses expose prompt/completion token counts and evaluation timings, allowing the estimator to incorporate model-specific token and compute weights. + +Policy overrides made in the web UI take effect immediately for new requests and are persisted in `policies.json`, so they survive restarts. The baseline `scheduler.policies` in configuration remains the fallback. + +## Worker routing + +Workers are polled using Ollama `/api/ps` and `/api/tags`. Installed-model ownership is used as a routing constraint when known; a worker that already has the requested model loaded receives an additional strong affinity preference, while its current active-slot ratio increases its routing score. + +### Model Placement / worker whitelist + +Placement is a **hard constraint before routing score calculation**. It answers "where may this model run?" while tenant policies answer "who may consume how much?". They are intentionally separate. + +Static worker defaults can be declared in `config.json`: + +```json +{ + "name": "rtx-4090", + "url": "http://10.10.11.124:11434", + "max_concurrent": 2, + "model_placement": { + "mode": "whitelist", + "allowed_models": [ + "qwen3:8b", + "gemma4:*", + "embeddinggemma:*" + ], + "denied_models": [ + "gemma4:e4b" + ] + } +} +``` + +`mode: "allow_all"` allows models by default; `mode: "whitelist"` blocks models unless an allow rule matches. Patterns are exact names, `*`, or a single trailing wildcard such as `gemma4:*`. The most specific rule wins; exact rules beat prefix rules and deny wins ties. This permits useful exceptions such as denying `gemma4:*` while explicitly allowing `gemma4:latest`. + +The admin UI exposes this under **Model Placement** as a model × worker matrix. A cell shows whether the model is allowed, installed and loaded. Clicking a cell creates/removes an exact persistent exception; the worker editor manages broad patterns and offers presets for **allow all**, **only installed**, and **block all**. UI overrides are written atomically to `/model-placement.json` and affect new requests immediately without a gateway restart. Resetting a worker restores its `config.json` baseline. + +Client-facing `/api/tags` and `/v1/models` only expose models installed on at least one worker on which placement permits them, so OpenWebUI does not offer a model that the gateway itself would refuse to route. Model-routed control requests such as `/api/show` use the same placement eligibility and cannot bypass it through the control worker. + +See [`docs/MODEL-PLACEMENT.md`](docs/MODEL-PLACEMENT.md) for operational examples and UI semantics. + +A two-worker M5-Ultra + RTX-4090 example using the model names from the development setup is included as `config.placement.example.json`. + +```json +"workers": [ + { + "name": "mac-studio-m5-ultra", + "url": "http://127.0.0.1:11434", + "max_concurrent": 4, + "health_interval": "5s", + "local_system_stats": true + } +] +``` + +The gateway's worker concurrency is enforced entirely with atomic counters in the process. No lease renewal or external slot backend is involved. + +`local_system_stats` can expose host RAM data for a **local** worker. For remote workers, leave it false and use `telemetry_url`; the shipped `cmd/worker-telemetry` agent can export host RAM plus NVIDIA or Linux AMDGPU sysfs metrics. See `docs/WORKER-TELEMETRY.md`. Ollama `size_vram` is shown as resident model footprint, not as GPU-utilization percentage. + +## Authentication + +Three API authentication methods can coexist: + +- OIDC bearer tokens; +- API keys from configuration and persistent UI-created API keys; +- trusted-IP bypass. + +Administrators can create and revoke keys in the embedded UI. The plaintext secret is returned exactly once at creation time; only its SHA-256 digest and non-secret metadata are written to the durable API-key store. Static keys loaded from the bootstrap configuration are shown as read-only and cannot be deleted from the UI. + +For IP bypass, the TCP peer is authoritative by default. `auth.trusted_proxies` still controls forwarded client-IP resolution for observability, but a forwarded address cannot satisfy credential-free IP bypass unless `auth.ip_bypass_use_forwarded_ip=true` is explicitly enabled. Keep bypass and proxy networks narrow. + +See `config.oidc.example.json` for a browser-OIDC configuration. + +## Web control plane + +Enable with: + +```json +"ui": { + "enabled": true, + "path": "/admin", + "title": "Ollama Fair Gateway" +} +``` + +The UI is embedded in the Go binary and has no CDN, npm or runtime frontend dependency. + +Important pages include: + +- Overview: queue, running requests, usage and workers. +- Live Flow: animated request lifecycle visualization. +- Infrastructure: local in-memory topology Tenant -> Gateway -> Queue -> Worker -> Model. +- Workers: health, active/per-model slots, GPU/VRAM telemetry and learned per-model token throughput. +- Models: inventory, detected capabilities/context length and administrative pull/stop/delete actions. **Stop** unloads a resident Ollama model using `POST /api/generate` with `keep_alive: 0`; it is not a request-cancellation mechanism. +- Model Placement: persistent worker whitelists/deny rules, current inventory/loaded state and one-click exact routing exceptions. +- Jobs: active queued/running/streaming inference requests with administrator-triggered cancellation. +- Fairness: scheduler limits, usage and persistent policy overrides. +- Usage: recent request metadata and token/credit accounting. +- Security: OIDC, API-key creation/revocation, static keys, IP bypass and trusted proxy configuration. +- Config: validated JSON editor for a persistent configuration override. Saves are atomic and take full effect after restart; storage-path changes remain bootstrap-only. + +Live metadata is delivered to the browser over authenticated Server-Sent Events. Animation is rendered with Canvas in the browser; the server sends only bounded metadata snapshots. + + +## Job cancellation + +Administrators can cancel active compute requests from **Admin -> Jobs** or directly from the **Live Flow** request list. The control-plane endpoints are: + +```text +GET /gateway/ui-api/jobs +POST /gateway/ui-api/jobs//cancel +``` + +Cancellation is implemented with the Go request context rather than a second Ollama-specific control request: + +- a **queued** job is removed from the fair-scheduler heap immediately and its reserved credits are released; +- a job waiting for a worker slot is cancelled immediately; +- a **running/streaming** job cancels the upstream Ollama HTTP context, releases the worker and scheduler slots, and reconciles any usage already observable from the response; +- live/accounting metadata records cancelled jobs with internal status `499`. If Ollama has already emitted HTTP `200` for a streaming response, that wire status cannot be changed; the client instead observes the stream ending early while the gateway records `499` internally. + +Model **Stop** and job **Cancel** are intentionally different operations. Model Stop only unloads a resident model. Job Cancel terminates a specific inference request. + +## Browser OIDC sessions + +After Authorization Code + PKCE login, the browser receives an opaque session identifier. The OIDC access token remains in the gateway's in-memory session store. Session cookies are HttpOnly and state-changing cookie-authenticated UI requests require CSRF protection. + +Because sessions are in memory, a gateway restart requires browser users to log in again. This is intentional in the single-process design. + +## Infrastructure API + +Admin-only local infrastructure endpoints: + +```text +GET /gateway/ui-api/infrastructure +GET /gateway/ui-api/infrastructure/stream +``` + +The snapshot contains one gateway process, all configured workers, loaded models and bounded live request metadata. It never contains prompt text or generated output. + +## Usage + +Gateway API endpoints: + +```text +GET /gateway/v1/usage/me +GET /gateway/v1/status +GET /gateway/v1/usage/tenant?tenant= # admin +``` + +Completed usage events are appended asynchronously to daily JSONL files. Tiered retention keeps request-level detail for a bounded window, then replaces it with dimension rollups so long-running gateways do not grow an unbounded journal: + +```json +"usage": { + "journal_dir": "./data/usage", + "buffer": 16384, + "flush_interval": "1s", + "retention": { + "detail_days": 30, + "daily_days": 400, + "monthly_months": 0, + "compaction_interval": "6h" + } +} +``` + +With these defaults, the last 30 days remain request-addressable JSONL detail, days 31–400 remain daily aggregates, and older data is folded into monthly aggregates indefinitely (`monthly_months: 0`). Rollups preserve request/error counts, tokens, credits, queue/service time, byte counts and prompt/output evaluation time for global, tenant, actor, application, model and worker dimensions. Performance (`tok/s`) is therefore still available after detail rows are removed. + +Historical UI/API queries are available through `GET /gateway/ui-api/usage/rollups?granularity=daily|monthly&dimension=...`. **Admin -> Persistenz -> Jetzt kompaktieren** triggers the same idempotent compactor manually. The supplied configurations use `./data/usage`; keep it on persistent storage if accounting history must survive container/host replacement. + +## Model capability preflight + +By default the gateway caches Ollama `/api/show` metadata for ten minutes and rejects known-incompatible capability requests before they consume a scheduler/worker slot. This produces a clear gateway error for cases such as a `tools` request sent to a completion-only model. Metadata lookup failure itself is fail-open. + +```json +"model_capabilities": { + "mode": "enforce", + "cache_ttl": "10m", + "context_guard": "reject", + "context": { + "max_requested_tokens": 32768, + "default_worker_tokens": 4096, + "estimation_margin_percent": 15, + "vision_reserve_tokens_per_image": 2048 + } +} +``` + +For heterogeneous workers, `default_context_tokens` describes that worker's effective Ollama default when a model is neither loaded nor configured with a Modelfile `num_ctx`. `context_limits` is a hard per-worker/model ceiling using the same exact/prefix/`*` matching style as model concurrency: + +```json +{ + "name": "gpu-1", + "default_context_tokens": 4096, + "context_limits": { + "qwen3:*": 16384, + "*": 32768 + } +} +``` + +The model table exposes **Model max**, **Configured** (Modelfile `num_ctx`) and **Loaded** (`/api/ps`) separately. Requests without a native `num_ctx` are routed only to workers whose effective context can hold the estimated request. + +## Adaptive routing and per-model concurrency + +Worker routing can combine model affinity with live GPU/VRAM telemetry and a per-worker/per-model EWMA of observed token throughput. Per-model limits prevent a large model from consuming the same concurrency as a much smaller model. + +```json +"model_concurrency": { + "qwen3:8b": 2, + "gemma4:*": 1, + "*": 1 +} +``` + +On NVIDIA Linux or Windows hosts, `"nvidia_smi": true` enables dependency-free GPU utilization, VRAM, temperature and power telemetry. Windows host RAM uses `GlobalMemoryStatusEx`; Linux uses `/proc/meminfo`. + +## Prometheus metrics + +`/metrics` exposes bounded-cardinality metrics for queue/running state, worker slots, request totals, GPU/VRAM telemetry, per-model active requests, learned token throughput, dropped usage-journal events and raw/daily/monthly retention file/byte gauges plus last-compaction timestamp/reclaimed-byte gauges. Tenant/user identifiers are deliberately not emitted as Prometheus labels. + +The exporter uses a **snapshot-then-render** path: registry/scheduler/worker locks are held only while copying state and are released before any HTTP response bytes are written. A slow or stalled Prometheus client therefore cannot block request completion or the inference hot path. + +## Operational model + +The design intentionally optimizes for one strong gateway process. Go can handle far more concurrent HTTP/streaming connections than a local LLM server can generally infer at once, so the GPU/model execution layer should become the bottleneck long before the gateway. + +If the gateway process is replicated independently behind a generic load balancer, each replica has its own queue, quotas and sessions. That setup does **not** provide strict global fairness. For strict fairness, run one authoritative gateway and scale Ollama workers behind it. + +## Tests + +```bash +node --check internal/webui/assets/app.js +go test ./... +go vet ./... +go test -race ./... +``` + +See also `docs/ARCHITECTURE.md`, `docs/SECURITY.md` and `docs/IMPROVEMENTS-2026.md`. A tuned RTX 4090 example is available as `config.rtx4090.example.json`. + +Multi-gateway HA is not part of the active roadmap after single-process performance proved sufficient for the current deployment. The deterministic benchmark and evidence tooling in `docs/HA-READINESS.md` is retained for future re-evaluation rather than removed. + +### OpenWebUI: "Modelle konnten nicht abgerufen werden" + +If OpenWebUI uses an Ollama connection with **Authentication: None**, its backend source IP must match an `auth.ip_bypass` CIDR. The default examples only bypass `127.0.0.1`/`::1`; Docker or another machine is therefore intentionally rejected. Authentication failures are logged with `client_ip=...` so an exact `/32` can be added. See `docs/OPENWEBUI.md`. + +## P2/P3 operations and durable work + +The current gateway includes the P2 operator/state features: + +- **Warm Models**: persistent `hot`/`warm`/`cold` residency policies with placement-aware preload, idle unload, model-operation locking, and VRAM eviction suggestions. See `docs/WARM-MODELS.md`. +- **Alerts**: persistent firing/resolved events for worker health, circuit state, queue depth/wait, VRAM, OOM, storage, and quota thresholds. Generic HMAC-signed webhooks use bounded asynchronous delivery and transient retries. See `docs/ALERTS.md`. +- **Optional Responses conversations**: disabled by default; when enabled, `previous_response_id` is resolved from an AES-256-GCM encrypted, retention-bounded tenant/actor-scoped store. `store:false` remains a per-request opt-out. See `docs/CONVERSATIONS.md`. +- **Policy simulator**: evaluates identity/model/capability/placement/routing decisions without running inference. +- **Durable Batch Jobs (P3.1)**: disabled by default; accepted compute requests can be persisted, executed later under service class `batch`, paused/resumed/cancelled, recovered after restart, retained for a bounded window and inspected in the dedicated Admin -> Batch Jobs page. See `docs/BATCH-JOBS.md`. + +Warm-model actions never override Model Placement or worker drain/disable state. Model unload/preload acquires a per-worker/per-model maintenance reservation so inference cannot race an unload operation. + +## Hardened production Compose + +For production Docker deployments, use `docker-compose.production.yml` with an explicit bootstrap config: + +```sh +cp .env.production.example .env +# edit .env so GATEWAY_CONFIG points at the production JSON +./scripts/production-preflight.sh +docker compose -f docker-compose.production.yml up -d --build +``` + +The production Compose publishes `127.0.0.1:9080` by default, requires `GATEWAY_CONFIG`, runs the scratch image as UID/GID 65532 with a read-only root filesystem, drops all capabilities and enables `no-new-privileges`. See `docs/DEPLOYMENT-HARDENING.md` before using a non-loopback bind. + +## Admin UI theme + +Checkpoint 24 ships the Admin Control Plane as a light, data-dense technical dashboard. The theme uses explicit light browser controls, high-contrast status colors, compact mono-accent typography, light tables/forms/dialogs and matching light Live Flow/Infrastructure canvases. It has no external font/CDN dependency: preferred local IBM Plex/JetBrains/Cascadia faces fall back to platform fonts automatically. + +The UI is embedded into the gateway binary. After upgrading from an earlier checkpoint, rebuild/redeploy the gateway image and perform one hard browser refresh if an old stylesheet is still cached. No gateway configuration or persistent-state migration is required for the theme update. + + +## Checkpoint 25 + +- Fixed sidebar navigation overflow on shorter viewport heights. +- Sidebar navigation now scrolls independently and keeps the footer visible. +- Added compact spacing rules for shorter screens. + +## Checkpoint 26 — Dialog close validation fix + +- Fixes **Tenant Policies → Override anlegen**: both the × button and **Abbrechen** now close the modal even while required fields are empty. +- Close controls are explicit `type="button"` actions using the shared `data-close-dialog` handler; only **Speichern** submits the policy form. +- Applies the same validation-independent close behavior to the existing **Modell pullen** dialog. +- Adds WebUI regression tests for close-target validity and required-field dialogs. +- No scheduler, proxy, auth, persistence, routing, quota, or inference behavior changed. + +## Checkpoint 27 — Context / num_ctx Hardening + +See `docs/CONTEXT-HARDENING.md`. The gateway now derives effective worker context from loaded `/api/ps`, Modelfile `num_ctx`, worker defaults and hard context caps; it also fixes Responses/output/instructions/suffix/vision context estimation. + +## Public status dashboard + +Checkpoint 28 adds an optional read-only public dashboard at `/status/`. It shows current queue/load, worker capacity, an infrastructure map and anonymized live request activity without requiring an API key. The public API is built from a separate allow-list and does not expose tenant/actor/application identities, worker URLs, policies, quotas, storage paths or error details. Worker and model names are anonymized by default and can be explicitly published or mapped to public display names. See `docs/PUBLIC-DASHBOARD.md`. diff --git a/cmd/bench/main.go b/cmd/bench/main.go new file mode 100644 index 0000000..103f6ff --- /dev/null +++ b/cmd/bench/main.go @@ -0,0 +1,339 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/http/httptrace" + "os" + "sort" + "sync" + "time" +) + +type result struct { + Latency time.Duration + TTFB time.Duration + Prompt int64 + Completion int64 + Bytes int64 + Status int + Err error +} + +type durationStats struct { + P50 float64 `json:"p50_ms"` + P95 float64 `json:"p95_ms"` + P99 float64 `json:"p99_ms"` + Max float64 `json:"max_ms"` +} + +type summary struct { + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + BaseURL string `json:"base_url"` + Model string `json:"model"` + Concurrency int `json:"concurrency"` + Requests int `json:"requests"` + WarmupRequests int `json:"warmup_requests"` + Successful int `json:"successful"` + Errors int `json:"errors"` + StatusCounts map[string]int `json:"status_counts"` + WallMS float64 `json:"wall_ms"` + ThroughputRPS float64 `json:"throughput_rps"` + Latency durationStats `json:"latency"` + TTFB durationStats `json:"ttfb"` + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + CompletionTokPS float64 `json:"completion_tokens_per_second"` + BytesReceived int64 `json:"bytes_received"` + BytesPerSecond float64 `json:"bytes_per_second"` + Stream bool `json:"stream"` + KeepAlive bool `json:"keep_alive"` + ServiceClass string `json:"service_class,omitempty"` + ErrorsByMessage map[string]int `json:"errors_by_message,omitempty"` +} + +type benchConfig struct { + Base string + Key string + Model string + Concurrency int + Requests int + Warmup int + MaxTokens int + Prompt string + Timeout time.Duration + Stream bool + DisableKeepAlive bool + ServiceClass string + JSONOut string +} + +func main() { + base := flag.String("base-url", "http://127.0.0.1:8080", "gateway base URL") + key := flag.String("api-key", "", "API key; can also use GATEWAY_BENCH_API_KEY") + model := flag.String("model", "qwen3:8b", "model name") + conc := flag.Int("concurrency", 4, "parallel clients") + n := flag.Int("requests", 20, "total measured requests") + warmup := flag.Int("warmup", 0, "warmup requests before measurement") + maxTokens := flag.Int("max-tokens", 128, "max completion tokens") + prompt := flag.String("prompt", "Explain in three concise paragraphs why fair scheduling matters for shared LLM inference.", "prompt") + timeout := flag.Duration("timeout", 2*time.Minute, "per-request HTTP timeout") + stream := flag.Bool("stream", false, "request OpenAI streaming responses") + disableKeepAlive := flag.Bool("disable-keepalive", false, "disable HTTP connection reuse") + serviceClass := flag.String("service-class", "", "optional X-Gateway-Service-Class override") + jsonOut := flag.String("json-out", "", "optional path for machine-readable JSON summary; '-' writes JSON to stdout") + flag.Parse() + if *key == "" { + *key = os.Getenv("GATEWAY_BENCH_API_KEY") + } + cfg := benchConfig{Base: *base, Key: *key, Model: *model, Concurrency: *conc, Requests: *n, Warmup: *warmup, MaxTokens: *maxTokens, Prompt: *prompt, Timeout: *timeout, Stream: *stream, DisableKeepAlive: *disableKeepAlive, ServiceClass: *serviceClass, JSONOut: *jsonOut} + if cfg.Concurrency < 1 || cfg.Requests < 1 || cfg.Warmup < 0 { + fmt.Fprintln(os.Stderr, "concurrency and requests must be positive; warmup must be non-negative") + os.Exit(2) + } + if cfg.Timeout <= 0 { + fmt.Fprintln(os.Stderr, "timeout must be positive") + os.Exit(2) + } + + client := newClient(cfg) + body, err := requestBody(cfg) + if err != nil { + fmt.Fprintln(os.Stderr, "request body:", err) + os.Exit(2) + } + ctx := context.Background() + if cfg.Warmup > 0 { + warm := run(ctx, client, cfg, body, cfg.Warmup) + for _, r := range warm { + if r.Err != nil { + fmt.Fprintln(os.Stderr, "warmup error:", r.Err) + } + } + } + + started := time.Now().UTC() + results := run(ctx, client, cfg, body, cfg.Requests) + finished := time.Now().UTC() + s := summarize(cfg, started, finished, results) + printHuman(s) + if cfg.JSONOut != "" { + if err := writeJSONSummary(cfg.JSONOut, s); err != nil { + fmt.Fprintln(os.Stderr, "write JSON summary:", err) + os.Exit(1) + } + } + if s.Successful == 0 { + os.Exit(1) + } +} + +func newClient(cfg benchConfig) *http.Client { + idle := max(256, cfg.Concurrency*2) + return &http.Client{ + Timeout: cfg.Timeout, + Transport: &http.Transport{ + MaxIdleConns: idle, + MaxIdleConnsPerHost: idle, + IdleConnTimeout: 90 * time.Second, + DisableKeepAlives: cfg.DisableKeepAlive, + }, + } +} + +func requestBody(cfg benchConfig) ([]byte, error) { + return json.Marshal(map[string]any{ + "model": cfg.Model, + "messages": []map[string]string{{"role": "user", "content": cfg.Prompt}}, + "max_tokens": cfg.MaxTokens, + "stream": cfg.Stream, + }) +} + +func run(ctx context.Context, client *http.Client, cfg benchConfig, body []byte, count int) []result { + jobs := make(chan struct{}) + results := make(chan result, count) + var wg sync.WaitGroup + for i := 0; i < cfg.Concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for range jobs { + results <- one(ctx, client, cfg, body) + } + }() + } + go func() { + for i := 0; i < count; i++ { + jobs <- struct{}{} + } + close(jobs) + wg.Wait() + close(results) + }() + out := make([]result, 0, count) + for r := range results { + out = append(out, r) + } + return out +} + +func one(ctx context.Context, client *http.Client, cfg benchConfig, body []byte) result { + started := time.Now() + var firstByte time.Time + trace := &httptrace.ClientTrace{GotFirstResponseByte: func() { firstByte = time.Now() }} + reqCtx := httptrace.WithClientTrace(ctx, trace) + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, stringsTrimRightSlash(cfg.Base)+"/v1/chat/completions", bytes.NewReader(body)) + if err != nil { + return result{Err: err} + } + req.Header.Set("Content-Type", "application/json") + if cfg.Key != "" { + req.Header.Set("Authorization", "Bearer "+cfg.Key) + } + if cfg.ServiceClass != "" { + req.Header.Set("X-Gateway-Service-Class", cfg.ServiceClass) + } + resp, err := client.Do(req) + if err != nil { + return result{Latency: time.Since(started), Err: err} + } + b, readErr := io.ReadAll(resp.Body) + closeErr := resp.Body.Close() + latency := time.Since(started) + ttfb := time.Duration(0) + if !firstByte.IsZero() { + ttfb = firstByte.Sub(started) + } + r := result{Latency: latency, TTFB: ttfb, Bytes: int64(len(b)), Status: resp.StatusCode} + if readErr != nil { + r.Err = readErr + return r + } + if closeErr != nil { + r.Err = closeErr + return r + } + if resp.StatusCode/100 != 2 { + r.Err = fmt.Errorf("HTTP %d: %s", resp.StatusCode, truncate(string(b), 240)) + return r + } + if !cfg.Stream { + var doc struct { + Usage struct { + Prompt int64 `json:"prompt_tokens"` + Completion int64 `json:"completion_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(b, &doc); err == nil { + r.Prompt = doc.Usage.Prompt + r.Completion = doc.Usage.Completion + } + } + return r +} + +func summarize(cfg benchConfig, started, finished time.Time, results []result) summary { + wall := finished.Sub(started) + latencies := make([]time.Duration, 0, len(results)) + ttfbs := make([]time.Duration, 0, len(results)) + statusCounts := map[string]int{} + errorsByMessage := map[string]int{} + var prompt, completion, received int64 + errs := 0 + for _, r := range results { + if r.Status != 0 { + statusCounts[fmt.Sprintf("%d", r.Status)]++ + } + if r.Err != nil { + errs++ + errorsByMessage[truncate(r.Err.Error(), 180)]++ + continue + } + latencies = append(latencies, r.Latency) + if r.TTFB > 0 { + ttfbs = append(ttfbs, r.TTFB) + } + prompt += r.Prompt + completion += r.Completion + received += r.Bytes + } + success := len(latencies) + seconds := wall.Seconds() + if seconds <= 0 { + seconds = 1e-9 + } + return summary{ + StartedAt: started, FinishedAt: finished, BaseURL: cfg.Base, Model: cfg.Model, + Concurrency: cfg.Concurrency, Requests: cfg.Requests, WarmupRequests: cfg.Warmup, + Successful: success, Errors: errs, StatusCounts: statusCounts, WallMS: float64(wall) / float64(time.Millisecond), + ThroughputRPS: float64(success) / seconds, Latency: stats(latencies), TTFB: stats(ttfbs), + PromptTokens: prompt, CompletionTokens: completion, CompletionTokPS: float64(completion) / seconds, + BytesReceived: received, BytesPerSecond: float64(received) / seconds, Stream: cfg.Stream, + KeepAlive: !cfg.DisableKeepAlive, ServiceClass: cfg.ServiceClass, ErrorsByMessage: errorsByMessage, + } +} + +func stats(ds []time.Duration) durationStats { + if len(ds) == 0 { + return durationStats{} + } + sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] }) + pct := func(p float64) time.Duration { + idx := int(float64(len(ds)-1) * p) + return ds[idx] + } + return durationStats{P50: msFloat(pct(.50)), P95: msFloat(pct(.95)), P99: msFloat(pct(.99)), Max: msFloat(ds[len(ds)-1])} +} + +func printHuman(s summary) { + fmt.Printf("successful=%d errors=%d concurrency=%d wall=%s keepalive=%t stream=%t\n", s.Successful, s.Errors, s.Concurrency, time.Duration(s.WallMS*float64(time.Millisecond)).Round(time.Millisecond), s.KeepAlive, s.Stream) + fmt.Printf("latency p50=%s p95=%s p99=%s max=%s\n", fmtMS(s.Latency.P50), fmtMS(s.Latency.P95), fmtMS(s.Latency.P99), fmtMS(s.Latency.Max)) + fmt.Printf("ttfb p50=%s p95=%s p99=%s max=%s\n", fmtMS(s.TTFB.P50), fmtMS(s.TTFB.P95), fmtMS(s.TTFB.P99), fmtMS(s.TTFB.Max)) + fmt.Printf("throughput=%.2f req/s bytes=%d bytes/s=%.0f prompt_tokens=%d completion_tokens=%d completion_tok/s=%.2f\n", s.ThroughputRPS, s.BytesReceived, s.BytesPerSecond, s.PromptTokens, s.CompletionTokens, s.CompletionTokPS) + if len(s.StatusCounts) > 0 { + b, _ := json.Marshal(s.StatusCounts) + fmt.Printf("status=%s\n", b) + } + if len(s.ErrorsByMessage) > 0 { + for msg, n := range s.ErrorsByMessage { + fmt.Fprintf(os.Stderr, "error x%d: %s\n", n, msg) + } + } +} + +func writeJSONSummary(path string, s summary) error { + b, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + if path == "-" { + _, err = os.Stdout.Write(b) + return err + } + return os.WriteFile(path, b, 0644) +} + +func msFloat(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) } +func fmtMS(v float64) string { + return (time.Duration(v * float64(time.Millisecond))).Round(time.Microsecond).String() +} +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} +func stringsTrimRightSlash(s string) string { + for len(s) > 0 && s[len(s)-1] == '/' { + s = s[:len(s)-1] + } + return s +} diff --git a/cmd/bench/main_test.go b/cmd/bench/main_test.go new file mode 100644 index 0000000..658732e --- /dev/null +++ b/cmd/bench/main_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestStats(t *testing.T) { + d := []time.Duration{10 * time.Millisecond, 50 * time.Millisecond, 20 * time.Millisecond, 40 * time.Millisecond, 30 * time.Millisecond} + s := stats(d) + if s.P50 != 30 || s.P95 != 40 || s.P99 != 40 || s.Max != 50 { + t.Fatalf("unexpected stats: %+v", s) + } +} + +func TestOneCapturesStatusUsageBytesAndHeaders(t *testing.T) { + var gotAuth, gotClass string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + gotClass = r.Header.Get("X-Gateway-Service-Class") + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"ok"}}],"usage":{"prompt_tokens":9,"completion_tokens":4}}`) + })) + defer ts.Close() + + cfg := benchConfig{Base: ts.URL + "/", Key: "secret", Model: "m", ServiceClass: "batch", Timeout: time.Second} + body, err := requestBody(cfg) + if err != nil { + t.Fatal(err) + } + r := one(context.Background(), newClient(cfg), cfg, body) + if r.Err != nil { + t.Fatal(r.Err) + } + if r.Status != http.StatusOK || r.Prompt != 9 || r.Completion != 4 || r.Bytes == 0 || r.Latency <= 0 || r.TTFB <= 0 { + t.Fatalf("unexpected result: %+v", r) + } + if gotAuth != "Bearer secret" || gotClass != "batch" { + t.Fatalf("headers auth=%q class=%q", gotAuth, gotClass) + } +} + +func TestSummarizeSeparatesErrors(t *testing.T) { + cfg := benchConfig{Base: "http://gateway", Model: "m", Concurrency: 2, Requests: 3, Warmup: 1} + start := time.Unix(1, 0).UTC() + finish := start.Add(time.Second) + s := summarize(cfg, start, finish, []result{ + {Latency: 10 * time.Millisecond, TTFB: 5 * time.Millisecond, Prompt: 8, Completion: 2, Bytes: 100, Status: 200}, + {Latency: 20 * time.Millisecond, TTFB: 7 * time.Millisecond, Prompt: 8, Completion: 3, Bytes: 110, Status: 200}, + {Latency: 2 * time.Millisecond, Status: 503, Err: fmt.Errorf("HTTP 503: unavailable")}, + }) + if s.Successful != 2 || s.Errors != 1 || s.StatusCounts["200"] != 2 || s.StatusCounts["503"] != 1 { + t.Fatalf("unexpected counts: %+v", s) + } + if s.ThroughputRPS != 2 || s.PromptTokens != 16 || s.CompletionTokens != 5 || s.BytesReceived != 210 { + t.Fatalf("unexpected throughput/usage: %+v", s) + } +} + +func TestStringsTrimRightSlash(t *testing.T) { + if got := stringsTrimRightSlash("http://x///"); got != "http://x" { + t.Fatalf("got %q", got) + } +} diff --git a/cmd/ha-report/main.go b/cmd/ha-report/main.go new file mode 100644 index 0000000..276d83e --- /dev/null +++ b/cmd/ha-report/main.go @@ -0,0 +1,396 @@ +package main + +import ( + "bufio" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/example/ollama-fair-gateway/internal/haresource" +) + +type durationStats struct { + P50 float64 `json:"p50_ms"` + P95 float64 `json:"p95_ms"` + P99 float64 `json:"p99_ms"` + Max float64 `json:"max_ms"` +} + +type benchSummary struct { + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + BaseURL string `json:"base_url"` + Model string `json:"model"` + Concurrency int `json:"concurrency"` + Requests int `json:"requests"` + WarmupRequests int `json:"warmup_requests"` + Successful int `json:"successful"` + Errors int `json:"errors"` + StatusCounts map[string]int `json:"status_counts"` + WallMS float64 `json:"wall_ms"` + ThroughputRPS float64 `json:"throughput_rps"` + Latency durationStats `json:"latency"` + TTFB durationStats `json:"ttfb"` + ServiceClass string `json:"service_class,omitempty"` + ErrorsByMessage map[string]int `json:"errors_by_message,omitempty"` +} + +type metricSample struct { + Name string + Labels string + Value float64 +} + +type resourceSnapshot = haresource.Snapshot +type resourceSamplingReport = haresource.SamplingReport + +type levelReport struct { + Concurrency int `json:"concurrency"` + Requests int `json:"requests"` + WarmupRequests int `json:"warmup_requests"` + Successful int `json:"successful"` + Errors int `json:"errors"` + ThroughputRPS float64 `json:"throughput_rps"` + Latency durationStats `json:"latency"` + TTFB durationStats `json:"ttfb"` + GatewayRequestsDelta float64 `json:"gateway_requests_delta"` + Gateway2xxDelta float64 `json:"gateway_2xx_delta"` + GatewayErrorDelta float64 `json:"gateway_error_delta"` + RetriesDelta float64 `json:"retries_delta"` + CircuitOpensDelta float64 `json:"circuit_opens_delta"` + PromptTokensDelta float64 `json:"prompt_tokens_delta"` + CompletionTokensDelta float64 `json:"completion_tokens_delta"` + QueueObservationsDelta float64 `json:"queue_observations_delta"` + ServiceObservationsDelta float64 `json:"service_observations_delta"` + AccountingMatches bool `json:"accounting_matches"` + ResourcesBefore *resourceSnapshot `json:"resources_before,omitempty"` + ResourcesAfter *resourceSnapshot `json:"resources_after,omitempty"` + ResourceSamples *resourceSamplingReport `json:"resource_samples,omitempty"` +} + +type evidenceReport struct { + GeneratedAt time.Time `json:"generated_at"` + InputDir string `json:"input_dir"` + Levels []levelReport `json:"levels"` + EvidenceComplete bool `json:"evidence_complete"` + ResourceEvidenceComplete bool `json:"resource_evidence_complete"` + SustainedResourceEvidenceComplete bool `json:"sustained_resource_evidence_complete"` + GateStatus string `json:"gate_status"` + GateReason string `json:"gate_reason"` +} + +func main() { + input := flag.String("input", "", "HA-readiness result directory") + jsonOut := flag.String("json-out", "", "JSON report path; defaults to /report.json") + markdownOut := flag.String("markdown-out", "", "Markdown report path; defaults to /report.md") + flag.Parse() + if *input == "" { + fmt.Fprintln(os.Stderr, "-input is required") + os.Exit(2) + } + if *jsonOut == "" { + *jsonOut = filepath.Join(*input, "report.json") + } + if *markdownOut == "" { + *markdownOut = filepath.Join(*input, "report.md") + } + r, err := buildReport(*input) + if err != nil { + fmt.Fprintln(os.Stderr, "build report:", err) + os.Exit(1) + } + if err := writeJSON(*jsonOut, r); err != nil { + fmt.Fprintln(os.Stderr, "write JSON report:", err) + os.Exit(1) + } + if err := os.WriteFile(*markdownOut, []byte(renderMarkdown(r)), 0o644); err != nil { + fmt.Fprintln(os.Stderr, "write Markdown report:", err) + os.Exit(1) + } + fmt.Printf("report: %s\nreport: %s\ngate: %s — %s\n", *jsonOut, *markdownOut, r.GateStatus, r.GateReason) +} + +func buildReport(dir string) (evidenceReport, error) { + matches, err := filepath.Glob(filepath.Join(dir, "concurrency-*.json")) + if err != nil { + return evidenceReport{}, err + } + if len(matches) == 0 { + return evidenceReport{}, errors.New("no concurrency-*.json files found") + } + levels := make([]levelReport, 0, len(matches)) + complete := true + resourceComplete := true + resourceSeen := false + sustainedComplete := true + sustainedSeen := false + for _, p := range matches { + b, err := os.ReadFile(p) + if err != nil { + return evidenceReport{}, err + } + var s benchSummary + if err := json.Unmarshal(b, &s); err != nil { + return evidenceReport{}, fmt.Errorf("%s: %w", p, err) + } + beforePath := filepath.Join(dir, fmt.Sprintf("metrics-before-c%d.prom", s.Concurrency)) + afterPath := filepath.Join(dir, fmt.Sprintf("metrics-after-c%d.prom", s.Concurrency)) + before, errBefore := parsePrometheusFile(beforePath) + after, errAfter := parsePrometheusFile(afterPath) + if errBefore != nil || errAfter != nil { + complete = false + } + l := levelReport{ + Concurrency: s.Concurrency, Requests: s.Requests, WarmupRequests: s.WarmupRequests, + Successful: s.Successful, Errors: s.Errors, ThroughputRPS: s.ThroughputRPS, + Latency: s.Latency, TTFB: s.TTFB, + } + resourceBefore, errResourceBefore := readResourceSnapshot(filepath.Join(dir, fmt.Sprintf("resources-before-c%d.json", s.Concurrency))) + resourceAfter, errResourceAfter := readResourceSnapshot(filepath.Join(dir, fmt.Sprintf("resources-after-c%d.json", s.Concurrency))) + if errResourceBefore == nil && errResourceAfter == nil { + l.ResourcesBefore = &resourceBefore + l.ResourcesAfter = &resourceAfter + resourceSeen = true + } else { + resourceComplete = false + } + resourceSamples, errResourceSamples := readResourceSamples(filepath.Join(dir, fmt.Sprintf("resources-samples-c%d.json", s.Concurrency))) + if errResourceSamples == nil && resourceSamples.Complete() { + l.ResourceSamples = &resourceSamples + sustainedSeen = true + } else { + sustainedComplete = false + } + if errBefore == nil && errAfter == nil { + l.GatewayRequestsDelta = deltaByName(before, after, "ollama_gateway_requests_total") + l.Gateway2xxDelta = deltaByNameLabelContains(before, after, "ollama_gateway_requests_total", `status_class="2xx"`) + l.GatewayErrorDelta = deltaByName(before, after, "ollama_gateway_errors_total") + // Older/current builds expose errors via requests_total status class rather than a dedicated counter. + if l.GatewayErrorDelta == 0 { + l.GatewayErrorDelta = l.GatewayRequestsDelta - l.Gateway2xxDelta + } + l.RetriesDelta = deltaByName(before, after, "ollama_gateway_retries_total") + l.CircuitOpensDelta = deltaByName(before, after, "ollama_gateway_circuit_opens_total") + l.PromptTokensDelta = deltaByName(before, after, "ollama_gateway_prompt_tokens_total") + l.CompletionTokensDelta = deltaByName(before, after, "ollama_gateway_completion_tokens_total") + l.QueueObservationsDelta = deltaByName(before, after, "ollama_gateway_queue_seconds_count") + l.ServiceObservationsDelta = deltaByName(before, after, "ollama_gateway_service_seconds_count") + expected := float64(s.Requests + s.WarmupRequests) + l.AccountingMatches = almostEqual(l.GatewayRequestsDelta, expected) && almostEqual(l.QueueObservationsDelta, expected) && almostEqual(l.ServiceObservationsDelta, expected) + if !l.AccountingMatches { + complete = false + } + } + levels = append(levels, l) + } + sort.Slice(levels, func(i, j int) bool { return levels[i].Concurrency < levels[j].Concurrency }) + if !resourceSeen { + resourceComplete = false + } + if !sustainedSeen { + sustainedComplete = false + } + r := evidenceReport{ + GeneratedAt: time.Now().UTC(), InputDir: filepath.Clean(dir), Levels: levels, + EvidenceComplete: complete, ResourceEvidenceComplete: resourceComplete, + SustainedResourceEvidenceComplete: sustainedComplete, + } + if !complete { + r.GateStatus = "incomplete" + r.GateReason = "benchmark and gateway metrics evidence are missing or do not reconcile" + } else if sustainedComplete { + r.GateStatus = "not-proven" + r.GateReason = "client/gateway accounting reconciles and sustained host/process resource sampling is complete; HA still requires an operator-demonstrated capacity, availability, or topology need" + } else if resourceComplete { + r.GateStatus = "not-proven" + r.GateReason = "client/gateway accounting reconciles and before/after host/process resource snapshots are present; sustained peak resource evidence is incomplete, and HA still requires an operator-demonstrated capacity, availability, or topology need" + } else { + r.GateStatus = "not-proven" + r.GateReason = "client and gateway counters reconcile; host/process resource evidence is incomplete, and HA still requires demonstrated capacity, availability, or topology need" + } + return r, nil +} + +func readResourceSnapshot(path string) (resourceSnapshot, error) { + b, err := os.ReadFile(path) + if err != nil { + return resourceSnapshot{}, err + } + var s resourceSnapshot + if err := json.Unmarshal(b, &s); err != nil { + return resourceSnapshot{}, err + } + if s.PID <= 0 { + return resourceSnapshot{}, fmt.Errorf("%s: invalid pid", path) + } + return s, nil +} + +func readResourceSamples(path string) (resourceSamplingReport, error) { + b, err := os.ReadFile(path) + if err != nil { + return resourceSamplingReport{}, err + } + var r resourceSamplingReport + if err := json.Unmarshal(b, &r); err != nil { + return resourceSamplingReport{}, err + } + if !r.Complete() { + return r, fmt.Errorf("%s: incomplete sustained resource report", path) + } + return r, nil +} + +func parsePrometheusFile(path string) ([]metricSample, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + var out []metricSample + s := bufio.NewScanner(f) + buf := make([]byte, 64*1024) + s.Buffer(buf, 4*1024*1024) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + parts := strings.Fields(line) + if len(parts) < 2 { + continue + } + v, err := strconv.ParseFloat(parts[1], 64) + if err != nil { + continue + } + key := parts[0] + name, labels := key, "" + if i := strings.IndexByte(key, '{'); i >= 0 { + name = key[:i] + labels = strings.TrimSuffix(key[i+1:], "}") + } + out = append(out, metricSample{Name: name, Labels: labels, Value: v}) + } + return out, s.Err() +} + +func deltaByName(before, after []metricSample, name string) float64 { + return sumBy(before, name, "", false, after) +} + +func deltaByNameLabelContains(before, after []metricSample, name, label string) float64 { + return sumBy(before, name, label, true, after) +} + +func sumBy(before []metricSample, name, label string, filter bool, after []metricSample) float64 { + sum := func(xs []metricSample) float64 { + var n float64 + for _, x := range xs { + if x.Name != name { + continue + } + if filter && !strings.Contains(x.Labels, label) { + continue + } + n += x.Value + } + return n + } + return sum(after) - sum(before) +} + +func almostEqual(a, b float64) bool { + d := a - b + if d < 0 { + d = -d + } + return d < 0.000001 +} + +func writeJSON(path string, v any) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + return os.WriteFile(path, b, 0o644) +} + +func renderMarkdown(r evidenceReport) string { + var b strings.Builder + fmt.Fprintln(&b, "# HA readiness evidence report") + fmt.Fprintln(&b) + fmt.Fprintf(&b, "Generated: `%s` \n", r.GeneratedAt.Format(time.RFC3339)) + fmt.Fprintf(&b, "Gateway accounting complete: **%t** \n", r.EvidenceComplete) + fmt.Fprintf(&b, "Resource snapshots complete: **%t** \n", r.ResourceEvidenceComplete) + fmt.Fprintf(&b, "Sustained resource sampling complete: **%t** \n", r.SustainedResourceEvidenceComplete) + fmt.Fprintf(&b, "P3.2 gate: **%s** — %s\n\n", r.GateStatus, r.GateReason) + fmt.Fprintln(&b, "| Concurrency | Success/Error | req/s | p95 latency | p95 TTFB | Gateway requests Δ | 2xx Δ | retries Δ | circuit opens Δ | counters reconcile | after RSS | sampled peak RSS | sampled peak CPU | sampled peak FDs | samples |") + fmt.Fprintln(&b, "|---:|---:|---:|---:|---:|---:|---:|---:|---:|:---:|---:|---:|---:|---:|---:|") + for _, l := range r.Levels { + afterRSS, _, _, _, _ := resourceCells(l.ResourcesAfter) + peakRSS, peakCPU, peakFDs, sampleCount := sustainedCells(l.ResourceSamples) + fmt.Fprintf(&b, "| %d | %d/%d | %.2f | %.2f ms | %.2f ms | %.0f | %.0f | %.0f | %.0f | %t | %s | %s | %s | %s | %s |\n", l.Concurrency, l.Successful, l.Errors, l.ThroughputRPS, l.Latency.P95, l.TTFB.P95, l.GatewayRequestsDelta, l.Gateway2xxDelta, l.RetriesDelta, l.CircuitOpensDelta, l.AccountingMatches, afterRSS, peakRSS, peakCPU, peakFDs, sampleCount) + } + fmt.Fprintln(&b, "\n## Interpretation") + fmt.Fprintln(&b) + fmt.Fprintln(&b, "This report verifies that client-side benchmark counts reconcile with gateway-side Prometheus counters. Before/after resource files remain endpoint evidence. When `resources-samples-cN.json` is present and complete, the sampled peak columns come from measurements taken throughout that benchmark level; on Linux process CPU is calculated from `/proc` process/host tick deltas and may exceed 100% when multiple logical CPUs are used.") + fmt.Fprintln(&b) + fmt.Fprintln(&b, "The report intentionally does not prove that HA is required. Correlate sustained process/host evidence with worker/GPU saturation and an explicit capacity, availability, or topology requirement before changing the P3.2 gate.") + return b.String() +} + +func resourceCells(s *resourceSnapshot) (rss, cpu, fds, threads, load1 string) { + if s == nil { + return "-", "-", "-", "-", "-" + } + if s.ProcessRSSBytes > 0 { + rss = fmt.Sprintf("%.1f MiB", float64(s.ProcessRSSBytes)/(1024*1024)) + } else { + rss = "-" + } + if s.ProcessCPUPercent > 0 { + cpu = fmt.Sprintf("%.1f%%", s.ProcessCPUPercent) + } else { + cpu = "0.0%" + } + if s.ProcessOpenFDs > 0 { + fds = strconv.FormatInt(s.ProcessOpenFDs, 10) + } else { + fds = "-" + } + if s.ProcessThreads > 0 { + threads = strconv.FormatInt(s.ProcessThreads, 10) + } else { + threads = "-" + } + load1 = fmt.Sprintf("%.2f", s.HostLoad1) + return +} + +func sustainedCells(r *resourceSamplingReport) (rss, cpu, fds, samples string) { + if r == nil { + return "-", "-", "-", "-" + } + if r.PeakProcessRSSBytes > 0 { + rss = fmt.Sprintf("%.1f MiB", float64(r.PeakProcessRSSBytes)/(1024*1024)) + } else { + rss = "-" + } + cpu = fmt.Sprintf("%.1f%%", r.PeakProcessCPUPercent) + if r.PeakProcessOpenFDs > 0 { + fds = strconv.FormatInt(r.PeakProcessOpenFDs, 10) + } else { + fds = "-" + } + samples = strconv.Itoa(len(r.Samples)) + return +} diff --git a/cmd/ha-report/main_test.go b/cmd/ha-report/main_test.go new file mode 100644 index 0000000..df2a21c --- /dev/null +++ b/cmd/ha-report/main_test.go @@ -0,0 +1,207 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestBuildReportReconcilesWarmupAndMeasuredRequests(t *testing.T) { + d := t.TempDir() + s := benchSummary{Concurrency: 4, Requests: 20, WarmupRequests: 3, Successful: 20, ThroughputRPS: 123.4, Latency: durationStats{P95: 8}, TTFB: durationStats{P95: 3}} + b, _ := json.Marshal(s) + if err := os.WriteFile(filepath.Join(d, "concurrency-4.json"), b, 0o600); err != nil { + t.Fatal(err) + } + before := `ollama_gateway_requests_total{api="openai",status_class="2xx"} 10 +ollama_gateway_queue_seconds_count 10 +ollama_gateway_service_seconds_count 10 +ollama_gateway_prompt_tokens_total 100 +ollama_gateway_completion_tokens_total 50 +` + after := `ollama_gateway_requests_total{api="openai",status_class="2xx"} 33 +ollama_gateway_queue_seconds_count 33 +ollama_gateway_service_seconds_count 33 +ollama_gateway_prompt_tokens_total 330 +ollama_gateway_completion_tokens_total 165 +ollama_gateway_retries_total{worker="w"} 0 +` + if err := os.WriteFile(filepath.Join(d, "metrics-before-c4.prom"), []byte(before), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, "metrics-after-c4.prom"), []byte(after), 0o600); err != nil { + t.Fatal(err) + } + r, err := buildReport(d) + if err != nil { + t.Fatal(err) + } + if !r.EvidenceComplete || len(r.Levels) != 1 { + t.Fatalf("unexpected report: %+v", r) + } + l := r.Levels[0] + if l.GatewayRequestsDelta != 23 || !l.AccountingMatches { + t.Fatalf("unexpected level: %+v", l) + } + if r.GateStatus != "not-proven" { + t.Fatalf("gate=%q", r.GateStatus) + } + if got := renderMarkdown(r); !strings.Contains(got, "| 4 | 20/0 |") { + t.Fatalf("markdown missing row: %s", got) + } +} + +func TestBuildReportMarksMissingMetricsIncomplete(t *testing.T) { + d := t.TempDir() + b, _ := json.Marshal(benchSummary{Concurrency: 1, Requests: 1, Successful: 1}) + if err := os.WriteFile(filepath.Join(d, "concurrency-1.json"), b, 0o600); err != nil { + t.Fatal(err) + } + r, err := buildReport(d) + if err != nil { + t.Fatal(err) + } + if r.EvidenceComplete || r.GateStatus != "incomplete" { + t.Fatalf("unexpected report: %+v", r) + } +} + +func TestPrometheusParserUsesFullOllamaGatewayNames(t *testing.T) { + p := filepath.Join(t.TempDir(), "m.prom") + content := "# HELP x x\nollama_gateway_requests_total{api=\"openai\",status_class=\"2xx\"} 7\n" + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + xs, err := parsePrometheusFile(p) + if err != nil { + t.Fatal(err) + } + if got := deltaByName(nil, xs, "ollama_gateway_requests_total"); got != 7 { + t.Fatalf("got %v", got) + } + if got := deltaByName(nil, xs, "gateway_requests_total"); got != 0 { + t.Fatalf("short metric name unexpectedly matched: %v", got) + } +} + +func TestBuildReportIncludesCompleteResourceSnapshots(t *testing.T) { + d := t.TempDir() + s := benchSummary{Concurrency: 8, Requests: 5, WarmupRequests: 2, Successful: 5, ThroughputRPS: 50, Latency: durationStats{P95: 10}, TTFB: durationStats{P95: 4}} + b, _ := json.Marshal(s) + if err := os.WriteFile(filepath.Join(d, "concurrency-8.json"), b, 0o600); err != nil { + t.Fatal(err) + } + beforeMetrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 10\nollama_gateway_queue_seconds_count 10\nollama_gateway_service_seconds_count 10\n" + afterMetrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 17\nollama_gateway_queue_seconds_count 17\nollama_gateway_service_seconds_count 17\n" + if err := os.WriteFile(filepath.Join(d, "metrics-before-c8.prom"), []byte(beforeMetrics), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, "metrics-after-c8.prom"), []byte(afterMetrics), 0o600); err != nil { + t.Fatal(err) + } + before := resourceSnapshot{PID: 4242, ProcessRSSBytes: 64 * 1024 * 1024, ProcessCPUPercent: 2.5, ProcessThreads: 8, ProcessOpenFDs: 21, HostLogicalCPUs: 8, HostLoad1: 0.5} + after := resourceSnapshot{PID: 4242, ProcessRSSBytes: 96 * 1024 * 1024, ProcessCPUPercent: 33.3, ProcessThreads: 12, ProcessOpenFDs: 40, HostLogicalCPUs: 8, HostLoad1: 1.25} + for name, snap := range map[string]resourceSnapshot{"resources-before-c8.json": before, "resources-after-c8.json": after} { + data, _ := json.Marshal(snap) + if err := os.WriteFile(filepath.Join(d, name), data, 0o600); err != nil { + t.Fatal(err) + } + } + r, err := buildReport(d) + if err != nil { + t.Fatal(err) + } + if !r.EvidenceComplete || !r.ResourceEvidenceComplete { + t.Fatalf("unexpected completeness: %+v", r) + } + if r.Levels[0].ResourcesAfter == nil || r.Levels[0].ResourcesAfter.ProcessOpenFDs != 40 { + t.Fatalf("resource snapshot missing: %+v", r.Levels[0]) + } + md := renderMarkdown(r) + if !strings.Contains(md, "96.0 MiB") || !strings.Contains(md, "Sustained resource sampling complete: **false**") || !strings.Contains(md, "Before/after resource files remain endpoint evidence") { + t.Fatalf("resource evidence missing from markdown: %s", md) + } +} + +func TestBuildReportMarksPartialResourceSnapshotsIncompleteWithoutBreakingAccounting(t *testing.T) { + d := t.TempDir() + b, _ := json.Marshal(benchSummary{Concurrency: 2, Requests: 1, Successful: 1}) + if err := os.WriteFile(filepath.Join(d, "concurrency-2.json"), b, 0o600); err != nil { + t.Fatal(err) + } + metrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 1\nollama_gateway_queue_seconds_count 1\nollama_gateway_service_seconds_count 1\n" + if err := os.WriteFile(filepath.Join(d, "metrics-before-c2.prom"), []byte("ollama_gateway_requests_total{status_class=\"2xx\"} 0\nollama_gateway_queue_seconds_count 0\nollama_gateway_service_seconds_count 0\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, "metrics-after-c2.prom"), []byte(metrics), 0o600); err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(resourceSnapshot{PID: 99, HostLogicalCPUs: 4}) + if err := os.WriteFile(filepath.Join(d, "resources-before-c2.json"), data, 0o600); err != nil { + t.Fatal(err) + } + r, err := buildReport(d) + if err != nil { + t.Fatal(err) + } + if !r.EvidenceComplete || r.ResourceEvidenceComplete || r.GateStatus != "not-proven" { + t.Fatalf("unexpected report: %+v", r) + } + if !strings.Contains(r.GateReason, "resource evidence is incomplete") { + t.Fatalf("unexpected gate reason: %s", r.GateReason) + } +} + +func TestBuildReportIncludesSustainedResourceSampling(t *testing.T) { + d := t.TempDir() + s := benchSummary{Concurrency: 16, Requests: 10, WarmupRequests: 2, Successful: 10, ThroughputRPS: 75, Latency: durationStats{P95: 12}, TTFB: durationStats{P95: 5}} + b, _ := json.Marshal(s) + if err := os.WriteFile(filepath.Join(d, "concurrency-16.json"), b, 0o600); err != nil { + t.Fatal(err) + } + beforeMetrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 4\nollama_gateway_queue_seconds_count 4\nollama_gateway_service_seconds_count 4\n" + afterMetrics := "ollama_gateway_requests_total{status_class=\"2xx\"} 16\nollama_gateway_queue_seconds_count 16\nollama_gateway_service_seconds_count 16\n" + if err := os.WriteFile(filepath.Join(d, "metrics-before-c16.prom"), []byte(beforeMetrics), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, "metrics-after-c16.prom"), []byte(afterMetrics), 0o600); err != nil { + t.Fatal(err) + } + for name, snap := range map[string]resourceSnapshot{ + "resources-before-c16.json": {PID: 77, ProcessRSSBytes: 50 * 1024 * 1024, HostLogicalCPUs: 8}, + "resources-after-c16.json": {PID: 77, ProcessRSSBytes: 60 * 1024 * 1024, HostLogicalCPUs: 8}, + } { + data, _ := json.Marshal(snap) + if err := os.WriteFile(filepath.Join(d, name), data, 0o600); err != nil { + t.Fatal(err) + } + } + samples := resourceSamplingReport{ + Version: 1, PID: 77, GOOS: "linux", PeakProcessRSSBytes: 80 * 1024 * 1024, + PeakProcessCPUPercent: 145.5, PeakProcessOpenFDs: 55, + Samples: []resourceSnapshot{{PID: 77}, {PID: 77}}, StopReason: "stop-file", + } + data, _ := json.Marshal(samples) + if err := os.WriteFile(filepath.Join(d, "resources-samples-c16.json"), data, 0o600); err != nil { + t.Fatal(err) + } + r, err := buildReport(d) + if err != nil { + t.Fatal(err) + } + if !r.EvidenceComplete || !r.ResourceEvidenceComplete || !r.SustainedResourceEvidenceComplete { + t.Fatalf("unexpected completeness: %+v", r) + } + if r.Levels[0].ResourceSamples == nil || r.Levels[0].ResourceSamples.PeakProcessOpenFDs != 55 { + t.Fatalf("missing sustained samples: %+v", r.Levels[0]) + } + if !strings.Contains(r.GateReason, "sustained host/process resource sampling is complete") { + t.Fatalf("unexpected gate reason: %s", r.GateReason) + } + md := renderMarkdown(r) + if !strings.Contains(md, "80.0 MiB") || !strings.Contains(md, "145.5%") || !strings.Contains(md, "| 16 | 10/0 |") { + t.Fatalf("sustained evidence missing from markdown: %s", md) + } +} diff --git a/cmd/ha-sampler/main.go b/cmd/ha-sampler/main.go new file mode 100644 index 0000000..500f283 --- /dev/null +++ b/cmd/ha-sampler/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/example/ollama-fair-gateway/internal/haresource" +) + +func main() { + pid := flag.Int("pid", 0, "gateway process PID") + interval := flag.Duration("interval", 250*time.Millisecond, "resource sample interval (minimum 50ms)") + maxDuration := flag.Duration("max-duration", 15*time.Minute, "safety limit for one sampling run") + stopFile := flag.String("stop-file", "", "stop sampling after this file appears") + out := flag.String("json-out", "-", "output JSON path; '-' writes stdout") + flag.Parse() + if *pid <= 0 || *stopFile == "" { + fmt.Fprintln(os.Stderr, "-pid must be positive and -stop-file is required") + os.Exit(2) + } + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + r, err := haresource.Sample(ctx, *pid, *interval, *maxDuration, *stopFile) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + b, err := json.MarshalIndent(r, "", " ") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + b = append(b, '\n') + if *out == "-" { + _, _ = os.Stdout.Write(b) + } else if err := os.WriteFile(*out, b, 0o644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if !r.Complete() { + fmt.Fprintln(os.Stderr, "sampling completed but sustained resource evidence is incomplete") + os.Exit(3) + } +} diff --git a/cmd/ha-snapshot/main.go b/cmd/ha-snapshot/main.go new file mode 100644 index 0000000..a2a8788 --- /dev/null +++ b/cmd/ha-snapshot/main.go @@ -0,0 +1,40 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/example/ollama-fair-gateway/internal/haresource" +) + +func main() { + pid := flag.Int("pid", 0, "gateway process PID") + out := flag.String("json-out", "-", "output JSON path; '-' writes stdout") + flag.Parse() + if *pid <= 0 { + fmt.Fprintln(os.Stderr, "-pid must be positive") + os.Exit(2) + } + s := haresource.Collect(*pid) + b, err := json.MarshalIndent(s, "", " ") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + b = append(b, '\n') + if *out == "-" { + _, _ = os.Stdout.Write(b) + return + } + if err := os.WriteFile(*out, b, 0o644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func outputName(dir string, concurrency int, phase string) string { + return filepath.Join(dir, fmt.Sprintf("resources-%s-c%d.json", phase, concurrency)) +} diff --git a/cmd/ha-snapshot/main_test.go b/cmd/ha-snapshot/main_test.go new file mode 100644 index 0000000..8bcec9c --- /dev/null +++ b/cmd/ha-snapshot/main_test.go @@ -0,0 +1,10 @@ +package main + +import "testing" + +func TestOutputName(t *testing.T) { + got := outputName("out", 8, "before") + if got != "out/resources-before-c8.json" { + t.Fatalf("outputName=%q", got) + } +} diff --git a/cmd/mock-ollama/main.go b/cmd/mock-ollama/main.go new file mode 100644 index 0000000..7654fbb --- /dev/null +++ b/cmd/mock-ollama/main.go @@ -0,0 +1,191 @@ +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "strings" + "sync/atomic" + "time" +) + +type server struct { + model string + delay time.Duration + streamDelay time.Duration + responseBytes int + promptTokens int64 + outputTokens int64 + failEvery int64 + requests atomic.Int64 +} + +func main() { + listen := flag.String("listen", "127.0.0.1:11435", "listen address") + model := flag.String("model", "qwen3:8b", "mock model name") + delay := flag.Duration("delay", 0, "delay before response headers") + streamDelay := flag.Duration("stream-delay", 0, "delay between streaming chunks") + responseBytes := flag.Int("response-bytes", 128, "approximate generated content bytes") + promptTokens := flag.Int64("prompt-tokens", 16, "reported prompt token count") + outputTokens := flag.Int64("output-tokens", 8, "reported completion token count") + failEvery := flag.Int64("fail-every", 0, "return HTTP 503 for every Nth inference request; 0 disables") + flag.Parse() + if *responseBytes < 0 || *promptTokens < 0 || *outputTokens < 0 || *failEvery < 0 { + log.Fatal("numeric flags must be non-negative") + } + s := &server{model: *model, delay: *delay, streamDelay: *streamDelay, responseBytes: *responseBytes, promptTokens: *promptTokens, outputTokens: *outputTokens, failEvery: *failEvery} + mux := http.NewServeMux() + mux.HandleFunc("/api/ps", s.ps) + mux.HandleFunc("/api/tags", s.tags) + mux.HandleFunc("/api/show", s.show) + mux.HandleFunc("/api/chat", s.nativeChat) + mux.HandleFunc("/api/generate", s.nativeGenerate) + mux.HandleFunc("/v1/chat/completions", s.openAIChat) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) }) + h := &http.Server{Addr: *listen, Handler: mux, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 2 * time.Minute} + log.Printf("mock Ollama listening on http://%s model=%s delay=%s response_bytes=%d", *listen, *model, *delay, *responseBytes) + log.Fatal(h.ListenAndServe()) +} + +func (s *server) ps(w http.ResponseWriter, r *http.Request) { + writeJSON(w, map[string]any{"models": []map[string]any{{"name": s.model, "model": s.model, "size": 1 << 30, "size_vram": 1 << 30, "context_length": 32768}}}) +} + +func (s *server) tags(w http.ResponseWriter, r *http.Request) { + writeJSON(w, map[string]any{"models": []map[string]any{{"name": s.model, "model": s.model, "size": 1 << 30, "details": map[string]any{"parameter_size": "8B", "quantization_level": "Q4_K_M"}}}}) +} + +func (s *server) show(w http.ResponseWriter, r *http.Request) { + writeJSON(w, map[string]any{"capabilities": []string{"completion", "tools", "thinking", "vision"}, "model_info": map[string]any{"mock.context_length": 32768}}) +} + +func (s *server) shouldFail() bool { + n := s.requests.Add(1) + return s.failEvery > 0 && n%s.failEvery == 0 +} + +func (s *server) openAIChat(w http.ResponseWriter, r *http.Request) { + if s.delay > 0 { + time.Sleep(s.delay) + } + if s.shouldFail() { + writeJSONStatus(w, http.StatusServiceUnavailable, map[string]any{"error": map[string]any{"message": "mock failure", "type": "server_error"}}) + return + } + var req struct { + Stream bool `json:"stream"` + } + _ = json.NewDecoder(io.LimitReader(r.Body, 8<<20)).Decode(&req) + content := strings.Repeat("x", s.responseBytes) + if req.Stream { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + f, _ := w.(http.Flusher) + parts := split(content, 4) + for i, p := range parts { + chunk := map[string]any{"id": "chatcmpl-mock", "object": "chat.completion.chunk", "choices": []map[string]any{{"index": 0, "delta": map[string]any{"content": p}, "finish_reason": nil}}} + if i == len(parts)-1 { + chunk["choices"] = []map[string]any{{"index": 0, "delta": map[string]any{}, "finish_reason": "stop"}} + } + b, _ := json.Marshal(chunk) + fmt.Fprintf(w, "data: %s\n\n", b) + if f != nil { + f.Flush() + } + if s.streamDelay > 0 { + time.Sleep(s.streamDelay) + } + } + io.WriteString(w, "data: [DONE]\n\n") + if f != nil { + f.Flush() + } + return + } + writeJSON(w, map[string]any{ + "id": "chatcmpl-mock", "object": "chat.completion", "created": time.Now().Unix(), "model": s.model, + "choices": []map[string]any{{"index": 0, "message": map[string]any{"role": "assistant", "content": content}, "finish_reason": "stop"}}, + "usage": map[string]any{"prompt_tokens": s.promptTokens, "completion_tokens": s.outputTokens, "total_tokens": s.promptTokens + s.outputTokens}, + }) +} + +func (s *server) nativeChat(w http.ResponseWriter, r *http.Request) { + s.native(w, r, "message") +} + +func (s *server) nativeGenerate(w http.ResponseWriter, r *http.Request) { + s.native(w, r, "response") +} + +func (s *server) native(w http.ResponseWriter, r *http.Request, field string) { + if s.delay > 0 { + time.Sleep(s.delay) + } + if s.shouldFail() { + writeJSONStatus(w, http.StatusServiceUnavailable, map[string]any{"error": "mock failure"}) + return + } + var req map[string]any + _ = json.NewDecoder(io.LimitReader(r.Body, 8<<20)).Decode(&req) + stream, _ := req["stream"].(bool) + content := strings.Repeat("x", s.responseBytes) + w.Header().Set("Content-Type", "application/x-ndjson") + bw := bufio.NewWriter(w) + if !stream { + doc := map[string]any{"model": s.model, "done": true, "prompt_eval_count": s.promptTokens, "eval_count": s.outputTokens} + if field == "message" { + doc[field] = map[string]any{"role": "assistant", "content": content} + } else { + doc[field] = content + } + _ = json.NewEncoder(bw).Encode(doc) + _ = bw.Flush() + return + } + f, _ := w.(http.Flusher) + for _, p := range split(content, 4) { + doc := map[string]any{"model": s.model, "done": false} + if field == "message" { + doc[field] = map[string]any{"role": "assistant", "content": p} + } else { + doc[field] = p + } + _ = json.NewEncoder(bw).Encode(doc) + _ = bw.Flush() + if f != nil { + f.Flush() + } + if s.streamDelay > 0 { + time.Sleep(s.streamDelay) + } + } + _ = json.NewEncoder(bw).Encode(map[string]any{"model": s.model, "done": true, "prompt_eval_count": s.promptTokens, "eval_count": s.outputTokens}) + _ = bw.Flush() + if f != nil { + f.Flush() + } +} + +func split(s string, n int) []string { + if n <= 1 || len(s) == 0 { + return []string{s} + } + out := make([]string, 0, n) + for i := 0; i < n; i++ { + start := len(s) * i / n + end := len(s) * (i + 1) / n + out = append(out, s[start:end]) + } + return out +} + +func writeJSON(w http.ResponseWriter, v any) { writeJSONStatus(w, http.StatusOK, v) } +func writeJSONStatus(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} diff --git a/cmd/mock-ollama/main_test.go b/cmd/mock-ollama/main_test.go new file mode 100644 index 0000000..7519b76 --- /dev/null +++ b/cmd/mock-ollama/main_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "bufio" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +func TestOpenAIChatNonStreaming(t *testing.T) { + s := &server{model: "mock:latest", responseBytes: 12, promptTokens: 7, outputTokens: 3} + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"stream":false}`)) + rr := httptest.NewRecorder() + s.openAIChat(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String()) + } + var doc struct { + Model string `json:"model"` + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + Usage struct { + Prompt int64 `json:"prompt_tokens"` + Output int64 `json:"completion_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &doc); err != nil { + t.Fatal(err) + } + if doc.Model != "mock:latest" || len(doc.Choices) != 1 || len(doc.Choices[0].Message.Content) != 12 { + t.Fatalf("unexpected response: %+v", doc) + } + if doc.Usage.Prompt != 7 || doc.Usage.Output != 3 { + t.Fatalf("unexpected usage: %+v", doc.Usage) + } +} + +func TestOpenAIChatStreaming(t *testing.T) { + s := &server{model: "mock:latest", responseBytes: 16} + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"stream":true}`)) + rr := httptest.NewRecorder() + s.openAIChat(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String()) + } + if ct := rr.Header().Get("Content-Type"); !strings.Contains(ct, "text/event-stream") { + t.Fatalf("content-type=%q", ct) + } + if !strings.Contains(rr.Body.String(), "data: [DONE]") { + t.Fatalf("missing done marker: %s", rr.Body.String()) + } +} + +func TestNativeStreamingEndsWithUsage(t *testing.T) { + s := &server{model: "mock:latest", responseBytes: 8, promptTokens: 11, outputTokens: 5} + req := httptest.NewRequest(http.MethodPost, "/api/chat", strings.NewReader(`{"stream":true}`)) + rr := httptest.NewRecorder() + s.nativeChat(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String()) + } + scanner := bufio.NewScanner(strings.NewReader(rr.Body.String())) + var last map[string]any + for scanner.Scan() { + if err := json.Unmarshal(scanner.Bytes(), &last); err != nil { + t.Fatal(err) + } + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + if done, _ := last["done"].(bool); !done { + t.Fatalf("last chunk not done: %#v", last) + } + if last["prompt_eval_count"] != float64(11) || last["eval_count"] != float64(5) { + t.Fatalf("unexpected usage: %#v", last) + } +} + +func TestShouldFailConcurrent(t *testing.T) { + const total = 1000 + s := &server{failEvery: 5} + var wg sync.WaitGroup + failures := make(chan struct{}, total) + for i := 0; i < total; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if s.shouldFail() { + failures <- struct{}{} + } + }() + } + wg.Wait() + close(failures) + if got, want := len(failures), total/5; got != want { + t.Fatalf("failures=%d want=%d", got, want) + } + if got := s.requests.Load(); got != total { + t.Fatalf("requests=%d want=%d", got, total) + } +} diff --git a/cmd/ollama-gateway/main.go b/cmd/ollama-gateway/main.go new file mode 100644 index 0000000..9227fd2 --- /dev/null +++ b/cmd/ollama-gateway/main.go @@ -0,0 +1,255 @@ +package main + +import ( + "context" + "flag" + "log/slog" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/example/ollama-fair-gateway/internal/alerts" + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/autotune" + "github.com/example/ollama-fair-gateway/internal/batch" + "github.com/example/ollama-fair-gateway/internal/conversation" + "github.com/example/ollama-fair-gateway/internal/cost" + "github.com/example/ollama-fair-gateway/internal/infrastructure" + "github.com/example/ollama-fair-gateway/internal/liveflow" + "github.com/example/ollama-fair-gateway/internal/metrics" + "github.com/example/ollama-fair-gateway/internal/proxy" + "github.com/example/ollama-fair-gateway/internal/quota" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/server" + "github.com/example/ollama-fair-gateway/internal/session" + "github.com/example/ollama-fair-gateway/internal/state" + "github.com/example/ollama-fair-gateway/internal/telemetry" + "github.com/example/ollama-fair-gateway/internal/usage" + "github.com/example/ollama-fair-gateway/internal/warm" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +func main() { + configPath := flag.String("config", "config.json", "configuration file") + checkConfig := flag.Bool("check-config", false, "validate effective configuration and storage, then exit") + probeURL := flag.String("probe", "", "probe an HTTP health/readiness URL, then exit") + probeTimeout := flag.Duration("probe-timeout", 2*time.Second, "timeout for -probe") + flag.Parse() + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + slog.SetDefault(log) + if *probeURL != "" { + if err := runProbe(*probeURL, *probeTimeout); err != nil { + log.Error("probe failed", "url", *probeURL, "error", err) + os.Exit(1) + } + return + } + if *checkConfig { + if err := runConfigCheck(*configPath, os.Stdout); err != nil { + log.Error("configuration preflight failed", "error", err) + os.Exit(2) + } + return + } + loaded, err := loadEffectiveConfig(*configPath) + if err != nil { + log.Error("configuration error", "error", err) + os.Exit(2) + } + cfg := loaded.Config + paths := loaded.Paths + configStore := loaded.Store + if loaded.Persistent { + log.Info("loaded persistent UI configuration", "path", configStore.Path()) + } + rootCtx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + apiKeyStore, err := state.NewAPIKeyStore(paths.APIKeys) + if err != nil { + log.Error("API key store initialization failed", "error", err) + os.Exit(2) + } + authenticator, err := auth.NewWithRuntimeStore(rootCtx, cfg.Auth, apiKeyStore) + if err != nil { + log.Error("authentication initialization failed", "error", err) + os.Exit(2) + } + + sched := scheduler.NewLocal(cfg.Scheduler.GlobalConcurrency, cfg.Scheduler.MaxQueue, cfg.Scheduler.MaxQueuePerActor) + var ledger quota.Ledger = quota.Disabled{} + var quotaMemory *quota.Memory + if cfg.Quota.Enabled { + quotaMemory = quota.NewMemory() + if err := quotaMemory.LoadPersistent(paths.Quota); err != nil { + log.Error("quota state initialization failed", "error", err) + os.Exit(2) + } + quotaMemory.StartPersistence(rootCtx, paths.Quota, cfg.Storage.FlushInterval.Value(), func(err error) { log.Error("quota persistence failed", "error", err) }) + ledger = quotaMemory + } + policyStore, err := state.NewPolicyStore(paths.Policies) + if err != nil { + log.Error("policy store initialization failed", "error", err) + os.Exit(2) + } + placementStore, err := state.NewModelPlacementStore(paths.ModelPlacement) + if err != nil { + log.Error("model placement store initialization failed", "error", err) + os.Exit(2) + } + workerStateStore, err := state.NewWorkerRuntimeStore(paths.WorkerState) + if err != nil { + log.Error("worker state store initialization failed", "error", err) + os.Exit(2) + } + uiSessions := session.NewMemory() + conversationStore, err := conversation.New(cfg.Conversations, paths.Conversations) + if err != nil { + log.Error("conversation store initialization failed", "error", err) + os.Exit(2) + } + conversationStore.StartCleanup(rootCtx, func(err error) { log.Error("conversation retention cleanup failed", "error", err) }) + batchManager, err := batch.New(cfg.BatchJobs, paths.BatchJobs, paths.BatchDir) + if err != nil { + log.Error("batch job manager initialization failed", "error", err) + os.Exit(2) + } + pool := worker.New(cfg.Workers, cfg.Native.ControlWorker) + pool.SetRoutingConfig(cfg.Routing) + pool.SetModelCapabilitiesConfig(cfg.ModelCapabilities) + pool.SetReliabilityConfig(cfg.Reliability) + if overrides, err := placementStore.List(rootCtx); err != nil { + log.Error("model placement overrides load failed", "error", err) + os.Exit(2) + } else { + for workerName, rule := range overrides { + if err := pool.SetPlacement(workerName, rule, true); err != nil { + // Keep orphaned rules durable when a worker is temporarily removed; + // they become active again if that worker name returns later. + log.Warn("ignoring model placement override for unknown worker", "worker", workerName, "error", err) + } + } + } + if modes, err := workerStateStore.List(rootCtx); err != nil { + log.Error("worker runtime state load failed", "error", err) + os.Exit(2) + } else { + for name, mode := range modes { + if err := pool.SetMaintenance(name, mode); err != nil { + log.Warn("ignoring worker state for unknown worker", "worker", name, "error", err) + } + } + } + if err := pool.LoadPerformance(paths.WorkerPerformance); err != nil { + log.Error("worker performance state initialization failed", "error", err) + os.Exit(2) + } + pool.StartPerformancePersistence(rootCtx, paths.WorkerPerformance, cfg.Storage.FlushInterval.Value(), func(err error) { log.Error("worker performance persistence failed", "error", err) }) + tuner, err := autotune.New(cfg.AutoTuning, pool, paths.AutoTune) + if err != nil { + log.Error("auto tuning state initialization failed", "error", err) + os.Exit(2) + } + for workerName, models := range tuner.Applied() { + for model, limit := range models { + if err := pool.SetModelConcurrency(workerName, model, limit); err != nil { + log.Warn("ignoring auto-tune override", "worker", workerName, "model", model, "error", err) + } + } + } + pool.Start(rootCtx) + warmManager, err := warm.New(cfg.WarmModels, pool, paths.WarmModels) + if err != nil { + log.Error("warm model manager initialization failed", "error", err) + os.Exit(2) + } + warmManager.Start(rootCtx) + alertManager, err := alerts.New(cfg.Alerts, paths.Alerts, func() alerts.Snapshot { + st := sched.Stats(context.Background()) + ws := pool.Snapshots() + aw := make([]alerts.Worker, 0, len(ws)) + for _, w := range ws { + aw = append(aw, alerts.Worker{Name: w.Name, Healthy: w.Healthy, CircuitState: w.CircuitState, LastCircuitError: w.LastCircuitError, VRAMUsedBytes: w.VRAMUsedBytes, VRAMTotalBytes: w.VRAMTotalBytes}) + } + return alerts.Snapshot{QueueDepth: int(st.Queued), QueueWait: st.OldestWait, Workers: aw, StorageBytes: alerts.DirSize(paths.DataDir)} + }) + if err != nil { + log.Error("alerts manager initialization failed", "error", err) + os.Exit(2) + } + alertManager.Start(rootCtx) + live := liveflow.New(10*time.Second, max(512, cfg.Infrastructure.MaxRequests)) + infra := infrastructure.New(cfg.Infrastructure, live, sched, pool) + infra.Start(rootCtx) + met := metrics.New() + if err := met.LoadPersistent(paths.Metrics); err != nil { + log.Error("metrics state initialization failed", "error", err) + os.Exit(2) + } + met.StartPersistence(rootCtx, paths.Metrics, cfg.Storage.FlushInterval.Value(), func(err error) { log.Error("metrics persistence failed", "error", err) }) + rec, err := usage.NewWithRetention(cfg.Usage.JournalDir, cfg.Usage.Buffer, cfg.Usage.FlushInterval.Value(), usage.RetentionConfig{DetailDays: cfg.Usage.Retention.DetailDays, DailyDays: cfg.Usage.Retention.DailyDays, MonthlyMonths: cfg.Usage.Retention.MonthlyMonths, CompactionInterval: cfg.Usage.Retention.CompactionInterval.Value()}, met.DropUsage) + if err != nil { + log.Error("usage recorder initialization failed", "error", err) + os.Exit(2) + } + rec.SetRecentCapacity(cfg.UI.RecentEvents) + defer rec.Close() + otelExporter := telemetry.New(cfg.OpenTelemetry) + if otelExporter != nil { + defer func() { + cctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := otelExporter.Close(cctx); err != nil { + log.Warn("OpenTelemetry shutdown", "error", err) + } + }() + } + srvHandler := server.New(cfg, server.Dependencies{Auth: authenticator, Scheduler: sched, Quota: ledger, Estimator: cost.New(cfg.Cost), Workers: pool, Proxy: proxy.New(), Usage: rec, Metrics: met, Policies: policyStore, Sessions: uiSessions, Live: live, Infrastructure: infra, Logger: log, ConfigStore: configStore, PlacementStore: placementStore, WorkerStateStore: workerStateStore, AutoTune: tuner, OpenTelemetry: otelExporter, WarmModels: warmManager, Alerts: alertManager, Conversations: conversationStore, BatchJobs: batchManager}) + batchManager.Start(rootCtx, srvHandler.ExecuteBatch) + hs := &http.Server{Addr: cfg.Server.Listen, Handler: srvHandler.Handler(), ReadHeaderTimeout: cfg.Server.ReadHeaderTimeout.Value(), IdleTimeout: cfg.Server.IdleTimeout.Value(), MaxHeaderBytes: 1 << 20} + go func() { + <-rootCtx.Done() + ctx, c := context.WithTimeout(context.Background(), 15*time.Second) + defer c() + _ = hs.Shutdown(ctx) + }() + log.Info("ollama fair gateway starting", "listen", cfg.Server.Listen, "workers", len(cfg.Workers), "coordination", "in-memory", "node", infra.NodeName(), "quota", cfg.Quota.Enabled, "storage", cfg.Storage.DataDir) + if cfg.Server.TLSCert != "" || cfg.Server.TLSKey != "" { + err = hs.ListenAndServeTLS(cfg.Server.TLSCert, cfg.Server.TLSKey) + } else { + err = hs.ListenAndServe() + } + if err != nil && err != http.ErrServerClosed { + log.Error("server failed", "error", err) + os.Exit(1) + } + if warmManager != nil { + wctx, wcancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := warmManager.Wait(wctx); err != nil { + log.Warn("warm model actions still active during shutdown", "error", err) + } + wcancel() + } + if batchManager != nil { + bctx, bcancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := batchManager.Wait(bctx); err != nil { + log.Warn("batch attempts still active during shutdown", "error", err) + } + bcancel() + } + rec.Close() + if quotaMemory != nil { + if err := quotaMemory.SavePersistent(paths.Quota); err != nil { + log.Error("final quota persistence failed", "error", err) + } + } + if err := met.SavePersistent(paths.Metrics); err != nil { + log.Error("final metrics persistence failed", "error", err) + } + if err := pool.SavePerformance(paths.WorkerPerformance); err != nil { + log.Error("final worker performance persistence failed", "error", err) + } + log.Info("gateway stopped") +} diff --git a/cmd/ollama-gateway/preflight.go b/cmd/ollama-gateway/preflight.go new file mode 100644 index 0000000..49a5ac2 --- /dev/null +++ b/cmd/ollama-gateway/preflight.go @@ -0,0 +1,222 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/state" +) + +type effectiveConfig struct { + Bootstrap *config.Config + Config *config.Config + Paths state.Paths + Store *state.ConfigStore + Persistent bool +} + +func loadEffectiveConfig(configPath string) (*effectiveConfig, error) { + bootstrapCfg, err := config.Load(configPath) + if err != nil { + return nil, fmt.Errorf("configuration error: %w", err) + } + paths := state.Resolve(bootstrapCfg.Storage) + configStore := state.NewConfigStore(paths.Config) + cfg := bootstrapCfg + persistentCfg, ok, err := configStore.LoadIfExists(bootstrapCfg) + if err != nil { + return nil, fmt.Errorf("persistent configuration error (%s): %w", configStore.Path(), err) + } + if ok { + if persistentCfg.Storage != bootstrapCfg.Storage { + return nil, fmt.Errorf("persistent configuration changes bootstrap-only storage settings: bootstrap=%+v persistent=%+v", bootstrapCfg.Storage, persistentCfg.Storage) + } + cfg = persistentCfg + } + return &effectiveConfig{ + Bootstrap: bootstrapCfg, + Config: cfg, + Paths: state.Resolve(cfg.Storage), + Store: configStore, + Persistent: ok, + }, nil +} + +type configCheckResult struct { + Status string `json:"status"` + ConfigPath string `json:"config_path"` + PersistentOverride bool `json:"persistent_override"` + PersistentPath string `json:"persistent_path"` + Workers int `json:"workers"` + DataDir string `json:"data_dir"` + DataDirAbsolute string `json:"data_dir_absolute"` + StorageWritable bool `json:"storage_writable"` + Warnings []string `json:"warnings,omitempty"` +} + +func runConfigCheck(configPath string, out io.Writer) error { + loaded, err := loadEffectiveConfig(configPath) + if err != nil { + return err + } + if err := checkWritableDir(loaded.Paths.DataDir); err != nil { + return fmt.Errorf("storage data_dir %q is not writable: %w", loaded.Paths.DataDir, err) + } + absDir, err := filepath.Abs(loaded.Paths.DataDir) + if err != nil { + absDir = loaded.Paths.DataDir + } + res := configCheckResult{ + Status: "ok", + ConfigPath: configPath, + PersistentOverride: loaded.Persistent, + PersistentPath: loaded.Store.Path(), + Workers: len(loaded.Config.Workers), + DataDir: loaded.Paths.DataDir, + DataDirAbsolute: absDir, + StorageWritable: true, + Warnings: configWarnings(loaded.Config), + } + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + return enc.Encode(res) +} + +func checkWritableDir(dir string) error { + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + f, err := os.CreateTemp(dir, ".gateway-preflight-*") + if err != nil { + return err + } + name := f.Name() + defer os.Remove(name) + if err := f.Chmod(0600); err != nil { + _ = f.Close() + return err + } + if _, err := f.WriteString("ok\n"); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + +func configWarnings(cfg *config.Config) []string { + var warnings []string + if control := strings.TrimSpace(cfg.Native.ControlWorker); control != "" { + found := false + for _, w := range cfg.Workers { + if w.Name == control { + found = true + break + } + } + if !found { + warnings = append(warnings, fmt.Sprintf("native.control_worker %q does not match any configured worker; management requests will fall back to another healthy worker", control)) + } + } + for _, w := range cfg.Workers { + if !w.LocalSystemStats { + continue + } + u, err := url.Parse(w.URL) + if err != nil { + continue + } + host := u.Hostname() + if host == "" || isLoopbackHost(host) { + continue + } + warnings = append(warnings, fmt.Sprintf("worker %q has local_system_stats=true but URL host %q is remote; local system stats describe the gateway host, not that worker (use telemetry_url or disable local_system_stats)", w.Name, host)) + } + if cfg.Auth.IPBypassUseForwardedIP && len(cfg.Auth.IPBypass) > 0 { + warnings = append(warnings, "auth.ip_bypass_use_forwarded_ip=true allows X-Forwarded-For-derived addresses to satisfy credential-free IP bypass; use only with tightly restricted trusted_proxies and network ACLs") + } + for _, raw := range cfg.Auth.TrustedProxies { + if broadTrustedProxyCIDR(raw) { + warnings = append(warnings, fmt.Sprintf("auth.trusted_proxies contains broad CIDR %q; any directly reachable peer in that range can influence X-Forwarded-For-derived client_ip, so prefer exact proxy addresses", raw)) + } + } + if cfg.UI.Enabled && !cfg.UI.OIDC.Enabled && len(cfg.Auth.APIKeys) == 0 { + warnings = append(warnings, "UI is enabled without UI OIDC and without bootstrap API keys; remote UI login depends entirely on IP bypass rules") + } + if cfg.UI.Enabled && !cfg.UI.SecureCookies && cfg.UI.OIDC.Enabled { + warnings = append(warnings, "ui.secure_cookies=false while UI OIDC is enabled; enable secure cookies when the browser reaches the gateway over HTTPS") + } + if cfg.Server.MetricsPublic { + warnings = append(warnings, "server.metrics_public=true exposes gateway metrics without authentication") + } + if cfg.ModelCapabilities.Context.MaxRequestedTokens == -1 { + warnings = append(warnings, "model_capabilities.context.max_requested_tokens=-1 removes the gateway-side context cap; large native num_ctx requests can cause substantial KV-cache/VRAM pressure") + } + if cfg.ModelCapabilities.Context.DefaultWorkerTokens == -1 { + warnings = append(warnings, "model_capabilities.context.default_worker_tokens=-1 falls back to the theoretical model maximum for unloaded models without Modelfile num_ctx; configure an explicit worker default for predictable memory use") + } + if !filepath.IsAbs(cfg.Storage.DataDir) { + warnings = append(warnings, fmt.Sprintf("storage.data_dir %q is relative and therefore depends on the process working directory", cfg.Storage.DataDir)) + } + return warnings +} + +func broadTrustedProxyCIDR(raw string) bool { + ip, n, err := net.ParseCIDR(strings.TrimSpace(raw)) + if err != nil || ip == nil || n == nil || ip.IsLoopback() { + return false + } + ones, bits := n.Mask.Size() + if bits == 32 { + return ones < 24 + } + if bits == 128 { + return ones < 64 + } + return false +} + +func isLoopbackHost(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +func runProbe(rawURL string, timeout time.Duration) error { + if timeout <= 0 { + return fmt.Errorf("probe timeout must be > 0") + } + u, err := url.Parse(rawURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("invalid probe URL %q", rawURL) + } + client := &http.Client{Timeout: timeout} + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + return err + } + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("probe returned HTTP %d", resp.StatusCode) + } + return nil +} diff --git a/cmd/ollama-gateway/preflight_test.go b/cmd/ollama-gateway/preflight_test.go new file mode 100644 index 0000000..947a0c6 --- /dev/null +++ b/cmd/ollama-gateway/preflight_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/state" +) + +func writeBootstrapConfig(t *testing.T, dataDir string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.json") + body := `{ + "auth":{"api_keys":[{"name":"admin","key":"01234567890123456789012345678901","tenant":"t","subject":"s","scopes":["gateway:admin"]}]}, + "workers":[{"name":"remote","url":"http://10.2.10.48:11434","local_system_stats":true}], + "ui":{"enabled":true,"path":"/admin","title":"Bootstrap"}, + "storage":{"data_dir":` + quoteJSON(dataDir) + `,"config_file":"gateway-config.json"} +}` + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + return path +} + +func quoteJSON(s string) string { + b, _ := json.Marshal(s) + return string(b) +} + +func TestLoadEffectiveConfigUsesPersistentOverrideButBootstrapSecrets(t *testing.T) { + dataDir := t.TempDir() + path := writeBootstrapConfig(t, dataDir) + base, err := config.Load(path) + if err != nil { + t.Fatal(err) + } + persistent := *base + persistent.UI.Title = "Persistent" + store := state.NewConfigStore(state.Resolve(base.Storage).Config) + if err := store.Save(&persistent); err != nil { + t.Fatal(err) + } + + loaded, err := loadEffectiveConfig(path) + if err != nil { + t.Fatal(err) + } + if !loaded.Persistent || loaded.Config.UI.Title != "Persistent" { + t.Fatalf("persistent override not loaded: %#v", loaded) + } + if got := loaded.Config.Auth.APIKeys[0].Key; got != "01234567890123456789012345678901" { + t.Fatalf("bootstrap secret was not restored, got %q", got) + } +} + +func TestRunConfigCheckReportsWritableStorageAndWarning(t *testing.T) { + dataDir := filepath.Join(t.TempDir(), "state") + path := writeBootstrapConfig(t, dataDir) + var out bytes.Buffer + if err := runConfigCheck(path, &out); err != nil { + t.Fatal(err) + } + var got configCheckResult + if err := json.Unmarshal(out.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Status != "ok" || !got.StorageWritable || got.Workers != 1 { + t.Fatalf("unexpected result: %+v", got) + } + if len(got.Warnings) == 0 || !strings.Contains(strings.Join(got.Warnings, "\n"), "local_system_stats=true") { + t.Fatalf("expected remote local-system-stats warning, got %#v", got.Warnings) + } +} + +func TestRunProbe(t *testing.T) { + ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer ok.Close() + if err := runProbe(ok.URL, time.Second); err != nil { + t.Fatal(err) + } + + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "not ready", http.StatusServiceUnavailable) + })) + defer bad.Close() + if err := runProbe(bad.URL, time.Second); err == nil || !strings.Contains(err.Error(), "503") { + t.Fatalf("expected HTTP 503 probe error, got %v", err) + } +} + +func TestComposeDoesNotRepeatEntrypointAndHasHealthcheck(t *testing.T) { + b, err := os.ReadFile("../../docker-compose.yml") + if err != nil { + t.Fatal(err) + } + s := string(b) + if strings.Contains(s, `command: ["/ollama-gateway"`) { + t.Fatal("compose command must not repeat the image ENTRYPOINT") + } + if !strings.Contains(s, `command: ["-config"`) { + t.Fatal("compose command must pass -config as an ENTRYPOINT argument") + } + if !strings.Contains(s, "healthcheck:") || !strings.Contains(s, "/healthz") { + t.Fatal("compose must define a liveness healthcheck") + } +} + +func TestConfigWarningsFlagForwardedBypassAndBroadTrustedProxy(t *testing.T) { + cfg := &config.Config{ + Auth: config.AuthConfig{ + IPBypassUseForwardedIP: true, + TrustedProxies: []string{"10.0.0.0/8", "127.0.0.1/8"}, + IPBypass: []config.IPBypassConfig{{ + CIDRs: []string{"127.0.0.1/32"}, + Tenant: "local", + }}, + }, + Storage: config.StorageConfig{DataDir: "/var/lib/ollama-gateway"}, + } + warnings := strings.Join(configWarnings(cfg), "\n") + if !strings.Contains(warnings, "ip_bypass_use_forwarded_ip=true") { + t.Fatalf("missing forwarded bypass warning: %s", warnings) + } + if !strings.Contains(warnings, `broad CIDR "10.0.0.0/8"`) { + t.Fatalf("missing broad trusted proxy warning: %s", warnings) + } + if strings.Contains(warnings, `broad CIDR "127.0.0.1/8"`) { + t.Fatalf("loopback trusted proxy should not be flagged as broad: %s", warnings) + } +} + +func TestConfigWarningsFlagUnlimitedContextPolicy(t *testing.T) { + cfg := &config.Config{ + ModelCapabilities: config.ModelCapabilitiesConfig{Context: config.ContextPolicyConfig{MaxRequestedTokens: -1, DefaultWorkerTokens: -1}}, + Storage: config.StorageConfig{DataDir: "/var/lib/ollama-gateway"}, + } + warnings := strings.Join(configWarnings(cfg), "\n") + if !strings.Contains(warnings, "max_requested_tokens=-1") { + t.Fatalf("missing unlimited context cap warning: %s", warnings) + } + if !strings.Contains(warnings, "default_worker_tokens=-1") { + t.Fatalf("missing model-max fallback warning: %s", warnings) + } +} diff --git a/cmd/worker-telemetry/main.go b/cmd/worker-telemetry/main.go new file mode 100644 index 0000000..86726ef --- /dev/null +++ b/cmd/worker-telemetry/main.go @@ -0,0 +1,180 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/example/ollama-fair-gateway/internal/hoststats" +) + +type telemetry struct { + MemoryUsedBytes int64 `json:"memory_used_bytes,omitempty"` + MemoryTotalBytes int64 `json:"memory_total_bytes,omitempty"` + VRAMUsedBytes int64 `json:"vram_used_bytes,omitempty"` + VRAMTotalBytes int64 `json:"vram_total_bytes,omitempty"` + GPUUtilizationPct float64 `json:"gpu_utilization_percent,omitempty"` + GPUTemperatureC float64 `json:"gpu_temperature_c,omitempty"` + GPUPowerWatts float64 `json:"gpu_power_watts,omitempty"` + Source string `json:"source,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` + Error string `json:"error,omitempty"` +} + +type collector struct { + nvidia bool + nvidiaGPU string + amd bool + amdDevice string +} + +func (c collector) collect(ctx context.Context) telemetry { + out := telemetry{UpdatedAt: time.Now().UTC()} + var errs []string + mctx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond) + m, err := hoststats.ReadMemory(mctx) + cancel() + if err != nil { + errs = append(errs, "memory: "+err.Error()) + } else { + out.MemoryTotalBytes, out.MemoryUsedBytes = m.TotalBytes, m.UsedBytes + out.Source = appendSource(out.Source, "host-memory") + } + if c.nvidia { + gctx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond) + g, err := hoststats.ReadNVIDIA(gctx, c.nvidiaGPU) + cancel() + if err != nil { + errs = append(errs, "nvidia: "+err.Error()) + } else { + out.VRAMUsedBytes, out.VRAMTotalBytes = g.MemoryUsedBytes, g.MemoryTotalBytes + out.GPUUtilizationPct, out.GPUTemperatureC, out.GPUPowerWatts = g.UtilizationPercent, g.TemperatureC, g.PowerWatts + out.Source = appendSource(out.Source, "nvidia-smi") + } + } + if c.amd { + gctx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond) + g, err := hoststats.ReadAMD(gctx, c.amdDevice) + cancel() + if err != nil { + errs = append(errs, "amd: "+err.Error()) + } else { + out.VRAMUsedBytes, out.VRAMTotalBytes = g.MemoryUsedBytes, g.MemoryTotalBytes + out.GPUUtilizationPct, out.GPUTemperatureC, out.GPUPowerWatts = g.UtilizationPercent, g.TemperatureC, g.PowerWatts + out.Source = appendSource(out.Source, "amdgpu-sysfs") + } + } + out.Error = strings.Join(errs, "; ") + return out +} + +func appendSource(cur, next string) string { + if cur == "" { + return next + } + return cur + "+" + next +} + +type cidrAllowlist struct{ nets []*net.IPNet } + +func parseCIDRs(raw string) (cidrAllowlist, error) { + var out cidrAllowlist + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + _, n, err := net.ParseCIDR(part) + if err != nil { + return out, fmt.Errorf("invalid allow CIDR %q: %w", part, err) + } + out.nets = append(out.nets, n) + } + if len(out.nets) == 0 { + return out, fmt.Errorf("at least one allow CIDR is required") + } + return out, nil +} + +func (a cidrAllowlist) allowed(remote string) bool { + host, _, err := net.SplitHostPort(remote) + if err != nil { + host = remote + } + ip := net.ParseIP(strings.Trim(host, "[]")) + if ip == nil { + return false + } + for _, n := range a.nets { + if n.Contains(ip) { + return true + } + } + return false +} + +func main() { + listen := flag.String("listen", "127.0.0.1:11500", "listen address") + path := flag.String("path", "/telemetry", "telemetry path") + allow := flag.String("allow-cidrs", "127.0.0.1/32,::1/128", "comma-separated client CIDRs allowed to read telemetry") + nvidia := flag.Bool("nvidia-smi", false, "collect NVIDIA telemetry with nvidia-smi") + nvidiaGPU := flag.String("nvidia-gpu", "", "optional nvidia-smi GPU selector") + amd := flag.Bool("amd-sysfs", false, "collect Linux AMDGPU telemetry from sysfs") + amdDevice := flag.String("amd-device", "", "optional AMDGPU device path such as /sys/class/drm/card0/device; empty auto-detects") + once := flag.Bool("once", false, "print one telemetry sample as JSON and exit") + flag.Parse() + if !strings.HasPrefix(*path, "/") { + log.Fatal("-path must begin with /") + } + acl, err := parseCIDRs(*allow) + if err != nil { + log.Fatal(err) + } + c := collector{nvidia: *nvidia, nvidiaGPU: *nvidiaGPU, amd: *amd, amdDevice: *amdDevice} + if *once { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(c.collect(context.Background())); err != nil { + log.Fatal(err) + } + return + } + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) }) + mux.HandleFunc(*path, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if !acl.allowed(r.RemoteAddr) { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Content-Type-Options", "nosniff") + _ = json.NewEncoder(w).Encode(c.collect(r.Context())) + }) + srv := &http.Server{Addr: *listen, Handler: mux, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: time.Minute} + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + go func() { + <-ctx.Done() + shutdown, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _ = srv.Shutdown(shutdown) + }() + log.Printf("worker telemetry listening on http://%s%s allowed=%s nvidia=%t amd=%t", *listen, *path, *allow, *nvidia, *amd) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatal(err) + } +} diff --git a/cmd/worker-telemetry/main_test.go b/cmd/worker-telemetry/main_test.go new file mode 100644 index 0000000..be7b461 --- /dev/null +++ b/cmd/worker-telemetry/main_test.go @@ -0,0 +1,19 @@ +package main + +import "testing" + +func TestCIDRAllowlist(t *testing.T) { + a, err := parseCIDRs("127.0.0.1/32,10.2.19.0/24") + if err != nil { + t.Fatal(err) + } + if !a.allowed("10.2.19.42:1234") || !a.allowed("127.0.0.1:1") || a.allowed("10.2.18.1:5") { + t.Fatal("unexpected allowlist result") + } +} + +func TestAppendSource(t *testing.T) { + if got := appendSource("host-memory", "amdgpu-sysfs"); got != "host-memory+amdgpu-sysfs" { + t.Fatal(got) + } +} diff --git a/config.example.json b/config.example.json new file mode 100644 index 0000000..30b9efc --- /dev/null +++ b/config.example.json @@ -0,0 +1,335 @@ +{ + "server": { + "listen": ":8080", + "read_header_timeout": "10s", + "idle_timeout": "2m", + "max_request_duration": "30m", + "max_body_bytes": 67108864, + "metrics_public": true + }, + "auth": { + "oidc": { + "enabled": false + }, + "api_keys": [], + "ip_bypass": [ + { + "cidrs": [ + "127.0.0.1/32", + "::1/128" + ], + "tenant": "local", + "subject": "localhost", + "application": "local-tools", + "scopes": [ + "gateway:admin" + ] + } + ], + "trusted_proxies": [], + "ip_bypass_use_forwarded_ip": false + }, + "scheduler": { + "global_concurrency": 4, + "max_queue": 2048, + "max_queue_per_actor": 64, + "queue_timeout": "10m", + "default_tenant_weight": 1, + "default_actor_weight": 1, + "policies": { + "*": { + "tenant_weight": 1, + "actor_weight": 1, + "actor_credits_per_minute": 60, + "actor_burst_credits": 180, + "tenant_credits_per_minute": 300, + "tenant_burst_credits": 900 + }, + "internal": { + "tenant_weight": 2, + "actor_weight": 2, + "actor_credits_per_minute": 180, + "actor_burst_credits": 540, + "tenant_credits_per_minute": 900, + "tenant_burst_credits": 2700 + } + }, + "compute_paths": [ + "/api/generate", + "/api/chat", + "/api/embed", + "/api/embeddings", + "/v1/chat/completions", + "/v1/completions", + "/v1/embeddings", + "/v1/responses", + "/v1/messages" + ] + }, + "quota": { + "enabled": true + }, + "cost": { + "default": { + "input_credits_per_1k": 1, + "output_credits_per_1k": 3, + "compute_credits_per_second": 0, + "cached_input_factor": 1.0 + }, + "models": { + "small-*": { + "input_credits_per_1k": 0.5, + "output_credits_per_1k": 1.5 + }, + "large-*": { + "input_credits_per_1k": 2, + "output_credits_per_1k": 6 + } + }, + "default_max_output_tokens": 1024 + }, + "model_capabilities": { + "mode": "enforce", + "cache_ttl": "10m", + "context_guard": "reject", + "context": { + "max_requested_tokens": 32768, + "default_worker_tokens": 4096, + "estimation_margin_percent": 15, + "vision_reserve_tokens_per_image": 2048 + } + }, + "routing": { + "loaded_bonus": 60, + "installed_bonus": 30, + "throughput_bonus": 20, + "vram_pressure_penalty": 35, + "gpu_utilization_penalty": 10, + "avoid_vram_percent": 97 + }, + "workers": [ + { + "name": "mac-studio-m5-ultra", + "url": "http://127.0.0.1:11434", + "max_concurrent": 4, + "health_interval": "5s", + "memory_capacity_bytes": 274877906944, + "vram_capacity_bytes": 274877906944, + "labels": { + "host": "mac-studio-m5-ultra", + "memory": "unified", + "accelerator": "apple-gpu" + }, + "local_system_stats": true, + "model_concurrency": { + "*": 4 + }, + "model_placement": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + } + } + ], + "usage": { + "journal_dir": "./data/usage", + "buffer": 16384, + "flush_interval": "1s", + "retention": { + "detail_days": 30, + "daily_days": 400, + "monthly_months": 0, + "compaction_interval": "6h" + } + }, + "native": { + "management_requires_admin": true, + "control_worker": "mac-studio-m5-ultra" + }, + "ui": { + "enabled": true, + "path": "/admin", + "title": "Ollama Fair Gateway", + "recent_events": 10000, + "secure_cookies": false, + "oidc": { + "enabled": false + } + }, + "public_dashboard": { + "enabled": false, + "path": "/status", + "title": "Ollama Gateway Status", + "subtitle": "Live-Auslastung und Infrastruktur", + "refresh_interval": "2s", + "max_live_requests": 64, + "show_worker_names": false, + "show_model_names": false, + "show_resource_metrics": true, + "worker_display_names": {} + }, + "infrastructure": { + "node_name": "mac-studio-gateway", + "refresh_interval": "250ms", + "max_requests": 256 + }, + "storage": { + "data_dir": "./data", + "config_file": "gateway-config.json", + "api_keys_file": "api-keys.json", + "policies_file": "policies.json", + "metrics_file": "metrics.json", + "quota_file": "quota.json", + "worker_performance_file": "worker-performance.json", + "flush_interval": "10s", + "model_placement_file": "model-placement.json", + "worker_state_file": "worker-state.json", + "auto_tune_file": "auto-tune.json", + "warm_models_file": "warm-models.json", + "alerts_file": "alerts.json", + "conversations_file": "conversations.enc.json", + "batch_jobs_file": "batch-jobs.json", + "batch_jobs_dir": "batch" + }, + "reliability": { + "enabled": true, + "failure_threshold": 3, + "open_duration": "30s", + "retry_attempts": 2, + "retry_backoff": "50ms" + }, + "model_access": { + "default": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + }, + "tenants": {} + }, + "model_aliases": { + "fast": { + "models": [ + "qwen3:8b", + "gemma3:12b" + ] + }, + "coding": { + "models": [ + "qwen3.6:27b-q4_K_M", + "qwen3:8b" + ], + "required_capabilities": [ + "completion" + ] + } + }, + "service_classes": { + "default": "interactive", + "header": "X-Gateway-Service-Class", + "override_scope": "gateway:service-class", + "classes": { + "interactive": { + "weight": 4, + "max_queue_wait": "30s", + "max_concurrent": 0 + }, + "system": { + "weight": 8, + "max_queue_wait": "30s", + "max_concurrent": 1 + }, + "background": { + "weight": 1, + "max_queue_wait": "10m", + "max_concurrent": 1 + }, + "batch": { + "weight": 0.5, + "max_queue_wait": "30m", + "max_concurrent": 1 + } + } + }, + "auto_tuning": { + "enabled": true, + "max_concurrency": 4, + "samples_per_level": 2, + "max_tokens": 96, + "timeout": "10m", + "prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.", + "ttft_weight": 0.25, + "throughput_weight": 1 + }, + "conversations": { + "enabled": false, + "encryption_key": "${GATEWAY_CONVERSATION_KEY}", + "retention": "24h", + "max_entries": 1000, + "max_content_bytes": 2097152 + }, + "batch_jobs": { + "enabled": false, + "retention": "168h", + "max_jobs": 1000, + "max_concurrent": 1, + "max_input_bytes": 16777216 + }, + "opentelemetry": { + "enabled": false, + "endpoint": "http://127.0.0.1:4318", + "headers": {}, + "service_name": "ollama-fair-gateway", + "service_version": "", + "sample_ratio": 1, + "batch_size": 128, + "flush_interval": "2s", + "capture_content": false + }, + "warm_models": { + "enabled": true, + "reconcile_interval": "30s", + "operation_timeout": "2m", + "policies": { + "qwen3:8b": { + "class": "warm", + "workers": [ + "mac-studio-m5-ultra" + ], + "replicas": 1, + "preload": true, + "idle_timeout": "30m" + }, + "gemma4:*": { + "class": "cold", + "workers": [ + "mac-studio-m5-ultra" + ], + "replicas": 1, + "preload": false, + "idle_timeout": "10m" + } + } + }, + "alerts": { + "enabled": true, + "evaluation_interval": "15s", + "cooldown": "5m", + "history_limit": 500, + "thresholds": { + "worker_down_for": "30s", + "circuit_open": true, + "queue_depth": 100, + "vram_percent": 95, + "storage_bytes": 0, + "quota_remaining_percent": 10, + "oom": true, + "queue_wait": "30s" + }, + "webhooks": [], + "webhook_timeout": "5s", + "webhook_max_concurrent": 4, + "webhook_queue": 1024, + "webhook_retry_attempts": 3, + "webhook_retry_backoff": "500ms" + } +} diff --git a/config.oidc.example.json b/config.oidc.example.json new file mode 100644 index 0000000..e53fa53 --- /dev/null +++ b/config.oidc.example.json @@ -0,0 +1,340 @@ +{ + "server": { + "listen": ":8080", + "read_header_timeout": "10s", + "idle_timeout": "2m", + "max_request_duration": "30m", + "max_body_bytes": 67108864, + "metrics_public": false + }, + "auth": { + "oidc": { + "enabled": true, + "issuer": "${OIDC_ISSUER}", + "audience": "${OIDC_AUDIENCE}", + "tenant_claim": "tenant_id", + "application_claim": "azp", + "groups_claim": "groups", + "admin_groups": [ + "llm-gateway-admins" + ], + "clock_skew": "60s", + "jwks_refresh_min_interval": "10s", + "allowed_algorithms": [ + "RS256", + "PS256", + "ES256", + "EdDSA" + ] + }, + "api_keys": [ + { + "name": "automation", + "key": "${GATEWAY_AUTOMATION_KEY}", + "tenant": "internal", + "subject": "automation", + "application": "automation", + "scopes": [], + "service_class": "interactive" + } + ], + "ip_bypass": [ + { + "cidrs": [ + "10.42.0.0/24" + ], + "tenant": "internal", + "subject": "trusted-lan", + "application": "lan-services", + "scopes": [] + } + ], + "trusted_proxies": [ + "10.0.0.0/8" + ] + }, + "scheduler": { + "global_concurrency": 4, + "max_queue": 2048, + "max_queue_per_actor": 64, + "queue_timeout": "10m", + "default_tenant_weight": 1, + "default_actor_weight": 1, + "policies": { + "*": { + "tenant_weight": 1, + "actor_weight": 1, + "actor_credits_per_minute": 60, + "actor_burst_credits": 180, + "tenant_credits_per_minute": 300, + "tenant_burst_credits": 900 + }, + "internal": { + "tenant_weight": 2, + "actor_weight": 2, + "actor_credits_per_minute": 180, + "actor_burst_credits": 540, + "tenant_credits_per_minute": 900, + "tenant_burst_credits": 2700 + } + }, + "compute_paths": [ + "/api/generate", + "/api/chat", + "/api/embed", + "/api/embeddings", + "/v1/chat/completions", + "/v1/completions", + "/v1/embeddings", + "/v1/responses", + "/v1/messages" + ] + }, + "quota": { + "enabled": true + }, + "cost": { + "default": { + "input_credits_per_1k": 1, + "output_credits_per_1k": 3, + "compute_credits_per_second": 0, + "cached_input_factor": 1.0 + }, + "models": { + "small-*": { + "input_credits_per_1k": 0.5, + "output_credits_per_1k": 1.5 + }, + "large-*": { + "input_credits_per_1k": 2, + "output_credits_per_1k": 6 + } + }, + "default_max_output_tokens": 1024 + }, + "model_capabilities": { + "mode": "enforce", + "cache_ttl": "10m", + "context_guard": "reject", + "context": { + "max_requested_tokens": 32768, + "default_worker_tokens": 4096, + "estimation_margin_percent": 15, + "vision_reserve_tokens_per_image": 2048 + } + }, + "routing": { + "loaded_bonus": 60, + "installed_bonus": 30, + "throughput_bonus": 20, + "vram_pressure_penalty": 35, + "gpu_utilization_penalty": 10, + "avoid_vram_percent": 97 + }, + "workers": [ + { + "name": "mac-studio-m5-ultra", + "url": "http://127.0.0.1:11434", + "max_concurrent": 4, + "health_interval": "5s", + "memory_capacity_bytes": 274877906944, + "vram_capacity_bytes": 274877906944, + "labels": { + "host": "mac-studio-m5-ultra", + "memory": "unified", + "accelerator": "apple-gpu" + }, + "local_system_stats": true, + "model_concurrency": { + "*": 4 + }, + "model_placement": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + } + } + ], + "usage": { + "journal_dir": "./data/usage", + "buffer": 16384, + "flush_interval": "1s", + "retention": { + "detail_days": 30, + "daily_days": 400, + "monthly_months": 0, + "compaction_interval": "6h" + } + }, + "native": { + "management_requires_admin": true, + "control_worker": "mac-studio-m5-ultra" + }, + "ui": { + "enabled": true, + "path": "/admin", + "title": "Ollama Fair Gateway", + "recent_events": 20000, + "session_secret": "${GATEWAY_UI_SESSION_SECRET}", + "secure_cookies": true, + "oidc": { + "enabled": true, + "client_id": "${OIDC_UI_CLIENT_ID}", + "client_secret": "${OIDC_UI_CLIENT_SECRET}", + "scopes": [ + "openid", + "profile", + "email" + ], + "redirect_url": "${GATEWAY_UI_REDIRECT_URL}" + } + }, + "infrastructure": { + "node_name": "mac-studio-gateway", + "refresh_interval": "250ms", + "max_requests": 256 + }, + "storage": { + "data_dir": "./data", + "config_file": "gateway-config.json", + "api_keys_file": "api-keys.json", + "policies_file": "policies.json", + "metrics_file": "metrics.json", + "quota_file": "quota.json", + "worker_performance_file": "worker-performance.json", + "flush_interval": "10s", + "model_placement_file": "model-placement.json", + "worker_state_file": "worker-state.json", + "auto_tune_file": "auto-tune.json", + "warm_models_file": "warm-models.json", + "alerts_file": "alerts.json", + "conversations_file": "conversations.enc.json", + "batch_jobs_file": "batch-jobs.json", + "batch_jobs_dir": "batch" + }, + "reliability": { + "enabled": true, + "failure_threshold": 3, + "open_duration": "30s", + "retry_attempts": 2, + "retry_backoff": "50ms" + }, + "model_access": { + "default": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + }, + "tenants": {} + }, + "model_aliases": {}, + "service_classes": { + "default": "interactive", + "header": "X-Gateway-Service-Class", + "override_scope": "gateway:service-class", + "classes": { + "interactive": { + "weight": 4, + "max_queue_wait": "30s", + "max_concurrent": 0 + }, + "system": { + "weight": 8, + "max_queue_wait": "30s", + "max_concurrent": 1 + }, + "background": { + "weight": 1, + "max_queue_wait": "10m", + "max_concurrent": 1 + }, + "batch": { + "weight": 0.5, + "max_queue_wait": "30m", + "max_concurrent": 1 + } + } + }, + "auto_tuning": { + "enabled": true, + "max_concurrency": 4, + "samples_per_level": 2, + "max_tokens": 96, + "timeout": "10m", + "prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.", + "ttft_weight": 0.25, + "throughput_weight": 1 + }, + "conversations": { + "enabled": false, + "encryption_key": "${GATEWAY_CONVERSATION_KEY}", + "retention": "24h", + "max_entries": 1000, + "max_content_bytes": 2097152 + }, + "batch_jobs": { + "enabled": false, + "retention": "168h", + "max_jobs": 1000, + "max_concurrent": 1, + "max_input_bytes": 16777216 + }, + "opentelemetry": { + "enabled": false, + "endpoint": "http://127.0.0.1:4318", + "headers": {}, + "service_name": "ollama-fair-gateway", + "service_version": "", + "sample_ratio": 1, + "batch_size": 128, + "flush_interval": "2s", + "capture_content": false + }, + "warm_models": { + "enabled": true, + "reconcile_interval": "30s", + "operation_timeout": "2m", + "policies": { + "qwen3:8b": { + "class": "warm", + "workers": [ + "mac-studio-m5-ultra" + ], + "replicas": 1, + "preload": true, + "idle_timeout": "30m" + }, + "gemma4:*": { + "class": "cold", + "workers": [ + "mac-studio-m5-ultra" + ], + "replicas": 1, + "preload": false, + "idle_timeout": "10m" + } + } + }, + "alerts": { + "enabled": true, + "evaluation_interval": "15s", + "cooldown": "5m", + "history_limit": 500, + "thresholds": { + "worker_down_for": "30s", + "circuit_open": true, + "queue_depth": 100, + "vram_percent": 95, + "storage_bytes": 0, + "quota_remaining_percent": 10, + "oom": true, + "queue_wait": "30s" + }, + "webhooks": [], + "webhook_timeout": "5s", + "webhook_max_concurrent": 4, + "webhook_queue": 1024, + "webhook_retry_attempts": 3, + "webhook_retry_backoff": "500ms" + } +} diff --git a/config.openwebui.example.json b/config.openwebui.example.json new file mode 100644 index 0000000..439b3f6 --- /dev/null +++ b/config.openwebui.example.json @@ -0,0 +1,316 @@ +{ + "server": { + "listen": ":8080", + "read_header_timeout": "10s", + "idle_timeout": "2m", + "max_request_duration": "30m", + "max_body_bytes": 67108864, + "metrics_public": true + }, + "auth": { + "oidc": { + "enabled": false + }, + "api_keys": [ + { + "name": "openwebui", + "key": "${OPENWEBUI_GATEWAY_KEY}", + "tenant": "interactive", + "subject": "openwebui", + "application": "openwebui", + "scopes": [], + "service_class": "interactive" + } + ], + "ip_bypass": [ + { + "cidrs": [ + "127.0.0.1/32", + "::1/128" + ], + "tenant": "local", + "subject": "localhost", + "application": "local-tools", + "scopes": [ + "gateway:admin" + ] + } + ], + "trusted_proxies": [] + }, + "scheduler": { + "global_concurrency": 4, + "max_queue": 2048, + "max_queue_per_actor": 64, + "queue_timeout": "10m", + "default_tenant_weight": 1, + "default_actor_weight": 1, + "policies": { + "*": { + "tenant_weight": 1, + "actor_weight": 1, + "actor_credits_per_minute": 60, + "actor_burst_credits": 180, + "tenant_credits_per_minute": 300, + "tenant_burst_credits": 900 + }, + "internal": { + "tenant_weight": 2, + "actor_weight": 2, + "actor_credits_per_minute": 180, + "actor_burst_credits": 540, + "tenant_credits_per_minute": 900, + "tenant_burst_credits": 2700 + } + }, + "compute_paths": [ + "/api/generate", + "/api/chat", + "/api/embed", + "/api/embeddings", + "/v1/chat/completions", + "/v1/completions", + "/v1/embeddings", + "/v1/responses", + "/v1/messages" + ] + }, + "quota": { + "enabled": true + }, + "cost": { + "default": { + "input_credits_per_1k": 1, + "output_credits_per_1k": 3, + "compute_credits_per_second": 0, + "cached_input_factor": 1.0 + }, + "models": { + "small-*": { + "input_credits_per_1k": 0.5, + "output_credits_per_1k": 1.5 + }, + "large-*": { + "input_credits_per_1k": 2, + "output_credits_per_1k": 6 + } + }, + "default_max_output_tokens": 1024 + }, + "model_capabilities": { + "mode": "enforce", + "cache_ttl": "10m", + "context_guard": "reject", + "context": { + "max_requested_tokens": 32768, + "default_worker_tokens": 4096, + "estimation_margin_percent": 15, + "vision_reserve_tokens_per_image": 2048 + } + }, + "routing": { + "loaded_bonus": 60, + "installed_bonus": 30, + "throughput_bonus": 20, + "vram_pressure_penalty": 35, + "gpu_utilization_penalty": 10, + "avoid_vram_percent": 97 + }, + "workers": [ + { + "name": "mac-studio-m5-ultra", + "url": "http://127.0.0.1:11434", + "max_concurrent": 4, + "health_interval": "5s", + "memory_capacity_bytes": 274877906944, + "vram_capacity_bytes": 274877906944, + "labels": { + "host": "mac-studio-m5-ultra", + "memory": "unified", + "accelerator": "apple-gpu" + }, + "local_system_stats": true, + "model_concurrency": { + "*": 4 + }, + "model_placement": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + } + } + ], + "usage": { + "journal_dir": "./data/usage", + "buffer": 16384, + "flush_interval": "1s", + "retention": { + "detail_days": 30, + "daily_days": 400, + "monthly_months": 0, + "compaction_interval": "6h" + } + }, + "native": { + "management_requires_admin": true, + "control_worker": "mac-studio-m5-ultra" + }, + "ui": { + "enabled": true, + "path": "/admin", + "title": "Ollama Fair Gateway", + "recent_events": 10000, + "secure_cookies": false, + "oidc": { + "enabled": false + } + }, + "infrastructure": { + "node_name": "mac-studio-gateway", + "refresh_interval": "250ms", + "max_requests": 256 + }, + "storage": { + "data_dir": "./data", + "config_file": "gateway-config.json", + "api_keys_file": "api-keys.json", + "policies_file": "policies.json", + "metrics_file": "metrics.json", + "quota_file": "quota.json", + "worker_performance_file": "worker-performance.json", + "flush_interval": "10s", + "model_placement_file": "model-placement.json", + "worker_state_file": "worker-state.json", + "auto_tune_file": "auto-tune.json", + "warm_models_file": "warm-models.json", + "alerts_file": "alerts.json", + "conversations_file": "conversations.enc.json", + "batch_jobs_file": "batch-jobs.json", + "batch_jobs_dir": "batch" + }, + "reliability": { + "enabled": true, + "failure_threshold": 3, + "open_duration": "30s", + "retry_attempts": 2, + "retry_backoff": "50ms" + }, + "model_access": { + "default": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + }, + "tenants": {} + }, + "model_aliases": {}, + "service_classes": { + "default": "interactive", + "header": "X-Gateway-Service-Class", + "override_scope": "gateway:service-class", + "classes": { + "interactive": { + "weight": 4, + "max_queue_wait": "30s", + "max_concurrent": 0 + }, + "system": { + "weight": 8, + "max_queue_wait": "30s", + "max_concurrent": 1 + }, + "background": { + "weight": 1, + "max_queue_wait": "10m", + "max_concurrent": 1 + }, + "batch": { + "weight": 0.5, + "max_queue_wait": "30m", + "max_concurrent": 1 + } + } + }, + "auto_tuning": { + "enabled": true, + "max_concurrency": 4, + "samples_per_level": 2, + "max_tokens": 96, + "timeout": "10m", + "prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.", + "ttft_weight": 0.25, + "throughput_weight": 1 + }, + "conversations": { + "enabled": false, + "encryption_key": "${GATEWAY_CONVERSATION_KEY}", + "retention": "24h", + "max_entries": 1000, + "max_content_bytes": 2097152 + }, + "batch_jobs": { + "enabled": false, + "retention": "168h", + "max_jobs": 1000, + "max_concurrent": 1, + "max_input_bytes": 16777216 + }, + "opentelemetry": { + "enabled": false, + "endpoint": "http://127.0.0.1:4318", + "headers": {}, + "service_name": "ollama-fair-gateway", + "service_version": "", + "sample_ratio": 1, + "batch_size": 128, + "flush_interval": "2s", + "capture_content": false + }, + "warm_models": { + "enabled": true, + "reconcile_interval": "30s", + "operation_timeout": "2m", + "policies": { + "qwen3:8b": { + "class": "warm", + "workers": [ + "mac-studio-m5-ultra" + ], + "replicas": 1, + "preload": true, + "idle_timeout": "30m" + }, + "gemma4:*": { + "class": "cold", + "workers": [ + "mac-studio-m5-ultra" + ], + "replicas": 1, + "preload": false, + "idle_timeout": "10m" + } + } + }, + "alerts": { + "enabled": true, + "evaluation_interval": "15s", + "cooldown": "5m", + "history_limit": 500, + "thresholds": { + "worker_down_for": "30s", + "circuit_open": true, + "queue_depth": 100, + "vram_percent": 95, + "storage_bytes": 0, + "quota_remaining_percent": 10, + "oom": true, + "queue_wait": "30s" + }, + "webhooks": [], + "webhook_timeout": "5s", + "webhook_max_concurrent": 4, + "webhook_queue": 1024, + "webhook_retry_attempts": 3, + "webhook_retry_backoff": "500ms" + } +} diff --git a/config.placement.example.json b/config.placement.example.json new file mode 100644 index 0000000..c77dcb9 --- /dev/null +++ b/config.placement.example.json @@ -0,0 +1,346 @@ +{ + "server": { + "listen": ":8080", + "read_header_timeout": "10s", + "idle_timeout": "2m", + "max_request_duration": "30m", + "max_body_bytes": 67108864, + "metrics_public": true + }, + "auth": { + "oidc": { + "enabled": false + }, + "api_keys": [], + "ip_bypass": [ + { + "cidrs": [ + "127.0.0.1/32", + "::1/128" + ], + "tenant": "local", + "subject": "localhost", + "application": "local-tools", + "scopes": [ + "gateway:admin" + ] + } + ], + "trusted_proxies": [] + }, + "scheduler": { + "global_concurrency": 6, + "max_queue": 2048, + "max_queue_per_actor": 64, + "queue_timeout": "10m", + "default_tenant_weight": 1, + "default_actor_weight": 1, + "policies": { + "*": { + "tenant_weight": 1, + "actor_weight": 1, + "actor_credits_per_minute": 60, + "actor_burst_credits": 180, + "tenant_credits_per_minute": 300, + "tenant_burst_credits": 900 + }, + "internal": { + "tenant_weight": 2, + "actor_weight": 2, + "actor_credits_per_minute": 180, + "actor_burst_credits": 540, + "tenant_credits_per_minute": 900, + "tenant_burst_credits": 2700 + } + }, + "compute_paths": [ + "/api/generate", + "/api/chat", + "/api/embed", + "/api/embeddings", + "/v1/chat/completions", + "/v1/completions", + "/v1/embeddings", + "/v1/responses", + "/v1/messages" + ] + }, + "quota": { + "enabled": true + }, + "cost": { + "default": { + "input_credits_per_1k": 1, + "output_credits_per_1k": 3, + "compute_credits_per_second": 0, + "cached_input_factor": 1.0 + }, + "models": { + "small-*": { + "input_credits_per_1k": 0.5, + "output_credits_per_1k": 1.5 + }, + "large-*": { + "input_credits_per_1k": 2, + "output_credits_per_1k": 6 + } + }, + "default_max_output_tokens": 1024 + }, + "model_capabilities": { + "mode": "enforce", + "cache_ttl": "10m", + "context_guard": "reject", + "context": { + "max_requested_tokens": 32768, + "default_worker_tokens": 4096, + "estimation_margin_percent": 15, + "vision_reserve_tokens_per_image": 2048 + } + }, + "routing": { + "loaded_bonus": 60, + "installed_bonus": 30, + "throughput_bonus": 20, + "vram_pressure_penalty": 35, + "gpu_utilization_penalty": 10, + "avoid_vram_percent": 97 + }, + "workers": [ + { + "name": "m5-ultra", + "url": "http://10.10.11.123:11434", + "max_concurrent": 4, + "health_interval": "5s", + "memory_capacity_bytes": 274877906944, + "vram_capacity_bytes": 274877906944, + "labels": { + "host": "m5-ultra", + "memory": "unified", + "accelerator": "apple-gpu" + }, + "local_system_stats": false, + "model_concurrency": { + "qwen3:8b": 4, + "gemma4:*": 2, + "qwen3.6:27b-*": 1, + "orcarouter/Qwen3.8-27B-Uncensored:*": 1, + "*": 2 + }, + "model_placement": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + } + }, + { + "name": "rtx-4090", + "url": "http://10.10.11.124:11434", + "max_concurrent": 2, + "health_interval": "5s", + "memory_capacity_bytes": 68719476736, + "vram_capacity_bytes": 25769803776, + "local_system_stats": false, + "nvidia_smi": false, + "labels": { + "host": "rtx-4090", + "memory": "system+dedicated-vram", + "accelerator": "nvidia-rtx-4090" + }, + "model_concurrency": { + "qwen3:8b": 2, + "gemma3:12b": 2, + "gemma4:*": 1, + "*": 1 + }, + "model_placement": { + "mode": "whitelist", + "allowed_models": [ + "qwen3:8b", + "gemma3:12b", + "gemma4:*", + "embeddinggemma:*", + "qwen3-embedding:*" + ], + "denied_models": [ + "qwen3.6:27b-*", + "orcarouter/Qwen3.8-27B-Uncensored:*" + ] + } + } + ], + "usage": { + "journal_dir": "./data/usage", + "buffer": 16384, + "flush_interval": "1s", + "retention": { + "detail_days": 30, + "daily_days": 400, + "monthly_months": 0, + "compaction_interval": "6h" + } + }, + "native": { + "management_requires_admin": true, + "control_worker": "m5-ultra" + }, + "ui": { + "enabled": true, + "path": "/admin", + "title": "Ollama Fair Gateway", + "recent_events": 10000, + "secure_cookies": false, + "oidc": { + "enabled": false + } + }, + "infrastructure": { + "node_name": "dual-worker-gateway", + "refresh_interval": "250ms", + "max_requests": 256 + }, + "storage": { + "data_dir": "./data", + "config_file": "gateway-config.json", + "api_keys_file": "api-keys.json", + "policies_file": "policies.json", + "metrics_file": "metrics.json", + "quota_file": "quota.json", + "worker_performance_file": "worker-performance.json", + "flush_interval": "10s", + "model_placement_file": "model-placement.json", + "worker_state_file": "worker-state.json", + "auto_tune_file": "auto-tune.json", + "warm_models_file": "warm-models.json", + "alerts_file": "alerts.json", + "conversations_file": "conversations.enc.json", + "batch_jobs_file": "batch-jobs.json", + "batch_jobs_dir": "batch" + }, + "reliability": { + "enabled": true, + "failure_threshold": 3, + "open_duration": "30s", + "retry_attempts": 2, + "retry_backoff": "50ms" + }, + "model_access": { + "default": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + }, + "tenants": {} + }, + "model_aliases": {}, + "service_classes": { + "default": "interactive", + "header": "X-Gateway-Service-Class", + "override_scope": "gateway:service-class", + "classes": { + "interactive": { + "weight": 4, + "max_queue_wait": "30s", + "max_concurrent": 0 + }, + "system": { + "weight": 8, + "max_queue_wait": "30s", + "max_concurrent": 1 + }, + "background": { + "weight": 1, + "max_queue_wait": "10m", + "max_concurrent": 1 + }, + "batch": { + "weight": 0.5, + "max_queue_wait": "30m", + "max_concurrent": 1 + } + } + }, + "auto_tuning": { + "enabled": true, + "max_concurrency": 4, + "samples_per_level": 2, + "max_tokens": 96, + "timeout": "10m", + "prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.", + "ttft_weight": 0.25, + "throughput_weight": 1 + }, + "conversations": { + "enabled": false, + "encryption_key": "${GATEWAY_CONVERSATION_KEY}", + "retention": "24h", + "max_entries": 1000, + "max_content_bytes": 2097152 + }, + "batch_jobs": { + "enabled": false, + "retention": "168h", + "max_jobs": 1000, + "max_concurrent": 1, + "max_input_bytes": 16777216 + }, + "opentelemetry": { + "enabled": false, + "endpoint": "http://127.0.0.1:4318", + "headers": {}, + "service_name": "ollama-fair-gateway", + "service_version": "", + "sample_ratio": 1, + "batch_size": 128, + "flush_interval": "2s", + "capture_content": false + }, + "warm_models": { + "enabled": true, + "reconcile_interval": "30s", + "operation_timeout": "2m", + "policies": { + "qwen3:8b": { + "class": "warm", + "workers": [ + "m5-ultra" + ], + "replicas": 1, + "preload": true, + "idle_timeout": "30m" + }, + "gemma4:*": { + "class": "cold", + "workers": [ + "m5-ultra", + "rtx-4090" + ], + "replicas": 1, + "preload": false, + "idle_timeout": "10m" + } + } + }, + "alerts": { + "enabled": true, + "evaluation_interval": "15s", + "cooldown": "5m", + "history_limit": 500, + "thresholds": { + "worker_down_for": "30s", + "circuit_open": true, + "queue_depth": 100, + "vram_percent": 95, + "storage_bytes": 0, + "quota_remaining_percent": 10, + "oom": true, + "queue_wait": "30s" + }, + "webhooks": [], + "webhook_timeout": "5s", + "webhook_max_concurrent": 4, + "webhook_queue": 1024, + "webhook_retry_attempts": 3, + "webhook_retry_backoff": "500ms" + } +} diff --git a/config.rtx4090.example.json b/config.rtx4090.example.json new file mode 100644 index 0000000..069f366 --- /dev/null +++ b/config.rtx4090.example.json @@ -0,0 +1,339 @@ +{ + "server": { + "listen": ":8080", + "read_header_timeout": "10s", + "idle_timeout": "2m", + "max_request_duration": "30m", + "max_body_bytes": 67108864, + "metrics_public": true + }, + "auth": { + "oidc": { + "enabled": false + }, + "api_keys": [ + { + "name": "openwebui", + "key": "${OPENWEBUI_GATEWAY_KEY}", + "tenant": "interactive", + "subject": "openwebui", + "application": "openwebui", + "scopes": [], + "service_class": "interactive" + } + ], + "ip_bypass": [ + { + "cidrs": [ + "127.0.0.1/32", + "::1/128" + ], + "tenant": "local", + "subject": "localhost", + "application": "local-tools", + "scopes": [ + "gateway:admin" + ] + } + ], + "trusted_proxies": [] + }, + "scheduler": { + "global_concurrency": 2, + "max_queue": 2048, + "max_queue_per_actor": 64, + "queue_timeout": "10m", + "default_tenant_weight": 1, + "default_actor_weight": 1, + "policies": { + "*": { + "tenant_weight": 1, + "actor_weight": 1, + "actor_credits_per_minute": 60, + "actor_burst_credits": 180, + "tenant_credits_per_minute": 300, + "tenant_burst_credits": 900 + }, + "internal": { + "tenant_weight": 2, + "actor_weight": 2, + "actor_credits_per_minute": 180, + "actor_burst_credits": 540, + "tenant_credits_per_minute": 900, + "tenant_burst_credits": 2700 + } + }, + "compute_paths": [ + "/api/generate", + "/api/chat", + "/api/embed", + "/api/embeddings", + "/v1/chat/completions", + "/v1/completions", + "/v1/embeddings", + "/v1/responses", + "/v1/messages" + ] + }, + "quota": { + "enabled": true + }, + "cost": { + "default": { + "input_credits_per_1k": 1, + "output_credits_per_1k": 3, + "compute_credits_per_second": 0, + "cached_input_factor": 1.0 + }, + "models": { + "small-*": { + "input_credits_per_1k": 0.5, + "output_credits_per_1k": 1.5 + }, + "large-*": { + "input_credits_per_1k": 2, + "output_credits_per_1k": 6 + } + }, + "default_max_output_tokens": 1024 + }, + "model_capabilities": { + "mode": "enforce", + "cache_ttl": "10m", + "context_guard": "reject", + "context": { + "max_requested_tokens": 32768, + "default_worker_tokens": 4096, + "estimation_margin_percent": 15, + "vision_reserve_tokens_per_image": 2048 + } + }, + "routing": { + "loaded_bonus": 60, + "installed_bonus": 30, + "throughput_bonus": 20, + "vram_pressure_penalty": 35, + "gpu_utilization_penalty": 10, + "avoid_vram_percent": 97 + }, + "workers": [ + { + "name": "rtx-4090", + "url": "http://127.0.0.1:11434", + "max_concurrent": 2, + "model_concurrency": { + "qwen3:8b": 2, + "gemma3:12b": 2, + "gemma4:*": 1, + "qwen3.6:27b-*": 1, + "orcarouter/Qwen3.8-27B-Uncensored:*": 1, + "*": 1 + }, + "health_interval": "5s", + "memory_capacity_bytes": 68719476736, + "vram_capacity_bytes": 25769803776, + "local_system_stats": true, + "nvidia_smi": true, + "nvidia_gpu": "0", + "labels": { + "host": "rtx-4090-pc", + "memory": "system+dedicated-vram", + "accelerator": "nvidia-rtx-4090" + }, + "model_placement": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + } + } + ], + "usage": { + "journal_dir": "./data/usage", + "buffer": 16384, + "flush_interval": "1s", + "retention": { + "detail_days": 30, + "daily_days": 400, + "monthly_months": 0, + "compaction_interval": "6h" + } + }, + "native": { + "management_requires_admin": true, + "control_worker": "rtx-4090" + }, + "ui": { + "enabled": true, + "path": "/admin", + "title": "Ollama Fair Gateway", + "recent_events": 10000, + "secure_cookies": false, + "oidc": { + "enabled": false + } + }, + "infrastructure": { + "node_name": "rtx-4090-gateway", + "refresh_interval": "250ms", + "max_requests": 256 + }, + "storage": { + "data_dir": "./data", + "config_file": "gateway-config.json", + "api_keys_file": "api-keys.json", + "policies_file": "policies.json", + "metrics_file": "metrics.json", + "quota_file": "quota.json", + "worker_performance_file": "worker-performance.json", + "flush_interval": "10s", + "model_placement_file": "model-placement.json", + "worker_state_file": "worker-state.json", + "auto_tune_file": "auto-tune.json", + "warm_models_file": "warm-models.json", + "alerts_file": "alerts.json", + "conversations_file": "conversations.enc.json", + "batch_jobs_file": "batch-jobs.json", + "batch_jobs_dir": "batch" + }, + "reliability": { + "enabled": true, + "failure_threshold": 3, + "open_duration": "30s", + "retry_attempts": 2, + "retry_backoff": "50ms" + }, + "model_access": { + "default": { + "mode": "allow_all", + "allowed_models": [], + "denied_models": [] + }, + "tenants": {} + }, + "model_aliases": { + "fast": { + "models": [ + "qwen3:8b", + "gemma3:12b" + ] + }, + "coding": { + "models": [ + "qwen3.6:27b-q4_K_M", + "qwen3:8b" + ], + "required_capabilities": [ + "completion" + ] + } + }, + "service_classes": { + "default": "interactive", + "header": "X-Gateway-Service-Class", + "override_scope": "gateway:service-class", + "classes": { + "interactive": { + "weight": 4, + "max_queue_wait": "30s", + "max_concurrent": 0 + }, + "system": { + "weight": 8, + "max_queue_wait": "30s", + "max_concurrent": 1 + }, + "background": { + "weight": 1, + "max_queue_wait": "10m", + "max_concurrent": 1 + }, + "batch": { + "weight": 0.5, + "max_queue_wait": "30m", + "max_concurrent": 1 + } + } + }, + "auto_tuning": { + "enabled": true, + "max_concurrency": 4, + "samples_per_level": 2, + "max_tokens": 96, + "timeout": "10m", + "prompt": "Write a short deterministic explanation of why bounded concurrency matters for local LLM inference.", + "ttft_weight": 0.25, + "throughput_weight": 1 + }, + "conversations": { + "enabled": false, + "encryption_key": "${GATEWAY_CONVERSATION_KEY}", + "retention": "24h", + "max_entries": 1000, + "max_content_bytes": 2097152 + }, + "batch_jobs": { + "enabled": false, + "retention": "168h", + "max_jobs": 1000, + "max_concurrent": 1, + "max_input_bytes": 16777216 + }, + "opentelemetry": { + "enabled": false, + "endpoint": "http://127.0.0.1:4318", + "headers": {}, + "service_name": "ollama-fair-gateway", + "service_version": "", + "sample_ratio": 1, + "batch_size": 128, + "flush_interval": "2s", + "capture_content": false + }, + "warm_models": { + "enabled": true, + "reconcile_interval": "30s", + "operation_timeout": "2m", + "policies": { + "qwen3:8b": { + "class": "warm", + "workers": [ + "rtx-4090" + ], + "replicas": 1, + "preload": true, + "idle_timeout": "30m" + }, + "gemma4:*": { + "class": "cold", + "workers": [ + "rtx-4090" + ], + "replicas": 1, + "preload": false, + "idle_timeout": "10m" + } + } + }, + "alerts": { + "enabled": true, + "evaluation_interval": "15s", + "cooldown": "5m", + "history_limit": 500, + "thresholds": { + "worker_down_for": "30s", + "circuit_open": true, + "queue_depth": 100, + "vram_percent": 95, + "storage_bytes": 0, + "quota_remaining_percent": 10, + "oom": true, + "queue_wait": "30s" + }, + "webhooks": [], + "webhook_timeout": "5s", + "webhook_max_concurrent": 4, + "webhook_queue": 1024, + "webhook_retry_attempts": 3, + "webhook_retry_backoff": "500ms" + } +} diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 0000000..940bb62 --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,32 @@ +services: + gateway: + build: . + command: ["-config", "/etc/ollama-gateway/config.json"] + ports: + - "${GATEWAY_PUBLISH_ADDRESS:-127.0.0.1}:${GATEWAY_PUBLISH_PORT:-9080}:8080" + volumes: + - ${GATEWAY_CONFIG:?set GATEWAY_CONFIG to the production bootstrap JSON}:/etc/ollama-gateway/config.json:ro + - gateway-data:/data + user: "65532:65532" + read_only: true + tmpfs: + - /tmp:size=64m,mode=1777 + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: ["CMD", "/ollama-gateway", "-probe", "http://127.0.0.1:8080/healthz", "-probe-timeout", "2s"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 10s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + restart: unless-stopped + +volumes: + gateway-data: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..446df39 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +services: + gateway: + build: . + command: ["-config", "/etc/ollama-gateway/config.json"] + ports: + - "${GATEWAY_PUBLISH_ADDRESS:-127.0.0.1}:${GATEWAY_PUBLISH_PORT:-9080}:8080" + volumes: + - ${GATEWAY_CONFIG:-./config.example.json}:/etc/ollama-gateway/config.json:ro + - gateway-data:/data + read_only: true + tmpfs: + - /tmp:size=64m,mode=1777 + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: ["CMD", "/ollama-gateway", "-probe", "http://127.0.0.1:8080/healthz", "-probe-timeout", "2s"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 10s + restart: unless-stopped + +volumes: + gateway-data: diff --git a/docs/ALERTS.md b/docs/ALERTS.md new file mode 100644 index 0000000..50d6165 --- /dev/null +++ b/docs/ALERTS.md @@ -0,0 +1,78 @@ +# Alerts and signed webhooks + +The alert manager evaluates bounded operational conditions and persists firing/resolved history. It never includes prompt, response, tool, or conversation content in webhook payloads. + +## Conditions + +Supported conditions include: + +- worker unhealthy for `worker_down_for`; +- circuit breaker open; +- queue depth threshold; +- oldest queue wait threshold; +- VRAM pressure; +- repeated OOM indication from the worker/circuit error; +- local gateway storage size; +- actor or tenant quota remaining percentage. + +Example: + +```json +{ + "alerts": { + "enabled": true, + "evaluation_interval": "15s", + "cooldown": "5m", + "history_limit": 500, + "webhook_timeout": "5s", + "webhook_max_concurrent": 4, + "webhook_queue": 1024, + "webhook_retry_attempts": 3, + "webhook_retry_backoff": "500ms", + "thresholds": { + "worker_down_for": "30s", + "circuit_open": true, + "queue_depth": 100, + "queue_wait": "30s", + "vram_percent": 95, + "storage_bytes": 0, + "quota_remaining_percent": 10, + "oom": true + }, + "webhooks": [] + } +} +``` + +## Webhook delivery + +A webhook payload has a stable event ID and is delivered asynchronously from a bounded queue. Transient transport failures, HTTP 429, and HTTP 5xx are retried with bounded exponential backoff. Other HTTP 4xx responses are treated as permanent failures. + +Headers: + +- `X-Ollama-Gateway-Event-ID` +- `X-Ollama-Gateway-Timestamp` +- `X-Ollama-Gateway-Delivery-Attempt` +- `X-Ollama-Gateway-Signature` when a secret is configured + +Signature format: + +```text +sha256=HMAC_SHA256(secret, timestamp + "." + raw_json_body) +``` + +Consumers should deduplicate by event ID because retries intentionally reuse the same event ID. + +## Hot-path isolation + +Quota observations happen during request admission, but disk persistence and webhook I/O are not performed synchronously on that request path. State persistence is coalesced asynchronously and webhook delivery uses a bounded queue. + +## Secrets + +Webhook secrets are redacted from the Admin JSON configuration view. OpenTelemetry header values are redacted there as well. Saving the redacted configuration restores the existing secret values instead of persisting the literal `` placeholder. + +## Metrics + +- `ollama_gateway_queue_oldest_wait_seconds` +- `ollama_gateway_alerts_active` +- `ollama_gateway_alerts_last_evaluate_timestamp_seconds` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..f2576a5 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,77 @@ +# Architecture + +## Request path + +```text +HTTP request + -> authentication / identity + -> compute request parsing + cost estimate + -> in-memory quota reservation + -> hierarchical in-memory WFQ + -> model-aware worker selection + -> streaming reverse proxy + -> actual usage reconciliation + -> in-memory usage aggregation + -> optional asynchronous JSONL journal +``` + +Management and blob endpoints do not enter the compute scheduler and keep their request body streaming end-to-end. + +## Single authoritative engine + +The gateway deliberately has no external coordination backend. All mutable control state belongs to one process. This means there is exactly one authoritative view of queue order, quota buckets and worker slots and therefore no distributed lock, lease renewal, consensus or fail-open state split. + +The hot-path structures are Go-native: + +- scheduler maps and heaps protected by one scheduler mutex; +- worker load using atomic counters; +- quota buckets protected by a small mutex; +- runtime policy and session maps behind RWMutex/Mutex; +- live request snapshots behind the lifecycle tracker; +- usage summaries updated in memory and journaled asynchronously. + +## Fair scheduler + +`internal/scheduler` implements two-level hierarchical weighted fair queueing. + +A tenant has a root service score. Inside that tenant each queued request has an actor virtual-finish score derived from request cost and actor weight. When capacity becomes free, the scheduler selects the tenant with the lowest root score and then its request with the lowest actor finish score. + +This yields two properties: + +- tenant fairness is not defeated by creating many actors; +- actors within one tenant still share service fairly. + +The scheduler is event-driven. A worker slot release signals the scheduler immediately; there is no polling backend. + +## Quotas + +`internal/quota` is an in-memory hierarchical token-bucket ledger whose bucket balances are periodically snapshotted for restart continuity. A request reserves estimated compute credits from actor and tenant buckets. Final usage reconciles the reservation. Token buckets refill lazily from monotonic wall-clock deltas. + +## Worker routing + +`internal/worker` maintains health and model residency snapshots for each Ollama endpoint. Local `active` counters are atomic and are the hard worker slot limiter. Candidate scoring combines active-slot pressure with a strong affinity bonus when the requested model is already loaded. + +There are no worker leases or renewal goroutines. + +## Live flow and infrastructure map + +`internal/liveflow` tracks bounded prompt-free lifecycle metadata. `internal/infrastructure` turns that local state into the UI topology snapshot. The browser receives it over SSE and renders the animated pulse map with Canvas. + +The infrastructure view is explicitly local to the process. It shows one gateway node plus all configured Ollama workers and models. + +## Runtime policy store and sessions + +Tenant policy overrides are backed by the local persistent policy store. Browser OIDC sessions remain in memory and disappear on restart by design. + +## Usage + +`internal/usage` maintains actor, tenant and global summaries in memory. The optional file journal runs asynchronously on a buffered channel and never participates in admission or request dispatch. + +## Scaling model + +Scale inference by adding Ollama workers to the one gateway. Running multiple independent gateway replicas would create independent fairness domains. If strict global fairness is required, requests must pass through the same gateway engine. + + +## Optional content-bearing conversation state + +The Responses conversation layer is separate from browser OIDC sessions. Browser sessions remain volatile. When explicitly enabled, Responses conversation contexts are tenant/actor scoped, encrypted at rest, retention bounded, and used only to expand `previous_response_id` before normal admission/routing. The ordinary proxy path does not capture response bodies; bounded capture is enabled only for store-eligible `/v1/responses` requests. diff --git a/docs/BATCH-JOBS.md b/docs/BATCH-JOBS.md new file mode 100644 index 0000000..4fdebab --- /dev/null +++ b/docs/BATCH-JOBS.md @@ -0,0 +1,140 @@ +# Durable batch jobs + +Durable batch jobs are the P3.1 background-execution layer. They are **disabled by default** because each submitted job intentionally persists its request body and, when available, its response body. + +## Configuration + +```json +"batch_jobs": { + "enabled": false, + "retention": "168h", + "max_jobs": 1000, + "max_concurrent": 1, + "max_input_bytes": 16777216 +} +``` + +The service class `batch` must also exist under `service_classes.classes`. `batch_jobs.max_input_bytes` must not exceed `server.max_body_bytes`. + +Storage paths are bootstrap-only: + +```json +"storage": { + "batch_jobs_file": "batch-jobs.json", + "batch_jobs_dir": "batch" +} +``` + +## Client API + +A batch job wraps one normal configured compute `POST` endpoint. The request body is the JSON body that the gateway will replay later. + +```http +POST /gateway/v1/batches +Authorization: Bearer +Content-Type: application/json + +{ + "path": "/api/chat", + "body": { + "model": "qwen3:8b", + "messages": [{"role":"user","content":"Summarize the report"}], + "stream": false + } +} +``` + +A successful submission returns `202 Accepted`, a `Location: /gateway/v1/batches/` header and the durable job metadata. + +Owner-scoped endpoints: + +```text +GET /gateway/v1/batches +GET /gateway/v1/batches/ +GET /gateway/v1/batches//output +POST /gateway/v1/batches//pause +POST /gateway/v1/batches//resume +POST /gateway/v1/batches//cancel +``` + +A caller can only see or control jobs created by the same authenticated tenant and scheduler actor. A cross-identity lookup is returned as not found. + +## Admin UI and API + +Administrators have a dedicated **Batch Jobs** page with all durable jobs, state, attempts, identity metadata, model/path, pause/resume/cancel controls and authenticated output download. + +Admin endpoints require `gateway:admin`: + +```text +GET /gateway/ui-api/batches +GET /gateway/ui-api/batches/ +GET /gateway/ui-api/batches//output +POST /gateway/ui-api/batches//pause +POST /gateway/ui-api/batches//resume +POST /gateway/ui-api/batches//cancel +``` + +## Execution semantics + +Every attempt is replayed into the same gateway compute path as an interactive request, with the original identity metadata and a forced service class of `batch`. Therefore the attempt still passes through: + +- tenant model ACLs and the submitted key-specific ACL snapshot; +- alias resolution, capability preflight, placement, maintenance state and circuit eligibility; +- quota reservation/reconciliation; +- weighted fair scheduling and the configured `batch` class concurrency ceiling; +- adaptive worker routing and normal safe pre-stream retry behavior; +- usage metering, request IDs, alerts and OpenTelemetry. + +The submission credential itself is never stored. The durable identity snapshot contains tenant/subject/application/scopes and the key-specific model ACL needed to execute an already accepted job. Current tenant policy, worker state, placement, quota state and routing configuration are evaluated again at execution time. Revoking the submitting API key does not retroactively cancel an already accepted durable job; cancel the job explicitly if that is required operationally. + +## States + +```text +queued -> running -> completed + | | \ + | | -> failed + | -> pausing -> paused -> queued + -> paused + -> cancelled +running/pausing -> cancelling -> cancelled +``` + +`attempts` increments when an execution attempt starts. A failed gateway/backend response is terminal; the durable batch manager itself does not blindly retry application failures. + +## Restart and shutdown behavior + +The metadata snapshot is written atomically. Input/output payloads live in separate spool files referenced from metadata. + +On startup: + +- `running` becomes `queued` and is eligible for retry; +- `pausing` becomes `paused`; +- `cancelling` becomes `cancelled`. + +During graceful shutdown the root context cancels active attempts, and the process waits for those attempts to persist their restart-safe `queued` state before final shutdown accounting is closed. A hard kill can still leave metadata in `running`; startup recovery converts that state to `queued`. + +This gives durable batches **at-least-once execution across an interruption boundary**, not exactly-once execution. If a worker completed side effects but the gateway did not durably commit the batch result before shutdown/crash, that job can run again after restart. Use idempotent batch workloads or application-level idempotency keys when duplicate execution would be harmful. + +## Storage, privacy and retention + +Layout: + +```text +/batch-jobs.json +/batch/input/.json +/batch/output/.response +``` + +Metadata and spool payload files are created with mode `0600`; spool directories use `0750`. Input/output references are stored in `batch-jobs.json`, not the potentially large payloads themselves. + +**Batch payloads are content-bearing and are not encrypted by the gateway.** This differs from the optional encrypted conversation store. Keep `batch_jobs.enabled=false` unless durable content persistence is intended, protect `storage.data_dir` with filesystem/disk encryption and access controls, and treat gateway backups as sensitive. + +Terminal jobs and their input/output files are removed after `batch_jobs.retention`. `max_jobs` caps retained metadata/jobs; `max_input_bytes` caps one submitted request body. + +## Accounting + +Each execution attempt receives its own normal `X-Request-ID`. The durable job stores the latest execution request ID and HTTP status. The attempt appears in the ordinary usage journal with `service_class: "batch"`; prompt/completion tokens and compute credits therefore use the same accounting and rollup path as other inference traffic. + +## Current scope + +P3.1 deliberately implements one durable compute request per job. It is not an OpenAI Batch API clone and does not yet ingest multi-request JSONL files. Global batch coordination across multiple gateway replicas remains intentionally deferred outside the active roadmap; the current design stays single-process and durable. diff --git a/docs/CONTEXT-HARDENING.md b/docs/CONTEXT-HARDENING.md new file mode 100644 index 0000000..eebfee0 --- /dev/null +++ b/docs/CONTEXT-HARDENING.md @@ -0,0 +1,69 @@ +# Context / `num_ctx` hardening + +Checkpoint 27 treats context size as a runtime resource rather than only a model capability. + +## Effective context + +For requests that cannot set native Ollama `options.num_ctx` (OpenAI `/v1/*`, Responses and Anthropic-compatible routes), each worker gets an effective context window from the strongest available evidence: + +1. loaded `/api/ps` `context_length`; +2. Modelfile `num_ctx` returned by `/api/show` `parameters`; +3. `workers[].default_context_tokens`; +4. `model_capabilities.context.default_worker_tokens`. + +The result is clamped by the theoretical model maximum and `workers[].context_limits`. The request is routed only to workers that can hold the estimated input + output budget. If no worker can hold it and `context_guard` is `reject`, admission fails before queueing or inference. + +## Native `options.num_ctx` + +`options.num_ctx` is accepted only on native Ollama `/api/*` requests. It must be a positive JSON integer. The gateway checks it against: + +- `model_capabilities.context.max_requested_tokens` (default 32768; `-1` removes the gateway cap); +- the model maximum discovered from `/api/show`; +- a matching `workers[].context_limits` entry. + +A model already loaded with a smaller context may still receive a larger explicit `num_ctx` because Ollama can resize/reload it, but that worker loses the loaded-model routing bonus for that request. + +## Conservative request estimate + +The context guard now includes: + +- `prompt`, messages and `system`; +- Responses `instructions` and `input`; +- native generate `suffix`; +- tool schema JSON; +- `max_tokens`, `max_completion_tokens`, `max_output_tokens` and native `num_predict`; +- a configurable reserve per image (`vision_reserve_tokens_per_image`); +- a configurable input safety margin (`estimation_margin_percent`). + +The estimator intentionally remains provider-independent and is not an exact tokenizer. The margin and image reserve compensate for that uncertainty. + +## Recommended configuration + +```json +"model_capabilities": { + "mode": "enforce", + "cache_ttl": "10m", + "context_guard": "reject", + "context": { + "max_requested_tokens": 32768, + "default_worker_tokens": 4096, + "estimation_margin_percent": 15, + "vision_reserve_tokens_per_image": 2048 + } +} +``` + +For a worker with a known different Ollama default or a stricter memory envelope: + +```json +{ + "name": "worker-a", + "default_context_tokens": 4096, + "context_limits": { + "large-*": 16384, + "*": 32768 + } +} +``` + +Use a Modelfile `num_ctx` when OpenAI-compatible clients need a larger stable context, because those APIs do not provide native per-request `options.num_ctx` semantics. diff --git a/docs/CONVERSATIONS.md b/docs/CONVERSATIONS.md new file mode 100644 index 0000000..5d6238e --- /dev/null +++ b/docs/CONVERSATIONS.md @@ -0,0 +1,67 @@ +# Optional encrypted Responses conversations + +`conversations` implements the P2.3 stateful layer for clients that use the OpenAI Responses API with `previous_response_id`. + +The feature is **disabled by default**. When disabled, the gateway does not parse, capture, persist, or reinterpret response content for conversation purposes; `/v1/responses` keeps its normal passthrough behavior. + +## Configuration + +```json +"conversations": { + "enabled": false, + "encryption_key": "${GATEWAY_CONVERSATION_KEY}", + "retention": "24h", + "max_entries": 1000, + "max_content_bytes": 2097152 +}, +"storage": { + "conversations_file": "conversations.enc.json" +} +``` + +When enabled: + +- `encryption_key` must contain at least 32 characters. Use an environment variable or another deployment secret source; do not commit it. +- `retention` is independent from usage-journal retention because this store contains prompt/output content. +- `max_entries` bounds the number of stored response contexts; oldest entries are evicted first. +- `max_content_bytes` bounds one flattened conversation context and also bounds the response capture used to construct it. + +The configured key is SHA-256-derived into an AES-256 key. The state snapshot is encrypted with AES-256-GCM and written mode `0600` through the same fsync + atomic-replace mechanism used by the other local state files. The on-disk JSON envelope contains only version/algorithm metadata, nonce, ciphertext, and update time; conversation content is not plaintext on disk. + +Changing or losing the encryption key makes the existing store unreadable. Startup fails closed if an enabled store cannot be decrypted. + +## `previous_response_id` behavior + +For an enabled store, a successful `/v1/responses` request is stored when the request does not set `"store": false`. + +For a later request containing `previous_response_id`: + +1. the ID is looked up only inside the authenticated tenant + actor boundary; +2. the prior flattened input/output item context is loaded; +3. the current `input` is appended; +4. `previous_response_id` is removed before forwarding upstream; +5. the resulting complete `input` is sent through the normal ACL, preflight, quota, queue, placement, routing and proxy path. + +A missing ID and an ID owned by another tenant/actor are intentionally indistinguishable to the caller. Both fail as an invalid previous response reference, preventing cross-identity enumeration. + +String `input` values are normalized to a user message item when history must be flattened. Existing array/object inputs are retained as items. Previous `instructions` are not copied into the stored conversation context; only input/output items are chained. + +## Streaming + +The proxy normally does not retain response bodies. Conversation capture is activated only for `/v1/responses` when conversations are enabled and the request is eligible for storage. + +For streaming Responses, the gateway reconstructs persisted output from `response.completed` (or completed output-item events when necessary). Client streaming remains unchanged. If the capture exceeds `max_content_bytes`, the inference response still succeeds but that response is not stored for future chaining. + +## Retention and deletion + +Expired entries are pruned on reads/writes and by a background cleanup loop. The cleanup interval is derived from the configured retention and capped at 15 minutes, so content does not depend on future traffic to expire. + +`store: false` is the per-request opt-out. It still allows a request to consume a valid prior response context, but the new response is not persisted as the next link. + +The store is intentionally not exposed as a content browser in the Admin UI. The persistence page reports only operational metadata such as whether it is enabled and the number of retained entries. + +## Backups and threat model + +The Admin storage backup includes the encrypted conversation state file when present. Treat backups as sensitive anyway: gateway configuration backups can contain deployment secrets, and possession of both the conversation ciphertext and its encryption key allows decryption. + +This feature protects content at rest from casual filesystem disclosure. It does not protect content from a compromised running gateway process, from an administrator who possesses the configured key, or from the Ollama worker that receives the expanded prompt context. diff --git a/docs/DEPLOYMENT-HARDENING.md b/docs/DEPLOYMENT-HARDENING.md new file mode 100644 index 0000000..5233b73 --- /dev/null +++ b/docs/DEPLOYMENT-HARDENING.md @@ -0,0 +1,117 @@ +# Deployment hardening and preflight + +Checkpoint 19 adds deployment checks intended to catch configuration/mount mistakes before production traffic reaches the gateway. These checks do not participate in the inference hot path. + +## Effective configuration preflight + +Run the gateway binary with `-check-config` before starting the service: + +```sh +./ollama-gateway -config ./gateway-config.json -check-config +``` + +The command validates the same strict bootstrap schema as normal startup, loads the persistent UI override from `storage.data_dir` when present, enforces the bootstrap-only storage rule, verifies that the effective state directory is writable, and prints a secret-free JSON summary. A non-zero exit code means the gateway should not be started with that configuration. + +The summary includes: + +- bootstrap config path, +- whether a persistent override was loaded, +- persistent override path, +- effective worker count, +- effective/absolute state directory, +- storage writability, +- non-fatal deployment warnings. + +Warnings currently cover common production foot-guns such as `local_system_stats=true` on a remote worker URL, an unknown `native.control_worker`, public metrics, a relative state directory and UI authentication/cookie combinations that deserve review. + +For Compose deployments, run the check in the exact image/user/mount context that will be used in production: + +```sh +GATEWAY_CONFIG=./gateway-config.json docker compose run --rm gateway \ + -config /etc/ollama-gateway/config.json -check-config +``` + +This is preferable to validating only on the Docker host because it also proves that the container user can read the bootstrap file and write the mounted state volume. + +## Built-in HTTP probe + +The scratch image contains no shell, curl or wget. The gateway binary therefore has a minimal HTTP probe mode: + +```sh +/ollama-gateway -probe http://127.0.0.1:8080/healthz -probe-timeout 2s +``` + +It exits 0 for any 2xx response and non-zero for connection errors, timeouts or non-2xx responses. Probe mode does not load gateway configuration or state. + +The image/Compose liveness healthcheck uses `/healthz`. Load balancers and rollout automation should additionally require `/readyz`; readiness checks scheduler, workers and persistent runtime stores and can return 503 while the process itself remains healthy. + +## Compose entrypoint rule + +The image already defines: + +```text +ENTRYPOINT ["/ollama-gateway"] +``` + +Therefore Compose `command` must contain only arguments: + +```yaml +command: ["-config", "/etc/ollama-gateway/config.json"] +``` + +Do **not** repeat `/ollama-gateway` in `command`. Repeating the executable turns it into the first positional argument passed to Go's flag parser and can prevent following flags from being interpreted as intended. + +The supplied Compose file supports selecting a bootstrap configuration without editing the file: + +```sh +GATEWAY_CONFIG=./gateway-config.json docker compose up -d --build +``` + +If `GATEWAY_CONFIG` is omitted, the development `config.example.json` is mounted. Do not mistake that default example for a production configuration; `-check-config` reports the effective worker count and makes that error easier to detect before rollout. + +## Recommended staged update + +1. Back up the bootstrap config and `storage.data_dir`. +2. Build/pull the candidate image. +3. Run `-check-config` in the candidate container with the production mounts. +4. Stop the old gateway gracefully. +5. Start the candidate and wait for the Docker liveness check. +6. Require `/readyz` before restoring traffic. +7. Verify authenticated `/gateway/ui-api/session`, one non-streaming request and one streaming request if used. +8. Keep the previous image/binary and state backup available until the observation window is complete. + +## Checkpoint 23: reverse-proxy boundary and container sandbox + +The supplied Compose file now publishes the gateway on host loopback by default: + +```text +127.0.0.1:9080 -> container :8080 +``` + +This is the preferred topology when the TLS/reverse proxy runs on the same Docker host. It prevents LAN clients from bypassing the proxy and reaching the gateway's published port directly. Override `GATEWAY_PUBLISH_ADDRESS` only when the real proxy is on another host. In that case bind to the specific host interface where possible and enforce a host firewall/ACL that permits only the real proxy source address. + +The Compose runtime also uses a read-only root filesystem, drops all Linux capabilities and enables `no-new-privileges`. `/data` remains the only persistent writable application state mount and `/tmp` is an ephemeral tmpfs. + +For the deployment observed during the checkpoint-22 rollout, browser traffic arrived at the container through Docker with TCP peer `172.30.3.1`. The hardened production configuration therefore replaces broad RFC1918 `trusted_proxies` ranges with the exact `172.30.3.1/32` peer. `ip_bypass_use_forwarded_ip` remains false. If the network path changes, re-observe the gateway's `remote_addr` and trust only the actual proxy/NAT peer that is expected to supply forwarded headers. + +A host-side preflight catches the common rollout mistakes before Compose starts production traffic: + +```sh +cp .env.production.example .env +GATEWAY_CONFIG=./gateway-config.json ./scripts/production-preflight.sh +``` + +It fails if the production config points to `config.example.json`, if the gateway is published outside loopback without explicit acknowledgement, if Compose rendering fails, or if the gateway's own `-check-config` rejects the effective configuration/state mounts. + +For a remote reverse proxy, use an explicit interface and firewall policy, for example: + +```sh +GATEWAY_PUBLISH_ADDRESS=10.2.10.20 \ +ALLOW_NON_LOOPBACK_BIND=1 \ +GATEWAY_CONFIG=./gateway-config.json \ +./scripts/production-preflight.sh +``` + +The explicit opt-in is not a substitute for a firewall. The published port should be reachable only from the reverse proxy. + +For production, prefer the stricter `docker-compose.production.yml`. Unlike the development Compose file, it requires `GATEWAY_CONFIG` to be set and never falls back to `config.example.json`. diff --git a/docs/HA-READINESS.md b/docs/HA-READINESS.md new file mode 100644 index 0000000..5f23ade --- /dev/null +++ b/docs/HA-READINESS.md @@ -0,0 +1,131 @@ +> **Status: archived diagnostic path.** Multi-gateway HA was removed from the active roadmap in checkpoint 28 because current single-process performance does not justify distributed coordination. Keep this tooling for future re-evaluation if requirements change. + +# P3.2 HA readiness and single-process limit characterization + +P3.2 is deliberately gated: do not add distributed consensus, global queue state or leader election until the single-process gateway is shown to be the limiting availability/capacity component rather than Ollama/GPU inference. + +This repository includes a small evidence toolchain for that decision: + +- `cmd/mock-ollama`: deterministic Ollama/OpenAI-compatible mock backend that removes model compute from the measurement; +- `cmd/bench`: concurrent OpenAI request driver with latency, TTFB, status, bytes, token throughput and JSON output; +- `scripts/ha-readiness.sh`: repeatable concurrency sweep that writes one JSON result per level and captures Prometheus snapshots before/after each level; +- `cmd/ha-snapshot`: optional host/process endpoint snapshotter for gateway RSS/CPU/threads/FDs and host CPU/memory/load evidence; +- `cmd/ha-sampler`: sustained resource sampler for each load level, including sampled peak RSS/CPU/threads/FDs and host-memory/load pressure; +- `cmd/ha-report`: reconciles client benchmark counts with gateway Prometheus deltas, attaches endpoint and sustained resource evidence, and writes machine-readable JSON plus a Markdown evidence summary. + +## 1. Build/run the mock backend + +```bash +go run ./cmd/mock-ollama \ + -listen 127.0.0.1:11435 \ + -model qwen3:8b \ + -delay 0 \ + -response-bytes 128 +``` + +Point one gateway worker at `http://127.0.0.1:11435`. The mock implements the worker inventory/metadata endpoints plus native and OpenAI chat endpoints required by the gateway. + +Useful mock controls: + +```text +-delay 5ms fixed pre-response latency +-stream-delay 10ms delay between streaming chunks +-response-bytes N generated payload size +-fail-every N deterministic HTTP 503 every Nth inference request +``` + +The zero-delay mode approximates gateway/control-plane overhead. Delay/stream-delay modes exercise many concurrent in-flight connections without requiring a GPU. + +## 2. Run a concurrency sweep + +For an authenticated gateway: + +```bash +export GATEWAY_BENCH_API_KEY='...' +export GATEWAY_PID="$(pgrep -n ollama-gateway)" +BASE_URL=http://127.0.0.1:8080 \ +MODEL=qwen3:8b \ +REQUESTS=1000 \ +WARMUP=50 \ +CONCURRENCIES='1 4 16 32 64 128' \ +./scripts/ha-readiness.sh +``` + +Each level prints a human summary and writes `concurrency-.json` plus `metrics-before-c.prom` and `metrics-after-c.prom`. When `GATEWAY_PID` is set, the sweep also writes `resources-before-c.json`, `resources-after-c.json`, and `resources-samples-c.json`. After the sweep, `cmd/ha-report` writes `report.json` and `report.md`. The report verifies that the gateway request, queue and service-observation deltas equal **warmup + measured requests** for every level and records endpoint-resource and sustained-resource evidence completeness separately. + +The sweep authenticates `/metrics` with `GATEWAY_BENCH_API_KEY` when that environment variable is set; the key is used only as an HTTP header and is never written to the result directory. Set `METRICS_URL` when metrics are exposed at a separate admin endpoint. Set `CAPTURE_METRICS=false` only for an intentionally client-only run; such a run does not produce reconciled HA evidence. + +Endpoint resource capture defaults to `auto`: it is enabled when `GATEWAY_PID` is present and otherwise skipped. Set `CAPTURE_RESOURCES=true` to require it explicitly, or `CAPTURE_RESOURCES=false` to suppress it. Sustained capture separately defaults to `auto` through `CAPTURE_SUSTAINED_RESOURCES`; set it to `false` when an external profiler already supplies the sustained evidence. `GATEWAY_PID` must identify the actual gateway process, not a shell wrapper. `RESOURCE_SAMPLE_INTERVAL` defaults to `250ms`, and `RESOURCE_SAMPLE_MAX_DURATION` defaults to `15m` as a safety bound for a stuck benchmark. + +The `resources-before/after` files are **endpoint snapshots immediately before and after a load level**. The `resources-samples` file is collected throughout the load level. On Linux, sampled process CPU uses deltas between `/proc//stat` and aggregate `/proc/stat`, scaled to all logical CPUs, so a multi-core process can legitimately exceed 100%. The sampler also records sampled peak RSS, threads, open FDs and load plus minimum available host memory. On other platforms the tool falls back to repeated `ps` sampling and leaves unavailable fields absent. + +Each benchmark JSON contains: + +- successful/error counts and HTTP status distribution; +- requests/second; +- latency p50/p95/p99/max; +- TTFB p50/p95/p99/max; +- bytes received and bytes/second; +- prompt/completion tokens and completion tokens/second when non-streaming usage is available; +- stream/keep-alive/service-class settings. + +To isolate connection setup cost, run an additional pass with `cmd/bench -disable-keepalive`. For long-lived response pressure, run with `STREAM=true` and configure a non-zero mock `-stream-delay`. + +## 3. Measure three distinct regimes + +### A. Gateway ceiling + +Mock Ollama with `-delay 0`, small response body, non-streaming. Increase concurrency until throughput flattens or latency/error rate rises sharply. This measures HTTP/auth/admission/scheduler/routing/proxy/accounting overhead rather than LLM inference. + +### B. Connection/stream pressure + +Mock Ollama with streaming enabled and `-stream-delay` long enough to keep many requests open. This tests goroutine/socket/buffer behavior and cancellation/shutdown paths. + +### C. Real inference + +Repeat against the actual Ollama workers. This gives the practical service curve and reveals whether GPU/VRAM/model residency saturates far earlier than the gateway process. + +## 4. Record gateway-side signals + +The default sweep captures `/metrics` automatically. `cmd/ha-report` consumes the full `ollama_gateway_*` Prometheus names and derives per-level deltas. Important signals include: + +- scheduler queued/running and queue wait; +- service-class queued/running; +- worker active/model-active slots; +- request/error/retry/circuit counters; +- usage-journal drop/storage gauges; +- process CPU/RSS/thread/file-descriptor and host load/memory endpoint snapshots from `cmd/ha-snapshot`; +- sampled peak process CPU/RSS/thread/file-descriptor and host pressure from `cmd/ha-sampler`, plus socket/runtime metrics from sustained host/container monitoring when deeper evidence is required. + +The benchmark numbers are meaningful only together with CPU, memory and worker saturation. A throughput plateau while GPU/worker slots are saturated is not evidence that HA is needed for gateway capacity. + +## 5. HA decision gate + +Proceed from readiness work to a real P3.2 coordinator/cluster design only when at least one of these is demonstrated: + +1. **Capacity:** gateway CPU/network/connection capacity saturates materially before the Ollama worker pool and cannot be resolved by tuning one process. +2. **Availability:** the required recovery objective cannot tolerate one authoritative gateway process, even with supervisor restart and durable local state. +3. **Operational topology:** multiple gateway instances are mandatory across failure domains, while strict global fairness/quota/batch semantics must still be preserved. + +Do **not** deploy independent replicas behind a generic load balancer and call that P3.2. Independent replicas create separate fair queues, quota buckets, transient job registries and batch dispatchers. They improve process redundancy only by weakening global correctness. + +### Evidence report gate states + +- `incomplete`: one or more metric snapshots are missing, or gateway request/queue/service counters do not reconcile with the expected warmup + measured request count. Fix the measurement before drawing conclusions. +- `not-proven`: client and gateway counters reconcile, but the evidence still does not demonstrate one of the capacity/availability/topology criteria above. `resource_evidence_complete` states whether every level has before/after snapshots; `sustained_resource_evidence_complete` states whether every level has a valid sustained sampler trace. This is the normal result of a mock-backend smoke test. + +The tool intentionally does **not** auto-promote the project into clustered HA based only on a throughput plateau. Sustained sampling closes the short-peak gap in endpoint snapshots, but it still must be interpreted with socket/runtime metrics and, for real inference runs, worker/GPU saturation. + +## 6. P3.2 design constraints once the gate is met + +The HA design must preserve the current hot-path properties: + +- one authoritative global order for fair admission; +- globally consistent quota and durable batch state; +- fencing/epochs so a stale leader cannot dispatch duplicate authoritative work; +- membership and leader failover without making every streamed token depend on consensus; +- local worker proxying remains direct after a request is admitted/assigned; +- content-bearing batch/conversation storage retains its current explicit privacy boundaries; +- no mandatory Redis dependency is introduced merely as a shortcut. + +A likely architecture is a small consensus-backed coordinator/control plane plus local proxy workers, not replicated copies of today’s independent in-memory scheduler. diff --git a/docs/IMPROVEMENTS-2026.md b/docs/IMPROVEMENTS-2026.md new file mode 100644 index 0000000..f1d393d --- /dev/null +++ b/docs/IMPROVEMENTS-2026.md @@ -0,0 +1,198 @@ +# Verbesserungen und Prioritäten (September 2026) + +Dieses Dokument beschreibt die nach einem Abgleich mit Ollama, OpenWebUI und etablierten LLM-Gateways priorisierten Verbesserungen. Ziel bleibt ein einzelner, sehr schneller Go-Prozess ohne Redis oder Datenbank im Inference-Hot-Path. + +## Priorität A – in diesem Release umgesetzt + +### 1. Capability-aware Model Preflight + +Ollama liefert über `POST /api/show` Modell-Capabilities und Modellinformationen. Das Gateway cached diese Metadaten pro Worker/Modell und erkennt aktuell: + +- `completion` +- `tools` +- `vision` +- `thinking` +- `embedding` + +Ein Request wird vor Admission gegen seine benötigten Capabilities geprüft. Das verhindert z. B., dass ein OpenWebUI-Tool-Request erst tief im Ollama-Backend mit `does not support tools` scheitert. + +Konfiguration: + +```json +"model_capabilities": { + "mode": "enforce", + "cache_ttl": "10m", + "context_guard": "reject", + "context": { + "max_requested_tokens": 32768, + "default_worker_tokens": 4096, + "estimation_margin_percent": 15, + "vision_reserve_tokens_per_image": 2048 + } +} +``` + +`mode`: + +- `enforce`: bekannte Capability-Konflikte mit HTTP 400 ablehnen. +- `observe`: nur Warnheader/Logs setzen. +- `off`: Metadaten-Preflight deaktivieren. + +Wenn `/api/show` temporär nicht erreichbar ist, lässt das Gateway den Request bewusst durch. Verfügbarkeit hat bei unbekannten Metadaten Vorrang; nur *bekannt inkompatible* Requests werden geblockt. + +### 2. Context Guard + +Der Gateway trennt das **theoretische Modellmaximum** von dem Kontext, den ein konkreter Worker tatsächlich bereitstellt. Für OpenAI-/Responses-/Anthropic-kompatible Requests ohne natives `num_ctx` gilt als effektive Quelle in dieser Reihenfolge: + +1. `context_length` des bereits geladenen Modells aus `/api/ps`; +2. `num_ctx` aus den Modelfile-Parametern von `/api/show`; +3. `workers[].default_context_tokens`; +4. `model_capabilities.context.default_worker_tokens`. + +Anschließend greifen `workers[].context_limits` und der Gateway-Cap. Ein natives `options.num_ctx` wird nur auf `/api/*` akzeptiert, muss ein positiver Integer sein und wird gegen Gateway-Cap, Modellmaximum und Worker-Cap geprüft. Ein kleinerer aktuell geladener Kontext darf für ein explizit größeres `num_ctx` neu allokiert werden, erhält dann aber keinen Loaded-Routingbonus. + +Die Admission-Schätzung zählt neben Prompt/Messages/Tool-Schemas jetzt auch `instructions`, `suffix`, `max_output_tokens`, eine konfigurierbare Vision-Reserve pro Bild und eine Sicherheitsmarge. Das ist bewusst eine konservative Schätzung und kein Tokenizer-Ersatz. + +### 3. Per-model Concurrency + +Ein 24-GB-Worker sollte kleine 8B-Modelle anders behandeln können als 27B-Modelle. Jeder Worker kann deshalb zusätzlich zu `max_concurrent` Modellgrenzen definieren: + +```json +"model_concurrency": { + "qwen3:8b": 2, + "gemma4:*": 1, + "*": 1 +} +``` + +Matching-Reihenfolge: + +1. exakter Modellname; +2. längster passender Prefix mit abschließendem `*`; +3. `*`; +4. `max_concurrent`. + +Der Slot wird im selben in-memory Worker-State geführt und bei Cancel/Fehler/Completion wieder freigegeben. + +### 4. Adaptive Worker Routing + +Worker-Routing berücksichtigt jetzt zusätzlich zu Health und aktiven Slots: + +- bereits geladenes Modell; +- auf dem Worker installiertes Modell; +- per-model Slot-Auslastung; +- gelernte Output-Tokenrate je Worker/Modell (EWMA), persistent über Neustarts; +- tatsächlichen VRAM-Druck, wenn Telemetrie verfügbar ist; +- GPU-Auslastung; +- eine harte Zusatzstrafe für das Laden eines neuen Modells bei sehr hohem VRAM-Druck. + +Die Gewichte sind konfigurierbar: + +```json +"routing": { + "loaded_bonus": 60, + "installed_bonus": 30, + "throughput_bonus": 20, + "vram_pressure_penalty": 35, + "gpu_utilization_penalty": 10, + "avoid_vram_percent": 97 +} +``` + +Durchsatzwerte werden ausschließlich aus echten abgeschlossenen Inferenzrequests gelernt und unter `storage.worker_performance_file` persistiert. Nach einem Neustart startet das Routing deshalb mit den zuletzt bekannten EWMA-Werten und lernt sie anschließend weiter. + +### 5. NVIDIA-Telemetrie ohne zusätzlichen Agenten + +Auf NVIDIA-Hosts kann ein Worker direkt `nvidia-smi` verwenden: + +```json +"nvidia_smi": true, +"nvidia_gpu": "0" +``` + +Erfasst werden: + +- GPU utilization +- VRAM used / total +- Temperatur +- Power Draw + +Die Abfrage läuft außerhalb des Request-Hot-Paths im normalen Worker-Health-Zyklus. Das Gateway bleibt `CGO_ENABLED=0` und benötigt keine NVML-Bibliothek. Ein vorhandenes `telemetry_url` kann weiterhin verwendet werden und externe Werte überschreiben. + +### 6. Operations UI und Metrics + +Die Modellansicht zeigt jetzt erkannte Capabilities und Context Length. Worker-Karten zeigen – soweit vorhanden – GPU, VRAM, Temperatur, Leistung und gelernte tok/s. + +Prometheus exportiert zusätzlich: + +```text +ollama_gateway_worker_memory_used_bytes +ollama_gateway_worker_memory_total_bytes +ollama_gateway_worker_vram_used_bytes +ollama_gateway_worker_vram_total_bytes +ollama_gateway_worker_gpu_utilization_percent +ollama_gateway_worker_gpu_temperature_celsius +ollama_gateway_worker_gpu_power_watts +ollama_gateway_worker_model_active +ollama_gateway_worker_model_prompt_tokens_per_second +ollama_gateway_worker_model_output_tokens_per_second +ollama_gateway_worker_model_performance_samples +``` + +Modelle sind eine begrenzte operative Dimension. Tenant-/User-/Request-IDs werden weiterhin nicht als Prometheus-Labels ausgegeben. + +### 7. Unbegrenzte Credits verständlicher im UI + +`0` bedeutet weiterhin unbegrenzt. Die Policy-Oberfläche zeigt dafür jetzt `∞` und bietet explizite Checkboxen für unbegrenzte Actor- bzw. Tenant-Credits. + +## RTX-4090-Startprofil + +`config.rtx4090.example.json` enthält einen Startpunkt für eine RTX 4090 mit 24 GiB VRAM und beispielhaft 64 GiB System-RAM. Den RAM-Wert bitte an den tatsächlichen Host anpassen. Das Profil funktioniert unter Linux und Windows; beide Plattformen erfassen Host-RAM nativ, und `nvidia-smi` liefert die GPU-Telemetrie, sofern es im `PATH` verfügbar ist. + +Für 24 GiB VRAM ist der wichtigste Unterschied zum 256-GB-Unified-Memory-Mac die Modell-spezifische Concurrency. Große ~17-GB-Modelle starten im Beispiel bei 1, kleine Modelle dürfen 2 parallele Slots erhalten. + +## Priorität B – als nächste Ausbaustufe empfohlen + +### Upstream Circuit Breaker + begrenzte Retries + +Retries sind bei LLM-Streaming gefährlich, weil eine bereits begonnene Generierung nicht transparent wiederholt werden darf. Sinnvoll wäre deshalb: + +- Retry nur vor dem ersten Response-Byte und nur bei eindeutig transienten Transportfehlern; +- Circuit Breaker pro Worker; +- Exponential Backoff bei Health-/Load-Fehlern; +- kein blindes Request Hedging, da es GPU-Arbeit dupliziert. + +### API-Key Model ACLs + +Pro API-Key/Tenant sollten erlaubte/verbotene Modell-Patterns möglich sein, z. B. `allowed_models: ["qwen3:*", "embeddinggemma:*"]`. Das ist besonders für teure oder unzensierte Modelle sinnvoll. + +### Capability Routing / expliziter Fallback + +Ein optionaler, explizit konfigurierter Fallback könnte Tool-Requests auf ein toolfähiges Modell umleiten. Default sollte `reject` bleiben, da ein stiller Modellwechsel semantisch überraschend ist. + +### Persistentes Audit-Log als optionales Modul + +Control-Plane-, Accounting- und Routing-Zustand ist inzwischen lokal persistent, während aktive Scheduling-/Socket-Zustände in-memory bleiben. Für Compliance wäre zusätzlich ein optionaler asynchroner Audit-Sink sinnvoll, der nur Admin-Aktionen und Metadaten schreibt und niemals Prompts/Antworten. + +### OpenTelemetry + +Prometheus deckt Aggregationen ab. Für verteilte Client-/Gateway-/Ollama-Latenz wäre optionales OTel Tracing sinnvoll, weiterhin ohne Prompt-Inhalte. + +## Referenzen + +- Ollama Show model details: https://docs.ollama.com/api-reference/show-model-details +- Ollama Tool calling: https://docs.ollama.com/capabilities/tool-calling +- Ollama Thinking: https://docs.ollama.com/capabilities/thinking +- Ollama Vision: https://docs.ollama.com/capabilities/vision +- Ollama Embeddings: https://docs.ollama.com/capabilities/embeddings +- OpenWebUI Ollama connection/context notes: https://docs.openwebui.com/getting-started/quick-start/connect-a-provider/starting-with-ollama/ +- NVIDIA `nvidia-smi` selective query: https://docs.nvidia.com/deploy/nvidia-smi/ + + +## Checkpoint 27 — Context / `num_ctx` Hardening + +Der Context Guard unterscheidet jetzt zwischen dem theoretischen Modellmaximum (`/api/show model_info.*.context_length`) und dem tatsächlich nutzbaren Kontext eines Workers. Für Requests ohne explizites natives `options.num_ctx` gilt in dieser Reihenfolge: geladenes `/api/ps context_length`, Modelfile-`num_ctx`, `workers[].default_context_tokens`, dann der globale `model_capabilities.context.default_worker_tokens`. `workers[].context_limits` kann Modelle pro Worker zusätzlich begrenzen. + +Native `options.num_ctx` muss ein positiver Integer sein. Es wird gegen Gateway-Cap, Modellmaximum und Worker-Cap geprüft. Ein bereits kleiner geladenes Modell bleibt für einen explizit größeren `num_ctx` grundsätzlich routbar, verliert aber den Loaded-Bonus, weil Ollama den KV-Cache neu allozieren muss. + +Die Preflight-Schätzung berücksichtigt nun außerdem `max_output_tokens`, `instructions`, `suffix`, einen konfigurierbaren Vision-Reservewert pro Bild sowie eine Sicherheitsmarge. Die Modellansicht zeigt Modell-Maximum, Modelfile-Kontext und geladenen Kontext getrennt. diff --git a/docs/JOBS.md b/docs/JOBS.md new file mode 100644 index 0000000..d67e62f --- /dev/null +++ b/docs/JOBS.md @@ -0,0 +1,47 @@ +# Jobs and cancellation + +The gateway distinguishes between **model lifecycle operations**, **transient inference jobs**, and **durable batch jobs**. + +## Model Stop + +The Models page uses **Stop** to unload a resident Ollama model. Ollama does not expose `/api/stop`; the gateway sends: + +```http +POST /api/generate +Content-Type: application/json + +{"model":"","keep_alive":0,"stream":false} +``` + +This asks Ollama to unload the model immediately. It does not cancel a specific request. + +## Inference jobs + +Every compute request receives an `X-Request-ID` and is registered as an in-memory job while it is queued, routing, running, or streaming. + +Admin endpoints: + +```text +GET /gateway/ui-api/jobs +POST /gateway/ui-api/jobs//cancel +``` + +The web UI exposes these under **Jobs** and also places an **Abbrechen** button next to active requests in Live Flow. + +### Cancellation semantics + +- **queued**: remove the ticket from the WFQ heap immediately and release the credit reservation; +- **waiting for worker**: cancel worker acquisition and release the scheduler lease; +- **running/streaming**: cancel the upstream HTTP context so the Ollama request terminates, then release worker/scheduler slots; +- reconcile any usage already observable from the response; +- store terminal live state `cancelled` with internal status `499`. + +For a streaming response whose upstream HTTP headers were already forwarded, the downstream client may still have HTTP status `200`; cancellation is observable as an early stream termination. Internal gateway telemetry records the request as `499`. + +Transient inference-job state is process-local and disappears on gateway restart, consistent with the in-memory hot-path design. + +## Durable batch jobs + +P3.1 adds a separate durable job type under `/gateway/v1/batches`. Batch definitions, state, input references and output references survive restart; an active execution attempt itself is cancelled on shutdown and converted back to a restart-safe queued state. + +The Admin UI exposes durable work on the dedicated **Batch Jobs** page rather than mixing it into the transient **Jobs** page. See `docs/BATCH-JOBS.md` for API, state-machine, persistence, privacy, retention and accounting semantics. diff --git a/docs/MIGRATION-INMEMORY.md b/docs/MIGRATION-INMEMORY.md new file mode 100644 index 0000000..e3a8518 --- /dev/null +++ b/docs/MIGRATION-INMEMORY.md @@ -0,0 +1,65 @@ +# Migration to the in-memory engine + +This release removes the external coordination backend entirely. + +## Removed configuration + +Delete these old top-level/settings from existing configuration files: + +```text +redis +scheduler.distributed +scheduler.lease_ttl +scheduler.poll_interval +workers[].request_lease_ttl +usage.redis_aggregates +cluster +``` + +Replace the old local/cluster visualization settings with: + +```json +"infrastructure": { + "node_name": "mac-studio-gateway", + "refresh_interval": "250ms", + "max_requests": 256 +} +``` + +The configuration parser uses `DisallowUnknownFields`, so obsolete settings fail fast instead of being silently ignored. + +## Runtime behavior + +The following state now exists only inside one gateway process: + +- weighted fair queue state; +- running concurrency; +- per-worker slots; +- actor and tenant quota buckets; +- runtime policy overrides; +- browser sessions; +- usage summaries; +- live/infrastructure telemetry. + +Active scheduling state still resets on restart. Durable control-plane state (UI-created API-key hashes, policy overrides, quota balances, metrics snapshots and usage history) is now stored locally under `storage.data_dir`; the inference scheduler itself remains in memory. + +## Multiple Ollama workers + +No change is required. One gateway may still route to many Ollama workers and maintains one fair queue across all of them. + +## Multiple gateway processes + +Do not place multiple independent gateway processes behind a generic load balancer if you require strict global fairness. Each process is now a separate fairness/quota/session domain. + +For the intended deployment, use one authoritative gateway and scale the Ollama worker pool behind it. + +## API change + +The former cluster visualization endpoints are now local infrastructure endpoints: + +```text +GET /gateway/ui-api/infrastructure +GET /gateway/ui-api/infrastructure/stream +``` + +The web UI has been updated accordingly. diff --git a/docs/MODEL-PLACEMENT.md b/docs/MODEL-PLACEMENT.md new file mode 100644 index 0000000..4493dd2 --- /dev/null +++ b/docs/MODEL-PLACEMENT.md @@ -0,0 +1,137 @@ +# Model Placement + +Model Placement is the gateway's hard model-to-worker routing policy. It is evaluated **before** adaptive worker scoring, so GPU load, VRAM pressure, model affinity or learned throughput can never override a placement prohibition. + +## Why it is separate from tenant policies + +Tenant/Fairness policies answer: + +> How much compute may a tenant or actor consume? + +Model Placement answers: + +> On which Ollama workers may a model execute? + +Keeping those concerns separate makes both the configuration and the UI predictable. + +## UI workflow + +Open **Admin -> Model Placement**. + +The matrix has one row per discovered model and one column per worker: + +- `✓ installed` — allowed and installed on this worker. +- `○ allowed` — policy allows it, but the model is not currently installed there. +- `● loaded` — allowed, installed and currently resident according to `/api/ps`. +- `⛔ blocked` — the effective placement rule denies the model. +- `↳` — the cell is resolved by an exact model rule. + +Click a cell to create an exact allow/deny exception. Clicking an exact UI exception again removes that exact rule so the broader prefix/default rule becomes effective again. + +The worker cards open a ruleset editor with: + +- **Allow all**: allow by default and optionally deny selected models/prefixes. +- **Whitelist**: deny by default and only allow selected models/prefixes. +- **Only installed** preset: builds a whitelist from the worker's current `/api/tags` inventory. +- **Block all** preset: an empty whitelist. +- **Reset to config default**: deletes the persistent UI override and immediately restores the worker's bootstrap/persistent-config baseline. + +All UI changes are written to `storage.model_placement_file` (default `model-placement.json`) using the gateway's atomic state writer and take effect for new requests immediately. + +## Pattern semantics + +Patterns support: + +```text +qwen3:8b exact model +gemma4:* prefix wildcard +orcarouter/* prefix wildcard +* all models +``` + +Wildcards are only supported as a single trailing `*`. + +Resolution uses specificity: + +1. exact model rule; +2. longest matching prefix rule; +3. worker mode (`allow_all` or `whitelist`). + +At equal specificity, deny wins. + +This means the following is valid and useful: + +```json +{ + "mode": "allow_all", + "allowed_models": ["gemma4:latest"], + "denied_models": ["gemma4:*"] +} +``` + +`gemma4:latest` is allowed because the exact rule is more specific, while other `gemma4:*` variants are blocked. + +## Example: model A on node 1, model B on node 1 + 2 + +```json +"workers": [ + { + "name": "node-1", + "url": "http://10.10.11.10:11434", + "max_concurrent": 4, + "model_placement": { + "mode": "whitelist", + "allowed_models": ["model-a:*", "model-b:*"], + "denied_models": [] + } + }, + { + "name": "node-2", + "url": "http://10.10.11.20:11434", + "max_concurrent": 2, + "model_placement": { + "mode": "whitelist", + "allowed_models": ["model-b:*"], + "denied_models": [] + } + } +] +``` + +The adaptive router then chooses between node 1 and node 2 for `model-b:*`, but `model-a:*` can only run on node 1. + +## Inventory behavior + +Each worker's `/api/tags` inventory is tracked separately from `/api/ps` loaded state. + +Routing behavior is: + +1. remove unhealthy workers; +2. remove workers blocked by Model Placement; +3. if inventory is known, require the model to be installed; +4. enforce global and per-model concurrency; +5. score the remaining workers by loaded affinity, load, learned tok/s, GPU utilization and VRAM pressure. + +If at least one eligible worker has a known installed copy, only those installed workers are considered. If all eligible inventories are known and none contains the model, the gateway returns a descriptive `model ... is not installed on any eligible worker` error instead of sending the request to an arbitrary node. + +A worker whose inventory has never been successfully retrieved is treated as **inventory unknown**, not as "model absent". That provides a controlled fail-open path for transient `/api/tags` discovery failures while still respecting hard placement rules. + +## Client model discovery + +`/api/tags` and `/v1/models` are gateway-native aggregated endpoints. Models that are installed only on placement-blocked workers are excluded from client discovery. This keeps OpenWebUI and other clients aligned with the gateway's actual routing policy. + +The admin model inventory remains unfiltered so operators can still see and manage models that are installed but deliberately blocked from inference. + +## Persistence precedence + +For a worker: + +```text +persistent UI placement override + ↓ (if absent) +config.json / persistent config baseline + ↓ +effective routing rule +``` + +Deleting the UI override restores the baseline immediately. The placement file is included in gateway ZIP backups and is shown on **Admin -> Persistenz**. diff --git a/docs/OPENWEBUI.md b/docs/OPENWEBUI.md new file mode 100644 index 0000000..799b99b --- /dev/null +++ b/docs/OPENWEBUI.md @@ -0,0 +1,134 @@ +# OpenWebUI integration + +## Recommended connection + +Use OpenWebUI's Ollama provider connection with the gateway base URL (no `/api` suffix) and a dedicated gateway API key. You can either configure a durable key in `auth.api_keys` or create a persistent key under **Admin -> Sicherheit -> API Keys**. + +Gateway configuration: + +```json +"auth": { + "oidc": { "enabled": false }, + "api_keys": [ + { + "name": "openwebui", + "key": "${OPENWEBUI_GATEWAY_KEY}", + "tenant": "interactive", + "subject": "openwebui", + "application": "openwebui", + "scopes": [] + } + ], + "ip_bypass": [], + "trusted_proxies": [] +} +``` + +### Create the key in the web interface + +Open **Admin -> Sicherheit -> API Keys -> API-Key erstellen** and use, for example: + +```text +Name: openwebui +Tenant: interactive +Application: openwebui +Scopes: (empty) +``` + +Copy the generated `ofg_...` secret immediately and paste it into OpenWebUI. The gateway does not retain the plaintext value. The plaintext is shown once; only its SHA-256 hash and metadata are persisted, so the same OpenWebUI credential remains valid across gateway restarts. + +OpenWebUI connection: + +```text +URL: http://host.docker.internal:8080 +API Key: +``` + +OpenWebUI's backend appends native Ollama paths itself. Do not configure the URL as `...:8080/api`. + +## Diagnostics + +From a machine that can reach the gateway: + +```bash +curl -i -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" http://GATEWAY:8080/api/version +curl -i -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" http://GATEWAY:8080/api/tags +``` + +Expected status for both is HTTP 200. `/api/tags` must contain a `models` array and every returned model is normalized to contain both `name` and `model`. + +OpenAI-compatible discovery is also available: + +```bash +curl -i -H "Authorization: Bearer $OPENWEBUI_GATEWAY_KEY" http://GATEWAY:8080/v1/models +``` + +## Docker networking + +The default example IP bypass only trusts loopback. An OpenWebUI Docker container normally reaches the gateway from a Docker/host network address, so loopback bypass does not apply. Prefer a dedicated API key over widening the bypass CIDR. + +On Docker Desktop for macOS, `host.docker.internal` normally resolves to the host. On Linux, use an address/service name reachable from the OpenWebUI backend or configure Docker's host-gateway mapping. + +## Gateway compatibility behavior + +The gateway owns model discovery instead of forwarding it to a single control worker: + +- `GET /api/tags`: parallel inventory across all workers, deduplicated by model ID. +- `GET /api/ps`: aggregated loaded-model view. +- `GET /v1/models`: OpenAI model list generated from the same aggregate inventory. +- `POST /api/show`: routed to a worker that owns the requested installed model when known. +- Compute requests are constrained to workers known to own the model; a loaded copy receives additional affinity preference. + +If one worker fails during discovery but another returns models, the gateway returns the available model set and sets `X-Gateway-Partial-Errors`. If no worker can provide tags, discovery returns HTTP 503 rather than an empty, misleading list. + +## OpenWebUI with "Authentication: None" and IP bypass + +If the OpenWebUI connection is configured with **Authentication: None**, the gateway must authenticate the OpenWebUI backend through `auth.ip_bypass`. The bypass is evaluated against the **source IP observed by the gateway**, not against the gateway URL. + +For example, if the gateway log shows: + +```text +authentication rejected client_ip=10.10.11.42 ... path=/api/version ... +``` + +add only that OpenWebUI host address when possible: + +```json +"ip_bypass": [ + { + "cidrs": ["10.10.11.42/32"], + "tenant": "interactive", + "subject": "openwebui", + "application": "openwebui", + "scopes": [] + }, + { + "cidrs": ["127.0.0.1/32", "::1/128"], + "tenant": "local", + "subject": "localhost", + "application": "local-tools", + "scopes": ["gateway:admin"] + } +] +``` + +Then OpenWebUI may remain configured with: + +```text +URL: http://10.10.11.123:8080 +Authentication: None +``` + +Do **not** blindly add `10.10.11.123/32` just because that is the gateway URL. That is the destination address. Use the `client_ip` value printed by the gateway when OpenWebUI performs `/api/version` or `/api/tags`. + +For Docker, the observed source may be a Docker bridge/VM address rather than the LAN address of the host. A dedicated static API key is more stable than allowing a broad Docker subnet. + +### Authentication diagnostics + +Authentication failures now log the resolved client address and include it in the `X-Gateway-Client-IP` response header. Native Ollama endpoints also return Ollama-compatible errors such as: + +```json +{"error":"authentication required"} +``` + +rather than an OpenAI-shaped nested object. This prevents OpenWebUI from rendering the gateway error as `[object Object]`. diff --git a/docs/PERSISTENCE.md b/docs/PERSISTENCE.md new file mode 100644 index 0000000..33c7b7d --- /dev/null +++ b/docs/PERSISTENCE.md @@ -0,0 +1,93 @@ +# Persistent local state + +The gateway deliberately separates the inference hot path from durable state. Fair queue ordering, active worker slots, live requests and browser OIDC sessions remain in memory. Restart-worthy control-plane/accounting data is stored under `storage.data_dir` using only the Go standard library. + +## Files + +| File | Contents | Write pattern | +|---|---|---| +| `gateway-config.json` | configuration override saved from Admin -> Konfiguration | atomic replace | +| `api-keys.json` | UI-created key metadata + SHA-256 hashes | atomic replace | +| `policies.json` | tenant policy overrides | atomic replace | +| `metrics.json` | Prometheus counters and histogram state | periodic atomic snapshot + shutdown | +| `quota.json` | actor/tenant credit bucket balances | periodic atomic snapshot + shutdown | +| `worker-performance.json` | learned per-worker/model prompt/output tok/s EWMA | periodic atomic snapshot + shutdown | +| `model-placement.json` | live per-worker model placement overrides from the admin UI | atomic replace | +| `conversations.enc.json` | optional encrypted Responses conversation contexts | synchronous encrypted atomic replace + retention cleanup | +| `batch-jobs.json` | optional durable batch definitions, identity metadata, state and spool references | synchronous atomic replace on state transitions | +| `batch/input/*.json` | durable batch request payloads | create + fsync + rename, mode 0600 | +| `batch/output/*.response` | durable batch response payloads | temp file + fsync + rename, mode 0600 | +| `usage/usage-YYYY-MM-DD.jsonl` | request-level accounting inside detail retention | append-only, buffered | +| `usage/rollups/daily/rollup-daily-YYYY-MM-DD.json` | daily dimensional usage aggregates | atomic replace during compaction | +| `usage/rollups/monthly/rollup-monthly-YYYY-MM.json` | long-term monthly dimensional aggregates | idempotent atomic replace | + +By default, no prompt text or generated response content is written by these stores. There are two explicit opt-in exceptions: `conversations.enabled=true` stores Responses context encrypted at rest, while `batch_jobs.enabled=true` allows submitted batch request/response content to be written to the local spool in plaintext files protected by filesystem permissions. See `docs/CONVERSATIONS.md` and `docs/BATCH-JOBS.md`. + +## Bootstrap and persistent configuration + +The file passed with `-config` is the bootstrap configuration. It determines `storage.data_dir` and the state filenames. If `/gateway-config.json` exists and validates, it becomes the effective configuration for the process. + +The web JSON editor saves a complete validated override but never exposes API-key secrets, the UI session secret or the browser OIDC client secret. Redacted secret values are preserved from the currently effective configuration. API keys themselves should be managed through the dedicated Security page. + +The `storage` section is bootstrap-only. This prevents the persistent configuration from moving the file that is needed to locate itself. To move the state directory, stop the gateway, update the bootstrap config and move/copy the state directory explicitly. + +## Crash and shutdown semantics + +Atomic state files are written through a temporary file, `fsync`, and rename. API keys, tenant policies, model-placement overrides and durable batch state transitions are saved synchronously when correctness requires it. Metrics, quota and learned worker-performance snapshots are refreshed every `storage.flush_interval` and once again during graceful shutdown. A hard process/host crash can therefore lose at most the most recent snapshot interval for those snapshot files; a batch attempt left as `running` is repaired to `queued` during startup recovery. + +Usage events are appended asynchronously and flushed according to `usage.flush_interval`. On normal shutdown the queue is drained and the current journal is flushed/synced. Before startup replay, retention compaction runs once so expired request rows are never needlessly loaded back into memory. Startup then reconstructs all-time counters from monthly rollups, daily rollups and the remaining detail journals. Only detail-journal events populate the recent-request UI. + + +## Tiered usage retention and aggregation + +The default retention policy is: + +```json +"retention": { + "detail_days": 30, + "daily_days": 400, + "monthly_months": 0, + "compaction_interval": "6h" +} +``` + +- **Detail**: raw request metadata for the most recent 30 calendar days. +- **Daily**: older request rows are replaced by one aggregate file per day until day 400. +- **Monthly**: older daily files are folded into monthly files. `monthly_months: 0` means keep them forever. + +Each rollup stores independent aggregate dimensions for global traffic, tenants, actors, applications, models and workers. Stored measures include requests/errors, prompt/completion/cached tokens, credits, queue/service duration, prompt/eval nanoseconds and transferred bytes. Prompt/output token throughput can therefore be derived after request details have expired. + +Monthly files keep their source day keyed internally. This is deliberate: if the process crashes after the monthly file is synced but before the daily source file is deleted, retrying compaction replaces that day contribution instead of adding it twice. + +Compaction only touches closed historical journal days. The active day's writer is never rewritten. The background interval can be configured, and admins can trigger the same compaction with `POST /gateway/ui-api/storage/compact` or **Admin -> Persistenz -> Jetzt kompaktieren**. + +Historical aggregate queries are exposed as: + +```text +GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=global&limit=120 +GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=tenant&name=team-a +GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=actor&name=team-a%00alice +GET /gateway/ui-api/usage/rollups?granularity=monthly&dimension=model&name=qwen3:8b +``` + +The Prometheus endpoint exports gauges for raw/daily/monthly file counts and byte footprints, plus the timestamp of the last compaction and the number of bytes reclaimed by that run. The cumulative Prometheus counter snapshot itself remains fixed-size and does not need retention compaction. + +## Intentionally non-persistent state + +- fair-queue heap and virtual clocks; +- running/streaming transient inference jobs and cancellation handles; +- active batch attempt contexts/cancellation handles (the durable batch definition and state remain persistent); +- active worker/model slots; +- live-flow/infrastructure animation state; +- browser OIDC sessions; +- transient model pull operation state. + +These objects are tied to sockets, requests or process-local execution and cannot safely be resumed after a restart. + +## Backup + +For a consistent offline backup, stop the gateway and copy `storage.data_dir`. For normal filesystem snapshots, the atomic JSON files and append-only usage journals are safe to copy while the process is running, though the newest buffered usage records may not yet be present on disk. + +## Admin storage controls + +**Admin -> Persistenz** shows every durable state file plus detail/daily/monthly usage and batch-spool footprint. **Jetzt flushen** forces metrics, quota, worker performance, encrypted conversation retention state, durable batch metadata, and buffered usage to disk. **Jetzt kompaktieren** applies usage/conversation/batch retention immediately. **Backup herunterladen** flushes first and streams a ZIP containing durable state, batch spool content when present, remaining detail journals and all rollups. Treat the backup as sensitive because configuration can contain deployment secrets and enabled batch jobs can contain prompt/response content. diff --git a/docs/PRODUCTION-UPDATE.md b/docs/PRODUCTION-UPDATE.md new file mode 100644 index 0000000..286ef72 --- /dev/null +++ b/docs/PRODUCTION-UPDATE.md @@ -0,0 +1,67 @@ +# Production update checklist + +This release is intended to be safe to stage as an update of an existing single-process Ollama Fair Gateway. The sustained HA-readiness work is isolated from the inference hot path: `cmd/ha-sampler`, the readiness shell wrapper and report tooling do not participate in normal request handling. + +## Scope of this checkpoint + +Relative to checkpoint 16, the gateway runtime behavior and persisted-state schemas are unchanged. The release adds/refines only HA-readiness tooling and documentation. The packaged gateway binaries should therefore remain byte-identical to checkpoint 16 when built with the same Go toolchain and flags; the release verification records this explicitly. + +If the production system is older than checkpoint 16, treat this as a normal application upgrade because earlier checkpoints added runtime alias/tenant ACL controls, conversations and durable batch state. The configuration loader supplies defaults for omitted optional sections, but a backup is still required before replacing an older binary. + +## Before updating + +1. Record the currently deployed binary checksum and keep the old binary available for rollback. +2. Back up the bootstrap configuration and `storage.data_dir`. For the most consistent backup, stop the gateway before copying the data directory; see `docs/PERSISTENCE.md`. +3. Validate the intended effective configuration with `ollama-gateway -config -check-config`. For containers, run this in the candidate image with the production config/state mounts so file permissions are tested too; see `docs/DEPLOYMENT-HARDENING.md`. +4. Confirm there is enough free space for the existing usage/batch/conversation retention settings. +5. Do not run the HA readiness sweep against production traffic unless the additional synthetic inference load is acceptable. + +## Update procedure + +1. Stop the supervised gateway gracefully with SIGTERM and wait for the `gateway stopped` log line. +2. Replace only the gateway executable appropriate for the host architecture. Verify its SHA-256 value against `dist/SHA256SUMS`. +3. Keep the existing bootstrap config and data directory in place. +4. Start the gateway under the same supervisor/service account. +5. Require both `/healthz` and `/readyz` to succeed before restoring external traffic. +6. Send at least one representative non-streaming and, if used in production, one streaming request through the normal authenticated client path. +7. Verify `/metrics`, the admin UI/API used operationally, and recent logs for persistence/configuration errors. + +## Rollback + +If startup/readiness or representative inference fails, stop the new process and restore the previous executable. When upgrading from checkpoint 16 specifically, no state-schema rollback is required by this checkpoint because the runtime/persistence code is unchanged. When upgrading from an older build, retain the pre-upgrade data-directory backup until the update has been observed successfully under normal load. + +## HA readiness tooling in production + +`GATEWAY_PID` must be the actual gateway PID. With it set, `scripts/ha-readiness.sh` captures before/after snapshots and starts `cmd/ha-sampler` during each load level. The sampler is read-only with respect to the gateway process; on Linux it reads `/proc` and uses `ps`, and it writes evidence only to the selected `OUT_DIR`. It does not write secrets or gateway state. + +## Checkpoint 18 Admin UI hotfix + +If upgrading from checkpoint 17, no configuration or persistent-state migration is required. Rebuild/redeploy the gateway executable or container because the Admin UI assets are embedded in the binary. Checkpoint 18 restores the Model Placement UI implementation and adds a regression test for missing render-dispatch functions. + + +## Checkpoint 19 deployment hardening + +Checkpoint 19 keeps P3.2/HA gated and instead hardens the single-node production update path. The shipped Compose command no longer repeats the image entrypoint, the scratch image has a built-in liveness healthcheck via the gateway's `-probe` mode, and `-check-config` validates the effective bootstrap+persistent configuration plus state-directory writability before startup. No persisted-state schema migration is introduced by this checkpoint. + +## Checkpoint 20 remote worker telemetry + +Checkpoint 20 adds an optional out-of-band worker telemetry agent and hardens the existing `workers[].telemetry_url` merge path. It does **not** change any persistent-state schema and the agent is not part of inference request execution. Existing deployments can upgrade without enabling the agent; behavior remains unchanged until `telemetry_url` is configured. + +For remote workers, prefer `local_system_stats: false` and point `telemetry_url` at the agent running on the actual Ollama host. Roll this out in two stages: first deploy/verify each agent with `-once` and its `/telemetry` endpoint, then change the gateway worker configuration. Keep the static `memory_capacity_bytes` and `vram_capacity_bytes` values as capacity hints even when runtime telemetry is enabled. + +If telemetry becomes unavailable, the gateway continues operating; the worker telemetry snapshot records the error and routing falls back to the remaining configured/observed signals. Do not expose the agent on an untrusted network. Restrict `-allow-cidrs` to the gateway source address or protect the endpoint with a TLS-authenticated reverse proxy. + +## Checkpoint 21 telemetry freshness hardening + +Checkpoint 21 does not change persistent-state schemas or inference semantics. It validates timestamps from `telemetry_url` exporters, rejects stale/future samples before they affect routing pressure, preserves pre-existing local collector warnings when external telemetry fails, and makes the shipped worker agent explicitly non-cacheable. Existing exporters without `updated_at` remain compatible. + + +## Checkpoint 22 IP-bypass hardening + +Checkpoint 22 changes only the authentication source used by credential-free `auth.ip_bypass`: the TCP peer is now authoritative by default. Forwarded client-IP resolution remains available for logs and usage attribution. Deployments that intentionally relied on an original `X-Forwarded-For` client address to satisfy IP bypass must explicitly set `auth.ip_bypass_use_forwarded_ip=true` and should restrict `auth.trusted_proxies` to exact proxy addresses plus network ACLs. API-key and OIDC authentication are unaffected. No persistent-state schema migration is introduced. + +## Checkpoint 23 reverse-proxy/deployment boundary hardening + +Checkpoint 23 does not change inference, scheduler, quota or persistent-state schemas. The supplied Compose topology now defaults to host-loopback publishing (`127.0.0.1:9080`), a read-only root filesystem, `no-new-privileges` and all Linux capabilities dropped. Deployments whose reverse proxy is on another host must explicitly opt into a non-loopback bind and should firewall the published port to that proxy only. + +Before rollout, run `scripts/production-preflight.sh` with the production bootstrap config. The script refuses the development `config.example.json` and non-loopback publishing unless explicitly acknowledged, then runs the gateway's secret-redacted `-check-config` inside the candidate container with the production mounts. diff --git a/docs/PUBLIC-DASHBOARD.md b/docs/PUBLIC-DASHBOARD.md new file mode 100644 index 0000000..042bb92 --- /dev/null +++ b/docs/PUBLIC-DASHBOARD.md @@ -0,0 +1,64 @@ +# Public Status Dashboard + +Checkpoint 28 adds an optional unauthenticated **read-only** dashboard for users who should be able to observe current gateway capacity without receiving admin access. + +## Enable + +```json +"public_dashboard": { + "enabled": true, + "path": "/status", + "title": "Ollama Gateway Status", + "subtitle": "Live-Auslastung und Infrastruktur", + "refresh_interval": "2s", + "max_live_requests": 64, + "show_worker_names": false, + "show_model_names": true, + "show_resource_metrics": true, + "worker_display_names": { + "internal-worker-name": "GPU Node A" + } +} +``` + +The dashboard is then available at `/status/` without a gateway credential. It is independent of the authenticated `/admin/` UI and also works when the admin UI itself is disabled. + +## Privacy boundary + +The public API does **not** reuse or serialize the admin snapshots. It builds a new response from an explicit allow-list. The public response excludes: + +- tenant, actor, subject and application identifiers; +- API keys, scopes and authentication metadata; +- worker URLs, IP labels and arbitrary worker labels; +- prompts, responses and request bodies; +- persistent storage paths and configuration; +- policies, quota buckets, aliases and ACLs; +- internal error strings, circuit error details and telemetry errors; +- cost/credit estimates. + +Request IDs are replaced by short SHA-256-derived opaque IDs. Worker names are anonymized by default and can be replaced with explicit public display names. Model names are also anonymized unless `show_model_names` is explicitly enabled. + +## Public data + +The dashboard exposes only operational information intended for status display: + +- current queue/running/streaming counts; +- gateway uptime; +- healthy/total worker count; +- number of loaded models; +- worker slot usage and health state; +- loaded model display names when enabled; +- numeric RAM/VRAM/GPU/temperature/power values when resource metrics are enabled; +- anonymized recent/live request state, queue/service timing and token counts. + +The snapshot endpoint is `GET /api/snapshot`. It is cacheable for one second to reduce repeated work behind a reverse proxy. The browser refresh interval is configurable but cannot be set below one second. + +## Reverse proxy + +Checkpoint 23+ binds the production Compose service to loopback by default. Expose `/status/` through the same trusted reverse proxy used for the public service. Do not expose the gateway container port directly merely to publish the dashboard. + +If the reverse proxy applies authentication globally, carve out only the configured public dashboard path. Keep `/admin/`, `/gateway/`, `/api/`, `/v1/` and `/metrics` under their existing gateway authentication rules. + +## Security headers + +The public UI sends a restrictive CSP, denies framing, disables referrer leakage and disables browser permissions such as camera, microphone, geolocation, payments and USB. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-17.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-17.md new file mode 100644 index 0000000..e514703 --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-17.md @@ -0,0 +1,47 @@ +# Checkpoint 17 — production update verification + +Validation date: 2026-09-08 +Validation toolchain: Go 1.23.2 on Linux/amd64 + +## Release scope + +Checkpoint 17 completes sustained HA-readiness resource sampling. It adds `cmd/ha-sampler`, shared read-only resource collection under `internal/haresource`, sweep integration, report integration, tests and operator documentation. + +The normal gateway request, scheduler, routing and persistence code is unchanged relative to checkpoint 16. Rebuilding with the same Go toolchain and release flags produced byte-identical gateway executables for all four packaged gateway targets. + +## Required verification results + +The following checks passed on the final source tree: + +- `go test ./...` +- `go test ./... -count=3` during the release cycle +- `go vet ./...` +- `go test -race ./...` +- `gofmt -l .` returned no files +- `sh -n scripts/ha-readiness.sh scripts/ollama-env.sh` +- all shipped JSON example configurations parse with `jq` +- all files in `dist/SHA256SUMS` verify successfully + +## Runtime smoke validation + +The packaged Linux/amd64 gateway was tested against the deterministic mock Ollama backend. + +1. **Persistent restart smoke:** two complete start/readiness/request/shutdown cycles reused the same `storage.data_dir`. Each cycle served a non-streaming and a streaming OpenAI-compatible request, logged `gateway stopped`, and exited with status 0. +2. **Authenticated readiness smoke:** private `/metrics` plus a Bearer API key completed the readiness sweep. Client counts reconciled with gateway request/queue/service counters, endpoint resource snapshots were complete, and sustained resource traces were complete. +3. **Secret-leak check:** the Bearer API key used by the authenticated sweep was not present in any generated readiness result file. +4. **Final sustained-sampler E2E:** concurrency 1 and 4 both completed with zero request errors, exact counter reconciliation, and `resource_samples.stop_reason == "stop-file"`. A sampler that exits because of `max-duration`, process exit, or interruption is deliberately not accepted as complete sustained evidence. + +## Gateway binary identity vs checkpoint 16 + +| Target | Checkpoint 17 SHA-256 | Compared with checkpoint 16 | +|---|---|---| +| Linux amd64 | `01efc1b236faa9b7b912e6bee6af8266f6a24722af419a6dd43b6f294c955f47` | byte-identical | +| Linux arm64 | `a364f03e67fe377f140936c2c0cdd2b480166e2c5d1fe519878d730268e74248` | byte-identical | +| macOS arm64 | `8f9185f561752656902ef8cf44f5f1c8f65b660f40fb5caf790c2d23773c14d8` | byte-identical | +| Windows amd64 | `5a5bbbf962f1a9fea88a76fc0ee1e28669170496800e9ea39d9ed11caf14b053` | byte-identical | + +This identity is the strongest update-safety property of this checkpoint for systems already on checkpoint 16: replacing the gateway executable with the checkpoint 17 gateway executable does not change the executable bytes. + +## Operator note + +For systems older than checkpoint 16, use `docs/PRODUCTION-UPDATE.md` and take a configuration/data-directory backup first. Earlier checkpoints contain real gateway runtime and persistence features, so the byte-identity statement above does not apply to an upgrade directly from an older production build. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-18.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-18.md new file mode 100644 index 0000000..8c74b5f --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-18.md @@ -0,0 +1,67 @@ +# Checkpoint 18 — Admin UI placement hotfix + +Validation date: 2026-09-08 + +## Release scope + +Checkpoint 18 is a production hotfix on top of checkpoint 17. It restores the complete Model Placement UI helper block that was accidentally removed during the runtime model-alias UI work introduced after checkpoint 13. + +The regression manifested after successful admin authentication as a browser error such as: + +`renderPlacement is not defined` + +The missing block also contained the generic `lines()` helper used by alias, placement and tenant model-access forms, so leaving only `renderPlacement` patched would have exposed follow-on UI failures when saving those forms. + +No gateway request, scheduler, routing, quota, persistence, batch, conversation, HA-sampling or worker-runtime behavior was intentionally changed by this hotfix. The gateway executables do change because the Admin UI JavaScript is embedded into the binary. + +## Fix + +Restored these UI helpers: + +- `placementRule` +- `placementRows` +- `placementCell` +- `placementSourceLabel` +- `renderPlacement` +- `placementCellHTML` +- `placementWorkerCard` +- `loadPlacement` +- `openPlacementDialog` +- `lines` +- `installedForWorker` +- `savePlacementRule` +- `placementExactAction` + +The existing placement matrix, worker-rule editor, exact allow/deny overrides, presets and reset controls now have their implementation functions again. + +## Regression protection + +`internal/webui/webui_test.go` now verifies that every `renderX()` function called by the main `render()` dispatcher is actually defined in the embedded Admin UI bundle. A second test verifies the complete Placement UI helper set. + +This specifically prevents the original `renderPlacement is not defined` class of regression from passing `go test ./...` again. + +## Required verification results + +The final source tree passed: + +- `go test ./...` +- `go vet ./...` +- `go test -race ./...` +- `node --check internal/webui/assets/app.js` +- release cross-builds for Linux amd64, Linux arm64, macOS arm64 and Windows amd64 +- `dist/SHA256SUMS` verification + +## Update guidance + +For Docker/Compose deployments, rebuild or replace the gateway image/binary. Copying only the source `app.js` beside an already-built binary does not fix the problem because `internal/webui/assets/app.js` is embedded at build time. + +The production configuration and persistent `/data` volume do not require a migration from checkpoint 17 to checkpoint 18. + +## Gateway binary SHA-256 + +| Target | SHA-256 | +|---|---| +| Linux amd64 | `550f481313e5869d884280dfc3248d70b6eabf51b4b3bb44083c36f622065cf2` | +| Linux arm64 | `27f4ab0b7a1dc9a2379e7e99471e177f9da9c715445b05b089d028011b8101aa` | +| macOS arm64 | `d72dd623844050b1b7396ae364e49a30ab76d2bee21a2ced22dad308c0e839bb` | +| Windows amd64 | `dedf06c3d42fdc8c2fe3de19cd9c00fcdb6eb6fab91442df3b18ac5e8a416072` | diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-19.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-19.md new file mode 100644 index 0000000..6dacd58 --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-19.md @@ -0,0 +1,58 @@ +# Checkpoint 19 — deployment preflight and container hardening + +Validation date: 2026-09-08 + +## Scope + +Checkpoint 19 follows the Checkpoint 18 Admin UI hotfix. It does not implement P3.2/HA and does not change any persistent-state schema. Its purpose is to reduce single-node production update risk after real deployment feedback exposed configuration mount/entrypoint and file-permission mistakes that were otherwise only visible after startup. + +The inference request/scheduler/quota/proxy path is unchanged by this checkpoint. Runtime changes are limited to startup configuration loading being factored through the same helper used by the new preflight, plus two explicit CLI-only modes (`-check-config` and `-probe`). + +## Changes + +- `-check-config`: strict effective configuration validation plus persistent override loading and state-directory write probe. +- Secret-free JSON preflight summary with worker count and deployment warnings. +- `-probe`: minimal HTTP probe implemented inside the gateway binary for scratch/container health checks. +- Dockerfile/Compose liveness healthcheck against public `/healthz`. +- Corrected Compose command: arguments only, no duplicate `/ollama-gateway` after the image `ENTRYPOINT`. +- `GATEWAY_CONFIG` Compose variable for selecting the bootstrap config explicitly. +- Regression tests for persistent-secret restoration, preflight output, HTTP probing and Compose entrypoint/healthcheck rules. +- Production deployment documentation. + +## Verification completed + +The final source tree passed: + +- `go test ./... -count=1` +- `go vet ./...` +- Race detector over all packages, split as `go test -race ./cmd/...` and `go test -race ./internal/...` so each command completed with an observable exit status. (The single monolithic invocation exceeded the execution harness timeout; no package was omitted from the split run.) +- `node --check internal/webui/assets/app.js` +- Compose YAML parse/assertions for the corrected command, `GATEWAY_CONFIG` mount and healthcheck. The Docker CLI itself was not available in the verification environment, so `docker compose config` was not claimed. +- Real Linux-amd64 binary `-check-config` run against the migrated production configuration: `status=ok`, exactly 2 effective workers, writable state directory, expected remote `local_system_stats` warnings, and no API-key secret in the preflight output. +- Real gateway + deterministic mock Ollama smoke: `/healthz` probe, `/readyz` probe, authenticated `/gateway/ui-api/session`, OpenAI-compatible inference, SIGTERM, and gateway exit code 0. +- Cross-builds for Linux amd64, Linux arm64, macOS arm64 and Windows amd64. +- `dist/SHA256SUMS` verification for every packaged binary/helper. + +## Gateway binary SHA-256 + +| Target | SHA-256 | +|---|---| +| Linux amd64 | `e30a7b01f8a96f6396c3582ab208bf8e4e330c6a882b724998448dc66dfa7cf7` | +| Linux arm64 | `675a72c054c21ba383c36891b9bdeb1b93cbd4690abedc22b74ae02c8bc081e4` | +| macOS arm64 | `586e489cf00c54aa5906a63f4ceada6ff22c90943a80d31fb1c864b557dde559` | +| Windows amd64 | `1d24976651a358bc80031f36eb2f59cca9233dfa89bf3189bb1bb4b6e9de96fe` | + +The HA helper binaries were not modified by this checkpoint and their existing checksums remain listed and verified in `dist/SHA256SUMS`. + +## Update guidance + +No configuration-schema or persistent-state migration is required from Checkpoint 18. Container deployments should rebuild/recreate the image because the Dockerfile healthcheck and gateway CLI modes are part of this release. + +Before starting production traffic, run the candidate container in preflight mode with the exact config/state mounts: + +```sh +GATEWAY_CONFIG=./gateway-config.json docker compose run --rm gateway \ + -config /etc/ollama-gateway/config.json -check-config +``` + +Then start the service, wait for liveness, require `/readyz`, authenticate to the admin UI/API and send representative inference traffic. See `docs/DEPLOYMENT-HARDENING.md` and `docs/PRODUCTION-UPDATE.md`. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-20.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-20.md new file mode 100644 index 0000000..dd6436f --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-20.md @@ -0,0 +1,69 @@ +# Checkpoint 20 — remote worker telemetry + +Date: 2026-09-08 + +## Scope + +Checkpoint 20 extends the single-node production hardening path with an optional `worker-telemetry` agent for remote Ollama hosts. It also fixes the existing external telemetry merge so omitted JSON fields no longer overwrite previously collected values with zero, while explicit zero values remain meaningful. + +No persistent-state schema changes are introduced. The inference, quota and scheduler request paths are unchanged. + +## New artifacts + +- `cmd/worker-telemetry` +- `dist/ollama-gateway-worker-telemetry-linux-amd64` +- `dist/ollama-gateway-worker-telemetry-linux-arm64` +- `docs/WORKER-TELEMETRY.md` +- Linux AMDGPU sysfs collector in `internal/hoststats` + +## Verification completed + +The final source tree passed: + +- `go test ./...` +- `go vet ./...` +- JavaScript syntax validation with `node --check internal/webui/assets/app.js` +- race tests across all command packages and all tested internal packages, including `internal/worker` and `internal/hoststats` +- SHA-256 verification for every file listed in `dist/SHA256SUMS` +- cross-builds for gateway Linux amd64/arm64, macOS arm64 and Windows amd64 +- cross-builds for worker telemetry Linux amd64/arm64 + +## Remote telemetry E2E + +A Linux-amd64 release gateway was started with: + +- one mock Ollama worker, +- `local_system_stats: false`, +- `telemetry_url` pointing at the release worker-telemetry agent, +- a temporary simulated AMDGPU sysfs tree. + +The gateway reported the worker healthy and exposed telemetry source: + +`telemetry-url:host-memory+amdgpu-sysfs` + +The simulated GPU values were preserved end-to-end: + +- VRAM used: 4,294,967,296 bytes +- VRAM total: 17,179,869,184 bytes +- GPU utilization: 73% +- GPU temperature: 62.5 C +- GPU power: 88 W + +Both the gateway and telemetry agent exited cleanly with status 0 after SIGTERM. + +## Merge regression coverage + +`externalTelemetry` uses pointer fields. This explicitly distinguishes an omitted JSON property from a supplied zero. Tests verify that omitted memory/GPU fields preserve existing telemetry while an explicit zero for VRAM usage or GPU utilization is applied. + +## Production rollout + +Enabling the agent is optional and can be staged independently from the gateway update. For a remote worker: + +```json +{ + "local_system_stats": false, + "telemetry_url": "http://WORKER-IP:11500/telemetry" +} +``` + +Validate the agent locally with `-once` first and restrict network access using `-allow-cidrs` plus the worker host firewall. See `docs/WORKER-TELEMETRY.md`. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-21.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-21.md new file mode 100644 index 0000000..24fa966 --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-21.md @@ -0,0 +1,40 @@ +# Checkpoint 21 — telemetry freshness hardening + +Date: 2026-09-08 + +## Scope + +Checkpoint 21 hardens the optional remote worker telemetry path introduced in checkpoint 20. No persistent-state schema changes are introduced and the inference/quota/scheduler hot paths are unchanged. + +Changes: + +- parse optional external `updated_at`, +- reject samples older than `max(30s, 6 x health_interval)`, +- reject timestamps more than 30 seconds in the future, +- keep backward compatibility for exporters without timestamps, +- append external HTTP/decode/freshness errors instead of overwriting an existing collector warning, +- send `Cache-Control: no-store` and `X-Content-Type-Options: nosniff` from the shipped telemetry agent. + +## Safety behavior + +A rejected external sample is not merged into `ResourceTelemetry`. Static worker capacity hints, Ollama health/inventory and all other routing inputs remain available. The worker remains usable; the telemetry error is surfaced for operators. + +## Verification + +Release verification includes unit tests for fresh/stale/future/missing timestamps, merge semantics, full repository tests/vet, race tests, release cross-builds, dist checksums and a real gateway + mock Ollama + telemetry-agent smoke path. + +## E2E freshness evidence + +Fresh-agent path: + +- agent response included `Cache-Control: no-store`, +- source was `telemetry-url:host-memory+amdgpu-sysfs`, +- simulated VRAM/GPU/temperature/power values were merged correctly. + +Stale-exporter path: + +- exporter was made ready before gateway startup, +- payload timestamp was intentionally more than one hour old, +- gateway reported `telemetry sample is stale: ... exceeds 30s`, +- stale VRAM/GPU pressure values were not merged, +- gateway still shut down cleanly with exit status 0. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-22.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-22.md new file mode 100644 index 0000000..1e51b64 --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-22.md @@ -0,0 +1,39 @@ +# Checkpoint 22 — IP-bypass / trusted-proxy hardening + +Date: 2026-09-08 + +## Security issue addressed + +Before checkpoint 22, `Authenticator.Authenticate` evaluated `auth.ip_bypass` against the fully resolved client IP. If the immediate peer was in `auth.trusted_proxies`, that address could originate from `X-Forwarded-For`. A deployment that trusted a broad reachable subnet could therefore allow a direct client in that subnet to present a bypass address such as `127.0.0.1`. + +## New default + +- `ClientIP()` still resolves trusted forwarding chains for observability. +- `auth.ip_bypass` is evaluated against the direct TCP peer by default. +- `Identity.ClientIP` remains the resolved client address for usage/logging. +- Legacy forwarded-IP bypass is available only with explicit `auth.ip_bypass_use_forwarded_ip=true`. +- Config preflight warns when forwarded bypass is enabled and when `auth.trusted_proxies` contains broad non-loopback CIDRs. + +API-key and OIDC authentication semantics are unchanged. No persistent-state schema change is introduced. + +## Regression coverage + +Tests prove that: + +1. a trusted peer with `X-Forwarded-For: 127.0.0.1` cannot satisfy loopback IP bypass by default, +2. forwarded client-IP resolution still reports `127.0.0.1` for observability in that test, +3. explicit compatibility mode restores forwarded-IP bypass, +4. a direct loopback peer still satisfies the normal loopback bypass, +5. preflight warns on compatibility mode and a broad `10.0.0.0/8` trusted-proxy range while not flagging loopback-only proxy ranges. + +## Built-binary spoof E2E + +A release Linux-amd64 gateway was started with a trusted loopback proxy peer and an IP-bypass rule for `192.0.2.123/32`. + +With the new default (`ip_bypass_use_forwarded_ip=false`), a request from the trusted TCP peer carrying `X-Forwarded-For: 192.0.2.123` returned **401 Unauthorized**. + +The same test with the explicit compatibility flag set to `true` returned **200 OK** with an `ip-bypass` admin identity and resolved `client_ip=192.0.2.123`. This proves both the secure default and the intentional compatibility escape hatch in the release binary. + +## Production-config preflight + +The user's two-worker configuration validates successfully with the checkpoint-22 candidate. Preflight reports broad trusted-proxy warnings for `10.0.0.0/8`, `172.16.0.0/12` and `192.168.0.0/16`; these no longer permit forwarded headers to satisfy IP bypass under the default, but should still be narrowed to improve client-IP attribution integrity. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-23.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-23.md new file mode 100644 index 0000000..30c7c4d --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-23.md @@ -0,0 +1,57 @@ +# Release verification: checkpoint 23 + +Checkpoint 23 hardens the Docker/reverse-proxy deployment boundary. It does not change the inference hot path or persisted-state schemas. + +## Scope verification + +`cmd/` and `internal/` were byte-for-byte/diff checked against checkpoint 22 after the deployment changes. No runtime Go source changed. Existing release binaries therefore remain unchanged and all entries in `dist/SHA256SUMS` still verify. + +## Deployment changes + +- Development Compose defaults to `127.0.0.1:9080 -> :8080` instead of publishing on all host interfaces. +- `docker-compose.production.yml` requires `GATEWAY_CONFIG`; there is no production fallback to `config.example.json`. +- Container root filesystem is read-only; `/data` is the persistent writable volume and `/tmp` is tmpfs. +- Runtime explicitly uses UID/GID `65532:65532`, drops all Linux capabilities and enables `no-new-privileges`. +- JSON-file logging is bounded to five 10 MiB files in the production Compose example. +- `scripts/production-preflight.sh` rejects `config.example.json`, rejects non-loopback publishing unless explicitly acknowledged, renders Compose, and invokes the gateway's secret-redacted `-check-config` in the candidate container. +- `scripts/production-preflight_test.sh` regression-tests loopback success, example-config rejection, wildcard-bind rejection and explicit remote-proxy opt-in. + +## Production config validation + +The hardened two-worker production configuration was validated with the release binary: + +- status: `ok` +- workers: `2` +- data directory: `/data` +- state directory writable: `true` +- broad trusted-proxy warnings: none + +The configuration narrows `auth.trusted_proxies` to loopback plus the observed Docker peer `172.30.3.1/32`; `auth.ip_bypass_use_forwarded_ip` remains false. + +## Test results + +- `go test ./...`: PASS +- `go vet ./...`: PASS +- `go test -race ./cmd/...`: PASS +- `go test -race ./internal/...`: PASS +- `sh -n scripts/production-preflight.sh`: PASS +- `sh -n scripts/production-preflight_test.sh`: PASS +- `scripts/production-preflight_test.sh`: PASS +- Compose YAML parse check: PASS +- `dist/SHA256SUMS`: all 10 packaged binaries PASS + +## Release-binary smoke test + +The shipped Linux-amd64 gateway binary was started against an isolated deterministic Ollama mock and temporary state directory. + +- `/healthz`: 200 / `status=ok` +- `/readyz`: 200 / `status=ready` +- authenticated `/gateway/ui-api/session`: admin identity returned +- authenticated `/v1/chat/completions`: successful response (`chatcmpl-mock`) +- SIGTERM gateway shutdown: exit code 0 and `gateway stopped` logged + +The mock process is test infrastructure and is not part of the production artifact. + +## Deployment recommendation + +For the observed topology, keep `GATEWAY_PUBLISH_ADDRESS=127.0.0.1` so only a reverse proxy on the Docker host can reach port 9080. If the reverse proxy is remote, bind to one specific host interface, set `ALLOW_NON_LOOPBACK_BIND=1` only for the preflight, and firewall the published port to the exact reverse-proxy source IP. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-24.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-24.md new file mode 100644 index 0000000..b87ecf8 --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-24.md @@ -0,0 +1,53 @@ +# Release verification — Checkpoint 24 + +Checkpoint 24 is an admin-WebUI-only visual refresh based on the production-hardened Checkpoint 23 runtime. It changes the embedded HTML/CSS/JavaScript presentation layer and the four gateway binaries that embed those assets. Scheduler, quota, routing, authentication, persistence, telemetry and inference semantics are unchanged. + +## Scope + +- Professional light control-plane theme inspired by dense technical/mission-database UIs. +- Offline-safe font stack: preferred local IBM Plex/JetBrains/Cascadia fonts with system fallbacks; no external font or CDN dependency. +- Explicit light color scheme for browser-native controls. +- Light sidebar, panels, KPI cards, tables, forms, dialogs, callouts, code/config editor and login screen. +- Light Live Flow and Infrastructure canvases, including canvas-drawn labels/nodes/routes rather than CSS-only restyling. +- Accessibility-oriented color tokens. Primary text, muted text, accent text, primary buttons and semantic status colors meet approximately WCAG AA 4.5:1 contrast against white for normal text. +- Regression tests assert that the light theme, light metadata and light canvas colors remain embedded. + +## Verification performed + +```sh +node --check internal/webui/assets/app.js +python -c 'import tinycss2; ... parse internal/webui/assets/app.css ...' +go test ./... +go vet ./... +go test -race ./... +``` + +All commands completed successfully. + +Four gateway release targets were rebuilt because WebUI assets are embedded in the gateway binary: + +- linux/amd64 +- linux/arm64 +- darwin/arm64 +- windows/amd64 + +`dist/SHA256SUMS` was regenerated and `sha256sum -c dist/SHA256SUMS` succeeded for all shipped binaries. + +## Release-binary smoke + +The rebuilt linux/amd64 binary was started with an isolated writable data directory and deterministic mock Ollama backend. The following checks passed: + +- `-check-config` +- `/readyz` +- embedded `/admin/` +- embedded `/admin/app.css` contains the Checkpoint 24 light-theme marker +- embedded `/admin/app.js` passes `node --check` +- `/gateway/ui-api/session` via direct loopback admin IP bypass returns an admin identity +- `/v1/chat/completions` returns a successful mock response +- SIGTERM shutdown exits with code 0 + +A headless Chromium screenshot was attempted, but the container's Chromium/DBus environment did not terminate reliably. This was not counted as a passed test. Asset parsing, embedding and runtime serving were verified independently as described above. + +## Deployment + +No configuration or persistent-state migration is required from Checkpoint 23. Rebuild/redeploy the gateway image because the WebUI is compiled into the binary. After update, hard-refresh the browser once to discard any previously cached `app.css` or `app.js`. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-26.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-26.md new file mode 100644 index 0000000..139ea6a --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-26.md @@ -0,0 +1,15 @@ +# Checkpoint 26 release verification + +Scope: embedded Admin WebUI only (dialog semantics + regression tests). + +Verified: + +- `go test ./...` +- `go vet ./...` +- `node --check internal/webui/assets/app.js` +- race tests across all packages (full run completed in two segments due execution timeout) +- rebuilt Gateway binaries for Linux amd64/arm64, macOS arm64, Windows amd64 +- `dist/SHA256SUMS` verified +- real Linux-amd64 release-binary smoke confirmed embedded policy/pull close controls and clean SIGTERM shutdown + +Regression covered: required HTML form controls must not prevent modal cancellation/close. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-27.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-27.md new file mode 100644 index 0000000..884320a --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-27.md @@ -0,0 +1,84 @@ +# Checkpoint 27 release verification + +Checkpoint 27 hardens effective context-window handling and `num_ctx` admission/routing. + +## Scope + +Runtime changes are intentionally limited to context estimation, model metadata, context-aware worker eligibility/routing, the policy simulator, configuration validation, and the corresponding Admin UI context columns. Scheduler fairness, authentication, persistence formats, batch execution, conversations, and quota accounting are otherwise unchanged. + +## Effective context model + +The gateway now keeps these values distinct: + +1. **Model maximum** from Ollama `/api/show` model metadata. +2. **Configured context** from a Modelfile `PARAMETER num_ctx`, when present. +3. **Loaded context** from Ollama `/api/ps context_length`, when the model is resident. +4. **Worker default** from `workers[].default_context_tokens` or the gateway context default. +5. **Administrative cap** from `workers[].context_limits` and `model_capabilities.context.max_requested_tokens`. + +For OpenAI-compatible requests, the effective usable context is derived from the actually loaded/configured/default context and administrative limits; the theoretical model maximum is no longer treated as the runtime context by itself. + +For native Ollama `/api/*` requests, a valid explicit `options.num_ctx` can request a resize/reload, but is still bounded by the model maximum, gateway cap, and worker context limit. A worker whose currently loaded context is smaller remains eligible only as a resize candidate and does not receive a misleading loaded-context routing advantage. + +## Request estimation fixes + +Context admission now includes: + +- Responses API `max_output_tokens`; +- Responses API `instructions`; +- native `/api/generate` `suffix`; +- configurable estimation margin; +- configurable per-image vision reserve; +- the largest positive output-token budget rather than silently preferring a smaller field. + +`options.num_ctx` is strictly validated as a positive integer. It is accepted only on native Ollama `/api/*` endpoints. OpenAI-compatible requests cannot inject it as a false context override. + +## Validation performed + +The release candidate passed: + +- `go test ./...` +- `go vet ./...` +- `node --check internal/webui/assets/app.js` +- `go test -race ./...` +- strict `-check-config` against the supplied two-M75q production configuration +- Linux amd64, Linux arm64, macOS arm64, and Windows amd64 gateway cross-builds +- `dist/SHA256SUMS` verification +- a real Linux-amd64 release-binary smoke test against the deterministic mock Ollama backend: + - `/readyz` ready + - Admin API-key session HTTP 200 + - `/v1/chat/completions` HTTP 200 + - native string-valued `options.num_ctx` HTTP 400 before backend execution + - OpenAI `options.num_ctx` spoof attempt HTTP 400 + - SIGTERM shutdown exit code 0 + +The final ZIP is additionally unpacked and tested again before release. + +## Recommended production settings for the current two-M75q deployment + +The supplied production config uses conservative starting bounds for the ~16 GiB AMD worker class: + +```json +"model_capabilities": { + "mode": "enforce", + "cache_ttl": "10m", + "context_guard": "reject", + "context": { + "max_requested_tokens": 32768, + "default_worker_tokens": 4096, + "estimation_margin_percent": 15, + "vision_reserve_tokens_per_image": 2048 + } +} +``` + +and per worker: + +```json +"default_context_tokens": 4096, +"context_limits": { + "*": 16384 +} +``` + +`16384` is an operational safety cap, not a claim that every model/workload will fit efficiently at 16k. Increase it only after measuring actual VRAM pressure, offload behavior, latency, and throughput on the target worker/model combination. diff --git a/docs/RELEASE-VERIFICATION-CHECKPOINT-28.md b/docs/RELEASE-VERIFICATION-CHECKPOINT-28.md new file mode 100644 index 0000000..b73d6d2 --- /dev/null +++ b/docs/RELEASE-VERIFICATION-CHECKPOINT-28.md @@ -0,0 +1,61 @@ +# Release verification — Checkpoint 28 public status dashboard + +Checkpoint 28 replaces the active P3.2 HA roadmap item with an optional public, unauthenticated, read-only operational dashboard. Existing HA-readiness tools remain packaged but no cluster/coordinator behavior is introduced. + +## Runtime scope + +Changed runtime areas: + +- strict configuration schema: new `public_dashboard` block; +- unauthenticated routing for the configured public dashboard path only; +- a dedicated sanitized public snapshot builder; +- embedded `internal/publicui` assets. + +Unchanged semantics: + +- inference/proxy protocol handling; +- scheduler/fairness/quota algorithms; +- worker selection and model placement; +- durable state schemas and storage files; +- admin authentication and admin UI authorization; +- batch/conversation persistence. + +No state migration is required. + +## Privacy verification + +The public API is not derived by JSON-marshalling admin/infrastructure objects. Tests require an explicit allow-list and fail if known sensitive fixture values appear in the public response. Verified absent fields/data include tenant, actor, application, internal request ID, worker URL, labels, internal error strings, telemetry errors and estimated credits. + +Worker names and model names default to anonymized display names. Worker aliases can be explicitly supplied in configuration. Resource metrics contain only numeric operational values. + +## Automated verification + +- `go test ./...` — PASS +- `go vet ./...` — PASS +- `go test -race ./...` — PASS +- `node --check internal/publicui/assets/app.js` — PASS +- `node --check internal/webui/assets/app.js` — PASS +- all four gateway release targets rebuilt — PASS +- every entry in `dist/SHA256SUMS` — PASS +- `config.example.json` strict parse test — PASS +- production-derived checkpoint-28 config through `-check-config` — PASS (2 workers, `/data` writable) + +## Release-binary smoke + +The packaged Linux amd64 gateway was started with a deterministic mock Ollama backend and an isolated state directory. + +Verified: + +- `/readyz` — 200 +- `/status/` without credentials — 200 +- `/status/api/snapshot` without credentials — 200 +- `/gateway/ui-api/session` without credentials — 401 +- `/gateway/ui-api/session` with admin API key — 200 +- `/v1/chat/completions` with API key — 200 +- post-request public snapshot contains the public worker alias and opaque `REQ-*` request ID +- post-request public snapshot does not contain private gateway/worker names, auth key, tenant/actor/application fields or worker URL +- SIGTERM shutdown — gateway exit code 0 + +## Deployment + +Enable the dashboard explicitly in configuration. A recommended production-derived example is supplied separately as `gateway-config.checkpoint28.public-dashboard.json`; it publishes model names but maps the two internal M75q worker names to `GPU Node A` and `GPU Node B`. diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md new file mode 100644 index 0000000..36cdbe3 --- /dev/null +++ b/docs/RELIABILITY.md @@ -0,0 +1,27 @@ +# Reliability and worker maintenance + +## Circuit breaker + +`reliability.enabled` activates a per-worker circuit breaker. Consecutive transport/backend failures increment the worker failure counter. At `failure_threshold` the circuit enters `open` for `open_duration`. After that period the next eligible request acts as the half-open probe; success closes the circuit, failure reopens it. + +```json +"reliability": { + "enabled": true, + "failure_threshold": 3, + "open_duration": "30s", + "retry_attempts": 2, + "retry_backoff": "50ms" +} +``` + +Retries are deliberately conservative: only transport failures before any upstream response is committed to the client are retried. A backend HTTP 5xx contributes to the circuit breaker but is not replayed once its headers/body have begun. + +## Worker maintenance + +The admin Worker page exposes: + +- `active`: accepts new inference jobs; +- `draining`: no new inference jobs, existing jobs continue; +- `disabled`: no new inference jobs until explicitly enabled. + +Drain/disable state is persisted in `/worker-state.json`. Circuit state is intentionally transient and can be manually reset from the Worker page. diff --git a/docs/RETENTION.md b/docs/RETENTION.md new file mode 100644 index 0000000..4b0768b --- /dev/null +++ b/docs/RETENTION.md @@ -0,0 +1,80 @@ +# Usage retention and long-term aggregation + +The gateway stores request-level usage as append-only daily JSONL journals and compacts old data into bounded rollups. + +## Default policy + +```json +"usage": { + "journal_dir": "./data/usage", + "buffer": 16384, + "flush_interval": "1s", + "retention": { + "detail_days": 30, + "daily_days": 400, + "monthly_months": 0, + "compaction_interval": "6h" + } +} +``` + +- `detail_days`: keep per-request records for this many calendar days. +- `daily_days`: retain one daily rollup until this age. Must be >= `detail_days`. +- `monthly_months`: retain monthly rollups for this many months. `0` means forever. +- `compaction_interval`: background compaction cadence; minimum one minute. + +## Storage layout + +```text +data/usage/ +├── usage-2026-09-06.jsonl +└── rollups/ + ├── daily/ + │ └── rollup-daily-2026-07-23.json + └── monthly/ + └── rollup-monthly-2025-08.json +``` + +Each daily/monthly rollup retains aggregate dimensions for global traffic, tenant, actor, application, model, and worker. + +Measures retained after request details expire include request/error counts, prompt/completion/cached tokens, credits, queue/service time, prompt/eval nanoseconds, bytes in/out, and last-request timestamp. Prompt and output tokens/second remain derivable from aggregate evaluation durations. + +## Crash/idempotency behavior + +Raw -> daily compaction writes and fsyncs the new daily file before deleting the source journal. Repeating this after a crash replaces the same daily period rather than summing it twice. + +Daily -> monthly files retain a map of their source days. If a process dies after the monthly file is committed but before the source daily file is deleted, replaying compaction replaces that day entry. This makes the monthly fold idempotent. + +## Startup and all-time accounting + +Before replaying request journals, startup applies retention once. It then reconstructs all-time usage from retained monthly rollups, retained daily rollups, and the remaining request-level journals. Only remaining request-level journals populate the recent-request table. + +If `monthly_months > 0`, data older than that window is deleted completely and removed from reconstructed all-time usage. + +## Admin/API + +```text +POST /gateway/ui-api/storage/compact +GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=global&limit=120 +GET /gateway/ui-api/usage/rollups?granularity=daily&dimension=tenant&name=team-a +GET /gateway/ui-api/usage/rollups?granularity=monthly&dimension=model&name=qwen3:8b +``` + +Allowed dimensions: `global`, `tenant`, `actor`, `application`, `model`, `worker`. + +The Admin UI exposes the same controls under **Persistenz** and historical aggregates under **Nutzung**. + +## Prometheus + +```text +ollama_gateway_usage_raw_files +ollama_gateway_usage_daily_rollup_files +ollama_gateway_usage_monthly_rollup_files +ollama_gateway_usage_raw_bytes +ollama_gateway_usage_daily_rollup_bytes +ollama_gateway_usage_monthly_rollup_bytes +ollama_gateway_usage_last_compaction_timestamp_seconds +ollama_gateway_usage_last_reclaimed_bytes +``` + +No tenant/user labels are added to `/metrics`; dimensional long-term analysis belongs to the rollup API/UI, avoiding unbounded Prometheus cardinality. diff --git a/docs/ROADMAP-P0-P3.md b/docs/ROADMAP-P0-P3.md new file mode 100644 index 0000000..96bcaf8 --- /dev/null +++ b/docs/ROADMAP-P0-P3.md @@ -0,0 +1,195 @@ +# Ollama Fair Gateway — P0–P3 implementation roadmap + +This roadmap turns the gateway from a local fair scheduler into a production-oriented LLM control plane while preserving the project's core properties: one Go binary, no mandatory external state service, streaming-first proxying, and no prompt/response content persistence by default. + +## Delivery principles + +1. **Never block the inference hot path on durable storage.** Control-plane writes may be synchronous when correctness requires it; usage/metrics remain buffered/snapshotted. +2. **Never retry after client-visible output starts.** A retry is permitted only before response headers/body are committed. +3. **Hard policies precede adaptive scoring.** Model ACL → alias resolution → capability preflight → placement → maintenance/circuit eligibility → adaptive routing. +4. **Every runtime control must be observable and persistent when operationally meaningful.** Drain/disable state, API keys, policies, placement, aliases and durable batch definitions survive restart; active sockets, transient inference jobs and in-flight batch attempt contexts do not. +5. **Protocol compatibility remains explicit.** Native Ollama, OpenAI-compatible and later Anthropic-compatible errors/streaming semantics are handled independently. + +--- + +## P0 — Production safety and policy foundation + +### P0.1 Circuit breaker + safe pre-stream retry + +**Goal:** stop repeatedly routing to unhealthy/OOM/transport-failing workers. + +- Worker circuit states: `closed`, `open`, `half_open`. +- Configurable consecutive-failure threshold and open duration. +- Transport failures and backend 5xx contribute to the circuit. +- Only transport failures that happen before response commitment are retried. +- Retry excludes workers that already failed the current request. +- Admin UI shows circuit state, last circuit error and manual reset. +- Prometheus counters planned for opens/retries/failures. + +**Acceptance:** a connection-reset worker opens its circuit and a request succeeds on another eligible worker without duplicate client-visible output. + +### P0.2 Virtual models / aliases + +**Goal:** clients use stable names such as `fast`, `coding`, `vision` rather than physical Ollama tags. + +- Ordered fallback list of real models. +- Optional required capability set. +- Alias participates in `/api/tags` and `/v1/models` discovery. +- Gateway rewrites the outbound model while returning diagnostic headers: + - `X-Gateway-Model-Alias` + - `X-Gateway-Resolved-Model` +- Alias target must still satisfy worker placement and inventory. + +**Acceptance:** OpenWebUI can select a virtual model and the backend receives the resolved physical model. + +### P0.3 Model ACLs + +**Goal:** answer “who may use which model?” independently from placement (“where may it run?”). + +- Tenant baseline rules. +- API-key-specific allow/deny rules. +- Exact and trailing `*` patterns; most specific match wins, deny wins ties. +- ACL is enforced for direct model requests and model discovery. +- API-key create UI exposes allow/deny lists. + +**Acceptance:** a key allowed only for `coding` cannot discover or directly call a denied physical model. + +### P0.4 Worker maintenance / drain + +**Goal:** take workers out of rotation without killing existing streams. + +- `active`: accepts new work. +- `draining`: no new work; active jobs finish. +- `disabled`: no new work until explicitly re-enabled. +- State persisted in `worker-state.json`. +- UI buttons: Drain, Disable, Activate; circuit reset beside them. + +**Acceptance:** a draining worker's active job completes while all new jobs route elsewhere. + +--- + +## P1 — Client reach, QoS and observability + +### P1.1 Anthropic `/v1/messages` + +- Native Anthropic-compatible request/stream passthrough to Ollama. +- Token/usage extraction and protocol-correct errors. +- Tool, vision and thinking capability preflight. +- Same ACL/quota/placement/scheduler pipeline as Ollama/OpenAI. + +### P1.2 Service classes / priority + +- Classes: `interactive`, `system`, `background`, `batch`. +- Weighted scheduling without starvation. +- Per-class queue wait and concurrency ceilings. +- API-key/default mapping and optional request header override with scope. + +### P1.3 Auto-tuning and benchmark profiles + +- Measure TTFT, prompt tok/s, output tok/s, throughput and VRAM by model/concurrency. +- Suggest `max_concurrent` / per-model concurrency. +- Optional “apply recommendation” workflow with audit entry. +- Never auto-change production settings unless explicitly enabled. + +### P1.4 OpenTelemetry + +- OTLP traces/metrics, content capture off by default. +- Spans: auth, admission, queue, route, upstream, first-byte/stream. +- Correlate with `X-Request-ID`. + +--- + +## P2 — Capacity automation and operator workflows + +### P2.1 Warm/preload policies — ✅ implemented + +- `hot`, `warm`, `cold` model classes. +- Explicit/preferred workers. +- Idle unload using Ollama `keep_alive: 0`. +- Optional pre-warm at gateway start/worker recovery. +- Memory-pressure-aware eviction suggestions. + +### P2.2 Alerts and webhooks — ✅ implemented + +- Conditions: worker down, circuit open, repeated OOM, queue depth/wait, storage growth, quota near exhaustion. +- Cooldown/deduplication. +- Generic signed webhook first; provider-specific integrations later. + +### P2.3 Optional stateful conversations — ✅ implemented + +- Opt-in conversation store for clients that need `previous_response_id` semantics. +- Separate encryption/retention policy because this stores content. +- Disabled by default to preserve current privacy posture. + +### P2.4 Policy simulator — ✅ implemented early in P0/P1 + +- Simulate identity/model/capabilities/context without executing inference. +- Explain ACL, alias, placement, inventory, circuit, drain, concurrency and final routing score. +- UI provides a step-by-step decision trace. + +--- + +## P3 — Batch and public operations visibility + +### P3.1 Batch jobs — ✅ implemented + +- Durable job definitions and status. +- Separate background scheduling class. +- Pause/resume/cancel. +- Input/output references rather than embedding large payloads in control state. +- Retention and accounting integrated with existing usage rollups. + +### P3.2 Public status dashboard — ✅ implemented + +- Separate unauthenticated, strictly read-only status surface. +- Current queue/load, worker capacity, infrastructure map and anonymized live flow. +- Explicit public-data allow-list; no tenant/actor/application identities or admin/control-plane state. +- Worker/model-name privacy controls and public worker aliases. +- HA/gateway clustering is removed from the active roadmap because observed single-process performance does not justify the distributed coordination cost. The HA-readiness tooling remains available if that operational assumption changes later. + +--- + +## Planned implementation sequence + +1. **Release A — P0 reliability foundation**: circuit breaker, safe retry, drain/disable persistence, UI controls. +2. **Release B — P0 policy surface**: model aliases and model ACLs with full runtime CRUD UI and persistence. +3. **Release C — P1 protocol/QoS**: Anthropic Messages + service classes. +4. **Release D — P1 observability/tuning**: OpenTelemetry + benchmark recommendations. +5. **Release E — P2 automation**: warm model manager, alerts, policy simulator. +6. **Release F — P2 stateful optional layer**: conversations with explicit content-retention controls. +7. **Release G — P3 batch**. +8. **Release H — P3 public status dashboard**; multi-gateway HA is deferred outside the active roadmap. + +## Current implementation status + +- P0.1: **implemented** (circuit breaker, safe pre-stream retry and reliability metrics). +- P0.2: **implemented** (aliases, discovery, capability-qualified resolution and persistent runtime CRUD from the admin API/UI with immediate publication for new requests). +- P0.3: **implemented** (tenant/API-key ACL enforcement, API-key ACL CRUD and persistent runtime Tenant Model Access CRUD from the admin API/UI). +- P0.4: **implemented** (persistent drain/disable + UI controls). +- P1.1: **implemented** (Anthropic Messages routing, metering and protocol-aware handling). +- P1.2: **implemented** (weighted service classes with queue/concurrency controls and scoped override header). +- P1.3: **implemented** (explicit benchmark profiles and apply-recommendation workflow). +- P1.4: **implemented** (OTLP/HTTP tracing with content capture off by default). +- P2.1: **implemented** (warm/preload policies). +- P2.2: **implemented** (alerts + signed webhooks). +- P2.3: **implemented** (optional AES-256-GCM encrypted Responses conversation store with retention, `store:false`, identity scoping and `previous_response_id` expansion). +- P2.4: **implemented** (policy simulator). +- P3.1: **implemented** (durable metadata + separate content spool, batch service-class execution through the normal gateway pipeline, pause/resume/cancel, restart recovery, retention, owner API, admin UI/API and accounting integration). +- P3.2: **implemented** as the public status dashboard in checkpoint 28. Multi-gateway HA is intentionally removed from the active roadmap after observed single-process performance showed no current capacity justification. The existing HA-readiness tools remain as diagnostic evidence tooling, not as a commitment to add distributed coordination. + +### Checkpoint 23 note + +At checkpoint 23, HA remained gated. Checkpoint 23 further reduces single-node production risk by hardening the Docker/reverse-proxy boundary: loopback-only publishing by default, an explicit production Compose file with no example-config fallback, read-only container rootfs/capability dropping/no-new-privileges, and a host preflight that rejects accidental public binding. This does not constitute HA implementation and does not change inference or persistent-state semantics. + +### Checkpoint 24 note + +At checkpoint 24, HA remained gated. Checkpoint 24 does not alter runtime scheduling, routing, persistence or HA semantics; it replaces the dark admin presentation with a professional light, data-dense control-plane theme. The embedded Live Flow and Infrastructure canvases were updated together with CSS so the UI remains visually coherent. No config/state migration is required. + +### Checkpoint 27 note + +At checkpoint 27, HA remained gated. Checkpoint 27 hardens context-window admission and routing before any HA work: the gateway no longer equates the theoretical `/api/show` model maximum with the context actually available on a worker. Effective context now derives from loaded `/api/ps context_length`, Modelfile `num_ctx`, explicit worker defaults and per-worker/model caps. Native `options.num_ctx` is validated/capped, while OpenAI/Responses estimation now covers `max_output_tokens`, `instructions`, `suffix`, vision reserves and an admission margin. Context-suitable workers are selected before inference, and the policy simulator/model UI expose the effective context evidence. + + +### Checkpoint 28 note + +Checkpoint 28 closes the active P3 roadmap with an optional public read-only status dashboard. `/status/` exposes current load, queue, worker capacity, an infrastructure map and anonymized live request state through a dedicated sanitized API. It never reuses the admin snapshot schema. HA remains available only as a future re-evaluation if measured availability or capacity requirements change. diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..4fe17af --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,76 @@ +# Security notes + +## OIDC + +The gateway validates bearer JWTs against OIDC discovery and JWKS. It rejects `alg=none`, restricts accepted signature algorithms, validates issuer, audience, expiry and `nbf`, and refreshes JWKS when a `kid` is unknown. API bearer-token validation is also the source of truth for the optional browser Authorization Code + PKCE login flow. + +## Trusted-IP bypass + +IP bypass is evaluated before bearer authentication. Starting with checkpoint 22, the **TCP peer address** is the authentication source for `auth.ip_bypass` by default, even when `X-Forwarded-For` is accepted for logging and usage attribution. This prevents a trusted-proxy chain from turning a spoofed forwarded address into a credential-free identity. + +Set `auth.ip_bypass_use_forwarded_ip=true` only when a deployment intentionally needs an original client address behind a reverse proxy to satisfy IP bypass. That mode requires a tightly restricted `auth.trusted_proxies` list plus network ACLs that prevent clients from reaching the gateway directly. Prefer narrow `/32` or `/128` bypass/proxy entries. Never configure a broad Internet-wide bypass for an exposed gateway. + +## API keys + +API keys are represented in the authenticator by their SHA-256 digest; plaintext runtime secrets are not retained. Clients may send either `X-API-Key` or `Authorization: Bearer ...`. + +There are two sources: + +- **Configuration keys** come from `auth.api_keys`. Put their secrets in environment variables and reference them with `${NAME}` in JSON. They survive restarts and are read-only in the web UI. +- **Runtime keys** are created by an administrator in the web UI or through `POST /gateway/ui-api/api-keys`. A cryptographically random `ofg_...` secret is returned once, then only the digest and metadata are persisted in the local key store. UI-created keys can be revoked immediately and survive process restarts until revoked. + +Only identities with `gateway:admin` may list, create or revoke API keys. Cookie-authenticated mutations are covered by the same CSRF checks as the rest of the control plane. Key secrets are never included in list responses, the redacted configuration endpoint or admin logs. + +## Ollama management + +With `native.management_requires_admin=true`, model-management operations require `gateway:admin`. Do not expose Ollama port 11434 directly to untrusted clients; expose only the gateway. + +## Web control plane + +Browser OIDC uses Authorization Code + PKCE. The transient verifier/state is HMAC-protected with `ui.session_secret`. After exchange, the browser receives only an opaque session identifier; the OIDC access token remains in the gateway's in-memory session store. + +A process restart invalidates all browser sessions. Cookie-authenticated state-changing requests require a same-origin CSRF token. The embedded UI uses a restrictive Content Security Policy and has no CDN, analytics or third-party JavaScript dependency. + +Policy overrides are persisted locally and take effect immediately for new requests. + +## In-memory security boundary + +Removing external state storage reduces the network attack surface: there is no coordination service, state-store credential or message-bus endpoint to protect. Sensitive runtime material such as OIDC session tokens and quota state exists only in process memory. + +A crash or restart clears browser sessions and active transient inference jobs, but API keys, policy overrides, quota state, usage history, metric counters and durable batch definitions are restored from local durable state. + +## Optional content-bearing conversations + +The default gateway privacy posture remains content-free: prompts and generated responses are not persisted unless `conversations.enabled=true` is explicitly configured. The optional Responses conversation store is isolated from usage retention and uses AES-256-GCM encryption at rest, a dedicated retention window, an entry cap, and a per-context byte cap. + +`previous_response_id` lookup is bound to the authenticated tenant and scheduler actor. A response ID from another identity is reported exactly like a nonexistent ID to avoid enumeration. `store:false` prevents persistence of the new response. The Admin UI exposes store status but not decrypted conversation content. See `docs/CONVERSATIONS.md`. + +The conversation encryption key is a deployment secret. Losing or changing it makes the existing ciphertext unreadable; exposing it together with the state file defeats at-rest confidentiality. Backups should therefore be protected as secrets. + +## Optional content-bearing batch jobs + +Durable batch jobs are also disabled by default. Enabling them does not persist content by itself, but every submitted job stores its request body and later its response body under `storage.batch_jobs_dir`. These spool payloads are **not encrypted by the gateway**; they rely on mode-0600 files, a mode-0750 spool directory and the deployment filesystem/disk security boundary. Use filesystem or full-disk encryption if at-rest content encryption is required. + +Batch metadata stores an authenticated identity/authorization snapshot but never the bearer token or API-key secret. Owner-facing `/gateway/v1/batches` endpoints are tenant+actor scoped; cross-identity IDs are hidden as not found. Global batch inspection/control and output download are restricted to the admin UI/API and therefore require `gateway:admin`. + +An accepted job can execute after the original credential has been revoked because the credential secret is intentionally not retained. Current tenant model policy, worker placement/maintenance/circuit state, quotas and scheduling are still evaluated when each attempt runs. Operationally, revoke the credential **and cancel its accepted durable jobs** when retroactive cancellation is required. + +Batch input/output files and backups can contain prompts, tool arguments, retrieved data and model output. Configure a short `batch_jobs.retention`, protect `storage.data_dir`, and treat backup archives as content-bearing secrets. See `docs/BATCH-JOBS.md`. + +## Live telemetry + +Live telemetry contains tenant/actor identifiers, request IDs, model names, worker names, timings, token counts and credit estimates, but never prompt text or generated model output. UI telemetry endpoints require an authenticated administrator. + +`workers[].telemetry_url` is administrator-controlled and fetched by the gateway. Point it only to a trusted exporter because it is an SSRF-capable configured URL, just like an Ollama worker URL. The optional shipped worker-telemetry agent should be bound to a trusted management interface and restricted with `-allow-cidrs`; it intentionally does not create a separate application-secret store. + +## Model access control + +Model access control is independent from worker placement. Tenant defaults are configured under `model_access`; API keys can carry a narrower or different allow/deny list. The gateway applies the identity ACL before scheduling and filters `/api/tags` and `/v1/models` accordingly. + +Patterns are exact model/alias names, `*`, or one trailing wildcard such as `qwen3:*`. The most specific matching rule wins; deny wins a specificity tie. API-key ACL metadata is persisted together with the key hash and is editable at creation time in **Admin -> Sicherheit -> API Keys**. + +## Reverse-proxy network boundary + +Prefer publishing the gateway only on loopback when the reverse proxy is local to the Docker host. A proxy should be the only network component able to reach the gateway's published port. Avoid trusting whole RFC1918 ranges merely because the deployment uses private addressing; list only concrete proxy/NAT peer addresses in `auth.trusted_proxies`. + +Checkpoint 23's Compose defaults additionally run with a read-only root filesystem, all Linux capabilities dropped and `no-new-privileges`. Persistent writes are constrained to `/data`; `/tmp` is ephemeral. These controls reduce the impact of a process compromise but do not replace API-key/OIDC authentication or host firewalling. diff --git a/docs/WARM-MODELS.md b/docs/WARM-MODELS.md new file mode 100644 index 0000000..6c9521a --- /dev/null +++ b/docs/WARM-MODELS.md @@ -0,0 +1,65 @@ +# Warm Model Management + +The warm-model manager controls Ollama model residency without putting model-management I/O on the inference hot path. + +## Classes + +- `hot`: keep the model resident on the requested number of eligible workers. A hot model is proactively preloaded. +- `warm`: optionally preload the model and unload it after `idle_timeout`. +- `cold`: do not preload and unload aggressively after `idle_timeout`. + +Rules support exact names and one trailing `*` wildcard. The most specific matching rule wins. + +```json +{ + "warm_models": { + "enabled": true, + "reconcile_interval": "30s", + "operation_timeout": "2m", + "policies": { + "qwen3:8b": { + "class": "hot", + "workers": ["rtx-4090"], + "replicas": 1, + "preload": true, + "idle_timeout": "30m" + }, + "gemma4:*": { + "class": "warm", + "replicas": 1, + "preload": false, + "idle_timeout": "20m" + } + } + } +} +``` + +## Safety rules + +Warm-model operations are subordinate to normal routing policy: + +1. worker must be healthy and in `active` maintenance mode; +2. preload must be allowed by Model Placement; +3. when worker inventory is known, preload requires the model to be installed; +4. workers with an open/half-open circuit are not used for proactive preload; +5. a preload/unload reserves a per-worker/per-model maintenance slot before calling Ollama; +6. inference acquisition for the same worker/model is blocked during that short operation; +7. unload is refused while the model has active inference jobs. + +The maintenance reservation closes the race where an idle unload could otherwise begin at the same moment a new request is routed to the model. + +## Persistence + +Runtime rules edited in **Admin → Warm Models** are stored in `warm-models.json`. The static config remains the baseline and can be restored with **Auf Config zurücksetzen**. + +Action history is retained with the same state file. Last-use timestamps are deliberately runtime state; after restart, currently loaded models receive a fresh grace period before idle eviction. + +## VRAM pressure + +The manager does not automatically evict arbitrary models under memory pressure. It exposes eviction suggestions for inactive, non-hot models. This avoids surprise unloads and lets the operator decide whether to act. + +## Metrics + +- `ollama_gateway_warm_actions_running` +- `ollama_gateway_warm_eviction_suggestions` diff --git a/docs/WORKER-TELEMETRY.md b/docs/WORKER-TELEMETRY.md new file mode 100644 index 0000000..e04d9aa --- /dev/null +++ b/docs/WORKER-TELEMETRY.md @@ -0,0 +1,127 @@ +# Remote worker telemetry + +Ollama Fair Gateway can consume optional out-of-band resource telemetry through `workers[].telemetry_url`. Checkpoint 20 ships a small `worker-telemetry` agent so remote Ollama hosts can report their own RAM/GPU state instead of accidentally reporting the gateway host through `local_system_stats`. + +The agent is **not** part of the inference request path. Gateway workers fetch it during the normal health/telemetry refresh cycle. + +## Why this matters for remote workers + +`local_system_stats: true` runs the memory collector inside the gateway process. It is correct only when the Ollama worker and gateway share the same host. For a worker such as `http://10.2.10.48:11434`, it otherwise reports the gateway machine's RAM as if it belonged to `10.2.10.48`. + +For remote workers, use: + +```json +{ + "name": "M75q - Gen5 - 1048", + "url": "http://10.2.10.48:11434", + "local_system_stats": false, + "telemetry_url": "http://10.2.10.48:11500/telemetry" +} +``` + +and run the agent on that Ollama host. + +## Validate locally first + +Host memory only: + +```sh +./ollama-gateway-worker-telemetry-linux-amd64 -once +``` + +AMD GPU through the Linux amdgpu sysfs interface: + +```sh +./ollama-gateway-worker-telemetry-linux-amd64 -once -amd-sysfs +``` + +If auto-detection picks the wrong GPU, specify the device explicitly: + +```sh +./ollama-gateway-worker-telemetry-linux-amd64 \ + -once \ + -amd-sysfs \ + -amd-device /sys/class/drm/card1/device +``` + +NVIDIA: + +```sh +./ollama-gateway-worker-telemetry-linux-amd64 -once -nvidia-smi +``` + +The JSON schema matches the existing gateway `telemetry_url` contract: + +```json +{ + "memory_used_bytes": 123, + "memory_total_bytes": 456, + "vram_used_bytes": 789, + "vram_total_bytes": 1024, + "gpu_utilization_percent": 42, + "gpu_temperature_c": 63.5, + "gpu_power_watts": 88, + "source": "host-memory+amdgpu-sysfs", + "updated_at": "2026-09-08T18:00:00Z" +} +``` + +A field may be absent when the operating system/driver does not expose it. The agent still returns the remaining usable metrics and puts collector failures in the `error` field. + +## Run as a service + +The listener defaults to loopback for safety. To expose it to the gateway, bind a worker interface and restrict clients by CIDR. Prefer an exact gateway host address when possible: + +```sh +./ollama-gateway-worker-telemetry-linux-amd64 \ + -listen 0.0.0.0:11500 \ + -allow-cidrs 10.2.19.1/32 \ + -amd-sysfs +``` + +Example systemd unit: + +```ini +[Unit] +Description=Ollama Gateway Worker Telemetry +After=network-online.target + +[Service] +ExecStart=/usr/local/bin/ollama-gateway-worker-telemetry-linux-amd64 -listen 0.0.0.0:11500 -allow-cidrs 10.2.19.1/32 -amd-sysfs +Restart=on-failure +User=nobody +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true + +[Install] +WantedBy=multi-user.target +``` + +Adjust the CIDR to the source IP the worker actually sees from the gateway. Do not use `0.0.0.0/0` merely to make the endpoint reachable. + +## AMD telemetry details + +On Linux the agent auto-detects the first `/sys/class/drm/card*/device` whose PCI vendor is `0x1002`. It reads standard amdgpu attributes when available: + +- `mem_info_vram_total` +- `mem_info_vram_used` +- `gpu_busy_percent` +- `hwmon/*/temp1_input` +- `hwmon/*/power1_average` + +No ROCm library, `rocm-smi`, NVML or CGO is required for the AMD path. Integrated/shared-memory GPUs may expose different or incomplete VRAM semantics; keep the statically configured `memory_capacity_bytes`/`vram_capacity_bytes` values as hard capacity hints and treat runtime telemetry as routing pressure information. + +## Security + +The agent intentionally has no application credential store. It relies on a narrow listener/firewall/CIDR allowlist so it remains dependency-free and does not introduce a second secret lifecycle. Resource telemetry can still reveal infrastructure details, so expose it only on a trusted management network, VPN or host firewall. For untrusted network segments, put the endpoint behind a TLS-authenticated reverse proxy rather than exposing it directly. + +`workers[].telemetry_url` remains administrator-controlled and is an SSRF-capable URL. Point it only to trusted worker exporters. + +## Freshness and caching + +Checkpoint 21 validates `updated_at` when the external exporter supplies it. The shipped agent always sends the field and also returns `Cache-Control: no-store`. + +A sample is rejected when it is older than `max(30s, 6 x workers[].health_interval)` or more than 30 seconds in the future. Rejected values are not merged into the routing telemetry; the reason is exposed through the worker telemetry error instead. This prevents a caching proxy or badly skewed exporter clock from silently presenting old resource pressure as current data. + +For compatibility, third-party exporters that omit `updated_at` are still accepted. Their freshness cannot be verified, so new integrations should always provide an RFC3339 timestamp. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..96a34cb --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/example/ollama-fair-gateway + +go 1.23 diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go new file mode 100644 index 0000000..50cef6e --- /dev/null +++ b/internal/alerts/alerts.go @@ -0,0 +1,591 @@ +package alerts + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/state" +) + +type Worker struct { + Name string + Healthy bool + CircuitState string + LastCircuitError string + VRAMUsedBytes int64 + VRAMTotalBytes int64 +} + +type Snapshot struct { + QueueDepth int + QueueWait time.Duration + Workers []Worker + StorageBytes int64 +} + +type Provider func() Snapshot + +type Event struct { + ID string `json:"id"` + Key string `json:"key"` + Type string `json:"type"` + Severity string `json:"severity"` + State string `json:"state"` // firing | resolved + Message string `json:"message"` + Worker string `json:"worker,omitempty"` + Tenant string `json:"tenant,omitempty"` + Actor string `json:"actor,omitempty"` + Value float64 `json:"value,omitempty"` + Threshold float64 `json:"threshold,omitempty"` + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` + ResolvedAt time.Time `json:"resolved_at,omitempty"` + Meta map[string]any `json:"meta,omitempty"` +} + +type Delivery struct { + Time time.Time `json:"time"` + EventID string `json:"event_id"` + Webhook string `json:"webhook"` + Attempt int `json:"attempt,omitempty"` + StatusCode int `json:"status_code,omitempty"` + Error string `json:"error,omitempty"` +} + +type deliveryTask struct { + Event Event + Webhook config.WebhookConfig +} + +type Status struct { + Enabled bool `json:"enabled"` + Active []Event `json:"active"` + History []Event `json:"history"` + Deliveries []Delivery `json:"deliveries"` + Webhooks []map[string]any `json:"webhooks"` + Thresholds config.AlertThresholds `json:"thresholds"` + LastEvaluate time.Time `json:"last_evaluate,omitempty"` + LastError string `json:"last_error,omitempty"` +} + +type persistent struct { + Active map[string]Event `json:"active"` + History []Event `json:"history"` + Deliveries []Delivery `json:"deliveries"` + LastSent map[string]time.Time `json:"last_sent"` +} + +type Manager struct { + mu sync.RWMutex + cfg config.AlertsConfig + file state.AtomicJSON + provider Provider + client *http.Client + active map[string]Event + history []Event + deliveries []Delivery + lastSent map[string]time.Time + downSince map[string]time.Time + lastEvaluate time.Time + lastError string + + deliveryQ chan deliveryTask + persistWake chan struct{} + started atomic.Bool +} + +func New(cfg config.AlertsConfig, path string, provider Provider) (*Manager, error) { + if cfg.WebhookTimeout.Value() <= 0 { + cfg.WebhookTimeout = config.Duration(5 * time.Second) + } + if cfg.WebhookMaxConcurrent <= 0 { + cfg.WebhookMaxConcurrent = 4 + } + if cfg.WebhookQueue <= 0 { + cfg.WebhookQueue = 1024 + } + if cfg.WebhookRetryAttempts <= 0 { + cfg.WebhookRetryAttempts = 3 + } + if cfg.WebhookRetryBackoff.Value() < 0 { + cfg.WebhookRetryBackoff = 0 + } + m := &Manager{cfg: cfg, file: state.AtomicJSON{Path: path, Mode: 0600}, provider: provider, client: &http.Client{Timeout: cfg.WebhookTimeout.Value()}, active: map[string]Event{}, lastSent: map[string]time.Time{}, downSince: map[string]time.Time{}, deliveryQ: make(chan deliveryTask, cfg.WebhookQueue), persistWake: make(chan struct{}, 1)} + var p persistent + if err := m.file.Load(&p); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else if err == nil { + if p.Active != nil { + m.active = p.Active + } + m.history = p.History + m.deliveries = p.Deliveries + if p.LastSent != nil { + m.lastSent = p.LastSent + } + m.trimLocked() + } + return m, nil +} + +func (m *Manager) Start(ctx context.Context) { + if m == nil || !m.cfg.Enabled || !m.started.CompareAndSwap(false, true) { + return + } + for i := 0; i < m.cfg.WebhookMaxConcurrent; i++ { + go m.deliveryWorker(ctx) + } + go m.persistenceWorker(ctx) + go func() { + m.Evaluate() + t := time.NewTicker(m.cfg.EvaluationInterval.Value()) + defer t.Stop() + for { + select { + case <-ctx.Done(): + _ = m.persistNow() + return + case <-t.C: + m.Evaluate() + } + } + }() +} + +func (m *Manager) Evaluate() { + if m == nil || !m.cfg.Enabled || m.provider == nil { + return + } + s := m.provider() + now := time.Now().UTC() + present := map[string]Event{} + th := m.cfg.Thresholds + if th.QueueDepth > 0 && s.QueueDepth >= th.QueueDepth { + e := newEvent("queue_depth", "warning", fmt.Sprintf("queue depth %d exceeds threshold %d", s.QueueDepth, th.QueueDepth)) + e.Value = float64(s.QueueDepth) + e.Threshold = float64(th.QueueDepth) + present[e.Key] = e + } + if th.QueueWait.Value() > 0 && s.QueueWait >= th.QueueWait.Value() { + e := newEvent("queue_wait", "warning", fmt.Sprintf("oldest queued request has waited %s (threshold %s)", s.QueueWait.Round(time.Millisecond), th.QueueWait.Value())) + e.Value = s.QueueWait.Seconds() + e.Threshold = th.QueueWait.Value().Seconds() + present[e.Key] = e + } + if th.StorageBytes > 0 && s.StorageBytes >= th.StorageBytes { + e := newEvent("storage_growth", "warning", fmt.Sprintf("gateway storage %d bytes exceeds threshold %d", s.StorageBytes, th.StorageBytes)) + e.Value = float64(s.StorageBytes) + e.Threshold = float64(th.StorageBytes) + present[e.Key] = e + } + m.mu.Lock() + for _, w := range s.Workers { + if !w.Healthy { + if m.downSince[w.Name].IsZero() { + m.downSince[w.Name] = now + } + if th.WorkerDownFor.Value() <= 0 || now.Sub(m.downSince[w.Name]) >= th.WorkerDownFor.Value() { + e := newEvent("worker_down:"+w.Name, "critical", fmt.Sprintf("worker %s is unhealthy", w.Name)) + e.Type = "worker_down" + e.Worker = w.Name + present[e.Key] = e + } + } else { + delete(m.downSince, w.Name) + } + if th.CircuitOpen && w.CircuitState == "open" { + e := newEvent("circuit_open:"+w.Name, "warning", fmt.Sprintf("worker %s circuit breaker is open", w.Name)) + e.Type = "circuit_open" + e.Worker = w.Name + present[e.Key] = e + } + if th.VRAMPercent > 0 && w.VRAMTotalBytes > 0 { + pct := 100 * float64(w.VRAMUsedBytes) / float64(w.VRAMTotalBytes) + if pct >= th.VRAMPercent { + e := newEvent("vram_pressure:"+w.Name, "warning", fmt.Sprintf("worker %s VRAM usage %.1f%% exceeds %.1f%%", w.Name, pct, th.VRAMPercent)) + e.Type = "vram_pressure" + e.Worker = w.Name + e.Value = pct + e.Threshold = th.VRAMPercent + present[e.Key] = e + } + } + if th.OOM && looksOOM(w.LastCircuitError) { + e := newEvent("oom:"+w.Name, "critical", fmt.Sprintf("worker %s reported an out-of-memory failure", w.Name)) + e.Type = "oom" + e.Worker = w.Name + e.Meta = map[string]any{"last_error": w.LastCircuitError} + present[e.Key] = e + } + } + toSend := m.reconcileLocked(present, now) + m.lastEvaluate = now + m.mu.Unlock() + m.schedulePersist() + for _, e := range toSend { + m.deliver(e) + } +} + +func newEvent(key, severity, msg string) Event { + now := time.Now().UTC() + return Event{ID: eventID(key, now), Key: key, Type: key, Severity: severity, State: "firing", Message: msg, StartedAt: now, UpdatedAt: now} +} + +func (m *Manager) reconcileLocked(present map[string]Event, now time.Time) []Event { + toSend := []Event{} + for k, next := range present { + if cur, ok := m.active[k]; ok { + next.ID = cur.ID + next.StartedAt = cur.StartedAt + next.UpdatedAt = now + m.active[k] = next + if m.cooldownReadyLocked(k, now) { + toSend = append(toSend, next) + m.lastSent[k] = now + } + continue + } + m.active[k] = next + m.history = append(m.history, next) + toSend = append(toSend, next) + m.lastSent[k] = now + } + for k, cur := range m.active { + if _, ok := present[k]; ok { + continue + } + cur.State = "resolved" + cur.UpdatedAt = now + cur.ResolvedAt = now + m.history = append(m.history, cur) + delete(m.active, k) + toSend = append(toSend, cur) + m.lastSent[k] = now + } + m.trimLocked() + return toSend +} +func (m *Manager) cooldownReadyLocked(k string, now time.Time) bool { + d := m.cfg.Cooldown.Value() + return d <= 0 || m.lastSent[k].IsZero() || now.Sub(m.lastSent[k]) >= d +} + +// ObserveQuota feeds the latest token-bucket remainder into alerting without +// putting webhook delivery on the request path. Delivery always happens in a goroutine. +func (m *Manager) ObserveQuota(tenant, actor string, remainingActor, remainingTenant, actorCap, tenantCap float64) { + if m == nil || !m.cfg.Enabled || m.cfg.Thresholds.QuotaRemainingPct <= 0 { + return + } + threshold := m.cfg.Thresholds.QuotaRemainingPct + now := time.Now().UTC() + events := []Event{} + check := func(kind, name string, remaining, cap float64) { + if cap <= 0 { + return + } + pct := 100 * remaining / cap + key := "quota_" + kind + ":" + name + m.mu.Lock() + if pct <= threshold { + e := newEvent(key, "warning", fmt.Sprintf("%s quota %s has %.1f%% credits remaining", kind, name, pct)) + e.Type = "quota_near_exhaustion" + e.Tenant = tenant + if kind == "actor" { + e.Actor = actor + } + e.Value = pct + e.Threshold = threshold + if cur, ok := m.active[key]; ok { + e.ID = cur.ID + e.StartedAt = cur.StartedAt + } + m.active[key] = e + if _, seen := m.lastSent[key]; !seen || m.cooldownReadyLocked(key, now) { + m.history = append(m.history, e) + m.lastSent[key] = now + events = append(events, e) + } + } else if cur, ok := m.active[key]; ok { + cur.State = "resolved" + cur.UpdatedAt = now + cur.ResolvedAt = now + m.history = append(m.history, cur) + delete(m.active, key) + events = append(events, cur) + } + m.trimLocked() + m.mu.Unlock() + m.schedulePersist() + } + check("actor", tenant+"/"+actor, remainingActor, actorCap) + check("tenant", tenant, remainingTenant, tenantCap) + for _, e := range events { + go m.deliver(e) + } +} + +func (m *Manager) Status() Status { + if m == nil { + return Status{} + } + m.mu.RLock() + defer m.mu.RUnlock() + active := make([]Event, 0, len(m.active)) + for _, e := range m.active { + active = append(active, e) + } + sort.Slice(active, func(i, j int) bool { return active[i].StartedAt.After(active[j].StartedAt) }) + hist := append([]Event(nil), m.history...) + sort.Slice(hist, func(i, j int) bool { return hist[i].UpdatedAt.After(hist[j].UpdatedAt) }) + if len(hist) > m.cfg.HistoryLimit { + hist = hist[:m.cfg.HistoryLimit] + } + ds := append([]Delivery(nil), m.deliveries...) + sort.Slice(ds, func(i, j int) bool { return ds[i].Time.After(ds[j].Time) }) + if len(ds) > 100 { + ds = ds[:100] + } + wh := make([]map[string]any, 0, len(m.cfg.Webhooks)) + for _, w := range m.cfg.Webhooks { + wh = append(wh, map[string]any{"name": w.Name, "url": w.URL, "enabled": w.Enabled, "signed": w.Secret != ""}) + } + return Status{Enabled: m.cfg.Enabled, Active: active, History: hist, Deliveries: ds, Webhooks: wh, Thresholds: m.cfg.Thresholds, LastEvaluate: m.lastEvaluate, LastError: m.lastError} +} + +func (m *Manager) TestWebhook(name string) error { + if m == nil { + return errors.New("alerts unavailable") + } + e := newEvent("webhook_test", "info", "Ollama Gateway webhook test") + e.Type = "test" + return m.deliverToNamed(e, name) +} + +func (m *Manager) deliver(e Event) { + for _, w := range m.cfg.Webhooks { + if !w.Enabled { + continue + } + task := deliveryTask{Event: e, Webhook: w} + if !m.started.Load() { + // A manager that has not been Start()ed has no owned worker lifecycle + // to join at shutdown. Deliver synchronously in that uncommon/test-only + // mode so no orphan goroutine can write persistence after its caller + // has torn down the state directory. Production managers use deliveryQ. + _ = m.deliverWithRetry(context.Background(), task) + continue + } + select { + case m.deliveryQ <- task: + default: + m.recordDelivery(Delivery{Time: time.Now().UTC(), EventID: e.ID, Webhook: w.Name, Error: "webhook delivery queue full"}) + } + } +} + +func (m *Manager) deliverToNamed(e Event, name string) error { + for _, w := range m.cfg.Webhooks { + if w.Enabled && (name == "" || w.Name == name) { + return m.deliverWithRetry(context.Background(), deliveryTask{Event: e, Webhook: w}) + } + } + return fmt.Errorf("enabled webhook %q not found", name) +} + +func (m *Manager) deliveryWorker(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case task := <-m.deliveryQ: + _ = m.deliverWithRetry(ctx, task) + } + } +} + +func (m *Manager) deliverWithRetry(ctx context.Context, task deliveryTask) error { + attempts := m.cfg.WebhookRetryAttempts + if attempts <= 0 { + attempts = 1 + } + var last error + for attempt := 1; attempt <= attempts; attempt++ { + retry, err := m.deliverOne(ctx, task.Event, task.Webhook, attempt) + if err == nil { + return nil + } + last = err + if !retry || attempt == attempts { + break + } + d := m.cfg.WebhookRetryBackoff.Value() + if d > 0 { + // Bounded exponential backoff keeps retry storms away from downstreams. + for i := 1; i < attempt && d < 30*time.Second; i++ { + d *= 2 + } + if d > 30*time.Second { + d = 30 * time.Second + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(d): + } + } + } + return last +} + +// deliverOne returns retry=true only for transient failures. Event IDs remain +// stable across attempts so webhook consumers can deduplicate deliveries. +func (m *Manager) deliverOne(ctx context.Context, e Event, w config.WebhookConfig, attempt int) (retry bool, err error) { + payload := map[string]any{"version": "1", "source": "ollama-fair-gateway", "event": e} + b, _ := json.Marshal(payload) + reqCtx := ctx + var cancel context.CancelFunc + if timeout := m.cfg.WebhookTimeout.Value(); timeout > 0 { + reqCtx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, w.URL, bytes.NewReader(b)) + if err != nil { + m.recordDelivery(Delivery{Time: time.Now().UTC(), EventID: e.ID, Webhook: w.Name, Attempt: attempt, Error: err.Error()}) + return false, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "ollama-fair-gateway/alerts") + ts := fmt.Sprint(time.Now().Unix()) + req.Header.Set("X-Ollama-Gateway-Timestamp", ts) + req.Header.Set("X-Ollama-Gateway-Event-ID", e.ID) + req.Header.Set("X-Ollama-Gateway-Delivery-Attempt", fmt.Sprint(attempt)) + if w.Secret != "" { + mac := hmac.New(sha256.New, []byte(w.Secret)) + _, _ = mac.Write([]byte(ts + ".")) + _, _ = mac.Write(b) + req.Header.Set("X-Ollama-Gateway-Signature", "sha256="+hex.EncodeToString(mac.Sum(nil))) + } + resp, err := m.client.Do(req) + d := Delivery{Time: time.Now().UTC(), EventID: e.ID, Webhook: w.Name, Attempt: attempt} + if err != nil { + d.Error = err.Error() + m.recordDelivery(d) + return true, err + } + d.StatusCode = resp.StatusCode + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + _ = resp.Body.Close() + if resp.StatusCode/100 != 2 { + err = fmt.Errorf("webhook HTTP %d", resp.StatusCode) + d.Error = err.Error() + m.recordDelivery(d) + return resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500, err + } + m.recordDelivery(d) + return false, nil +} + +func (m *Manager) recordDelivery(d Delivery) { + m.mu.Lock() + m.deliveries = append(m.deliveries, d) + if d.Error != "" { + m.lastError = d.Error + } + m.trimLocked() + m.mu.Unlock() + m.schedulePersist() +} + +func (m *Manager) schedulePersist() { + if m == nil { + return + } + if !m.started.Load() { + _ = m.persistNow() + return + } + select { + case m.persistWake <- struct{}{}: + default: + } +} + +func (m *Manager) persistenceWorker(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-m.persistWake: + if err := m.persistNow(); err != nil { + m.mu.Lock() + m.lastError = err.Error() + m.mu.Unlock() + } + } + } +} + +func (m *Manager) persistNow() error { + m.mu.RLock() + active := make(map[string]Event, len(m.active)) + for k, v := range m.active { + active[k] = v + } + history := append([]Event(nil), m.history...) + deliveries := append([]Delivery(nil), m.deliveries...) + lastSent := make(map[string]time.Time, len(m.lastSent)) + for k, v := range m.lastSent { + lastSent[k] = v + } + m.mu.RUnlock() + return m.file.Save(persistent{Active: active, History: history, Deliveries: deliveries, LastSent: lastSent}) +} + +func (m *Manager) trimLocked() { + limit := m.cfg.HistoryLimit + if limit <= 0 { + limit = 500 + } + if len(m.history) > limit { + m.history = append([]Event(nil), m.history[len(m.history)-limit:]...) + } + if len(m.deliveries) > 200 { + m.deliveries = append([]Delivery(nil), m.deliveries[len(m.deliveries)-200:]...) + } +} +func looksOOM(s string) bool { + s = strings.ToLower(s) + return strings.Contains(s, "out of memory") || strings.Contains(s, "oom") || strings.Contains(s, "cuda error: out of memory") +} +func eventID(k string, t time.Time) string { + h := sha256.Sum256([]byte(fmt.Sprintf("%s:%d", k, t.UnixNano()))) + return hex.EncodeToString(h[:8]) +} + +func DirSize(root string) int64 { + var total int64 + _ = filepath.Walk(root, func(_ string, info os.FileInfo, err error) error { + if err == nil && info != nil && !info.IsDir() { + total += info.Size() + } + return nil + }) + return total +} diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go new file mode 100644 index 0000000..6d13ad1 --- /dev/null +++ b/internal/alerts/alerts_test.go @@ -0,0 +1,127 @@ +package alerts + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +func TestSignedWebhookAndResolution(t *testing.T) { + var mu sync.Mutex + healthy := false + received := make(chan string, 4) + secret := "test-secret" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + tsv := r.Header.Get("X-Ollama-Gateway-Timestamp") + mac := hmac.New(sha256.New, []byte(secret)) + _, _ = mac.Write([]byte(tsv + ".")) + _, _ = mac.Write(b) + want := "sha256=" + hex.EncodeToString(mac.Sum(nil)) + if r.Header.Get("X-Ollama-Gateway-Signature") != want { + t.Errorf("bad signature") + } + received <- string(b) + w.WriteHeader(204) + })) + defer ts.Close() + cfg := config.AlertsConfig{Enabled: true, EvaluationInterval: config.Duration(time.Second), Cooldown: config.Duration(time.Hour), HistoryLimit: 20, Webhooks: []config.WebhookConfig{{Name: "test", URL: ts.URL, Secret: secret, Enabled: true}}, Thresholds: config.AlertThresholds{WorkerDownFor: config.Duration(time.Nanosecond)}} + m, err := New(cfg, t.TempDir()+"/alerts.json", func() Snapshot { + mu.Lock() + h := healthy + mu.Unlock() + return Snapshot{Workers: []Worker{{Name: "w", Healthy: h}}} + }) + if err != nil { + t.Fatal(err) + } + m.Evaluate() + time.Sleep(time.Millisecond) + m.Evaluate() + select { + case <-received: + case <-time.After(2 * time.Second): + t.Fatal("firing webhook missing") + } + if len(m.Status().Active) != 1 { + t.Fatalf("active=%#v", m.Status().Active) + } + mu.Lock() + healthy = true + mu.Unlock() + m.Evaluate() + select { + case <-received: + case <-time.After(2 * time.Second): + t.Fatal("resolved webhook missing") + } + if len(m.Status().Active) != 0 { + t.Fatal("alert did not resolve") + } +} + +func TestQuotaNearExhaustion(t *testing.T) { + cfg := config.AlertsConfig{Enabled: true, Cooldown: config.Duration(time.Hour), HistoryLimit: 20, Thresholds: config.AlertThresholds{QuotaRemainingPct: 10}} + m, err := New(cfg, t.TempDir()+"/alerts.json", nil) + if err != nil { + t.Fatal(err) + } + m.ObserveQuota("t", "a", 5, 50, 100, 100) + st := m.Status() + if len(st.Active) != 1 || st.Active[0].Type != "quota_near_exhaustion" { + t.Fatalf("active=%#v", st.Active) + } + m.ObserveQuota("t", "a", 50, 50, 100, 100) + if len(m.Status().Active) != 0 { + t.Fatal("quota alert did not resolve") + } +} + +func TestWebhookRetriesTransientFailure(t *testing.T) { + var mu sync.Mutex + attempts := 0 + seen := make(chan int, 4) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + attempts++ + n := attempts + mu.Unlock() + seen <- n + if n < 3 { + http.Error(w, "temporary", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + })) + defer ts.Close() + cfg := config.AlertsConfig{ + Enabled: true, EvaluationInterval: config.Duration(time.Hour), Cooldown: config.Duration(time.Hour), HistoryLimit: 20, + WebhookTimeout: config.Duration(time.Second), WebhookMaxConcurrent: 1, WebhookQueue: 8, WebhookRetryAttempts: 3, WebhookRetryBackoff: config.Duration(time.Millisecond), + Webhooks: []config.WebhookConfig{{Name: "retry", URL: ts.URL, Enabled: true}}, + } + m, err := New(cfg, t.TempDir()+"/alerts.json", nil) + if err != nil { + t.Fatal(err) + } + if err := m.TestWebhook("retry"); err != nil { + t.Fatal(err) + } + mu.Lock() + got := attempts + mu.Unlock() + if got != 3 { + t.Fatalf("attempts=%d want 3", got) + } + st := m.Status() + if len(st.Deliveries) < 3 || st.Deliveries[0].StatusCode != http.StatusNoContent { + t.Fatalf("deliveries=%#v", st.Deliveries) + } +} diff --git a/internal/auth/identity.go b/internal/auth/identity.go new file mode 100644 index 0000000..8d3cca1 --- /dev/null +++ b/internal/auth/identity.go @@ -0,0 +1,492 @@ +package auth + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "net" + "net/http" + "sort" + "strings" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +type Identity struct { + Tenant string `json:"tenant"` + Subject string `json:"subject"` + Application string `json:"application,omitempty"` + AuthType string `json:"auth_type"` + Scopes map[string]bool `json:"-"` + ClientIP string `json:"client_ip"` + ModelACLSet bool `json:"-"` + ModelAccess config.ModelAccessRule `json:"-"` + ServiceClass string `json:"-"` +} + +func (i Identity) Actor() string { + // Interactive OIDC identities are fair-scheduled by subject, so one user + // cannot gain extra shares by using multiple OAuth clients. Static API-key + // and trusted-IP identities are normally applications and use that identity. + if i.AuthType != "oidc" && i.Application != "" { + return "app:" + i.Application + } + if i.Subject != "" { + return i.Subject + } + if i.Application != "" { + return "app:" + i.Application + } + return "anonymous" +} +func (i Identity) HasScope(s string) bool { return i.Scopes[s] || i.Scopes["*"] } +func (i Identity) IsAdmin() bool { return i.HasScope("gateway:admin") } + +type ctxKey struct{} + +func WithIdentity(ctx context.Context, i Identity) context.Context { + return context.WithValue(ctx, ctxKey{}, i) +} +func FromContext(ctx context.Context) (Identity, bool) { + i, ok := ctx.Value(ctxKey{}).(Identity) + return i, ok +} + +// APIKeyInfo is safe to return through the admin API. It never contains the +// key secret. UI-created keys may be backed by a durable RuntimeKeyStore. +type APIKeyInfo struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Tenant string `json:"tenant"` + Subject string `json:"subject"` + Application string `json:"application,omitempty"` + Scopes []string `json:"scopes"` + AllowedModels []string `json:"allowed_models,omitempty"` + DeniedModels []string `json:"denied_models,omitempty"` + ServiceClass string `json:"service_class,omitempty"` + Source string `json:"source"` // config | runtime | persistent + KeyHint string `json:"key_hint,omitempty"` + CreatedAt *time.Time `json:"created_at,omitempty"` + Deletable bool `json:"deletable"` +} + +type APIKeyCreate struct { + Name string + Tenant string + Subject string + Application string + Scopes []string + AllowedModels []string + DeniedModels []string + ServiceClass string +} + +// StoredAPIKey is the durable representation of a UI-created API key. Only +// the SHA-256 hash is persisted; the plaintext secret never leaves CreateAPIKey. +type StoredAPIKey struct { + ID string `json:"id"` + Name string `json:"name"` + Tenant string `json:"tenant"` + Subject string `json:"subject"` + Application string `json:"application,omitempty"` + Scopes []string `json:"scopes"` + AllowedModels []string `json:"allowed_models,omitempty"` + DeniedModels []string `json:"denied_models,omitempty"` + ServiceClass string `json:"service_class,omitempty"` + HashHex string `json:"hash_sha256"` + KeyHint string `json:"key_hint,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +type RuntimeKeyStore interface { + Load() ([]StoredAPIKey, error) + Put(StoredAPIKey) error + Delete(string) error + Health(context.Context) error +} + +type managedKey struct { + identity Identity + info APIKeyInfo +} + +type bypassRule struct { + nets []*net.IPNet + identity Identity +} + +type Authenticator struct { + oidc *OIDCVerifier + mu sync.RWMutex + keys map[[32]byte]managedKey + runtime map[string][32]byte + bypass []bypassRule + trusted []*net.IPNet + bypassForwarded bool + store RuntimeKeyStore +} + +func New(ctx context.Context, cfg config.AuthConfig) (*Authenticator, error) { + return NewWithRuntimeStore(ctx, cfg, nil) +} + +func NewWithRuntimeStore(ctx context.Context, cfg config.AuthConfig, store RuntimeKeyStore) (*Authenticator, error) { + a := &Authenticator{keys: make(map[[32]byte]managedKey), runtime: make(map[string][32]byte), bypassForwarded: cfg.IPBypassUseForwardedIP, store: store} + for _, c := range cfg.TrustedProxies { + _, n, _ := net.ParseCIDR(c) + a.trusted = append(a.trusted, n) + } + for _, b := range cfg.IPBypass { + r := bypassRule{identity: Identity{Tenant: b.Tenant, Subject: b.Subject, Application: b.Application, AuthType: "ip-bypass", Scopes: scopeMap(b.Scopes)}} + if r.identity.Subject == "" { + r.identity.Subject = "ip-bypass" + } + for _, c := range b.CIDRs { + _, n, _ := net.ParseCIDR(c) + r.nets = append(r.nets, n) + } + a.bypass = append(a.bypass, r) + } + for _, k := range cfg.APIKeys { + if k.Key == "" { + return nil, fmt.Errorf("api key %q is empty (is its environment variable set?)", k.Name) + } + id := Identity{Tenant: k.Tenant, Subject: k.Subject, Application: k.Application, AuthType: "api-key", Scopes: scopeMap(k.Scopes), ModelACLSet: len(k.AllowedModels) > 0 || len(k.DeniedModels) > 0, ModelAccess: config.ModelAccessRule{Mode: "allow_all", AllowedModels: append([]string(nil), k.AllowedModels...), DeniedModels: append([]string(nil), k.DeniedModels...)}, ServiceClass: strings.TrimSpace(k.ServiceClass)} + if id.Tenant == "" { + return nil, fmt.Errorf("api key %q has no tenant", k.Name) + } + if id.Subject == "" { + id.Subject = "apikey:" + k.Name + } + h := sha256.Sum256([]byte(k.Key)) + a.keys[h] = managedKey{identity: id, info: APIKeyInfo{Name: k.Name, Tenant: id.Tenant, Subject: id.Subject, Application: id.Application, Scopes: sortedScopes(k.Scopes), AllowedModels: append([]string(nil), k.AllowedModels...), DeniedModels: append([]string(nil), k.DeniedModels...), ServiceClass: strings.TrimSpace(k.ServiceClass), Source: "config", Deletable: false}} + } + if store != nil { + records, err := store.Load() + if err != nil { + return nil, fmt.Errorf("load persistent API keys: %w", err) + } + for _, rec := range records { + hb, err := hex.DecodeString(rec.HashHex) + if err != nil || len(hb) != sha256.Size { + return nil, fmt.Errorf("persistent API key %q has invalid hash", rec.ID) + } + var h [32]byte + copy(h[:], hb) + if rec.ID == "" || rec.Tenant == "" { + return nil, fmt.Errorf("persistent API key has missing id or tenant") + } + created := rec.CreatedAt + info := APIKeyInfo{ID: rec.ID, Name: rec.Name, Tenant: rec.Tenant, Subject: rec.Subject, Application: rec.Application, Scopes: sortedScopes(rec.Scopes), AllowedModels: append([]string(nil), rec.AllowedModels...), DeniedModels: append([]string(nil), rec.DeniedModels...), ServiceClass: rec.ServiceClass, Source: "persistent", KeyHint: rec.KeyHint, CreatedAt: &created, Deletable: true} + id := Identity{Tenant: rec.Tenant, Subject: rec.Subject, Application: rec.Application, AuthType: "api-key", Scopes: scopeMap(rec.Scopes), ModelACLSet: len(rec.AllowedModels) > 0 || len(rec.DeniedModels) > 0, ModelAccess: config.ModelAccessRule{Mode: "allow_all", AllowedModels: append([]string(nil), rec.AllowedModels...), DeniedModels: append([]string(nil), rec.DeniedModels...)}, ServiceClass: rec.ServiceClass} + if id.Subject == "" { + id.Subject = "apikey:" + rec.Name + info.Subject = id.Subject + } + if _, exists := a.keys[h]; exists { + return nil, fmt.Errorf("persistent API key hash collision for %q", rec.ID) + } + if _, exists := a.runtime[rec.ID]; exists { + return nil, fmt.Errorf("duplicate persistent API key id %q", rec.ID) + } + a.keys[h] = managedKey{identity: id, info: info} + a.runtime[rec.ID] = h + } + } + + if cfg.OIDC.Enabled { + v, err := NewOIDCVerifier(ctx, cfg.OIDC) + if err != nil { + return nil, err + } + a.oidc = v + } + return a, nil +} + +func scopeMap(in []string) map[string]bool { + m := map[string]bool{} + for _, s := range in { + s = strings.TrimSpace(s) + if s != "" { + m[s] = true + } + } + return m +} + +func sortedScopes(in []string) []string { + m := map[string]struct{}{} + for _, s := range in { + if s = strings.TrimSpace(s); s != "" { + m[s] = struct{}{} + } + } + out := make([]string, 0, len(m)) + for s := range m { + out = append(out, s) + } + sort.Strings(out) + return out +} + +func (a *Authenticator) Authenticate(r *http.Request) (Identity, error) { + ip := a.ClientIP(r) + bypassIP := a.PeerIP(r) + if a.bypassForwarded { + bypassIP = ip + } + for _, rule := range a.bypass { + if containsAny(rule.nets, net.ParseIP(bypassIP)) { + id := rule.identity + // Keep the resolved client IP for observability even though the + // authentication decision defaults to the TCP peer address. + id.ClientIP = ip + return id, nil + } + } + token := "" + if x := strings.TrimSpace(r.Header.Get("X-API-Key")); x != "" { + token = x + } + if token == "" { + h := r.Header.Get("Authorization") + if len(h) > 7 && strings.EqualFold(h[:7], "Bearer ") { + token = strings.TrimSpace(h[7:]) + } + } + if token != "" { + sum := sha256.Sum256([]byte(token)) + a.mu.RLock() + k, ok := a.keys[sum] + a.mu.RUnlock() + if ok { + id := k.identity + id.ClientIP = ip + return id, nil + } + if a.oidc != nil { + id, err := a.oidc.Verify(r.Context(), token) + if err == nil { + id.ClientIP = ip + return id, nil + } + return Identity{}, fmt.Errorf("invalid bearer token: %w", err) + } + } + return Identity{}, ErrUnauthorized +} + +var ErrUnauthorized = fmt.Errorf("authentication required") + +// CreateAPIKey creates an API key. With a RuntimeKeyStore configured, only the +// key hash and metadata are persisted. The returned secret is the +// only copy of the plaintext key and must be shown to the administrator once. +func (a *Authenticator) CreateAPIKey(in APIKeyCreate) (APIKeyInfo, string, error) { + if a == nil { + return APIKeyInfo{}, "", errors.New("authenticator unavailable") + } + in.Name = strings.TrimSpace(in.Name) + in.Tenant = strings.TrimSpace(in.Tenant) + in.Subject = strings.TrimSpace(in.Subject) + in.Application = strings.TrimSpace(in.Application) + if in.Name == "" || len(in.Name) > 128 { + return APIKeyInfo{}, "", errors.New("name is required and must be at most 128 characters") + } + if in.Tenant == "" || len(in.Tenant) > 256 { + return APIKeyInfo{}, "", errors.New("tenant is required and must be at most 256 characters") + } + if len(in.Subject) > 256 || len(in.Application) > 256 { + return APIKeyInfo{}, "", errors.New("subject and application must be at most 256 characters") + } + if in.Subject == "" { + in.Subject = "apikey:" + in.Name + } + scopes := sortedScopes(in.Scopes) + for _, s := range scopes { + if len(s) > 128 { + return APIKeyInfo{}, "", errors.New("scope must be at most 128 characters") + } + } + if err := config.ValidateModelAccessRule(config.ModelAccessRule{Mode: "allow_all", AllowedModels: in.AllowedModels, DeniedModels: in.DeniedModels}); err != nil { + return APIKeyInfo{}, "", fmt.Errorf("model ACL: %w", err) + } + + secretBytes := make([]byte, 32) + if _, err := rand.Read(secretBytes); err != nil { + return APIKeyInfo{}, "", fmt.Errorf("generate key: %w", err) + } + secret := "ofg_" + base64.RawURLEncoding.EncodeToString(secretBytes) + idBytes := make([]byte, 12) + if _, err := rand.Read(idBytes); err != nil { + return APIKeyInfo{}, "", fmt.Errorf("generate key id: %w", err) + } + id := base64.RawURLEncoding.EncodeToString(idBytes) + h := sha256.Sum256([]byte(secret)) + createdAt := time.Now().UTC() + info := APIKeyInfo{ID: id, Name: in.Name, Tenant: in.Tenant, Subject: in.Subject, Application: in.Application, Scopes: scopes, AllowedModels: append([]string(nil), in.AllowedModels...), DeniedModels: append([]string(nil), in.DeniedModels...), ServiceClass: strings.TrimSpace(in.ServiceClass), Source: map[bool]string{true: "persistent", false: "runtime"}[a.store != nil], KeyHint: keyHint(secret), CreatedAt: &createdAt, Deletable: true} + identity := Identity{Tenant: in.Tenant, Subject: in.Subject, Application: in.Application, AuthType: "api-key", Scopes: scopeMap(scopes), ModelACLSet: len(in.AllowedModels) > 0 || len(in.DeniedModels) > 0, ModelAccess: config.ModelAccessRule{Mode: "allow_all", AllowedModels: append([]string(nil), in.AllowedModels...), DeniedModels: append([]string(nil), in.DeniedModels...)}, ServiceClass: strings.TrimSpace(in.ServiceClass)} + + a.mu.Lock() + defer a.mu.Unlock() + for _, existing := range a.keys { + if existing.info.Name == in.Name && existing.info.Tenant == in.Tenant { + return APIKeyInfo{}, "", fmt.Errorf("an API key named %q already exists for tenant %q", in.Name, in.Tenant) + } + } + if _, exists := a.keys[h]; exists { + return APIKeyInfo{}, "", errors.New("generated API key collision") + } + if _, exists := a.runtime[id]; exists { + return APIKeyInfo{}, "", errors.New("generated API key id collision") + } + if a.store != nil { + rec := StoredAPIKey{ID: id, Name: in.Name, Tenant: in.Tenant, Subject: in.Subject, Application: in.Application, Scopes: scopes, AllowedModels: append([]string(nil), in.AllowedModels...), DeniedModels: append([]string(nil), in.DeniedModels...), ServiceClass: strings.TrimSpace(in.ServiceClass), HashHex: hex.EncodeToString(h[:]), KeyHint: info.KeyHint, CreatedAt: createdAt} + if err := a.store.Put(rec); err != nil { + return APIKeyInfo{}, "", fmt.Errorf("persist API key: %w", err) + } + } + a.keys[h] = managedKey{identity: identity, info: info} + a.runtime[id] = h + return info, secret, nil +} + +func keyHint(secret string) string { + if len(secret) <= 12 { + return secret + } + return secret[:8] + "…" + secret[len(secret)-4:] +} + +func (a *Authenticator) APIKeys() []APIKeyInfo { + if a == nil { + return nil + } + a.mu.RLock() + out := make([]APIKeyInfo, 0, len(a.keys)) + for _, k := range a.keys { + i := k.info + i.Scopes = append([]string(nil), i.Scopes...) + i.AllowedModels = append([]string(nil), i.AllowedModels...) + i.DeniedModels = append([]string(nil), i.DeniedModels...) + out = append(out, i) + } + a.mu.RUnlock() + sort.Slice(out, func(i, j int) bool { + if out[i].Source != out[j].Source { + return out[i].Source < out[j].Source + } + if out[i].Tenant != out[j].Tenant { + return out[i].Tenant < out[j].Tenant + } + return out[i].Name < out[j].Name + }) + return out +} + +func (a *Authenticator) DeleteAPIKey(id string) (APIKeyInfo, bool, error) { + if a == nil || id == "" { + return APIKeyInfo{}, false, nil + } + a.mu.Lock() + defer a.mu.Unlock() + h, ok := a.runtime[id] + if !ok { + return APIKeyInfo{}, false, nil + } + k, ok := a.keys[h] + if !ok { + delete(a.runtime, id) + return APIKeyInfo{}, false, nil + } + if a.store != nil { + if err := a.store.Delete(id); err != nil { + return APIKeyInfo{}, true, fmt.Errorf("delete persistent API key: %w", err) + } + } + delete(a.keys, h) + delete(a.runtime, id) + return k.info, true, nil +} + +func (a *Authenticator) HasPersistentRuntimeStore() bool { return a != nil && a.store != nil } + +func (a *Authenticator) RuntimeStoreHealth(ctx context.Context) error { + if a == nil || a.store == nil { + return nil + } + return a.store.Health(ctx) +} + +func (a *Authenticator) PeerIP(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host = r.RemoteAddr + } + return strings.TrimSpace(host) +} + +func (a *Authenticator) ClientIP(r *http.Request) string { + host := a.PeerIP(r) + peer := net.ParseIP(host) + if peer == nil || !containsAny(a.trusted, peer) { + return host + } + parts := strings.Split(r.Header.Get("X-Forwarded-For"), ",") + chain := make([]net.IP, 0, len(parts)+1) + for _, p := range parts { + if ip := net.ParseIP(strings.TrimSpace(p)); ip != nil { + chain = append(chain, ip) + } + } + chain = append(chain, peer) + for i := len(chain) - 1; i >= 0; i-- { + if !containsAny(a.trusted, chain[i]) { + return chain[i].String() + } + } + if len(chain) > 0 { + return chain[0].String() + } + return strings.TrimSpace(host) +} +func containsAny(nets []*net.IPNet, ip net.IP) bool { + if ip == nil { + return false + } + for _, n := range nets { + if n.Contains(ip) { + return true + } + } + return false +} + +func (a *Authenticator) OIDCEnabled() bool { return a != nil && a.oidc != nil } + +func (a *Authenticator) OIDCBrowserEndpoints() (BrowserEndpoints, bool) { + if a == nil || a.oidc == nil { + return BrowserEndpoints{}, false + } + return a.oidc.BrowserEndpoints(), true +} + +func (a *Authenticator) ExchangeOIDCCode(ctx context.Context, code, redirectURI, clientID, clientSecret, verifier string) (TokenExchange, error) { + if a == nil || a.oidc == nil { + return TokenExchange{}, errors.New("OIDC is disabled") + } + return a.oidc.ExchangeCode(ctx, code, redirectURI, clientID, clientSecret, verifier) +} + +func (a *Authenticator) VerifyOIDCToken(ctx context.Context, token string) (Identity, error) { + if a == nil || a.oidc == nil { + return Identity{}, errors.New("OIDC is disabled") + } + return a.oidc.Verify(ctx, token) +} diff --git a/internal/auth/identity_test.go b/internal/auth/identity_test.go new file mode 100644 index 0000000..c7dce7d --- /dev/null +++ b/internal/auth/identity_test.go @@ -0,0 +1,103 @@ +package auth + +import ( + "context" + "net/http/httptest" + "strings" + "testing" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +func TestRuntimeAPIKeyCreateAuthenticateDelete(t *testing.T) { + a, err := New(context.Background(), config.AuthConfig{APIKeys: []config.APIKeyConfig{{Name: "static", Key: "static-secret", Tenant: "ops", Scopes: []string{"gateway:admin"}}}}) + if err != nil { + t.Fatal(err) + } + info, secret, err := a.CreateAPIKey(APIKeyCreate{Name: "openwebui", Tenant: "interactive", Application: "openwebui", Scopes: []string{"models:read", "models:read"}}) + if err != nil { + t.Fatal(err) + } + if info.ID == "" || secret == "" || !strings.HasPrefix(secret, "ofg_") || !info.Deletable || info.Source != "runtime" { + t.Fatalf("unexpected created key: %#v secret=%q", info, secret) + } + if len(info.Scopes) != 1 || info.Scopes[0] != "models:read" { + t.Fatalf("unexpected scopes: %#v", info.Scopes) + } + + r := httptest.NewRequest("GET", "http://gateway/api/tags", nil) + r.Header.Set("Authorization", "Bearer "+secret) + id, err := a.Authenticate(r) + if err != nil { + t.Fatal(err) + } + if id.Tenant != "interactive" || id.Application != "openwebui" || id.Actor() != "app:openwebui" { + t.Fatalf("unexpected identity: %#v", id) + } + + keys := a.APIKeys() + if len(keys) != 2 { + t.Fatalf("keys=%d want 2: %#v", len(keys), keys) + } + for _, k := range keys { + if strings.Contains(k.KeyHint, secret) { + t.Fatalf("key listing leaked secret: %#v", k) + } + } + + deleted, ok, err := a.DeleteAPIKey(info.ID) + if err != nil { + t.Fatal(err) + } + if !ok || deleted.Name != "openwebui" { + t.Fatalf("delete failed: ok=%v info=%#v", ok, deleted) + } + if _, err := a.Authenticate(r); err == nil { + t.Fatal("deleted runtime key still authenticates") + } + if _, ok, err := a.DeleteAPIKey(info.ID); err != nil || ok { + t.Fatal("second delete unexpectedly succeeded") + } +} + +func TestRuntimeAPIKeyRejectsDuplicateTenantName(t *testing.T) { + a, err := New(context.Background(), config.AuthConfig{}) + if err != nil { + t.Fatal(err) + } + if _, _, err := a.CreateAPIKey(APIKeyCreate{Name: "client", Tenant: "team"}); err != nil { + t.Fatal(err) + } + if _, _, err := a.CreateAPIKey(APIKeyCreate{Name: "client", Tenant: "team"}); err == nil { + t.Fatal("expected duplicate error") + } + if _, _, err := a.CreateAPIKey(APIKeyCreate{Name: "client", Tenant: "other"}); err != nil { + t.Fatalf("same name in other tenant should be allowed: %v", err) + } +} + +func TestRuntimeAPIKeyCarriesModelACL(t *testing.T) { + a, err := New(context.Background(), config.AuthConfig{}) + if err != nil { + t.Fatal(err) + } + info, secret, err := a.CreateAPIKey(APIKeyCreate{Name: "limited", Tenant: "team", AllowedModels: []string{"coding", "qwen3:*"}, DeniedModels: []string{"qwen3:70b*"}}) + if err != nil { + t.Fatal(err) + } + if len(info.AllowedModels) != 2 || len(info.DeniedModels) != 1 { + t.Fatalf("info=%#v", info) + } + r := httptest.NewRequest("GET", "http://gateway/api/tags", nil) + r.Header.Set("Authorization", "Bearer "+secret) + id, err := a.Authenticate(r) + if err != nil { + t.Fatal(err) + } + if !id.ModelACLSet { + t.Fatal("expected model ACL on identity") + } + if !config.ModelAccessAllowed(id.ModelAccess, "coding") || config.ModelAccessAllowed(id.ModelAccess, "qwen3:70b-q4") { + t.Fatalf("unexpected ACL: %#v", id.ModelAccess) + } +} diff --git a/internal/auth/ip_bypass_test.go b/internal/auth/ip_bypass_test.go new file mode 100644 index 0000000..9383631 --- /dev/null +++ b/internal/auth/ip_bypass_test.go @@ -0,0 +1,71 @@ +package auth + +import ( + "context" + "net/http/httptest" + "testing" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +func bypassTestConfig(useForwarded bool) config.AuthConfig { + return config.AuthConfig{ + IPBypassUseForwardedIP: useForwarded, + TrustedProxies: []string{"10.0.0.0/8"}, + IPBypass: []config.IPBypassConfig{{ + CIDRs: []string{"127.0.0.1/32"}, + Tenant: "local", + Subject: "localhost", + Scopes: []string{"gateway:admin"}, + }}, + } +} + +func TestIPBypassDefaultsToDirectPeerNotForwardedHeader(t *testing.T) { + a, err := New(context.Background(), bypassTestConfig(false)) + if err != nil { + t.Fatal(err) + } + r := httptest.NewRequest("GET", "http://gateway/admin", nil) + r.RemoteAddr = "10.2.3.4:12345" + r.Header.Set("X-Forwarded-For", "127.0.0.1") + if got := a.ClientIP(r); got != "127.0.0.1" { + t.Fatalf("resolved client ip changed: got %q", got) + } + if _, err := a.Authenticate(r); err == nil { + t.Fatal("spoofed forwarded loopback unexpectedly satisfied ip_bypass") + } +} + +func TestIPBypassForwardedCompatibilityMustBeExplicit(t *testing.T) { + a, err := New(context.Background(), bypassTestConfig(true)) + if err != nil { + t.Fatal(err) + } + r := httptest.NewRequest("GET", "http://gateway/admin", nil) + r.RemoteAddr = "10.2.3.4:12345" + r.Header.Set("X-Forwarded-For", "127.0.0.1") + id, err := a.Authenticate(r) + if err != nil { + t.Fatal(err) + } + if !id.IsAdmin() || id.AuthType != "ip-bypass" || id.ClientIP != "127.0.0.1" { + t.Fatalf("unexpected identity: %#v", id) + } +} + +func TestIPBypassStillAcceptsDirectPeer(t *testing.T) { + a, err := New(context.Background(), bypassTestConfig(false)) + if err != nil { + t.Fatal(err) + } + r := httptest.NewRequest("GET", "http://gateway/admin", nil) + r.RemoteAddr = "127.0.0.1:12345" + id, err := a.Authenticate(r) + if err != nil { + t.Fatal(err) + } + if !id.IsAdmin() || id.ClientIP != "127.0.0.1" { + t.Fatalf("unexpected identity: %#v", id) + } +} diff --git a/internal/auth/oidc.go b/internal/auth/oidc.go new file mode 100644 index 0000000..5ea30d0 --- /dev/null +++ b/internal/auth/oidc.go @@ -0,0 +1,499 @@ +package auth + +import ( + "context" + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rsa" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +type discoveryDoc struct { + Issuer string `json:"issuer"` + JWKSURI string `json:"jwks_uri"` + AuthorizationEndpoint string `json:"authorization_endpoint"` + TokenEndpoint string `json:"token_endpoint"` +} +type jwksDoc struct { + Keys []json.RawMessage `json:"keys"` +} +type jwkHeader struct { + Kty string `json:"kty"` + Kid string `json:"kid"` + Alg string `json:"alg"` + Use string `json:"use"` + N string `json:"n"` + E string `json:"e"` + Crv string `json:"crv"` + X string `json:"x"` + Y string `json:"y"` +} +type jwtHeader struct { + Alg string `json:"alg"` + Kid string `json:"kid"` + Typ string `json:"typ"` +} + +type OIDCVerifier struct { + cfg config.OIDCConfig + issuer string + jwksURI string + authorizationEndpoint string + tokenEndpoint string + client *http.Client + mu sync.RWMutex + keys map[string]crypto.PublicKey + lastRefresh time.Time + allowed map[string]bool + adminGroups map[string]bool +} + +func NewOIDCVerifier(ctx context.Context, cfg config.OIDCConfig) (*OIDCVerifier, error) { + issuer := strings.TrimSuffix(cfg.Issuer, "/") + v := &OIDCVerifier{cfg: cfg, issuer: issuer, client: &http.Client{Timeout: 10 * time.Second}, keys: map[string]crypto.PublicKey{}, allowed: map[string]bool{}, adminGroups: map[string]bool{}} + for _, a := range cfg.AllowedAlgorithms { + v.allowed[a] = true + } + for _, g := range cfg.AdminGroups { + v.adminGroups[g] = true + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, issuer+"/.well-known/openid-configuration", nil) + if err != nil { + return nil, err + } + resp, err := v.client.Do(req) + if err != nil { + return nil, fmt.Errorf("oidc discovery: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("oidc discovery: HTTP %d", resp.StatusCode) + } + var d discoveryDoc + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&d); err != nil { + return nil, err + } + if strings.TrimSuffix(d.Issuer, "/") != issuer { + return nil, fmt.Errorf("oidc discovery issuer mismatch: got %q", d.Issuer) + } + if d.JWKSURI == "" { + return nil, errors.New("oidc discovery returned no jwks_uri") + } + v.jwksURI = d.JWKSURI + v.authorizationEndpoint = d.AuthorizationEndpoint + v.tokenEndpoint = d.TokenEndpoint + if err := v.refresh(ctx, true); err != nil { + return nil, err + } + return v, nil +} + +func (v *OIDCVerifier) Verify(ctx context.Context, token string) (Identity, error) { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return Identity{}, errors.New("token is not a JWT") + } + hb, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil { + return Identity{}, err + } + var h jwtHeader + if err := json.Unmarshal(hb, &h); err != nil { + return Identity{}, err + } + if h.Alg == "none" || !v.allowed[h.Alg] { + return Identity{}, fmt.Errorf("JWT algorithm %q is not allowed", h.Alg) + } + key := v.key(h.Kid) + if key == nil { + if err := v.refresh(ctx, false); err != nil { + return Identity{}, err + } + key = v.key(h.Kid) + } + if key == nil { + return Identity{}, fmt.Errorf("no signing key for kid %q", h.Kid) + } + sig, err := base64.RawURLEncoding.DecodeString(parts[2]) + if err != nil { + return Identity{}, err + } + signingInput := []byte(parts[0] + "." + parts[1]) + if err := verifySignature(h.Alg, key, signingInput, sig); err != nil { + // Some providers rotate key material while reusing a kid. Refresh once + // (subject to the anti-thundering-herd interval) before rejecting. + _ = v.refresh(ctx, false) + key = v.key(h.Kid) + if key == nil { + return Identity{}, err + } + if err2 := verifySignature(h.Alg, key, signingInput, sig); err2 != nil { + return Identity{}, err2 + } + } + pb, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return Identity{}, err + } + dec := json.NewDecoder(strings.NewReader(string(pb))) + dec.UseNumber() + claims := map[string]any{} + if err := dec.Decode(&claims); err != nil { + return Identity{}, err + } + now := time.Now() + skew := v.cfg.ClockSkew.Value() + if iss, _ := claims["iss"].(string); strings.TrimSuffix(iss, "/") != v.issuer { + return Identity{}, errors.New("issuer mismatch") + } + if !audContains(claims["aud"], v.cfg.Audience) { + return Identity{}, errors.New("audience mismatch") + } + if exp, ok := numericTime(claims["exp"]); !ok || now.After(exp.Add(skew)) { + return Identity{}, errors.New("token expired or missing exp") + } + if nbf, ok := numericTime(claims["nbf"]); ok && now.Add(skew).Before(nbf) { + return Identity{}, errors.New("token not valid yet") + } + sub, _ := claims["sub"].(string) + if sub == "" { + return Identity{}, errors.New("token has no sub") + } + tenant := claimString(claims, v.cfg.TenantClaim) + if tenant == "" { + tenant = sub + } + app := claimString(claims, v.cfg.ApplicationClaim) + scopes := map[string]bool{} + for _, s := range claimStrings(claims["scope"]) { + for _, p := range strings.Fields(s) { + scopes[p] = true + } + } + for _, s := range claimStrings(claims["scp"]) { + for _, p := range strings.Fields(s) { + scopes[p] = true + } + } + groups := claimStrings(claimValue(claims, v.cfg.GroupsClaim)) + for _, g := range groups { + if v.adminGroups[g] { + scopes["gateway:admin"] = true + } + } + return Identity{Tenant: tenant, Subject: sub, Application: app, AuthType: "oidc", Scopes: scopes}, nil +} + +func (v *OIDCVerifier) key(kid string) crypto.PublicKey { + v.mu.RLock() + defer v.mu.RUnlock() + if kid != "" { + return v.keys[kid] + } + if len(v.keys) == 1 { + for _, k := range v.keys { + return k + } + } + return nil +} +func (v *OIDCVerifier) refresh(ctx context.Context, force bool) error { + v.mu.Lock() + defer v.mu.Unlock() + if !force && time.Since(v.lastRefresh) < v.cfg.JWKSRefreshMinInterval.Value() { + return nil + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, v.jwksURI, nil) + if err != nil { + return err + } + resp, err := v.client.Do(req) + if err != nil { + return fmt.Errorf("fetch jwks: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return fmt.Errorf("fetch jwks: HTTP %d", resp.StatusCode) + } + var d jwksDoc + if err := json.NewDecoder(io.LimitReader(resp.Body, 2<<20)).Decode(&d); err != nil { + return err + } + keys := map[string]crypto.PublicKey{} + for _, raw := range d.Keys { + var j jwkHeader + if err := json.Unmarshal(raw, &j); err != nil { + continue + } + k, err := parseJWK(j) + if err != nil { + continue + } + keys[j.Kid] = k + } + if len(keys) == 0 { + return errors.New("jwks contained no supported signing keys") + } + v.keys = keys + v.lastRefresh = time.Now() + return nil +} +func parseJWK(j jwkHeader) (crypto.PublicKey, error) { + dec := base64.RawURLEncoding.DecodeString + switch j.Kty { + case "RSA": + nB, err := dec(j.N) + if err != nil { + return nil, err + } + eB, err := dec(j.E) + if err != nil { + return nil, err + } + e := 0 + for _, b := range eB { + e = e<<8 + int(b) + } + if e == 0 { + return nil, errors.New("bad RSA exponent") + } + return &rsa.PublicKey{N: new(big.Int).SetBytes(nB), E: e}, nil + case "EC": + xb, err := dec(j.X) + if err != nil { + return nil, err + } + yb, err := dec(j.Y) + if err != nil { + return nil, err + } + var c elliptic.Curve + switch j.Crv { + case "P-256": + c = elliptic.P256() + case "P-384": + c = elliptic.P384() + case "P-521": + c = elliptic.P521() + default: + return nil, errors.New("unsupported EC curve") + } + x, y := new(big.Int).SetBytes(xb), new(big.Int).SetBytes(yb) + if !c.IsOnCurve(x, y) { + return nil, errors.New("EC key is not on curve") + } + return &ecdsa.PublicKey{Curve: c, X: x, Y: y}, nil + case "OKP": + if j.Crv != "Ed25519" { + return nil, errors.New("unsupported OKP curve") + } + b, err := dec(j.X) + if err != nil { + return nil, err + } + if len(b) != ed25519.PublicKeySize { + return nil, errors.New("bad Ed25519 key") + } + return ed25519.PublicKey(b), nil + default: + return nil, errors.New("unsupported key type") + } +} +func verifySignature(alg string, key crypto.PublicKey, msg, sig []byte) error { + var hash crypto.Hash + var digest []byte + switch alg { + case "RS256", "PS256", "ES256": + h := sha256.Sum256(msg) + hash = crypto.SHA256 + digest = h[:] + case "RS384", "PS384", "ES384": + h := sha512.Sum384(msg) + hash = crypto.SHA384 + digest = h[:] + case "RS512", "PS512", "ES512": + h := sha512.Sum512(msg) + hash = crypto.SHA512 + digest = h[:] + case "EdDSA": + pk, ok := key.(ed25519.PublicKey) + if !ok || !ed25519.Verify(pk, msg, sig) { + return errors.New("invalid JWT signature") + } + return nil + default: + return errors.New("unsupported JWT algorithm") + } + if strings.HasPrefix(alg, "RS") { + pk, ok := key.(*rsa.PublicKey) + if !ok { + return errors.New("JWT key type mismatch") + } + if err := rsa.VerifyPKCS1v15(pk, hash, digest, sig); err != nil { + return errors.New("invalid JWT signature") + } + return nil + } + if strings.HasPrefix(alg, "PS") { + pk, ok := key.(*rsa.PublicKey) + if !ok { + return errors.New("JWT key type mismatch") + } + if err := rsa.VerifyPSS(pk, hash, digest, sig, &rsa.PSSOptions{SaltLength: rsa.PSSSaltLengthEqualsHash, Hash: hash}); err != nil { + return errors.New("invalid JWT signature") + } + return nil + } + pk, ok := key.(*ecdsa.PublicKey) + if !ok { + return errors.New("JWT key type mismatch") + } + n := (pk.Curve.Params().BitSize + 7) / 8 + if len(sig) != 2*n { + return errors.New("bad ECDSA JWT signature length") + } + r, s := new(big.Int).SetBytes(sig[:n]), new(big.Int).SetBytes(sig[n:]) + if !ecdsa.Verify(pk, digest, r, s) { + return errors.New("invalid JWT signature") + } + return nil +} +func audContains(v any, want string) bool { + for _, s := range claimStrings(v) { + if s == want { + return true + } + } + return false +} +func claimStrings(v any) []string { + switch x := v.(type) { + case string: + return []string{x} + case []any: + out := []string{} + for _, z := range x { + if s, ok := z.(string); ok { + out = append(out, s) + } + } + return out + case []string: + return x + } + return nil +} +func numericTime(v any) (time.Time, bool) { + var n int64 + switch x := v.(type) { + case json.Number: + i, e := x.Int64() + if e != nil { + return time.Time{}, false + } + n = i + case float64: + n = int64(x) + case int64: + n = x + default: + return time.Time{}, false + } + return time.Unix(n, 0), true +} +func claimValue(m map[string]any, path string) any { + if path == "" { + return nil + } + var cur any = m + for _, p := range strings.Split(path, ".") { + mm, ok := cur.(map[string]any) + if !ok { + return nil + } + cur = mm[p] + } + return cur +} +func claimString(m map[string]any, path string) string { + if s, ok := claimValue(m, path).(string); ok { + return s + } + return "" +} + +type BrowserEndpoints struct { + Authorization string `json:"authorization_endpoint"` + Token string `json:"token_endpoint"` +} + +type TokenExchange struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + ExpiresIn int64 `json:"expires_in"` + IDToken string `json:"id_token,omitempty"` + Scope string `json:"scope,omitempty"` +} + +func (v *OIDCVerifier) BrowserEndpoints() BrowserEndpoints { + return BrowserEndpoints{Authorization: v.authorizationEndpoint, Token: v.tokenEndpoint} +} + +func (v *OIDCVerifier) ExchangeCode(ctx context.Context, code, redirectURI, clientID, clientSecret, verifier string) (TokenExchange, error) { + if v.tokenEndpoint == "" { + return TokenExchange{}, errors.New("OIDC discovery returned no token_endpoint") + } + form := url.Values{} + form.Set("grant_type", "authorization_code") + form.Set("code", code) + form.Set("redirect_uri", redirectURI) + form.Set("client_id", clientID) + if verifier != "" { + form.Set("code_verifier", verifier) + } + if clientSecret != "" { + form.Set("client_secret", clientSecret) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, v.tokenEndpoint, strings.NewReader(form.Encode())) + if err != nil { + return TokenExchange{}, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Accept", "application/json") + resp, err := v.client.Do(req) + if err != nil { + return TokenExchange{}, fmt.Errorf("OIDC token exchange: %w", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + return TokenExchange{}, err + } + if resp.StatusCode/100 != 2 { + return TokenExchange{}, fmt.Errorf("OIDC token exchange: HTTP %d", resp.StatusCode) + } + var out TokenExchange + if err := json.Unmarshal(body, &out); err != nil { + return TokenExchange{}, err + } + if out.AccessToken == "" { + return TokenExchange{}, errors.New("OIDC token response contains no access_token") + } + return out, nil +} diff --git a/internal/auth/oidc_test.go b/internal/auth/oidc_test.go new file mode 100644 index 0000000..f7e10c7 --- /dev/null +++ b/internal/auth/oidc_test.go @@ -0,0 +1,63 @@ +package auth + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +func TestOIDCVerifierRS256(t *testing.T) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + var issuer string + mux := http.NewServeMux() + srv := httptest.NewServer(mux) + defer srv.Close() + issuer = srv.URL + mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"issuer": issuer, "jwks_uri": issuer + "/jwks"}) + }) + mux.HandleFunc("/jwks", func(w http.ResponseWriter, r *http.Request) { + e := big.NewInt(int64(key.PublicKey.E)).Bytes() + _ = json.NewEncoder(w).Encode(map[string]any{"keys": []any{map[string]any{"kty": "RSA", "kid": "k1", "alg": "RS256", "n": base64.RawURLEncoding.EncodeToString(key.PublicKey.N.Bytes()), "e": base64.RawURLEncoding.EncodeToString(e)}}}) + }) + v, err := NewOIDCVerifier(context.Background(), config.OIDCConfig{Enabled: true, Issuer: issuer, Audience: "gateway", TenantClaim: "tenant", ApplicationClaim: "azp", GroupsClaim: "groups", AdminGroups: []string{"admins"}, ClockSkew: config.Duration(time.Second), JWKSRefreshMinInterval: config.Duration(time.Second), AllowedAlgorithms: []string{"RS256"}}) + if err != nil { + t.Fatal(err) + } + now := time.Now() + tok := signRS256(t, key, "k1", map[string]any{"iss": issuer, "aud": "gateway", "sub": "alice", "tenant": "team-a", "azp": "web", "groups": []string{"admins"}, "exp": now.Add(time.Minute).Unix(), "nbf": now.Add(-time.Second).Unix()}) + id, err := v.Verify(context.Background(), tok) + if err != nil { + t.Fatal(err) + } + if id.Tenant != "team-a" || id.Subject != "alice" || id.Application != "web" || !id.IsAdmin() { + t.Fatalf("unexpected identity: %#v", id) + } +} +func signRS256(t *testing.T, key *rsa.PrivateKey, kid string, claims map[string]any) string { + t.Helper() + h, _ := json.Marshal(map[string]any{"alg": "RS256", "typ": "JWT", "kid": kid}) + p, _ := json.Marshal(claims) + a := base64.RawURLEncoding.EncodeToString(h) + "." + base64.RawURLEncoding.EncodeToString(p) + sum := sha256.Sum256([]byte(a)) + sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, sum[:]) + if err != nil { + t.Fatal(err) + } + return fmt.Sprintf("%s.%s", a, base64.RawURLEncoding.EncodeToString(sig)) +} diff --git a/internal/autotune/autotune.go b/internal/autotune/autotune.go new file mode 100644 index 0000000..931ec4c --- /dev/null +++ b/internal/autotune/autotune.go @@ -0,0 +1,532 @@ +package autotune + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math" + "net/http" + "net/url" + "os" + "sort" + "strings" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/hoststats" + "github.com/example/ollama-fair-gateway/internal/state" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +type Sample struct { + TTFTMS float64 `json:"ttft_ms"` + ServiceMS float64 `json:"service_ms"` + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + PromptTPS float64 `json:"prompt_tps"` + OutputTPS float64 `json:"output_tps"` + Status int `json:"status"` + Error string `json:"error,omitempty"` +} + +type Level struct { + Concurrency int `json:"concurrency"` + Requests int `json:"requests"` + Successful int `json:"successful"` + TTFTP50MS float64 `json:"ttft_p50_ms"` + TTFTP95MS float64 `json:"ttft_p95_ms"` + MeanServiceMS float64 `json:"mean_service_ms"` + MeanPromptTPS float64 `json:"mean_prompt_tps"` + MeanOutputTPS float64 `json:"mean_output_tps"` + AggregateOutputTPS float64 `json:"aggregate_output_tps"` + PeakVRAMBytes int64 `json:"peak_vram_bytes,omitempty"` + PeakGPUPercent float64 `json:"peak_gpu_percent,omitempty"` + Score float64 `json:"score"` + Samples []Sample `json:"samples,omitempty"` +} + +type Profile struct { + ID string `json:"id"` + Worker string `json:"worker"` + Model string `json:"model"` + Status string `json:"status"` // queued|running|completed|failed|cancelled + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at,omitempty"` + Levels []Level `json:"levels,omitempty"` + RecommendedConcurrency int `json:"recommended_concurrency,omitempty"` + Applied bool `json:"applied,omitempty"` + AppliedAt time.Time `json:"applied_at,omitempty"` + Error string `json:"error,omitempty"` +} + +type persistentState struct { + Version int `json:"version"` + SavedAt time.Time `json:"saved_at"` + Profiles []Profile `json:"profiles"` + Applied map[string]map[string]int `json:"applied"` +} + +type StartRequest struct { + Worker string `json:"worker"` + Model string `json:"model"` + MaxConcurrency int `json:"max_concurrency,omitempty"` + SamplesPerLevel int `json:"samples_per_level,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` + Prompt string `json:"prompt,omitempty"` +} + +type Manager struct { + mu sync.RWMutex + cfg config.AutoTuningConfig + workers *worker.Pool + path string + client *http.Client + profiles map[string]Profile + order []string + applied map[string]map[string]int + cancel map[string]context.CancelFunc +} + +func New(cfg config.AutoTuningConfig, workers *worker.Pool, path string) (*Manager, error) { + m := &Manager{cfg: cfg, workers: workers, path: path, client: &http.Client{Transport: &http.Transport{Proxy: http.ProxyFromEnvironment, MaxIdleConns: 64, MaxIdleConnsPerHost: 32, IdleConnTimeout: 30 * time.Second}}, profiles: map[string]Profile{}, applied: map[string]map[string]int{}, cancel: map[string]context.CancelFunc{}} + var ps persistentState + if path != "" { + err := (state.AtomicJSON{Path: path, Mode: 0640}).Load(&ps) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + } + for _, p := range ps.Profiles { + if p.ID == "" { + continue + } + if p.Status == "running" || p.Status == "queued" { + p.Status = "failed" + p.Error = "gateway restarted while benchmark was active" + p.FinishedAt = time.Now().UTC() + } + m.profiles[p.ID] = p + m.order = append(m.order, p.ID) + } + if ps.Applied != nil { + m.applied = ps.Applied + } + return m, nil +} + +func (m *Manager) saveLocked() error { + if m.path == "" { + return nil + } + profiles := make([]Profile, 0, len(m.order)) + for _, id := range m.order { + if p, ok := m.profiles[id]; ok { + profiles = append(profiles, p) + } + } + ps := persistentState{Version: 1, SavedAt: time.Now().UTC(), Profiles: profiles, Applied: m.applied} + return (state.AtomicJSON{Path: m.path, Mode: 0640}).Save(ps) +} + +func (m *Manager) Applied() map[string]map[string]int { + m.mu.RLock() + defer m.mu.RUnlock() + out := map[string]map[string]int{} + for w, mm := range m.applied { + out[w] = map[string]int{} + for model, n := range mm { + out[w][model] = n + } + } + return out +} + +func (m *Manager) List() []Profile { + m.mu.RLock() + defer m.mu.RUnlock() + out := make([]Profile, 0, len(m.order)) + for i := len(m.order) - 1; i >= 0; i-- { + if p, ok := m.profiles[m.order[i]]; ok { + out = append(out, cloneProfile(p)) + } + } + return out +} +func (m *Manager) Get(id string) (Profile, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + p, ok := m.profiles[id] + return cloneProfile(p), ok +} + +func cloneProfile(p Profile) Profile { + p.Levels = append([]Level(nil), p.Levels...) + for i := range p.Levels { + p.Levels[i].Samples = append([]Sample(nil), p.Levels[i].Samples...) + } + return p +} + +func (m *Manager) Start(parent context.Context, in StartRequest) (Profile, error) { + if !m.cfg.Enabled { + return Profile{}, errors.New("auto tuning is disabled") + } + in.Worker = strings.TrimSpace(in.Worker) + in.Model = strings.TrimSpace(in.Model) + if in.Worker == "" || in.Model == "" { + return Profile{}, errors.New("worker and model are required") + } + wc, ok := m.workers.WorkerConfig(in.Worker) + if !ok { + return Profile{}, fmt.Errorf("unknown worker %q", in.Worker) + } + if in.MaxConcurrency <= 0 { + in.MaxConcurrency = m.cfg.MaxConcurrency + } + in.MaxConcurrency = min(in.MaxConcurrency, wc.MaxConcurrent) + if in.MaxConcurrency < 1 { + in.MaxConcurrency = 1 + } + if in.SamplesPerLevel <= 0 { + in.SamplesPerLevel = m.cfg.SamplesPerLevel + } + if in.SamplesPerLevel > 20 { + in.SamplesPerLevel = 20 + } + if in.MaxTokens <= 0 { + in.MaxTokens = m.cfg.MaxTokens + } + if in.MaxTokens > 4096 { + in.MaxTokens = 4096 + } + if strings.TrimSpace(in.Prompt) == "" { + in.Prompt = m.cfg.Prompt + } + id := fmt.Sprintf("tune-%d", time.Now().UnixNano()) + p := Profile{ID: id, Worker: in.Worker, Model: in.Model, Status: "queued", StartedAt: time.Now().UTC()} + m.mu.Lock() + m.profiles[id] = p + m.order = append(m.order, id) + _ = m.saveLocked() + m.mu.Unlock() + ctx, cancel := context.WithTimeout(context.WithoutCancel(parent), m.cfg.Timeout.Value()) + m.mu.Lock() + m.cancel[id] = cancel + m.mu.Unlock() + go m.run(ctx, id, in, wc) + return p, nil +} + +func (m *Manager) Cancel(id string) bool { + m.mu.RLock() + c := m.cancel[id] + m.mu.RUnlock() + if c == nil { + return false + } + c() + return true +} + +func (m *Manager) Apply(id string) (Profile, error) { + m.mu.Lock() + defer m.mu.Unlock() + p, ok := m.profiles[id] + if !ok { + return Profile{}, errors.New("benchmark profile not found") + } + if p.Status != "completed" || p.RecommendedConcurrency <= 0 { + return Profile{}, errors.New("benchmark has no applicable recommendation") + } + if m.applied[p.Worker] == nil { + m.applied[p.Worker] = map[string]int{} + } + m.applied[p.Worker][p.Model] = p.RecommendedConcurrency + p.Applied = true + p.AppliedAt = time.Now().UTC() + m.profiles[id] = p + if err := m.saveLocked(); err != nil { + return Profile{}, err + } + return cloneProfile(p), nil +} +func (m *Manager) Reset(workerName, model string) error { + m.mu.Lock() + defer m.mu.Unlock() + if mm := m.applied[workerName]; mm != nil { + delete(mm, model) + if len(mm) == 0 { + delete(m.applied, workerName) + } + } + return m.saveLocked() +} + +func (m *Manager) run(ctx context.Context, id string, in StartRequest, wc config.WorkerConfig) { + m.update(id, func(p *Profile) { p.Status = "running" }) + levels := make([]Level, 0, in.MaxConcurrency) + for c := 1; c <= in.MaxConcurrency; c++ { + if ctx.Err() != nil { + m.finishCancelled(id, ctx.Err()) + return + } + lvl := m.runLevel(ctx, wc, in.Model, in.Prompt, in.MaxTokens, c, in.SamplesPerLevel) + levels = append(levels, lvl) + m.update(id, func(p *Profile) { p.Levels = append([]Level(nil), levels...) }) + } + if len(levels) == 0 { + m.finishFailed(id, "no benchmark levels completed") + return + } + maxAgg, maxP95 := 0.0, 0.0 + for _, l := range levels { + if l.AggregateOutputTPS > maxAgg { + maxAgg = l.AggregateOutputTPS + } + if l.TTFTP95MS > maxP95 { + maxP95 = l.TTFTP95MS + } + } + best, bestScore := 1, -math.MaxFloat64 + for i := range levels { + throughput := 0.0 + if maxAgg > 0 { + throughput = levels[i].AggregateOutputTPS / maxAgg + } + latency := 0.0 + if maxP95 > 0 { + latency = levels[i].TTFTP95MS / maxP95 + } + failure := 1.0 - float64(levels[i].Successful)/float64(max(1, levels[i].Requests)) + levels[i].Score = m.cfg.ThroughputWeight*throughput - m.cfg.TTFTWeight*latency - failure + if levels[i].Successful == levels[i].Requests && levels[i].Score > bestScore { + bestScore = levels[i].Score + best = levels[i].Concurrency + } + } + m.mu.Lock() + p := m.profiles[id] + p.Status = "completed" + p.Levels = levels + p.RecommendedConcurrency = best + p.FinishedAt = time.Now().UTC() + m.profiles[id] = p + delete(m.cancel, id) + _ = m.saveLocked() + m.mu.Unlock() +} + +func (m *Manager) update(id string, fn func(*Profile)) { + m.mu.Lock() + p := m.profiles[id] + fn(&p) + m.profiles[id] = p + _ = m.saveLocked() + m.mu.Unlock() +} +func (m *Manager) finishCancelled(id string, err error) { + m.mu.Lock() + p := m.profiles[id] + p.Status = "cancelled" + if err != nil { + p.Error = err.Error() + } + p.FinishedAt = time.Now().UTC() + m.profiles[id] = p + delete(m.cancel, id) + _ = m.saveLocked() + m.mu.Unlock() +} +func (m *Manager) finishFailed(id, msg string) { + m.mu.Lock() + p := m.profiles[id] + p.Status = "failed" + p.Error = msg + p.FinishedAt = time.Now().UTC() + m.profiles[id] = p + delete(m.cancel, id) + _ = m.saveLocked() + m.mu.Unlock() +} + +func (m *Manager) runLevel(ctx context.Context, wc config.WorkerConfig, model, prompt string, maxTokens, concurrency, repeats int) Level { + lvl := Level{Concurrency: concurrency, Requests: concurrency * repeats} + var samples []Sample + var totalTokens int64 + var totalWall time.Duration + var peakVRAM int64 + var peakGPU float64 + var peakMu sync.Mutex + for rep := 0; rep < repeats; rep++ { + start := time.Now() + ch := make(chan Sample, concurrency) + var wg sync.WaitGroup + for i := 0; i < concurrency; i++ { + wg.Add(1) + go func() { defer wg.Done(); ch <- m.one(ctx, wc.URL, model, prompt, maxTokens) }() + } + done := make(chan struct{}) + go func() { wg.Wait(); close(ch); close(done) }() + var telemetryWG sync.WaitGroup + if wc.NVIDIASMI { + telemetryWG.Add(1) + go func() { + defer telemetryWG.Done() + t := time.NewTicker(200 * time.Millisecond) + defer t.Stop() + for { + select { + case <-done: + return + case <-ctx.Done(): + return + case <-t.C: + c, cancel := context.WithTimeout(ctx, time.Second) + n, e := hoststats.ReadNVIDIA(c, wc.NVIDIAGPU) + cancel() + if e == nil { + peakMu.Lock() + if n.MemoryUsedBytes > peakVRAM { + peakVRAM = n.MemoryUsedBytes + } + if n.UtilizationPercent > peakGPU { + peakGPU = n.UtilizationPercent + } + peakMu.Unlock() + } + } + } + }() + } + for s := range ch { + samples = append(samples, s) + if s.Status >= 200 && s.Status < 300 && s.Error == "" { + lvl.Successful++ + totalTokens += s.CompletionTokens + } + } + telemetryWG.Wait() + totalWall += time.Since(start) + } + lvl.Samples = samples + lvl.PeakVRAMBytes = peakVRAM + lvl.PeakGPUPercent = peakGPU + var ttfts, services []float64 + var pTPS, oTPS float64 + for _, s := range samples { + if s.Error != "" || s.Status < 200 || s.Status >= 300 { + continue + } + ttfts = append(ttfts, s.TTFTMS) + services = append(services, s.ServiceMS) + pTPS += s.PromptTPS + oTPS += s.OutputTPS + } + lvl.TTFTP50MS = percentile(ttfts, .50) + lvl.TTFTP95MS = percentile(ttfts, .95) + lvl.MeanServiceMS = mean(services) + if lvl.Successful > 0 { + lvl.MeanPromptTPS = pTPS / float64(lvl.Successful) + lvl.MeanOutputTPS = oTPS / float64(lvl.Successful) + } + if totalWall > 0 { + lvl.AggregateOutputTPS = float64(totalTokens) / totalWall.Seconds() + } + return lvl +} + +func (m *Manager) one(ctx context.Context, base, model, prompt string, maxTokens int) Sample { + u, err := url.Parse(strings.TrimRight(base, "/") + "/api/generate") + if err != nil { + return Sample{Error: err.Error()} + } + b, _ := json.Marshal(map[string]any{"model": model, "prompt": prompt, "stream": true, "keep_alive": "5m", "options": map[string]any{"num_predict": maxTokens, "temperature": 0}}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewReader(b)) + if err != nil { + return Sample{Error: err.Error()} + } + req.Header.Set("Content-Type", "application/json") + start := time.Now() + resp, err := m.client.Do(req) + if err != nil { + return Sample{Error: err.Error(), ServiceMS: float64(time.Since(start).Microseconds()) / 1000} + } + defer resp.Body.Close() + out := Sample{Status: resp.StatusCode} + scan := bufio.NewScanner(resp.Body) + buf := make([]byte, 64*1024) + scan.Buffer(buf, 4<<20) + first := true + for scan.Scan() { + if first { + out.TTFTMS = float64(time.Since(start).Microseconds()) / 1000 + first = false + } + var v map[string]any + if json.Unmarshal(scan.Bytes(), &v) != nil { + continue + } + if n := asI64(v["prompt_eval_count"]); n > 0 { + out.PromptTokens = n + } + if n := asI64(v["eval_count"]); n > 0 { + out.CompletionTokens = n + } + pn := asI64(v["prompt_eval_duration"]) + en := asI64(v["eval_duration"]) + if pn > 0 && out.PromptTokens > 0 { + out.PromptTPS = float64(out.PromptTokens) / (float64(pn) / 1e9) + } + if en > 0 && out.CompletionTokens > 0 { + out.OutputTPS = float64(out.CompletionTokens) / (float64(en) / 1e9) + } + } + if err := scan.Err(); err != nil { + out.Error = err.Error() + } + out.ServiceMS = float64(time.Since(start).Microseconds()) / 1000 + return out +} +func asI64(v any) int64 { + switch x := v.(type) { + case float64: + return int64(x) + case json.Number: + n, _ := x.Int64() + return n + case int64: + return x + case int: + return int64(x) + } + return 0 +} +func mean(xs []float64) float64 { + if len(xs) == 0 { + return 0 + } + s := 0.0 + for _, x := range xs { + s += x + } + return s / float64(len(xs)) +} +func percentile(xs []float64, p float64) float64 { + if len(xs) == 0 { + return 0 + } + ys := append([]float64(nil), xs...) + sort.Float64s(ys) + i := int(math.Ceil(p*float64(len(ys)))) - 1 + if i < 0 { + i = 0 + } + if i >= len(ys) { + i = len(ys) - 1 + } + return ys[i] +} diff --git a/internal/autotune/autotune_test.go b/internal/autotune/autotune_test.go new file mode 100644 index 0000000..08bab8f --- /dev/null +++ b/internal/autotune/autotune_test.go @@ -0,0 +1,80 @@ +package autotune + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +func waitProfile(t *testing.T, m *Manager, id string) Profile { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + p, ok := m.Get(id) + if ok && p.Status != "queued" && p.Status != "running" { + return p + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("profile %s did not finish", id) + return Profile{} +} + +func TestBenchmarkRecommendationApplyAndReload(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/generate" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/x-ndjson") + f, _ := w.(http.Flusher) + fmt.Fprintln(w, `{"response":"x","done":false}`) + if f != nil { + f.Flush() + } + time.Sleep(20 * time.Millisecond) + fmt.Fprintln(w, `{"done":true,"prompt_eval_count":10,"prompt_eval_duration":10000000,"eval_count":20,"eval_duration":20000000}`) + })) + defer ts.Close() + + wc := config.WorkerConfig{Name: "gpu", URL: ts.URL, MaxConcurrent: 2} + pool := worker.New([]config.WorkerConfig{wc}, "gpu") + path := filepath.Join(t.TempDir(), "auto-tune.json") + cfg := config.AutoTuningConfig{Enabled: true, MaxConcurrency: 2, SamplesPerLevel: 1, MaxTokens: 32, Timeout: config.Duration(5 * time.Second), Prompt: "test", TTFTWeight: .05, ThroughputWeight: 1} + m, err := New(cfg, pool, path) + if err != nil { + t.Fatal(err) + } + p, err := m.Start(context.Background(), StartRequest{Worker: "gpu", Model: "qwen:test"}) + if err != nil { + t.Fatal(err) + } + p = waitProfile(t, m, p.ID) + if p.Status != "completed" { + t.Fatalf("status=%s err=%s", p.Status, p.Error) + } + if len(p.Levels) != 2 { + t.Fatalf("levels=%d", len(p.Levels)) + } + if p.RecommendedConcurrency != 2 { + t.Fatalf("recommendation=%d want 2; levels=%+v", p.RecommendedConcurrency, p.Levels) + } + if _, err := m.Apply(p.ID); err != nil { + t.Fatal(err) + } + + m2, err := New(cfg, pool, path) + if err != nil { + t.Fatal(err) + } + if got := m2.Applied()["gpu"]["qwen:test"]; got != 2 { + t.Fatalf("reloaded applied=%d want 2", got) + } +} diff --git a/internal/batch/manager.go b/internal/batch/manager.go new file mode 100644 index 0000000..b6f0e6f --- /dev/null +++ b/internal/batch/manager.go @@ -0,0 +1,692 @@ +package batch + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/state" +) + +const ( + StateQueued = "queued" + StateRunning = "running" + StatePaused = "paused" + StatePausing = "pausing" + StateCancelling = "cancelling" + StateCompleted = "completed" + StateFailed = "failed" + StateCancelled = "cancelled" +) + +var ( + ErrDisabled = errors.New("durable batch jobs are disabled") + ErrNotFound = errors.New("batch job not found") + ErrInvalidState = errors.New("batch job state does not allow this operation") + ErrFull = errors.New("batch job retention store is full") + ErrInputTooLarge = errors.New("batch input exceeds configured max_input_bytes") +) + +// IdentitySnapshot preserves the authenticated metadata required to re-run a +// durable request through the normal gateway authorization/routing pipeline. +// It intentionally contains no bearer/API-key secret. +type IdentitySnapshot struct { + Tenant string `json:"tenant"` + Subject string `json:"subject"` + Actor string `json:"actor"` + Application string `json:"application,omitempty"` + AuthType string `json:"auth_type"` + ClientIP string `json:"client_ip,omitempty"` + Scopes []string `json:"scopes,omitempty"` + ModelACLSet bool `json:"model_acl_set,omitempty"` + ModelAccess config.ModelAccessRule `json:"model_access,omitempty"` +} + +type Job struct { + ID string `json:"id"` + Identity IdentitySnapshot `json:"identity"` + Path string `json:"path"` + Model string `json:"model,omitempty"` + ServiceClass string `json:"service_class"` + InputRef string `json:"input_ref"` + OutputRef string `json:"output_ref,omitempty"` + State string `json:"state"` + Attempts int `json:"attempts"` + HTTPStatus int `json:"http_status,omitempty"` + ResponseContentType string `json:"response_content_type,omitempty"` + ExecutionRequestID string `json:"execution_request_id,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Error string `json:"error,omitempty"` +} + +type RunResult struct { + HTTPStatus int + ResponseContentType string + RequestID string + Error string +} + +type Runner func(context.Context, Job, io.Reader, io.Writer) RunResult + +type snapshot struct { + Version int `json:"version"` + Jobs map[string]Job `json:"jobs"` +} + +type Manager struct { + mu sync.Mutex + cfg config.BatchJobsConfig + file state.AtomicJSON + dir string + jobs map[string]Job + cancel map[string]context.CancelCauseFunc + wake chan struct{} + active int + runner Runner + ctx context.Context + now func() time.Time + wg sync.WaitGroup +} + +var ( + errPause = errors.New("batch pause requested") + errCancel = errors.New("batch cancel requested") +) + +func New(cfg config.BatchJobsConfig, metadataPath, dir string) (*Manager, error) { + m := &Manager{cfg: cfg, file: state.AtomicJSON{Path: metadataPath, Mode: 0600}, dir: dir, jobs: map[string]Job{}, cancel: map[string]context.CancelCauseFunc{}, wake: make(chan struct{}, 1), now: time.Now} + if !cfg.Enabled { + return m, nil + } + if err := os.MkdirAll(filepath.Join(dir, "input"), 0750); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Join(dir, "output"), 0750); err != nil { + return nil, err + } + var snap snapshot + if err := m.file.Load(&snap); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + } else { + if snap.Version != 1 { + return nil, fmt.Errorf("unsupported batch metadata version %d", snap.Version) + } + if snap.Jobs != nil { + m.jobs = snap.Jobs + } + } + changed := m.recoverLocked() + changed = m.pruneLocked(m.now().UTC()) || changed + if changed { + if err := m.saveLocked(); err != nil { + return nil, err + } + } + return m, nil +} + +func (m *Manager) Enabled() bool { return m != nil && m.cfg.Enabled } + +func (m *Manager) Start(ctx context.Context, runner Runner) { + if !m.Enabled() || runner == nil { + return + } + m.mu.Lock() + m.ctx = ctx + m.runner = runner + m.mu.Unlock() + go m.loop(ctx) + m.signal() +} + +// Wait blocks until all currently running batch attempts have returned. It is +// intended for graceful shutdown after the root context has been cancelled, so +// no new attempts can be dispatched while waiting. Each finishing attempt +// persists its final restart-safe state before it releases the wait group. +func (m *Manager) Wait(ctx context.Context) error { + if !m.Enabled() { + return nil + } + done := make(chan struct{}) + go func() { + m.wg.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (m *Manager) Create(id IdentitySnapshot, path, model string, body []byte) (Job, error) { + if !m.Enabled() { + return Job{}, ErrDisabled + } + if id.Tenant == "" || id.Actor == "" { + return Job{}, errors.New("batch identity requires tenant and actor") + } + if path == "" || !strings.HasPrefix(path, "/") { + return Job{}, errors.New("batch path must be an absolute gateway path") + } + if int64(len(body)) > m.cfg.MaxInputBytes { + return Job{}, ErrInputTooLarge + } + if len(body) == 0 { + return Job{}, errors.New("batch input body is empty") + } + + m.mu.Lock() + defer m.mu.Unlock() + now := m.now().UTC() + m.pruneLocked(now) + if len(m.jobs) >= m.cfg.MaxJobs { + return Job{}, ErrFull + } + jobID, err := newID() + if err != nil { + return Job{}, err + } + inputRef := filepath.ToSlash(filepath.Join("input", jobID+".json")) + if err := m.writeInputLocked(inputRef, body); err != nil { + return Job{}, err + } + j := Job{ID: jobID, Identity: id, Path: path, Model: model, ServiceClass: "batch", InputRef: inputRef, State: StateQueued, CreatedAt: now, UpdatedAt: now} + m.jobs[j.ID] = j + if err := m.saveLocked(); err != nil { + delete(m.jobs, j.ID) + _ = os.Remove(m.refPath(inputRef)) + return Job{}, err + } + m.signal() + return cloneJob(j), nil +} + +func (m *Manager) List(tenant, actor string, all bool) []Job { + if !m.Enabled() { + return nil + } + m.mu.Lock() + defer m.mu.Unlock() + if m.pruneLocked(m.now().UTC()) { + _ = m.saveLocked() + } + out := make([]Job, 0, len(m.jobs)) + for _, j := range m.jobs { + if all || (j.Identity.Tenant == tenant && j.Identity.Actor == actor) { + out = append(out, cloneJob(j)) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) }) + return out +} + +func (m *Manager) Get(id, tenant, actor string, all bool) (Job, bool) { + if !m.Enabled() { + return Job{}, false + } + m.mu.Lock() + defer m.mu.Unlock() + j, ok := m.jobs[id] + if !ok || (!all && (j.Identity.Tenant != tenant || j.Identity.Actor != actor)) { + return Job{}, false + } + return cloneJob(j), true +} + +func (m *Manager) Pause(id, tenant, actor string, all bool) (Job, error) { + return m.control(id, tenant, actor, all, "pause") +} + +func (m *Manager) Resume(id, tenant, actor string, all bool) (Job, error) { + return m.control(id, tenant, actor, all, "resume") +} + +func (m *Manager) Cancel(id, tenant, actor string, all bool) (Job, error) { + return m.control(id, tenant, actor, all, "cancel") +} + +func (m *Manager) control(id, tenant, actor string, all bool, action string) (Job, error) { + if !m.Enabled() { + return Job{}, ErrDisabled + } + m.mu.Lock() + j, ok := m.jobs[id] + if !ok || (!all && (j.Identity.Tenant != tenant || j.Identity.Actor != actor)) { + m.mu.Unlock() + return Job{}, ErrNotFound + } + now := m.now().UTC() + var cancel context.CancelCauseFunc + switch action { + case "pause": + switch j.State { + case StateQueued: + j.State = StatePaused + case StateRunning: + j.State = StatePausing + cancel = m.cancel[id] + case StatePaused, StatePausing: + // idempotent + default: + m.mu.Unlock() + return Job{}, ErrInvalidState + } + case "resume": + if j.State != StatePaused { + m.mu.Unlock() + return Job{}, ErrInvalidState + } + j.State = StateQueued + j.Error = "" + j.HTTPStatus = 0 + j.ResponseContentType = "" + j.ExecutionRequestID = "" + j.OutputRef = "" + j.StartedAt = nil + j.FinishedAt = nil + case "cancel": + switch j.State { + case StateQueued, StatePaused: + j.State = StateCancelled + j.FinishedAt = ptrTime(now) + case StateRunning, StatePausing: + j.State = StateCancelling + cancel = m.cancel[id] + case StateCancelling, StateCancelled: + // idempotent + default: + m.mu.Unlock() + return Job{}, ErrInvalidState + } + default: + m.mu.Unlock() + return Job{}, errors.New("unknown batch control action") + } + j.UpdatedAt = now + m.jobs[id] = j + if err := m.saveLocked(); err != nil { + m.mu.Unlock() + return Job{}, err + } + m.mu.Unlock() + if cancel != nil { + if action == "pause" { + cancel(errPause) + } else { + cancel(errCancel) + } + } + if action == "resume" { + m.signal() + } + return cloneJob(j), nil +} + +func (m *Manager) OpenOutput(id, tenant, actor string, all bool) (*os.File, Job, error) { + if !m.Enabled() { + return nil, Job{}, ErrDisabled + } + m.mu.Lock() + j, ok := m.jobs[id] + if !ok || (!all && (j.Identity.Tenant != tenant || j.Identity.Actor != actor)) { + m.mu.Unlock() + return nil, Job{}, ErrNotFound + } + ref := j.OutputRef + m.mu.Unlock() + if ref == "" { + return nil, cloneJob(j), os.ErrNotExist + } + f, err := os.Open(m.refPath(ref)) + return f, cloneJob(j), err +} + +func (m *Manager) Compact() error { + if !m.Enabled() { + return nil + } + m.mu.Lock() + defer m.mu.Unlock() + m.pruneLocked(m.now().UTC()) + return m.saveLocked() +} + +func (m *Manager) loop(ctx context.Context) { + t := time.NewTicker(time.Minute) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-m.wake: + m.dispatch(ctx) + case <-t.C: + m.mu.Lock() + changed := m.pruneLocked(m.now().UTC()) + if changed { + _ = m.saveLocked() + } + m.mu.Unlock() + m.dispatch(ctx) + } + } +} + +func (m *Manager) dispatch(root context.Context) { + for { + m.mu.Lock() + if m.runner == nil || m.active >= m.cfg.MaxConcurrent || root.Err() != nil { + m.mu.Unlock() + return + } + var chosen *Job + for _, j := range m.jobs { + if j.State != StateQueued { + continue + } + if chosen == nil || j.CreatedAt.Before(chosen.CreatedAt) { + jc := j + chosen = &jc + } + } + if chosen == nil { + m.mu.Unlock() + return + } + now := m.now().UTC() + j := *chosen + j.State = StateRunning + j.Attempts++ + j.StartedAt = ptrTime(now) + j.FinishedAt = nil + j.UpdatedAt = now + j.Error = "" + m.jobs[j.ID] = j + ctx, cancel := context.WithCancelCause(root) + m.cancel[j.ID] = cancel + m.active++ + if err := m.saveLocked(); err != nil { + m.active-- + delete(m.cancel, j.ID) + j.State = StateQueued + m.jobs[j.ID] = j + m.mu.Unlock() + return + } + runner := m.runner + m.wg.Add(1) + m.mu.Unlock() + go func() { + defer m.wg.Done() + m.runOne(ctx, j, runner) + }() + } +} + +func (m *Manager) runOne(ctx context.Context, j Job, runner Runner) { + input, err := os.Open(m.refPath(j.InputRef)) + if err != nil { + m.finishRun(j.ID, RunResult{Error: "open input: " + err.Error()}, "", true) + return + } + defer input.Close() + + outDir := filepath.Join(m.dir, "output") + tmp, err := os.CreateTemp(outDir, ".batch-output-*.tmp") + if err != nil { + m.finishRun(j.ID, RunResult{Error: "create output: " + err.Error()}, "", true) + return + } + tmpPath := tmp.Name() + if err := tmp.Chmod(0600); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpPath) + m.finishRun(j.ID, RunResult{Error: "chmod output: " + err.Error()}, "", true) + return + } + res := runner(ctx, cloneJob(j), input, tmp) + if err := tmp.Sync(); err != nil && res.Error == "" { + res.Error = "sync output: " + err.Error() + } + if err := tmp.Close(); err != nil && res.Error == "" { + res.Error = "close output: " + err.Error() + } + m.finishRun(j.ID, res, tmpPath, false) +} + +func (m *Manager) finishRun(id string, res RunResult, tmpPath string, setupFailure bool) { + m.mu.Lock() + defer m.mu.Unlock() + j, ok := m.jobs[id] + if !ok { + if tmpPath != "" { + _ = os.Remove(tmpPath) + } + return + } + delete(m.cancel, id) + if m.active > 0 { + m.active-- + } + now := m.now().UTC() + j.UpdatedAt = now + j.HTTPStatus = res.HTTPStatus + j.ResponseContentType = res.ResponseContentType + j.ExecutionRequestID = res.RequestID + + switch { + case j.State == StateCancelling: + if tmpPath != "" { + _ = os.Remove(tmpPath) + } + j.State = StateCancelled + j.FinishedAt = ptrTime(now) + j.Error = "cancelled" + case j.State == StatePausing: + if tmpPath != "" { + _ = os.Remove(tmpPath) + } + j.State = StatePaused + j.StartedAt = nil + j.Error = "paused" + case m.ctx != nil && m.ctx.Err() != nil: + if tmpPath != "" { + _ = os.Remove(tmpPath) + } + j.State = StateQueued + j.StartedAt = nil + j.Error = "interrupted by gateway shutdown; queued for retry" + case setupFailure: + j.State = StateFailed + j.FinishedAt = ptrTime(now) + j.Error = res.Error + default: + finalRef := filepath.ToSlash(filepath.Join("output", id+".response")) + finalPath := m.refPath(finalRef) + _ = os.Remove(finalPath) + if tmpPath != "" { + if err := os.Rename(tmpPath, finalPath); err != nil { + j.State = StateFailed + j.FinishedAt = ptrTime(now) + j.Error = "commit output: " + err.Error() + break + } + j.OutputRef = finalRef + } + if res.Error != "" || res.HTTPStatus < 200 || res.HTTPStatus >= 400 { + j.State = StateFailed + j.Error = res.Error + if j.Error == "" && res.HTTPStatus != 0 { + j.Error = fmt.Sprintf("gateway HTTP %d", res.HTTPStatus) + } + } else { + j.State = StateCompleted + j.Error = "" + } + j.FinishedAt = ptrTime(now) + } + m.jobs[id] = j + _ = m.saveLocked() + m.signal() +} + +func (m *Manager) recoverLocked() bool { + changed := false + now := m.now().UTC() + for id, j := range m.jobs { + switch j.State { + case StateRunning: + j.State = StateQueued + j.StartedAt = nil + j.Error = "recovered after gateway restart; queued for retry" + j.UpdatedAt = now + m.jobs[id] = j + changed = true + case StatePausing: + j.State = StatePaused + j.StartedAt = nil + j.Error = "pause recovered after gateway restart" + j.UpdatedAt = now + m.jobs[id] = j + changed = true + case StateCancelling: + j.State = StateCancelled + j.FinishedAt = ptrTime(now) + j.Error = "cancel recovered after gateway restart" + j.UpdatedAt = now + m.jobs[id] = j + changed = true + } + } + return changed +} + +func (m *Manager) pruneLocked(now time.Time) bool { + if m.cfg.Retention.Value() <= 0 { + return false + } + changed := false + for id, j := range m.jobs { + if !isTerminal(j.State) || j.FinishedAt == nil || now.Sub(*j.FinishedAt) < m.cfg.Retention.Value() { + continue + } + _ = os.Remove(m.refPath(j.InputRef)) + if j.OutputRef != "" { + _ = os.Remove(m.refPath(j.OutputRef)) + } + delete(m.jobs, id) + changed = true + } + return changed +} + +func (m *Manager) saveLocked() error { + copyMap := make(map[string]Job, len(m.jobs)) + for id, j := range m.jobs { + copyMap[id] = cloneJob(j) + } + return m.file.Save(snapshot{Version: 1, Jobs: copyMap}) +} + +func (m *Manager) writeInputLocked(ref string, body []byte) error { + path := m.refPath(ref) + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0750); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, ".batch-input-*.tmp") + if err != nil { + return err + } + name := tmp.Name() + ok := false + defer func() { + _ = tmp.Close() + if !ok { + _ = os.Remove(name) + } + }() + if err := tmp.Chmod(0600); err != nil { + return err + } + if _, err := tmp.Write(body); err != nil { + return err + } + if err := tmp.Sync(); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(name, path); err != nil { + return err + } + ok = true + return nil +} + +func (m *Manager) refPath(ref string) string { + ref = filepath.Clean(filepath.FromSlash(ref)) + if ref == "." || filepath.IsAbs(ref) || ref == ".." || strings.HasPrefix(ref, ".."+string(filepath.Separator)) { + return filepath.Join(m.dir, "invalid-ref") + } + return filepath.Join(m.dir, ref) +} + +func (m *Manager) signal() { + select { + case m.wake <- struct{}{}: + default: + } +} + +func isTerminal(state string) bool { + switch state { + case StateCompleted, StateFailed, StateCancelled: + return true + default: + return false + } +} + +func newID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return "batch_" + hex.EncodeToString(b), nil +} + +func cloneJob(j Job) Job { + j.Identity.Scopes = append([]string(nil), j.Identity.Scopes...) + j.Identity.ModelAccess.AllowedModels = append([]string(nil), j.Identity.ModelAccess.AllowedModels...) + j.Identity.ModelAccess.DeniedModels = append([]string(nil), j.Identity.ModelAccess.DeniedModels...) + if j.StartedAt != nil { + t := *j.StartedAt + j.StartedAt = &t + } + if j.FinishedAt != nil { + t := *j.FinishedAt + j.FinishedAt = &t + } + return j +} + +func ptrTime(t time.Time) *time.Time { return &t } diff --git a/internal/batch/manager_test.go b/internal/batch/manager_test.go new file mode 100644 index 0000000..52b044d --- /dev/null +++ b/internal/batch/manager_test.go @@ -0,0 +1,216 @@ +package batch + +import ( + "context" + "io" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +func batchTestConfig() config.BatchJobsConfig { + return config.BatchJobsConfig{Enabled: true, Retention: config.Duration(time.Hour), MaxJobs: 100, MaxConcurrent: 1, MaxInputBytes: 1 << 20} +} + +func waitState(t *testing.T, m *Manager, id, state string) Job { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + j, ok := m.Get(id, "t", "u", false) + if ok && j.State == state { + return j + } + time.Sleep(5 * time.Millisecond) + } + j, _ := m.Get(id, "t", "u", false) + t.Fatalf("job %s did not reach %s; got %#v", id, state, j) + return Job{} +} + +func TestCreateRunPersistAndOutput(t *testing.T) { + dir := t.TempDir() + meta := filepath.Join(dir, "batch-jobs.json") + spool := filepath.Join(dir, "batch") + m, err := New(batchTestConfig(), meta, spool) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + m.Start(ctx, func(ctx context.Context, j Job, in io.Reader, out io.Writer) RunResult { + b, _ := io.ReadAll(in) + if string(b) != `{"model":"m","input":"x"}` { + t.Errorf("input=%s", b) + } + io.WriteString(out, `{"id":"ok"}`) + return RunResult{HTTPStatus: 200, ResponseContentType: "application/json", RequestID: "req-1"} + }) + j, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/v1/responses", "m", []byte(`{"model":"m","input":"x"}`)) + if err != nil { + t.Fatal(err) + } + j = waitState(t, m, j.ID, StateCompleted) + if j.OutputRef == "" || j.ExecutionRequestID != "req-1" || j.HTTPStatus != 200 { + t.Fatalf("job=%#v", j) + } + f, _, err := m.OpenOutput(j.ID, "t", "u", false) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(f) + f.Close() + if string(b) != `{"id":"ok"}` { + t.Fatalf("output=%s", b) + } + + m2, err := New(batchTestConfig(), meta, spool) + if err != nil { + t.Fatal(err) + } + got, ok := m2.Get(j.ID, "t", "u", false) + if !ok || got.State != StateCompleted { + t.Fatalf("reloaded=%#v ok=%v", got, ok) + } + if _, ok := m2.Get(j.ID, "t", "other", false); ok { + t.Fatal("cross-actor job lookup must be hidden") + } +} + +func TestPauseResumeAndCancel(t *testing.T) { + dir := t.TempDir() + m, err := New(batchTestConfig(), filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool")) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var attempts atomic.Int32 + m.Start(ctx, func(ctx context.Context, j Job, in io.Reader, out io.Writer) RunResult { + n := attempts.Add(1) + if n == 1 { + <-ctx.Done() + return RunResult{HTTPStatus: 499, Error: context.Cause(ctx).Error()} + } + io.WriteString(out, "done") + return RunResult{HTTPStatus: 200, RequestID: "req-done"} + }) + j, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/api/chat", "m", []byte(`{"model":"m"}`)) + if err != nil { + t.Fatal(err) + } + waitState(t, m, j.ID, StateRunning) + if _, err := m.Pause(j.ID, "t", "u", false); err != nil { + t.Fatal(err) + } + waitState(t, m, j.ID, StatePaused) + if _, err := m.Resume(j.ID, "t", "u", false); err != nil { + t.Fatal(err) + } + waitState(t, m, j.ID, StateCompleted) + if attempts.Load() != 2 { + t.Fatalf("attempts=%d", attempts.Load()) + } + + j2, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/api/chat", "m", []byte(`{"model":"m"}`)) + if err != nil { + t.Fatal(err) + } + // The runner completes quickly on later attempts, so cancel while queued by + // first pausing it synchronously. + if _, err := m.Pause(j2.ID, "t", "u", false); err != nil && err != ErrInvalidState { + t.Fatal(err) + } + cur, _ := m.Get(j2.ID, "t", "u", false) + if cur.State == StatePaused { + if _, err := m.Cancel(j2.ID, "t", "u", false); err != nil { + t.Fatal(err) + } + waitState(t, m, j2.ID, StateCancelled) + } +} + +func TestRetentionDeletesContentFiles(t *testing.T) { + dir := t.TempDir() + cfg := batchTestConfig() + cfg.Retention = config.Duration(time.Minute) + m, err := New(cfg, filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool")) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 9, 8, 8, 0, 0, 0, time.UTC) + m.now = func() time.Time { return now } + j, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/api/chat", "m", []byte(`{"model":"m"}`)) + if err != nil { + t.Fatal(err) + } + if _, err := m.Cancel(j.ID, "t", "u", false); err != nil { + t.Fatal(err) + } + job, _ := m.Get(j.ID, "t", "u", false) + inputPath := m.refPath(job.InputRef) + if _, err := os.Stat(inputPath); err != nil { + t.Fatal(err) + } + now = now.Add(2 * time.Minute) + if err := m.Compact(); err != nil { + t.Fatal(err) + } + if _, ok := m.Get(j.ID, "t", "u", false); ok { + t.Fatal("expired job retained") + } + if _, err := os.Stat(inputPath); !os.IsNotExist(err) { + t.Fatalf("input still exists: %v", err) + } +} + +func TestInputLimit(t *testing.T) { + dir := t.TempDir() + cfg := batchTestConfig() + cfg.MaxInputBytes = 4 + m, _ := New(cfg, filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool")) + _, err := m.Create(IdentitySnapshot{Tenant: "t", Actor: "u"}, "/api/chat", "m", []byte(strings.Repeat("x", 5))) + if err != ErrInputTooLarge { + t.Fatalf("err=%v", err) + } +} + +func TestWaitPersistsRestartSafeStateAfterShutdown(t *testing.T) { + dir := t.TempDir() + m, err := New(batchTestConfig(), filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool")) + if err != nil { + t.Fatal(err) + } + root, cancel := context.WithCancel(context.Background()) + m.Start(root, func(ctx context.Context, j Job, in io.Reader, out io.Writer) RunResult { + <-ctx.Done() + return RunResult{HTTPStatus: 499, Error: context.Cause(ctx).Error()} + }) + j, err := m.Create(IdentitySnapshot{Tenant: "t", Subject: "u", Actor: "u", AuthType: "oidc"}, "/api/chat", "m", []byte(`{"model":"m"}`)) + if err != nil { + t.Fatal(err) + } + waitState(t, m, j.ID, StateRunning) + cancel() + wctx, wcancel := context.WithTimeout(context.Background(), time.Second) + defer wcancel() + if err := m.Wait(wctx); err != nil { + t.Fatal(err) + } + got, ok := m.Get(j.ID, "t", "u", false) + if !ok || got.State != StateQueued || !strings.Contains(got.Error, "shutdown") { + t.Fatalf("restart-safe state not persisted: %#v ok=%v", got, ok) + } + m2, err := New(batchTestConfig(), filepath.Join(dir, "jobs.json"), filepath.Join(dir, "spool")) + if err != nil { + t.Fatal(err) + } + reloaded, ok := m2.Get(j.ID, "t", "u", false) + if !ok || reloaded.State != StateQueued { + t.Fatalf("reloaded state=%#v ok=%v", reloaded, ok) + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..f4bec18 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,1314 @@ +package config + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "time" +) + +type Duration time.Duration + +func (d *Duration) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err == nil { + v, err := time.ParseDuration(s) + if err != nil { + return err + } + *d = Duration(v) + return nil + } + var n int64 + if err := json.Unmarshal(b, &n); err != nil { + return errors.New("duration must be a Go duration string or nanoseconds") + } + *d = Duration(time.Duration(n)) + return nil +} +func (d Duration) Value() time.Duration { return time.Duration(d) } +func (d Duration) MarshalJSON() ([]byte, error) { return json.Marshal(time.Duration(d).String()) } + +type Config struct { + Server ServerConfig `json:"server"` + Auth AuthConfig `json:"auth"` + Scheduler SchedulerConfig `json:"scheduler"` + Quota QuotaConfig `json:"quota"` + Cost CostConfig `json:"cost"` + Workers []WorkerConfig `json:"workers"` + Usage UsageConfig `json:"usage"` + Native NativeConfig `json:"native"` + UI UIConfig `json:"ui"` + PublicDashboard PublicDashboardConfig `json:"public_dashboard"` + Infrastructure InfrastructureConfig `json:"infrastructure"` + ModelCapabilities ModelCapabilitiesConfig `json:"model_capabilities"` + Routing RoutingConfig `json:"routing"` + Reliability ReliabilityConfig `json:"reliability"` + ServiceClasses ServiceClassesConfig `json:"service_classes"` + AutoTuning AutoTuningConfig `json:"auto_tuning"` + OpenTelemetry OpenTelemetryConfig `json:"opentelemetry"` + WarmModels WarmModelsConfig `json:"warm_models"` + Alerts AlertsConfig `json:"alerts"` + Conversations ConversationsConfig `json:"conversations"` + BatchJobs BatchJobsConfig `json:"batch_jobs"` + ModelAliases map[string]ModelAliasConfig `json:"model_aliases"` + ModelAccess ModelAccessConfig `json:"model_access"` + Storage StorageConfig `json:"storage"` +} + +type ServerConfig struct { + Listen string `json:"listen"` + ReadHeaderTimeout Duration `json:"read_header_timeout"` + IdleTimeout Duration `json:"idle_timeout"` + MaxRequestDuration Duration `json:"max_request_duration"` + MaxBodyBytes int64 `json:"max_body_bytes"` + MetricsPublic bool `json:"metrics_public"` + TLSCert string `json:"tls_cert"` + TLSKey string `json:"tls_key"` +} + +type AuthConfig struct { + OIDC OIDCConfig `json:"oidc"` + APIKeys []APIKeyConfig `json:"api_keys"` + IPBypass []IPBypassConfig `json:"ip_bypass"` + TrustedProxies []string `json:"trusted_proxies"` + IPBypassUseForwardedIP bool `json:"ip_bypass_use_forwarded_ip,omitempty"` +} + +type OIDCConfig struct { + Enabled bool `json:"enabled"` + Issuer string `json:"issuer"` + Audience string `json:"audience"` + TenantClaim string `json:"tenant_claim"` + ApplicationClaim string `json:"application_claim"` + GroupsClaim string `json:"groups_claim"` + AdminGroups []string `json:"admin_groups"` + ClockSkew Duration `json:"clock_skew"` + JWKSRefreshMinInterval Duration `json:"jwks_refresh_min_interval"` + AllowedAlgorithms []string `json:"allowed_algorithms"` +} + +type APIKeyConfig struct { + Name string `json:"name"` + Key string `json:"key"` + Tenant string `json:"tenant"` + Subject string `json:"subject"` + Application string `json:"application"` + Scopes []string `json:"scopes"` + AllowedModels []string `json:"allowed_models,omitempty"` + DeniedModels []string `json:"denied_models,omitempty"` + ServiceClass string `json:"service_class,omitempty"` +} + +type IPBypassConfig struct { + CIDRs []string `json:"cidrs"` + Tenant string `json:"tenant"` + Subject string `json:"subject"` + Application string `json:"application"` + Scopes []string `json:"scopes"` +} + +type SchedulerConfig struct { + GlobalConcurrency int `json:"global_concurrency"` + MaxQueue int `json:"max_queue"` + MaxQueuePerActor int `json:"max_queue_per_actor"` + QueueTimeout Duration `json:"queue_timeout"` + DefaultTenantWeight float64 `json:"default_tenant_weight"` + DefaultActorWeight float64 `json:"default_actor_weight"` + Policies map[string]TenantPolicy `json:"policies"` + ComputePaths []string `json:"compute_paths"` +} + +type TenantPolicy struct { + TenantWeight float64 `json:"tenant_weight"` + ActorWeight float64 `json:"actor_weight"` + ActorCreditsPerMinute float64 `json:"actor_credits_per_minute"` + ActorBurstCredits float64 `json:"actor_burst_credits"` + TenantCreditsPerMinute float64 `json:"tenant_credits_per_minute"` + TenantBurstCredits float64 `json:"tenant_burst_credits"` +} + +type QuotaConfig struct { + Enabled bool `json:"enabled"` +} + +type CostConfig struct { + Default ModelRate `json:"default"` + Models map[string]ModelRate `json:"models"` + DefaultMaxOutputTokens int `json:"default_max_output_tokens"` +} + +type ModelRate struct { + InputCreditsPer1K float64 `json:"input_credits_per_1k"` + CachedInputFactor float64 `json:"cached_input_factor"` + OutputCreditsPer1K float64 `json:"output_credits_per_1k"` + ComputeCreditsPerSecond float64 `json:"compute_credits_per_second"` + ExpectedPromptTokensPerSecond float64 `json:"expected_prompt_tokens_per_second"` + ExpectedOutputTokensPerSecond float64 `json:"expected_output_tokens_per_second"` +} + +type WorkerConfig struct { + Name string `json:"name"` + URL string `json:"url"` + MaxConcurrent int `json:"max_concurrent"` + ModelConcurrency map[string]int `json:"model_concurrency,omitempty"` + ContextLimits map[string]int64 `json:"context_limits,omitempty"` + DefaultContextTokens int64 `json:"default_context_tokens,omitempty"` + ModelPlacement ModelPlacementRule `json:"model_placement,omitempty"` + HealthInterval Duration `json:"health_interval"` + MemoryCapacityBytes int64 `json:"memory_capacity_bytes,omitempty"` + VRAMCapacityBytes int64 `json:"vram_capacity_bytes,omitempty"` + LocalSystemStats bool `json:"local_system_stats,omitempty"` + TelemetryURL string `json:"telemetry_url,omitempty"` + NVIDIASMI bool `json:"nvidia_smi,omitempty"` + NVIDIAGPU string `json:"nvidia_gpu,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +// ModelPlacementRule is a hard routing constraint applied before adaptive +// worker scoring. "allow_all" permits every model unless a deny rule matches; +// "whitelist" permits only models matched by allowed_models. Exact matches +// are more specific than prefix rules ("gemma4:*") and therefore can be used +// as exceptions to broader rules. At equal specificity, deny wins. +type ModelPlacementRule struct { + Mode string `json:"mode,omitempty"` // allow_all | whitelist + AllowedModels []string `json:"allowed_models,omitempty"` + DeniedModels []string `json:"denied_models,omitempty"` +} + +type UsageConfig struct { + JournalDir string `json:"journal_dir"` + Buffer int `json:"buffer"` + FlushInterval Duration `json:"flush_interval"` + Retention UsageRetentionConfig `json:"retention"` +} + +type UsageRetentionConfig struct { + DetailDays int `json:"detail_days"` + DailyDays int `json:"daily_days"` + MonthlyMonths int `json:"monthly_months"` + CompactionInterval Duration `json:"compaction_interval"` +} + +type NativeConfig struct { + ManagementRequiresAdmin bool `json:"management_requires_admin"` + ControlWorker string `json:"control_worker"` +} + +type UIConfig struct { + Enabled bool `json:"enabled"` + Path string `json:"path"` + Title string `json:"title"` + RecentEvents int `json:"recent_events"` + SessionSecret string `json:"session_secret"` + SecureCookies bool `json:"secure_cookies"` + OIDC UIOIDCConfig `json:"oidc"` +} + +type UIOIDCConfig struct { + Enabled bool `json:"enabled"` + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + Scopes []string `json:"scopes"` + RedirectURL string `json:"redirect_url"` +} + +// PublicDashboardConfig controls the optional unauthenticated, strictly +// read-only status dashboard. The public API is deliberately sanitized and +// never exposes tenant/actor/application identities, worker URLs, labels, +// error strings, API keys, policies, quotas or persistent-state paths. +type PublicDashboardConfig struct { + Enabled bool `json:"enabled"` + Path string `json:"path"` + Title string `json:"title"` + Subtitle string `json:"subtitle"` + RefreshInterval Duration `json:"refresh_interval"` + MaxLiveRequests int `json:"max_live_requests"` + ShowWorkerNames bool `json:"show_worker_names"` + ShowModelNames bool `json:"show_model_names"` + ShowResourceMetrics bool `json:"show_resource_metrics"` + WorkerDisplayNames map[string]string `json:"worker_display_names,omitempty"` +} + +type InfrastructureConfig struct { + NodeID string `json:"node_id"` + NodeName string `json:"node_name"` + RefreshInterval Duration `json:"refresh_interval"` + MaxRequests int `json:"max_requests"` +} + +// ModelCapabilitiesConfig controls model metadata discovery through Ollama +// /api/show and request preflight. Unknown capability metadata is allowed by +// default so a temporarily unavailable metadata call never takes inference +// offline. +type ModelCapabilitiesConfig struct { + Mode string `json:"mode"` // enforce | observe | off + CacheTTL Duration `json:"cache_ttl"` + ContextGuard string `json:"context_guard"` // reject | warn | off + Context ContextPolicyConfig `json:"context"` +} + +// ContextPolicyConfig controls how the gateway turns a model's theoretical +// context window into a safe, routable effective context window. The defaults +// intentionally favor predictable memory use on mixed/local Ollama fleets. +// Set max_requested_tokens to -1 only when an operator explicitly wants to +// remove the gateway-side cap. Set default_worker_tokens to -1 only when the +// gateway should fall back to the model maximum for unloaded models without a +// Modelfile num_ctx. +type ContextPolicyConfig struct { + MaxRequestedTokens int64 `json:"max_requested_tokens"` + DefaultWorkerTokens int64 `json:"default_worker_tokens"` + EstimationMarginPercent float64 `json:"estimation_margin_percent"` + VisionReserveTokensPerImage int64 `json:"vision_reserve_tokens_per_image"` +} + +// RoutingConfig tunes the local worker scoring function. Scores are relative; +// lower is better. Throughput is learned from completed requests in memory. + +// ReliabilityConfig controls worker circuit breaking and safe pre-stream retries. +// Retries are only attempted when the upstream request failed before any +// response headers/body were committed to the client. +type ReliabilityConfig struct { + Enabled bool `json:"enabled"` + FailureThreshold int `json:"failure_threshold"` + OpenDuration Duration `json:"open_duration"` + RetryAttempts int `json:"retry_attempts"` + RetryBackoff Duration `json:"retry_backoff"` +} + +// ModelAliasConfig exposes a stable virtual model name to clients and resolves +// it to the first currently routable real model in Models. +type ModelAliasConfig struct { + Models []string `json:"models"` + RequiredCapabilities []string `json:"required_capabilities,omitempty"` + Visible *bool `json:"visible,omitempty"` +} + +// ModelAccessRule controls which models an identity may request. Exact names +// and one trailing '*' wildcard are supported. At equal specificity deny wins. +type ModelAccessRule struct { + Mode string `json:"mode,omitempty"` // allow_all | whitelist + AllowedModels []string `json:"allowed_models,omitempty"` + DeniedModels []string `json:"denied_models,omitempty"` +} + +type ModelAccessConfig struct { + Default ModelAccessRule `json:"default"` + Tenants map[string]ModelAccessRule `json:"tenants,omitempty"` +} + +// ServiceClassConfig provides a request-level QoS hint inside the tenant fairness boundary. +type ServiceClassConfig struct { + Weight float64 `json:"weight"` + MaxQueueWait Duration `json:"max_queue_wait"` + MaxConcurrent int `json:"max_concurrent"` +} + +type ServiceClassesConfig struct { + Default string `json:"default"` + Header string `json:"header"` + OverrideScope string `json:"override_scope"` + Classes map[string]ServiceClassConfig `json:"classes"` +} + +// AutoTuningConfig controls the explicit admin-triggered benchmark workflow. +// Auto tuning never changes production concurrency unless Apply is called by an admin. +type AutoTuningConfig struct { + Enabled bool `json:"enabled"` + MaxConcurrency int `json:"max_concurrency"` + SamplesPerLevel int `json:"samples_per_level"` + MaxTokens int `json:"max_tokens"` + Timeout Duration `json:"timeout"` + Prompt string `json:"prompt"` + TTFTWeight float64 `json:"ttft_weight"` + ThroughputWeight float64 `json:"throughput_weight"` +} + +type BatchJobsConfig struct { + Enabled bool `json:"enabled"` + Retention Duration `json:"retention"` + MaxJobs int `json:"max_jobs"` + MaxConcurrent int `json:"max_concurrent"` + MaxInputBytes int64 `json:"max_input_bytes"` +} + +type ConversationsConfig struct { + Enabled bool `json:"enabled"` + EncryptionKey string `json:"encryption_key,omitempty"` + Retention Duration `json:"retention"` + MaxEntries int `json:"max_entries"` + MaxContentBytes int64 `json:"max_content_bytes"` +} + +type OpenTelemetryConfig struct { + Enabled bool `json:"enabled"` + Endpoint string `json:"endpoint"` + Headers map[string]string `json:"headers,omitempty"` + ServiceName string `json:"service_name"` + ServiceVersion string `json:"service_version,omitempty"` + SampleRatio float64 `json:"sample_ratio"` + BatchSize int `json:"batch_size"` + FlushInterval Duration `json:"flush_interval"` + CaptureContent bool `json:"capture_content"` +} + +// WarmModelsConfig controls proactive model residency and idle unloading. +// Policies use exact model names or a single trailing '*' prefix wildcard. +type WarmModelsConfig struct { + Enabled bool `json:"enabled"` + ReconcileInterval Duration `json:"reconcile_interval"` + OperationTimeout Duration `json:"operation_timeout"` + Policies map[string]WarmModelPolicy `json:"policies"` +} + +type WarmModelPolicy struct { + Class string `json:"class"` // hot | warm | cold + Workers []string `json:"workers,omitempty"` + Replicas int `json:"replicas,omitempty"` + Preload bool `json:"preload,omitempty"` + IdleTimeout Duration `json:"idle_timeout,omitempty"` +} + +// AlertsConfig evaluates bounded operational conditions and can deliver a +// signed generic webhook. Payloads never include prompts or model output. +type AlertsConfig struct { + Enabled bool `json:"enabled"` + EvaluationInterval Duration `json:"evaluation_interval"` + Cooldown Duration `json:"cooldown"` + HistoryLimit int `json:"history_limit"` + WebhookTimeout Duration `json:"webhook_timeout"` + WebhookMaxConcurrent int `json:"webhook_max_concurrent"` + WebhookQueue int `json:"webhook_queue"` + WebhookRetryAttempts int `json:"webhook_retry_attempts"` + WebhookRetryBackoff Duration `json:"webhook_retry_backoff"` + Webhooks []WebhookConfig `json:"webhooks"` + Thresholds AlertThresholds `json:"thresholds"` +} + +type WebhookConfig struct { + Name string `json:"name"` + URL string `json:"url"` + Secret string `json:"secret,omitempty"` + Enabled bool `json:"enabled"` +} + +type AlertThresholds struct { + WorkerDownFor Duration `json:"worker_down_for"` + CircuitOpen bool `json:"circuit_open"` + QueueDepth int `json:"queue_depth"` + QueueWait Duration `json:"queue_wait"` + VRAMPercent float64 `json:"vram_percent"` + StorageBytes int64 `json:"storage_bytes"` + QuotaRemainingPct float64 `json:"quota_remaining_percent"` + OOM bool `json:"oom"` +} + +// StorageConfig defines the local durable state directory. The gateway keeps +// active scheduling state in memory; only restart-worthy state is persisted. +type StorageConfig struct { + DataDir string `json:"data_dir"` + ConfigFile string `json:"config_file"` + APIKeysFile string `json:"api_keys_file"` + PoliciesFile string `json:"policies_file"` + MetricsFile string `json:"metrics_file"` + QuotaFile string `json:"quota_file"` + WorkerPerformanceFile string `json:"worker_performance_file"` + ModelPlacementFile string `json:"model_placement_file"` + WorkerStateFile string `json:"worker_state_file"` + AutoTuneFile string `json:"auto_tune_file"` + WarmModelsFile string `json:"warm_models_file"` + AlertsFile string `json:"alerts_file"` + ConversationsFile string `json:"conversations_file"` + BatchJobsFile string `json:"batch_jobs_file"` + BatchJobsDir string `json:"batch_jobs_dir"` + FlushInterval Duration `json:"flush_interval"` +} + +type RoutingConfig struct { + LoadedBonus float64 `json:"loaded_bonus"` + InstalledBonus float64 `json:"installed_bonus"` + ThroughputBonus float64 `json:"throughput_bonus"` + VRAMPressurePenalty float64 `json:"vram_pressure_penalty"` + GPUUtilizationPenalty float64 `json:"gpu_utilization_penalty"` + AvoidVRAMPercent float64 `json:"avoid_vram_percent"` +} + +func Load(path string) (*Config, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return ParseBytes(b) +} + +// ParseBytes parses a complete gateway configuration using the same strict +// validation used at process startup. Environment variables are expanded so +// persisted configurations can continue to reference deployment secrets. +func ParseBytes(b []byte) (*Config, error) { + b = []byte(os.ExpandEnv(string(b))) + var c Config + dec := json.NewDecoder(strings.NewReader(string(b))) + dec.DisallowUnknownFields() + if err := dec.Decode(&c); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + c.defaults() + if err := c.Validate(); err != nil { + return nil, err + } + return &c, nil +} + +func (c *Config) defaults() { + if c.Server.Listen == "" { + c.Server.Listen = ":8080" + } + if c.Server.ReadHeaderTimeout == 0 { + c.Server.ReadHeaderTimeout = Duration(10 * time.Second) + } + if c.Server.IdleTimeout == 0 { + c.Server.IdleTimeout = Duration(2 * time.Minute) + } + if c.Server.MaxRequestDuration == 0 { + c.Server.MaxRequestDuration = Duration(30 * time.Minute) + } + if c.Server.MaxBodyBytes == 0 { + c.Server.MaxBodyBytes = 64 << 20 + } + if c.Scheduler.GlobalConcurrency <= 0 { + for _, w := range c.Workers { + c.Scheduler.GlobalConcurrency += max(1, w.MaxConcurrent) + } + if c.Scheduler.GlobalConcurrency == 0 { + c.Scheduler.GlobalConcurrency = 1 + } + } + if c.Scheduler.MaxQueue <= 0 { + c.Scheduler.MaxQueue = 1024 + } + if c.Scheduler.MaxQueuePerActor <= 0 { + c.Scheduler.MaxQueuePerActor = 64 + } + if c.Scheduler.QueueTimeout == 0 { + c.Scheduler.QueueTimeout = Duration(10 * time.Minute) + } + if c.Scheduler.DefaultTenantWeight <= 0 { + c.Scheduler.DefaultTenantWeight = 1 + } + if c.Scheduler.DefaultActorWeight <= 0 { + c.Scheduler.DefaultActorWeight = 1 + } + if len(c.Scheduler.ComputePaths) == 0 { + c.Scheduler.ComputePaths = []string{"/api/generate", "/api/chat", "/api/embed", "/api/embeddings", "/v1/chat/completions", "/v1/completions", "/v1/embeddings", "/v1/responses", "/v1/messages"} + } + if c.Cost.Default.InputCreditsPer1K <= 0 { + c.Cost.Default.InputCreditsPer1K = 1 + } + if c.Cost.Default.OutputCreditsPer1K <= 0 { + c.Cost.Default.OutputCreditsPer1K = 3 + } + if c.Cost.Default.CachedInputFactor <= 0 { + c.Cost.Default.CachedInputFactor = 1 + } + if c.Cost.DefaultMaxOutputTokens <= 0 { + c.Cost.DefaultMaxOutputTokens = 1024 + } + if c.Storage.DataDir == "" { + c.Storage.DataDir = "./data" + } + if c.Storage.ConfigFile == "" { + c.Storage.ConfigFile = "gateway-config.json" + } + if c.Storage.APIKeysFile == "" { + c.Storage.APIKeysFile = "api-keys.json" + } + if c.Storage.PoliciesFile == "" { + c.Storage.PoliciesFile = "policies.json" + } + if c.Storage.MetricsFile == "" { + c.Storage.MetricsFile = "metrics.json" + } + if c.Storage.QuotaFile == "" { + c.Storage.QuotaFile = "quota.json" + } + if c.Storage.WorkerPerformanceFile == "" { + c.Storage.WorkerPerformanceFile = "worker-performance.json" + } + if c.Storage.ModelPlacementFile == "" { + c.Storage.ModelPlacementFile = "model-placement.json" + } + if c.Storage.WorkerStateFile == "" { + c.Storage.WorkerStateFile = "worker-state.json" + } + if c.Storage.AutoTuneFile == "" { + c.Storage.AutoTuneFile = "auto-tune.json" + } + if c.Storage.WarmModelsFile == "" { + c.Storage.WarmModelsFile = "warm-models.json" + } + if c.Storage.AlertsFile == "" { + c.Storage.AlertsFile = "alerts.json" + } + if c.Storage.ConversationsFile == "" { + c.Storage.ConversationsFile = "conversations.enc.json" + } + if c.Storage.BatchJobsFile == "" { + c.Storage.BatchJobsFile = "batch-jobs.json" + } + if c.Storage.BatchJobsDir == "" { + c.Storage.BatchJobsDir = "batch" + } + if c.Storage.FlushInterval == 0 { + c.Storage.FlushInterval = Duration(10 * time.Second) + } + if c.Usage.JournalDir == "" { + c.Usage.JournalDir = filepath.Join(c.Storage.DataDir, "usage") + } + if c.Usage.Buffer <= 0 { + c.Usage.Buffer = 8192 + } + if c.Usage.FlushInterval == 0 { + c.Usage.FlushInterval = Duration(time.Second) + } + if c.Usage.Retention.DetailDays <= 0 { + c.Usage.Retention.DetailDays = 30 + } + if c.Usage.Retention.DailyDays <= 0 { + c.Usage.Retention.DailyDays = 400 + } + if c.Usage.Retention.CompactionInterval == 0 { + c.Usage.Retention.CompactionInterval = Duration(6 * time.Hour) + } + if c.Auth.OIDC.ClockSkew == 0 { + c.Auth.OIDC.ClockSkew = Duration(60 * time.Second) + } + if c.Auth.OIDC.JWKSRefreshMinInterval == 0 { + c.Auth.OIDC.JWKSRefreshMinInterval = Duration(10 * time.Second) + } + if c.UI.Path == "" { + c.UI.Path = "/admin" + } + if !strings.HasPrefix(c.UI.Path, "/") { + c.UI.Path = "/" + c.UI.Path + } + c.UI.Path = strings.TrimRight(c.UI.Path, "/") + if c.UI.Title == "" { + c.UI.Title = "Ollama Fair Gateway" + } + if c.UI.RecentEvents <= 0 { + c.UI.RecentEvents = 10000 + } + if c.UI.OIDC.Enabled && len(c.UI.OIDC.Scopes) == 0 { + c.UI.OIDC.Scopes = []string{"openid", "profile", "email"} + } + if c.PublicDashboard.Path == "" { + c.PublicDashboard.Path = "/status" + } + if !strings.HasPrefix(c.PublicDashboard.Path, "/") { + c.PublicDashboard.Path = "/" + c.PublicDashboard.Path + } + c.PublicDashboard.Path = strings.TrimRight(c.PublicDashboard.Path, "/") + if c.PublicDashboard.Title == "" { + c.PublicDashboard.Title = c.UI.Title + if c.PublicDashboard.Title == "" { + c.PublicDashboard.Title = "Ollama Gateway Status" + } + } + if c.PublicDashboard.Subtitle == "" { + c.PublicDashboard.Subtitle = "Live-Auslastung und Infrastruktur" + } + if c.PublicDashboard.RefreshInterval == 0 { + c.PublicDashboard.RefreshInterval = Duration(2 * time.Second) + } + if c.PublicDashboard.MaxLiveRequests <= 0 { + c.PublicDashboard.MaxLiveRequests = 64 + } + if c.Infrastructure.RefreshInterval == 0 { + c.Infrastructure.RefreshInterval = Duration(250 * time.Millisecond) + } + if c.Infrastructure.MaxRequests <= 0 { + c.Infrastructure.MaxRequests = 256 + } + if len(c.Auth.OIDC.AllowedAlgorithms) == 0 { + c.Auth.OIDC.AllowedAlgorithms = []string{"RS256", "PS256", "ES256", "EdDSA"} + } + if c.ModelCapabilities.Mode == "" { + c.ModelCapabilities.Mode = "enforce" + } + if c.ModelCapabilities.CacheTTL == 0 { + c.ModelCapabilities.CacheTTL = Duration(10 * time.Minute) + } + if c.ModelCapabilities.ContextGuard == "" { + c.ModelCapabilities.ContextGuard = "reject" + } + if c.ModelCapabilities.Context.MaxRequestedTokens == 0 { + c.ModelCapabilities.Context.MaxRequestedTokens = 32768 + } + if c.ModelCapabilities.Context.DefaultWorkerTokens == 0 { + c.ModelCapabilities.Context.DefaultWorkerTokens = 4096 + } + if c.ModelCapabilities.Context.EstimationMarginPercent == 0 { + c.ModelCapabilities.Context.EstimationMarginPercent = 15 + } + if c.ModelCapabilities.Context.VisionReserveTokensPerImage == 0 { + c.ModelCapabilities.Context.VisionReserveTokensPerImage = 2048 + } + if c.Routing.LoadedBonus == 0 { + c.Routing.LoadedBonus = 60 + } + if c.Routing.InstalledBonus == 0 { + c.Routing.InstalledBonus = 30 + } + if c.Routing.ThroughputBonus == 0 { + c.Routing.ThroughputBonus = 20 + } + if c.Routing.VRAMPressurePenalty == 0 { + c.Routing.VRAMPressurePenalty = 35 + } + if c.Routing.GPUUtilizationPenalty == 0 { + c.Routing.GPUUtilizationPenalty = 10 + } + if c.Routing.AvoidVRAMPercent == 0 { + c.Routing.AvoidVRAMPercent = 97 + } + if c.Reliability.FailureThreshold <= 0 { + c.Reliability.FailureThreshold = 3 + } + if c.Reliability.OpenDuration == 0 { + c.Reliability.OpenDuration = Duration(30 * time.Second) + } + if c.Reliability.RetryAttempts <= 0 { + c.Reliability.RetryAttempts = 2 + } + if c.Reliability.RetryBackoff == 0 { + c.Reliability.RetryBackoff = Duration(50 * time.Millisecond) + } + if c.ServiceClasses.Default == "" { + c.ServiceClasses.Default = "interactive" + } + if c.ServiceClasses.Header == "" { + c.ServiceClasses.Header = "X-Gateway-Service-Class" + } + if c.ServiceClasses.OverrideScope == "" { + c.ServiceClasses.OverrideScope = "gateway:service-class" + } + if len(c.ServiceClasses.Classes) == 0 { + c.ServiceClasses.Classes = map[string]ServiceClassConfig{ + "interactive": {Weight: 4, MaxQueueWait: Duration(30 * time.Second)}, + "system": {Weight: 8, MaxQueueWait: Duration(30 * time.Second), MaxConcurrent: 1}, + "background": {Weight: 1, MaxQueueWait: Duration(10 * time.Minute), MaxConcurrent: 1}, + "batch": {Weight: .5, MaxQueueWait: Duration(30 * time.Minute), MaxConcurrent: 1}, + } + } + for name, sc := range c.ServiceClasses.Classes { + if sc.Weight <= 0 { + sc.Weight = 1 + } + if sc.MaxQueueWait == 0 { + sc.MaxQueueWait = c.Scheduler.QueueTimeout + } + c.ServiceClasses.Classes[name] = sc + } + if c.AutoTuning.MaxConcurrency <= 0 { + c.AutoTuning.MaxConcurrency = 4 + } + if c.AutoTuning.SamplesPerLevel <= 0 { + c.AutoTuning.SamplesPerLevel = 2 + } + if c.AutoTuning.MaxTokens <= 0 { + c.AutoTuning.MaxTokens = 96 + } + if c.AutoTuning.Timeout == 0 { + c.AutoTuning.Timeout = Duration(10 * time.Minute) + } + if c.AutoTuning.Prompt == "" { + c.AutoTuning.Prompt = "Write a concise explanation of why deterministic benchmarking matters for local LLM serving." + } + if c.AutoTuning.TTFTWeight <= 0 { + c.AutoTuning.TTFTWeight = .25 + } + if c.AutoTuning.ThroughputWeight <= 0 { + c.AutoTuning.ThroughputWeight = 1 + } + if c.OpenTelemetry.ServiceName == "" { + c.OpenTelemetry.ServiceName = "ollama-fair-gateway" + } + if c.OpenTelemetry.SampleRatio <= 0 || c.OpenTelemetry.SampleRatio > 1 { + c.OpenTelemetry.SampleRatio = 1 + } + if c.OpenTelemetry.BatchSize <= 0 { + c.OpenTelemetry.BatchSize = 128 + } + if c.OpenTelemetry.FlushInterval == 0 { + c.OpenTelemetry.FlushInterval = Duration(2 * time.Second) + } + if c.WarmModels.ReconcileInterval == 0 { + c.WarmModels.ReconcileInterval = Duration(30 * time.Second) + } + if c.WarmModels.OperationTimeout == 0 { + c.WarmModels.OperationTimeout = Duration(2 * time.Minute) + } + for pattern, wp := range c.WarmModels.Policies { + if wp.Class == "" { + wp.Class = "warm" + } + if wp.Replicas <= 0 { + wp.Replicas = 1 + } + if wp.IdleTimeout == 0 { + if wp.Class == "cold" { + wp.IdleTimeout = Duration(5 * time.Minute) + } else { + wp.IdleTimeout = Duration(30 * time.Minute) + } + } + c.WarmModels.Policies[pattern] = wp + } + if c.BatchJobs.Retention == 0 { + c.BatchJobs.Retention = Duration(7 * 24 * time.Hour) + } + if c.BatchJobs.MaxJobs <= 0 { + c.BatchJobs.MaxJobs = 1000 + } + if c.BatchJobs.MaxConcurrent <= 0 { + c.BatchJobs.MaxConcurrent = 1 + } + if c.BatchJobs.MaxInputBytes <= 0 { + c.BatchJobs.MaxInputBytes = 16 << 20 + } + if c.Conversations.Retention == 0 { + c.Conversations.Retention = Duration(24 * time.Hour) + } + if c.Conversations.MaxEntries <= 0 { + c.Conversations.MaxEntries = 1000 + } + if c.Conversations.MaxContentBytes <= 0 { + c.Conversations.MaxContentBytes = 2 << 20 + } + if c.Alerts.EvaluationInterval == 0 { + c.Alerts.EvaluationInterval = Duration(15 * time.Second) + } + if c.Alerts.Cooldown == 0 { + c.Alerts.Cooldown = Duration(5 * time.Minute) + } + if c.Alerts.HistoryLimit <= 0 { + c.Alerts.HistoryLimit = 500 + } + if c.Alerts.WebhookTimeout == 0 { + c.Alerts.WebhookTimeout = Duration(5 * time.Second) + } + if c.Alerts.WebhookMaxConcurrent <= 0 { + c.Alerts.WebhookMaxConcurrent = 4 + } + if c.Alerts.WebhookQueue <= 0 { + c.Alerts.WebhookQueue = 1024 + } + if c.Alerts.WebhookRetryAttempts <= 0 { + c.Alerts.WebhookRetryAttempts = 3 + } + if c.Alerts.WebhookRetryBackoff == 0 { + c.Alerts.WebhookRetryBackoff = Duration(500 * time.Millisecond) + } + if c.Alerts.Thresholds.WorkerDownFor == 0 { + c.Alerts.Thresholds.WorkerDownFor = Duration(30 * time.Second) + } + if c.Alerts.Thresholds.VRAMPercent == 0 { + c.Alerts.Thresholds.VRAMPercent = 95 + } + if c.Alerts.Thresholds.QuotaRemainingPct == 0 { + c.Alerts.Thresholds.QuotaRemainingPct = 10 + } + if c.ModelAccess.Default.Mode == "" { + c.ModelAccess.Default.Mode = "allow_all" + } + for i := range c.Workers { + if c.Workers[i].Name == "" { + c.Workers[i].Name = fmt.Sprintf("worker-%d", i+1) + } + if c.Workers[i].MaxConcurrent <= 0 { + c.Workers[i].MaxConcurrent = 1 + } + if c.Workers[i].HealthInterval == 0 { + c.Workers[i].HealthInterval = Duration(5 * time.Second) + } + if c.Workers[i].ModelPlacement.Mode == "" { + c.Workers[i].ModelPlacement.Mode = "allow_all" + } + } +} + +func pathPrefixesOverlap(a, b string) bool { + a = strings.TrimRight(a, "/") + b = strings.TrimRight(b, "/") + if a == "" || b == "" { + return false + } + return a == b || strings.HasPrefix(a, b+"/") || strings.HasPrefix(b, a+"/") +} + +func (c *Config) Validate() error { + if len(c.Workers) == 0 { + return errors.New("at least one worker is required") + } + names := map[string]bool{} + for _, w := range c.Workers { + if w.URL == "" { + return fmt.Errorf("worker %q has empty url", w.Name) + } + if names[w.Name] { + return fmt.Errorf("duplicate worker name %q", w.Name) + } + names[w.Name] = true + } + if c.Auth.OIDC.Enabled && (c.Auth.OIDC.Issuer == "" || c.Auth.OIDC.Audience == "") { + return errors.New("auth.oidc.issuer and auth.oidc.audience are required when OIDC is enabled") + } + if !c.Auth.OIDC.Enabled && len(c.Auth.APIKeys) == 0 && len(c.Auth.IPBypass) == 0 { + return errors.New("no authentication method configured") + } + for _, p := range c.Auth.TrustedProxies { + if _, _, err := net.ParseCIDR(p); err != nil { + return fmt.Errorf("invalid trusted proxy CIDR %q: %w", p, err) + } + } + for _, b := range c.Auth.IPBypass { + if b.Tenant == "" { + return errors.New("ip_bypass tenant must not be empty") + } + for _, s := range b.CIDRs { + if _, _, err := net.ParseCIDR(s); err != nil { + return fmt.Errorf("invalid bypass CIDR %q: %w", s, err) + } + } + } + if c.Infrastructure.RefreshInterval.Value() < 50*time.Millisecond { + return errors.New("infrastructure.refresh_interval must be at least 50ms") + } + if c.PublicDashboard.Path == "" || c.PublicDashboard.Path == "/" { + return errors.New("public_dashboard.path must be a non-root path") + } + if c.PublicDashboard.Enabled { + if pathPrefixesOverlap(c.PublicDashboard.Path, c.UI.Path) { + return errors.New("public_dashboard.path must not overlap ui.path") + } + for _, reserved := range []string{"/healthz", "/readyz", "/metrics", "/api", "/v1", "/gateway"} { + if pathPrefixesOverlap(c.PublicDashboard.Path, reserved) { + return errors.New("public_dashboard.path conflicts with a reserved gateway path") + } + } + if c.PublicDashboard.RefreshInterval.Value() < time.Second { + return errors.New("public_dashboard.refresh_interval must be at least 1s") + } + if c.PublicDashboard.MaxLiveRequests < 1 || c.PublicDashboard.MaxLiveRequests > 512 { + return errors.New("public_dashboard.max_live_requests must be between 1 and 512") + } + } + switch c.ModelCapabilities.Mode { + case "enforce", "observe", "off": + default: + return errors.New("model_capabilities.mode must be enforce, observe, or off") + } + switch c.ModelCapabilities.ContextGuard { + case "reject", "warn", "off": + default: + return errors.New("model_capabilities.context_guard must be reject, warn, or off") + } + if c.ModelCapabilities.CacheTTL.Value() < time.Second { + return errors.New("model_capabilities.cache_ttl must be at least 1s") + } + if c.ModelCapabilities.Context.MaxRequestedTokens < -1 { + return errors.New("model_capabilities.context.max_requested_tokens must be -1 or >= 1") + } + if c.ModelCapabilities.Context.DefaultWorkerTokens < -1 { + return errors.New("model_capabilities.context.default_worker_tokens must be -1 or >= 1") + } + if c.ModelCapabilities.Context.EstimationMarginPercent < 0 || c.ModelCapabilities.Context.EstimationMarginPercent > 100 { + return errors.New("model_capabilities.context.estimation_margin_percent must be between 0 and 100") + } + if c.ModelCapabilities.Context.VisionReserveTokensPerImage < 0 { + return errors.New("model_capabilities.context.vision_reserve_tokens_per_image must be >= 0") + } + for _, w := range c.Workers { + if w.DefaultContextTokens < 0 { + return fmt.Errorf("worker %s default_context_tokens must be >= 0", w.Name) + } + for pattern, limit := range w.ContextLimits { + if strings.TrimSpace(pattern) == "" || limit <= 0 { + return fmt.Errorf("worker %s context_limits entries require a non-empty pattern and positive token limit", w.Name) + } + } + } + if c.Routing.AvoidVRAMPercent < 0 || c.Routing.AvoidVRAMPercent > 100 { + return errors.New("routing.avoid_vram_percent must be between 0 and 100") + } + if strings.TrimSpace(c.Storage.DataDir) == "" { + return errors.New("storage.data_dir must not be empty") + } + for name, file := range map[string]string{ + "config_file": c.Storage.ConfigFile, "api_keys_file": c.Storage.APIKeysFile, + "policies_file": c.Storage.PoliciesFile, "metrics_file": c.Storage.MetricsFile, "quota_file": c.Storage.QuotaFile, + "worker_performance_file": c.Storage.WorkerPerformanceFile, "model_placement_file": c.Storage.ModelPlacementFile, "worker_state_file": c.Storage.WorkerStateFile, "auto_tune_file": c.Storage.AutoTuneFile, "warm_models_file": c.Storage.WarmModelsFile, "alerts_file": c.Storage.AlertsFile, "conversations_file": c.Storage.ConversationsFile, "batch_jobs_file": c.Storage.BatchJobsFile, "batch_jobs_dir": c.Storage.BatchJobsDir, + } { + if file == "" || filepath.IsAbs(file) || filepath.Base(file) != file || file == "." || file == ".." { + return fmt.Errorf("storage.%s must be a simple relative filename", name) + } + } + if c.Storage.FlushInterval.Value() < time.Second { + return errors.New("storage.flush_interval must be at least 1s") + } + if c.Usage.Retention.DetailDays < 1 { + return errors.New("usage.retention.detail_days must be at least 1") + } + if c.Usage.Retention.DailyDays < c.Usage.Retention.DetailDays { + return errors.New("usage.retention.daily_days must be >= detail_days") + } + if c.Usage.Retention.MonthlyMonths < 0 { + return errors.New("usage.retention.monthly_months must be >= 0 (0 keeps monthly rollups forever)") + } + if c.Usage.Retention.CompactionInterval.Value() < time.Minute { + return errors.New("usage.retention.compaction_interval must be at least 1m") + } + if c.Reliability.FailureThreshold < 1 { + return errors.New("reliability.failure_threshold must be >= 1") + } + if c.Reliability.OpenDuration.Value() < time.Second { + return errors.New("reliability.open_duration must be at least 1s") + } + if c.Reliability.RetryAttempts < 1 || c.Reliability.RetryAttempts > 5 { + return errors.New("reliability.retry_attempts must be between 1 and 5") + } + if c.Reliability.RetryBackoff.Value() < 0 { + return errors.New("reliability.retry_backoff must be >= 0") + } + if err := ValidateModelAccessRule(c.ModelAccess.Default); err != nil { + return fmt.Errorf("model_access.default: %w", err) + } + for tenant, rule := range c.ModelAccess.Tenants { + if strings.TrimSpace(tenant) == "" { + return errors.New("model_access.tenants contains empty tenant") + } + if err := ValidateModelAccessRule(rule); err != nil { + return fmt.Errorf("model_access.tenants[%q]: %w", tenant, err) + } + } + if _, ok := c.ServiceClasses.Classes[c.ServiceClasses.Default]; !ok { + return fmt.Errorf("service_classes.default %q is not defined", c.ServiceClasses.Default) + } + for name, sc := range c.ServiceClasses.Classes { + if strings.TrimSpace(name) == "" || sc.Weight <= 0 || sc.MaxConcurrent < 0 || sc.MaxQueueWait.Value() < 0 { + return fmt.Errorf("invalid service class %q", name) + } + } + for _, k := range c.Auth.APIKeys { + if cls := strings.TrimSpace(k.ServiceClass); cls != "" { + if _, ok := c.ServiceClasses.Classes[cls]; !ok { + return fmt.Errorf("auth.api_keys[%q].service_class %q is not defined", k.Name, cls) + } + } + } + if c.AutoTuning.MaxConcurrency < 1 || c.AutoTuning.MaxConcurrency > 32 || c.AutoTuning.SamplesPerLevel < 1 || c.AutoTuning.SamplesPerLevel > 20 || c.AutoTuning.MaxTokens < 1 { + return errors.New("auto_tuning limits are invalid") + } + if c.BatchJobs.Enabled { + if c.BatchJobs.Retention.Value() < time.Minute { + return errors.New("batch_jobs.retention must be at least 1m") + } + if c.BatchJobs.MaxJobs < 1 || c.BatchJobs.MaxJobs > 1000000 { + return errors.New("batch_jobs.max_jobs must be between 1 and 1000000") + } + if c.BatchJobs.MaxConcurrent < 1 || c.BatchJobs.MaxConcurrent > 32 { + return errors.New("batch_jobs.max_concurrent must be between 1 and 32") + } + if c.BatchJobs.MaxInputBytes < 1024 || c.BatchJobs.MaxInputBytes > 64<<20 { + return errors.New("batch_jobs.max_input_bytes must be between 1KiB and 64MiB") + } + if c.Server.MaxBodyBytes > 0 && c.BatchJobs.MaxInputBytes > c.Server.MaxBodyBytes { + return errors.New("batch_jobs.max_input_bytes must be <= server.max_body_bytes") + } + if _, ok := c.ServiceClasses.Classes["batch"]; len(c.ServiceClasses.Classes) > 0 && !ok { + return errors.New("batch_jobs.enabled requires service_classes.classes.batch") + } + } + if c.Conversations.Enabled { + if len(c.Conversations.EncryptionKey) < 32 { + return errors.New("conversations.encryption_key must contain at least 32 characters when conversations are enabled") + } + if c.Conversations.Retention.Value() < time.Minute { + return errors.New("conversations.retention must be at least 1m") + } + if c.Conversations.MaxEntries < 1 || c.Conversations.MaxEntries > 1000000 { + return errors.New("conversations.max_entries must be between 1 and 1000000") + } + if c.Conversations.MaxContentBytes < 1024 || c.Conversations.MaxContentBytes > 64<<20 { + return errors.New("conversations.max_content_bytes must be between 1KiB and 64MiB") + } + } + if c.OpenTelemetry.Enabled && strings.TrimSpace(c.OpenTelemetry.Endpoint) == "" { + return errors.New("opentelemetry.endpoint is required when enabled") + } + if c.WarmModels.ReconcileInterval.Value() < time.Second { + return errors.New("warm_models.reconcile_interval must be at least 1s") + } + if c.WarmModels.OperationTimeout.Value() < time.Second { + return errors.New("warm_models.operation_timeout must be at least 1s") + } + for pattern, wp := range c.WarmModels.Policies { + if err := validateSimpleModelPattern(pattern); err != nil { + return fmt.Errorf("warm_models.policies[%q]: %w", pattern, err) + } + switch wp.Class { + case "hot", "warm", "cold": + default: + return fmt.Errorf("warm_models.policies[%q].class must be hot, warm, or cold", pattern) + } + if wp.Replicas < 1 { + return fmt.Errorf("warm_models.policies[%q].replicas must be >= 1", pattern) + } + if wp.IdleTimeout.Value() < 0 { + return fmt.Errorf("warm_models.policies[%q].idle_timeout must be >= 0", pattern) + } + for _, wn := range wp.Workers { + if !names[wn] { + return fmt.Errorf("warm_models.policies[%q] references unknown worker %q", pattern, wn) + } + } + } + if c.Alerts.EvaluationInterval.Value() < time.Second { + return errors.New("alerts.evaluation_interval must be at least 1s") + } + if c.Alerts.Cooldown.Value() < 0 { + return errors.New("alerts.cooldown must be >= 0") + } + if c.Alerts.HistoryLimit < 1 || c.Alerts.HistoryLimit > 10000 { + return errors.New("alerts.history_limit must be between 1 and 10000") + } + if c.Alerts.WebhookTimeout.Value() < time.Second || c.Alerts.WebhookTimeout.Value() > 2*time.Minute { + return errors.New("alerts.webhook_timeout must be between 1s and 2m") + } + if c.Alerts.WebhookMaxConcurrent < 1 || c.Alerts.WebhookMaxConcurrent > 64 { + return errors.New("alerts.webhook_max_concurrent must be between 1 and 64") + } + if c.Alerts.WebhookQueue < 1 || c.Alerts.WebhookQueue > 100000 { + return errors.New("alerts.webhook_queue must be between 1 and 100000") + } + if c.Alerts.WebhookRetryAttempts < 1 || c.Alerts.WebhookRetryAttempts > 10 { + return errors.New("alerts.webhook_retry_attempts must be between 1 and 10") + } + if c.Alerts.WebhookRetryBackoff.Value() < 0 || c.Alerts.WebhookRetryBackoff.Value() > time.Minute { + return errors.New("alerts.webhook_retry_backoff must be between 0 and 1m") + } + if c.Alerts.Thresholds.QueueWait.Value() < 0 { + return errors.New("alerts.thresholds.queue_wait must be >= 0") + } + if c.Alerts.Thresholds.VRAMPercent < 0 || c.Alerts.Thresholds.VRAMPercent > 100 { + return errors.New("alerts.thresholds.vram_percent must be between 0 and 100") + } + if c.Alerts.Thresholds.QuotaRemainingPct < 0 || c.Alerts.Thresholds.QuotaRemainingPct > 100 { + return errors.New("alerts.thresholds.quota_remaining_percent must be between 0 and 100") + } + for i, wh := range c.Alerts.Webhooks { + if !wh.Enabled { + continue + } + if strings.TrimSpace(wh.URL) == "" { + return fmt.Errorf("alerts.webhooks[%d].url is required when enabled", i) + } + if !(strings.HasPrefix(wh.URL, "http://") || strings.HasPrefix(wh.URL, "https://")) { + return fmt.Errorf("alerts.webhooks[%d].url must be http(s)", i) + } + } + for alias, a := range c.ModelAliases { + if strings.TrimSpace(alias) == "" { + return errors.New("model_aliases contains empty alias") + } + if len(a.Models) == 0 { + return fmt.Errorf("model_aliases[%q] requires at least one model", alias) + } + for _, m := range a.Models { + if strings.TrimSpace(m) == "" { + return fmt.Errorf("model_aliases[%q] contains empty model", alias) + } + } + } + for _, k := range c.Auth.APIKeys { + if err := ValidateModelAccessRule(ModelAccessRule{Mode: "allow_all", AllowedModels: k.AllowedModels, DeniedModels: k.DeniedModels}); err != nil { + return fmt.Errorf("api key %q model ACL: %w", k.Name, err) + } + } + for _, w := range c.Workers { + for pattern, limit := range w.ModelConcurrency { + if strings.TrimSpace(pattern) == "" || limit <= 0 { + return fmt.Errorf("worker %q model_concurrency entries require a non-empty pattern and limit > 0", w.Name) + } + } + if err := ValidateModelPlacementRule(w.ModelPlacement); err != nil { + return fmt.Errorf("worker %q model_placement: %w", w.Name, err) + } + } + if c.UI.Enabled { + if c.UI.Path == "" || c.UI.Path == "/" { + return errors.New("ui.path must be a non-root path") + } + if c.UI.OIDC.Enabled { + if !c.Auth.OIDC.Enabled { + return errors.New("ui.oidc.enabled requires auth.oidc.enabled") + } + if c.UI.OIDC.ClientID == "" { + return errors.New("ui.oidc.client_id is required when browser OIDC login is enabled") + } + if len(c.UI.SessionSecret) < 32 { + return errors.New("ui.session_secret must contain at least 32 characters when browser OIDC login is enabled") + } + } + } + return nil +} + +func ValidateWarmModelPolicies(policies map[string]WarmModelPolicy, workers map[string]bool) error { + for pattern, wp := range policies { + if err := validateSimpleModelPattern(pattern); err != nil { + return fmt.Errorf("policy %q: %w", pattern, err) + } + switch wp.Class { + case "hot", "warm", "cold": + default: + return fmt.Errorf("policy %q class must be hot, warm, or cold", pattern) + } + if wp.Replicas < 1 { + return fmt.Errorf("policy %q replicas must be >= 1", pattern) + } + if wp.IdleTimeout.Value() < 0 { + return fmt.Errorf("policy %q idle_timeout must be >= 0", pattern) + } + for _, wn := range wp.Workers { + if len(workers) > 0 && !workers[wn] { + return fmt.Errorf("policy %q references unknown worker %q", pattern, wn) + } + } + } + return nil +} + +func validateSimpleModelPattern(p string) error { + p = strings.TrimSpace(p) + if p == "" { + return errors.New("model pattern must not be empty") + } + if strings.Count(p, "*") > 1 || (strings.Contains(p, "*") && !strings.HasSuffix(p, "*")) { + return errors.New("model pattern supports only one trailing * wildcard") + } + return nil +} + +// ValidateModelPlacementRule validates the compact pattern syntax used by +// model placement. Patterns may be exact model names, "*", or a single +// trailing wildcard such as "gemma4:*". This deliberately matches the model +// pattern semantics already used by cost and per-model concurrency rules. +func ValidateModelPlacementRule(r ModelPlacementRule) error { + mode := strings.TrimSpace(r.Mode) + if mode == "" { + mode = "allow_all" + } + if mode != "allow_all" && mode != "whitelist" { + return errors.New("mode must be allow_all or whitelist") + } + for _, group := range []struct { + name string + vals []string + }{{"allowed_models", r.AllowedModels}, {"denied_models", r.DeniedModels}} { + seen := map[string]bool{} + for _, raw := range group.vals { + p := strings.TrimSpace(raw) + if p == "" { + return fmt.Errorf("%s contains an empty pattern", group.name) + } + if strings.Count(p, "*") > 1 || (strings.Contains(p, "*") && !strings.HasSuffix(p, "*")) { + return fmt.Errorf("%s pattern %q must be exact, '*' or use one trailing '*'", group.name, p) + } + if seen[p] { + return fmt.Errorf("%s contains duplicate pattern %q", group.name, p) + } + seen[p] = true + } + } + return nil +} + +// ValidateModelAccessRule uses the same compact pattern syntax as placement. +func ValidateModelAccessRule(r ModelAccessRule) error { + return ValidateModelPlacementRule(ModelPlacementRule{Mode: r.Mode, AllowedModels: r.AllowedModels, DeniedModels: r.DeniedModels}) +} + +// ModelAccessAllowed evaluates exact and trailing-wildcard rules. More specific +// rules win; at equal specificity deny wins. +func ModelAccessAllowed(r ModelAccessRule, model string) bool { + mode := strings.TrimSpace(r.Mode) + if mode == "" { + mode = "allow_all" + } + model = strings.TrimSpace(model) + plain := strings.TrimSuffix(model, ":latest") + match := func(pattern string) (int, bool) { + pattern = strings.TrimSpace(pattern) + if pattern == model || pattern == plain { + return 100000 + len(pattern), true + } + if pattern == "*" { + return 0, true + } + if strings.HasSuffix(pattern, "*") { + prefix := strings.TrimSuffix(pattern, "*") + if strings.HasPrefix(model, prefix) || strings.HasPrefix(plain, prefix) { + return len(prefix), true + } + } + return -1, false + } + bestAllow, bestDeny := -1, -1 + for _, p := range r.AllowedModels { + if n, ok := match(p); ok && n > bestAllow { + bestAllow = n + } + } + for _, p := range r.DeniedModels { + if n, ok := match(p); ok && n > bestDeny { + bestDeny = n + } + } + if bestAllow >= 0 || bestDeny >= 0 { + return bestAllow > bestDeny + } + return mode != "whitelist" +} + +func (c *Config) ModelAccessRuleForTenant(tenant string) ModelAccessRule { + if r, ok := c.ModelAccess.Tenants[tenant]; ok { + if r.Mode == "" { + r.Mode = "allow_all" + } + return r + } + r := c.ModelAccess.Default + if r.Mode == "" { + r.Mode = "allow_all" + } + return r +} + +func (c *Config) Policy(tenant string) TenantPolicy { + p, ok := c.Scheduler.Policies[tenant] + if !ok { + p = c.Scheduler.Policies["*"] + } + if p.TenantWeight <= 0 { + p.TenantWeight = c.Scheduler.DefaultTenantWeight + } + if p.ActorWeight <= 0 { + p.ActorWeight = c.Scheduler.DefaultActorWeight + } + return p +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..ba481e7 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,181 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestLoadRejectsRemovedRedisConfig(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.json") + body := `{ + "auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]}, + "workers":[{"name":"w","url":"http://127.0.0.1:11434"}], + "redis":{"url":"redis://127.0.0.1:6379"} + }` + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("expected removed redis field to be rejected, got %v", err) + } +} + +func TestLoadInMemoryExampleShape(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.json") + body := `{ + "auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]}, + "scheduler":{"global_concurrency":4,"max_queue":100,"max_queue_per_actor":10}, + "workers":[{"name":"w","url":"http://127.0.0.1:11434","max_concurrent":4}], + "infrastructure":{"node_name":"gw","refresh_interval":"250ms","max_requests":64} + }` + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if cfg.Infrastructure.NodeName != "gw" || cfg.Scheduler.GlobalConcurrency != 4 { + t.Fatalf("unexpected config: %+v", cfg) + } +} + +func TestExampleConfigsStrictParse(t *testing.T) { + t.Setenv("OIDC_ISSUER", "https://issuer.example.test") + t.Setenv("OIDC_AUDIENCE", "gateway") + t.Setenv("GATEWAY_AUTOMATION_KEY", "01234567890123456789012345678901") + t.Setenv("GATEWAY_UI_SESSION_SECRET", "01234567890123456789012345678901") + t.Setenv("OIDC_UI_CLIENT_ID", "gateway-ui") + t.Setenv("OIDC_UI_CLIENT_SECRET", "01234567890123456789012345678901") + t.Setenv("GATEWAY_UI_REDIRECT_URL", "https://gateway.example.test/admin/callback") + t.Setenv("OPENWEBUI_GATEWAY_KEY", "01234567890123456789012345678901") + for _, path := range []string{ + "../../config.example.json", + "../../config.oidc.example.json", + "../../config.openwebui.example.json", + "../../config.rtx4090.example.json", + "../../config.placement.example.json", + } { + if _, err := Load(path); err != nil { + t.Fatalf("%s: %v", path, err) + } + } +} + +func TestModelPlacementValidation(t *testing.T) { + good := ModelPlacementRule{Mode: "whitelist", AllowedModels: []string{"qwen3:8b", "gemma4:*"}, DeniedModels: []string{"gemma4:e4b"}} + if err := ValidateModelPlacementRule(good); err != nil { + t.Fatalf("good rule rejected: %v", err) + } + for _, bad := range []ModelPlacementRule{ + {Mode: "sometimes"}, + {Mode: "allow_all", AllowedModels: []string{"*gemma*"}}, + {Mode: "allow_all", DeniedModels: []string{"gemma*4*"}}, + } { + if err := ValidateModelPlacementRule(bad); err == nil { + t.Fatalf("bad rule accepted: %#v", bad) + } + } +} + +func TestConversationsRequireEncryptionKeyWhenEnabled(t *testing.T) { + body := `{ + "auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]}, + "workers":[{"name":"w","url":"http://127.0.0.1:11434"}], + "conversations":{"enabled":true,"retention":"24h","max_entries":100,"max_content_bytes":4096} + }` + _, err := ParseBytes([]byte(body)) + if err == nil || !strings.Contains(err.Error(), "encryption_key") { + t.Fatalf("expected encryption key validation error, got %v", err) + } +} + +func TestConversationsEnabledConfigParses(t *testing.T) { + body := `{ + "auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]}, + "workers":[{"name":"w","url":"http://127.0.0.1:11434"}], + "conversations":{"enabled":true,"encryption_key":"01234567890123456789012345678901","retention":"2h","max_entries":100,"max_content_bytes":4096} + }` + cfg, err := ParseBytes([]byte(body)) + if err != nil { + t.Fatal(err) + } + if !cfg.Conversations.Enabled || cfg.Storage.ConversationsFile != "conversations.enc.json" { + t.Fatalf("unexpected conversation config: %+v storage=%+v", cfg.Conversations, cfg.Storage) + } +} + +func TestContextPolicySecureDefaults(t *testing.T) { + body := `{ + "auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]}, + "workers":[{"name":"w","url":"http://127.0.0.1:11434"}] + }` + cfg, err := ParseBytes([]byte(body)) + if err != nil { + t.Fatal(err) + } + got := cfg.ModelCapabilities.Context + if got.MaxRequestedTokens != 32768 || got.DefaultWorkerTokens != 4096 || got.EstimationMarginPercent != 15 || got.VisionReserveTokensPerImage != 2048 { + t.Fatalf("unexpected context defaults: %+v", got) + } +} + +func TestWorkerContextLimitsValidation(t *testing.T) { + body := `{ + "auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]}, + "workers":[{"name":"w","url":"http://127.0.0.1:11434","context_limits":{"qwen:*":0}}] + }` + _, err := ParseBytes([]byte(body)) + if err == nil || !strings.Contains(err.Error(), "context_limits") { + t.Fatalf("expected context limit validation error, got %v", err) + } +} + +func TestPublicDashboardDefaultsAndValidation(t *testing.T) { + body := `{ + "auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]}, + "workers":[{"name":"w","url":"http://127.0.0.1:11434"}], + "public_dashboard":{"enabled":true} + }` + cfg, err := ParseBytes([]byte(body)) + if err != nil { + t.Fatal(err) + } + if cfg.PublicDashboard.Path != "/status" || cfg.PublicDashboard.RefreshInterval.Value() != 2*time.Second || cfg.PublicDashboard.MaxLiveRequests != 64 { + t.Fatalf("unexpected public dashboard defaults: %+v", cfg.PublicDashboard) + } + + bad := `{ + "auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]}, + "workers":[{"name":"w","url":"http://127.0.0.1:11434"}], + "ui":{"path":"/admin"}, + "public_dashboard":{"enabled":true,"path":"/admin","refresh_interval":"2s"} + }` + if _, err := ParseBytes([]byte(bad)); err == nil || !strings.Contains(err.Error(), "must not overlap") { + t.Fatalf("expected public/admin path conflict, got %v", err) + } + + tooFast := `{ + "auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]}, + "workers":[{"name":"w","url":"http://127.0.0.1:11434"}], + "public_dashboard":{"enabled":true,"path":"/status","refresh_interval":"250ms"} + }` + if _, err := ParseBytes([]byte(tooFast)); err == nil || !strings.Contains(err.Error(), "at least 1s") { + t.Fatalf("expected refresh interval validation, got %v", err) + } +} + +func TestPublicDashboardRejectsReservedNamespaceOverlap(t *testing.T) { + for _, publicPath := range []string{"/gateway/status", "/api/status", "/v1/status", "/admin/public"} { + body := `{"auth":{"ip_bypass":[{"cidrs":["127.0.0.1/32"],"tenant":"t","subject":"s"}]},"workers":[{"name":"w","url":"http://127.0.0.1:11434"}],"ui":{"path":"/admin"},"public_dashboard":{"enabled":true,"path":"` + publicPath + `","refresh_interval":"2s"}}` + if _, err := ParseBytes([]byte(body)); err == nil { + t.Fatalf("expected public path %q to be rejected", publicPath) + } + } +} diff --git a/internal/conversation/store.go b/internal/conversation/store.go new file mode 100644 index 0000000..ef5378b --- /dev/null +++ b/internal/conversation/store.go @@ -0,0 +1,322 @@ +package conversation + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sort" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/state" +) + +var ErrContextTooLarge = errors.New("conversation context exceeds configured max_content_bytes") + +// Entry is the minimum content-bearing state required to implement +// previous_response_id semantics for the OpenAI Responses API. Context is a +// JSON array containing the flattened input/output items through this response. +type Entry struct { + ID string `json:"id"` + Tenant string `json:"tenant"` + Actor string `json:"actor"` + Model string `json:"model,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` + Context json.RawMessage `json:"context"` +} + +type Status struct { + Enabled bool `json:"enabled"` + Entries int `json:"entries"` + Oldest time.Time `json:"oldest_at,omitempty"` + Newest time.Time `json:"newest_at,omitempty"` + ExpiresAt time.Time `json:"next_expiry_at,omitempty"` +} + +type Store struct { + mu sync.Mutex + cfg config.ConversationsConfig + file state.AtomicJSON + key [32]byte + now func() time.Time + data map[string]Entry +} + +type diskEnvelope struct { + Version int `json:"version"` + Algorithm string `json:"algorithm"` + Nonce string `json:"nonce"` + Ciphertext string `json:"ciphertext"` + UpdatedAt time.Time `json:"updated_at"` +} + +type diskPlaintext struct { + Version int `json:"version"` + Entries map[string]Entry `json:"entries"` +} + +func New(cfg config.ConversationsConfig, path string) (*Store, error) { + s := &Store{cfg: cfg, file: state.AtomicJSON{Path: path, Mode: 0600}, now: time.Now, data: map[string]Entry{}} + s.key = sha256.Sum256([]byte(cfg.EncryptionKey)) + if !cfg.Enabled { + return s, nil + } + if err := s.load(); err != nil { + return nil, err + } + return s, nil +} + +func (s *Store) Enabled() bool { return s != nil && s.cfg.Enabled } + +// StartCleanup enforces retention even when no requests touch the store. +func (s *Store) StartCleanup(ctx context.Context, onError func(error)) { + if !s.Enabled() { + return + } + interval := s.cfg.Retention.Value() / 4 + if interval < time.Minute { + interval = time.Minute + } + if interval > 15*time.Minute { + interval = 15 * time.Minute + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := s.pruneAndSave(); err != nil && onError != nil { + onError(err) + } + } + } + }() +} + +func (s *Store) pruneAndSave() error { + s.mu.Lock() + defer s.mu.Unlock() + if !s.pruneLocked(s.now().UTC()) { + return nil + } + return s.saveLocked() +} + +func (s *Store) Get(id, tenant, actor string) (Entry, bool, error) { + if !s.Enabled() || id == "" { + return Entry{}, false, nil + } + s.mu.Lock() + defer s.mu.Unlock() + changed := s.pruneLocked(s.now().UTC()) + e, ok := s.data[id] + if !ok || e.Tenant != tenant || e.Actor != actor { + if changed { + _ = s.saveLocked() + } + return Entry{}, false, nil + } + if changed { + if err := s.saveLocked(); err != nil { + return Entry{}, false, err + } + } + e.Context = append(json.RawMessage(nil), e.Context...) + return e, true, nil +} + +func (s *Store) Put(e Entry) error { + if !s.Enabled() { + return nil + } + if e.ID == "" || e.Tenant == "" || e.Actor == "" { + return errors.New("conversation entry requires id, tenant, and actor") + } + if len(e.Context) == 0 || !json.Valid(e.Context) { + return errors.New("conversation entry context must contain valid JSON") + } + if s.cfg.MaxContentBytes > 0 && int64(len(e.Context)) > s.cfg.MaxContentBytes { + return ErrContextTooLarge + } + now := s.now().UTC() + if e.CreatedAt.IsZero() { + e.CreatedAt = now + } else { + e.CreatedAt = e.CreatedAt.UTC() + } + if e.ExpiresAt.IsZero() { + e.ExpiresAt = e.CreatedAt.Add(s.cfg.Retention.Value()) + } else { + e.ExpiresAt = e.ExpiresAt.UTC() + } + e.Context = append(json.RawMessage(nil), e.Context...) + + s.mu.Lock() + defer s.mu.Unlock() + s.pruneLocked(now) + s.data[e.ID] = e + s.enforceLimitLocked() + return s.saveLocked() +} + +func (s *Store) Delete(id, tenant, actor string) error { + if !s.Enabled() || id == "" { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + e, ok := s.data[id] + if !ok || e.Tenant != tenant || e.Actor != actor { + return nil + } + delete(s.data, id) + return s.saveLocked() +} + +func (s *Store) Compact() error { + if !s.Enabled() { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + s.pruneLocked(s.now().UTC()) + s.enforceLimitLocked() + return s.saveLocked() +} + +func (s *Store) Status() Status { + st := Status{Enabled: s.Enabled()} + if !s.Enabled() { + return st + } + s.mu.Lock() + defer s.mu.Unlock() + now := s.now().UTC() + _ = s.pruneLocked(now) + st.Entries = len(s.data) + for _, e := range s.data { + if st.Oldest.IsZero() || e.CreatedAt.Before(st.Oldest) { + st.Oldest = e.CreatedAt + } + if st.Newest.IsZero() || e.CreatedAt.After(st.Newest) { + st.Newest = e.CreatedAt + } + if st.ExpiresAt.IsZero() || e.ExpiresAt.Before(st.ExpiresAt) { + st.ExpiresAt = e.ExpiresAt + } + } + return st +} + +func (s *Store) load() error { + var env diskEnvelope + if err := s.file.Load(&env); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if env.Version != 1 || env.Algorithm != "AES-256-GCM" { + return fmt.Errorf("unsupported conversation store envelope version/algorithm") + } + nonce, err := base64.StdEncoding.DecodeString(env.Nonce) + if err != nil { + return fmt.Errorf("decode conversation nonce: %w", err) + } + ct, err := base64.StdEncoding.DecodeString(env.Ciphertext) + if err != nil { + return fmt.Errorf("decode conversation ciphertext: %w", err) + } + block, err := aes.NewCipher(s.key[:]) + if err != nil { + return err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return err + } + plain, err := gcm.Open(nil, nonce, ct, []byte("ollama-fair-gateway/conversations/v1")) + if err != nil { + return errors.New("decrypt conversation store: encryption key mismatch or file corruption") + } + var snap diskPlaintext + if err := json.Unmarshal(plain, &snap); err != nil { + return fmt.Errorf("decode conversation store: %w", err) + } + if snap.Version != 1 { + return fmt.Errorf("unsupported conversation plaintext version %d", snap.Version) + } + if snap.Entries != nil { + s.data = snap.Entries + } + s.pruneLocked(s.now().UTC()) + s.enforceLimitLocked() + return nil +} + +func (s *Store) saveLocked() error { + plain, err := json.Marshal(diskPlaintext{Version: 1, Entries: s.data}) + if err != nil { + return err + } + block, err := aes.NewCipher(s.key[:]) + if err != nil { + return err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return err + } + ct := gcm.Seal(nil, nonce, plain, []byte("ollama-fair-gateway/conversations/v1")) + env := diskEnvelope{Version: 1, Algorithm: "AES-256-GCM", Nonce: base64.StdEncoding.EncodeToString(nonce), Ciphertext: base64.StdEncoding.EncodeToString(ct), UpdatedAt: s.now().UTC()} + return s.file.Save(env) +} + +func (s *Store) pruneLocked(now time.Time) bool { + changed := false + for id, e := range s.data { + if !e.ExpiresAt.IsZero() && !e.ExpiresAt.After(now) { + delete(s.data, id) + changed = true + } + } + return changed +} + +func (s *Store) enforceLimitLocked() { + max := s.cfg.MaxEntries + if max <= 0 || len(s.data) <= max { + return + } + type pair struct { + id string + t time.Time + } + xs := make([]pair, 0, len(s.data)) + for id, e := range s.data { + xs = append(xs, pair{id: id, t: e.CreatedAt}) + } + sort.Slice(xs, func(i, j int) bool { return xs[i].t.Before(xs[j].t) }) + for len(s.data) > max && len(xs) > 0 { + delete(s.data, xs[0].id) + xs = xs[1:] + } +} diff --git a/internal/conversation/store_test.go b/internal/conversation/store_test.go new file mode 100644 index 0000000..b716e1f --- /dev/null +++ b/internal/conversation/store_test.go @@ -0,0 +1,103 @@ +package conversation + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +func testConfig() config.ConversationsConfig { + return config.ConversationsConfig{Enabled: true, EncryptionKey: strings.Repeat("k", 32), Retention: config.Duration(time.Hour), MaxEntries: 2, MaxContentBytes: 4096} +} + +func TestEncryptedRoundTripAndIdentityBoundary(t *testing.T) { + path := filepath.Join(t.TempDir(), "conversations.enc") + s, err := New(testConfig(), path) + if err != nil { + t.Fatal(err) + } + ctx := json.RawMessage(`[{"role":"user","content":"secret prompt"}]`) + if err := s.Put(Entry{ID: "resp_1", Tenant: "t1", Actor: "a1", Context: ctx}); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(b), "secret prompt") { + t.Fatalf("plaintext content leaked to disk: %s", b) + } + + s2, err := New(testConfig(), path) + if err != nil { + t.Fatal(err) + } + got, ok, err := s2.Get("resp_1", "t1", "a1") + if err != nil || !ok { + t.Fatalf("get: ok=%v err=%v", ok, err) + } + if string(got.Context) != string(ctx) { + t.Fatalf("context = %s", got.Context) + } + if _, ok, _ := s2.Get("resp_1", "t1", "other"); ok { + t.Fatal("cross-actor lookup must be hidden") + } + if _, ok, _ := s2.Get("resp_1", "other", "a1"); ok { + t.Fatal("cross-tenant lookup must be hidden") + } +} + +func TestWrongKeyFailsClosed(t *testing.T) { + path := filepath.Join(t.TempDir(), "conversations.enc") + s, _ := New(testConfig(), path) + if err := s.Put(Entry{ID: "resp_1", Tenant: "t", Actor: "a", Context: json.RawMessage(`[]`)}); err != nil { + t.Fatal(err) + } + cfg := testConfig() + cfg.EncryptionKey = strings.Repeat("z", 32) + if _, err := New(cfg, path); err == nil { + t.Fatal("expected wrong key to fail") + } +} + +func TestRetentionAndEntryLimit(t *testing.T) { + path := filepath.Join(t.TempDir(), "conversations.enc") + cfg := testConfig() + cfg.Retention = config.Duration(time.Minute) + s, err := New(cfg, path) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 9, 8, 8, 0, 0, 0, time.UTC) + s.now = func() time.Time { return now } + for i, id := range []string{"r1", "r2", "r3"} { + if err := s.Put(Entry{ID: id, Tenant: "t", Actor: "a", CreatedAt: now.Add(time.Duration(i) * time.Second), Context: json.RawMessage(`[]`)}); err != nil { + t.Fatal(err) + } + } + if _, ok, _ := s.Get("r1", "t", "a"); ok { + t.Fatal("oldest entry should be evicted") + } + now = now.Add(2 * time.Minute) + if _, ok, _ := s.Get("r3", "t", "a"); ok { + t.Fatal("expired entry should be pruned") + } +} + +func TestContentLimit(t *testing.T) { + path := filepath.Join(t.TempDir(), "conversations.enc") + cfg := testConfig() + cfg.MaxContentBytes = 2 + s, _ := New(cfg, path) + if err := s.Put(Entry{ID: "r", Tenant: "t", Actor: "a", Context: json.RawMessage(`[]`)}); err != nil { + t.Fatalf("2-byte context should fit: %v", err) + } + if err := s.Put(Entry{ID: "r2", Tenant: "t", Actor: "a", Context: json.RawMessage(`[1]`)}); err != ErrContextTooLarge { + t.Fatalf("err=%v", err) + } +} diff --git a/internal/cost/estimator.go b/internal/cost/estimator.go new file mode 100644 index 0000000..d7ac463 --- /dev/null +++ b/internal/cost/estimator.go @@ -0,0 +1,214 @@ +package cost + +import ( + "bytes" + "encoding/json" + "math" + "strings" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +type Estimate struct { + Model string + InputTokens int64 + OutputTokens int64 + Credits float64 +} + +type Usage struct { + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + CachedPromptTokens int64 `json:"cached_prompt_tokens,omitempty"` + PromptEvalNS int64 `json:"prompt_eval_ns,omitempty"` + EvalNS int64 `json:"eval_ns,omitempty"` + LoadNS int64 `json:"load_ns,omitempty"` + TotalNS int64 `json:"total_ns,omitempty"` + Approximate bool `json:"approximate,omitempty"` +} + +type Estimator struct{ cfg config.CostConfig } + +func New(cfg config.CostConfig) *Estimator { return &Estimator{cfg: cfg} } + +func (e *Estimator) Rate(model string) config.ModelRate { + if r, ok := e.cfg.Models[model]; ok { + return fillRate(r, e.cfg.Default) + } + // permit convenient prefix entries such as "qwen3:*" + best := "" + var out config.ModelRate + for p, r := range e.cfg.Models { + if strings.HasSuffix(p, "*") && strings.HasPrefix(model, strings.TrimSuffix(p, "*")) && len(p) > len(best) { + best = p + out = r + } + } + if best != "" { + return fillRate(out, e.cfg.Default) + } + return e.cfg.Default +} +func fillRate(r, d config.ModelRate) config.ModelRate { + if r.InputCreditsPer1K <= 0 { + r.InputCreditsPer1K = d.InputCreditsPer1K + } + if r.OutputCreditsPer1K <= 0 { + r.OutputCreditsPer1K = d.OutputCreditsPer1K + } + if r.CachedInputFactor <= 0 { + r.CachedInputFactor = d.CachedInputFactor + if r.CachedInputFactor <= 0 { + r.CachedInputFactor = 1 + } + } + if r.ComputeCreditsPerSecond < 0 { + r.ComputeCreditsPerSecond = 0 + } + return r +} + +type wireReq struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + Suffix string `json:"suffix"` + System json.RawMessage `json:"system"` + Instructions json.RawMessage `json:"instructions"` + Input json.RawMessage `json:"input"` + Tools json.RawMessage `json:"tools"` + Messages []struct { + Content json.RawMessage `json:"content"` + } `json:"messages"` + MaxTokens int `json:"max_tokens"` + MaxCompletionTokens int `json:"max_completion_tokens"` + MaxOutputTokens int `json:"max_output_tokens"` + Options struct { + NumPredict int `json:"num_predict"` + NumCtx int `json:"num_ctx"` + } `json:"options"` +} + +func (e *Estimator) Estimate(path string, body []byte) Estimate { + var r wireReq + _ = json.Unmarshal(body, &r) + textBytes := len(r.Prompt) + len(r.Suffix) + textualBytes(r.System) + textBytes += textualBytes(r.Instructions) + textBytes += textualBytes(r.Input) + // Tool schemas are injected into the model context. Counting their raw JSON + // bytes gives a conservative pre-run estimate without attempting to tokenize + // provider-specific JSON schema dialects. + if raw := bytes.TrimSpace(r.Tools); len(raw) > 2 && !bytes.Equal(raw, []byte("[]")) && !bytes.Equal(raw, []byte("null")) { + textBytes += len(raw) + } + for _, m := range r.Messages { + textBytes += textualBytes(m.Content) + } + in := roughTokens(textBytes) + if in < 1 { // bounded fallback for unusual request shapes + n := len(body) + if n > 64<<10 { + n = 64 << 10 + } + in = roughTokens(n) + } + // Use the largest explicitly requested output budget. Different compatible + // APIs use different field names, and choosing the largest is conservative + // when a client sends more than one of them. + out := maxInt(r.MaxCompletionTokens, r.MaxOutputTokens, r.MaxTokens, r.Options.NumPredict) + if out <= 0 { + out = e.cfg.DefaultMaxOutputTokens + } + if strings.Contains(path, "embeddings") || strings.Contains(path, "/embed") { + out = 0 + } + rate := e.Rate(r.Model) + credits := creditsFor(rate, in, 0, int64(out), 0) + if rate.ComputeCreditsPerSecond > 0 { + var sec float64 + if rate.ExpectedPromptTokensPerSecond > 0 { + sec += float64(in) / rate.ExpectedPromptTokensPerSecond + } + if out > 0 && rate.ExpectedOutputTokensPerSecond > 0 { + sec += float64(out) / rate.ExpectedOutputTokensPerSecond + } + credits += sec * rate.ComputeCreditsPerSecond + } + if credits <= 0 { + credits = .001 + } + return Estimate{Model: r.Model, InputTokens: in, OutputTokens: int64(out), Credits: credits} +} + +func (e *Estimator) Actual(model string, u Usage) float64 { + r := e.Rate(model) + sec := float64(u.PromptEvalNS+u.EvalNS) / 1e9 + return creditsFor(r, u.PromptTokens, u.CachedPromptTokens, u.CompletionTokens, sec) +} +func creditsFor(r config.ModelRate, in, cached, out int64, sec float64) float64 { + if cached < 0 { + cached = 0 + } + if cached > in { + cached = in + } + uncached := in - cached + input := (float64(uncached) + float64(cached)*r.CachedInputFactor) / 1000 * r.InputCreditsPer1K + return input + float64(out)/1000*r.OutputCreditsPer1K + sec*r.ComputeCreditsPerSecond +} +func maxInt(values ...int) int { + m := 0 + for _, v := range values { + if v > m { + m = v + } + } + return m +} + +func roughTokens(n int) int64 { + if n <= 0 { + return 0 + } + return int64(math.Ceil(float64(n) / 4)) +} + +func textualBytes(raw json.RawMessage) int { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { + return 0 + } + var s string + if raw[0] == '"' && json.Unmarshal(raw, &s) == nil { + return len(s) + } + var v any + dec := json.NewDecoder(bytes.NewReader(raw)) + if dec.Decode(&v) != nil { + return 0 + } + return walkText(v) +} +func walkText(v any) int { + switch x := v.(type) { + case string: + return len(x) + case []any: + n := 0 + for _, z := range x { + n += walkText(z) + } + return n + case map[string]any: + // Follow only known textual fields. URLs/base64 image payloads are + // deliberately excluded from the pre-run token estimate. + n := 0 + if v, ok := x["text"]; ok { + n += walkText(v) + } + if v, ok := x["content"]; ok { + n += walkText(v) + } + return n + } + return 0 +} diff --git a/internal/cost/estimator_test.go b/internal/cost/estimator_test.go new file mode 100644 index 0000000..548b531 --- /dev/null +++ b/internal/cost/estimator_test.go @@ -0,0 +1,52 @@ +package cost + +import ( + "github.com/example/ollama-fair-gateway/internal/config" + "strings" + "testing" +) + +func TestEstimateIgnoresImagePayload(t *testing.T) { + e := New(config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 100}) + b := []byte(`{"model":"m","messages":[{"content":[{"type":"text","text":"hello world"},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAAAAAAAAAAAAAAAAAAAAAA"}}]}],"max_tokens":10}`) + x := e.Estimate("/v1/chat/completions", b) + if x.InputTokens > 20 { + t.Fatalf("image data inflated estimate: %d", x.InputTokens) + } + if x.OutputTokens != 10 { + t.Fatalf("output=%d", x.OutputTokens) + } +} + +func TestActualDiscountsCachedPromptTokens(t *testing.T) { + e := New(config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 2, OutputCreditsPer1K: 4, CachedInputFactor: 0.25}}) + got := e.Actual("m", Usage{PromptTokens: 1000, CachedPromptTokens: 800, CompletionTokens: 100}) + // input: (200 + 800*0.25)/1000*2 = 0.8; output: 0.1*4 = 0.4 + if got < 1.199999 || got > 1.200001 { + t.Fatalf("credits=%f, want 1.2", got) + } +} + +func TestEstimateCountsResponsesInstructionsAndMaxOutputTokens(t *testing.T) { + e := New(config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 100}) + b := []byte(`{"model":"m","instructions":"` + strings.Repeat("i", 400) + `","input":"hello","max_output_tokens":8192}`) + x := e.Estimate("/v1/responses", b) + if x.InputTokens < 100 { + t.Fatalf("instructions not counted, input=%d", x.InputTokens) + } + if x.OutputTokens != 8192 { + t.Fatalf("output=%d, want 8192", x.OutputTokens) + } +} + +func TestEstimateCountsGenerateSuffixAndLargestOutputBudget(t *testing.T) { + e := New(config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 100}) + b := []byte(`{"model":"m","prompt":"abc","suffix":"` + strings.Repeat("s", 400) + `","max_tokens":12,"max_completion_tokens":24,"max_output_tokens":48,"options":{"num_predict":36}}`) + x := e.Estimate("/api/generate", b) + if x.InputTokens < 100 { + t.Fatalf("suffix not counted, input=%d", x.InputTokens) + } + if x.OutputTokens != 48 { + t.Fatalf("output=%d, want 48", x.OutputTokens) + } +} diff --git a/internal/haresource/collect.go b/internal/haresource/collect.go new file mode 100644 index 0000000..455c444 --- /dev/null +++ b/internal/haresource/collect.go @@ -0,0 +1,141 @@ +package haresource + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + "time" +) + +// Snapshot is a point-in-time view of gateway-process and host resources. +// Fields that cannot be collected on the current platform are omitted from JSON. +type Snapshot struct { + CapturedAt time.Time `json:"captured_at"` + GOOS string `json:"goos"` + PID int `json:"pid"` + ProcessRSSBytes int64 `json:"process_rss_bytes,omitempty"` + ProcessCPUPercent float64 `json:"process_cpu_percent,omitempty"` + ProcessThreads int64 `json:"process_threads,omitempty"` + ProcessOpenFDs int64 `json:"process_open_fds,omitempty"` + HostLogicalCPUs int `json:"host_logical_cpus"` + HostMemoryTotalBytes int64 `json:"host_memory_total_bytes,omitempty"` + HostMemoryAvailBytes int64 `json:"host_memory_available_bytes,omitempty"` + HostLoad1 float64 `json:"host_load1,omitempty"` + HostLoad5 float64 `json:"host_load5,omitempty"` + HostLoad15 float64 `json:"host_load15,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +// Collect captures one resource snapshot for pid without modifying the process. +func Collect(pid int) Snapshot { + s := Snapshot{CapturedAt: time.Now().UTC(), GOOS: runtime.GOOS, PID: pid, HostLogicalCPUs: runtime.NumCPU()} + if rss, cpu, err := psProcess(pid); err == nil { + s.ProcessRSSBytes = rss + s.ProcessCPUPercent = cpu + } else { + s.Warnings = append(s.Warnings, "ps: "+err.Error()) + } + if runtime.GOOS == "linux" { + if b, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid)); err == nil { + vmRSS, threads := ParseProcStatus(string(b)) + if vmRSS > 0 { + s.ProcessRSSBytes = vmRSS + } + s.ProcessThreads = threads + } else { + s.Warnings = append(s.Warnings, "proc status: "+err.Error()) + } + if entries, err := os.ReadDir(fmt.Sprintf("/proc/%d/fd", pid)); err == nil { + s.ProcessOpenFDs = int64(len(entries)) + } else { + s.Warnings = append(s.Warnings, "proc fd: "+err.Error()) + } + if b, err := os.ReadFile("/proc/meminfo"); err == nil { + s.HostMemoryTotalBytes, s.HostMemoryAvailBytes = ParseMeminfo(string(b)) + } + if b, err := os.ReadFile("/proc/loadavg"); err == nil { + s.HostLoad1, s.HostLoad5, s.HostLoad15 = ParseLoadavg(string(b)) + } + } + return s +} + +func psProcess(pid int) (rssBytes int64, cpuPercent float64, err error) { + cmd := exec.Command("ps", "-o", "rss=", "-o", "%cpu=", "-p", strconv.Itoa(pid)) + b, err := cmd.Output() + if err != nil { + return 0, 0, err + } + fields := strings.Fields(string(b)) + if len(fields) < 2 { + return 0, 0, fmt.Errorf("unexpected ps output %q", strings.TrimSpace(string(b))) + } + rssKB, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + return 0, 0, err + } + cpu, err := strconv.ParseFloat(strings.ReplaceAll(fields[1], ",", "."), 64) + if err != nil { + return 0, 0, err + } + return rssKB * 1024, cpu, nil +} + +// ParseProcStatus extracts VmRSS and Threads from Linux /proc//status text. +func ParseProcStatus(s string) (rssBytes, threads int64) { + scan := bufio.NewScanner(strings.NewReader(s)) + for scan.Scan() { + line := strings.TrimSpace(scan.Text()) + if strings.HasPrefix(line, "VmRSS:") { + f := strings.Fields(line) + if len(f) >= 2 { + n, _ := strconv.ParseInt(f[1], 10, 64) + rssBytes = n * 1024 + } + } + if strings.HasPrefix(line, "Threads:") { + f := strings.Fields(line) + if len(f) >= 2 { + threads, _ = strconv.ParseInt(f[1], 10, 64) + } + } + } + return rssBytes, threads +} + +// ParseMeminfo extracts MemTotal and MemAvailable from Linux /proc/meminfo text. +func ParseMeminfo(s string) (total, available int64) { + for _, line := range strings.Split(s, "\n") { + f := strings.Fields(line) + if len(f) < 2 { + continue + } + n, _ := strconv.ParseInt(f[1], 10, 64) + switch strings.TrimSuffix(f[0], ":") { + case "MemTotal": + total = n * 1024 + case "MemAvailable": + available = n * 1024 + } + } + return total, available +} + +// ParseLoadavg extracts the 1, 5 and 15 minute load averages. +func ParseLoadavg(s string) (one, five, fifteen float64) { + f := strings.Fields(s) + if len(f) > 0 { + one, _ = strconv.ParseFloat(f[0], 64) + } + if len(f) > 1 { + five, _ = strconv.ParseFloat(f[1], 64) + } + if len(f) > 2 { + fifteen, _ = strconv.ParseFloat(f[2], 64) + } + return +} diff --git a/internal/haresource/collect_test.go b/internal/haresource/collect_test.go new file mode 100644 index 0000000..55ab867 --- /dev/null +++ b/internal/haresource/collect_test.go @@ -0,0 +1,21 @@ +package haresource + +import "testing" + +func TestParseProcStatus(t *testing.T) { + rss, threads := ParseProcStatus("Name:\tx\nVmRSS:\t 1234 kB\nThreads:\t7\n") + if rss != 1234*1024 || threads != 7 { + t.Fatalf("rss=%d threads=%d", rss, threads) + } +} + +func TestParseMeminfoAndLoadavg(t *testing.T) { + total, avail := ParseMeminfo("MemTotal: 1000 kB\nMemAvailable: 400 kB\n") + if total != 1000*1024 || avail != 400*1024 { + t.Fatalf("total=%d avail=%d", total, avail) + } + a, b, c := ParseLoadavg("1.25 0.50 0.10 1/100 1\n") + if a != 1.25 || b != .5 || c != .1 { + t.Fatalf("load=%v %v %v", a, b, c) + } +} diff --git a/internal/haresource/sample.go b/internal/haresource/sample.go new file mode 100644 index 0000000..4034013 --- /dev/null +++ b/internal/haresource/sample.go @@ -0,0 +1,240 @@ +package haresource + +import ( + "context" + "errors" + "fmt" + "os" + "runtime" + "strconv" + "strings" + "time" +) + +const SamplingSchemaVersion = 1 + +// SamplingReport is a bounded sustained-resource trace around one benchmark level. +type SamplingReport struct { + Version int `json:"version"` + PID int `json:"pid"` + GOOS string `json:"goos"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + IntervalMS int64 `json:"interval_ms"` + CPUMethod string `json:"cpu_method"` + Samples []Snapshot `json:"samples"` + PeakProcessRSSBytes int64 `json:"peak_process_rss_bytes,omitempty"` + PeakProcessCPUPercent float64 `json:"peak_process_cpu_percent,omitempty"` + PeakProcessThreads int64 `json:"peak_process_threads,omitempty"` + PeakProcessOpenFDs int64 `json:"peak_process_open_fds,omitempty"` + MinHostMemoryAvailableBytes int64 `json:"min_host_memory_available_bytes,omitempty"` + PeakHostLoad1 float64 `json:"peak_host_load1,omitempty"` + ProcessExited bool `json:"process_exited,omitempty"` + StopReason string `json:"stop_reason"` + Warnings []string `json:"warnings,omitempty"` +} + +// Complete reports whether the trace is suitable as sustained resource evidence. +func (r SamplingReport) Complete() bool { + return r.Version == SamplingSchemaVersion && r.PID > 0 && len(r.Samples) >= 2 && !r.ProcessExited && r.PeakProcessRSSBytes > 0 && r.StopReason == "stop-file" +} + +type cpuCounters struct { + process uint64 + total uint64 +} + +// Sample collects until stopFile appears, maxDuration expires, or ctx is canceled. +// stopFile is polled so shell wrappers do not need to forward signals reliably. +func Sample(ctx context.Context, pid int, interval, maxDuration time.Duration, stopFile string) (SamplingReport, error) { + if pid <= 0 { + return SamplingReport{}, errors.New("pid must be positive") + } + if interval < 50*time.Millisecond { + return SamplingReport{}, errors.New("interval must be at least 50ms") + } + if maxDuration <= 0 { + return SamplingReport{}, errors.New("max duration must be positive") + } + if strings.TrimSpace(stopFile) == "" { + return SamplingReport{}, errors.New("stop file is required") + } + if _, err := os.Stat(stopFile); err == nil { + return SamplingReport{}, fmt.Errorf("stop file already exists: %s", stopFile) + } else if !errors.Is(err, os.ErrNotExist) { + return SamplingReport{}, fmt.Errorf("check stop file: %w", err) + } + + r := SamplingReport{ + Version: SamplingSchemaVersion, + PID: pid, + GOOS: runtime.GOOS, + StartedAt: time.Now().UTC(), + IntervalMS: interval.Milliseconds(), + CPUMethod: "ps_process_percent_sample", + } + var prev *cpuCounters + if runtime.GOOS == "linux" { + if c, err := readLinuxCPUCounters(pid); err == nil { + prev = &c + r.CPUMethod = "linux_procfs_interval_all_cpus_percent" + } else { + r.Warnings = append(r.Warnings, "initial procfs cpu counters: "+err.Error()) + } + } + + collectOne := func() bool { + s := Collect(pid) + if runtime.GOOS == "linux" && prev != nil { + if cur, err := readLinuxCPUCounters(pid); err == nil { + if cur.total > prev.total && cur.process >= prev.process { + dProc := float64(cur.process - prev.process) + dTotal := float64(cur.total - prev.total) + s.ProcessCPUPercent = (dProc / dTotal) * float64(max(1, s.HostLogicalCPUs)) * 100 + } + *prev = cur + } else { + r.Warnings = append(r.Warnings, "procfs cpu counters: "+err.Error()) + if errors.Is(err, os.ErrNotExist) { + r.ProcessExited = true + } + } + } + if processMissing(s) { + r.ProcessExited = true + } + r.Samples = append(r.Samples, s) + updatePeaks(&r, s) + return !r.ProcessExited + } + + if !collectOne() { + r.StopReason = "process-exited" + r.FinishedAt = time.Now().UTC() + return r, nil + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + deadline := time.NewTimer(maxDuration) + defer deadline.Stop() + for { + select { + case <-ctx.Done(): + r.StopReason = "context-canceled" + r.FinishedAt = time.Now().UTC() + return r, nil + case <-deadline.C: + _ = collectOne() + r.StopReason = "max-duration" + r.FinishedAt = time.Now().UTC() + return r, nil + case <-ticker.C: + if _, err := os.Stat(stopFile); err == nil { + _ = collectOne() + r.StopReason = "stop-file" + r.FinishedAt = time.Now().UTC() + return r, nil + } else if !errors.Is(err, os.ErrNotExist) { + r.Warnings = append(r.Warnings, "check stop file: "+err.Error()) + } + if !collectOne() { + r.StopReason = "process-exited" + r.FinishedAt = time.Now().UTC() + return r, nil + } + } + } +} + +func updatePeaks(r *SamplingReport, s Snapshot) { + if s.ProcessRSSBytes > r.PeakProcessRSSBytes { + r.PeakProcessRSSBytes = s.ProcessRSSBytes + } + if s.ProcessCPUPercent > r.PeakProcessCPUPercent { + r.PeakProcessCPUPercent = s.ProcessCPUPercent + } + if s.ProcessThreads > r.PeakProcessThreads { + r.PeakProcessThreads = s.ProcessThreads + } + if s.ProcessOpenFDs > r.PeakProcessOpenFDs { + r.PeakProcessOpenFDs = s.ProcessOpenFDs + } + if s.HostMemoryAvailBytes > 0 && (r.MinHostMemoryAvailableBytes == 0 || s.HostMemoryAvailBytes < r.MinHostMemoryAvailableBytes) { + r.MinHostMemoryAvailableBytes = s.HostMemoryAvailBytes + } + if s.HostLoad1 > r.PeakHostLoad1 { + r.PeakHostLoad1 = s.HostLoad1 + } +} + +func processMissing(s Snapshot) bool { + if runtime.GOOS == "linux" { + for _, w := range s.Warnings { + if strings.HasPrefix(w, "proc status:") && (strings.Contains(w, "no such file") || strings.Contains(w, "not found")) { + return true + } + } + } + return false +} + +func readLinuxCPUCounters(pid int) (cpuCounters, error) { + proc, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)) + if err != nil { + return cpuCounters{}, err + } + procTicks, err := ParseProcStatTicks(string(proc)) + if err != nil { + return cpuCounters{}, err + } + host, err := os.ReadFile("/proc/stat") + if err != nil { + return cpuCounters{}, err + } + totalTicks, err := ParseHostCPUTicks(string(host)) + if err != nil { + return cpuCounters{}, err + } + return cpuCounters{process: procTicks, total: totalTicks}, nil +} + +// ParseProcStatTicks returns utime+stime from Linux /proc//stat. +func ParseProcStatTicks(s string) (uint64, error) { + end := strings.LastIndexByte(strings.TrimSpace(s), ')') + if end < 0 || end+2 >= len(s) { + return 0, errors.New("malformed proc stat") + } + fields := strings.Fields(s[end+1:]) + // fields[0] is field 3 (state); utime/stime are fields 14/15. + if len(fields) <= 12 { + return 0, errors.New("proc stat has too few fields") + } + utime, err := strconv.ParseUint(fields[11], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse utime: %w", err) + } + stime, err := strconv.ParseUint(fields[12], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse stime: %w", err) + } + return utime + stime, nil +} + +// ParseHostCPUTicks sums the aggregate Linux /proc/stat cpu line. +func ParseHostCPUTicks(s string) (uint64, error) { + line, _, _ := strings.Cut(s, "\n") + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != "cpu" { + return 0, errors.New("missing aggregate cpu line") + } + var total uint64 + for _, field := range fields[1:] { + n, err := strconv.ParseUint(field, 10, 64) + if err != nil { + return 0, fmt.Errorf("parse host cpu tick: %w", err) + } + total += n + } + return total, nil +} diff --git a/internal/haresource/sample_test.go b/internal/haresource/sample_test.go new file mode 100644 index 0000000..33895d9 --- /dev/null +++ b/internal/haresource/sample_test.go @@ -0,0 +1,69 @@ +package haresource + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +func TestParseProcStatTicksHandlesSpacesInComm(t *testing.T) { + // fields 3..15 after the closing parenthesis; utime=11 and stime=13. + line := "123 (gateway worker (test)) S 1 2 3 4 5 6 7 8 9 10 11 13 0 0 0" + got, err := ParseProcStatTicks(line) + if err != nil { + t.Fatal(err) + } + if got != 24 { + t.Fatalf("ticks=%d want=24", got) + } +} + +func TestParseHostCPUTicks(t *testing.T) { + got, err := ParseHostCPUTicks("cpu 1 2 3 4 5 6 7 8 9 10\ncpu0 1 2 3\n") + if err != nil { + t.Fatal(err) + } + if got != 55 { + t.Fatalf("ticks=%d want=55", got) + } +} + +func TestSamplingReportComplete(t *testing.T) { + r := SamplingReport{Version: SamplingSchemaVersion, PID: 1, Samples: []Snapshot{{}, {}}, PeakProcessRSSBytes: 1, StopReason: "stop-file"} + if !r.Complete() { + t.Fatal("expected complete report") + } + r.ProcessExited = true + if r.Complete() { + t.Fatal("process-exited report must be incomplete") + } + r.ProcessExited = false + r.StopReason = "max-duration" + if r.Complete() { + t.Fatal("max-duration report must be incomplete") + } +} + +func TestSampleCurrentProcessStopsOnFile(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("ps-based collector is not portable to Windows test hosts") + } + stop := filepath.Join(t.TempDir(), "stop") + go func() { + time.Sleep(140 * time.Millisecond) + _ = os.WriteFile(stop, []byte("stop\n"), 0o600) + }() + r, err := Sample(context.Background(), os.Getpid(), 50*time.Millisecond, 2*time.Second, stop) + if err != nil { + t.Fatal(err) + } + if r.StopReason != "stop-file" || len(r.Samples) < 2 || r.PeakProcessRSSBytes <= 0 { + t.Fatalf("unexpected report: %+v", r) + } + if runtime.GOOS == "linux" && r.CPUMethod != "linux_procfs_interval_all_cpus_percent" { + t.Fatalf("cpu method=%q", r.CPUMethod) + } +} diff --git a/internal/hoststats/amd.go b/internal/hoststats/amd.go new file mode 100644 index 0000000..6d31afa --- /dev/null +++ b/internal/hoststats/amd.go @@ -0,0 +1,15 @@ +package hoststats + +import "context" + +// AMD contains lightweight AMDGPU telemetry. Linux reads the standard amdgpu +// sysfs counters and therefore needs no ROCm library, daemon or CGO. +type AMD struct { + UtilizationPercent float64 + MemoryUsedBytes int64 + MemoryTotalBytes int64 + TemperatureC float64 + PowerWatts float64 +} + +func ReadAMD(ctx context.Context, device string) (AMD, error) { return readAMD(ctx, device) } diff --git a/internal/hoststats/amd_linux.go b/internal/hoststats/amd_linux.go new file mode 100644 index 0000000..b989649 --- /dev/null +++ b/internal/hoststats/amd_linux.go @@ -0,0 +1,87 @@ +//go:build linux + +package hoststats + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +func readAMD(ctx context.Context, device string) (AMD, error) { + if err := ctx.Err(); err != nil { + return AMD{}, err + } + if strings.TrimSpace(device) == "" { + var err error + device, err = findAMDDevice("/sys/class/drm") + if err != nil { + return AMD{}, err + } + } + return readAMDDevice(ctx, device) +} + +func findAMDDevice(root string) (string, error) { + cards, err := filepath.Glob(filepath.Join(root, "card*", "device")) + if err != nil { + return "", err + } + for _, device := range cards { + b, err := os.ReadFile(filepath.Join(device, "vendor")) + if err != nil { + continue + } + if strings.EqualFold(strings.TrimSpace(string(b)), "0x1002") { + return device, nil + } + } + return "", fmt.Errorf("no AMDGPU sysfs device found") +} + +func readAMDDevice(ctx context.Context, device string) (AMD, error) { + if err := ctx.Err(); err != nil { + return AMD{}, err + } + var out AMD + var found bool + if v, ok := readInt64(filepath.Join(device, "mem_info_vram_total")); ok { + out.MemoryTotalBytes = v + found = true + } + if v, ok := readInt64(filepath.Join(device, "mem_info_vram_used")); ok { + out.MemoryUsedBytes = v + found = true + } + if v, ok := readInt64(filepath.Join(device, "gpu_busy_percent")); ok { + out.UtilizationPercent = float64(v) + found = true + } + hwmons, _ := filepath.Glob(filepath.Join(device, "hwmon", "hwmon*")) + for _, hw := range hwmons { + if v, ok := readInt64(filepath.Join(hw, "temp1_input")); ok && out.TemperatureC == 0 { + out.TemperatureC = float64(v) / 1000 + found = true + } + if v, ok := readInt64(filepath.Join(hw, "power1_average")); ok && out.PowerWatts == 0 { + out.PowerWatts = float64(v) / 1_000_000 + found = true + } + } + if !found { + return AMD{}, fmt.Errorf("no readable AMDGPU telemetry in %s", device) + } + return out, nil +} + +func readInt64(path string) (int64, bool) { + b, err := os.ReadFile(path) + if err != nil { + return 0, false + } + v, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) + return v, err == nil +} diff --git a/internal/hoststats/amd_other.go b/internal/hoststats/amd_other.go new file mode 100644 index 0000000..8f7abdc --- /dev/null +++ b/internal/hoststats/amd_other.go @@ -0,0 +1,12 @@ +//go:build !linux + +package hoststats + +import ( + "context" + "fmt" +) + +func readAMD(context.Context, string) (AMD, error) { + return AMD{}, fmt.Errorf("AMDGPU sysfs telemetry is supported only on Linux") +} diff --git a/internal/hoststats/amd_test.go b/internal/hoststats/amd_test.go new file mode 100644 index 0000000..acd146d --- /dev/null +++ b/internal/hoststats/amd_test.go @@ -0,0 +1,51 @@ +//go:build linux + +package hoststats + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func writeStat(t *testing.T, path, value string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(value), 0644); err != nil { + t.Fatal(err) + } +} + +func TestReadAMDDevice(t *testing.T) { + device := filepath.Join(t.TempDir(), "device") + writeStat(t, filepath.Join(device, "mem_info_vram_total"), "17179869184\n") + writeStat(t, filepath.Join(device, "mem_info_vram_used"), "4294967296\n") + writeStat(t, filepath.Join(device, "gpu_busy_percent"), "73\n") + writeStat(t, filepath.Join(device, "hwmon", "hwmon0", "temp1_input"), "62500\n") + writeStat(t, filepath.Join(device, "hwmon", "hwmon0", "power1_average"), "88000000\n") + + got, err := readAMDDevice(context.Background(), device) + if err != nil { + t.Fatal(err) + } + if got.MemoryTotalBytes != 17179869184 || got.MemoryUsedBytes != 4294967296 || got.UtilizationPercent != 73 || got.TemperatureC != 62.5 || got.PowerWatts != 88 { + t.Fatalf("unexpected AMD telemetry: %+v", got) + } +} + +func TestFindAMDDevice(t *testing.T) { + root := t.TempDir() + writeStat(t, filepath.Join(root, "card0", "device", "vendor"), "0x10de\n") + writeStat(t, filepath.Join(root, "card1", "device", "vendor"), "0x1002\n") + got, err := findAMDDevice(root) + if err != nil { + t.Fatal(err) + } + want := filepath.Join(root, "card1", "device") + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} diff --git a/internal/hoststats/memory.go b/internal/hoststats/memory.go new file mode 100644 index 0000000..1d81178 --- /dev/null +++ b/internal/hoststats/memory.go @@ -0,0 +1,10 @@ +package hoststats + +import "context" + +type Memory struct { + TotalBytes int64 + UsedBytes int64 +} + +func ReadMemory(ctx context.Context) (Memory, error) { return readMemory(ctx) } diff --git a/internal/hoststats/memory_darwin.go b/internal/hoststats/memory_darwin.go new file mode 100644 index 0000000..011ca09 --- /dev/null +++ b/internal/hoststats/memory_darwin.go @@ -0,0 +1,66 @@ +//go:build darwin + +package hoststats + +import ( + "bufio" + "context" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" +) + +var pageSizeRE = regexp.MustCompile(`page size of ([0-9]+) bytes`) + +func readMemory(ctx context.Context) (Memory, error) { + totalOut, err := exec.CommandContext(ctx, "/usr/sbin/sysctl", "-n", "hw.memsize").Output() + if err != nil { + return Memory{}, err + } + total, err := strconv.ParseInt(strings.TrimSpace(string(totalOut)), 10, 64) + if err != nil || total <= 0 { + return Memory{}, fmt.Errorf("invalid hw.memsize") + } + out, err := exec.CommandContext(ctx, "/usr/bin/vm_stat").Output() + if err != nil { + return Memory{}, err + } + pageSize := int64(4096) + var freePages int64 + s := bufio.NewScanner(strings.NewReader(string(out))) + first := true + for s.Scan() { + line := s.Text() + if first { + first = false + if m := pageSizeRE.FindStringSubmatch(line); len(m) == 2 { + if v, e := strconv.ParseInt(m[1], 10, 64); e == nil { + pageSize = v + } + } + continue + } + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + continue + } + name := strings.TrimSpace(parts[0]) + v := strings.TrimSpace(strings.TrimSuffix(parts[1], ".")) + n, _ := strconv.ParseInt(v, 10, 64) + switch name { + case "Pages free", "Pages inactive", "Pages speculative": + freePages += n + } + } + available := freePages * pageSize + used := total - available + if used < 0 { + used = 0 + } + if used > total { + used = total + } + return Memory{TotalBytes: total, UsedBytes: used}, nil +} diff --git a/internal/hoststats/memory_linux.go b/internal/hoststats/memory_linux.go new file mode 100644 index 0000000..dbc5f26 --- /dev/null +++ b/internal/hoststats/memory_linux.go @@ -0,0 +1,46 @@ +//go:build linux + +package hoststats + +import ( + "bufio" + "context" + "fmt" + "os" + "strconv" + "strings" +) + +func readMemory(context.Context) (Memory, error) { + f, err := os.Open("/proc/meminfo") + if err != nil { + return Memory{}, err + } + defer f.Close() + var total, available int64 + s := bufio.NewScanner(f) + for s.Scan() { + fields := strings.Fields(s.Text()) + if len(fields) < 2 { + continue + } + v, _ := strconv.ParseInt(fields[1], 10, 64) + switch strings.TrimSuffix(fields[0], ":") { + case "MemTotal": + total = v * 1024 + case "MemAvailable": + available = v * 1024 + } + } + if err := s.Err(); err != nil { + return Memory{}, err + } + if total <= 0 { + return Memory{}, fmt.Errorf("MemTotal unavailable") + } + used := total - available + if used < 0 { + used = 0 + } + return Memory{TotalBytes: total, UsedBytes: used}, nil +} diff --git a/internal/hoststats/memory_other.go b/internal/hoststats/memory_other.go new file mode 100644 index 0000000..a0ee7db --- /dev/null +++ b/internal/hoststats/memory_other.go @@ -0,0 +1,12 @@ +//go:build !linux && !darwin && !windows + +package hoststats + +import ( + "context" + "fmt" +) + +func readMemory(context.Context) (Memory, error) { + return Memory{}, fmt.Errorf("host memory telemetry unsupported on this OS") +} diff --git a/internal/hoststats/memory_windows.go b/internal/hoststats/memory_windows.go new file mode 100644 index 0000000..1280106 --- /dev/null +++ b/internal/hoststats/memory_windows.go @@ -0,0 +1,41 @@ +//go:build windows + +package hoststats + +import ( + "context" + "fmt" + "syscall" + "unsafe" +) + +type memoryStatusEx struct { + Length uint32 + MemoryLoad uint32 + TotalPhys uint64 + AvailPhys uint64 + TotalPageFile uint64 + AvailPageFile uint64 + TotalVirtual uint64 + AvailVirtual uint64 + AvailExtendedVirtual uint64 +} + +func readMemory(ctx context.Context) (Memory, error) { + if err := ctx.Err(); err != nil { + return Memory{}, err + } + var st memoryStatusEx + st.Length = uint32(unsafe.Sizeof(st)) + kernel32 := syscall.NewLazyDLL("kernel32.dll") + proc := kernel32.NewProc("GlobalMemoryStatusEx") + r1, _, callErr := proc.Call(uintptr(unsafe.Pointer(&st))) + if r1 == 0 { + if callErr != syscall.Errno(0) { + return Memory{}, fmt.Errorf("GlobalMemoryStatusEx: %w", callErr) + } + return Memory{}, fmt.Errorf("GlobalMemoryStatusEx failed") + } + used := int64(st.TotalPhys - st.AvailPhys) + return Memory{TotalBytes: int64(st.TotalPhys), UsedBytes: used}, nil +} diff --git a/internal/hoststats/nvidia.go b/internal/hoststats/nvidia.go new file mode 100644 index 0000000..2690074 --- /dev/null +++ b/internal/hoststats/nvidia.go @@ -0,0 +1,69 @@ +package hoststats + +import ( + "context" + "encoding/csv" + "fmt" + "os/exec" + "strconv" + "strings" +) + +// NVIDIA contains lightweight local GPU telemetry obtained from nvidia-smi. +// Values are intentionally limited to stable fields supported by the selective +// --query-gpu interface; the gateway does not require NVML or CGO. +type NVIDIA struct { + UtilizationPercent float64 + MemoryUsedBytes int64 + MemoryTotalBytes int64 + TemperatureC float64 + PowerWatts float64 +} + +func ReadNVIDIA(ctx context.Context, gpu string) (NVIDIA, error) { + args := []string{"--query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw", "--format=csv,noheader,nounits"} + if strings.TrimSpace(gpu) != "" { + args = append(args, "-i", strings.TrimSpace(gpu)) + } + out, err := exec.CommandContext(ctx, "nvidia-smi", args...).Output() + if err != nil { + if ctx.Err() != nil { + return NVIDIA{}, ctx.Err() + } + return NVIDIA{}, fmt.Errorf("nvidia-smi: %w", err) + } + r := csv.NewReader(strings.NewReader(strings.TrimSpace(string(out)))) + rec, err := r.Read() + if err != nil || len(rec) < 5 { + return NVIDIA{}, fmt.Errorf("nvidia-smi returned an unexpected row") + } + parse := func(s string) (float64, error) { + s = strings.TrimSpace(s) + if s == "" || strings.EqualFold(s, "N/A") || strings.EqualFold(s, "[Not Supported]") { + return 0, nil + } + return strconv.ParseFloat(s, 64) + } + util, err := parse(rec[0]) + if err != nil { + return NVIDIA{}, fmt.Errorf("parse gpu utilization: %w", err) + } + usedMiB, err := parse(rec[1]) + if err != nil { + return NVIDIA{}, fmt.Errorf("parse gpu memory used: %w", err) + } + totalMiB, err := parse(rec[2]) + if err != nil { + return NVIDIA{}, fmt.Errorf("parse gpu memory total: %w", err) + } + temp, err := parse(rec[3]) + if err != nil { + return NVIDIA{}, fmt.Errorf("parse gpu temperature: %w", err) + } + power, err := parse(rec[4]) + if err != nil { + return NVIDIA{}, fmt.Errorf("parse gpu power: %w", err) + } + const mib = int64(1024 * 1024) + return NVIDIA{UtilizationPercent: util, MemoryUsedBytes: int64(usedMiB * float64(mib)), MemoryTotalBytes: int64(totalMiB * float64(mib)), TemperatureC: temp, PowerWatts: power}, nil +} diff --git a/internal/hoststats/nvidia_test.go b/internal/hoststats/nvidia_test.go new file mode 100644 index 0000000..3ed7a52 --- /dev/null +++ b/internal/hoststats/nvidia_test.go @@ -0,0 +1,12 @@ +package hoststats + +import "testing" + +// Parsing is exercised indirectly through the command-backed function in +// integration environments with nvidia-smi. Keep this package dependency-free. +func TestNVIDIAStruct(t *testing.T) { + n := NVIDIA{UtilizationPercent: 50, MemoryUsedBytes: 1, MemoryTotalBytes: 2} + if n.MemoryTotalBytes != 2 { + t.Fatal("unexpected struct value") + } +} diff --git a/internal/infrastructure/hub.go b/internal/infrastructure/hub.go new file mode 100644 index 0000000..6ff9e54 --- /dev/null +++ b/internal/infrastructure/hub.go @@ -0,0 +1,201 @@ +package infrastructure + +import ( + "context" + "crypto/rand" + "encoding/hex" + "os" + "sort" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/liveflow" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +type Gateway struct { + NodeID string `json:"node_id"` + NodeName string `json:"node_name"` + Hostname string `json:"hostname"` + StartedAt time.Time `json:"started_at"` + LastSeen time.Time `json:"last_seen"` + Queued int64 `json:"queued"` + Running int64 `json:"running"` + LiveActive int `json:"live_active"` + Workers int `json:"workers"` + Healthy bool `json:"healthy"` +} + +type Request struct { + liveflow.Request + GatewayID string `json:"gateway_id"` + GatewayName string `json:"gateway_name"` +} + +type Worker struct { + worker.Snapshot + Gateways []string `json:"gateways"` +} + +type Counts struct { + Gateways int `json:"gateways"` + Workers int `json:"workers"` + Models int `json:"models"` + Active int `json:"active"` + Queued int `json:"queued"` + Routing int `json:"routing"` + Running int `json:"running"` + Streaming int `json:"streaming"` +} + +type Snapshot struct { + GeneratedAt time.Time `json:"generated_at"` + Version uint64 `json:"version"` + Mode string `json:"mode"` + Queue int64 `json:"queue"` + Running int64 `json:"running"` + Counts Counts `json:"counts"` + Gateways []Gateway `json:"gateways"` + Workers []Worker `json:"workers"` + Requests []Request `json:"requests"` +} + +type Hub struct { + cfg config.InfrastructureConfig + live *liveflow.Tracker + sched scheduler.Scheduler + workers *worker.Pool + startedAt time.Time + nodeID string + nodeName string + hostname string + + mu sync.RWMutex + version uint64 + notify chan struct{} +} + +func New(cfg config.InfrastructureConfig, live *liveflow.Tracker, sched scheduler.Scheduler, workers *worker.Pool) *Hub { + host, _ := os.Hostname() + if host == "" { + host = "gateway" + } + id := cfg.NodeID + if id == "" { + var b [6]byte + _, _ = rand.Read(b[:]) + id = host + "-" + hex.EncodeToString(b[:]) + } + name := cfg.NodeName + if name == "" { + name = host + } + return &Hub{cfg: cfg, live: live, sched: sched, workers: workers, startedAt: time.Now().UTC(), nodeID: id, nodeName: name, hostname: host, notify: make(chan struct{})} +} + +func (h *Hub) NodeID() string { return h.nodeID } +func (h *Hub) NodeName() string { return h.nodeName } + +func (h *Hub) Start(ctx context.Context) { + interval := h.cfg.RefreshInterval.Value() + if interval <= 0 { + interval = 250 * time.Millisecond + } + go func() { + t := time.NewTicker(interval) + defer t.Stop() + changed := h.live.Changed() + for { + select { + case <-ctx.Done(): + return + case <-changed: + changed = h.live.Changed() + h.bump() + case <-t.C: + h.bump() + } + } + }() +} + +func (h *Hub) bump() { + h.mu.Lock() + h.version++ + close(h.notify) + h.notify = make(chan struct{}) + h.mu.Unlock() +} + +func (h *Hub) Changed() <-chan struct{} { + h.mu.RLock() + ch := h.notify + h.mu.RUnlock() + return ch +} + +func (h *Hub) Snapshot() Snapshot { + now := time.Now().UTC() + live := h.live.Snapshot() + if max := h.cfg.MaxRequests; max > 0 && len(live.Requests) > max { + active := make([]liveflow.Request, 0, max) + recent := make([]liveflow.Request, 0, max) + for _, r := range live.Requests { + if r.State == liveflow.StateCompleted || r.State == liveflow.StateFailed { + recent = append(recent, r) + } else { + active = append(active, r) + } + } + if len(active) >= max { + live.Requests = active[:max] + } else { + need := max - len(active) + if need > len(recent) { + need = len(recent) + } + live.Requests = append(active, recent[len(recent)-need:]...) + } + } + st := h.sched.Stats(context.Background()) + ws := h.workers.Snapshots() + h.mu.RLock() + version := h.version + h.mu.RUnlock() + + out := Snapshot{GeneratedAt: now, Version: version, Mode: "in-memory", Queue: st.Queued, Running: st.Running} + out.Gateways = []Gateway{{NodeID: h.nodeID, NodeName: h.nodeName, Hostname: h.hostname, StartedAt: h.startedAt, LastSeen: now, Queued: st.Queued, Running: st.Running, LiveActive: live.Counts.Active, Workers: len(ws), Healthy: true}} + modelSet := map[string]bool{} + for _, w := range ws { + out.Workers = append(out.Workers, Worker{Snapshot: w, Gateways: []string{h.nodeName}}) + for _, m := range w.LoadedModels { + if m.Name != "" { + modelSet[m.Name] = true + } + } + } + for _, r := range live.Requests { + out.Requests = append(out.Requests, Request{Request: r, GatewayID: h.nodeID, GatewayName: h.nodeName}) + switch r.State { + case liveflow.StateQueued: + out.Counts.Queued++ + case liveflow.StateRouting: + out.Counts.Routing++ + case liveflow.StateRunning: + out.Counts.Running++ + case liveflow.StateStreaming: + out.Counts.Streaming++ + } + if r.State != liveflow.StateCompleted && r.State != liveflow.StateFailed { + out.Counts.Active++ + } + } + sort.Slice(out.Workers, func(i, j int) bool { return out.Workers[i].Name < out.Workers[j].Name }) + sort.Slice(out.Requests, func(i, j int) bool { return out.Requests[i].QueuedAt.Before(out.Requests[j].QueuedAt) }) + out.Counts.Gateways = 1 + out.Counts.Workers = len(out.Workers) + out.Counts.Models = len(modelSet) + return out +} diff --git a/internal/infrastructure/hub_test.go b/internal/infrastructure/hub_test.go new file mode 100644 index 0000000..957cffb --- /dev/null +++ b/internal/infrastructure/hub_test.go @@ -0,0 +1,48 @@ +package infrastructure + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/liveflow" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +func TestSnapshotIsLocalInMemoryTopology(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/ps" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"models":[{"name":"qwen3:8b","size":100,"size_vram":80,"context_length":32768}]}`) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer backend.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + pool := worker.New([]config.WorkerConfig{{Name: "ollama", URL: backend.URL, MaxConcurrent: 2, HealthInterval: config.Duration(time.Hour)}}, "ollama") + pool.Start(ctx) + sched := scheduler.NewLocal(2, 16, 4) + live := liveflow.New(10*time.Second, 32) + live.Begin(liveflow.Request{ID: "r1", Tenant: "team", Actor: "app", Worker: "ollama", Model: "qwen3:8b", State: liveflow.StateStreaming}) + h := New(config.InfrastructureConfig{NodeName: "gateway", RefreshInterval: config.Duration(50 * time.Millisecond), MaxRequests: 32}, live, sched, pool) + h.Start(ctx) + + s := h.Snapshot() + if s.Mode != "in-memory" || len(s.Gateways) != 1 || s.Gateways[0].NodeName != "gateway" { + t.Fatalf("unexpected gateway snapshot: %+v", s) + } + if len(s.Workers) != 1 || s.Counts.Models != 1 { + t.Fatalf("unexpected worker/model snapshot: %+v", s) + } + if len(s.Requests) != 1 || s.Requests[0].GatewayName != "gateway" { + t.Fatalf("unexpected request snapshot: %+v", s.Requests) + } +} diff --git a/internal/liveflow/tracker.go b/internal/liveflow/tracker.go new file mode 100644 index 0000000..fe327c2 --- /dev/null +++ b/internal/liveflow/tracker.go @@ -0,0 +1,335 @@ +package liveflow + +import ( + "sort" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/cost" +) + +const ( + StateQueued = "queued" + StateRouting = "routing" + StateRunning = "running" + StateStreaming = "streaming" + StateCompleted = "completed" + StateCancelled = "cancelled" + StateFailed = "failed" +) + +type Request struct { + ID string `json:"id"` + Tenant string `json:"tenant"` + Actor string `json:"actor"` + Application string `json:"application,omitempty"` + ServiceClass string `json:"service_class,omitempty"` + API string `json:"api"` + Path string `json:"path"` + Model string `json:"model,omitempty"` + Worker string `json:"worker,omitempty"` + State string `json:"state"` + EstimatedCredits float64 `json:"estimated_credits"` + ActualCredits float64 `json:"actual_credits,omitempty"` + EstimatedPromptTokens int64 `json:"estimated_prompt_tokens,omitempty"` + PromptTokens int64 `json:"prompt_tokens,omitempty"` + CompletionTokens int64 `json:"completion_tokens,omitempty"` + BytesOut int64 `json:"bytes_out,omitempty"` + Status int `json:"status,omitempty"` + QueuedAt time.Time `json:"queued_at"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` + QueueMS int64 `json:"queue_ms,omitempty"` + ServiceMS int64 `json:"service_ms,omitempty"` +} + +type Counts struct { + Total int `json:"total"` + Active int `json:"active"` + Queued int `json:"queued"` + Routing int `json:"routing"` + Running int `json:"running"` + Streaming int `json:"streaming"` + Completed int `json:"completed"` + Cancelled int `json:"cancelled"` + Failed int `json:"failed"` +} + +type Snapshot struct { + GeneratedAt time.Time `json:"generated_at"` + Version uint64 `json:"version"` + Counts Counts `json:"counts"` + Truncated bool `json:"truncated,omitempty"` + Requests []Request `json:"requests"` +} + +type Tracker struct { + mu sync.RWMutex + active map[string]Request + recent []Request + recentTTL time.Duration + maxRecent int + version uint64 + notify chan struct{} +} + +func New(recentTTL time.Duration, maxRecent int) *Tracker { + if recentTTL <= 0 { + recentTTL = 8 * time.Second + } + if maxRecent <= 0 { + maxRecent = 256 + } + return &Tracker{active: make(map[string]Request), recentTTL: recentTTL, maxRecent: maxRecent, notify: make(chan struct{})} +} + +func (t *Tracker) Begin(r Request) { + now := time.Now().UTC() + r.State = StateQueued + r.QueuedAt = now + r.UpdatedAt = now + t.mu.Lock() + t.active[r.ID] = r + t.bumpLocked() + t.mu.Unlock() +} + +func (t *Tracker) MarkRouting(id, worker string, queue time.Duration) { + t.update(id, func(r *Request, now time.Time) { + r.State = StateRouting + r.Worker = worker + r.QueueMS = queue.Milliseconds() + r.UpdatedAt = now + }) +} + +func (t *Tracker) MarkRunning(id string) { + t.update(id, func(r *Request, now time.Time) { + r.State = StateRunning + if r.StartedAt == nil { + x := now + r.StartedAt = &x + } + r.UpdatedAt = now + }) +} + +func (t *Tracker) Progress(id string, bytesOut int64, u cost.Usage) { + t.update(id, func(r *Request, now time.Time) { + if bytesOut > 0 { + r.BytesOut = bytesOut + if r.State == StateRunning || r.State == StateRouting { + r.State = StateStreaming + } + } + if u.PromptTokens > 0 { + r.PromptTokens = u.PromptTokens + } + if u.CompletionTokens > 0 { + r.CompletionTokens = u.CompletionTokens + } else if bytesOut > 0 { + // Purely for live visualization. Final accounting still comes from + // Ollama/OpenAI usage metadata in the normal request recorder. + approx := (bytesOut + 15) / 16 + if approx > r.CompletionTokens { + r.CompletionTokens = approx + } + } + r.UpdatedAt = now + }) +} + +func (t *Tracker) Finish(id string, status int, actualCredits float64, u cost.Usage, service time.Duration) { + now := time.Now().UTC() + t.mu.Lock() + r, ok := t.active[id] + if !ok { + t.mu.Unlock() + return + } + delete(t.active, id) + if status >= 200 && status < 400 { + r.State = StateCompleted + } else { + r.State = StateFailed + } + r.Status = status + r.ActualCredits = actualCredits + r.ServiceMS = service.Milliseconds() + if u.PromptTokens > 0 { + r.PromptTokens = u.PromptTokens + } + if u.CompletionTokens > 0 { + r.CompletionTokens = u.CompletionTokens + } + x := now + r.FinishedAt = &x + r.UpdatedAt = now + t.pruneLocked(now) + t.recent = append(t.recent, r) + if len(t.recent) > t.maxRecent { + t.recent = append([]Request(nil), t.recent[len(t.recent)-t.maxRecent:]...) + } + t.bumpLocked() + t.mu.Unlock() +} + +func (t *Tracker) Cancel(id string, status int, actualCredits float64, u cost.Usage, service time.Duration) { + now := time.Now().UTC() + t.mu.Lock() + r, ok := t.active[id] + if !ok { + t.mu.Unlock() + return + } + delete(t.active, id) + r.State = StateCancelled + r.Status = status + r.ActualCredits = actualCredits + r.ServiceMS = service.Milliseconds() + if u.PromptTokens > 0 { + r.PromptTokens = u.PromptTokens + } + if u.CompletionTokens > 0 { + r.CompletionTokens = u.CompletionTokens + } + x := now + r.FinishedAt = &x + r.UpdatedAt = now + t.pruneLocked(now) + t.recent = append(t.recent, r) + if len(t.recent) > t.maxRecent { + t.recent = append([]Request(nil), t.recent[len(t.recent)-t.maxRecent:]...) + } + t.bumpLocked() + t.mu.Unlock() +} + +func (t *Tracker) Drop(id string, status int) { + t.Finish(id, status, 0, cost.Usage{}, 0) +} + +func (t *Tracker) Snapshot() Snapshot { + now := time.Now().UTC() + t.mu.Lock() + t.pruneLocked(now) + out := make([]Request, 0, len(t.active)+len(t.recent)) + counts := Counts{} + for _, r := range t.active { + out = append(out, r) + addCount(&counts, r.State) + } + for _, r := range t.recent { + out = append(out, r) + addCount(&counts, r.State) + } + counts.Total = len(t.active) + len(t.recent) + counts.Active = len(t.active) + version := t.version + limit := t.maxRecent + t.mu.Unlock() + + truncated := limit > 0 && len(out) > limit + if truncated { + sort.SliceStable(out, func(i, j int) bool { + pi, pj := statePriority(out[i].State), statePriority(out[j].State) + if pi != pj { + return pi < pj + } + if out[i].State == StateCompleted || out[i].State == StateCancelled || out[i].State == StateFailed { + return out[i].UpdatedAt.After(out[j].UpdatedAt) + } + return out[i].QueuedAt.Before(out[j].QueuedAt) + }) + out = out[:limit] + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].QueuedAt.Equal(out[j].QueuedAt) { + return out[i].ID < out[j].ID + } + return out[i].QueuedAt.Before(out[j].QueuedAt) + }) + return Snapshot{GeneratedAt: now, Version: version, Counts: counts, Truncated: truncated, Requests: out} +} + +func addCount(c *Counts, state string) { + switch state { + case StateQueued: + c.Queued++ + case StateRouting: + c.Routing++ + case StateRunning: + c.Running++ + case StateStreaming: + c.Streaming++ + case StateCompleted: + c.Completed++ + case StateCancelled: + c.Cancelled++ + case StateFailed: + c.Failed++ + } +} + +func statePriority(state string) int { + switch state { + case StateStreaming: + return 0 + case StateRunning: + return 1 + case StateRouting: + return 2 + case StateQueued: + return 3 + case StateCompleted: + return 4 + case StateCancelled: + return 5 + default: + return 5 + } +} + +func (t *Tracker) Changed() <-chan struct{} { + t.mu.RLock() + ch := t.notify + t.mu.RUnlock() + return ch +} + +func (t *Tracker) update(id string, fn func(*Request, time.Time)) { + now := time.Now().UTC() + t.mu.Lock() + r, ok := t.active[id] + if ok { + fn(&r, now) + t.active[id] = r + t.bumpLocked() + } + t.mu.Unlock() +} + +func (t *Tracker) pruneLocked(now time.Time) { + if len(t.recent) == 0 { + return + } + cutoff := now.Add(-t.recentTTL) + first := 0 + for first < len(t.recent) { + r := t.recent[first] + if r.FinishedAt == nil || r.FinishedAt.After(cutoff) { + break + } + first++ + } + if first > 0 { + t.recent = append([]Request(nil), t.recent[first:]...) + } +} + +func (t *Tracker) bumpLocked() { + t.version++ + close(t.notify) + t.notify = make(chan struct{}) +} diff --git a/internal/liveflow/tracker_test.go b/internal/liveflow/tracker_test.go new file mode 100644 index 0000000..b650ac2 --- /dev/null +++ b/internal/liveflow/tracker_test.go @@ -0,0 +1,51 @@ +package liveflow + +import ( + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/cost" +) + +func TestLifecycle(t *testing.T) { + tr := New(time.Second, 10) + tr.Begin(Request{ID: "r1", Tenant: "a", Actor: "u", Model: "m", EstimatedCredits: 2}) + snap := tr.Snapshot() + if len(snap.Requests) != 1 || snap.Requests[0].State != StateQueued { + t.Fatalf("unexpected initial snapshot: %#v", snap) + } + tr.MarkRouting("r1", "w1", 25*time.Millisecond) + tr.MarkRunning("r1") + tr.Progress("r1", 160, cost.Usage{PromptTokens: 12}) + snap = tr.Snapshot() + if got := snap.Requests[0]; got.State != StateStreaming || got.Worker != "w1" || got.PromptTokens != 12 || got.CompletionTokens == 0 { + t.Fatalf("unexpected progress: %#v", got) + } + tr.Finish("r1", 200, 1.25, cost.Usage{PromptTokens: 12, CompletionTokens: 8}, 50*time.Millisecond) + snap = tr.Snapshot() + if got := snap.Requests[0]; got.State != StateCompleted || got.Status != 200 || got.ActualCredits != 1.25 || got.CompletionTokens != 8 { + t.Fatalf("unexpected final: %#v", got) + } +} + +func TestChangedNotifies(t *testing.T) { + tr := New(time.Second, 10) + ch := tr.Changed() + tr.Begin(Request{ID: "r"}) + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("tracker did not notify") + } +} + +func TestSnapshotIsBoundedButCountsRemainExact(t *testing.T) { + tr := New(time.Second, 2) + tr.Begin(Request{ID: "a"}) + tr.Begin(Request{ID: "b"}) + tr.Begin(Request{ID: "c"}) + s := tr.Snapshot() + if !s.Truncated || len(s.Requests) != 2 || s.Counts.Active != 3 || s.Counts.Queued != 3 { + t.Fatalf("unexpected bounded snapshot: %#v", s) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..726cf57 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,387 @@ +package metrics + +import ( + "fmt" + "io" + "math" + "sort" + "strings" + "sync" + "sync/atomic" + "time" +) + +type histogram struct { + buckets []float64 + counts []atomic.Uint64 + sum atomic.Uint64 + total atomic.Uint64 +} + +func newHistogram(b []float64) *histogram { + return &histogram{buckets: b, counts: make([]atomic.Uint64, len(b))} +} +func (h *histogram) observe(v float64) { + h.total.Add(1) + atomicAddFloat(&h.sum, v) + for i, b := range h.buckets { + if v <= b { + h.counts[i].Add(1) + } + } +} + +type Registry struct { + mu sync.RWMutex + dynamicMu sync.RWMutex + requests map[string]*atomic.Uint64 + errors map[string]*atomic.Uint64 + queue *histogram + service *histogram + prompt atomic.Uint64 + completion atomic.Uint64 + creditsBits atomic.Uint64 + bytesIn atomic.Uint64 + bytesOut atomic.Uint64 + usageDropped atomic.Uint64 + upstreamFailures map[string]*atomic.Uint64 + retries map[string]*atomic.Uint64 + circuitOpens map[string]*atomic.Uint64 + circuitResets map[string]*atomic.Uint64 + dynamic func() Dynamic +} +type Dynamic struct { + Queued, Running int64 + OldestQueueWaitSeconds float64 + Workers []WorkerMetric + UsageRawFiles int + UsageDailyFiles int + UsageMonthlyFiles int + UsageRawBytes int64 + UsageDailyBytes int64 + UsageMonthlyBytes int64 + UsageLastCompactionUnix int64 + UsageLastReclaimedBytes int64 + ServiceClasses map[string]ServiceClassMetric + OTelExportedSpans uint64 + OTelFailedSpans uint64 + OTelDroppedSpans uint64 + WarmActionsRunning int + WarmEvictionSuggestions int + AlertsActive int + AlertsLastEvaluateUnix int64 +} +type ServiceClassMetric struct{ Queued, Running int64 } +type WorkerMetric struct { + Name string + Healthy bool + Active int64 + Max int + MemoryUsedBytes int64 + MemoryTotalBytes int64 + VRAMUsedBytes int64 + VRAMTotalBytes int64 + GPUUtilizationPct float64 + GPUTemperatureC float64 + GPUPowerWatts float64 + ModelActive map[string]int + Performance []ModelPerformanceMetric + CircuitState string + Maintenance string +} + +type ModelPerformanceMetric struct { + Model string + PromptTPS float64 + OutputTPS float64 + Samples int64 +} + +func New() *Registry { + return &Registry{requests: map[string]*atomic.Uint64{}, errors: map[string]*atomic.Uint64{}, upstreamFailures: map[string]*atomic.Uint64{}, retries: map[string]*atomic.Uint64{}, circuitOpens: map[string]*atomic.Uint64{}, circuitResets: map[string]*atomic.Uint64{}, queue: newHistogram([]float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60, 120, 300}), service: newHistogram([]float64{.1, .25, .5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600, 1200})} +} +func (r *Registry) SetDynamic(f func() Dynamic) { + r.dynamicMu.Lock() + r.dynamic = f + r.dynamicMu.Unlock() +} +func (r *Registry) Record(api string, status int, queue, service time.Duration, prompt, completion int64, credits float64, in, out int64) { + class := fmt.Sprintf("%dxx", status/100) + key := api + "|" + class + r.mu.Lock() + c := r.requests[key] + if c == nil { + c = &atomic.Uint64{} + r.requests[key] = c + } + c.Add(1) + if status >= 400 { + e := r.errors[key] + if e == nil { + e = &atomic.Uint64{} + r.errors[key] = e + } + e.Add(1) + } + r.mu.Unlock() + r.queue.observe(queue.Seconds()) + r.service.observe(service.Seconds()) + if prompt > 0 { + r.prompt.Add(uint64(prompt)) + } + if completion > 0 { + r.completion.Add(uint64(completion)) + } + atomicAddFloat(&r.creditsBits, credits) + if in > 0 { + r.bytesIn.Add(uint64(in)) + } + if out > 0 { + r.bytesOut.Add(uint64(out)) + } +} +func (r *Registry) DropUsage() { r.usageDropped.Add(1) } + +func (r *Registry) incBounded(m map[string]*atomic.Uint64, key string) { + r.mu.Lock() + c := m[key] + if c == nil { + c = &atomic.Uint64{} + m[key] = c + } + c.Add(1) + r.mu.Unlock() +} +func (r *Registry) RecordUpstreamFailure(worker, class string) { + r.incBounded(r.upstreamFailures, worker+"|"+class) +} +func (r *Registry) RecordRetry(worker string) { r.incBounded(r.retries, worker) } +func (r *Registry) RecordCircuitOpen(worker string) { r.incBounded(r.circuitOpens, worker) } +func (r *Registry) RecordCircuitReset(worker string) { r.incBounded(r.circuitResets, worker) } + +type requestMetric struct { + API string + StatusClass string + Value uint64 +} + +type namedCounter struct { + Key string + Value uint64 +} +type prometheusSnapshot struct { + Requests []requestMetric + UpstreamFailures []namedCounter + Retries []namedCounter + CircuitOpens []namedCounter + CircuitResets []namedCounter + Queue histogramState + Service histogramState + Prompt uint64 + Completion uint64 + Credits float64 + BytesIn uint64 + BytesOut uint64 + UsageDropped uint64 + Dynamic Dynamic +} + +// snapshotPrometheus copies all state needed by the exporter before any bytes +// are written to the client. In particular, no registry lock is ever held +// across network I/O. This matters because a slow or stalled Prometheus client +// must not be able to block Record() calls in the inference hot path. +func (r *Registry) snapshotPrometheus() prometheusSnapshot { + s := prometheusSnapshot{ + Queue: histogramSnapshot(r.queue), + Service: histogramSnapshot(r.service), + Prompt: r.prompt.Load(), + Completion: r.completion.Load(), + Credits: atomicLoadFloat(&r.creditsBits), + BytesIn: r.bytesIn.Load(), + BytesOut: r.bytesOut.Load(), + UsageDropped: r.usageDropped.Load(), + } + + // Only the map shape requires a lock. Counter values themselves are atomic. + r.mu.RLock() + keys := make([]string, 0, len(r.requests)) + for k := range r.requests { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + p := strings.SplitN(k, "|", 2) + if len(p) != 2 { + continue + } + s.Requests = append(s.Requests, requestMetric{API: p[0], StatusClass: p[1], Value: r.requests[k].Load()}) + } + copyCounters := func(m map[string]*atomic.Uint64) []namedCounter { + ks := make([]string, 0, len(m)) + for k := range m { + ks = append(ks, k) + } + sort.Strings(ks) + out := make([]namedCounter, 0, len(ks)) + for _, k := range ks { + out = append(out, namedCounter{Key: k, Value: m[k].Load()}) + } + return out + } + s.UpstreamFailures = copyCounters(r.upstreamFailures) + s.Retries = copyCounters(r.retries) + s.CircuitOpens = copyCounters(r.circuitOpens) + s.CircuitResets = copyCounters(r.circuitResets) + r.mu.RUnlock() + + // The dynamic callback snapshots scheduler/worker state. It is deliberately + // invoked after releasing the metrics registry lock, so it cannot form a + // lock-order cycle with request completion, worker telemetry, or persistence. + r.dynamicMu.RLock() + dynamic := r.dynamic + r.dynamicMu.RUnlock() + if dynamic != nil { + s.Dynamic = dynamic() + } + return s +} + +func (r *Registry) WritePrometheus(w io.Writer) { + s := r.snapshotPrometheus() + io.WriteString(w, "# HELP ollama_gateway_requests_total Requests handled by API and status class.\n# TYPE ollama_gateway_requests_total counter\n") + for _, x := range s.Requests { + fmt.Fprintf(w, "ollama_gateway_requests_total{api=%q,status_class=%q} %d\n", x.API, x.StatusClass, x.Value) + } + writeHistState(w, "ollama_gateway_queue_seconds", "Queue wait time.", s.Queue) + writeHistState(w, "ollama_gateway_service_seconds", "Backend service time.", s.Service) + fmt.Fprintf(w, "# TYPE ollama_gateway_prompt_tokens_total counter\nollama_gateway_prompt_tokens_total %d\n", s.Prompt) + fmt.Fprintf(w, "# TYPE ollama_gateway_completion_tokens_total counter\nollama_gateway_completion_tokens_total %d\n", s.Completion) + fmt.Fprintf(w, "# TYPE ollama_gateway_credits_total counter\nollama_gateway_credits_total %.6f\n", s.Credits) + fmt.Fprintf(w, "# TYPE ollama_gateway_bytes_in_total counter\nollama_gateway_bytes_in_total %d\n", s.BytesIn) + fmt.Fprintf(w, "# TYPE ollama_gateway_bytes_out_total counter\nollama_gateway_bytes_out_total %d\n", s.BytesOut) + fmt.Fprintf(w, "# TYPE ollama_gateway_usage_events_dropped_total counter\nollama_gateway_usage_events_dropped_total %d\n", s.UsageDropped) + fmt.Fprintln(w, "# TYPE ollama_gateway_upstream_failures_total counter") + for _, x := range s.UpstreamFailures { + p := strings.SplitN(x.Key, "|", 2) + class := "other" + if len(p) > 1 { + class = p[1] + } + fmt.Fprintf(w, "ollama_gateway_upstream_failures_total{worker=%q,class=%q} %d\n", p[0], class, x.Value) + } + fmt.Fprintln(w, "# TYPE ollama_gateway_retries_total counter") + for _, x := range s.Retries { + fmt.Fprintf(w, "ollama_gateway_retries_total{worker=%q} %d\n", x.Key, x.Value) + } + fmt.Fprintln(w, "# TYPE ollama_gateway_circuit_opens_total counter") + for _, x := range s.CircuitOpens { + fmt.Fprintf(w, "ollama_gateway_circuit_opens_total{worker=%q} %d\n", x.Key, x.Value) + } + fmt.Fprintln(w, "# TYPE ollama_gateway_circuit_resets_total counter") + for _, x := range s.CircuitResets { + fmt.Fprintf(w, "ollama_gateway_circuit_resets_total{worker=%q} %d\n", x.Key, x.Value) + } + + d := s.Dynamic + fmt.Fprintf(w, "# TYPE ollama_gateway_queue_depth gauge\nollama_gateway_queue_depth %d\n# TYPE ollama_gateway_running gauge\nollama_gateway_running %d\n", d.Queued, d.Running) + classNames := make([]string, 0, len(d.ServiceClasses)) + for n := range d.ServiceClasses { + classNames = append(classNames, n) + } + sort.Strings(classNames) + for _, n := range classNames { + x := d.ServiceClasses[n] + fmt.Fprintf(w, "ollama_gateway_service_class_queued{name=%q} %d\nollama_gateway_service_class_running{name=%q} %d\n", n, x.Queued, n, x.Running) + } + fmt.Fprintf(w, "# TYPE ollama_gateway_usage_raw_files gauge\nollama_gateway_usage_raw_files %d\n", d.UsageRawFiles) + fmt.Fprintf(w, "# TYPE ollama_gateway_usage_daily_rollup_files gauge\nollama_gateway_usage_daily_rollup_files %d\n", d.UsageDailyFiles) + fmt.Fprintf(w, "# TYPE ollama_gateway_usage_monthly_rollup_files gauge\nollama_gateway_usage_monthly_rollup_files %d\n", d.UsageMonthlyFiles) + fmt.Fprintf(w, "# TYPE ollama_gateway_usage_raw_bytes gauge\nollama_gateway_usage_raw_bytes %d\n", d.UsageRawBytes) + fmt.Fprintf(w, "# TYPE ollama_gateway_usage_daily_rollup_bytes gauge\nollama_gateway_usage_daily_rollup_bytes %d\n", d.UsageDailyBytes) + fmt.Fprintf(w, "# TYPE ollama_gateway_usage_monthly_rollup_bytes gauge\nollama_gateway_usage_monthly_rollup_bytes %d\n", d.UsageMonthlyBytes) + fmt.Fprintf(w, "# TYPE ollama_gateway_usage_last_compaction_timestamp_seconds gauge\nollama_gateway_usage_last_compaction_timestamp_seconds %d\n", d.UsageLastCompactionUnix) + fmt.Fprintf(w, "# TYPE ollama_gateway_usage_last_reclaimed_bytes gauge\nollama_gateway_usage_last_reclaimed_bytes %d\n", d.UsageLastReclaimedBytes) + fmt.Fprintf(w, "# TYPE ollama_gateway_otel_exported_spans_total counter\nollama_gateway_otel_exported_spans_total %d\n", d.OTelExportedSpans) + fmt.Fprintf(w, "# TYPE ollama_gateway_otel_failed_spans_total counter\nollama_gateway_otel_failed_spans_total %d\n", d.OTelFailedSpans) + fmt.Fprintf(w, "# TYPE ollama_gateway_otel_dropped_spans_total counter\nollama_gateway_otel_dropped_spans_total %d\n", d.OTelDroppedSpans) + for _, x := range d.Workers { + h := 0 + if x.Healthy { + h = 1 + } + fmt.Fprintf(w, "ollama_gateway_worker_healthy{name=%q} %d\nollama_gateway_worker_active{name=%q} %d\nollama_gateway_worker_capacity{name=%q} %d\n", x.Name, h, x.Name, x.Active, x.Name, x.Max) + if x.MemoryTotalBytes > 0 { + fmt.Fprintf(w, "ollama_gateway_worker_memory_used_bytes{name=%q} %d\nollama_gateway_worker_memory_total_bytes{name=%q} %d\n", x.Name, x.MemoryUsedBytes, x.Name, x.MemoryTotalBytes) + } + if x.VRAMTotalBytes > 0 { + fmt.Fprintf(w, "ollama_gateway_worker_vram_used_bytes{name=%q} %d\nollama_gateway_worker_vram_total_bytes{name=%q} %d\n", x.Name, x.VRAMUsedBytes, x.Name, x.VRAMTotalBytes) + } + if x.GPUUtilizationPct > 0 { + fmt.Fprintf(w, "ollama_gateway_worker_gpu_utilization_percent{name=%q} %.3f\n", x.Name, x.GPUUtilizationPct) + } + if x.GPUTemperatureC > 0 { + fmt.Fprintf(w, "ollama_gateway_worker_gpu_temperature_celsius{name=%q} %.3f\n", x.Name, x.GPUTemperatureC) + } + if x.GPUPowerWatts > 0 { + fmt.Fprintf(w, "ollama_gateway_worker_gpu_power_watts{name=%q} %.3f\n", x.Name, x.GPUPowerWatts) + } + open, half, accepting := 0, 0, 1 + if x.CircuitState == "open" { + open = 1 + accepting = 0 + } + if x.CircuitState == "half_open" { + half = 1 + } + if x.Maintenance != "active" { + accepting = 0 + } + fmt.Fprintf(w, "ollama_gateway_worker_circuit_open{name=%q} %d\nollama_gateway_worker_circuit_half_open{name=%q} %d\nollama_gateway_worker_accepting_new{name=%q} %d\n", x.Name, open, x.Name, half, x.Name, accepting) + models := make([]string, 0, len(x.ModelActive)) + for model := range x.ModelActive { + models = append(models, model) + } + sort.Strings(models) + for _, model := range models { + fmt.Fprintf(w, "ollama_gateway_worker_model_active{name=%q,model=%q} %d\n", x.Name, model, x.ModelActive[model]) + } + for _, perf := range x.Performance { + if perf.PromptTPS > 0 { + fmt.Fprintf(w, "ollama_gateway_worker_model_prompt_tokens_per_second{name=%q,model=%q} %.6f\n", x.Name, perf.Model, perf.PromptTPS) + } + if perf.OutputTPS > 0 { + fmt.Fprintf(w, "ollama_gateway_worker_model_output_tokens_per_second{name=%q,model=%q} %.6f\n", x.Name, perf.Model, perf.OutputTPS) + } + fmt.Fprintf(w, "ollama_gateway_worker_model_performance_samples{name=%q,model=%q} %d\n", x.Name, perf.Model, perf.Samples) + } + } +} + +func writeHistState(w io.Writer, name, help string, h histogramState) { + fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s histogram\n", name, help, name) + for i, b := range h.Buckets { + var count uint64 + if i < len(h.Counts) { + count = h.Counts[i] + } + fmt.Fprintf(w, "%s_bucket{le=\"%g\"} %d\n", name, b, count) + } + fmt.Fprintf(w, "%s_bucket{le=\"+Inf\"} %d\n%s_sum %.9f\n%s_count %d\n", name, h.Total, name, h.Sum, name, h.Total) +} + +// Atomic float helpers use CAS so simultaneous observations do not lose increments. +func atomicAddFloat(a *atomic.Uint64, v float64) { + for { + old := a.Load() + nv := mathFloat64frombits(old) + v + if a.CompareAndSwap(old, mathFloat64bits(nv)) { + return + } + } +} + +func atomicLoadFloat(a *atomic.Uint64) float64 { return mathFloat64frombits(a.Load()) } + +// Tiny wrappers keep the package dependency-free while using math's bit representation. +func mathFloat64bits(f float64) uint64 { return math.Float64bits(f) } +func mathFloat64frombits(b uint64) float64 { return math.Float64frombits(b) } diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 0000000..6aed910 --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,96 @@ +package metrics + +import ( + "bytes" + "strings" + "sync" + "testing" + "time" +) + +func TestPrometheusIncludesWorkerTelemetryAndModelThroughput(t *testing.T) { + r := New() + r.SetDynamic(func() Dynamic { + return Dynamic{UsageRawFiles: 3, UsageDailyFiles: 12, UsageMonthlyFiles: 7, UsageRawBytes: 100, UsageDailyBytes: 200, UsageMonthlyBytes: 300, UsageLastCompactionUnix: 123456, UsageLastReclaimedBytes: 789, Workers: []WorkerMetric{{ + Name: "gpu-1", Healthy: true, Active: 1, Max: 2, + VRAMUsedBytes: 12, VRAMTotalBytes: 24, GPUUtilizationPct: 87.5, + GPUTemperatureC: 63, GPUPowerWatts: 310, + ModelActive: map[string]int{"qwen3:8b": 1}, + Performance: []ModelPerformanceMetric{{Model: "qwen3:8b", OutputTPS: 101.25, Samples: 3}}, + }}} + }) + var b bytes.Buffer + r.WritePrometheus(&b) + out := b.String() + for _, want := range []string{ + `ollama_gateway_worker_vram_used_bytes{name="gpu-1"} 12`, + `ollama_gateway_worker_gpu_utilization_percent{name="gpu-1"} 87.500`, + `ollama_gateway_worker_model_active{name="gpu-1",model="qwen3:8b"} 1`, + `ollama_gateway_worker_model_output_tokens_per_second{name="gpu-1",model="qwen3:8b"} 101.250000`, + `ollama_gateway_usage_raw_files 3`, + `ollama_gateway_usage_daily_rollup_files 12`, + `ollama_gateway_usage_monthly_rollup_bytes 300`, + `ollama_gateway_usage_last_compaction_timestamp_seconds 123456`, + `ollama_gateway_usage_last_reclaimed_bytes 789`, + } { + if !strings.Contains(out, want) { + t.Fatalf("missing %q in metrics:\n%s", want, out) + } + } +} + +// blockingWriter simulates a Prometheus client that stops reading after the +// exporter has started writing request metrics. WritePrometheus must not hold +// the registry mutex while this writer is blocked, otherwise all request +// completions stall in Record(). +type blockingWriter struct { + started chan struct{} + release chan struct{} + once sync.Once +} + +func (w *blockingWriter) Write(p []byte) (int, error) { + if strings.Contains(string(p), "ollama_gateway_requests_total{") { + w.once.Do(func() { close(w.started) }) + <-w.release + } + return len(p), nil +} + +func TestPrometheusSlowClientDoesNotBlockRecord(t *testing.T) { + r := New() + r.Record("ollama", 200, 0, time.Millisecond, 1, 1, 1, 1, 1) + + bw := &blockingWriter{started: make(chan struct{}), release: make(chan struct{})} + done := make(chan struct{}) + go func() { + r.WritePrometheus(bw) + close(done) + }() + + select { + case <-bw.started: + case <-time.After(time.Second): + t.Fatal("metrics writer did not reach blocking write") + } + + recorded := make(chan struct{}) + go func() { + r.Record("ollama", 200, 0, time.Millisecond, 1, 1, 1, 1, 1) + close(recorded) + }() + select { + case <-recorded: + // success: no registry lock is held by the stalled exporter + case <-time.After(250 * time.Millisecond): + close(bw.release) + t.Fatal("Record blocked behind a stalled /metrics writer") + } + + close(bw.release) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("metrics writer did not finish after release") + } +} diff --git a/internal/metrics/persistence.go b/internal/metrics/persistence.go new file mode 100644 index 0000000..db0d022 --- /dev/null +++ b/internal/metrics/persistence.go @@ -0,0 +1,160 @@ +package metrics + +import ( + "context" + "errors" + "os" + "sync/atomic" + "time" + + "github.com/example/ollama-fair-gateway/internal/state" +) + +type histogramState struct { + Buckets []float64 `json:"buckets"` + Counts []uint64 `json:"counts"` + Sum float64 `json:"sum"` + Total uint64 `json:"total"` +} + +type PersistentState struct { + Version int `json:"version"` + SavedAt time.Time `json:"saved_at"` + Requests map[string]uint64 `json:"requests"` + Errors map[string]uint64 `json:"errors"` + Queue histogramState `json:"queue"` + Service histogramState `json:"service"` + Prompt uint64 `json:"prompt_tokens"` + Completion uint64 `json:"completion_tokens"` + Credits float64 `json:"credits"` + BytesIn uint64 `json:"bytes_in"` + BytesOut uint64 `json:"bytes_out"` + UsageDropped uint64 `json:"usage_dropped"` + UpstreamFailures map[string]uint64 `json:"upstream_failures,omitempty"` + Retries map[string]uint64 `json:"retries,omitempty"` + CircuitOpens map[string]uint64 `json:"circuit_opens,omitempty"` + CircuitResets map[string]uint64 `json:"circuit_resets,omitempty"` +} + +func histogramSnapshot(h *histogram) histogramState { + x := histogramState{Buckets: append([]float64(nil), h.buckets...), Counts: make([]uint64, len(h.counts)), Sum: atomicLoadFloat(&h.sum), Total: h.total.Load()} + for i := range h.counts { + x.Counts[i] = h.counts[i].Load() + } + return x +} +func (r *Registry) SnapshotPersistent() PersistentState { + r.mu.Lock() + defer r.mu.Unlock() + s := PersistentState{Version: 1, SavedAt: time.Now().UTC(), Requests: map[string]uint64{}, Errors: map[string]uint64{}, Queue: histogramSnapshot(r.queue), Service: histogramSnapshot(r.service), Prompt: r.prompt.Load(), Completion: r.completion.Load(), Credits: atomicLoadFloat(&r.creditsBits), BytesIn: r.bytesIn.Load(), BytesOut: r.bytesOut.Load(), UsageDropped: r.usageDropped.Load(), UpstreamFailures: map[string]uint64{}, Retries: map[string]uint64{}, CircuitOpens: map[string]uint64{}, CircuitResets: map[string]uint64{}} + for k, v := range r.requests { + s.Requests[k] = v.Load() + } + for k, v := range r.errors { + s.Errors[k] = v.Load() + } + for k, v := range r.upstreamFailures { + s.UpstreamFailures[k] = v.Load() + } + for k, v := range r.retries { + s.Retries[k] = v.Load() + } + for k, v := range r.circuitOpens { + s.CircuitOpens[k] = v.Load() + } + for k, v := range r.circuitResets { + s.CircuitResets[k] = v.Load() + } + return s +} +func restoreHistogram(dst *histogram, src histogramState) { + if len(src.Counts) != len(dst.counts) { + return + } + for i := range dst.counts { + dst.counts[i].Store(src.Counts[i]) + } + dst.total.Store(src.Total) + dst.sum.Store(mathFloat64bits(src.Sum)) +} +func (r *Registry) RestorePersistent(s PersistentState) { + if s.Version != 1 { + return + } + r.mu.Lock() + defer r.mu.Unlock() + r.requests = map[string]*atomic.Uint64{} + for k, v := range s.Requests { + x := &atomic.Uint64{} + x.Store(v) + r.requests[k] = x + } + r.errors = map[string]*atomic.Uint64{} + for k, v := range s.Errors { + x := &atomic.Uint64{} + x.Store(v) + r.errors[k] = x + } + restoreMap := func(src map[string]uint64) map[string]*atomic.Uint64 { + out := map[string]*atomic.Uint64{} + for k, v := range src { + x := &atomic.Uint64{} + x.Store(v) + out[k] = x + } + return out + } + r.upstreamFailures = restoreMap(s.UpstreamFailures) + r.retries = restoreMap(s.Retries) + r.circuitOpens = restoreMap(s.CircuitOpens) + r.circuitResets = restoreMap(s.CircuitResets) + restoreHistogram(r.queue, s.Queue) + restoreHistogram(r.service, s.Service) + r.prompt.Store(s.Prompt) + r.completion.Store(s.Completion) + r.creditsBits.Store(mathFloat64bits(s.Credits)) + r.bytesIn.Store(s.BytesIn) + r.bytesOut.Store(s.BytesOut) + r.usageDropped.Store(s.UsageDropped) +} + +func (r *Registry) LoadPersistent(path string) error { + var s PersistentState + err := (state.AtomicJSON{Path: path, Mode: 0640}).Load(&s) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + r.RestorePersistent(s) + return nil +} +func (r *Registry) SavePersistent(path string) error { + return (state.AtomicJSON{Path: path, Mode: 0640}).Save(r.SnapshotPersistent()) +} + +func (r *Registry) StartPersistence(ctx context.Context, path string, interval time.Duration, onError func(error)) { + if interval < time.Second { + interval = 10 * time.Second + } + go func() { + t := time.NewTicker(interval) + defer t.Stop() + defer func() { + if err := r.SavePersistent(path); err != nil && onError != nil { + onError(err) + } + }() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := r.SavePersistent(path); err != nil && onError != nil { + onError(err) + } + } + } + }() +} diff --git a/internal/metrics/persistence_test.go b/internal/metrics/persistence_test.go new file mode 100644 index 0000000..cbe3f30 --- /dev/null +++ b/internal/metrics/persistence_test.go @@ -0,0 +1,29 @@ +package metrics + +import ( + "path/filepath" + "testing" + "time" +) + +func TestPersistentMetricsRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "metrics.json") + a := New() + a.Record("ollama", 200, 25*time.Millisecond, 2*time.Second, 123, 45, 7.25, 1000, 2000) + a.Record("openai", 500, time.Second, 3*time.Second, 5, 1, 1.5, 10, 20) + a.DropUsage() + if err := a.SavePersistent(path); err != nil { + t.Fatal(err) + } + b := New() + if err := b.LoadPersistent(path); err != nil { + t.Fatal(err) + } + x := b.SnapshotPersistent() + if x.Prompt != 128 || x.Completion != 46 || x.Credits != 8.75 || x.BytesIn != 1010 || x.BytesOut != 2020 || x.UsageDropped != 1 { + t.Fatalf("restored=%#v", x) + } + if x.Requests["ollama|2xx"] != 1 || x.Requests["openai|5xx"] != 1 || x.Errors["openai|5xx"] != 1 { + t.Fatalf("counters=%#v errors=%#v", x.Requests, x.Errors) + } +} diff --git a/internal/policy/store.go b/internal/policy/store.go new file mode 100644 index 0000000..18cc926 --- /dev/null +++ b/internal/policy/store.go @@ -0,0 +1,64 @@ +package policy + +import ( + "context" + "sort" + "sync" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +type Store interface { + Get(context.Context, string) (config.TenantPolicy, bool, error) + Put(context.Context, string, config.TenantPolicy) error + Delete(context.Context, string) error + List(context.Context) (map[string]config.TenantPolicy, error) + Health(context.Context) error +} + +type Memory struct { + mu sync.RWMutex + m map[string]config.TenantPolicy +} + +func NewMemory() *Memory { return &Memory{m: map[string]config.TenantPolicy{}} } + +func (m *Memory) Get(_ context.Context, tenant string) (config.TenantPolicy, bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + p, ok := m.m[tenant] + return p, ok, nil +} +func (m *Memory) Put(_ context.Context, tenant string, p config.TenantPolicy) error { + m.mu.Lock() + m.m[tenant] = p + m.mu.Unlock() + return nil +} +func (m *Memory) Delete(_ context.Context, tenant string) error { + m.mu.Lock() + delete(m.m, tenant) + m.mu.Unlock() + return nil +} +func (m *Memory) List(context.Context) (map[string]config.TenantPolicy, error) { + m.mu.RLock() + defer m.mu.RUnlock() + out := make(map[string]config.TenantPolicy, len(m.m)) + for k, v := range m.m { + out[k] = v + } + return out, nil +} +func (m *Memory) Health(context.Context) error { return nil } + +// Names returns the stable sorted tenant names in a policy map. Kept here so +// both API and UI can present deterministic policy tables. +func Names(m map[string]config.TenantPolicy) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go new file mode 100644 index 0000000..8a05c7f --- /dev/null +++ b/internal/proxy/proxy.go @@ -0,0 +1,428 @@ +package proxy + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/cost" +) + +type Proxy struct { + client *http.Client + buffers sync.Pool +} +type ProgressFunc func(bytesOut int64, usage cost.Usage) + +type Result struct { + Status int + BytesIn int64 + BytesOut int64 + Usage cost.Usage + Err error + Started bool + FirstByte time.Duration + Captured []byte + CaptureTruncated bool +} + +func New() *Proxy { + tr := &http.Transport{Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{Timeout: 5 * time.Second, KeepAlive: 30 * time.Second}).DialContext, ForceAttemptHTTP2: false, MaxIdleConns: 1024, MaxIdleConnsPerHost: 256, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 5 * time.Second, ExpectContinueTimeout: time.Second, DisableCompression: true} + p := &Proxy{client: &http.Client{Transport: tr}} + p.buffers.New = func() any { b := make([]byte, 32<<10); return &b } + return p +} + +func (p *Proxy) Forward(ctx context.Context, w http.ResponseWriter, in *http.Request, target *url.URL, body io.Reader, api string, estimatedInput int64, progress ...ProgressFunc) Result { + return p.forward(ctx, w, in, target, body, api, estimatedInput, 0, progress...) +} + +// ForwardCapture behaves like Forward but retains up to captureLimit bytes of +// the upstream response for post-response control-plane processing. It is used +// only by explicitly enabled content-bearing features such as conversation +// persistence; the normal inference path keeps response bodies uncaptured. +func (p *Proxy) ForwardCapture(ctx context.Context, w http.ResponseWriter, in *http.Request, target *url.URL, body io.Reader, api string, estimatedInput int64, captureLimit int64, progress ...ProgressFunc) Result { + return p.forward(ctx, w, in, target, body, api, estimatedInput, captureLimit, progress...) +} + +func (p *Proxy) forward(ctx context.Context, w http.ResponseWriter, in *http.Request, target *url.URL, body io.Reader, api string, estimatedInput int64, captureLimit int64, progress ...ProgressFunc) Result { + u := *target + u.Path = singleJoiningSlash(target.Path, in.URL.Path) + u.RawQuery = in.URL.RawQuery + cr := &countingReader{r: body} + var requestBody io.Reader + if body != nil { + requestBody = cr + } + req, err := http.NewRequestWithContext(ctx, in.Method, u.String(), requestBody) + if err != nil { + return Result{Status: 502, BytesIn: cr.n, Err: err} + } + if body != nil && in.ContentLength >= 0 { + req.ContentLength = in.ContentLength + } + copyHeader(req.Header, in.Header) + stripHop(req.Header) + req.Header.Del("Authorization") + req.Header.Del("X-API-Key") + req.Header.Del("X-Gateway-Service-Class") + req.Host = target.Host + if ip, _, e := net.SplitHostPort(in.RemoteAddr); e == nil { + prior := req.Header.Get("X-Forwarded-For") + if prior != "" { + req.Header.Set("X-Forwarded-For", prior+", "+ip) + } else { + req.Header.Set("X-Forwarded-For", ip) + } + } + if in.TLS != nil { + req.Header.Set("X-Forwarded-Proto", "https") + } else { + req.Header.Set("X-Forwarded-Proto", "http") + } + requestStarted := time.Now() + resp, err := p.client.Do(req) + if err != nil { + return Result{Status: 502, BytesIn: cr.n, Err: err} + } + defer resp.Body.Close() + copyHeader(w.Header(), resp.Header) + stripHop(w.Header()) + w.WriteHeader(resp.StatusCode) + meter := newMeter(api, estimatedInput) + ct := strings.ToLower(resp.Header.Get("Content-Type")) + stream := strings.Contains(ct, "event-stream") || strings.Contains(ct, "ndjson") || strings.Contains(ct, "stream") + bp := p.buffers.Get().(*[]byte) + defer p.buffers.Put(bp) + buf := *bp + var out int64 + var firstByte time.Duration + var captured []byte + var captureTruncated bool + if captureLimit > 0 { + capHint := captureLimit + if capHint > 1<<20 { + capHint = 1 << 20 + } + captured = make([]byte, 0, int(capHint)) + } + var observer ProgressFunc + if len(progress) > 0 { + observer = progress[0] + } + var lastProgress time.Time + notifyProgress := func(force bool, usage cost.Usage) { + if observer == nil { + return + } + now := time.Now() + if force || lastProgress.IsZero() || now.Sub(lastProgress) >= 200*time.Millisecond { + observer(out, usage) + lastProgress = now + } + } + for { + n, re := resp.Body.Read(buf) + if n > 0 { + if firstByte == 0 { + firstByte = time.Since(requestStarted) + } + chunk := buf[:n] + meter.Feed(chunk) + if captureLimit > 0 { + remain := captureLimit - int64(len(captured)) + if remain > 0 { + take := int64(len(chunk)) + if take > remain { + take = remain + } + captured = append(captured, chunk[:int(take)]...) + } + if int64(len(chunk)) > remain { + captureTruncated = true + } + } + wn, we := w.Write(chunk) + out += int64(wn) + if stream { + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + } + notifyProgress(false, meter.usage) + if we != nil { + u := meter.Finish(out) + notifyProgress(true, u) + return Result{Status: resp.StatusCode, BytesIn: cr.n, BytesOut: out, Usage: u, Err: we, Started: true, FirstByte: firstByte, Captured: captured, CaptureTruncated: captureTruncated} + } + } + if re != nil { + if re == io.EOF { + break + } + u := meter.Finish(out) + notifyProgress(true, u) + return Result{Status: resp.StatusCode, BytesIn: cr.n, BytesOut: out, Usage: u, Err: re, Started: true, FirstByte: firstByte, Captured: captured, CaptureTruncated: captureTruncated} + } + } + finalUsage := meter.Finish(out) + notifyProgress(true, finalUsage) + return Result{Status: resp.StatusCode, BytesIn: cr.n, BytesOut: out, Usage: finalUsage, Started: true, FirstByte: firstByte, Captured: captured, CaptureTruncated: captureTruncated} +} +func copyHeader(dst, src http.Header) { + for k, vv := range src { + for _, v := range vv { + dst.Add(k, v) + } + } +} + +var hopHeaders = []string{"Connection", "Proxy-Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "Te", "Trailer", "Transfer-Encoding", "Upgrade"} + +func stripHop(h http.Header) { + if c := h.Get("Connection"); c != "" { + for _, f := range strings.Split(c, ",") { + h.Del(strings.TrimSpace(f)) + } + } + for _, k := range hopHeaders { + h.Del(k) + } +} +func singleJoiningSlash(a, b string) string { + as := strings.HasSuffix(a, "/") + bs := strings.HasPrefix(b, "/") + switch { + case as && bs: + return a + b[1:] + case !as && !bs: + return a + "/" + b + default: + return a + b + } +} + +type meter struct { + api string + line []byte + usage cost.Usage + found bool + outTextBytes int64 + estimatedInput int64 +} + +func newMeter(api string, estimatedInput int64) *meter { + return &meter{api: api, estimatedInput: estimatedInput} +} +func (m *meter) Feed(p []byte) { + m.line = append(m.line, p...) + for { + idx := bytes.IndexByte(m.line, '\n') + if idx < 0 { + if len(m.line) > 2<<20 { + m.line = append([]byte(nil), m.line[len(m.line)-(1<<20):]...) + } + return + } + line := bytes.TrimSpace(m.line[:idx]) + m.process(line) + m.line = append(m.line[:0], m.line[idx+1:]...) + } +} +func (m *meter) Finish(bytesOut int64) cost.Usage { + if len(bytes.TrimSpace(m.line)) > 0 { + m.process(bytes.TrimSpace(m.line)) + } + if !m.found { + m.usage.PromptTokens = m.estimatedInput + if m.usage.CompletionTokens == 0 { + if m.outTextBytes > 0 { + m.usage.CompletionTokens = (m.outTextBytes + 3) / 4 + } else if bytesOut > 0 { + m.usage.CompletionTokens = (bytesOut + 15) / 16 + } + } + m.usage.Approximate = true + } + return m.usage +} +func (m *meter) process(line []byte) { + if len(line) == 0 { + return + } + if bytes.HasPrefix(line, []byte("data:")) { + line = bytes.TrimSpace(bytes.TrimPrefix(line, []byte("data:"))) + if bytes.Equal(line, []byte("[DONE]")) { + return + } + } + var v map[string]any + dec := json.NewDecoder(bytes.NewReader(line)) + dec.UseNumber() + if dec.Decode(&v) != nil { + return + } + if m.api == "ollama" { + m.parseNative(v) + } else if m.api == "anthropic" { + m.parseAnthropic(v) + } else { + m.parseOpenAI(v) + } +} +func (m *meter) parseNative(v map[string]any) { + u := cost.Usage{PromptTokens: i64(v["prompt_eval_count"]), CachedPromptTokens: i64(v["prompt_eval_cached_count"]), CompletionTokens: i64(v["eval_count"]), PromptEvalNS: i64(v["prompt_eval_duration"]), EvalNS: i64(v["eval_duration"]), LoadNS: i64(v["load_duration"]), TotalNS: i64(v["total_duration"])} + if u.PromptTokens > 0 || u.CompletionTokens > 0 || u.TotalNS > 0 { + m.usage = u + m.found = true + } +} +func (m *meter) parseOpenAI(v map[string]any) { + if u := findUsage(v); u != nil { + pt := firstI64(u, "prompt_tokens", "input_tokens") + ct := firstI64(u, "completion_tokens", "output_tokens") + cached := int64(0) + if d, ok := u["prompt_tokens_details"].(map[string]any); ok { + cached = i64(d["cached_tokens"]) + } + m.usage.PromptTokens = pt + m.usage.CompletionTokens = ct + m.usage.CachedPromptTokens = cached + m.found = true + } + m.outTextBytes += deltaTextBytes(v) +} +func (m *meter) parseAnthropic(v map[string]any) { + if u := findUsage(v); u != nil { + pt := firstI64(u, "input_tokens", "prompt_tokens") + ct := firstI64(u, "output_tokens", "completion_tokens") + cached := firstI64(u, "cache_read_input_tokens", "cached_tokens") + if pt > 0 { + m.usage.PromptTokens = pt + } + if ct > 0 { + m.usage.CompletionTokens = ct + } + if cached > 0 { + m.usage.CachedPromptTokens = cached + } + if pt > 0 || ct > 0 { + m.found = true + } + } + if d, ok := v["delta"].(map[string]any); ok { + if text, ok := d["text"].(string); ok { + m.outTextBytes += int64(len(text)) + } + } + if cb, ok := v["content_block"].(map[string]any); ok { + if text, ok := cb["text"].(string); ok { + m.outTextBytes += int64(len(text)) + } + } +} + +func findUsage(v any) map[string]any { + switch x := v.(type) { + case map[string]any: + if u, ok := x["usage"].(map[string]any); ok { + return u + } + for _, key := range []string{"response", "message"} { + if r, ok := x[key]; ok { + if u := findUsage(r); u != nil { + return u + } + } + } + case []any: + for _, z := range x { + if u := findUsage(z); u != nil { + return u + } + } + } + return nil +} +func deltaTextBytes(v map[string]any) int64 { + var n int64 + if s, ok := v["delta"].(string); ok { + n += int64(len(s)) + } + if choices, ok := v["choices"].([]any); ok { + for _, c := range choices { + cm, _ := c.(map[string]any) + d, _ := cm["delta"].(map[string]any) + if s, ok := d["content"].(string); ok { + n += int64(len(s)) + } + } + } + return n +} +func i64(v any) int64 { + switch x := v.(type) { + case json.Number: + n, _ := x.Int64() + return n + case float64: + return int64(x) + case int64: + return x + case int: + return int64(x) + case string: + n, _ := strconv.ParseInt(x, 10, 64) + return n + } + return 0 +} +func firstI64(m map[string]any, keys ...string) int64 { + for _, k := range keys { + if n := i64(m[k]); n != 0 { + return n + } + } + return 0 +} + +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + if c.r == nil { + return 0, io.EOF + } + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +func WriteJSONError(w http.ResponseWriter, status int, code, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"code": code, "message": msg, "type": "gateway_error"}}) +} +func FormatRetryAfter(d time.Duration) string { + if d <= 0 { + return "1" + } + return strconv.Itoa(max(1, int(d.Round(time.Second)/time.Second))) +} +func BackendError(err error) string { + if err == nil { + return "" + } + return fmt.Sprintf("Ollama backend error: %v", err) +} diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go new file mode 100644 index 0000000..df8b8b0 --- /dev/null +++ b/internal/proxy/proxy_test.go @@ -0,0 +1,65 @@ +package proxy + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/example/ollama-fair-gateway/internal/cost" +) + +func TestNativeMeter(t *testing.T) { + m := newMeter("ollama", 0) + m.Feed([]byte("{\"response\":\"x\",\"done\":false}\n{\"done\":true,\"prompt_eval_count\":12,\"eval_count\":4,\"eval_duration\":100}\n")) + u := m.Finish(100) + if u.PromptTokens != 12 || u.CompletionTokens != 4 || u.EvalNS != 100 || u.Approximate { + t.Fatalf("bad usage %#v", u) + } +} +func TestOpenAIMeter(t *testing.T) { + m := newMeter("openai", 9) + m.Feed([]byte("data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}}]}\n\ndata: {\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":2}}\n")) + u := m.Finish(100) + if u.PromptTokens != 10 || u.CompletionTokens != 2 || u.Approximate { + t.Fatalf("bad usage %#v", u) + } +} + +func TestAnthropicMeter(t *testing.T) { + m := newMeter("anthropic", 0) + m.Feed([]byte("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"usage\":{\"input_tokens\":11,\"output_tokens\":0}}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"hello\"}}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":5}}\n\n")) + u := m.Finish(100) + if u.PromptTokens != 11 || u.CompletionTokens != 5 || u.Approximate { + t.Fatalf("bad anthropic usage %#v", u) + } +} + +func TestForwardProgressObserver(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/x-ndjson") + _, _ = io.WriteString(w, "{\"message\":{\"content\":\"hi\"},\"done\":false}\n") + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + _, _ = io.WriteString(w, "{\"done\":true,\"prompt_eval_count\":10,\"eval_count\":2}\n") + })) + defer backend.Close() + target, _ := url.Parse(backend.URL) + in := httptest.NewRequest(http.MethodPost, "http://gateway/api/chat", strings.NewReader(`{"model":"x"}`)) + out := httptest.NewRecorder() + var calls int + var lastBytes int64 + var last cost.Usage + res := New().Forward(context.Background(), out, in, target, strings.NewReader(`{"model":"x"}`), "ollama", 0, func(n int64, u cost.Usage) { + calls++ + lastBytes = n + last = u + }) + if res.Status != http.StatusOK || calls == 0 || lastBytes == 0 || last.PromptTokens != 10 || last.CompletionTokens != 2 { + t.Fatalf("status=%d calls=%d bytes=%d usage=%#v", res.Status, calls, lastBytes, last) + } +} diff --git a/internal/publicui/assets/app.css b/internal/publicui/assets/app.css new file mode 100644 index 0000000..a07f69c --- /dev/null +++ b/internal/publicui/assets/app.css @@ -0,0 +1,3 @@ +:root{--bg:#edf2f6;--panel:#fff;--panel2:#f6f9fb;--line:#d6dfe8;--line2:#c4d0dc;--text:#25384b;--muted:#718397;--cyan:#147fa5;--cyan2:#2aa2c9;--green:#1a805f;--amber:#a56600;--red:#b83d4b;--violet:#6b5bb5;--mono:"IBM Plex Mono","JetBrains Mono","Cascadia Mono",SFMono-Regular,Consolas,monospace;--ui:"IBM Plex Sans Condensed","Roboto Condensed","Arial Narrow",Inter,system-ui,sans-serif;--shadow:0 10px 30px rgba(38,58,77,.07)} +*.hidden{display:none!important}*{box-sizing:border-box}html,body{margin:0;min-height:100%;background:var(--bg);color:var(--text)}body{font-family:var(--ui);font-size:14px;background-image:linear-gradient(rgba(96,116,136,.035) 1px,transparent 1px),linear-gradient(90deg,rgba(96,116,136,.025) 1px,transparent 1px);background-size:32px 32px}.shell{width:min(1500px,100%);margin:auto;padding:26px 30px 20px}.hero{display:grid;grid-template-columns:auto 1fr auto;gap:30px;align-items:center;padding:8px 0 22px;border-bottom:1px solid var(--line)}.brand{display:flex;align-items:center;gap:10px}.mark{width:38px;height:38px;display:grid;place-items:center;border:1px solid #a8d2df;border-radius:7px;background:#f9fdfe;color:var(--cyan);font:700 12px var(--mono)}.brand strong{display:block;font:700 12px var(--mono);letter-spacing:.02em}.brand small{display:block;margin-top:3px;color:#8595a5;font:9px var(--mono);letter-spacing:.12em}.hero-copy{border-left:1px solid var(--line);padding-left:28px}.hero h1{margin:0;font:700 22px var(--mono);letter-spacing:.01em}.hero p{margin:5px 0 0;color:var(--muted);font-size:12px}.status-wrap{display:flex;align-items:center;gap:10px;padding:9px 12px;border:1px solid var(--line);border-radius:7px;background:rgba(255,255,255,.8)}.status-dot{width:9px;height:9px;border-radius:50%;background:var(--amber);box-shadow:0 0 0 4px rgba(165,102,0,.09)}.status-wrap.ok .status-dot{background:var(--green);box-shadow:0 0 0 4px rgba(26,128,95,.09)}.status-wrap.bad .status-dot{background:var(--red);box-shadow:0 0 0 4px rgba(184,61,75,.09)}.status-wrap strong{display:block;font:700 10px var(--mono);text-transform:uppercase}.status-wrap small{display:block;margin-top:2px;color:var(--muted);font:9px var(--mono)}main{padding:20px 0}.kpis{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));gap:11px;margin-bottom:12px}.kpis article,.panel{background:var(--panel);border:1px solid var(--line);border-radius:8px;box-shadow:0 1px 2px rgba(39,57,77,.035)}.kpis article{padding:14px 15px;min-width:0}.kpis span,.eyebrow{display:block;color:#798b9d;font:9px var(--mono);text-transform:uppercase;letter-spacing:.09em}.kpis strong{display:block;margin-top:7px;color:#29465d;font:700 23px var(--mono)}.kpis small{display:block;margin-top:5px;color:#8796a5;font:9px var(--mono);white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.grid{display:grid;gap:12px}.map-grid{grid-template-columns:minmax(0,2fr) minmax(320px,.8fr);margin-bottom:12px}.panel{padding:16px}.panel-head{display:flex;justify-content:space-between;align-items:flex-start;gap:15px;margin-bottom:13px}.panel-head h2{margin:3px 0 0;font:700 12px var(--mono);text-transform:uppercase;letter-spacing:.04em}.panel-head p{margin:5px 0 0;color:var(--muted);font-size:11px}.badge{display:inline-flex;align-items:center;border:1px solid #b7d9e5;border-radius:5px;background:#e9f6f9;color:var(--cyan);padding:4px 7px;font:700 9px var(--mono);letter-spacing:.06em}.badge.muted{border-color:var(--line);background:#f1f4f7;color:#6d7f91}.map-stage{height:510px;position:relative;overflow:hidden;border:1px solid var(--line2);border-radius:7px;background:radial-gradient(circle at 48% 48%,rgba(20,127,165,.065),transparent 40%),linear-gradient(180deg,#fbfdfe,#edf3f7)}.map-stage:before{content:"";position:absolute;inset:0;background-image:linear-gradient(rgba(79,105,126,.08) 1px,transparent 1px),linear-gradient(90deg,rgba(79,105,126,.08) 1px,transparent 1px);background-size:30px 30px;pointer-events:none}.map-stage canvas{position:absolute;inset:0;width:100%;height:100%}.map-empty{position:absolute;left:50%;top:50%;transform:translate(-50%,-50%);padding:8px 11px;border:1px solid var(--line);border-radius:6px;background:rgba(255,255,255,.9);color:var(--muted);font:10px var(--mono)}.legend{display:flex;gap:14px;flex-wrap:wrap;margin-top:10px;color:var(--muted);font:9px var(--mono)}.legend span{display:flex;gap:5px;align-items:center}.legend i{width:7px;height:7px;border-radius:50%;background:var(--cyan)}.legend .q{background:var(--amber)}.legend .w{background:var(--green)}.legend .r{background:var(--violet)}.worker-list{display:flex;flex-direction:column;gap:9px;max-height:555px;overflow:auto;padding-right:2px}.worker{border:1px solid #dce3ea;border-radius:7px;background:var(--panel2);padding:11px}.worker-head{display:flex;justify-content:space-between;gap:10px;align-items:flex-start}.worker-head strong{font:700 11px var(--mono)}.worker-head small{display:block;margin-top:3px;color:var(--muted);font:9px var(--mono)}.chip{border:1px solid #c1d9cf;border-radius:4px;background:#edf6f2;color:var(--green);padding:3px 6px;font:8px var(--mono);text-transform:uppercase}.chip.bad{border-color:#e1bec3;background:#fbf0f2;color:var(--red)}.metrics{display:grid;grid-template-columns:repeat(2,1fr);gap:6px;margin-top:9px}.metric{background:#fff;border:1px solid #e0e6ec;border-radius:5px;padding:7px}.metric span{color:#8090a1;font:8px var(--mono);text-transform:uppercase}.metric b{display:block;margin-top:2px;font:700 11px var(--mono)}.bar{height:4px;margin-top:5px;border-radius:9px;background:#e5ebf0;overflow:hidden}.bar i{display:block;height:100%;background:linear-gradient(90deg,var(--cyan),var(--green))}.models{display:flex;flex-wrap:wrap;gap:4px;margin-top:8px}.models span{max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;border:1px solid #d5dee6;border-radius:4px;background:#fff;padding:3px 5px;color:#5b7084;font:8px var(--mono)}.live-panel{margin-bottom:4px}.live-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(250px,1fr));gap:8px}.event{position:relative;border:1px solid #dce3ea;border-radius:7px;background:#f8fafb;padding:10px 11px;overflow:hidden}.event:before{content:"";position:absolute;left:0;top:0;bottom:0;width:3px;background:var(--cyan)}.event.queued:before{background:var(--amber)}.event.streaming:before{background:var(--violet)}.event.failed:before,.event.cancelled:before{background:var(--red)}.event.done:before{background:var(--green)}.event-head{display:flex;justify-content:space-between;gap:8px}.event strong{font:700 10px var(--mono)}.event .state{font:8px var(--mono);text-transform:uppercase;color:var(--cyan)}.event.queued .state{color:var(--amber)}.event.streaming .state{color:var(--violet)}.event small{display:block;margin-top:5px;color:var(--muted);font:9px var(--mono)}.event-meta{display:flex;gap:10px;margin-top:8px;color:#607487;font:8px var(--mono)}.empty{grid-column:1/-1;border:1px dashed #cbd6e0;border-radius:7px;padding:22px;text-align:center;color:var(--muted);font:10px var(--mono)}footer{display:flex;justify-content:space-between;gap:15px;padding:12px 1px 0;border-top:1px solid var(--line);color:#8090a1;font:9px var(--mono);text-transform:uppercase;letter-spacing:.06em} +@media(max-width:1050px){.kpis{grid-template-columns:repeat(3,1fr)}.map-grid{grid-template-columns:1fr}.worker-list{max-height:none}.map-stage{height:430px}}@media(max-width:680px){.shell{padding:16px 14px}.hero{grid-template-columns:1fr auto;gap:14px}.hero-copy{grid-column:1/-1;grid-row:2;border-left:0;border-top:1px solid var(--line);padding:14px 0 0}.status-wrap{padding:7px 9px}.kpis{grid-template-columns:repeat(2,1fr)}.map-stage{height:350px}.live-grid{grid-template-columns:1fr}.panel{padding:13px}}@media(prefers-reduced-motion:reduce){*{scroll-behavior:auto!important}} diff --git a/internal/publicui/assets/app.js b/internal/publicui/assets/app.js new file mode 100644 index 0000000..8ee0c43 --- /dev/null +++ b/internal/publicui/assets/app.js @@ -0,0 +1,17 @@ +const $=s=>document.querySelector(s);let snapshot=null,timer=null,raf=0,lastFetch=0; +const esc=s=>String(s??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); +const pct=(a,b)=>b>0?Math.max(0,Math.min(100,a/b*100)):0;const fmtPct=v=>Number.isFinite(v)?`${Math.round(v)}%`:'–'; +const uptime=s=>{s=Math.max(0,Math.floor(s||0));const d=Math.floor(s/86400);s%=86400;const h=Math.floor(s/3600);s%=3600;const m=Math.floor(s/60);return d?`${d}d ${h}h`:h?`${h}h ${m}m`:`${m}m`}; +const stateLabel=s=>({queued:'Queue',routing:'Routing',running:'Compute',streaming:'Streaming',completed:'Fertig',failed:'Fehler',cancelled:'Abbruch'}[s]||s||'–'); +function setText(id,v){const e=$(id);if(e)e.textContent=v} +function render(s){snapshot=s;document.title=s.title||'Gateway Status';setText('#title',s.title||'Gateway Status');setText('#subtitle',s.subtitle||'Live-Auslastung und Infrastruktur');setText('#k-queue',s.counts?.queued??0);setText('#k-active',s.counts?.active??0);setText('#k-workers',`${s.counts?.healthy_workers??0}/${s.counts?.workers??0}`);setText('#k-models',s.counts?.models??0);setText('#k-streaming',s.counts?.streaming??0);setText('#k-uptime',uptime(s.uptime_seconds));setText('#live-count',`${s.requests?.length||0} EVENTS`);setText('#footer-refresh',`Refresh ${Math.round((s.refresh_interval_ms||2000)/100)/10}s`);const when=new Date(s.generated_at);setText('#updated',Number.isNaN(when.getTime())?'–':`Update ${when.toLocaleTimeString()}`);const wrap=$('.status-wrap');wrap?.classList.remove('ok','bad');const healthy=(s.counts?.workers||0)>0&&(s.counts?.healthy_workers||0)===(s.counts?.workers||0);if(healthy){wrap?.classList.add('ok');setText('#status-text',(s.counts?.queued||0)>0?'Ausgelastet':'Operational')}else{wrap?.classList.add('bad');setText('#status-text','Degraded')}renderWorkers(s.workers||[]);renderLive(s.requests||[]);$('#map-empty')?.classList.toggle('hidden',(s.workers||[]).length>0);drawMap();} +function renderWorkers(ws){const root=$('#workers');if(!root)return;if(!ws.length){root.innerHTML='
Keine Worker-Telemetrie verfügbar.
';return}root.innerHTML=ws.map(w=>{const slots=`${w.active||0}/${w.max_concurrent||0} Slots`;const mods=(w.loaded_models||[]).map(m=>`${esc(m)}`).join('');const metrics=[];if(w.resource_metrics){const r=w.resource_metrics;if(r.vram_total_bytes>0)metrics.push(metric('VRAM',fmtPct(pct(r.vram_used_bytes,r.vram_total_bytes)),pct(r.vram_used_bytes,r.vram_total_bytes)));if(r.memory_total_bytes>0)metrics.push(metric('RAM',fmtPct(pct(r.memory_used_bytes,r.memory_total_bytes)),pct(r.memory_used_bytes,r.memory_total_bytes)));if(Number.isFinite(r.gpu_utilization_percent))metrics.push(metric('GPU',fmtPct(r.gpu_utilization_percent),r.gpu_utilization_percent));if(r.gpu_temperature_c>0)metrics.push(metric('Temp',`${Math.round(r.gpu_temperature_c)}°C`,Math.min(100,r.gpu_temperature_c)));}return `
${esc(w.name)}${esc(slots)} · ${esc(w.maintenance||'active')}
${w.healthy?'online':'offline'}
${metrics.length?`
${metrics.join('')}
`:''}${mods?`
${mods}
`:''}
`}).join('')} +function metric(k,v,p){return `
${esc(k)}${esc(v)}
`} +function renderLive(rs){const root=$('#live');if(!root)return;if(!rs.length){root.innerHTML='
Aktuell keine sichtbare Request-Aktivität.
';return}root.innerHTML=rs.map(r=>{const cls=r.state==='completed'?'done':r.state;const model=r.model?` · ${esc(r.model)}`:'';const worker=r.worker?esc(r.worker):'wartet';const toks=(r.prompt_tokens||0)+(r.completion_tokens||0);return `
${esc(r.id)}${esc(stateLabel(r.state))}
${worker}${model}
Queue ${r.queue_ms||0}ms${toks?`${toks} tok`:'–'}
`}).join('')} +async function poll(){try{const r=await fetch('./api/snapshot',{headers:{Accept:'application/json'}});if(!r.ok)throw new Error(`HTTP ${r.status}`);const s=await r.json();render(s);lastFetch=Date.now();const delay=Math.max(1000,s.refresh_interval_ms||2000);clearTimeout(timer);timer=setTimeout(poll,delay)}catch(e){const wrap=$('.status-wrap');wrap?.classList.remove('ok');wrap?.classList.add('bad');setText('#status-text','Unavailable');setText('#updated',e.message);clearTimeout(timer);timer=setTimeout(poll,5000)}} +function fitCanvas(){const c=$('#map-canvas'),stage=$('#map-stage');if(!c||!stage)return null;const dpr=Math.min(devicePixelRatio||1,2),r=stage.getBoundingClientRect();const w=Math.max(1,Math.floor(r.width)),h=Math.max(1,Math.floor(r.height));if(c.width!==Math.floor(w*dpr)||c.height!==Math.floor(h*dpr)){c.width=Math.floor(w*dpr);c.height=Math.floor(h*dpr)}const x=c.getContext('2d');x.setTransform(dpr,0,0,dpr,0,0);return{x,w,h}} +function drawMap(ts=performance.now()){const fit=fitCanvas();if(!fit){raf=requestAnimationFrame(drawMap);return}const {x,w,h}=fit;x.clearRect(0,0,w,h);const s=snapshot;if(s){const ws=s.workers||[],req=s.requests||[];const queue={x:w*.18,y:h*.5},gateway={x:w*.48,y:h*.5};const nodes=ws.map((v,i)=>({v,x:w*.80,y:h*(i+1)/(ws.length+1)}));line(x,queue,gateway,'#b9c7d3',1.4);for(const n of nodes)line(x,gateway,n,'#b9c7d3',1.2);node(x,queue,54,'#a56600','QUEUE',String(s.counts?.queued||0));node(x,gateway,66,'#147fa5','GATEWAY',`${s.counts?.active||0} active`);for(const n of nodes)node(x,n,52,n.v.healthy?'#1a805f':'#b83d4b',n.v.name,`${n.v.active||0}/${n.v.max_concurrent||0}`);req.slice(0,40).forEach((r,i)=>{let a=queue,b=gateway;if(r.worker){const n=nodes.find(n=>n.v.name===r.worker);if(n){a=gateway;b=n}}const phase=((ts/1800)+(hash(r.id)%100)/100)%1;const p=r.state==='queued'?Math.min(.25,phase*.25):phase;const px=a.x+(b.x-a.x)*p,py=a.y+(b.y-a.y)*p;const col=r.state==='streaming'?'#6b5bb5':r.state==='queued'?'#a56600':'#147fa5';x.beginPath();x.fillStyle=col;x.arc(px,py,3.5,0,Math.PI*2);x.fill()})}raf=requestAnimationFrame(drawMap)} +function line(x,a,b,c,l){x.beginPath();x.moveTo(a.x,a.y);x.lineTo(b.x,b.y);x.strokeStyle=c;x.lineWidth=l;x.stroke()} +function node(x,n,r,c,title,sub){x.save();x.beginPath();x.fillStyle='#fff';x.strokeStyle=c;x.lineWidth=2;x.arc(n.x,n.y,r,0,Math.PI*2);x.fill();x.stroke();x.textAlign='center';x.fillStyle='#2a4054';x.font='700 9px '+getComputedStyle(document.documentElement).getPropertyValue('--mono');x.fillText(String(title).slice(0,18),n.x,n.y-2);x.fillStyle='#7b8c9d';x.font='8px '+getComputedStyle(document.documentElement).getPropertyValue('--mono');x.fillText(String(sub).slice(0,18),n.x,n.y+12);x.restore()} +function hash(s){let h=2166136261;for(const c of String(s))h=Math.imul(h^c.charCodeAt(0),16777619);return h>>>0} +document.addEventListener('visibilitychange',()=>{if(!document.hidden&&Date.now()-lastFetch>3000)poll()});poll();drawMap(); diff --git a/internal/publicui/assets/index.html b/internal/publicui/assets/index.html new file mode 100644 index 0000000..ffc0009 --- /dev/null +++ b/internal/publicui/assets/index.html @@ -0,0 +1,51 @@ + + + + + + + + Gateway Status + + + +
+
+
OF
Gateway StatusPUBLIC STATUS
+

Systemstatus

Live-Auslastung und Infrastruktur

+
Verbinde …–
+
+ +
+
+
Queue–wartend
+
Aktiv–laufende Requests
+
Worker–gesund
+
Modelle–aktuell geladen
+
Streaming–aktive Streams
+
Uptime–Gateway-Prozess
+
+ +
+
+
INFRASTRUCTURE

Live Topology

Read-only Ansicht von Gateway, Queue und verfügbaren Ollama-Workern.

LIVE
+
Warte auf Telemetrie …
+
GatewayQueueWorkerRequest
+
+
+
CAPACITY

Worker

Slots und optionale Ressourcenmetriken.

+
+
+
+ +
+
LIVE FLOW

Aktuelle Aktivität

Anonymisierte Request-Metadaten. Keine Prompts, Antworten, Nutzer-, Tenant- oder API-Key-Daten.

0 EVENTS
+
+
+
+ +
Read-only public status–
+
+ + + diff --git a/internal/publicui/publicui.go b/internal/publicui/publicui.go new file mode 100644 index 0000000..2972874 --- /dev/null +++ b/internal/publicui/publicui.go @@ -0,0 +1,15 @@ +package publicui + +import ( + "embed" + "io/fs" + "net/http" +) + +//go:embed assets/* +var embedded embed.FS + +func Handler() http.Handler { + sub, _ := fs.Sub(embedded, "assets") + return http.FileServer(http.FS(sub)) +} diff --git a/internal/quota/persistence.go b/internal/quota/persistence.go new file mode 100644 index 0000000..8904e09 --- /dev/null +++ b/internal/quota/persistence.go @@ -0,0 +1,85 @@ +package quota + +import ( + "context" + "errors" + "os" + "time" + + "github.com/example/ollama-fair-gateway/internal/state" +) + +type BucketState struct { + Balance float64 `json:"balance"` + Updated time.Time `json:"updated"` +} + +type PersistentState struct { + Version int `json:"version"` + SavedAt time.Time `json:"saved_at"` + Buckets map[string]BucketState `json:"buckets"` +} + +func (m *Memory) SnapshotPersistent() PersistentState { + m.mu.Lock() + defer m.mu.Unlock() + out := PersistentState{Version: 1, SavedAt: time.Now().UTC(), Buckets: make(map[string]BucketState, len(m.buckets))} + for k, b := range m.buckets { + out.Buckets[k] = BucketState{Balance: b.balance, Updated: b.updated} + } + return out +} + +func (m *Memory) RestorePersistent(s PersistentState) { + if s.Version != 1 { + return + } + m.mu.Lock() + defer m.mu.Unlock() + m.buckets = make(map[string]bucket, len(s.Buckets)) + for k, b := range s.Buckets { + m.buckets[k] = bucket{balance: b.Balance, updated: b.Updated} + } +} + +func (m *Memory) LoadPersistent(path string) error { + var s PersistentState + err := (state.AtomicJSON{Path: path, Mode: 0600}).Load(&s) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + m.RestorePersistent(s) + return nil +} + +func (m *Memory) SavePersistent(path string) error { + return (state.AtomicJSON{Path: path, Mode: 0600}).Save(m.SnapshotPersistent()) +} + +func (m *Memory) StartPersistence(ctx context.Context, path string, interval time.Duration, onError func(error)) { + if interval < time.Second { + interval = 10 * time.Second + } + go func() { + t := time.NewTicker(interval) + defer t.Stop() + defer func() { + if err := m.SavePersistent(path); err != nil && onError != nil { + onError(err) + } + }() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := m.SavePersistent(path); err != nil && onError != nil { + onError(err) + } + } + } + }() +} diff --git a/internal/quota/persistence_test.go b/internal/quota/persistence_test.go new file mode 100644 index 0000000..9ff6550 --- /dev/null +++ b/internal/quota/persistence_test.go @@ -0,0 +1,37 @@ +package quota + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +func TestQuotaPersistenceRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "quota.json") + a := NewMemory() + lim := Limits{ActorCreditsPerMinute: 60, ActorBurstCredits: 120, TenantCreditsPerMinute: 300, TenantBurstCredits: 600} + d, err := a.Reserve(context.Background(), "team", "user", 50, lim) + if err != nil || !d.Allowed { + t.Fatalf("reserve allowed=%v err=%v", d.Allowed, err) + } + if err := a.SavePersistent(path); err != nil { + t.Fatal(err) + } + b := NewMemory() + if err := b.LoadPersistent(path); err != nil { + t.Fatal(err) + } + d2, err := b.Reserve(context.Background(), "team", "user", 80, lim) + if err != nil { + t.Fatal(err) + } + // Actor started with 120, spent 50, so an immediate 80-credit request + // remains over budget after restart (apart from negligible refill time). + if d2.Allowed { + t.Fatalf("quota reset across restart: %#v", d2) + } + if d2.RetryAfter <= 0 || d2.RetryAfter > 20*time.Second { + t.Fatalf("unexpected retry: %v", d2.RetryAfter) + } +} diff --git a/internal/quota/quota.go b/internal/quota/quota.go new file mode 100644 index 0000000..928d32b --- /dev/null +++ b/internal/quota/quota.go @@ -0,0 +1,135 @@ +package quota + +import ( + "context" + "sync" + "time" +) + +type Limits struct { + ActorCreditsPerMinute float64 + ActorBurstCredits float64 + TenantCreditsPerMinute float64 + TenantBurstCredits float64 +} + +type Reservation struct { + Tenant, Actor string + Amount float64 + Limits Limits +} +type Decision struct { + Allowed bool + RetryAfter time.Duration + RemainingActor float64 + RemainingTenant float64 + Reservation Reservation +} +type Ledger interface { + Reserve(context.Context, string, string, float64, Limits) (Decision, error) + Reconcile(context.Context, Reservation, float64) error + Health(context.Context) error +} + +// Disabled permits all requests. +type Disabled struct{} + +func (Disabled) Reserve(_ context.Context, t, a string, amt float64, l Limits) (Decision, error) { + return Decision{Allowed: true, Reservation: Reservation{Tenant: t, Actor: a, Amount: amt, Limits: l}}, nil +} +func (Disabled) Reconcile(context.Context, Reservation, float64) error { return nil } +func (Disabled) Health(context.Context) error { return nil } + +type bucket struct { + balance float64 + updated time.Time +} +type Memory struct { + mu sync.Mutex + buckets map[string]bucket +} + +func NewMemory() *Memory { return &Memory{buckets: map[string]bucket{}} } +func (m *Memory) Health(context.Context) error { return nil } +func (m *Memory) Reserve(_ context.Context, tenant, actor string, amount float64, l Limits) (Decision, error) { + m.mu.Lock() + defer m.mu.Unlock() + now := time.Now() + ab, _ := m.refill("a:"+tenant+":"+actor, now, l.ActorCreditsPerMinute, l.ActorBurstCredits) + tb, _ := m.refill("t:"+tenant, now, l.TenantCreditsPerMinute, l.TenantBurstCredits) + allowed := (l.ActorCreditsPerMinute <= 0 || ab >= amount) && (l.TenantCreditsPerMinute <= 0 || tb >= amount) + retry := time.Duration(0) + if l.ActorCreditsPerMinute > 0 && ab < amount { + retry = maxDur(retry, time.Duration((amount-ab)/(l.ActorCreditsPerMinute/60)*float64(time.Second))) + } + if l.TenantCreditsPerMinute > 0 && tb < amount { + retry = maxDur(retry, time.Duration((amount-tb)/(l.TenantCreditsPerMinute/60)*float64(time.Second))) + } + if allowed { + if l.ActorCreditsPerMinute > 0 { + ab -= amount + m.buckets["a:"+tenant+":"+actor] = bucket{ab, now} + } + if l.TenantCreditsPerMinute > 0 { + tb -= amount + m.buckets["t:"+tenant] = bucket{tb, now} + } + retry = 0 + } + return Decision{Allowed: allowed, RetryAfter: retry, RemainingActor: ab, RemainingTenant: tb, Reservation: Reservation{tenant, actor, amount, l}}, nil +} +func (m *Memory) refill(key string, now time.Time, rateMin, cap float64) (float64, time.Duration) { + if rateMin <= 0 { + return 1e18, 0 + } + if cap <= 0 { + cap = rateMin + } + b, ok := m.buckets[key] + if !ok { + return cap, 0 + } + bal := b.balance + now.Sub(b.updated).Minutes()*rateMin + if bal > cap { + bal = cap + } + wait := time.Duration(0) + if bal < 0 { + wait = time.Duration((-bal) / (rateMin / 60) * float64(time.Second)) + } + return bal, wait +} +func (m *Memory) Reconcile(_ context.Context, r Reservation, actual float64) error { + delta := r.Amount - actual + if delta == 0 { + return nil + } + m.mu.Lock() + defer m.mu.Unlock() + now := time.Now() + for _, x := range []struct { + key string + rate, cap float64 + }{{"a:" + r.Tenant + ":" + r.Actor, r.Limits.ActorCreditsPerMinute, r.Limits.ActorBurstCredits}, {"t:" + r.Tenant, r.Limits.TenantCreditsPerMinute, r.Limits.TenantBurstCredits}} { + if x.rate <= 0 { + continue + } + bal, _ := m.refill(x.key, now, x.rate, x.cap) + bal += delta + if x.cap <= 0 { + x.cap = x.rate + } + if bal > x.cap { + bal = x.cap + } + m.buckets[x.key] = bucket{bal, now} + } + return nil +} + +func maxDur(a, b time.Duration) time.Duration { + if a > b { + return a + } + return b +} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go new file mode 100644 index 0000000..6385f61 --- /dev/null +++ b/internal/scheduler/scheduler.go @@ -0,0 +1,360 @@ +package scheduler + +import ( + "container/heap" + "context" + "errors" + "math" + "sync" + "time" +) + +var ErrQueueFull = errors.New("scheduler queue full") +var ErrActorQueueFull = errors.New("actor queue full") + +type Request struct { + Tenant, Actor string + Cost float64 + TenantWeight, ActorWeight float64 + Timeout time.Duration + ServiceClass string + ClassWeight float64 + ClassMaxConcurrent int +} + +type ClassStats struct { + Queued int64 `json:"queued"` + Running int64 `json:"running"` +} + +type Stats struct { + Queued int64 `json:"queued"` + Running int64 `json:"running"` + OldestWait time.Duration `json:"oldest_wait,omitempty"` + Classes map[string]ClassStats `json:"classes,omitempty"` +} + +type Scheduler interface { + Acquire(context.Context, Request) (*Lease, error) + Stats(context.Context) Stats + Health(context.Context) error +} + +type Lease struct { + Wait time.Duration + release func() + once sync.Once +} + +func (l *Lease) Release() { + if l != nil && l.release != nil { + l.once.Do(l.release) + } +} + +// Local implements two-level hierarchical weighted fair queueing. Tenants are +// the root fairness boundary. Within a tenant, actor virtual finish time is +// adjusted by an optional service-class weight. Class concurrency caps are +// global across tenants; saturated classes are skipped instead of causing +// head-of-line blocking for other classes. +type localTicket struct { + ctx context.Context + req Request + enqueued time.Time + actorFinish float64 + cancelled bool + dispatched bool + ready chan struct{} + index int +} + +type ticketHeap []*localTicket + +func (h ticketHeap) Len() int { return len(h) } +func (h ticketHeap) Less(i, j int) bool { + if h[i].actorFinish == h[j].actorFinish { + return h[i].enqueued.Before(h[j].enqueued) + } + return h[i].actorFinish < h[j].actorFinish +} +func (h ticketHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i]; h[i].index = i; h[j].index = j } +func (h *ticketHeap) Push(x any) { t := x.(*localTicket); t.index = len(*h); *h = append(*h, t) } +func (h *ticketHeap) Pop() any { + old := *h + n := len(old) + t := old[n-1] + t.index = -1 + *h = old[:n-1] + return t +} + +type tenantState struct { + queue ticketHeap + service float64 + rootScore float64 + active bool +} + +type Local struct { + mu sync.Mutex + tenants map[string]*tenantState + lastActor map[string]float64 + actorQueued map[string]int + classQueued map[string]int + classRunning map[string]int + queued, running int + maxRunning, maxQueue int + maxActor int + virtualTime float64 + wake chan struct{} +} + +func NewLocal(maxRunning, maxQueue, maxActor int) *Local { + l := &Local{maxRunning: maxRunning, maxQueue: maxQueue, maxActor: maxActor, tenants: map[string]*tenantState{}, lastActor: map[string]float64{}, actorQueued: map[string]int{}, classQueued: map[string]int{}, classRunning: map[string]int{}, wake: make(chan struct{}, 1)} + go l.loop() + return l +} +func (l *Local) Health(context.Context) error { return nil } +func (l *Local) Stats(context.Context) Stats { + l.mu.Lock() + defer l.mu.Unlock() + classes := map[string]ClassStats{} + for k, q := range l.classQueued { + classes[k] = ClassStats{Queued: int64(q), Running: int64(l.classRunning[k])} + } + for k, r := range l.classRunning { + if _, ok := classes[k]; !ok { + classes[k] = ClassStats{Running: int64(r)} + } + } + var oldest time.Time + for _, ts := range l.tenants { + for _, ticket := range ts.queue { + if ticket == nil || ticket.index < 0 { + continue + } + if oldest.IsZero() || ticket.enqueued.Before(oldest) { + oldest = ticket.enqueued + } + } + } + oldestWait := time.Duration(0) + if !oldest.IsZero() { + oldestWait = time.Since(oldest) + } + return Stats{Queued: int64(l.queued), Running: int64(l.running), OldestWait: oldestWait, Classes: classes} +} +func (l *Local) Acquire(ctx context.Context, r Request) (*Lease, error) { + normalize(&r) + t := &localTicket{ctx: ctx, req: r, enqueued: time.Now(), ready: make(chan struct{})} + actorKey := r.Tenant + "\x00" + r.Actor + l.mu.Lock() + if l.queued >= l.maxQueue { + l.mu.Unlock() + return nil, ErrQueueFull + } + if l.actorQueued[actorKey] >= l.maxActor { + l.mu.Unlock() + return nil, ErrActorQueueFull + } + ts := l.tenants[r.Tenant] + if ts == nil { + ts = &tenantState{} + heap.Init(&ts.queue) + l.tenants[r.Tenant] = ts + } + base := math.Max(l.lastActor[actorKey], math.Max(ts.service, l.virtualTime)) + t.actorFinish = base + r.Cost/(r.ActorWeight*r.ClassWeight) + l.lastActor[actorKey] = t.actorFinish + heap.Push(&ts.queue, t) + if !ts.active { + ts.rootScore = math.Max(ts.service, l.virtualTime) + ts.active = true + } + l.queued++ + l.actorQueued[actorKey]++ + l.classQueued[r.ServiceClass]++ + l.mu.Unlock() + l.signal() + + var timer *time.Timer + var timeout <-chan time.Time + if r.Timeout > 0 { + timer = time.NewTimer(r.Timeout) + timeout = timer.C + defer timer.Stop() + } + select { + case <-t.ready: + return &Lease{Wait: time.Since(t.enqueued), release: func() { + l.mu.Lock() + if l.running > 0 { + l.running-- + } + if l.classRunning[r.ServiceClass] > 0 { + l.classRunning[r.ServiceClass]-- + } + l.mu.Unlock() + l.signal() + }}, nil + case <-ctx.Done(): + l.cancel(t) + return nil, ctx.Err() + case <-timeout: + l.cancel(t) + return nil, context.DeadlineExceeded + } +} +func (l *Local) cancel(t *localTicket) { + l.mu.Lock() + if t.cancelled { + l.mu.Unlock() + return + } + t.cancelled = true + if t.index >= 0 { + if ts := l.tenants[t.req.Tenant]; ts != nil { + heap.Remove(&ts.queue, t.index) + l.decQueuedLocked(t) + if ts.queue.Len() == 0 { + ts.active = false + delete(l.tenants, t.req.Tenant) + } + } + } else if t.dispatched { + // Acquire selected ctx.Done after dispatch but before consuming ready. + // Reclaim the slot here because no Lease will be returned to release it. + if l.running > 0 { + l.running-- + } + if l.classRunning[t.req.ServiceClass] > 0 { + l.classRunning[t.req.ServiceClass]-- + } + } + l.mu.Unlock() + l.signal() +} +func (l *Local) decQueuedLocked(t *localTicket) { + if l.queued > 0 { + l.queued-- + } + actorKey := t.req.Tenant + "\x00" + t.req.Actor + if l.actorQueued[actorKey] > 0 { + l.actorQueued[actorKey]-- + } + if l.classQueued[t.req.ServiceClass] > 0 { + l.classQueued[t.req.ServiceClass]-- + } +} +func (l *Local) signal() { + select { + case l.wake <- struct{}{}: + default: + } +} +func (l *Local) loop() { + for range l.wake { + for { + l.mu.Lock() + if l.running >= l.maxRunning || l.queued == 0 { + l.mu.Unlock() + break + } + name, ts := l.nextTenantLocked() + if ts == nil { + l.mu.Unlock() + break + } + t := l.nextTicketLocked(name, ts) + if t == nil { + l.mu.Unlock() + continue + } + base := math.Max(ts.rootScore, l.virtualTime) + ts.service = base + t.req.Cost/t.req.TenantWeight + l.virtualTime = math.Max(l.virtualTime, base) + if ts.queue.Len() > 0 { + ts.rootScore = ts.service + } else { + ts.active = false + } + t.dispatched = true + l.running++ + l.classRunning[t.req.ServiceClass]++ + close(t.ready) + l.mu.Unlock() + } + } +} +func (l *Local) classEligibleLocked(t *localTicket) bool { + if t.cancelled || t.ctx.Err() != nil { + return false + } + return t.req.ClassMaxConcurrent <= 0 || l.classRunning[t.req.ServiceClass] < t.req.ClassMaxConcurrent +} +func (l *Local) bestEligibleIndexLocked(ts *tenantState) int { + best := -1 + for i, t := range ts.queue { + if !l.classEligibleLocked(t) { + continue + } + if best < 0 || ts.queue.Less(i, best) { + best = i + } + } + return best +} +func (l *Local) nextTenantLocked() (string, *tenantState) { + var bestName string + var best *tenantState + var bestTicket *localTicket + for name, ts := range l.tenants { + if !ts.active || ts.queue.Len() == 0 { + continue + } + idx := l.bestEligibleIndexLocked(ts) + if idx < 0 { + continue + } + t := ts.queue[idx] + if best == nil || ts.rootScore < best.rootScore || (ts.rootScore == best.rootScore && t.enqueued.Before(bestTicket.enqueued)) { + bestName, best, bestTicket = name, ts, t + } + } + return bestName, best +} +func (l *Local) nextTicketLocked(name string, ts *tenantState) *localTicket { + for ts.queue.Len() > 0 { + idx := l.bestEligibleIndexLocked(ts) + if idx < 0 { + return nil + } + t := heap.Remove(&ts.queue, idx).(*localTicket) + l.decQueuedLocked(t) + if t.cancelled || t.ctx.Err() != nil { + continue + } + return t + } + ts.active = false + delete(l.tenants, name) + return nil +} + +func normalize(r *Request) { + if r.TenantWeight <= 0 { + r.TenantWeight = 1 + } + if r.ActorWeight <= 0 { + r.ActorWeight = 1 + } + if r.ClassWeight <= 0 { + r.ClassWeight = 1 + } + if r.ServiceClass == "" { + r.ServiceClass = "default" + } + if r.Cost <= 0 { + r.Cost = .001 + } +} diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go new file mode 100644 index 0000000..a88595e --- /dev/null +++ b/internal/scheduler/scheduler_test.go @@ -0,0 +1,194 @@ +package scheduler + +import ( + "context" + "testing" + "time" +) + +type acquired struct { + name string + lease *Lease + err error +} + +func waitQueued(t *testing.T, s *Local, n int) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if s.Stats(context.Background()).Queued == int64(n) { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("queue never reached %d", n) +} + +func TestLocalFairnessWithinTenant(t *testing.T) { + s := NewLocal(1, 20, 20) + ctx := context.Background() + first, err := s.Acquire(ctx, Request{Tenant: "T", Actor: "T/A", Cost: 1, TenantWeight: 1, ActorWeight: 1, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + ch := make(chan acquired, 2) + go func() { + l, e := s.Acquire(ctx, Request{Tenant: "T", Actor: "T/A", Cost: 10, TenantWeight: 1, ActorWeight: 1, Timeout: time.Second}) + ch <- acquired{"A-heavy", l, e} + }() + waitQueued(t, s, 1) + go func() { + l, e := s.Acquire(ctx, Request{Tenant: "T", Actor: "T/B", Cost: 1, TenantWeight: 1, ActorWeight: 1, Timeout: time.Second}) + ch <- acquired{"B-small", l, e} + }() + waitQueued(t, s, 2) + first.Release() + got := <-ch + if got.err != nil { + t.Fatal(got.err) + } + defer got.lease.Release() + if got.name != "B-small" { + t.Fatalf("expected small second actor first, got %s", got.name) + } +} + +func TestLocalTenantFairness(t *testing.T) { + s := NewLocal(1, 20, 20) + ctx := context.Background() + first, err := s.Acquire(ctx, Request{Tenant: "A", Actor: "A/u", Cost: 1, TenantWeight: 1, ActorWeight: 1, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + ch := make(chan acquired, 2) + go func() { + l, e := s.Acquire(ctx, Request{Tenant: "A", Actor: "A/u", Cost: 1, TenantWeight: 1, ActorWeight: 1, Timeout: time.Second}) + ch <- acquired{"A2", l, e} + }() + waitQueued(t, s, 1) + go func() { + l, e := s.Acquire(ctx, Request{Tenant: "B", Actor: "B/u", Cost: 1, TenantWeight: 1, ActorWeight: 1, Timeout: time.Second}) + ch <- acquired{"B1", l, e} + }() + waitQueued(t, s, 2) + first.Release() + got := <-ch + if got.err != nil { + t.Fatal(got.err) + } + defer got.lease.Release() + if got.name != "B1" { + t.Fatalf("expected other tenant next, got %s", got.name) + } +} + +func TestActorQueueIsolatedByTenant(t *testing.T) { + s := NewLocal(1, 20, 1) + ctx := context.Background() + first, err := s.Acquire(ctx, Request{Tenant: "root", Actor: "holder", Cost: 1, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer first.Release() + + ch := make(chan error, 2) + go func() { + l, e := s.Acquire(ctx, Request{Tenant: "A", Actor: "same", Cost: 1, Timeout: time.Second}) + if l != nil { + l.Release() + } + ch <- e + }() + waitQueued(t, s, 1) + go func() { + l, e := s.Acquire(ctx, Request{Tenant: "B", Actor: "same", Cost: 1, Timeout: time.Second}) + if l != nil { + l.Release() + } + ch <- e + }() + waitQueued(t, s, 2) + first.Release() + for i := 0; i < 2; i++ { + if e := <-ch; e != nil { + t.Fatalf("tenant-isolated actor queue failed: %v", e) + } + } +} + +func TestCancelRemovesQueuedTicketImmediately(t *testing.T) { + s := NewLocal(1, 20, 20) + holder, err := s.Acquire(context.Background(), Request{Tenant: "A", Actor: "holder", Cost: 1, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer holder.Release() + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + l, err := s.Acquire(ctx, Request{Tenant: "B", Actor: "user", Cost: 1, Timeout: time.Second}) + if l != nil { + l.Release() + } + done <- err + }() + waitQueued(t, s, 1) + cancel() + select { + case err := <-done: + if err != context.Canceled { + t.Fatalf("err=%v want context.Canceled", err) + } + case <-time.After(time.Second): + t.Fatal("cancelled Acquire did not return") + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if s.Stats(context.Background()).Queued == 0 { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("queue still contains cancelled ticket: %+v", s.Stats(context.Background())) +} + +func TestServiceClassCapDoesNotHeadOfLineBlock(t *testing.T) { + s := NewLocal(2, 20, 20) + ctx := context.Background() + bg, err := s.Acquire(ctx, Request{Tenant: "T", Actor: "bg-holder", Cost: 1, ServiceClass: "background", ClassWeight: 1, ClassMaxConcurrent: 1, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer bg.Release() + sys, err := s.Acquire(ctx, Request{Tenant: "T", Actor: "sys-holder", Cost: 1, ServiceClass: "system", ClassWeight: 1, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + + ch := make(chan acquired, 2) + go func() { + l, e := s.Acquire(ctx, Request{Tenant: "T", Actor: "bg2", Cost: 1, ServiceClass: "background", ClassWeight: 10, ClassMaxConcurrent: 1, Timeout: time.Second}) + ch <- acquired{"background", l, e} + }() + waitQueued(t, s, 1) + go func() { + l, e := s.Acquire(ctx, Request{Tenant: "T", Actor: "chat", Cost: 1, ServiceClass: "interactive", ClassWeight: 1, Timeout: time.Second}) + ch <- acquired{"interactive", l, e} + }() + waitQueued(t, s, 2) + + sys.Release() + select { + case got := <-ch: + if got.err != nil { + t.Fatal(got.err) + } + if got.name != "interactive" { + t.Fatalf("expected interactive to bypass saturated background class, got %s", got.name) + } + got.lease.Release() + case <-time.After(time.Second): + t.Fatal("interactive request was head-of-line blocked") + } +} diff --git a/internal/server/aliases.go b/internal/server/aliases.go new file mode 100644 index 0000000..d26c14d --- /dev/null +++ b/internal/server/aliases.go @@ -0,0 +1,183 @@ +package server + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "sort" + "strings" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/proxy" +) + +type configOverrideLoader interface { + LoadWithBootstrap(*config.Config) (*config.Config, error) +} + +func cloneModelAliases(in map[string]config.ModelAliasConfig) map[string]config.ModelAliasConfig { + out := make(map[string]config.ModelAliasConfig, len(in)) + for name, a := range in { + a.Models = append([]string(nil), a.Models...) + a.RequiredCapabilities = append([]string(nil), a.RequiredCapabilities...) + if a.Visible != nil { + v := *a.Visible + a.Visible = &v + } + out[name] = a + } + return out +} + +func (s *Server) aliasSnapshot() map[string]config.ModelAliasConfig { + if s == nil { + return map[string]config.ModelAliasConfig{} + } + v := s.aliases.Load() + if v == nil { + return cloneModelAliases(s.cfg.ModelAliases) + } + return cloneModelAliases(v.(map[string]config.ModelAliasConfig)) +} + +func (s *Server) aliasConfig(name string) (config.ModelAliasConfig, bool) { + if s == nil { + return config.ModelAliasConfig{}, false + } + v := s.aliases.Load() + if v == nil { + a, ok := s.cfg.ModelAliases[name] + return a, ok + } + a, ok := v.(map[string]config.ModelAliasConfig)[name] + return a, ok +} + +func (s *Server) storeAliases(next map[string]config.ModelAliasConfig) error { + if s.configStore == nil { + return errors.New("persistent configuration store unavailable") + } + base := s.cfg + if loader, ok := s.configStore.(configOverrideLoader); ok { + if loaded, err := loader.LoadWithBootstrap(s.cfg); err == nil { + base = loaded + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + } + b, err := json.Marshal(base) + if err != nil { + return err + } + var candidate config.Config + if err := json.Unmarshal(b, &candidate); err != nil { + return err + } + candidate.ModelAliases = cloneModelAliases(next) + if err := validateAliasSet(candidate.ModelAliases); err != nil { + return err + } + if err := s.configStore.Save(&candidate); err != nil { + return err + } + // Publish only after persistence succeeds. Readers never observe a map that + // would be lost on restart, and the stored map is never mutated in place. + s.aliases.Store(cloneModelAliases(next)) + return nil +} + +func validateAliasSet(aliases map[string]config.ModelAliasConfig) error { + for name, a := range aliases { + if strings.TrimSpace(name) == "" { + return errors.New("model alias name is required") + } + if len(a.Models) == 0 { + return fmt.Errorf("model alias %q requires at least one model", name) + } + for _, model := range a.Models { + if strings.TrimSpace(model) == "" { + return fmt.Errorf("model alias %q contains an empty model", name) + } + } + } + return nil +} + +func (s *Server) uiModelAliases(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if r.URL.Path == "/gateway/ui-api/model-aliases" { + if r.Method != http.MethodGet { + proxy.WriteJSONError(w, http.StatusMethodNotAllowed, "method_not_allowed", "GET required") + return + } + aliases := s.aliasSnapshot() + names := make([]string, 0, len(aliases)) + for name := range aliases { + names = append(names, name) + } + sort.Strings(names) + writeJSON(w, http.StatusOK, map[string]any{"aliases": aliases, "names": names, "runtime": true, "persistent": s.configStore != nil}) + return + } + + raw := strings.TrimPrefix(r.URL.Path, "/gateway/ui-api/model-aliases/") + name, err := url.PathUnescape(raw) + name = strings.TrimSpace(name) + if err != nil || name == "" || strings.Contains(name, "/") || len(name) > 256 { + proxy.WriteJSONError(w, http.StatusBadRequest, "bad_alias", "invalid model alias name") + return + } + if s.configStore == nil { + proxy.WriteJSONError(w, http.StatusServiceUnavailable, "config_store", "persistent configuration store unavailable") + return + } + + s.aliasMu.Lock() + defer s.aliasMu.Unlock() + next := s.aliasSnapshot() + switch r.Method { + case http.MethodPut: + var in config.ModelAliasConfig + if err := decodeJSON(r, &in, 128<<10); err != nil { + proxy.WriteJSONError(w, http.StatusBadRequest, "bad_alias", err.Error()) + return + } + in.Models = cleanStrings(in.Models) + in.RequiredCapabilities = cleanStrings(in.RequiredCapabilities) + next[name] = in + if err := s.storeAliases(next); err != nil { + proxy.WriteJSONError(w, http.StatusBadRequest, "alias_store", err.Error()) + return + } + s.log.Info("model alias saved", "alias", name, "models", len(in.Models), "admin_subject", actor.Subject, "admin_auth_type", actor.AuthType) + writeJSON(w, http.StatusOK, map[string]any{"name": name, "alias": in, "restart_required": false}) + case http.MethodDelete: + if _, ok := next[name]; !ok { + proxy.WriteJSONError(w, http.StatusNotFound, "alias_not_found", fmt.Sprintf("model alias %q not found", name)) + return + } + delete(next, name) + if err := s.storeAliases(next); err != nil { + proxy.WriteJSONError(w, http.StatusBadRequest, "alias_store", err.Error()) + return + } + s.log.Info("model alias deleted", "alias", name, "admin_subject", actor.Subject, "admin_auth_type", actor.AuthType) + writeJSON(w, http.StatusOK, map[string]any{"deleted": true, "name": name, "restart_required": false}) + default: + proxy.WriteJSONError(w, http.StatusMethodNotAllowed, "method_not_allowed", "PUT or DELETE required") + } +} + +func cleanStrings(in []string) []string { + out := make([]string, 0, len(in)) + for _, v := range in { + v = strings.TrimSpace(v) + if v != "" { + out = append(out, v) + } + } + return out +} diff --git a/internal/server/batch_api.go b/internal/server/batch_api.go new file mode 100644 index 0000000..fcf8894 --- /dev/null +++ b/internal/server/batch_api.go @@ -0,0 +1,298 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "os" + "sort" + "strconv" + "strings" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/batch" +) + +type batchCreateRequest struct { + Path string `json:"path"` + Body json.RawMessage `json:"body"` +} + +func (s *Server) batchAPI(w http.ResponseWriter, r *http.Request, id auth.Identity) { + if s.batchJobs == nil || !s.batchJobs.Enabled() { + writeProtocolError(w, r, http.StatusNotFound, "batch_disabled", "durable batch jobs are disabled") + return + } + base := "/gateway/v1/batches" + rest := strings.TrimPrefix(r.URL.Path, base) + if rest == "" || rest == "/" { + switch r.Method { + case http.MethodGet: + writeJSON(w, http.StatusOK, map[string]any{"jobs": s.batchJobs.List(id.Tenant, id.Actor(), false)}) + case http.MethodPost: + s.batchCreate(w, r, id) + default: + writeProtocolError(w, r, http.StatusMethodNotAllowed, "method_not_allowed", "method not allowed") + } + return + } + parts := strings.Split(strings.Trim(rest, "/"), "/") + if len(parts) == 0 || parts[0] == "" { + writeProtocolError(w, r, http.StatusNotFound, "not_found", "batch job not found") + return + } + jobID := parts[0] + if len(parts) == 1 && r.Method == http.MethodGet { + j, ok := s.batchJobs.Get(jobID, id.Tenant, id.Actor(), false) + if !ok { + writeProtocolError(w, r, http.StatusNotFound, "batch_not_found", "batch job not found") + return + } + writeJSON(w, http.StatusOK, j) + return + } + if len(parts) == 2 && parts[1] == "output" && r.Method == http.MethodGet { + s.batchOutput(w, r, id, jobID, false) + return + } + if len(parts) == 2 && r.Method == http.MethodPost { + var ( + j batch.Job + err error + ) + switch parts[1] { + case "pause": + j, err = s.batchJobs.Pause(jobID, id.Tenant, id.Actor(), false) + case "resume": + j, err = s.batchJobs.Resume(jobID, id.Tenant, id.Actor(), false) + case "cancel": + j, err = s.batchJobs.Cancel(jobID, id.Tenant, id.Actor(), false) + default: + writeProtocolError(w, r, http.StatusNotFound, "not_found", "unknown batch operation") + return + } + if err != nil { + s.writeBatchError(w, r, err) + return + } + writeJSON(w, http.StatusOK, j) + return + } + writeProtocolError(w, r, http.StatusNotFound, "not_found", "unknown batch endpoint") +} + +func (s *Server) batchCreate(w http.ResponseWriter, r *http.Request, id auth.Identity) { + limit := s.cfg.BatchJobs.MaxInputBytes + 64<<10 + if limit <= 0 { + limit = 16 << 20 + } + var req batchCreateRequest + if err := decodeJSON(r, &req, limit); err != nil { + writeProtocolError(w, r, http.StatusBadRequest, "bad_batch", err.Error()) + return + } + if !s.isCompute(http.MethodPost, req.Path) { + writeProtocolError(w, r, http.StatusBadRequest, "bad_batch_path", "batch path must be a configured compute POST endpoint") + return + } + if len(req.Body) == 0 || string(req.Body) == "null" || !json.Valid(req.Body) { + writeProtocolError(w, r, http.StatusBadRequest, "bad_batch_body", "batch body must contain a valid JSON request body") + return + } + model := modelFromBody(req.Body) + if _, _, err := s.resolveModel(r.Context(), id, model); err != nil { + if errors.Is(err, ErrModelAccessDenied) { + writeProtocolError(w, r, http.StatusForbidden, "model_access_denied", err.Error()) + } else { + writeProtocolError(w, r, http.StatusNotFound, "model_alias_unavailable", err.Error()) + } + return + } + j, err := s.batchJobs.Create(batchIdentitySnapshot(id), req.Path, model, req.Body) + if err != nil { + s.writeBatchError(w, r, err) + return + } + w.Header().Set("Location", "/gateway/v1/batches/"+j.ID) + writeJSON(w, http.StatusAccepted, j) +} + +func (s *Server) batchOutput(w http.ResponseWriter, r *http.Request, id auth.Identity, jobID string, all bool) { + f, j, err := s.batchJobs.OpenOutput(jobID, id.Tenant, id.Actor(), all) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + writeProtocolError(w, r, http.StatusConflict, "batch_output_unavailable", "batch output is not available yet") + return + } + s.writeBatchError(w, r, err) + return + } + defer f.Close() + ct := strings.TrimSpace(j.ResponseContentType) + if ct == "" { + ct = "application/octet-stream" + } + w.Header().Set("Content-Type", ct) + w.Header().Set("Cache-Control", "no-store") + if st, err := f.Stat(); err == nil { + w.Header().Set("Content-Length", strconv.FormatInt(st.Size(), 10)) + } + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, f) +} + +func (s *Server) writeBatchError(w http.ResponseWriter, r *http.Request, err error) { + switch { + case errors.Is(err, batch.ErrDisabled): + writeProtocolError(w, r, http.StatusNotFound, "batch_disabled", err.Error()) + case errors.Is(err, batch.ErrNotFound): + writeProtocolError(w, r, http.StatusNotFound, "batch_not_found", "batch job not found") + case errors.Is(err, batch.ErrInvalidState): + writeProtocolError(w, r, http.StatusConflict, "batch_state", err.Error()) + case errors.Is(err, batch.ErrFull): + writeProtocolError(w, r, http.StatusTooManyRequests, "batch_full", err.Error()) + case errors.Is(err, batch.ErrInputTooLarge): + writeProtocolError(w, r, http.StatusRequestEntityTooLarge, "batch_input_too_large", err.Error()) + default: + writeProtocolError(w, r, http.StatusInternalServerError, "batch_error", err.Error()) + } +} + +// ExecuteBatch is the runner used by the durable batch manager. The request is +// replayed directly into the normal compute path with the original identity +// metadata but the batch service class, so quotas, ACLs, scheduling, routing, +// metering, alerts and OpenTelemetry stay consistent with interactive traffic. +func (s *Server) ExecuteBatch(ctx context.Context, j batch.Job, input io.Reader, output io.Writer) batch.RunResult { + id := auth.Identity{ + Tenant: j.Identity.Tenant, + Subject: j.Identity.Subject, + Application: j.Identity.Application, + AuthType: j.Identity.AuthType, + ClientIP: j.Identity.ClientIP, + Scopes: make(map[string]bool, len(j.Identity.Scopes)), + ModelACLSet: j.Identity.ModelACLSet, + ModelAccess: j.Identity.ModelAccess, + ServiceClass: "batch", + } + for _, scope := range j.Identity.Scopes { + id.Scopes[scope] = true + } + r, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://gateway.local"+j.Path, input) + if err != nil { + return batch.RunResult{Error: err.Error()} + } + r.Header.Set("Content-Type", "application/json") + r.RemoteAddr = "127.0.0.1:0" + rw := &batchResponseWriter{header: make(http.Header), out: output} + s.forward(rw, r, id) + status := rw.status + if status == 0 { + status = http.StatusOK + } + return batch.RunResult{HTTPStatus: status, ResponseContentType: rw.header.Get("Content-Type"), RequestID: rw.header.Get("X-Request-ID")} +} + +type batchResponseWriter struct { + header http.Header + out io.Writer + status int +} + +func (w *batchResponseWriter) Header() http.Header { return w.header } +func (w *batchResponseWriter) WriteHeader(status int) { + if w.status == 0 { + w.status = status + } +} +func (w *batchResponseWriter) Write(p []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + return w.out.Write(p) +} +func (w *batchResponseWriter) Flush() {} + +func batchIdentitySnapshot(id auth.Identity) batch.IdentitySnapshot { + scopes := make([]string, 0, len(id.Scopes)) + for scope, ok := range id.Scopes { + if ok { + scopes = append(scopes, scope) + } + } + sort.Strings(scopes) + return batch.IdentitySnapshot{Tenant: id.Tenant, Subject: id.Subject, Actor: id.Actor(), Application: id.Application, AuthType: id.AuthType, ClientIP: id.ClientIP, Scopes: scopes, ModelACLSet: id.ModelACLSet, ModelAccess: id.ModelAccess} +} + +func (s *Server) uiBatchJobs(w http.ResponseWriter, r *http.Request, id auth.Identity) { + base := "/gateway/ui-api/batches" + if s.batchJobs == nil || !s.batchJobs.Enabled() { + if r.URL.Path == base && r.Method == http.MethodGet { + writeJSON(w, http.StatusOK, map[string]any{"enabled": false, "jobs": []batch.Job{}}) + return + } + writeProtocolError(w, r, http.StatusNotFound, "batch_disabled", "durable batch jobs are disabled") + return + } + rest := strings.TrimPrefix(r.URL.Path, base) + if rest == "" || rest == "/" { + if r.Method != http.MethodGet { + writeProtocolError(w, r, http.StatusMethodNotAllowed, "method_not_allowed", "method not allowed") + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "enabled": true, + "jobs": s.batchJobs.List("", "", true), + "retention": s.cfg.BatchJobs.Retention.Value().String(), + "max_jobs": s.cfg.BatchJobs.MaxJobs, + "max_concurrent": s.cfg.BatchJobs.MaxConcurrent, + "max_input_bytes": s.cfg.BatchJobs.MaxInputBytes, + }) + return + } + parts := strings.Split(strings.Trim(rest, "/"), "/") + if len(parts) == 0 || parts[0] == "" { + writeProtocolError(w, r, http.StatusNotFound, "batch_not_found", "batch job not found") + return + } + jobID := parts[0] + if len(parts) == 1 && r.Method == http.MethodGet { + j, ok := s.batchJobs.Get(jobID, "", "", true) + if !ok { + writeProtocolError(w, r, http.StatusNotFound, "batch_not_found", "batch job not found") + return + } + writeJSON(w, http.StatusOK, j) + return + } + if len(parts) == 2 && parts[1] == "output" && r.Method == http.MethodGet { + s.batchOutput(w, r, id, jobID, true) + return + } + if len(parts) == 2 && r.Method == http.MethodPost { + var ( + j batch.Job + err error + ) + switch parts[1] { + case "pause": + j, err = s.batchJobs.Pause(jobID, "", "", true) + case "resume": + j, err = s.batchJobs.Resume(jobID, "", "", true) + case "cancel": + j, err = s.batchJobs.Cancel(jobID, "", "", true) + default: + writeProtocolError(w, r, http.StatusNotFound, "not_found", "unknown batch operation") + return + } + if err != nil { + s.writeBatchError(w, r, err) + return + } + s.log.Info("durable batch control", "batch_id", jobID, "action", parts[1], "admin_subject", id.Subject, "admin_auth_type", id.AuthType) + writeJSON(w, http.StatusOK, j) + return + } + writeProtocolError(w, r, http.StatusNotFound, "not_found", "unknown batch UI endpoint") +} diff --git a/internal/server/batch_e2e_test.go b/internal/server/batch_e2e_test.go new file mode 100644 index 0000000..a7bfb6e --- /dev/null +++ b/internal/server/batch_e2e_test.go @@ -0,0 +1,165 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/batch" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/cost" + "github.com/example/ollama-fair-gateway/internal/metrics" + px "github.com/example/ollama-fair-gateway/internal/proxy" + "github.com/example/ollama-fair-gateway/internal/quota" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/usage" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +func TestDurableBatchEndToEndThroughGatewayPipeline(t *testing.T) { + var upstreamBody string + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"models":[{"name":"qwen3:8b","model":"qwen3:8b"}]}`) + case "/api/tags": + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"models":[{"name":"qwen3:8b"}]}`) + case "/api/chat": + b, _ := io.ReadAll(r.Body) + upstreamBody = string(b) + w.Header().Set("Content-Type", "application/x-ndjson") + _, _ = io.WriteString(w, "{\"message\":{\"content\":\"batch-ok\"},\"done\":true,\"prompt_eval_count\":3,\"eval_count\":2}\n") + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer backend.Close() + + cfg := &config.Config{ + Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute)}, + Auth: config.AuthConfig{APIKeys: []config.APIKeyConfig{{Name: "client", Key: "batch-key", Tenant: "team-a", Subject: "alice"}}}, + Scheduler: config.SchedulerConfig{GlobalConcurrency: 2, MaxQueue: 32, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, ComputePaths: []string{"/api/chat"}}, + Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 16}, + Workers: []config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 2, HealthInterval: config.Duration(time.Hour)}}, + ServiceClasses: config.ServiceClassesConfig{Default: "interactive", Classes: map[string]config.ServiceClassConfig{ + "interactive": {Weight: 1, MaxQueueWait: config.Duration(time.Second), MaxConcurrent: 2}, + "batch": {Weight: 0.25, MaxQueueWait: config.Duration(time.Second), MaxConcurrent: 1}, + }}, + BatchJobs: config.BatchJobsConfig{Enabled: true, Retention: config.Duration(time.Hour), MaxJobs: 100, MaxConcurrent: 1, MaxInputBytes: 1 << 20}, + } + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "w") + root, cancel := context.WithCancel(context.Background()) + defer cancel() + wp.Start(root) + rec, err := usage.New("", 100, time.Second, nil) + if err != nil { + t.Fatal(err) + } + defer rec.Close() + dir := t.TempDir() + bm, err := batch.New(cfg.BatchJobs, filepath.Join(dir, "batch-jobs.json"), filepath.Join(dir, "batch")) + if err != nil { + t.Fatal(err) + } + sv := New(cfg, Dependencies{Auth: a, Scheduler: scheduler.NewLocal(2, 32, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: px.New(), Usage: rec, Metrics: metrics.New(), Logger: slog.Default(), BatchJobs: bm}) + bm.Start(root, sv.ExecuteBatch) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + payload := []byte(`{"path":"/api/chat","body":{"model":"qwen3:8b","messages":[{"role":"user","content":"run batch"}]}}`) + req, _ := http.NewRequest(http.MethodPost, front.URL+"/gateway/v1/batches", bytes.NewReader(payload)) + req.Header.Set("Authorization", "Bearer batch-key") + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("create status=%d body=%s", resp.StatusCode, body) + } + var created batch.Job + if err := json.Unmarshal(body, &created); err != nil { + t.Fatal(err) + } + if created.ID == "" || created.ServiceClass != "batch" || created.Identity.Tenant != "team-a" || created.Identity.Actor != "alice" { + t.Fatalf("created=%#v", created) + } + if !strings.HasPrefix(resp.Header.Get("Location"), "/gateway/v1/batches/") { + t.Fatalf("location=%q", resp.Header.Get("Location")) + } + + var completed batch.Job + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + req, _ = http.NewRequest(http.MethodGet, front.URL+"/gateway/v1/batches/"+created.ID, nil) + req.Header.Set("Authorization", "Bearer batch-key") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("get status=%d body=%s", resp.StatusCode, body) + } + if err := json.Unmarshal(body, &completed); err != nil { + t.Fatal(err) + } + if completed.State == batch.StateCompleted { + break + } + if completed.State == batch.StateFailed || completed.State == batch.StateCancelled { + t.Fatalf("unexpected terminal state: %#v", completed) + } + time.Sleep(10 * time.Millisecond) + } + if completed.State != batch.StateCompleted || completed.HTTPStatus != http.StatusOK || completed.ExecutionRequestID == "" || completed.OutputRef == "" { + t.Fatalf("completed=%#v", completed) + } + if !strings.Contains(upstreamBody, `"model":"qwen3:8b"`) || !strings.Contains(upstreamBody, `"run batch"`) { + t.Fatalf("upstream body=%s", upstreamBody) + } + + req, _ = http.NewRequest(http.MethodGet, front.URL+"/gateway/v1/batches/"+created.ID+"/output", nil) + req.Header.Set("Authorization", "Bearer batch-key") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"batch-ok"`) { + t.Fatalf("output status=%d body=%s", resp.StatusCode, body) + } + + events := rec.Recent(10) + found := false + for _, e := range events { + if e.ID == completed.ExecutionRequestID { + found = true + if e.ServiceClass != "batch" || e.Tenant != "team-a" || e.Actor != "alice" || e.Usage.PromptTokens != 3 || e.Usage.CompletionTokens != 2 { + t.Fatalf("usage event=%#v", e) + } + } + } + if !found { + t.Fatalf("execution request %s missing from usage: %#v", completed.ExecutionRequestID, events) + } +} diff --git a/internal/server/conversations.go b/internal/server/conversations.go new file mode 100644 index 0000000..6bdd58c --- /dev/null +++ b/internal/server/conversations.go @@ -0,0 +1,185 @@ +package server + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/conversation" +) + +type responseConversationPlan struct { + Store bool + RequestItems []any +} + +// prepareResponseConversation expands previous_response_id into a flattened +// Responses API input history when the optional encrypted conversation store +// is enabled. When disabled the request is left untouched, preserving the +// gateway's content-free default behavior and any upstream-native semantics. +func (s *Server) prepareResponseConversation(body []byte, id auth.Identity) ([]byte, *responseConversationPlan, error) { + if s.conversations == nil || !s.conversations.Enabled() || len(body) == 0 { + return body, nil, nil + } + var req map[string]any + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + if err := dec.Decode(&req); err != nil { + return nil, nil, fmt.Errorf("decode Responses request: %w", err) + } + store := true + if v, ok := req["store"].(bool); ok && !v { + store = false + } + current, err := responseInputItems(req["input"]) + if err != nil { + return nil, nil, err + } + items := current + if rawPrev, ok := req["previous_response_id"]; ok { + prev, ok := rawPrev.(string) + prev = strings.TrimSpace(prev) + if !ok || prev == "" { + return nil, nil, errors.New("previous_response_id must be a non-empty string") + } + parent, found, err := s.conversations.Get(prev, id.Tenant, id.Actor()) + if err != nil { + return nil, nil, fmt.Errorf("load previous response: %w", err) + } + if !found { + return nil, nil, fmt.Errorf("previous_response_id %q was not found for this identity", prev) + } + var prior []any + if err := json.Unmarshal(parent.Context, &prior); err != nil { + return nil, nil, fmt.Errorf("decode stored conversation context: %w", err) + } + items = make([]any, 0, len(prior)+len(current)) + items = append(items, prior...) + items = append(items, current...) + req["input"] = items + delete(req, "previous_response_id") + } + out, err := json.Marshal(req) + if err != nil { + return nil, nil, fmt.Errorf("encode expanded Responses request: %w", err) + } + return out, &responseConversationPlan{Store: store, RequestItems: append([]any(nil), items...)}, nil +} + +func responseInputItems(v any) ([]any, error) { + if v == nil { + return nil, nil + } + switch x := v.(type) { + case string: + return []any{map[string]any{"role": "user", "content": x}}, nil + case []any: + return append([]any(nil), x...), nil + case map[string]any: + return []any{x}, nil + default: + return nil, errors.New("Responses input must be a string, object, or array") + } +} + +func (s *Server) persistResponseConversation(plan *responseConversationPlan, captured []byte, truncated bool, id auth.Identity, model string) { + if plan == nil || !plan.Store || s.conversations == nil || !s.conversations.Enabled() { + return + } + if truncated { + s.log.Warn("conversation response not stored because capture exceeded limit", "tenant", id.Tenant, "actor", id.Actor(), "model", model) + return + } + responseID, output, err := parseResponsesOutput(captured) + if err != nil { + s.log.Warn("conversation response not stored", "tenant", id.Tenant, "actor", id.Actor(), "model", model, "error", err) + return + } + if responseID == "" { + s.log.Warn("conversation response not stored because response id was absent", "tenant", id.Tenant, "actor", id.Actor(), "model", model) + return + } + ctx := make([]any, 0, len(plan.RequestItems)+len(output)) + ctx = append(ctx, plan.RequestItems...) + ctx = append(ctx, output...) + raw, err := json.Marshal(ctx) + if err != nil { + s.log.Warn("conversation context encode failed", "response_id", responseID, "error", err) + return + } + if err := s.conversations.Put(conversation.Entry{ID: responseID, Tenant: id.Tenant, Actor: id.Actor(), Model: model, CreatedAt: time.Now().UTC(), Context: raw}); err != nil { + s.log.Warn("conversation persistence failed", "response_id", responseID, "tenant", id.Tenant, "actor", id.Actor(), "error", err) + } +} + +func parseResponsesOutput(body []byte) (string, []any, error) { + body = bytes.TrimSpace(body) + if len(body) == 0 { + return "", nil, errors.New("empty Responses response") + } + if body[0] == '{' { + var v map[string]any + if err := json.Unmarshal(body, &v); err != nil { + return "", nil, err + } + return responseIDAndOutput(v) + } + var responseID string + var completedOutput []any + var doneOutput []any + for _, rawLine := range bytes.Split(body, []byte{'\n'}) { + line := bytes.TrimSpace(rawLine) + if !bytes.HasPrefix(line, []byte("data:")) { + continue + } + data := bytes.TrimSpace(bytes.TrimPrefix(line, []byte("data:"))) + if len(data) == 0 || bytes.Equal(data, []byte("[DONE]")) { + continue + } + var ev map[string]any + if json.Unmarshal(data, &ev) != nil { + continue + } + typ, _ := ev["type"].(string) + if resp, ok := ev["response"].(map[string]any); ok { + id, out, _ := responseIDAndOutput(resp) + if id != "" { + responseID = id + } + if typ == "response.completed" && out != nil { + completedOutput = out + } + } + if typ == "response.output_item.done" { + if item, ok := ev["item"].(map[string]any); ok { + doneOutput = append(doneOutput, item) + } + } + } + if responseID == "" { + return "", nil, errors.New("stream did not contain a response id") + } + if completedOutput != nil { + return responseID, completedOutput, nil + } + if len(doneOutput) > 0 { + return responseID, doneOutput, nil + } + return "", nil, errors.New("stream did not contain completed output items") +} + +func responseIDAndOutput(v map[string]any) (string, []any, error) { + id, _ := v["id"].(string) + out, _ := v["output"].([]any) + if id == "" { + return "", nil, errors.New("response id is missing") + } + if out == nil { + return id, nil, errors.New("response output is missing") + } + return id, out, nil +} diff --git a/internal/server/conversations_test.go b/internal/server/conversations_test.go new file mode 100644 index 0000000..f0d8ea6 --- /dev/null +++ b/internal/server/conversations_test.go @@ -0,0 +1,222 @@ +package server + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/conversation" + "github.com/example/ollama-fair-gateway/internal/cost" + "github.com/example/ollama-fair-gateway/internal/metrics" + "github.com/example/ollama-fair-gateway/internal/proxy" + "github.com/example/ollama-fair-gateway/internal/quota" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/usage" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +func newConversationTestServer(t *testing.T) (*Server, *conversation.Store) { + t.Helper() + cc := config.ConversationsConfig{Enabled: true, EncryptionKey: strings.Repeat("k", 32), Retention: config.Duration(time.Hour), MaxEntries: 100, MaxContentBytes: 1 << 20} + store, err := conversation.New(cc, filepath.Join(t.TempDir(), "conversations.enc.json")) + if err != nil { + t.Fatal(err) + } + return &Server{cfg: &config.Config{Conversations: cc}, conversations: store, log: slog.New(slog.NewTextHandler(io.Discard, nil))}, store +} + +func TestPrepareResponseConversationExpandsPreviousResponse(t *testing.T) { + s, store := newConversationTestServer(t) + id := auth.Identity{Tenant: "tenant-a", Subject: "user-a", AuthType: "oidc"} + prior := json.RawMessage(`[{"role":"user","content":"first"},{"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer"}]}]`) + if err := store.Put(conversation.Entry{ID: "resp_prev", Tenant: id.Tenant, Actor: id.Actor(), Context: prior}); err != nil { + t.Fatal(err) + } + + body, plan, err := s.prepareResponseConversation([]byte(`{"model":"qwen3:8b","previous_response_id":"resp_prev","input":"follow up"}`), id) + if err != nil { + t.Fatal(err) + } + if plan == nil || !plan.Store || len(plan.RequestItems) != 3 { + t.Fatalf("plan=%#v", plan) + } + var req map[string]any + if err := json.Unmarshal(body, &req); err != nil { + t.Fatal(err) + } + if _, exists := req["previous_response_id"]; exists { + t.Fatal("previous_response_id must not be forwarded after expansion") + } + items, ok := req["input"].([]any) + if !ok || len(items) != 3 { + t.Fatalf("expanded input=%#v", req["input"]) + } + last := items[2].(map[string]any) + if last["role"] != "user" || last["content"] != "follow up" { + t.Fatalf("last item=%#v", last) + } +} + +func TestPrepareResponseConversationDoesNotCrossIdentityBoundary(t *testing.T) { + s, store := newConversationTestServer(t) + if err := store.Put(conversation.Entry{ID: "resp_prev", Tenant: "tenant-a", Actor: "user-a", Context: json.RawMessage(`[]`)}); err != nil { + t.Fatal(err) + } + _, _, err := s.prepareResponseConversation([]byte(`{"previous_response_id":"resp_prev","input":"x"}`), auth.Identity{Tenant: "tenant-a", Subject: "user-b", AuthType: "oidc"}) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("err=%v", err) + } +} + +func TestPrepareResponseConversationRespectsStoreFalse(t *testing.T) { + s, _ := newConversationTestServer(t) + _, plan, err := s.prepareResponseConversation([]byte(`{"input":"x","store":false}`), auth.Identity{Tenant: "t", Subject: "u", AuthType: "oidc"}) + if err != nil { + t.Fatal(err) + } + if plan == nil || plan.Store { + t.Fatalf("plan=%#v", plan) + } +} + +func TestParseResponsesOutputJSONAndSSE(t *testing.T) { + id, out, err := parseResponsesOutput([]byte(`{"id":"resp_1","output":[{"type":"message","role":"assistant"}]}`)) + if err != nil || id != "resp_1" || len(out) != 1 { + t.Fatalf("json: id=%q out=%#v err=%v", id, out, err) + } + + sse := "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_2\",\"output\":[]}}\n\n" + + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"message\",\"role\":\"assistant\"}}\n\n" + + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_2\",\"output\":[{\"type\":\"message\",\"role\":\"assistant\"}]}}\n\n" + id, out, err = parseResponsesOutput([]byte(sse)) + if err != nil || id != "resp_2" || len(out) != 1 { + t.Fatalf("sse: id=%q out=%#v err=%v", id, out, err) + } +} + +func TestResponsesPreviousResponseEndToEnd(t *testing.T) { + var responseCalls int + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"models":[{"name":"qwen3:8b","model":"qwen3:8b"}]}`) + case "/api/tags": + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"models":[{"name":"qwen3:8b","model":"qwen3:8b"}]}`) + case "/v1/responses": + responseCalls++ + var req map[string]any + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("decode upstream request: %v", err) + w.WriteHeader(400) + return + } + if _, exists := req["previous_response_id"]; exists { + t.Errorf("previous_response_id leaked upstream on call %d", responseCalls) + } + if responseCalls == 1 { + if req["input"] != "first" { + t.Errorf("first input=%#v", req["input"]) + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"id":"resp_1","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"one"}]}],"usage":{"input_tokens":1,"output_tokens":1}}`) + return + } + items, ok := req["input"].([]any) + if !ok || len(items) != 3 { + t.Errorf("expanded second input=%#v", req["input"]) + } else { + first := items[0].(map[string]any) + assistant := items[1].(map[string]any) + last := items[2].(map[string]any) + if first["role"] != "user" || first["content"] != "first" { + t.Errorf("first history item=%#v", first) + } + if assistant["role"] != "assistant" { + t.Errorf("assistant history item=%#v", assistant) + } + if last["role"] != "user" || last["content"] != "second" { + t.Errorf("last history item=%#v", last) + } + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"id":"resp_2","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"two"}]}],"usage":{"input_tokens":3,"output_tokens":1}}`) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer backend.Close() + + cc := config.ConversationsConfig{Enabled: true, EncryptionKey: strings.Repeat("k", 32), Retention: config.Duration(time.Hour), MaxEntries: 100, MaxContentBytes: 1 << 20} + conv, err := conversation.New(cc, filepath.Join(t.TempDir(), "conversations.enc.json")) + if err != nil { + t.Fatal(err) + } + cfg := &config.Config{ + Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute)}, + Auth: config.AuthConfig{IPBypass: []config.IPBypassConfig{{CIDRs: []string{"127.0.0.1/32"}, Tenant: "test", Subject: "u"}}}, + Scheduler: config.SchedulerConfig{GlobalConcurrency: 1, MaxQueue: 8, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, ComputePaths: []string{"/v1/responses"}}, + Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 16}, + Workers: []config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}}, + ModelCapabilities: config.ModelCapabilitiesConfig{Mode: "off", ContextGuard: "off"}, + Conversations: cc, + } + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wp.SetModelCapabilitiesConfig(cfg.ModelCapabilities) + wp.Start(ctx) + rec, _ := usage.New("", 100, time.Second, nil) + defer rec.Close() + sv := New(cfg, Dependencies{Auth: a, Scheduler: scheduler.NewLocal(1, 8, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: proxy.New(), Usage: rec, Metrics: metrics.New(), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), Conversations: conv}) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + resp, err := http.Post(front.URL+"/v1/responses", "application/json", strings.NewReader(`{"model":"qwen3:8b","input":"first"}`)) + if err != nil { + t.Fatal(err) + } + firstBody, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 || !strings.Contains(string(firstBody), `"id":"resp_1"`) { + t.Fatalf("first status=%d body=%s", resp.StatusCode, firstBody) + } + if _, ok, err := conv.Get("resp_1", "test", "u"); err != nil || !ok { + t.Fatalf("first response not stored: ok=%v err=%v", ok, err) + } + + resp, err = http.Post(front.URL+"/v1/responses", "application/json", strings.NewReader(`{"model":"qwen3:8b","previous_response_id":"resp_1","input":"second"}`)) + if err != nil { + t.Fatal(err) + } + secondBody, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 || !strings.Contains(string(secondBody), `"id":"resp_2"`) { + t.Fatalf("second status=%d body=%s", resp.StatusCode, secondBody) + } + stored, ok, err := conv.Get("resp_2", "test", "u") + if err != nil || !ok { + t.Fatalf("second response not stored: ok=%v err=%v", ok, err) + } + var final []any + if err := json.Unmarshal(stored.Context, &final); err != nil { + t.Fatal(err) + } + if len(final) != 4 { + t.Fatalf("stored final context has %d items: %s", len(final), stored.Context) + } +} diff --git a/internal/server/jobs.go b/internal/server/jobs.go new file mode 100644 index 0000000..56e515b --- /dev/null +++ b/internal/server/jobs.go @@ -0,0 +1,103 @@ +package server + +import ( + "context" + "errors" + "sort" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/liveflow" +) + +var errJobCancelled = errors.New("job cancelled by administrator") + +type jobEntry struct { + ID string `json:"id"` + Tenant string `json:"tenant"` + Actor string `json:"actor"` + Application string `json:"application,omitempty"` + ServiceClass string `json:"service_class,omitempty"` + Model string `json:"model,omitempty"` + Path string `json:"path"` + API string `json:"api"` + Worker string `json:"worker,omitempty"` + CreatedAt time.Time `json:"created_at"` + CancelledAt *time.Time `json:"cancelled_at,omitempty"` + Cancelling bool `json:"cancelling,omitempty"` +} + +type jobView struct { + liveflow.Request + Cancellable bool `json:"cancellable"` + Cancelling bool `json:"cancelling,omitempty"` +} + +type jobManager struct { + mu sync.RWMutex + jobs map[string]jobEntry + cancel map[string]context.CancelCauseFunc +} + +func newJobManager() *jobManager { + return &jobManager{jobs: make(map[string]jobEntry), cancel: make(map[string]context.CancelCauseFunc)} +} + +func (m *jobManager) register(j jobEntry, cancel context.CancelCauseFunc) { + m.mu.Lock() + defer m.mu.Unlock() + m.jobs[j.ID] = j + m.cancel[j.ID] = cancel +} + +func (m *jobManager) setWorker(id, worker string) { + m.mu.Lock() + defer m.mu.Unlock() + j, ok := m.jobs[id] + if !ok { + return + } + j.Worker = worker + m.jobs[id] = j +} + +func (m *jobManager) finish(id string) { + m.mu.Lock() + delete(m.jobs, id) + delete(m.cancel, id) + m.mu.Unlock() +} + +func (m *jobManager) cancelJob(id string) bool { + m.mu.Lock() + cancel := m.cancel[id] + j, ok := m.jobs[id] + if cancel == nil || !ok { + m.mu.Unlock() + return false + } + if !j.Cancelling { + now := time.Now().UTC() + j.Cancelling = true + j.CancelledAt = &now + m.jobs[id] = j + } + m.mu.Unlock() + cancel(errJobCancelled) + return true +} + +func (m *jobManager) list() []jobEntry { + m.mu.RLock() + out := make([]jobEntry, 0, len(m.jobs)) + for _, j := range m.jobs { + out = append(out, j) + } + m.mu.RUnlock() + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.Before(out[j].CreatedAt) }) + return out +} + +func isAdminJobCancel(ctx context.Context) bool { + return errors.Is(context.Cause(ctx), errJobCancelled) +} diff --git a/internal/server/model_access.go b/internal/server/model_access.go new file mode 100644 index 0000000..c2ee185 --- /dev/null +++ b/internal/server/model_access.go @@ -0,0 +1,179 @@ +package server + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "sort" + "strings" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/proxy" +) + +func cloneModelAccess(in config.ModelAccessConfig) config.ModelAccessConfig { + cloneRule := func(r config.ModelAccessRule) config.ModelAccessRule { + r.AllowedModels = append([]string(nil), r.AllowedModels...) + r.DeniedModels = append([]string(nil), r.DeniedModels...) + return r + } + out := config.ModelAccessConfig{Default: cloneRule(in.Default), Tenants: make(map[string]config.ModelAccessRule, len(in.Tenants))} + for name, r := range in.Tenants { + out.Tenants[name] = cloneRule(r) + } + return out +} + +func (s *Server) modelAccessSnapshot() config.ModelAccessConfig { + if s == nil { + return config.ModelAccessConfig{Tenants: map[string]config.ModelAccessRule{}} + } + if v := s.modelAccess.Load(); v != nil { + return cloneModelAccess(v.(config.ModelAccessConfig)) + } + return cloneModelAccess(s.cfg.ModelAccess) +} + +func (s *Server) runtimeTenantModelAccessRule(tenant string) config.ModelAccessRule { + m := s.modelAccessSnapshot() + if r, ok := m.Tenants[tenant]; ok { + if r.Mode == "" { + r.Mode = "allow_all" + } + return r + } + r := m.Default + if r.Mode == "" { + r.Mode = "allow_all" + } + return r +} + +func normalizeAccessRule(r config.ModelAccessRule) config.ModelAccessRule { + r.Mode = strings.TrimSpace(r.Mode) + if r.Mode == "" { + r.Mode = "allow_all" + } + r.AllowedModels = cleanStrings(r.AllowedModels) + r.DeniedModels = cleanStrings(r.DeniedModels) + return r +} + +func validateModelAccessConfig(m config.ModelAccessConfig) error { + if err := config.ValidateModelAccessRule(normalizeAccessRule(m.Default)); err != nil { + return fmt.Errorf("default model access: %w", err) + } + for tenant, r := range m.Tenants { + if strings.TrimSpace(tenant) == "" { + return errors.New("tenant name is required") + } + if err := config.ValidateModelAccessRule(normalizeAccessRule(r)); err != nil { + return fmt.Errorf("tenant %q model access: %w", tenant, err) + } + } + return nil +} + +func (s *Server) storeModelAccess(next config.ModelAccessConfig) error { + if s.configStore == nil { + return errors.New("persistent configuration store unavailable") + } + base := s.cfg + if loader, ok := s.configStore.(configOverrideLoader); ok { + if loaded, err := loader.LoadWithBootstrap(s.cfg); err == nil { + base = loaded + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + } + b, err := json.Marshal(base) + if err != nil { + return err + } + var candidate config.Config + if err := json.Unmarshal(b, &candidate); err != nil { + return err + } + next = cloneModelAccess(next) + if next.Tenants == nil { + next.Tenants = map[string]config.ModelAccessRule{} + } + candidate.ModelAccess = next + if err := validateModelAccessConfig(next); err != nil { + return err + } + if err := s.configStore.Save(&candidate); err != nil { + return err + } + s.modelAccess.Store(cloneModelAccess(next)) + return nil +} + +func (s *Server) uiModelAccess(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if r.URL.Path == "/gateway/ui-api/model-access" { + if r.Method != http.MethodGet { + proxy.WriteJSONError(w, http.StatusMethodNotAllowed, "method_not_allowed", "GET required") + return + } + m := s.modelAccessSnapshot() + names := make([]string, 0, len(m.Tenants)) + for n := range m.Tenants { + names = append(names, n) + } + sort.Strings(names) + writeJSON(w, http.StatusOK, map[string]any{"default": m.Default, "tenants": m.Tenants, "tenant_names": names, "runtime": true, "persistent": s.configStore != nil}) + return + } + raw := strings.TrimPrefix(r.URL.Path, "/gateway/ui-api/model-access/") + tenant, err := url.PathUnescape(raw) + tenant = strings.TrimSpace(tenant) + if err != nil || tenant == "" || strings.Contains(tenant, "/") || len(tenant) > 256 { + proxy.WriteJSONError(w, 400, "bad_tenant", "invalid tenant") + return + } + if s.configStore == nil { + proxy.WriteJSONError(w, 503, "config_store", "persistent configuration store unavailable") + return + } + s.modelAccessMu.Lock() + defer s.modelAccessMu.Unlock() + next := s.modelAccessSnapshot() + switch r.Method { + case http.MethodPut: + var in config.ModelAccessRule + if err := decodeJSON(r, &in, 128<<10); err != nil { + proxy.WriteJSONError(w, 400, "bad_model_access", err.Error()) + return + } + in = normalizeAccessRule(in) + if err := config.ValidateModelAccessRule(in); err != nil { + proxy.WriteJSONError(w, 400, "bad_model_access", err.Error()) + return + } + next.Tenants[tenant] = in + if err := s.storeModelAccess(next); err != nil { + proxy.WriteJSONError(w, 400, "model_access_store", err.Error()) + return + } + s.log.Info("tenant model access saved", "tenant", tenant, "mode", in.Mode, "admin_subject", actor.Subject) + writeJSON(w, 200, map[string]any{"tenant": tenant, "rule": in, "restart_required": false}) + case http.MethodDelete: + if _, ok := next.Tenants[tenant]; !ok { + proxy.WriteJSONError(w, 404, "model_access_not_found", fmt.Sprintf("tenant model access %q not found", tenant)) + return + } + delete(next.Tenants, tenant) + if err := s.storeModelAccess(next); err != nil { + proxy.WriteJSONError(w, 400, "model_access_store", err.Error()) + return + } + s.log.Info("tenant model access reset", "tenant", tenant, "admin_subject", actor.Subject) + writeJSON(w, 200, map[string]any{"deleted": true, "tenant": tenant, "restart_required": false}) + default: + proxy.WriteJSONError(w, http.StatusMethodNotAllowed, "method_not_allowed", "PUT or DELETE required") + } +} diff --git a/internal/server/models.go b/internal/server/models.go new file mode 100644 index 0000000..53eb394 --- /dev/null +++ b/internal/server/models.go @@ -0,0 +1,107 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +var ErrModelAccessDenied = errors.New("model access denied") +var ErrAliasUnavailable = errors.New("model alias has no routable target") + +func (s *Server) tenantModelAccessRule(id auth.Identity) config.ModelAccessRule { + return s.runtimeTenantModelAccessRule(id.Tenant) +} + +func (s *Server) modelAllowed(id auth.Identity, model string) bool { + if strings.TrimSpace(model) == "" { + return true + } + // Tenant policy is the outer security boundary. An API-key ACL may narrow + // that boundary, but must never widen it. + if !config.ModelAccessAllowed(s.tenantModelAccessRule(id), model) { + return false + } + if id.ModelACLSet && !config.ModelAccessAllowed(id.ModelAccess, model) { + return false + } + return true +} + +func (s *Server) resolveModel(ctx context.Context, id auth.Identity, requested string) (string, string, error) { + requested = strings.TrimSpace(requested) + if requested == "" { + return "", "", nil + } + if !s.modelAllowed(id, requested) { + return "", "", fmt.Errorf("%w: model %q is not allowed for this identity", ErrModelAccessDenied, requested) + } + a, ok := s.aliasConfig(requested) + if !ok { + return requested, "", nil + } + for _, m := range a.Models { + m = strings.TrimSpace(m) + if m == "" || !s.workers.CanRoute(m) { + continue + } + if len(a.RequiredCapabilities) > 0 { + meta, _, err := s.workers.Metadata(ctx, m) + if err != nil { + continue + } + okCaps := true + for _, capName := range a.RequiredCapabilities { + if !worker.HasCapability(meta, capName) { + okCaps = false + break + } + } + if !okCaps { + continue + } + } + return m, requested, nil + } + return "", requested, fmt.Errorf("%w: alias %q has no healthy/eligible installed target", ErrAliasUnavailable, requested) +} + +func rewriteModelBody(body []byte, model string) ([]byte, error) { + if len(body) == 0 || strings.TrimSpace(model) == "" { + return body, nil + } + var v map[string]any + if err := json.Unmarshal(body, &v); err != nil { + return nil, err + } + v["model"] = model + if _, ok := v["name"]; ok { + v["name"] = model + } + return json.Marshal(v) +} + +func (s *Server) visibleAliases(ctx context.Context, id auth.Identity) []string { + aliases := s.aliasSnapshot() + out := make([]string, 0, len(aliases)) + for name, a := range aliases { + if a.Visible != nil && !*a.Visible { + continue + } + if !s.modelAllowed(id, name) { + continue + } + if _, _, err := s.resolveModel(ctx, id, name); err == nil { + out = append(out, name) + } + } + sort.Strings(out) + return out +} diff --git a/internal/server/openwebui_test.go b/internal/server/openwebui_test.go new file mode 100644 index 0000000..7435662 --- /dev/null +++ b/internal/server/openwebui_test.go @@ -0,0 +1,164 @@ +package server + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/cost" + "github.com/example/ollama-fair-gateway/internal/metrics" + px "github.com/example/ollama-fair-gateway/internal/proxy" + "github.com/example/ollama-fair-gateway/internal/quota" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/usage" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +func TestOpenWebUIOllamaCompatibility(t *testing.T) { + var betaShowOnB atomic.Bool + var betaChatOnB atomic.Bool + + backendA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + writeJSON(w, 200, map[string]any{"models": []any{}}) + case "/api/tags": + // Deliberately omit "model" to verify gateway normalization for + // clients such as OpenWebUI that key discovery by that field. + io.WriteString(w, `{"models":[{"name":"alpha:latest","size":100,"details":{"family":"alpha"}}]}`) + case "/api/version": + io.WriteString(w, `{"version":"0.99.0"}`) + case "/api/show": + io.WriteString(w, `{"error":"model not found"}`) + case "/api/chat": + w.WriteHeader(http.StatusNotFound) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer backendA.Close() + + backendB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + writeJSON(w, 200, map[string]any{"models": []any{}}) + case "/api/tags": + io.WriteString(w, `{"models":[{"name":"beta:latest","model":"beta:latest","size":200,"details":{"family":"beta"}}]}`) + case "/api/version": + io.WriteString(w, `{"version":"0.99.0"}`) + case "/api/show": + b, _ := io.ReadAll(r.Body) + var v map[string]any + _ = json.Unmarshal(b, &v) + if v["model"] == "beta:latest" { + betaShowOnB.Store(true) + io.WriteString(w, `{"modelfile":"FROM beta"}`) + return + } + w.WriteHeader(http.StatusNotFound) + case "/api/chat": + betaChatOnB.Store(true) + w.Header().Set("Content-Type", "application/x-ndjson") + io.WriteString(w, "{\"message\":{\"content\":\"ok\"},\"done\":true,\"prompt_eval_count\":2,\"eval_count\":1}\n") + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer backendB.Close() + + cfg := &config.Config{ + Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute)}, + Auth: config.AuthConfig{APIKeys: []config.APIKeyConfig{{ + Name: "openwebui", Key: "owui-secret", Tenant: "apps", Subject: "openwebui", Application: "openwebui", + }}}, + Scheduler: config.SchedulerConfig{ + GlobalConcurrency: 2, MaxQueue: 16, MaxQueuePerActor: 8, + QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, + ComputePaths: []string{"/api/chat", "/api/generate", "/api/embed", "/api/embeddings", "/v1/chat/completions"}, + }, + Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 1}, DefaultMaxOutputTokens: 16}, + Workers: []config.WorkerConfig{ + {Name: "a", URL: backendA.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}, + {Name: "b", URL: backendB.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}, + }, + Native: config.NativeConfig{ControlWorker: "a"}, + } + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "a") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wp.Start(ctx) + rec, _ := usage.New("", 128, time.Second, nil) + sv := New(cfg, Dependencies{ + Auth: a, Scheduler: scheduler.NewLocal(2, 16, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), + Workers: wp, Proxy: px.New(), Usage: rec, Metrics: metrics.New(), Logger: slog.Default(), + }) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + do := func(method, path, body string, withKey bool) (int, string) { + req, _ := http.NewRequest(method, front.URL+path, strings.NewReader(body)) + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + if withKey { + req.Header.Set("Authorization", "Bearer owui-secret") + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return resp.StatusCode, string(b) + } + + if status, body := do(http.MethodGet, "/api/tags", "", false); status != http.StatusUnauthorized { + t.Fatalf("unauthenticated /api/tags status=%d, want 401", status) + } else if !strings.Contains(body, `"error":"authentication required"`) { + t.Fatalf("native Ollama error shape is not compatible: %s", body) + } + + if status, body := do(http.MethodGet, "/v1/models", "", false); status != http.StatusUnauthorized { + t.Fatalf("unauthenticated /v1/models status=%d, want 401", status) + } else if !strings.Contains(body, `"message":"authentication required"`) { + t.Fatalf("OpenAI error shape changed unexpectedly: %s", body) + } + + status, body := do(http.MethodGet, "/api/version", "", true) + if status != 200 || !strings.Contains(body, `"version":"0.99.0"`) { + t.Fatalf("version status=%d body=%s", status, body) + } + + status, body = do(http.MethodGet, "/api/tags", "", true) + if status != 200 || !strings.Contains(body, `"model":"alpha:latest"`) || !strings.Contains(body, `"model":"beta:latest"`) { + t.Fatalf("tags status=%d body=%s", status, body) + } + + status, body = do(http.MethodGet, "/v1/models", "", true) + if status != 200 || !strings.Contains(body, `"id":"alpha:latest"`) || !strings.Contains(body, `"id":"beta:latest"`) || !strings.Contains(body, `"object":"list"`) { + t.Fatalf("v1 models status=%d body=%s", status, body) + } + + status, body = do(http.MethodPost, "/api/show", `{"model":"beta:latest"}`, true) + if status != 200 || !betaShowOnB.Load() { + t.Fatalf("show was not model-routed to backend B: status=%d body=%s", status, body) + } + + status, body = do(http.MethodPost, "/api/chat", `{"model":"beta:latest","messages":[{"role":"user","content":"hi"}]}`, true) + if status != 200 || !betaChatOnB.Load() || !strings.Contains(body, `"content":"ok"`) { + t.Fatalf("chat was not model-routed to backend B: status=%d body=%s", status, body) + } +} diff --git a/internal/server/operations.go b/internal/server/operations.go new file mode 100644 index 0000000..ce48551 --- /dev/null +++ b/internal/server/operations.go @@ -0,0 +1,229 @@ +package server + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "sync" + "time" +) + +type adminOperation struct { + ID string `json:"id"` + Type string `json:"type"` + Worker string `json:"worker"` + Model string `json:"model"` + Status string `json:"status"` + Message string `json:"message,omitempty"` + Completed int64 `json:"completed,omitempty"` + Total int64 `json:"total,omitempty"` + Progress float64 `json:"progress"` + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` + Error string `json:"error,omitempty"` +} + +type operationManager struct { + mu sync.RWMutex + ops map[string]adminOperation + cancel map[string]context.CancelFunc + maxKeep int + client *http.Client +} + +func newOperationManager() *operationManager { + return &operationManager{ops: map[string]adminOperation{}, cancel: map[string]context.CancelFunc{}, maxKeep: 100, client: &http.Client{Timeout: 0}} +} + +func (m *operationManager) list() []adminOperation { + m.mu.RLock() + defer m.mu.RUnlock() + out := make([]adminOperation, 0, len(m.ops)) + for _, op := range m.ops { + out = append(out, op) + } + sort.Slice(out, func(i, j int) bool { return out[i].StartedAt.After(out[j].StartedAt) }) + return out +} + +func (m *operationManager) update(id string, fn func(*adminOperation)) { + m.mu.Lock() + defer m.mu.Unlock() + op, ok := m.ops[id] + if !ok { + return + } + fn(&op) + op.UpdatedAt = time.Now().UTC() + m.ops[id] = op +} + +func (m *operationManager) add(op adminOperation) { + m.mu.Lock() + defer m.mu.Unlock() + m.ops[op.ID] = op + if len(m.ops) <= m.maxKeep { + return + } + var oldest string + var oldestTime time.Time + for id, x := range m.ops { + if x.Status == "running" || x.Status == "queued" { + continue + } + if oldest == "" || x.StartedAt.Before(oldestTime) { + oldest, oldestTime = id, x.StartedAt + } + } + if oldest != "" { + delete(m.ops, oldest) + } +} + +func (m *operationManager) startPull(base *url.URL, worker, model string) adminOperation { + now := time.Now().UTC() + op := adminOperation{ID: operationID(), Type: "pull", Worker: worker, Model: model, Status: "queued", StartedAt: now, UpdatedAt: now} + m.add(op) + ctx, cancel := context.WithCancel(context.Background()) + m.mu.Lock() + m.cancel[op.ID] = cancel + m.mu.Unlock() + go m.runPull(ctx, op.ID, base, model) + return op +} + +func (m *operationManager) runPull(ctx context.Context, id string, base *url.URL, model string) { + defer func() { + m.mu.Lock() + delete(m.cancel, id) + m.mu.Unlock() + }() + m.update(id, func(op *adminOperation) { op.Status = "running"; op.Message = "starting pull" }) + body, _ := json.Marshal(map[string]any{"model": model, "stream": true}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, base.String()+"/api/pull", bytes.NewReader(body)) + if err != nil { + m.fail(id, err) + return + } + req.Header.Set("Content-Type", "application/json") + resp, err := m.client.Do(req) + if err != nil { + if ctx.Err() != nil { + m.update(id, func(op *adminOperation) { op.Status = "cancelled"; op.Message = "cancelled" }) + return + } + m.fail(id, err) + return + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + m.fail(id, fmt.Errorf("Ollama HTTP %d: %s", resp.StatusCode, string(b))) + return + } + sc := bufio.NewScanner(resp.Body) + buf := make([]byte, 0, 64<<10) + sc.Buffer(buf, 2<<20) + for sc.Scan() { + var x struct { + Status string `json:"status"` + Digest string `json:"digest"` + Total int64 `json:"total"` + Completed int64 `json:"completed"` + Error string `json:"error"` + } + if json.Unmarshal(sc.Bytes(), &x) != nil { + continue + } + if x.Error != "" { + m.fail(id, fmt.Errorf("%s", x.Error)) + return + } + m.update(id, func(op *adminOperation) { + op.Message = x.Status + if x.Total > 0 { + op.Total = x.Total + } + if x.Completed > 0 { + op.Completed = x.Completed + } + if op.Total > 0 { + op.Progress = float64(op.Completed) / float64(op.Total) + if op.Progress > 1 { + op.Progress = 1 + } + } + }) + } + if err := sc.Err(); err != nil { + if ctx.Err() != nil { + m.update(id, func(op *adminOperation) { op.Status = "cancelled"; op.Message = "cancelled" }) + return + } + m.fail(id, err) + return + } + m.update(id, func(op *adminOperation) { op.Status = "completed"; op.Message = "success"; op.Progress = 1 }) +} + +func (m *operationManager) fail(id string, err error) { + m.update(id, func(op *adminOperation) { op.Status = "failed"; op.Error = err.Error(); op.Message = "failed" }) +} + +func (m *operationManager) cancelOperation(id string) bool { + m.mu.RLock() + cancel := m.cancel[id] + m.mu.RUnlock() + if cancel == nil { + return false + } + cancel() + return true +} + +func (m *operationManager) modelAction(ctx context.Context, base *url.URL, action, model string) error { + var method, path string + var payload any + switch action { + case "delete": + method, path = http.MethodDelete, "/api/delete" + payload = map[string]string{"model": model} + case "stop": + // Ollama unloads a resident model by issuing a generate request with + // keep_alive=0. There is no native /api/stop endpoint. + method, path = http.MethodPost, "/api/generate" + payload = map[string]any{"model": model, "keep_alive": 0, "stream": false} + default: + return fmt.Errorf("unsupported model action %q", action) + } + body, _ := json.Marshal(payload) + req, err := http.NewRequestWithContext(ctx, method, base.String()+path, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := m.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + return fmt.Errorf("Ollama HTTP %d: %s", resp.StatusCode, string(b)) + } + return nil +} + +func operationID() string { + b := make([]byte, 12) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/internal/server/operations_test.go b/internal/server/operations_test.go new file mode 100644 index 0000000..fb9b74c --- /dev/null +++ b/internal/server/operations_test.go @@ -0,0 +1,43 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +func TestModelStopUsesGenerateKeepAliveZero(t *testing.T) { + var gotMethod, gotPath string + var got map[string]any + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod, gotPath = r.Method, r.URL.Path + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Errorf("decode body: %v", err) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"done":true}`)) + })) + defer backend.Close() + u, _ := url.Parse(backend.URL) + + m := newOperationManager() + if err := m.modelAction(context.Background(), u, "stop", "gemma4:latest"); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPost || gotPath != "/api/generate" { + t.Fatalf("method/path=%s %s", gotMethod, gotPath) + } + if got["model"] != "gemma4:latest" { + t.Fatalf("model=%v", got["model"]) + } + if v, ok := got["keep_alive"].(float64); !ok || v != 0 { + t.Fatalf("keep_alive=%#v", got["keep_alive"]) + } + if v, ok := got["stream"].(bool); !ok || v { + t.Fatalf("stream=%#v", got["stream"]) + } +} diff --git a/internal/server/policy_simulator.go b/internal/server/policy_simulator.go new file mode 100644 index 0000000..1457ad0 --- /dev/null +++ b/internal/server/policy_simulator.go @@ -0,0 +1,284 @@ +package server + +import ( + "context" + "fmt" + "math" + "sort" + "strings" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +type policySimulationRequest struct { + Tenant string `json:"tenant"` + APIKeyID string `json:"api_key_id,omitempty"` + APIKeyName string `json:"api_key_name,omitempty"` + Model string `json:"model"` + RequiredCapabilities []string `json:"required_capabilities,omitempty"` + InputTokens int64 `json:"input_tokens,omitempty"` + OutputTokens int64 `json:"output_tokens,omitempty"` + ServiceClass string `json:"service_class,omitempty"` +} + +type accessExplanation struct { + TenantAllowed bool `json:"tenant_allowed"` + KeyApplied bool `json:"key_applied"` + KeyAllowed bool `json:"key_allowed"` + Allowed bool `json:"allowed"` + TenantRule config.ModelAccessRule `json:"tenant_rule"` + KeyRule config.ModelAccessRule `json:"key_rule,omitempty"` +} + +type aliasCandidateExplanation struct { + Model string `json:"model"` + Routable bool `json:"routable"` + Capabilities []string `json:"capabilities,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type policySimulationResult struct { + RequestedModel string `json:"requested_model"` + ResolvedModel string `json:"resolved_model,omitempty"` + Alias string `json:"alias,omitempty"` + AliasCandidates []aliasCandidateExplanation `json:"alias_candidates,omitempty"` + Access accessExplanation `json:"access"` + CapabilitiesRequired []string `json:"capabilities_required,omitempty"` + CapabilitiesKnown []string `json:"capabilities_known,omitempty"` + CapabilitiesOK bool `json:"capabilities_ok"` + ContextLength int64 `json:"context_length,omitempty"` + ContextEffectiveMax int64 `json:"context_effective_max,omitempty"` + ContextRequested int64 `json:"context_requested,omitempty"` + ContextOK bool `json:"context_ok"` + EstimatedCredits float64 `json:"estimated_credits"` + CostRate config.ModelRate `json:"cost_rate"` + TenantPolicy config.TenantPolicy `json:"tenant_policy"` + ServiceClass string `json:"service_class"` + ServiceClassConfig config.ServiceClassConfig `json:"service_class_config"` + Workers []worker.RoutingExplanation `json:"workers"` + SelectedWorker string `json:"selected_worker,omitempty"` + Decision string `json:"decision"` + Errors []string `json:"errors,omitempty"` +} + +func (s *Server) simulatedIdentity(in policySimulationRequest) (auth.Identity, *auth.APIKeyInfo, error) { + tenant := strings.TrimSpace(in.Tenant) + var selected *auth.APIKeyInfo + if strings.TrimSpace(in.APIKeyID) != "" || strings.TrimSpace(in.APIKeyName) != "" { + for _, k := range s.auth.APIKeys() { + idMatch := in.APIKeyID != "" && k.ID == in.APIKeyID + nameMatch := in.APIKeyID == "" && in.APIKeyName != "" && k.Name == in.APIKeyName && (tenant == "" || k.Tenant == tenant) + if idMatch || nameMatch { + kk := k + selected = &kk + break + } + } + if selected == nil { + return auth.Identity{}, nil, fmt.Errorf("API key not found") + } + if tenant == "" { + tenant = selected.Tenant + } else if selected.Tenant != tenant { + return auth.Identity{}, nil, fmt.Errorf("API key belongs to tenant %q, not %q", selected.Tenant, tenant) + } + } + if tenant == "" { + tenant = "default" + } + id := auth.Identity{Tenant: tenant, Subject: "policy-simulator", Application: "admin-ui", AuthType: "simulation", Scopes: map[string]bool{}} + if selected != nil { + id.Subject = selected.Subject + id.Application = selected.Application + id.ServiceClass = selected.ServiceClass + if len(selected.AllowedModels) > 0 || len(selected.DeniedModels) > 0 { + id.ModelACLSet = true + id.ModelAccess = config.ModelAccessRule{Mode: "allow_all", AllowedModels: append([]string(nil), selected.AllowedModels...), DeniedModels: append([]string(nil), selected.DeniedModels...)} + } + for _, scope := range selected.Scopes { + id.Scopes[scope] = true + } + } + return id, selected, nil +} + +func (s *Server) simulatePolicy(ctx context.Context, in policySimulationRequest) (policySimulationResult, error) { + in.Model = strings.TrimSpace(in.Model) + if in.Model == "" { + return policySimulationResult{}, fmt.Errorf("model is required") + } + if in.InputTokens < 0 || in.OutputTokens < 0 { + return policySimulationResult{}, fmt.Errorf("token estimates must be >= 0") + } + id, _, err := s.simulatedIdentity(in) + if err != nil { + return policySimulationResult{}, err + } + result := policySimulationResult{RequestedModel: in.Model, CapabilitiesOK: true, ContextOK: true} + tenantRule := s.tenantModelAccessRule(id) + result.Access = accessExplanation{TenantAllowed: config.ModelAccessAllowed(tenantRule, in.Model), TenantRule: tenantRule, KeyAllowed: true} + if id.ModelACLSet { + result.Access.KeyApplied = true + result.Access.KeyRule = id.ModelAccess + result.Access.KeyAllowed = config.ModelAccessAllowed(id.ModelAccess, in.Model) + } + result.Access.Allowed = result.Access.TenantAllowed && result.Access.KeyAllowed + if !result.Access.Allowed { + result.Decision = "denied_model_access" + return result, nil + } + + resolved := in.Model + if alias, ok := s.aliasConfig(in.Model); ok { + result.Alias = in.Model + for _, candidate := range alias.Models { + candidate = strings.TrimSpace(candidate) + if candidate == "" { + continue + } + x := aliasCandidateExplanation{Model: candidate, Routable: s.workers.CanRoute(candidate)} + if !x.Routable { + x.Reason = "no_eligible_worker" + } + if len(alias.RequiredCapabilities) > 0 { + meta, _, metaErr := s.workers.Metadata(ctx, candidate) + if metaErr != nil { + x.Reason = "metadata_unavailable" + } else { + x.Capabilities = append([]string(nil), meta.Capabilities...) + for _, c := range alias.RequiredCapabilities { + if !worker.HasCapability(meta, c) { + x.Routable = false + x.Reason = "missing_alias_capability:" + c + break + } + } + } + } + result.AliasCandidates = append(result.AliasCandidates, x) + if resolved == in.Model && x.Routable { + resolved = candidate + } + } + if resolved == in.Model { + result.Decision = "alias_unavailable" + return result, nil + } + } + result.ResolvedModel = resolved + + requiredMap := map[string]bool{} + for _, c := range in.RequiredCapabilities { + if c = strings.TrimSpace(c); c != "" { + requiredMap[c] = true + } + } + if alias, ok := s.aliasConfig(in.Model); ok { + for _, c := range alias.RequiredCapabilities { + if c = strings.TrimSpace(c); c != "" { + requiredMap[c] = true + } + } + } + for c := range requiredMap { + result.CapabilitiesRequired = append(result.CapabilitiesRequired, c) + } + sort.Strings(result.CapabilitiesRequired) + if len(result.CapabilitiesRequired) > 0 || in.InputTokens+in.OutputTokens > 0 { + meta, _, metaErr := s.workers.Metadata(ctx, resolved) + if metaErr != nil { + result.Errors = append(result.Errors, "model metadata unavailable: "+metaErr.Error()) + } else { + result.CapabilitiesKnown = append([]string(nil), meta.Capabilities...) + result.ContextLength = meta.ContextLength + for _, c := range result.CapabilitiesRequired { + if !worker.HasCapability(meta, c) { + result.CapabilitiesOK = false + result.Errors = append(result.Errors, "missing capability: "+c) + } + } + } + } + margin := int64(0) + if pct := s.cfg.ModelCapabilities.Context.EstimationMarginPercent; pct > 0 && in.InputTokens > 0 { + margin = int64(math.Ceil(float64(in.InputTokens) * pct / 100)) + } + result.ContextRequested = in.InputTokens + margin + in.OutputTokens + contextAllowed := map[string]bool{} + for _, cw := range s.workers.ContextWindows(ctx, resolved) { + effective := cw.EffectiveTokens + if cap := s.cfg.ModelCapabilities.Context.MaxRequestedTokens; cap > 0 { + effective = minPositive64(effective, cap) + } + if effective > result.ContextEffectiveMax { + result.ContextEffectiveMax = effective + } + if effective > 0 && result.ContextRequested <= effective { + contextAllowed[cw.Worker] = true + } + } + if cap := s.cfg.ModelCapabilities.Context.MaxRequestedTokens; cap > 0 && result.ContextRequested > cap { + result.ContextOK = false + result.Errors = append(result.Errors, fmt.Sprintf("context %d exceeds gateway cap %d", result.ContextRequested, cap)) + } else if result.ContextEffectiveMax > 0 && result.ContextRequested > result.ContextEffectiveMax { + result.ContextOK = false + result.Errors = append(result.Errors, fmt.Sprintf("context %d exceeds effective worker context %d", result.ContextRequested, result.ContextEffectiveMax)) + } else if result.ContextLength > 0 && result.ContextRequested > result.ContextLength { + result.ContextOK = false + result.Errors = append(result.Errors, fmt.Sprintf("context %d exceeds model limit %d", result.ContextRequested, result.ContextLength)) + } + + rate := s.estimator.Rate(resolved) + result.CostRate = rate + result.EstimatedCredits = float64(in.InputTokens)/1000*rate.InputCreditsPer1K + float64(in.OutputTokens)/1000*rate.OutputCreditsPer1K + if rate.ComputeCreditsPerSecond > 0 { + sec := 0.0 + if rate.ExpectedPromptTokensPerSecond > 0 { + sec += float64(in.InputTokens) / rate.ExpectedPromptTokensPerSecond + } + if rate.ExpectedOutputTokensPerSecond > 0 { + sec += float64(in.OutputTokens) / rate.ExpectedOutputTokensPerSecond + } + result.EstimatedCredits += sec * rate.ComputeCreditsPerSecond + } + result.TenantPolicy = s.policyFor(ctx, id.Tenant) + className := strings.TrimSpace(in.ServiceClass) + if className == "" { + className = strings.TrimSpace(id.ServiceClass) + } + if className == "" { + className = s.cfg.ServiceClasses.Default + } + if className == "" { + className = "interactive" + } + result.ServiceClass = className + if sc, ok := s.cfg.ServiceClasses.Classes[className]; ok { + result.ServiceClassConfig = sc + } else { + result.Errors = append(result.Errors, "unknown service class: "+className) + } + + if len(contextAllowed) == 0 { + contextAllowed = nil + } + result.Workers = s.workers.ExplainRoutingAllowed(resolved, contextAllowed, 0) + for _, w := range result.Workers { + if w.Eligible { + result.SelectedWorker = w.Worker + break + } + } + if !result.CapabilitiesOK { + result.Decision = "denied_capability" + } else if !result.ContextOK { + result.Decision = "denied_context" + } else if result.SelectedWorker == "" { + result.Decision = "no_eligible_worker" + } else { + result.Decision = "would_route" + } + return result, nil +} diff --git a/internal/server/preflight.go b/internal/server/preflight.go new file mode 100644 index 0000000..b36b81c --- /dev/null +++ b/internal/server/preflight.go @@ -0,0 +1,343 @@ +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "net/http" + "sort" + "strconv" + "strings" + + "github.com/example/ollama-fair-gateway/internal/cost" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +type requestRequirements struct { + Model string + Capabilities []string + RequestedContext int64 +} + +type modelPreflight struct { + AllowedWorkers map[string]bool + RequestedContext int64 + RequiredContext int64 +} + +func requirementsFor(path string, body []byte) (requestRequirements, error) { + var doc map[string]json.RawMessage + _ = json.Unmarshal(body, &doc) + var model string + _ = json.Unmarshal(doc["model"], &model) + req := requestRequirements{Model: model} + caps := map[string]bool{} + if strings.Contains(path, "embed") { + caps["embedding"] = true + } + if strings.Contains(path, "chat") || strings.Contains(path, "completion") || strings.Contains(path, "responses") || strings.Contains(path, "messages") || strings.Contains(path, "generate") { + caps["completion"] = true + } + if nonEmptyJSONList(doc["tools"]) { + caps["tools"] = true + } + if thinkingRequested(doc) { + caps["thinking"] = true + } + if containsVision(body) { + caps["vision"] = true + } + if raw := bytes.TrimSpace(doc["options"]); len(raw) > 0 && !bytes.Equal(raw, []byte("null")) { + var opts map[string]json.RawMessage + if err := json.Unmarshal(raw, &opts); err != nil { + return req, fmt.Errorf("options must be a JSON object: %w", err) + } + if rawNum, ok := opts["num_ctx"]; ok { + if !strings.HasPrefix(path, "/api/") { + return req, fmt.Errorf("options.num_ctx is only supported on native Ollama /api endpoints") + } + n, err := strictPositiveInt64(rawNum) + if err != nil { + return req, fmt.Errorf("options.num_ctx must be a positive integer: %w", err) + } + req.RequestedContext = n + } + } + for c := range caps { + req.Capabilities = append(req.Capabilities, c) + } + sort.Strings(req.Capabilities) + return req, nil +} + +func strictPositiveInt64(raw json.RawMessage) (int64, error) { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) || raw[0] == '"' { + return 0, fmt.Errorf("not an integer") + } + n, err := strconv.ParseInt(string(raw), 10, 64) + if err != nil || n <= 0 { + if err == nil { + err = fmt.Errorf("must be greater than zero") + } + return 0, err + } + return n, nil +} + +func nonEmptyJSONList(raw json.RawMessage) bool { + raw = bytes.TrimSpace(raw) + return len(raw) > 2 && !bytes.Equal(raw, []byte("null")) && !bytes.Equal(raw, []byte("[]")) +} + +func thinkingRequested(doc map[string]json.RawMessage) bool { + if raw := bytes.TrimSpace(doc["think"]); len(raw) > 0 && !bytes.Equal(raw, []byte("false")) && !bytes.Equal(raw, []byte("null")) && !bytes.Equal(raw, []byte(`"none"`)) { + return true + } + var effort string + _ = json.Unmarshal(doc["reasoning_effort"], &effort) + if effort != "" && !strings.EqualFold(effort, "none") { + return true + } + if raw := doc["thinking"]; len(raw) > 0 { + var x struct { + Type string `json:"type"` + } + if json.Unmarshal(raw, &x) == nil && strings.EqualFold(x.Type, "enabled") { + return true + } + } + if raw := doc["reasoning"]; len(raw) > 0 { + var x struct { + Effort string `json:"effort"` + } + if json.Unmarshal(raw, &x) == nil && x.Effort != "" && !strings.EqualFold(x.Effort, "none") { + return true + } + } + return false +} + +func countVisionInputs(body []byte) int64 { + var v any + if json.Unmarshal(body, &v) != nil { + return 0 + } + var walk func(any) int64 + walk = func(x any) int64 { + switch z := x.(type) { + case []any: + var n int64 + for _, item := range z { + n += walk(item) + } + return n + case map[string]any: + var n int64 + if imgs, ok := z["images"].([]any); ok { + n += int64(len(imgs)) + } + typ, _ := z["type"].(string) + isImageObject := typ == "image_url" || typ == "input_image" || typ == "image" + if isImageObject { + n++ + } + for k, item := range z { + // Payloads can be very large. The image object/array itself already + // reserves context tokens, so never recurse into base64/URLs. + if k == "image_url" || k == "images" || (isImageObject && (k == "url" || k == "data" || k == "image")) { + continue + } + n += walk(item) + } + return n + } + return 0 + } + return walk(v) +} + +func containsVision(body []byte) bool { return countVisionInputs(body) > 0 } + +func requiredContextTokens(est cost.Estimate, body []byte, visionReserve int64, marginPercent float64) (required, vision, margin int64) { + images := countVisionInputs(body) + if images > 0 && visionReserve > 0 { + vision = images * visionReserve + } + base := est.InputTokens + vision + if marginPercent > 0 && base > 0 { + margin = int64(math.Ceil(float64(base) * marginPercent / 100)) + } + return base + margin + est.OutputTokens, vision, margin +} + +func minPositive64(values ...int64) int64 { + var out int64 + for _, v := range values { + if v <= 0 { + continue + } + if out == 0 || v < out { + out = v + } + } + return out +} + +func contextWindowsSummary(windows []worker.ContextWindow) string { + parts := make([]string, 0, len(windows)) + for _, x := range windows { + if x.EffectiveTokens > 0 { + parts = append(parts, fmt.Sprintf("%s=%d(%s)", x.Worker, x.EffectiveTokens, x.EffectiveSource)) + } else { + parts = append(parts, x.Worker+"=unknown") + } + } + return strings.Join(parts, ", ") +} + +// preflightModel validates capability-sensitive requests and derives the set of +// workers that can actually serve the request's context budget. The model +// maximum from /api/show is only one input: loaded /api/ps context_length, +// Modelfile num_ctx, worker defaults and per-worker caps are stronger runtime +// evidence for OpenAI-compatible requests that cannot set num_ctx themselves. +func (s *Server) preflightModel(w http.ResponseWriter, r *http.Request, body []byte, est cost.Estimate) (modelPreflight, bool) { + out := modelPreflight{} + cfg := s.cfg.ModelCapabilities + if cfg.Mode == "off" || strings.TrimSpace(est.Model) == "" { + return out, true + } + req, reqErr := requirementsFor(r.URL.Path, body) + if reqErr != nil { + writeProtocolError(w, r, http.StatusBadRequest, "invalid_num_ctx", reqErr.Error()) + return out, false + } + out.RequestedContext = req.RequestedContext + meta, metaWorker, err := s.workers.Metadata(r.Context(), est.Model) + if err != nil { + w.Header().Set("X-Gateway-Model-Metadata", "unavailable") + s.log.Warn("model metadata unavailable; capability checks are best-effort", "model", est.Model, "error", err) + } else { + if metaWorker != "" { + w.Header().Set("X-Gateway-Model-Metadata-Worker", metaWorker) + } + if len(meta.Capabilities) > 0 { + w.Header().Set("X-Gateway-Model-Capabilities", strings.Join(meta.Capabilities, ",")) + } + if meta.ContextLength > 0 { + w.Header().Set("X-Gateway-Model-Context", strconv.FormatInt(meta.ContextLength, 10)) + } + if meta.ConfiguredContextLength > 0 { + w.Header().Set("X-Gateway-Model-Configured-Context", strconv.FormatInt(meta.ConfiguredContextLength, 10)) + } + } + + unsupported := make([]string, 0) + if err == nil && len(meta.Capabilities) > 0 { + for _, capability := range req.Capabilities { + if !worker.HasCapability(meta, capability) { + unsupported = append(unsupported, capability) + } + } + } + if len(unsupported) > 0 { + msg := fmt.Sprintf("model %s does not support required capability: %s", est.Model, strings.Join(unsupported, ", ")) + if cfg.Mode == "enforce" { + writeProtocolError(w, r, http.StatusBadRequest, "unsupported_capability", msg) + return out, false + } + w.Header().Set("X-Gateway-Capability-Warning", strings.Join(unsupported, ",")) + s.log.Warn("unsupported model capability observed", "model", est.Model, "required", unsupported) + } + + if cfg.ContextGuard == "off" { + return out, true + } + + required, visionReserve, margin := requiredContextTokens(est, body, cfg.Context.VisionReserveTokensPerImage, cfg.Context.EstimationMarginPercent) + out.RequiredContext = required + w.Header().Set("X-Gateway-Context-Required", strconv.FormatInt(required, 10)) + if req.RequestedContext > 0 { + w.Header().Set("X-Gateway-Requested-Context", strconv.FormatInt(req.RequestedContext, 10)) + } + if visionReserve > 0 { + w.Header().Set("X-Gateway-Context-Vision-Reserve", strconv.FormatInt(visionReserve, 10)) + } + if margin > 0 { + w.Header().Set("X-Gateway-Context-Margin", strconv.FormatInt(margin, 10)) + } + + var warning string + gatewayCap := cfg.Context.MaxRequestedTokens + if gatewayCap > 0 && req.RequestedContext > gatewayCap { + warning = fmt.Sprintf("requested num_ctx %d exceeds gateway context cap %d", req.RequestedContext, gatewayCap) + } else if gatewayCap > 0 && required > gatewayCap { + warning = fmt.Sprintf("estimated request context %d tokens exceeds gateway context cap %d", required, gatewayCap) + } else if req.RequestedContext > 0 && meta.ContextLength > 0 && req.RequestedContext > meta.ContextLength { + warning = fmt.Sprintf("requested num_ctx %d exceeds model context length %d", req.RequestedContext, meta.ContextLength) + } else if req.RequestedContext > 0 && required > req.RequestedContext { + warning = fmt.Sprintf("estimated request context %d tokens exceeds explicit num_ctx %d", required, req.RequestedContext) + } + + windows := s.workers.ContextWindows(r.Context(), est.Model) + allowed := make(map[string]bool, len(windows)) + var maxEffective int64 + for _, x := range windows { + eligible := false + if req.RequestedContext > 0 { + // Explicit native num_ctx can resize/reload the model, so current loaded + // context does not hard-block the worker. The theoretical model maximum + // and the operator's per-worker context cap still do. + capacity := minPositive64(x.ModelMaxTokens, x.WorkerLimitTokens) + if gatewayCap > 0 { + capacity = minPositive64(capacity, gatewayCap) + } + if capacity > maxEffective { + maxEffective = capacity + } + eligible = (x.ModelMaxTokens <= 0 || req.RequestedContext <= x.ModelMaxTokens) && + (x.WorkerLimitTokens <= 0 || req.RequestedContext <= x.WorkerLimitTokens) + } else { + effective := x.EffectiveTokens + if gatewayCap > 0 { + effective = minPositive64(effective, gatewayCap) + } + if effective > maxEffective { + maxEffective = effective + } + eligible = effective > 0 && required <= effective + } + if eligible { + allowed[x.Worker] = true + } + } + if maxEffective > 0 { + w.Header().Set("X-Gateway-Effective-Context-Max", strconv.FormatInt(maxEffective, 10)) + } + if len(windows) > 0 { + w.Header().Set("X-Gateway-Context-Eligible-Workers", fmt.Sprintf("%d/%d", len(allowed), len(windows))) + } + if warning == "" && len(windows) > 0 && len(allowed) == 0 { + if req.RequestedContext > 0 { + warning = fmt.Sprintf("requested num_ctx %d cannot be served by any eligible worker; contexts: %s", req.RequestedContext, contextWindowsSummary(windows)) + } else { + warning = fmt.Sprintf("estimated request context %d tokens exceeds every effective worker context; contexts: %s", required, contextWindowsSummary(windows)) + } + } + + if warning != "" { + if cfg.ContextGuard == "reject" { + writeProtocolError(w, r, http.StatusBadRequest, "context_window_exceeded", warning) + return out, false + } + w.Header().Set("X-Gateway-Context-Warning", warning) + s.log.Warn("context guard warning", "model", est.Model, "warning", warning) + } + // Even in warn mode, prefer context-suitable workers when at least one exists. + // If none exists, warn mode intentionally preserves legacy routing behavior. + if len(allowed) > 0 { + out.AllowedWorkers = allowed + } + return out, true +} diff --git a/internal/server/preflight_test.go b/internal/server/preflight_test.go new file mode 100644 index 0000000..268e34a --- /dev/null +++ b/internal/server/preflight_test.go @@ -0,0 +1,344 @@ +package server + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/cost" + "github.com/example/ollama-fair-gateway/internal/metrics" + px "github.com/example/ollama-fair-gateway/internal/proxy" + "github.com/example/ollama-fair-gateway/internal/quota" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/usage" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +func newPreflightServer(t *testing.T, capabilities []string, contextLength int64) *httptest.Server { + t.Helper() + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "codellama:7b", "model": "codellama:7b"}}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "codellama:7b", "model": "codellama:7b"}}}) + case "/api/show": + _ = json.NewEncoder(w).Encode(map[string]any{"capabilities": capabilities, "model_info": map[string]any{"llama.context_length": contextLength}}) + case "/api/chat": + _ = json.NewEncoder(w).Encode(map[string]any{"done": true, "message": map[string]any{"content": "ok"}, "prompt_eval_count": 2, "eval_count": 1}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(backend.Close) + cfg := &config.Config{ + Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute)}, + Auth: config.AuthConfig{IPBypass: []config.IPBypassConfig{{CIDRs: []string{"127.0.0.1/32"}, Tenant: "t", Subject: "u"}}}, + Scheduler: config.SchedulerConfig{GlobalConcurrency: 1, MaxQueue: 8, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, ComputePaths: []string{"/api/chat"}}, + Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3, CachedInputFactor: 1}, DefaultMaxOutputTokens: 16}, + Workers: []config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}}, + ModelCapabilities: config.ModelCapabilitiesConfig{Mode: "enforce", CacheTTL: config.Duration(time.Hour), ContextGuard: "reject"}, + } + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "w") + wp.SetModelCapabilitiesConfig(cfg.ModelCapabilities) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + wp.Start(ctx) + met := metrics.New() + rec, _ := usage.New("", 100, time.Second, nil) + sv := New(cfg, Dependencies{Auth: a, Scheduler: scheduler.NewLocal(1, 8, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: px.New(), Usage: rec, Metrics: met, Logger: slog.Default()}) + front := httptest.NewServer(sv.Handler()) + t.Cleanup(front.Close) + return front +} + +func TestRejectsUnsupportedToolsBeforeOllama(t *testing.T) { + front := newPreflightServer(t, []string{"completion"}, 16384) + resp, err := http.Post(front.URL+"/api/chat", "application/json", strings.NewReader(`{"model":"codellama:7b","messages":[{"role":"user","content":"x"}],"tools":[{"type":"function","function":{"name":"f"}}]}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 400 { + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } + if !strings.Contains(string(b), "does not support required capability: tools") { + t.Fatalf("unexpected body: %s", b) + } +} + +func TestAllowsSupportedTools(t *testing.T) { + front := newPreflightServer(t, []string{"completion", "tools"}, 16384) + resp, err := http.Post(front.URL+"/api/chat", "application/json", strings.NewReader(`{"model":"codellama:7b","messages":[{"role":"user","content":"x"}],"tools":[{"type":"function","function":{"name":"f"}}]}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + b, _ := io.ReadAll(resp.Body) + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } + if got := resp.Header.Get("X-Gateway-Model-Capabilities"); !strings.Contains(got, "tools") { + t.Fatalf("capability header=%q", got) + } +} + +func TestRejectsExplicitContextOverModelMaximum(t *testing.T) { + front := newPreflightServer(t, []string{"completion"}, 4096) + resp, err := http.Post(front.URL+"/api/chat", "application/json", strings.NewReader(`{"model":"codellama:7b","messages":[{"role":"user","content":"x"}],"options":{"num_ctx":8192}}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 400 || !strings.Contains(string(b), "exceeds model context length") { + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } +} + +func newContextGuardServer(t *testing.T, modelMax, loadedContext int64, parameters string, ctxCfg config.ContextPolicyConfig, workerLimits map[string]int64) (*httptest.Server, *atomic.Int64) { + t.Helper() + var inferenceCalls atomic.Int64 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + models := []any{} + if loadedContext > 0 { + models = append(models, map[string]any{"name": "ctx:latest", "model": "ctx:latest", "context_length": loadedContext}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"models": models}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "ctx:latest", "model": "ctx:latest"}}}) + case "/api/show": + _ = json.NewEncoder(w).Encode(map[string]any{"capabilities": []string{"completion", "vision"}, "model_info": map[string]any{"ctx.context_length": modelMax}, "parameters": parameters}) + case "/api/chat", "/v1/chat/completions", "/v1/responses", "/api/generate": + inferenceCalls.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"done": true, "message": map[string]any{"content": "ok"}, "choices": []any{}, "usage": map[string]any{"prompt_tokens": 2, "completion_tokens": 1}, "prompt_eval_count": 2, "eval_count": 1}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(backend.Close) + cfg := &config.Config{ + Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute)}, + Auth: config.AuthConfig{IPBypass: []config.IPBypassConfig{{CIDRs: []string{"127.0.0.1/32"}, Tenant: "t", Subject: "u"}}}, + Scheduler: config.SchedulerConfig{GlobalConcurrency: 1, MaxQueue: 8, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, ComputePaths: []string{"/api/chat", "/api/generate", "/v1/chat/completions", "/v1/responses"}}, + Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3, CachedInputFactor: 1}, DefaultMaxOutputTokens: 16}, + Workers: []config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour), ContextLimits: workerLimits}}, + ModelCapabilities: config.ModelCapabilitiesConfig{Mode: "enforce", CacheTTL: config.Duration(time.Hour), ContextGuard: "reject", Context: ctxCfg}, + } + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "w") + wp.SetModelCapabilitiesConfig(cfg.ModelCapabilities) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + wp.Start(ctx) + rec, _ := usage.New("", 100, time.Second, nil) + sv := New(cfg, Dependencies{Auth: a, Scheduler: scheduler.NewLocal(1, 8, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: px.New(), Usage: rec, Metrics: metrics.New(), Logger: slog.Default()}) + front := httptest.NewServer(sv.Handler()) + t.Cleanup(front.Close) + return front, &inferenceCalls +} + +func TestInvalidNumCtxRejectedBeforeBackend(t *testing.T) { + front, calls := newContextGuardServer(t, 131072, 0, "", config.ContextPolicyConfig{MaxRequestedTokens: 32768, DefaultWorkerTokens: 4096, EstimationMarginPercent: 15, VisionReserveTokensPerImage: 2048}, nil) + for _, raw := range []string{`-1`, `0`, `"8192"`, `1.5`, `null`} { + resp, err := http.Post(front.URL+"/api/chat", "application/json", strings.NewReader(`{"model":"ctx:latest","messages":[{"role":"user","content":"x"}],"options":{"num_ctx":`+raw+`}}`)) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 400 || !strings.Contains(string(b), "options.num_ctx must be a positive integer") { + t.Fatalf("num_ctx=%s status=%d body=%s", raw, resp.StatusCode, b) + } + } + if calls.Load() != 0 { + t.Fatalf("backend called %d times", calls.Load()) + } +} + +func TestResponsesInstructionsAndMaxOutputRespectEffectiveContext(t *testing.T) { + front, calls := newContextGuardServer(t, 131072, 4096, "", config.ContextPolicyConfig{MaxRequestedTokens: 32768, DefaultWorkerTokens: 4096, EstimationMarginPercent: 15, VisionReserveTokensPerImage: 2048}, nil) + body := `{"model":"ctx:latest","instructions":"` + strings.Repeat("x", 8000) + `","input":"hello","max_output_tokens":3000}` + resp, err := http.Post(front.URL+"/v1/responses", "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 400 || !strings.Contains(string(b), "effective worker context") { + t.Fatalf("status=%d body=%s headers=%v", resp.StatusCode, b, resp.Header) + } + if calls.Load() != 0 { + t.Fatalf("backend called %d times", calls.Load()) + } +} + +func TestOpenAIUsesModelfileNumCtxWhenModelNotLoaded(t *testing.T) { + front, calls := newContextGuardServer(t, 131072, 0, "num_ctx 16384\ntemperature 0.7", config.ContextPolicyConfig{MaxRequestedTokens: 32768, DefaultWorkerTokens: 4096, EstimationMarginPercent: 10, VisionReserveTokensPerImage: 2048}, nil) + body := `{"model":"ctx:latest","messages":[{"role":"user","content":"` + strings.Repeat("x", 12000) + `"}],"max_tokens":1024}` + resp, err := http.Post(front.URL+"/v1/chat/completions", "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 { + t.Fatalf("status=%d body=%s headers=%v", resp.StatusCode, b, resp.Header) + } + if calls.Load() != 1 { + t.Fatalf("backend calls=%d", calls.Load()) + } + if got := resp.Header.Get("X-Gateway-Model-Configured-Context"); got != "16384" { + t.Fatalf("configured context header=%q", got) + } +} + +func TestExplicitNumCtxHonorsGatewayAndWorkerCaps(t *testing.T) { + front, calls := newContextGuardServer(t, 131072, 4096, "", config.ContextPolicyConfig{MaxRequestedTokens: 32768, DefaultWorkerTokens: 4096, EstimationMarginPercent: 10, VisionReserveTokensPerImage: 2048}, map[string]int64{"*": 16384}) + resp, err := http.Post(front.URL+"/api/chat", "application/json", strings.NewReader(`{"model":"ctx:latest","messages":[{"role":"user","content":"x"}],"options":{"num_ctx":20000}}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 400 || !strings.Contains(string(b), "cannot be served by any eligible worker") { + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } + if calls.Load() != 0 { + t.Fatalf("backend called %d times", calls.Load()) + } +} + +func TestVisionReserveParticipatesInContextGuard(t *testing.T) { + front, calls := newContextGuardServer(t, 131072, 4096, "", config.ContextPolicyConfig{MaxRequestedTokens: 32768, DefaultWorkerTokens: 4096, EstimationMarginPercent: 0, VisionReserveTokensPerImage: 3000}, nil) + body := `{"model":"ctx:latest","messages":[{"role":"user","content":[{"type":"text","text":"hello"},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]}],"max_tokens":1500}` + resp, err := http.Post(front.URL+"/v1/chat/completions", "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 400 || resp.Header.Get("X-Gateway-Context-Vision-Reserve") != "3000" { + t.Fatalf("status=%d reserve=%q body=%s", resp.StatusCode, resp.Header.Get("X-Gateway-Context-Vision-Reserve"), b) + } + if calls.Load() != 0 { + t.Fatalf("backend called %d times", calls.Load()) + } +} + +func TestContextGuardRoutesToWorkerWithSufficientLoadedContext(t *testing.T) { + var smallCalls, largeCalls atomic.Int64 + backend := func(ctxTokens int64, calls *atomic.Int64) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "ctx:latest", "model": "ctx:latest", "context_length": ctxTokens}}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "ctx:latest", "model": "ctx:latest"}}}) + case "/api/show": + _ = json.NewEncoder(w).Encode(map[string]any{"capabilities": []string{"completion"}, "model_info": map[string]any{"ctx.context_length": 131072}}) + case "/v1/chat/completions": + calls.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{}, "usage": map[string]any{"prompt_tokens": 3000, "completion_tokens": 100}}) + default: + http.NotFound(w, r) + } + })) + } + small := backend(4096, &smallCalls) + defer small.Close() + large := backend(16384, &largeCalls) + defer large.Close() + cfg := &config.Config{ + Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute)}, + Auth: config.AuthConfig{IPBypass: []config.IPBypassConfig{{CIDRs: []string{"127.0.0.1/32"}, Tenant: "t", Subject: "u"}}}, + Scheduler: config.SchedulerConfig{GlobalConcurrency: 2, MaxQueue: 8, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, ComputePaths: []string{"/v1/chat/completions"}}, + Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3, CachedInputFactor: 1}, DefaultMaxOutputTokens: 16}, + Workers: []config.WorkerConfig{ + {Name: "small", URL: small.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}, + {Name: "large", URL: large.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}, + }, + ModelCapabilities: config.ModelCapabilitiesConfig{Mode: "enforce", CacheTTL: config.Duration(time.Hour), ContextGuard: "reject", Context: config.ContextPolicyConfig{MaxRequestedTokens: 32768, DefaultWorkerTokens: 4096, EstimationMarginPercent: 15, VisionReserveTokensPerImage: 2048}}, + } + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "small") + wp.SetModelCapabilitiesConfig(cfg.ModelCapabilities) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wp.Start(ctx) + rec, _ := usage.New("", 100, time.Second, nil) + sv := New(cfg, Dependencies{Auth: a, Scheduler: scheduler.NewLocal(2, 8, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: px.New(), Usage: rec, Metrics: metrics.New(), Logger: slog.Default()}) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + body := `{"model":"ctx:latest","messages":[{"role":"user","content":"` + strings.Repeat("x", 12000) + `"}],"max_tokens":4096}` + resp, err := http.Post(front.URL+"/v1/chat/completions", "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 { + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } + if got := resp.Header.Get("X-Gateway-Worker"); got != "large" { + t.Fatalf("worker=%q, want large; small=%d large=%d", got, smallCalls.Load(), largeCalls.Load()) + } + if smallCalls.Load() != 0 || largeCalls.Load() != 1 { + t.Fatalf("unexpected backend calls small=%d large=%d", smallCalls.Load(), largeCalls.Load()) + } +} + +func TestOpenAICannotSpoofEffectiveContextWithOptionsNumCtx(t *testing.T) { + front, calls := newContextGuardServer(t, 131072, 4096, "", config.ContextPolicyConfig{MaxRequestedTokens: 32768, DefaultWorkerTokens: 4096, EstimationMarginPercent: 15, VisionReserveTokensPerImage: 2048}, nil) + resp, err := http.Post(front.URL+"/v1/chat/completions", "application/json", strings.NewReader(`{"model":"ctx:latest","messages":[{"role":"user","content":"x"}],"options":{"num_ctx":16384}}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 400 || !strings.Contains(string(b), "only supported on native Ollama /api endpoints") { + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } + if calls.Load() != 0 { + t.Fatalf("backend called %d times", calls.Load()) + } +} + +func TestGenerateSuffixParticipatesInContextGuard(t *testing.T) { + front, calls := newContextGuardServer(t, 131072, 4096, "", config.ContextPolicyConfig{MaxRequestedTokens: 32768, DefaultWorkerTokens: 4096, EstimationMarginPercent: 10, VisionReserveTokensPerImage: 2048}, nil) + body := `{"model":"ctx:latest","prompt":"hello","suffix":"` + strings.Repeat("s", 14000) + `","options":{"num_predict":1024}}` + resp, err := http.Post(front.URL+"/api/generate", "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 400 || !strings.Contains(string(b), "effective worker context") { + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } + if calls.Load() != 0 { + t.Fatalf("backend called %d times", calls.Load()) + } +} diff --git a/internal/server/public_dashboard.go b/internal/server/public_dashboard.go new file mode 100644 index 0000000..3368aa8 --- /dev/null +++ b/internal/server/public_dashboard.go @@ -0,0 +1,344 @@ +package server + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/example/ollama-fair-gateway/internal/liveflow" + "github.com/example/ollama-fair-gateway/internal/publicui" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +type publicDashboardCounts struct { + Workers int `json:"workers"` + HealthyWorkers int `json:"healthy_workers"` + Models int `json:"models"` + Active int `json:"active"` + Queued int64 `json:"queued"` + Routing int `json:"routing"` + Running int64 `json:"running"` + Streaming int `json:"streaming"` +} + +type publicResourceMetrics struct { + MemoryUsedBytes int64 `json:"memory_used_bytes,omitempty"` + MemoryTotalBytes int64 `json:"memory_total_bytes,omitempty"` + VRAMUsedBytes int64 `json:"vram_used_bytes,omitempty"` + VRAMTotalBytes int64 `json:"vram_total_bytes,omitempty"` + GPUUtilizationPct float64 `json:"gpu_utilization_percent,omitempty"` + GPUTemperatureC float64 `json:"gpu_temperature_c,omitempty"` + GPUPowerWatts float64 `json:"gpu_power_watts,omitempty"` +} + +type publicDashboardWorker struct { + Name string `json:"name"` + Healthy bool `json:"healthy"` + Active int64 `json:"active"` + MaxConcurrent int `json:"max_concurrent"` + AcceptingNew bool `json:"accepting_new"` + Maintenance string `json:"maintenance,omitempty"` + CircuitState string `json:"circuit_state,omitempty"` + LoadedModels []string `json:"loaded_models,omitempty"` + ResourceMetrics *publicResourceMetrics `json:"resource_metrics,omitempty"` +} + +type publicDashboardRequest struct { + ID string `json:"id"` + State string `json:"state"` + Model string `json:"model,omitempty"` + Worker string `json:"worker,omitempty"` + QueueMS int64 `json:"queue_ms,omitempty"` + ServiceMS int64 `json:"service_ms,omitempty"` + PromptTokens int64 `json:"prompt_tokens,omitempty"` + CompletionTokens int64 `json:"completion_tokens,omitempty"` +} + +type publicDashboardSnapshot struct { + SchemaVersion int `json:"schema_version"` + GeneratedAt time.Time `json:"generated_at"` + Title string `json:"title"` + Subtitle string `json:"subtitle"` + RefreshIntervalMS int64 `json:"refresh_interval_ms"` + UptimeSeconds float64 `json:"uptime_seconds"` + Counts publicDashboardCounts `json:"counts"` + Workers []publicDashboardWorker `json:"workers"` + Requests []publicDashboardRequest `json:"requests"` +} + +// handlePublicDashboard serves a separate unauthenticated read-only surface. +// It is intentionally evaluated before authentication. The snapshot builder +// only copies an explicit allow-list of fields; never return admin snapshots +// directly from this handler. +func (s *Server) handlePublicDashboard(w http.ResponseWriter, r *http.Request) bool { + base := s.cfg.PublicDashboard.Path + if base == "" { + base = "/status" + } + if r.URL.Path != base && !strings.HasPrefix(r.URL.Path, base+"/") { + return false + } + if !s.cfg.PublicDashboard.Enabled { + http.NotFound(w, r) + return true + } + if r.URL.Path == base { + http.Redirect(w, r, base+"/", http.StatusTemporaryRedirect) + return true + } + + setPublicDashboardHeaders(w) + rel := strings.TrimPrefix(r.URL.Path, base+"/") + if rel == "api/snapshot" { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return true + } + w.Header().Set("Cache-Control", "public, max-age=1, stale-while-revalidate=2") + if r.Method == http.MethodHead { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + return true + } + writeJSON(w, http.StatusOK, s.publicDashboardSnapshot()) + return true + } + if r.Method != http.MethodGet && r.Method != http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return true + } + w.Header().Set("Cache-Control", "public, max-age=300") + h := publicui.Handler() + r2 := r.Clone(r.Context()) + r2.URL.Path = "/" + rel + h.ServeHTTP(w, r2) + return true +} + +func setPublicDashboardHeaders(w http.ResponseWriter) { + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Referrer-Policy", "no-referrer") + w.Header().Set("X-Frame-Options", "DENY") + w.Header().Set("Permissions-Policy", "camera=(), microphone=(), geolocation=(), payment=(), usb=()") + w.Header().Set("Content-Security-Policy", "default-src 'self'; connect-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; script-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'") +} + +func (s *Server) publicDashboardSnapshot() publicDashboardSnapshot { + cfg := s.cfg.PublicDashboard + st := s.sched.Stats(context.Background()) + ws := s.workers.Snapshots() + live := s.live.Snapshot() + sort.Slice(ws, func(i, j int) bool { return ws[i].Name < ws[j].Name }) + + workerNames := make(map[string]string, len(ws)) + for i, w := range ws { + workerNames[w.Name] = publicWorkerDisplayName(cfg.ShowWorkerNames, cfg.WorkerDisplayNames, w.Name, i) + } + modelNames := publicModelDisplayNames(cfg.ShowModelNames, ws, live.Requests) + + out := publicDashboardSnapshot{ + SchemaVersion: 1, + GeneratedAt: time.Now().UTC(), + Title: cfg.Title, + Subtitle: cfg.Subtitle, + RefreshIntervalMS: cfg.RefreshInterval.Value().Milliseconds(), + UptimeSeconds: time.Since(s.startedAt).Seconds(), + Counts: publicDashboardCounts{ + Workers: len(ws), + Active: live.Counts.Active, + Queued: st.Queued, + Routing: live.Counts.Routing, + Running: st.Running, + Streaming: live.Counts.Streaming, + }, + } + + modelSet := make(map[string]struct{}) + for _, w := range ws { + pw := publicDashboardWorker{ + Name: workerNames[w.Name], + Healthy: w.Healthy, + Active: w.Active, + MaxConcurrent: w.MaxConcurrent, + AcceptingNew: w.AcceptingNew, + Maintenance: publicMaintenance(w.Maintenance), + CircuitState: publicCircuitState(w.CircuitState), + } + if w.Healthy { + out.Counts.HealthyWorkers++ + } + for _, m := range w.LoadedModels { + if m.Name == "" { + continue + } + modelSet[m.Name] = struct{}{} + pw.LoadedModels = append(pw.LoadedModels, modelNames[m.Name]) + } + sort.Strings(pw.LoadedModels) + if cfg.ShowResourceMetrics { + memTotal := firstPositive(w.MemoryTotalBytes, w.MemoryCapacityBytes) + vramTotal := firstPositive(w.VRAMTotalBytes, w.VRAMCapacityBytes) + pw.ResourceMetrics = &publicResourceMetrics{ + MemoryUsedBytes: w.MemoryUsedBytes, + MemoryTotalBytes: memTotal, + VRAMUsedBytes: w.VRAMUsedBytes, + VRAMTotalBytes: vramTotal, + GPUUtilizationPct: w.GPUUtilizationPct, + GPUTemperatureC: w.GPUTemperatureC, + GPUPowerWatts: w.GPUPowerWatts, + } + } + out.Workers = append(out.Workers, pw) + } + out.Counts.Models = len(modelSet) + + requests := publicVisibleRequests(live.Requests, cfg.MaxLiveRequests) + for _, r := range requests { + pr := publicDashboardRequest{ + ID: publicRequestID(r.ID), + State: publicRequestState(r.State), + QueueMS: maxInt64(0, r.QueueMS), + ServiceMS: maxInt64(0, r.ServiceMS), + PromptTokens: maxInt64(0, r.PromptTokens), + CompletionTokens: maxInt64(0, r.CompletionTokens), + } + if r.Model != "" { + pr.Model = modelNames[r.Model] + } + if r.Worker != "" { + pr.Worker = workerNames[r.Worker] + if pr.Worker == "" { + pr.Worker = "Worker" + } + } + out.Requests = append(out.Requests, pr) + } + return out +} + +func publicWorkerDisplayName(show bool, aliases map[string]string, actual string, index int) string { + if alias := strings.TrimSpace(aliases[actual]); alias != "" { + return alias + } + if show && strings.TrimSpace(actual) != "" { + return actual + } + return "Worker " + twoDigits(index+1) +} + +func publicModelDisplayNames(show bool, ws []worker.Snapshot, requests []liveflow.Request) map[string]string { + set := map[string]struct{}{} + for _, w := range ws { + for _, m := range w.LoadedModels { + if m.Name != "" { + set[m.Name] = struct{}{} + } + } + } + for _, r := range requests { + if r.Model != "" { + set[r.Model] = struct{}{} + } + } + names := make([]string, 0, len(set)) + for name := range set { + names = append(names, name) + } + sort.Strings(names) + out := make(map[string]string, len(names)) + for i, name := range names { + if show { + out[name] = name + } else { + out[name] = "Model " + twoDigits(i+1) + } + } + return out +} + +func publicVisibleRequests(in []liveflow.Request, limit int) []liveflow.Request { + if limit <= 0 || len(in) <= limit { + return append([]liveflow.Request(nil), in...) + } + active := make([]liveflow.Request, 0, limit) + recent := make([]liveflow.Request, 0, limit) + for _, r := range in { + switch r.State { + case liveflow.StateCompleted, liveflow.StateCancelled, liveflow.StateFailed: + recent = append(recent, r) + default: + active = append(active, r) + } + } + if len(active) >= limit { + return active[:limit] + } + need := limit - len(active) + if need > len(recent) { + need = len(recent) + } + return append(active, recent[len(recent)-need:]...) +} + +func publicRequestID(id string) string { + sum := sha256.Sum256([]byte(id)) + return "REQ-" + strings.ToUpper(hex.EncodeToString(sum[:5])) +} + +func publicRequestState(state string) string { + switch state { + case liveflow.StateQueued, liveflow.StateRouting, liveflow.StateRunning, liveflow.StateStreaming, liveflow.StateCompleted, liveflow.StateCancelled, liveflow.StateFailed: + return state + default: + return "running" + } +} + +func publicMaintenance(v string) string { + switch strings.ToLower(strings.TrimSpace(v)) { + case "drain", "draining": + return "draining" + case "disabled", "offline": + return "disabled" + default: + return "active" + } +} + +func publicCircuitState(v string) string { + switch strings.ToLower(strings.TrimSpace(v)) { + case "open", "half-open", "half_open": + return strings.ReplaceAll(v, "_", "-") + default: + return "closed" + } +} + +func firstPositive(a, b int64) int64 { + if a > 0 { + return a + } + if b > 0 { + return b + } + return 0 +} + +func maxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} + +func twoDigits(n int) string { + if n < 10 { + return "0" + string(rune('0'+n)) + } + return strconv.Itoa(n) +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..d5a0f02 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,883 @@ +package server + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/example/ollama-fair-gateway/internal/alerts" + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/autotune" + "github.com/example/ollama-fair-gateway/internal/batch" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/conversation" + "github.com/example/ollama-fair-gateway/internal/cost" + "github.com/example/ollama-fair-gateway/internal/infrastructure" + "github.com/example/ollama-fair-gateway/internal/liveflow" + "github.com/example/ollama-fair-gateway/internal/metrics" + "github.com/example/ollama-fair-gateway/internal/policy" + "github.com/example/ollama-fair-gateway/internal/proxy" + "github.com/example/ollama-fair-gateway/internal/quota" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/session" + "github.com/example/ollama-fair-gateway/internal/telemetry" + "github.com/example/ollama-fair-gateway/internal/usage" + "github.com/example/ollama-fair-gateway/internal/warm" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +type ConfigStore interface { + Save(*config.Config) error + Delete() error + Path() string +} + +type WorkerStateStore interface { + List(context.Context) (map[string]string, error) + Put(context.Context, string, string) error + Delete(context.Context, string) error + Health(context.Context) error +} + +type ModelPlacementStore interface { + Get(context.Context, string) (config.ModelPlacementRule, bool, error) + List(context.Context) (map[string]config.ModelPlacementRule, error) + Put(context.Context, string, config.ModelPlacementRule) error + Delete(context.Context, string) error + Health(context.Context) error +} + +func unixOrZero(t time.Time) int64 { + if t.IsZero() { + return 0 + } + return t.Unix() +} + +type Server struct { + cfg *config.Config + auth *auth.Authenticator + sched scheduler.Scheduler + quota quota.Ledger + estimator *cost.Estimator + workers *worker.Pool + proxy *proxy.Proxy + usage *usage.Recorder + metrics *metrics.Registry + live *liveflow.Tracker + infrastructure *infrastructure.Hub + policies policy.Store + sessions session.Store + ops *operationManager + jobs *jobManager + startedAt time.Time + log *slog.Logger + configStore ConfigStore + placementStore ModelPlacementStore + workerStateStore WorkerStateStore + autoTune *autotune.Manager + otel *telemetry.Exporter + warm *warm.Manager + alerts *alerts.Manager + conversations *conversation.Store + batchJobs *batch.Manager + aliasMu sync.Mutex + aliases atomic.Value // immutable map[string]config.ModelAliasConfig + modelAccessMu sync.Mutex + modelAccess atomic.Value // immutable config.ModelAccessConfig +} + +type Dependencies struct { + Auth *auth.Authenticator + Scheduler scheduler.Scheduler + Quota quota.Ledger + Estimator *cost.Estimator + Workers *worker.Pool + Proxy *proxy.Proxy + Usage *usage.Recorder + Metrics *metrics.Registry + Live *liveflow.Tracker + Infrastructure *infrastructure.Hub + Policies policy.Store + Sessions session.Store + Logger *slog.Logger + ConfigStore ConfigStore + PlacementStore ModelPlacementStore + WorkerStateStore WorkerStateStore + AutoTune *autotune.Manager + OpenTelemetry *telemetry.Exporter + WarmModels *warm.Manager + Alerts *alerts.Manager + Conversations *conversation.Store + BatchJobs *batch.Manager +} + +func New(cfg *config.Config, d Dependencies) *Server { + s := &Server{cfg: cfg, auth: d.Auth, sched: d.Scheduler, quota: d.Quota, estimator: d.Estimator, workers: d.Workers, proxy: d.Proxy, usage: d.Usage, metrics: d.Metrics, live: d.Live, infrastructure: d.Infrastructure, policies: d.Policies, sessions: d.Sessions, ops: newOperationManager(), jobs: newJobManager(), startedAt: time.Now(), log: d.Logger, configStore: d.ConfigStore, placementStore: d.PlacementStore, workerStateStore: d.WorkerStateStore, autoTune: d.AutoTune, otel: d.OpenTelemetry, warm: d.WarmModels, alerts: d.Alerts, conversations: d.Conversations, batchJobs: d.BatchJobs} + s.aliases.Store(cloneModelAliases(cfg.ModelAliases)) + s.modelAccess.Store(cloneModelAccess(cfg.ModelAccess)) + if s.live == nil { + s.live = liveflow.New(10*time.Second, 512) + } + if s.policies == nil { + s.policies = policy.NewMemory() + } + if s.sessions == nil { + s.sessions = session.NewMemory() + } + s.metrics.SetDynamic(func() metrics.Dynamic { + st := s.sched.Stats(context.Background()) + ws := s.workers.Snapshots() + classes := make(map[string]metrics.ServiceClassMetric, len(st.Classes)) + for name, cs := range st.Classes { + classes[name] = metrics.ServiceClassMetric{Queued: cs.Queued, Running: cs.Running} + } + wm := make([]metrics.WorkerMetric, 0, len(ws)) + for _, w := range ws { + perf := make([]metrics.ModelPerformanceMetric, 0, len(w.Performance)) + for _, p := range w.Performance { + perf = append(perf, metrics.ModelPerformanceMetric{Model: p.Model, PromptTPS: p.PromptTPS, OutputTPS: p.OutputTPS, Samples: p.Samples}) + } + wm = append(wm, metrics.WorkerMetric{Name: w.Name, Healthy: w.Healthy, Active: w.Active, Max: w.MaxConcurrent, MemoryUsedBytes: w.MemoryUsedBytes, MemoryTotalBytes: w.MemoryTotalBytes, VRAMUsedBytes: w.VRAMUsedBytes, VRAMTotalBytes: w.VRAMTotalBytes, GPUUtilizationPct: w.GPUUtilizationPct, GPUTemperatureC: w.GPUTemperatureC, GPUPowerWatts: w.GPUPowerWatts, ModelActive: w.ModelActive, Performance: perf, CircuitState: w.CircuitState, Maintenance: w.Maintenance}) + } + ret := s.usage.RetentionStatus() + d := metrics.Dynamic{Queued: st.Queued, Running: st.Running, OldestQueueWaitSeconds: st.OldestWait.Seconds(), Workers: wm, UsageRawFiles: ret.RawFiles, UsageDailyFiles: ret.DailyFiles, UsageMonthlyFiles: ret.MonthlyFiles, UsageRawBytes: ret.RawBytes, UsageDailyBytes: ret.DailyBytes, UsageMonthlyBytes: ret.MonthlyBytes, UsageLastCompactionUnix: unixOrZero(ret.LastCompaction), UsageLastReclaimedBytes: ret.LastReclaimedBytes, ServiceClasses: classes} + if s.otel != nil { + d.OTelExportedSpans = s.otel.Exported() + d.OTelFailedSpans = s.otel.Failed() + d.OTelDroppedSpans = s.otel.Dropped() + } + if s.warm != nil { + ws := s.warm.Status() + d.WarmEvictionSuggestions = len(ws.Suggestions) + for _, a := range ws.Actions { + if a.Status == "running" { + d.WarmActionsRunning++ + } + } + } + if s.alerts != nil { + as := s.alerts.Status() + d.AlertsActive = len(as.Active) + d.AlertsLastEvaluateUnix = unixOrZero(as.LastEvaluate) + } + return d + }) + return s +} +func (s *Server) Handler() http.Handler { return http.HandlerFunc(s.serveHTTP) } + +func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) { + if s.handlePublicDashboard(w, r) { + return + } + if s.handleUIPublic(w, r) { + return + } + switch r.URL.Path { + case "/healthz": + writeJSON(w, 200, map[string]any{"status": "ok"}) + return + case "/readyz": + s.ready(w, r) + return + case "/metrics": + if s.cfg.Server.MetricsPublic { + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + s.metrics.WritePrometheus(w) + return + } + } + sessionCookieAuth := s.injectUISession(r) + id, err := s.auth.Authenticate(r) + if err != nil { + clientIP := s.auth.ClientIP(r) + w.Header().Set("WWW-Authenticate", `Bearer realm="ollama-gateway"`) + w.Header().Set("X-Gateway-Client-IP", clientIP) + s.log.Warn("authentication rejected", "client_ip", clientIP, "remote_addr", r.RemoteAddr, "method", r.Method, "path", r.URL.Path, "user_agent", r.UserAgent()) + writeProtocolError(w, r, 401, "unauthorized", "authentication required") + return + } + r = r.WithContext(auth.WithIdentity(r.Context(), id)) + if s.handleModelDiscovery(w, r, id) { + return + } + if r.URL.Path == "/metrics" { + if !id.IsAdmin() { + writeProtocolError(w, r, 403, "forbidden", "gateway:admin scope required") + return + } + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + s.metrics.WritePrometheus(w) + return + } + if strings.HasPrefix(r.URL.Path, "/gateway/ui-api/") { + s.uiAPI(w, r, id, sessionCookieAuth) + return + } + if strings.HasPrefix(r.URL.Path, "/gateway/v1/batches") { + s.batchAPI(w, r, id) + return + } + if strings.HasPrefix(r.URL.Path, "/gateway/v1/") { + s.gatewayAPI(w, r, id) + return + } + if !strings.HasPrefix(r.URL.Path, "/api/") && !strings.HasPrefix(r.URL.Path, "/v1/") { + writeProtocolError(w, r, 404, "not_found", "unknown endpoint") + return + } + if s.cfg.Native.ManagementRequiresAdmin && isManagement(r.Method, r.URL.Path) && !id.IsAdmin() { + writeProtocolError(w, r, 403, "forbidden", "model management requires gateway:admin scope") + return + } + s.forward(w, r, id) +} + +func (s *Server) handleModelDiscovery(w http.ResponseWriter, r *http.Request, id auth.Identity) bool { + if r.Method != http.MethodGet { + return false + } + switch r.URL.Path { + case "/api/tags": + models, errs := s.workers.Tags(r.Context()) + if len(errs) > 0 { + w.Header().Set("X-Gateway-Partial-Errors", strconv.Itoa(len(errs))) + s.log.Warn("partial model discovery", "endpoint", r.URL.Path, "errors", strings.Join(errs, "; ")) + if len(models) == 0 { + writeProtocolError(w, r, 503, "worker_unavailable", "unable to retrieve Ollama model tags") + return true + } + } + filtered := make([]worker.TagModel, 0, len(models)+len(s.aliasSnapshot())) + for _, m := range models { + if s.modelAllowed(id, m.Model) { + filtered = append(filtered, m) + } + } + for _, alias := range s.visibleAliases(r.Context(), id) { + filtered = append(filtered, worker.TagModel{Name: alias, Model: alias, Digest: "virtual"}) + } + sort.Slice(filtered, func(i, j int) bool { return filtered[i].Model < filtered[j].Model }) + writeJSON(w, http.StatusOK, map[string]any{"models": filtered}) + return true + case "/api/ps": + loaded := s.workers.Loaded() + filtered := loaded[:0] + for _, m := range loaded { + if s.modelAllowed(id, m.Model) { + filtered = append(filtered, m) + } + } + writeJSON(w, http.StatusOK, map[string]any{"models": filtered}) + return true + case "/v1/models": + models, errs := s.workers.Tags(r.Context()) + if len(errs) > 0 { + w.Header().Set("X-Gateway-Partial-Errors", strconv.Itoa(len(errs))) + s.log.Warn("partial model discovery", "endpoint", r.URL.Path, "errors", strings.Join(errs, "; ")) + if len(models) == 0 { + writeProtocolError(w, r, 503, "worker_unavailable", "unable to retrieve Ollama models") + return true + } + } + data := make([]map[string]any, 0, len(models)+len(s.aliasSnapshot())) + for _, m := range models { + if !s.modelAllowed(id, m.Model) { + continue + } + created := int64(0) + if !m.ModifiedAt.IsZero() { + created = m.ModifiedAt.Unix() + } + data = append(data, map[string]any{"id": m.Model, "object": "model", "created": created, "owned_by": "ollama"}) + } + for _, alias := range s.visibleAliases(r.Context(), id) { + data = append(data, map[string]any{"id": alias, "object": "model", "created": int64(0), "owned_by": "ollama-gateway"}) + } + sort.Slice(data, func(i, j int) bool { return data[i]["id"].(string) < data[j]["id"].(string) }) + writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data}) + return true + } + return false +} + +func (s *Server) ready(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + checks := map[string]string{} + ok := true + checksFn := map[string]func(context.Context) error{"workers": s.workers.Health, "scheduler": s.sched.Health, "quota": s.quota.Health, "usage": s.usage.Health, "policy": s.policies.Health, "api_keys": s.auth.RuntimeStoreHealth} + if s.placementStore != nil { + checksFn["model_placement"] = s.placementStore.Health + } + if s.workerStateStore != nil { + checksFn["worker_state"] = s.workerStateStore.Health + } + for name, fn := range checksFn { + if err := fn(ctx); err != nil { + checks[name] = err.Error() + ok = false + } else { + checks[name] = "ok" + } + } + status := 200 + if !ok { + status = 503 + } + writeJSON(w, status, map[string]any{"status": map[bool]string{true: "ready", false: "not_ready"}[ok], "checks": checks}) +} +func (s *Server) gatewayAPI(w http.ResponseWriter, r *http.Request, id auth.Identity) { + switch r.URL.Path { + case "/gateway/v1/usage/me": + writeJSON(w, 200, s.usage.Actor(r.Context(), id.Tenant, id.Actor())) + case "/gateway/v1/status": + if !id.IsAdmin() { + writeProtocolError(w, r, 403, "forbidden", "gateway:admin scope required") + return + } + st := s.sched.Stats(r.Context()) + writeJSON(w, 200, map[string]any{"scheduler": st, "workers": s.workers.Snapshots()}) + case "/gateway/v1/usage/tenant": + if !id.IsAdmin() { + writeProtocolError(w, r, 403, "forbidden", "gateway:admin scope required") + return + } + t := r.URL.Query().Get("tenant") + if t == "" { + t = id.Tenant + } + writeJSON(w, 200, s.usage.Tenant(r.Context(), t)) + default: + writeProtocolError(w, r, 404, "not_found", "unknown gateway endpoint") + } +} + +func (s *Server) forward(w http.ResponseWriter, r *http.Request, id auth.Identity) { + started := time.Now() + ctx := r.Context() + if d := s.cfg.Server.MaxRequestDuration.Value(); d > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, d) + defer cancel() + } + api := "ollama" + if r.URL.Path == "/v1/messages" { + api = "anthropic" + } else if strings.HasPrefix(r.URL.Path, "/v1/") { + api = "openai" + } + compute := s.isCompute(r.Method, r.URL.Path) + var body []byte + var outboundBody io.Reader + controlModel := "" + if compute || isModelRoutedControlRequest(r.Method, r.URL.Path) { + var err error + body, err = readBody(r, s.cfg.Server.MaxBodyBytes) + if err != nil { + writeProtocolError(w, r, 413, "request_too_large", err.Error()) + return + } + if body != nil { + outboundBody = bytes.NewReader(body) + } + if !compute { + controlModel = modelFromBody(body) + } + } else if r.Body != nil && r.Body != http.NoBody { + // Large native management/blob endpoints remain true streaming + // passthroughs. Only compute and small model-introspection requests are + // buffered so they can be routed to a worker that actually owns the model. + outboundBody = r.Body + } + var conversationPlan *responseConversationPlan + if compute && r.URL.Path == "/v1/responses" && s.conversations != nil && s.conversations.Enabled() { + var err error + body, conversationPlan, err = s.prepareResponseConversation(body, id) + if err != nil { + writeProtocolError(w, r, 400, "invalid_previous_response_id", err.Error()) + return + } + r.ContentLength = int64(len(body)) + outboundBody = bytes.NewReader(body) + w.Header().Set("X-Gateway-Conversations", "enabled") + } + requestedModel := modelFromBody(body) + resolvedModel, aliasName, resolveErr := s.resolveModel(ctx, id, requestedModel) + if resolveErr != nil { + if errors.Is(resolveErr, ErrModelAccessDenied) { + writeProtocolError(w, r, 403, "model_access_denied", resolveErr.Error()) + } else { + writeProtocolError(w, r, 404, "model_alias_unavailable", resolveErr.Error()) + } + return + } + if resolvedModel != "" && resolvedModel != requestedModel { + var err error + body, err = rewriteModelBody(body, resolvedModel) + if err != nil { + writeProtocolError(w, r, 400, "bad_model_alias", err.Error()) + return + } + r.ContentLength = int64(len(body)) + outboundBody = bytes.NewReader(body) + if !compute { + controlModel = resolvedModel + } + w.Header().Set("X-Gateway-Model-Alias", aliasName) + w.Header().Set("X-Gateway-Resolved-Model", resolvedModel) + } else if !compute { + controlModel = resolvedModel + } + est := cost.Estimate{} + preflight := modelPreflight{} + serviceClass := "" + serviceCfg := config.ServiceClassConfig{} + if compute { + est = s.estimator.Estimate(r.URL.Path, body) + var ok bool + preflight, ok = s.preflightModel(w, r, body, est) + if !ok { + return + } + var classErr error + serviceClass, serviceCfg, classErr = s.serviceClassFor(r, id) + if classErr != nil { + writeProtocolError(w, r, 403, "service_class_denied", classErr.Error()) + return + } + if h := strings.TrimSpace(s.cfg.ServiceClasses.Header); h != "" { + r.Header.Del(h) + } + w.Header().Set("X-Gateway-Service-Class", serviceClass) + } + requestID := newID() + w.Header().Set("X-Request-ID", requestID) + traceCtx := telemetry.Context{} + if compute && s.otel != nil { + traceCtx = s.otel.NewTrace(r.Header.Get("traceparent")) + if traceCtx.Sampled { + w.Header().Set("X-Gateway-Trace-ID", traceCtx.TraceID) + if tp := s.otel.TraceParent(traceCtx); tp != "" { + r.Header.Set("traceparent", tp) + } + } + } + var jobCancel context.CancelCauseFunc + if compute { + ctx, jobCancel = context.WithCancelCause(ctx) + defer jobCancel(nil) + } + var qlease *scheduler.Lease + var reservation quota.Reservation + queueDur := time.Duration(0) + workerName := "" + var workerLease *worker.Lease + var targetURL any + var queueStart, queueEnd, routeEnd time.Time + if compute { + pol := s.policyFor(ctx, id.Tenant) + limits := quota.Limits{ActorCreditsPerMinute: pol.ActorCreditsPerMinute, ActorBurstCredits: pol.ActorBurstCredits, TenantCreditsPerMinute: pol.TenantCreditsPerMinute, TenantBurstCredits: pol.TenantBurstCredits} + decision, err := s.quota.Reserve(ctx, id.Tenant, id.Actor(), est.Credits, limits) + if err != nil { + writeProtocolError(w, r, 503, "quota_unavailable", err.Error()) + return + } + if !decision.Allowed { + w.Header().Set("Retry-After", proxy.FormatRetryAfter(decision.RetryAfter)) + writeProtocolError(w, r, 429, "quota_exceeded", "compute-credit quota exceeded") + return + } + reservation = decision.Reservation + if s.alerts != nil { + s.alerts.ObserveQuota(id.Tenant, id.Actor(), decision.RemainingActor, decision.RemainingTenant, pol.ActorBurstCredits, pol.TenantBurstCredits) + } + s.live.Begin(liveflow.Request{ID: requestID, Tenant: id.Tenant, Actor: id.Actor(), Application: id.Application, ServiceClass: serviceClass, API: api, Path: r.URL.Path, Model: est.Model, EstimatedCredits: est.Credits, EstimatedPromptTokens: est.InputTokens}) + s.jobs.register(jobEntry{ID: requestID, Tenant: id.Tenant, Actor: id.Actor(), Application: id.Application, ServiceClass: serviceClass, Model: est.Model, Path: r.URL.Path, API: api, CreatedAt: time.Now().UTC()}, jobCancel) + defer s.jobs.finish(requestID) + queueTimeout := s.cfg.Scheduler.QueueTimeout.Value() + if d := serviceCfg.MaxQueueWait.Value(); d > 0 && (queueTimeout <= 0 || d < queueTimeout) { + queueTimeout = d + } + queueStart = time.Now() + qlease, err = s.sched.Acquire(ctx, scheduler.Request{Tenant: id.Tenant, Actor: id.Actor(), Cost: est.Credits, TenantWeight: pol.TenantWeight, ActorWeight: pol.ActorWeight, Timeout: queueTimeout, ServiceClass: serviceClass, ClassWeight: serviceCfg.Weight, ClassMaxConcurrent: serviceCfg.MaxConcurrent}) + if err != nil { + _ = s.quota.Reconcile(context.Background(), reservation, 0) + status := 503 + if errors.Is(err, scheduler.ErrQueueFull) || errors.Is(err, scheduler.ErrActorQueueFull) { + status = 429 + w.Header().Set("Retry-After", "1") + writeProtocolError(w, r, status, "queue_full", err.Error()) + } else if errors.Is(err, context.DeadlineExceeded) { + writeProtocolError(w, r, status, "queue_timeout", "request exceeded queue timeout") + } else if errors.Is(err, context.Canceled) { + status = 499 + if isAdminJobCancel(ctx) { + writeProtocolError(w, r, status, "request_cancelled", "request cancelled by administrator") + } else { + writeProtocolError(w, r, status, "request_cancelled", "request cancelled") + } + s.live.Cancel(requestID, status, 0, cost.Usage{}, 0) + return + } else { + writeProtocolError(w, r, status, "scheduler_unavailable", err.Error()) + } + s.live.Drop(requestID, status) + return + } + defer qlease.Release() + queueEnd = time.Now() + s.live.MarkRouting(requestID, "", qlease.Wait) + workerLease, err = s.workers.AcquireAllowed(ctx, est.Model, preflight.AllowedWorkers, preflight.RequestedContext) + if err != nil { + _ = s.quota.Reconcile(context.Background(), reservation, 0) + if errors.Is(err, context.Canceled) { + writeProtocolError(w, r, 499, "request_cancelled", "request cancelled") + s.live.Cancel(requestID, 499, 0, cost.Usage{}, 0) + return + } + if errors.Is(err, worker.ErrModelPlacementBlocked) { + writeProtocolError(w, r, 403, "model_placement_denied", err.Error()) + s.live.Drop(requestID, 403) + return + } + if errors.Is(err, worker.ErrModelNotInstalled) { + writeProtocolError(w, r, 404, "model_not_found", err.Error()) + s.live.Drop(requestID, 404) + return + } + writeProtocolError(w, r, 503, "worker_unavailable", err.Error()) + s.live.Drop(requestID, 503) + return + } + defer func() { + if workerLease != nil { + workerLease.Release() + } + }() + workerName = workerLease.Name() + if s.warm != nil { + s.warm.Touch(workerName, est.Model) + } + routeEnd = time.Now() + s.jobs.setWorker(requestID, workerName) + targetURL = workerLease.URL() + queueDur = time.Since(started) + s.live.MarkRouting(requestID, workerName, queueDur) + w.Header().Set("X-Gateway-Queue-Ms", strconv.FormatInt(queueDur.Milliseconds(), 10)) + w.Header().Set("X-Gateway-Worker", workerName) + w.Header().Set("X-Gateway-Estimated-Credits", strconv.FormatFloat(est.Credits, 'f', 4, 64)) + } else { + u, name, err := s.workers.ControlForModel(controlModel) + if err != nil { + if errors.Is(err, worker.ErrModelPlacementBlocked) { + writeProtocolError(w, r, 403, "model_placement_denied", err.Error()) + return + } + if errors.Is(err, worker.ErrModelNotInstalled) { + writeProtocolError(w, r, 404, "model_not_found", err.Error()) + return + } + writeProtocolError(w, r, 503, "worker_unavailable", err.Error()) + return + } + targetURL = u + workerName = name + } + target := targetURL.(*url.URL) + serviceStart := time.Now() + var progress proxy.ProgressFunc + if compute { + s.live.MarkRunning(requestID) + progress = func(bytesOut int64, u cost.Usage) { s.live.Progress(requestID, bytesOut, u) } + } + attempts := 1 + excluded := map[string]bool{} + reportedFailures := map[string]bool{} + forwardBody := outboundBody + if compute && body != nil { + forwardBody = bytes.NewReader(body) + } + forwardRequest := func(target *url.URL, requestBody io.Reader) proxy.Result { + if conversationPlan != nil && conversationPlan.Store { + return s.proxy.ForwardCapture(ctx, w, r, target, requestBody, api, est.InputTokens, s.cfg.Conversations.MaxContentBytes, progress) + } + return s.proxy.Forward(ctx, w, r, target, requestBody, api, est.InputTokens, progress) + } + res := forwardRequest(target, forwardBody) + for compute && s.cfg.Reliability.Enabled && res.Err != nil && !res.Started && attempts < s.cfg.Reliability.RetryAttempts && ctx.Err() == nil { + opened := s.workers.ReportResult(workerName, true, res.Err.Error()) + s.metrics.RecordUpstreamFailure(workerName, "transport") + if opened { + s.metrics.RecordCircuitOpen(workerName) + } + reportedFailures[workerName] = true + excluded[workerName] = true + if workerLease != nil { + workerLease.Release() + workerLease = nil + } + if d := s.cfg.Reliability.RetryBackoff.Value(); d > 0 { + select { + case <-ctx.Done(): + break + case <-time.After(d): + } + } + next, err := s.workers.AcquireAllowedExcluding(ctx, est.Model, preflight.AllowedWorkers, excluded, preflight.RequestedContext) + if err != nil { + break + } + workerLease = next + workerName = next.Name() + if s.warm != nil { + s.warm.Touch(workerName, est.Model) + } + target = next.URL() + s.jobs.setWorker(requestID, workerName) + s.live.MarkRouting(requestID, workerName, queueDur) + attempts++ + s.metrics.RecordRetry(workerName) + w.Header().Set("X-Gateway-Retry-Count", strconv.Itoa(attempts-1)) + w.Header().Set("X-Gateway-Worker", workerName) + res = forwardRequest(target, bytes.NewReader(body)) + } + if compute && workerName != "" { + failedWorker := (res.Err != nil && !errors.Is(ctx.Err(), context.Canceled)) || res.Status >= 500 + errText := "" + if res.Err != nil { + errText = res.Err.Error() + } else if res.Status >= 500 { + errText = fmt.Sprintf("HTTP %d", res.Status) + } + if !failedWorker || !reportedFailures[workerName] { + opened := s.workers.ReportResult(workerName, failedWorker, errText) + if failedWorker { + class := "http_5xx" + if res.Err != nil { + class = "transport" + } + s.metrics.RecordUpstreamFailure(workerName, class) + } + if opened { + s.metrics.RecordCircuitOpen(workerName) + } + } + } + serviceDur := time.Since(serviceStart) + cancelled := compute && errors.Is(ctx.Err(), context.Canceled) + if compute && res.Status < 400 && res.Usage.PromptEvalNS+res.Usage.EvalNS == 0 { + // Ollama's native API exposes exact eval durations. OpenAI-compatible + // responses generally do not, so use worker-slot wall time as a + // conservative compute-duration approximation when duration credits are enabled. + res.Usage.EvalNS = serviceDur.Nanoseconds() + res.Usage.Approximate = true + } + if res.Err != nil && !res.Started { + if cancelled { + writeProtocolError(w, r, 499, "request_cancelled", "request cancelled") + } else { + writeProtocolError(w, r, 502, "backend_error", proxy.BackendError(res.Err)) + } + } + actual := 0.0 + if compute && res.Status < 400 { + actual = s.estimator.Actual(est.Model, res.Usage) + if actual <= 0 { + actual = est.Credits + } + } + if compute { + if err := s.quota.Reconcile(context.Background(), reservation, actual); err != nil { + s.log.Warn("quota reconcile failed", "request_id", requestID, "error", err) + } + } + status := res.Status + if cancelled { + status = 499 + } else if status == 0 { + status = 502 + } + if compute { + if workerName != "" && status < 500 { + s.workers.Observe(workerName, est.Model, res.Usage.PromptTokens, res.Usage.CompletionTokens, res.Usage.PromptEvalNS, res.Usage.EvalNS, serviceDur) + } + if cancelled { + s.live.Cancel(requestID, status, actual, res.Usage, serviceDur) + } else { + s.live.Finish(requestID, status, actual, res.Usage, serviceDur) + } + } + if compute && r.URL.Path == "/v1/responses" && status < 400 { + s.persistResponseConversation(conversationPlan, res.Captured, res.CaptureTruncated, id, est.Model) + } + s.metrics.Record(api, status, queueDur, serviceDur, res.Usage.PromptTokens, res.Usage.CompletionTokens, actual, res.BytesIn, res.BytesOut) + if compute && s.otel != nil && traceCtx.Sampled { + intervals := []telemetry.Interval{} + if !queueStart.IsZero() { + intervals = append(intervals, telemetry.Interval{Name: "gateway.admission", Start: started, End: queueStart}) + } + if !queueStart.IsZero() && !queueEnd.IsZero() { + intervals = append(intervals, telemetry.Interval{Name: "gateway.queue", Start: queueStart, End: queueEnd, Attrs: map[string]any{"ollama.gateway.service_class": serviceClass}}) + } + if !queueEnd.IsZero() && !routeEnd.IsZero() { + intervals = append(intervals, telemetry.Interval{Name: "gateway.route", Start: queueEnd, End: routeEnd, Attrs: map[string]any{"server.address": workerName}}) + } + intervals = append(intervals, telemetry.Interval{Name: "ollama.upstream", Start: serviceStart, End: serviceStart.Add(serviceDur), Attrs: map[string]any{"server.address": workerName}}) + errText := "" + if res.Err != nil { + errText = res.Err.Error() + } + s.otel.Record(telemetry.Record{Trace: traceCtx, RequestID: requestID, API: api, Path: r.URL.Path, Tenant: id.Tenant, Actor: id.Actor(), Application: id.Application, ServiceClass: serviceClass, Model: est.Model, Alias: aliasName, Worker: workerName, Started: started, Finished: time.Now().UTC(), FirstByte: res.FirstByte, Status: status, PromptTokens: res.Usage.PromptTokens, OutputTokens: res.Usage.CompletionTokens, CachedTokens: res.Usage.CachedPromptTokens, Credits: actual, Error: errText, Intervals: intervals}) + } + s.usage.Record(usage.Event{ID: requestID, Time: time.Now().UTC(), Tenant: id.Tenant, Subject: id.Subject, Actor: id.Actor(), Application: id.Application, ServiceClass: serviceClass, AuthType: id.AuthType, ClientIP: id.ClientIP, API: api, Path: r.URL.Path, Model: est.Model, Worker: workerName, Status: status, QueueMS: queueDur.Milliseconds(), ServiceMS: serviceDur.Milliseconds(), EstimatedCredits: est.Credits, ActualCredits: actual, Usage: res.Usage, BytesIn: res.BytesIn, BytesOut: res.BytesOut}) + s.log.Info("request", "request_id", requestID, "tenant", id.Tenant, "subject", id.Subject, "api", api, "path", r.URL.Path, "model", est.Model, "worker", workerName, "service_class", serviceClass, "status", status, "queue_ms", queueDur.Milliseconds(), "service_ms", serviceDur.Milliseconds(), "credits", actual) +} + +func (s *Server) serviceClassFor(r *http.Request, id auth.Identity) (string, config.ServiceClassConfig, error) { + name := strings.TrimSpace(id.ServiceClass) + if name == "" { + name = strings.TrimSpace(s.cfg.ServiceClasses.Default) + } + if name == "" { + name = "interactive" + } + if len(s.cfg.ServiceClasses.Classes) == 0 { + return name, config.ServiceClassConfig{Weight: 1, MaxQueueWait: s.cfg.Scheduler.QueueTimeout}, nil + } + header := strings.TrimSpace(s.cfg.ServiceClasses.Header) + if header != "" { + if requested := strings.TrimSpace(r.Header.Get(header)); requested != "" && requested != name { + if !id.HasScope(s.cfg.ServiceClasses.OverrideScope) { + return "", config.ServiceClassConfig{}, fmt.Errorf("service class override requires scope %s", s.cfg.ServiceClasses.OverrideScope) + } + name = requested + } + } + cfg, ok := s.cfg.ServiceClasses.Classes[name] + if !ok { + return "", config.ServiceClassConfig{}, fmt.Errorf("unknown service class %q", name) + } + return name, cfg, nil +} + +func readBody(r *http.Request, maxBytes int64) ([]byte, error) { + if r.Body == nil { + return nil, nil + } + defer r.Body.Close() + b, err := io.ReadAll(io.LimitReader(r.Body, maxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > maxBytes { + return nil, fmt.Errorf("request body exceeds %d bytes", maxBytes) + } + return b, nil +} +func (s *Server) isCompute(method, path string) bool { + if method != http.MethodPost { + return false + } + for _, p := range s.cfg.Scheduler.ComputePaths { + if path == p { + return true + } + } + return false +} +func isModelRoutedControlRequest(method, path string) bool { + if method != http.MethodPost && method != http.MethodDelete { + return false + } + switch path { + case "/api/show", "/api/delete": + return true + default: + return false + } +} + +func modelFromBody(body []byte) string { + if len(body) == 0 { + return "" + } + var v struct { + Model string `json:"model"` + Name string `json:"name"` + } + if json.Unmarshal(body, &v) != nil { + return "" + } + if strings.TrimSpace(v.Model) != "" { + return strings.TrimSpace(v.Model) + } + return strings.TrimSpace(v.Name) +} + +func isManagement(method, path string) bool { + if !strings.HasPrefix(path, "/api/") { + return false + } + for _, p := range []string{"/api/pull", "/api/push", "/api/create", "/api/copy", "/api/delete", "/api/stop", "/api/blobs"} { + if path == p || strings.HasPrefix(path, p+"/") { + return true + } + } + return false +} +func writeProtocolError(w http.ResponseWriter, r *http.Request, status int, code, msg string) { + if r != nil && r.URL.Path == "/v1/messages" { + typ := "api_error" + switch status { + case 400: + typ = "invalid_request_error" + case 401: + typ = "authentication_error" + case 403: + typ = "permission_error" + case 404: + typ = "not_found_error" + case 429: + typ = "rate_limit_error" + case 500, 502, 503, 504: + typ = "api_error" + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{"type": "error", "error": map[string]any{"type": typ, "message": msg}}) + return + } + // Ollama's native API uses a string-valued "error" field. OpenWebUI + // relies on that shape when verifying Ollama connections; returning the + // OpenAI-style nested object causes UI messages such as "[object Object]". + if r != nil && strings.HasPrefix(r.URL.Path, "/api/") { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{"error": msg}) + return + } + proxy.WriteJSONError(w, status, code, msg) +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} +func newID() string { b := make([]byte, 16); _, _ = rand.Read(b); return hex.EncodeToString(b) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000..5f2c286 --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,297 @@ +package server + +import ( + "context" + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/cost" + "github.com/example/ollama-fair-gateway/internal/metrics" + px "github.com/example/ollama-fair-gateway/internal/proxy" + "github.com/example/ollama-fair-gateway/internal/quota" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/usage" + "github.com/example/ollama-fair-gateway/internal/worker" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestNativeStreamingPassthroughAndMetering(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/ps" { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"models":[{"name":"qwen3:8b"}]}`) + return + } + if r.URL.Path == "/api/chat" { + w.Header().Set("Content-Type", "application/x-ndjson") + io.WriteString(w, "{\"message\":{\"content\":\"hi\"},\"done\":false}\n{\"done\":true,\"prompt_eval_count\":10,\"eval_count\":2,\"eval_duration\":1000}\n") + return + } + w.WriteHeader(404) + })) + defer backend.Close() + cfg := &config.Config{Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute), MetricsPublic: true}, Auth: config.AuthConfig{IPBypass: []config.IPBypassConfig{{CIDRs: []string{"127.0.0.1/32"}, Tenant: "test", Subject: "u"}}}, Scheduler: config.SchedulerConfig{GlobalConcurrency: 1, MaxQueue: 8, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1}, Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 16}, Workers: []config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}}} + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wp.Start(ctx) + met := metrics.New() + rec, _ := usage.New("", 100, time.Second, nil) + sv := New(cfg, Dependencies{Auth: a, Scheduler: scheduler.NewLocal(1, 8, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: px.New(), Usage: rec, Metrics: met, Logger: slog.Default()}) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + resp, err := http.Post(front.URL+"/api/chat", "application/json", strings.NewReader(`{"model":"qwen3:8b","messages":[{"role":"user","content":"x"}]}`)) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + want := "{\"message\":{\"content\":\"hi\"},\"done\":false}\n{\"done\":true,\"prompt_eval_count\":10,\"eval_count\":2,\"eval_duration\":1000}\n" + if string(b) != want { + t.Fatalf("body changed:\n%s", b) + } + var s usage.Summary + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + s = rec.Actor(context.Background(), "test", "u") + if s.PromptTokens == 10 && s.CompletionTokens == 2 { + break + } + time.Sleep(time.Millisecond) + } + if s.PromptTokens != 10 || s.CompletionTokens != 2 { + t.Fatalf("usage not metered: %#v", s) + } +} + +func TestNativeNonComputeRequestBodyStreamsPastComputeLimit(t *testing.T) { + const bodySize = 256 << 10 + gotSize := make(chan int64, 1) + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"models":[]}`) + case "/api/blobs/sha256:test": + n, _ := io.Copy(io.Discard, r.Body) + gotSize <- n + w.WriteHeader(http.StatusCreated) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer backend.Close() + + cfg := &config.Config{ + Server: config.ServerConfig{MaxBodyBytes: 32, MaxRequestDuration: config.Duration(time.Minute)}, + Auth: config.AuthConfig{IPBypass: []config.IPBypassConfig{{CIDRs: []string{"127.0.0.1/32"}, Tenant: "test", Subject: "u", Scopes: []string{"gateway:admin"}}}}, + Scheduler: config.SchedulerConfig{GlobalConcurrency: 1, MaxQueue: 8, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, ComputePaths: []string{"/api/chat"}}, + Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 16}, + Workers: []config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}}, + Native: config.NativeConfig{ManagementRequiresAdmin: true, ControlWorker: "w"}, + } + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "w") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wp.Start(ctx) + met := metrics.New() + rec, _ := usage.New("", 100, time.Second, nil) + sv := New(cfg, Dependencies{Auth: a, Scheduler: scheduler.NewLocal(1, 8, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: px.New(), Usage: rec, Metrics: met, Logger: slog.Default()}) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + req, _ := http.NewRequest(http.MethodPut, front.URL+"/api/blobs/sha256:test", io.LimitReader(strings.NewReader(strings.Repeat("x", bodySize)), bodySize)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status=%d", resp.StatusCode) + } + select { + case n := <-gotSize: + if n != bodySize { + t.Fatalf("backend received %d bytes, want %d", n, bodySize) + } + case <-time.After(time.Second): + t.Fatal("backend did not receive streamed body") + } +} + +func TestModelAliasAndTenantACL(t *testing.T) { + seen := make(chan string, 1) + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + io.WriteString(w, `{"models":[{"name":"real:1","model":"real:1"}]}`) + case "/api/tags": + io.WriteString(w, `{"models":[{"name":"real:1","model":"real:1"}]}`) + case "/api/show": + io.WriteString(w, `{"capabilities":["completion"]}`) + case "/api/chat": + b, _ := io.ReadAll(r.Body) + seen <- string(b) + w.Header().Set("Content-Type", "application/x-ndjson") + io.WriteString(w, `{"done":true,"prompt_eval_count":1,"eval_count":1}`+"\n") + default: + http.NotFound(w, r) + } + })) + defer backend.Close() + visible := true + cfg := &config.Config{ + Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute)}, + Auth: config.AuthConfig{IPBypass: []config.IPBypassConfig{{CIDRs: []string{"127.0.0.1/32"}, Tenant: "t", Subject: "u"}}}, + Scheduler: config.SchedulerConfig{GlobalConcurrency: 1, MaxQueue: 8, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, ComputePaths: []string{"/api/chat"}}, + Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 16}, + Workers: []config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}}, + ModelAliases: map[string]config.ModelAliasConfig{"fast": {Models: []string{"real:1"}, Visible: &visible}}, + ModelAccess: config.ModelAccessConfig{Default: config.ModelAccessRule{Mode: "whitelist", AllowedModels: []string{"fast"}}}, + ModelCapabilities: config.ModelCapabilitiesConfig{Mode: "enforce", CacheTTL: config.Duration(time.Hour), ContextGuard: "off"}, + } + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "") + wp.SetModelCapabilitiesConfig(cfg.ModelCapabilities) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wp.Start(ctx) + rec, _ := usage.New("", 100, time.Second, nil) + sv := New(cfg, Dependencies{Auth: a, Scheduler: scheduler.NewLocal(1, 8, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: px.New(), Usage: rec, Metrics: metrics.New(), Logger: slog.Default()}) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + resp, err := http.Get(front.URL + "/api/tags") + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if !strings.Contains(string(b), `"model":"fast"`) || strings.Contains(string(b), `"model":"real:1"`) { + t.Fatalf("unexpected discovery: %s", b) + } + resp, err = http.Post(front.URL+"/api/chat", "application/json", strings.NewReader(`{"model":"fast","messages":[{"role":"user","content":"x"}]}`)) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("alias status=%d", resp.StatusCode) + } + select { + case body := <-seen: + if !strings.Contains(body, `"model":"real:1"`) { + t.Fatalf("backend body=%s", body) + } + case <-time.After(time.Second): + t.Fatal("backend not called") + } + resp, err = http.Post(front.URL+"/api/chat", "application/json", strings.NewReader(`{"model":"real:1","messages":[]}`)) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode != 403 { + t.Fatalf("real model should be ACL denied, got %d", resp.StatusCode) + } +} + +func TestSafeRetryBeforeResponseAndCircuitOpen(t *testing.T) { + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + io.WriteString(w, `{"models":[]}`) + case "/api/tags": + io.WriteString(w, `{"models":[{"name":"m","model":"m"}]}`) + case "/api/show": + io.WriteString(w, `{"capabilities":["completion"]}`) + case "/api/chat": + c, _, _ := w.(http.Hijacker).Hijack() + _ = c.Close() + default: + http.NotFound(w, r) + } + })) + defer bad.Close() + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + io.WriteString(w, `{"models":[]}`) + case "/api/tags": + io.WriteString(w, `{"models":[{"name":"m","model":"m"}]}`) + case "/api/show": + io.WriteString(w, `{"capabilities":["completion"]}`) + case "/api/chat": + w.Header().Set("Content-Type", "application/x-ndjson") + io.WriteString(w, `{"done":true,"prompt_eval_count":2,"eval_count":1}`+"\n") + default: + http.NotFound(w, r) + } + })) + defer good.Close() + cfg := &config.Config{Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute)}, Auth: config.AuthConfig{IPBypass: []config.IPBypassConfig{{CIDRs: []string{"127.0.0.1/32"}, Tenant: "t", Subject: "u"}}}, Scheduler: config.SchedulerConfig{GlobalConcurrency: 1, MaxQueue: 8, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, ComputePaths: []string{"/api/chat"}}, Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 8}, Workers: []config.WorkerConfig{{Name: "bad", URL: bad.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}, {Name: "good", URL: good.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}}, Reliability: config.ReliabilityConfig{Enabled: true, FailureThreshold: 1, OpenDuration: config.Duration(time.Hour), RetryAttempts: 2}, ModelCapabilities: config.ModelCapabilitiesConfig{Mode: "off", ContextGuard: "off"}} + a, _ := auth.New(context.Background(), cfg.Auth) + wp := worker.New(cfg.Workers, "") + wp.SetReliabilityConfig(cfg.Reliability) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wp.Start(ctx) + rec, _ := usage.New("", 100, time.Second, nil) + sv := New(cfg, Dependencies{Auth: a, Scheduler: scheduler.NewLocal(1, 8, 8), Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: px.New(), Usage: rec, Metrics: metrics.New(), Logger: slog.Default()}) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + resp, err := http.Post(front.URL+"/api/chat", "application/json", strings.NewReader(`{"model":"m","messages":[]}`)) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("status=%d", resp.StatusCode) + } + if got := resp.Header.Get("X-Gateway-Retry-Count"); got != "1" { + t.Fatalf("retry header=%q", got) + } + if got := resp.Header.Get("X-Gateway-Worker"); got != "good" { + t.Fatalf("worker=%q", got) + } + for _, snap := range wp.Snapshots() { + if snap.Name == "bad" && snap.CircuitState != "open" { + t.Fatalf("bad circuit=%s", snap.CircuitState) + } + } +} + +func TestAPIKeyModelACLCanNarrowButNotWidenTenantACL(t *testing.T) { + cfg := &config.Config{} + cfg.ModelAccess = config.ModelAccessConfig{Default: config.ModelAccessRule{Mode: "whitelist", AllowedModels: []string{"fast", "qwen3:8b"}}} + s := &Server{cfg: cfg} + id := auth.Identity{Tenant: "team", ModelACLSet: true, ModelAccess: config.ModelAccessRule{Mode: "whitelist", AllowedModels: []string{"fast", "gemma4:*"}}} + if !s.modelAllowed(id, "fast") { + t.Fatal("expected intersection to allow fast") + } + if s.modelAllowed(id, "qwen3:8b") { + t.Fatal("API key ACL should narrow tenant ACL") + } + if s.modelAllowed(id, "gemma4:latest") { + t.Fatal("API key ACL must not widen tenant ACL") + } +} diff --git a/internal/server/storage.go b/internal/server/storage.go new file mode 100644 index 0000000..5f71a2e --- /dev/null +++ b/internal/server/storage.go @@ -0,0 +1,288 @@ +package server + +import ( + "archive/zip" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/example/ollama-fair-gateway/internal/state" +) + +type persistentSaver interface { + SavePersistent(string) error +} + +type storageFileStatus struct { + Name string `json:"name"` + Kind string `json:"kind"` + Path string `json:"path"` + Exists bool `json:"exists"` + Size int64 `json:"size_bytes,omitempty"` + Modified time.Time `json:"modified_at,omitempty"` +} + +func fileStatus(name, kind, path string) storageFileStatus { + x := storageFileStatus{Name: name, Kind: kind, Path: path} + st, err := os.Stat(path) + if err == nil && st.Mode().IsRegular() { + x.Exists = true + x.Size = st.Size() + x.Modified = st.ModTime().UTC() + } + return x +} + +func (s *Server) storageStatus() map[string]any { + paths := state.Resolve(s.cfg.Storage) + files := []storageFileStatus{ + fileStatus("configuration override", "config", paths.Config), + fileStatus("API keys", "security", paths.APIKeys), + fileStatus("tenant policies", "policy", paths.Policies), + fileStatus("metrics snapshot", "metrics", paths.Metrics), + fileStatus("quota buckets", "quota", paths.Quota), + fileStatus("worker performance", "routing", paths.WorkerPerformance), + fileStatus("model placement", "routing", paths.ModelPlacement), + fileStatus("worker runtime state", "routing", paths.WorkerState), + fileStatus("warm model policies", "capacity", paths.WarmModels), + fileStatus("alerts history", "alerts", paths.Alerts), + fileStatus("encrypted conversations", "content", paths.Conversations), + fileStatus("durable batch jobs", "batch", paths.BatchJobs), + } + usageDir := s.cfg.Usage.JournalDir + usageFiles := 0 + var usageBytes int64 + var usageNewest time.Time + if usageDir != "" { + matches, _ := filepath.Glob(filepath.Join(usageDir, "usage-*.jsonl")) + for _, p := range matches { + if st, err := os.Stat(p); err == nil && st.Mode().IsRegular() { + usageFiles++ + usageBytes += st.Size() + if st.ModTime().After(usageNewest) { + usageNewest = st.ModTime().UTC() + } + } + } + } + batchFiles := 0 + var batchBytes int64 + var batchNewest time.Time + _ = filepath.Walk(paths.BatchDir, func(path string, info os.FileInfo, err error) error { + if err == nil && info.Mode().IsRegular() { + batchFiles++ + batchBytes += info.Size() + if info.ModTime().After(batchNewest) { + batchNewest = info.ModTime().UTC() + } + } + return nil + }) + retention := s.usage.RetentionStatus() + var total int64 + for _, f := range files { + total += f.Size + } + total += usageBytes + retention.DailyBytes + retention.MonthlyBytes + batchBytes + return map[string]any{ + "mode": "local-persistent", + "data_dir": paths.DataDir, + "flush_interval": s.cfg.Storage.FlushInterval.Value().String(), + "config_override_active": s.configStore != nil && fileStatus("", "", paths.Config).Exists, + "files": files, + "usage": map[string]any{ + "directory": usageDir, + "files": usageFiles, + "size_bytes": usageBytes, + "modified_at": usageNewest, + "retention": retention, + }, + "batch": map[string]any{ + "enabled": s.batchJobs != nil && s.batchJobs.Enabled(), + "directory": paths.BatchDir, + "files": batchFiles, + "size_bytes": batchBytes, + "modified_at": batchNewest, + }, + "conversations": func() any { + if s.conversations == nil { + return map[string]any{"enabled": false} + } + return s.conversations.Status() + }(), + "total_bytes": total, + "volatile": []string{ + "active transient inference jobs and cancellation handles", + "active durable-batch attempt contexts (batch metadata remains persistent)", + "fair-queue heap and virtual clocks", + "active worker/model slots", + "browser OIDC sessions", + "live-flow animation state", + }, + } +} + +func (s *Server) flushPersistentState(ctx context.Context) error { + paths := state.Resolve(s.cfg.Storage) + var errs []string + if err := s.metrics.SavePersistent(paths.Metrics); err != nil { + errs = append(errs, "metrics: "+err.Error()) + } + if saver, ok := s.quota.(persistentSaver); ok { + if err := saver.SavePersistent(paths.Quota); err != nil { + errs = append(errs, "quota: "+err.Error()) + } + } + if err := s.workers.SavePerformance(paths.WorkerPerformance); err != nil { + errs = append(errs, "worker performance: "+err.Error()) + } + if err := s.usage.Flush(ctx); err != nil { + errs = append(errs, "usage: "+err.Error()) + } + if s.conversations != nil && s.conversations.Enabled() { + if err := s.conversations.Compact(); err != nil { + errs = append(errs, "conversations: "+err.Error()) + } + } + if s.batchJobs != nil && s.batchJobs.Enabled() { + if err := s.batchJobs.Compact(); err != nil { + errs = append(errs, "batch jobs: "+err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("persistent flush failed: %s", strings.Join(errs, "; ")) + } + return nil +} + +func (s *Server) uiStorageFlush(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + if err := s.flushPersistentState(ctx); err != nil { + writeProtocolError(w, r, http.StatusServiceUnavailable, "storage_flush", err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]any{"flushed": true, "at": time.Now().UTC(), "storage": s.storageStatus()}) +} + +func (s *Server) uiStorageCompact(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Minute) + defer cancel() + if err := s.flushPersistentState(ctx); err != nil { + writeProtocolError(w, r, http.StatusServiceUnavailable, "storage_flush", err.Error()) + return + } + status, err := s.usage.Compact(ctx) + if err != nil { + writeProtocolError(w, r, http.StatusInternalServerError, "usage_compaction", err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]any{"compacted": true, "at": time.Now().UTC(), "retention": status, "storage": s.storageStatus()}) +} + +func (s *Server) uiStorageBackup(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + if err := s.flushPersistentState(ctx); err != nil { + writeProtocolError(w, r, http.StatusServiceUnavailable, "storage_flush", err.Error()) + return + } + name := "ollama-gateway-backup-" + time.Now().UTC().Format("20060102-150405") + ".zip" + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`) + w.Header().Set("Cache-Control", "no-store") + zw := zip.NewWriter(w) + defer zw.Close() + + paths := state.Resolve(s.cfg.Storage) + known := []struct{ archive, path string }{ + {"state/" + filepath.Base(paths.Config), paths.Config}, + {"state/" + filepath.Base(paths.APIKeys), paths.APIKeys}, + {"state/" + filepath.Base(paths.Policies), paths.Policies}, + {"state/" + filepath.Base(paths.Metrics), paths.Metrics}, + {"state/" + filepath.Base(paths.Quota), paths.Quota}, + {"state/" + filepath.Base(paths.WorkerPerformance), paths.WorkerPerformance}, + {"state/" + filepath.Base(paths.ModelPlacement), paths.ModelPlacement}, + {"state/" + filepath.Base(paths.WorkerState), paths.WorkerState}, + {"state/" + filepath.Base(paths.WarmModels), paths.WarmModels}, + {"state/" + filepath.Base(paths.Alerts), paths.Alerts}, + {"state/" + filepath.Base(paths.Conversations), paths.Conversations}, + {"state/" + filepath.Base(paths.BatchJobs), paths.BatchJobs}, + } + for _, f := range known { + if err := zipFile(zw, f.archive, f.path); err != nil && !os.IsNotExist(err) { + return + } + } + if paths.BatchDir != "" { + var matches []string + _ = filepath.Walk(paths.BatchDir, func(path string, info os.FileInfo, err error) error { + if err == nil && info.Mode().IsRegular() { + matches = append(matches, path) + } + return nil + }) + sort.Strings(matches) + for _, p := range matches { + rel, err := filepath.Rel(paths.BatchDir, p) + if err != nil { + continue + } + if err := zipFile(zw, "batch/"+filepath.ToSlash(rel), p); err != nil && !os.IsNotExist(err) { + return + } + } + } + if s.cfg.Usage.JournalDir != "" { + var matches []string + _ = filepath.Walk(s.cfg.Usage.JournalDir, func(path string, info os.FileInfo, err error) error { + if err == nil && info.Mode().IsRegular() && (strings.HasSuffix(info.Name(), ".jsonl") || strings.HasSuffix(info.Name(), ".json")) { + matches = append(matches, path) + } + return nil + }) + sort.Strings(matches) + for _, p := range matches { + rel, err := filepath.Rel(s.cfg.Usage.JournalDir, p) + if err != nil { + continue + } + if err := zipFile(zw, "usage/"+filepath.ToSlash(rel), p); err != nil && !os.IsNotExist(err) { + return + } + } + } +} + +func zipFile(zw *zip.Writer, archiveName, path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return err + } + if !st.Mode().IsRegular() { + return nil + } + h, err := zip.FileInfoHeader(st) + if err != nil { + return err + } + h.Name = filepath.ToSlash(archiveName) + h.Method = zip.Deflate + dst, err := zw.CreateHeader(h) + if err != nil { + return err + } + _, err = io.Copy(dst, f) + return err +} diff --git a/internal/server/ui.go b/internal/server/ui.go new file mode 100644 index 0000000..3d888fb --- /dev/null +++ b/internal/server/ui.go @@ -0,0 +1,1423 @@ +package server + +import ( + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/autotune" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/proxy" + "github.com/example/ollama-fair-gateway/internal/webui" +) + +const ( + uiSessionCookie = "ofg_session" + uiStateCookie = "ofg_oidc_state" + uiCSRFCookie = "ofg_csrf" +) + +type oidcLoginState struct { + State string `json:"state"` + Verifier string `json:"verifier"` + Redirect string `json:"redirect"` + Expires int64 `json:"expires"` +} + +func (s *Server) handleUIPublic(w http.ResponseWriter, r *http.Request) bool { + if !s.cfg.UI.Enabled { + return false + } + if r.URL.Path == "/gateway/ui-api/bootstrap" { + writeJSON(w, 200, map[string]any{ + "title": s.cfg.UI.Title, + "ui_path": s.cfg.UI.Path, + "oidc_enabled": s.cfg.UI.OIDC.Enabled, + "oidc_login_url": s.cfg.UI.Path + "/login/oidc", + }) + return true + } + base := s.cfg.UI.Path + if r.URL.Path == base { + http.Redirect(w, r, base+"/", http.StatusTemporaryRedirect) + return true + } + if !strings.HasPrefix(r.URL.Path, base+"/") { + return false + } + rel := strings.TrimPrefix(r.URL.Path, base+"/") + switch rel { + case "login/oidc": + s.uiOIDCLogin(w, r) + return true + case "callback": + s.uiOIDCCallback(w, r) + return true + case "logout": + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return true + } + s.clearUICookies(w, r) + w.WriteHeader(http.StatusNoContent) + return true + } + if r.Method != http.MethodGet && r.Method != http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return true + } + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Referrer-Policy", "no-referrer") + w.Header().Set("X-Frame-Options", "DENY") + w.Header().Set("Content-Security-Policy", "default-src 'self'; connect-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; script-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'") + w.Header().Set("Cache-Control", "no-store") + h := webui.Handler() + r2 := r.Clone(r.Context()) + r2.URL.Path = "/" + rel + h.ServeHTTP(w, r2) + return true +} + +func (s *Server) injectUISession(r *http.Request) bool { + if !s.cfg.UI.Enabled || r.Header.Get("Authorization") != "" || r.Header.Get("X-API-Key") != "" { + return false + } + c, err := r.Cookie(uiSessionCookie) + if err != nil || c.Value == "" { + return false + } + token, err := s.sessions.Get(r.Context(), c.Value) + if err != nil || token == "" { + return false + } + r.Header.Set("Authorization", "Bearer "+token) + return true +} + +func (s *Server) uiOIDCLogin(w http.ResponseWriter, r *http.Request) { + if !s.cfg.UI.OIDC.Enabled || !s.auth.OIDCEnabled() { + proxy.WriteJSONError(w, 404, "oidc_disabled", "browser OIDC login is disabled") + return + } + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + ep, ok := s.auth.OIDCBrowserEndpoints() + if !ok || ep.Authorization == "" { + proxy.WriteJSONError(w, 503, "oidc_unavailable", "OIDC authorization endpoint unavailable") + return + } + state := randomURLToken(24) + verifier := randomURLToken(48) + redirect := s.uiRedirectURL(r) + payload := oidcLoginState{State: state, Verifier: verifier, Redirect: redirect, Expires: time.Now().Add(10 * time.Minute).Unix()} + signed, err := s.signUIState(payload) + if err != nil { + proxy.WriteJSONError(w, 500, "oidc_state", err.Error()) + return + } + http.SetCookie(w, &http.Cookie{Name: uiStateCookie, Value: signed, Path: s.cfg.UI.Path + "/", MaxAge: 600, HttpOnly: true, Secure: s.uiSecureCookie(r), SameSite: http.SameSiteLaxMode}) + h := sha256.Sum256([]byte(verifier)) + q := url.Values{} + q.Set("response_type", "code") + q.Set("client_id", s.cfg.UI.OIDC.ClientID) + q.Set("redirect_uri", redirect) + q.Set("scope", strings.Join(s.cfg.UI.OIDC.Scopes, " ")) + q.Set("state", state) + q.Set("code_challenge", base64.RawURLEncoding.EncodeToString(h[:])) + q.Set("code_challenge_method", "S256") + target := ep.Authorization + sep := "?" + if strings.Contains(target, "?") { + sep = "&" + } + http.Redirect(w, r, target+sep+q.Encode(), http.StatusFound) +} + +func (s *Server) uiOIDCCallback(w http.ResponseWriter, r *http.Request) { + if !s.cfg.UI.OIDC.Enabled || r.Method != http.MethodGet { + w.WriteHeader(http.StatusNotFound) + return + } + if e := r.URL.Query().Get("error"); e != "" { + http.Redirect(w, r, s.cfg.UI.Path+"/?login_error="+url.QueryEscape(e), http.StatusFound) + return + } + cookie, err := r.Cookie(uiStateCookie) + if err != nil { + proxy.WriteJSONError(w, 400, "oidc_state", "missing OIDC state cookie") + return + } + st, err := s.verifyUIState(cookie.Value) + if err != nil || st.Expires < time.Now().Unix() || subtle.ConstantTimeCompare([]byte(st.State), []byte(r.URL.Query().Get("state"))) != 1 { + proxy.WriteJSONError(w, 400, "oidc_state", "invalid or expired OIDC state") + return + } + code := r.URL.Query().Get("code") + if code == "" { + proxy.WriteJSONError(w, 400, "oidc_code", "missing authorization code") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + tok, err := s.auth.ExchangeOIDCCode(ctx, code, st.Redirect, s.cfg.UI.OIDC.ClientID, s.cfg.UI.OIDC.ClientSecret, st.Verifier) + if err != nil { + proxy.WriteJSONError(w, 502, "oidc_exchange", err.Error()) + return + } + if _, err := s.auth.VerifyOIDCToken(ctx, tok.AccessToken); err != nil { + proxy.WriteJSONError(w, 401, "oidc_token", "OIDC access token is not valid for this gateway") + return + } + maxAge := int(tok.ExpiresIn) + if maxAge <= 0 || maxAge > 86400 { + maxAge = 3600 + } + sessionID, err := s.sessions.Create(ctx, tok.AccessToken, time.Duration(maxAge)*time.Second) + if err != nil { + proxy.WriteJSONError(w, 503, "session_store", "could not create UI session") + return + } + http.SetCookie(w, &http.Cookie{Name: uiSessionCookie, Value: sessionID, Path: "/", MaxAge: maxAge, HttpOnly: true, Secure: s.uiSecureCookie(r), SameSite: http.SameSiteLaxMode}) + csrf := randomURLToken(24) + http.SetCookie(w, &http.Cookie{Name: uiCSRFCookie, Value: csrf, Path: "/", MaxAge: maxAge, HttpOnly: false, Secure: s.uiSecureCookie(r), SameSite: http.SameSiteLaxMode}) + http.SetCookie(w, &http.Cookie{Name: uiStateCookie, Value: "", Path: s.cfg.UI.Path + "/", MaxAge: -1, HttpOnly: true, Secure: s.uiSecureCookie(r), SameSite: http.SameSiteLaxMode}) + http.Redirect(w, r, s.cfg.UI.Path+"/", http.StatusFound) +} + +func (s *Server) uiRedirectURL(r *http.Request) string { + if s.cfg.UI.OIDC.RedirectURL != "" { + return s.cfg.UI.OIDC.RedirectURL + } + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + return scheme + "://" + r.Host + s.cfg.UI.Path + "/callback" +} +func (s *Server) uiSecureCookie(r *http.Request) bool { return s.cfg.UI.SecureCookies || r.TLS != nil } +func (s *Server) clearUICookies(w http.ResponseWriter, r *http.Request) { + if c, err := r.Cookie(uiSessionCookie); err == nil && c.Value != "" { + _ = s.sessions.Delete(r.Context(), c.Value) + } + for _, c := range []http.Cookie{ + {Name: uiSessionCookie, Path: "/", MaxAge: -1, HttpOnly: true}, + {Name: uiCSRFCookie, Path: "/", MaxAge: -1}, + {Name: uiStateCookie, Path: s.cfg.UI.Path + "/", MaxAge: -1, HttpOnly: true}, + } { + c.Secure = s.uiSecureCookie(r) + c.SameSite = http.SameSiteLaxMode + http.SetCookie(w, &c) + } +} +func (s *Server) signUIState(st oidcLoginState) (string, error) { + b, err := json.Marshal(st) + if err != nil { + return "", err + } + p := base64.RawURLEncoding.EncodeToString(b) + m := hmac.New(sha256.New, []byte(s.cfg.UI.SessionSecret)) + _, _ = m.Write([]byte(p)) + return p + "." + base64.RawURLEncoding.EncodeToString(m.Sum(nil)), nil +} +func (s *Server) verifyUIState(v string) (oidcLoginState, error) { + parts := strings.Split(v, ".") + if len(parts) != 2 { + return oidcLoginState{}, errors.New("bad state") + } + sig, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return oidcLoginState{}, err + } + m := hmac.New(sha256.New, []byte(s.cfg.UI.SessionSecret)) + _, _ = m.Write([]byte(parts[0])) + if !hmac.Equal(sig, m.Sum(nil)) { + return oidcLoginState{}, errors.New("bad state signature") + } + b, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil { + return oidcLoginState{}, err + } + var out oidcLoginState + err = json.Unmarshal(b, &out) + return out, err +} +func randomURLToken(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return base64.RawURLEncoding.EncodeToString(b) +} + +func (s *Server) checkUICSRF(r *http.Request, sessionCookieAuth bool) bool { + if !sessionCookieAuth || r.Method == http.MethodGet || r.Method == http.MethodHead || r.Method == http.MethodOptions { + return true + } + c, err := r.Cookie(uiCSRFCookie) + if err != nil || c.Value == "" || subtle.ConstantTimeCompare([]byte(c.Value), []byte(r.Header.Get("X-CSRF-Token"))) != 1 { + return false + } + if origin := r.Header.Get("Origin"); origin != "" { + u, err := url.Parse(origin) + if err != nil || !strings.EqualFold(u.Host, r.Host) { + return false + } + } + return true +} + +func (s *Server) uiAPI(w http.ResponseWriter, r *http.Request, id auth.Identity, sessionCookieAuth bool) { + if !s.cfg.UI.Enabled { + proxy.WriteJSONError(w, 404, "not_found", "web UI disabled") + return + } + if !s.checkUICSRF(r, sessionCookieAuth) { + proxy.WriteJSONError(w, 403, "csrf", "invalid CSRF token") + return + } + if r.URL.Path == "/gateway/ui-api/session" { + scopes := make([]string, 0, len(id.Scopes)) + for x := range id.Scopes { + scopes = append(scopes, x) + } + sort.Strings(scopes) + writeJSON(w, 200, map[string]any{"tenant": id.Tenant, "subject": id.Subject, "application": id.Application, "auth_type": id.AuthType, "actor": id.Actor(), "admin": id.IsAdmin(), "scopes": scopes, "client_ip": id.ClientIP}) + return + } + if !id.IsAdmin() { + proxy.WriteJSONError(w, 403, "forbidden", "gateway:admin scope required") + return + } + switch { + case r.URL.Path == "/gateway/ui-api/live" && r.Method == http.MethodGet: + writeJSON(w, 200, s.live.Snapshot()) + case r.URL.Path == "/gateway/ui-api/live/stream" && r.Method == http.MethodGet: + s.uiLiveStream(w, r) + case r.URL.Path == "/gateway/ui-api/infrastructure" && r.Method == http.MethodGet: + if s.infrastructure == nil { + proxy.WriteJSONError(w, 503, "infrastructure_unavailable", "infrastructure live view is unavailable") + return + } + writeJSON(w, 200, s.infrastructure.Snapshot()) + case r.URL.Path == "/gateway/ui-api/infrastructure/stream" && r.Method == http.MethodGet: + s.uiInfrastructureStream(w, r) + case r.URL.Path == "/gateway/ui-api/policy-simulator" && r.Method == http.MethodPost: + var in policySimulationRequest + if err := decodeJSON(r, &in, 1<<20); err != nil { + proxy.WriteJSONError(w, 400, "bad_request", err.Error()) + return + } + result, err := s.simulatePolicy(r.Context(), in) + if err != nil { + proxy.WriteJSONError(w, 400, "simulation_failed", err.Error()) + return + } + writeJSON(w, 200, result) + case r.URL.Path == "/gateway/ui-api/overview" && r.Method == http.MethodGet: + st := s.sched.Stats(r.Context()) + writeJSON(w, 200, map[string]any{ + "scheduler": st, + "scheduler_config": map[string]any{"global_concurrency": s.cfg.Scheduler.GlobalConcurrency, "max_queue": s.cfg.Scheduler.MaxQueue, "max_queue_per_actor": s.cfg.Scheduler.MaxQueuePerActor, "queue_timeout": s.cfg.Scheduler.QueueTimeout.Value().String(), "mode": "in-memory", "routing": s.cfg.Routing, "model_capabilities": s.cfg.ModelCapabilities}, + "workers": s.workers.Snapshots(), "usage": s.usage.Global(), "tenants": s.usage.LocalTenants(), "uptime_seconds": time.Since(s.startedAt).Seconds(), + "storage": map[string]any{"mode": "local-persistent", "data_dir": s.cfg.Storage.DataDir, "usage_journal": s.cfg.Usage.JournalDir}, + "model_aliases": s.aliasSnapshot(), "model_access": s.modelAccessSnapshot(), "reliability": s.cfg.Reliability, + "service_classes": s.cfg.ServiceClasses, "auto_tuning": s.cfg.AutoTuning, + "warm_models": func() any { + if s.warm != nil { + return s.warm.Status() + } + return map[string]any{"enabled": false} + }(), + "alerts": func() any { + if s.alerts != nil { + return s.alerts.Status() + } + return map[string]any{"enabled": false} + }(), + "opentelemetry": map[string]any{"enabled": s.cfg.OpenTelemetry.Enabled, "endpoint": s.cfg.OpenTelemetry.Endpoint, "service_name": s.cfg.OpenTelemetry.ServiceName, "sample_ratio": s.cfg.OpenTelemetry.SampleRatio, "exported_spans": func() uint64 { + if s.otel != nil { + return s.otel.Exported() + } + return 0 + }(), "failed_spans": func() uint64 { + if s.otel != nil { + return s.otel.Failed() + } + return 0 + }(), "dropped_spans": func() uint64 { + if s.otel != nil { + return s.otel.Dropped() + } + return 0 + }()}, + }) + case r.URL.Path == "/gateway/ui-api/recent" && r.Method == http.MethodGet: + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + if limit <= 0 || limit > s.cfg.UI.RecentEvents { + limit = min(1000, s.cfg.UI.RecentEvents) + } + writeJSON(w, 200, map[string]any{"events": s.usage.Recent(limit), "scope": "persistent-journal"}) + case r.URL.Path == "/gateway/ui-api/usage/rollups" && r.Method == http.MethodGet: + granularity := r.URL.Query().Get("granularity") + if granularity == "" { + granularity = "daily" + } + if granularity != "daily" && granularity != "monthly" { + proxy.WriteJSONError(w, 400, "bad_granularity", "granularity must be daily or monthly") + return + } + dimension := r.URL.Query().Get("dimension") + if dimension == "" { + dimension = "global" + } + switch dimension { + case "global", "tenant", "actor", "application", "model", "worker": + default: + proxy.WriteJSONError(w, 400, "bad_dimension", "dimension must be global, tenant, actor, application, model, or worker") + return + } + name := r.URL.Query().Get("name") + if dimension != "global" && name == "" { + proxy.WriteJSONError(w, 400, "missing_name", "name is required for the selected dimension") + return + } + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + writeJSON(w, 200, map[string]any{"granularity": granularity, "dimension": dimension, "name": name, "points": s.usage.Series(granularity, dimension, name, limit), "retention": s.usage.RetentionStatus()}) + case r.URL.Path == "/gateway/ui-api/autotune" && r.Method == http.MethodGet: + s.uiAutoTuneGet(w, r) + case r.URL.Path == "/gateway/ui-api/autotune/start" && r.Method == http.MethodPost: + s.uiAutoTuneStart(w, r, id) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/autotune/"): + s.uiAutoTuneAction(w, r, id) + case r.URL.Path == "/gateway/ui-api/warm-models" && r.Method == http.MethodGet: + s.uiWarmModelsGet(w, r) + case r.URL.Path == "/gateway/ui-api/warm-models" && r.Method == http.MethodPut: + s.uiWarmModelsSave(w, r, id) + case r.URL.Path == "/gateway/ui-api/warm-models" && r.Method == http.MethodDelete: + s.uiWarmModelsReset(w, r, id) + case r.URL.Path == "/gateway/ui-api/warm-models/reconcile" && r.Method == http.MethodPost: + s.uiWarmModelsReconcile(w, r, id) + case r.URL.Path == "/gateway/ui-api/alerts" && r.Method == http.MethodGet: + s.uiAlertsGet(w, r) + case r.URL.Path == "/gateway/ui-api/alerts/test" && r.Method == http.MethodPost: + s.uiAlertsTest(w, r, id) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/workers/"): + s.uiWorkerRuntime(w, r, id) + case r.URL.Path == "/gateway/ui-api/models" && r.Method == http.MethodGet: + writeJSON(w, 200, map[string]any{"inventories": s.workers.Inventories(r.Context())}) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/model-aliases"): + s.uiModelAliases(w, r, id) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/model-access"): + s.uiModelAccess(w, r, id) + case r.URL.Path == "/gateway/ui-api/placement" && r.Method == http.MethodGet: + s.uiPlacementGet(w, r) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/placement/"): + s.uiPlacementRule(w, r, id) + case r.URL.Path == "/gateway/ui-api/models/pull" && r.Method == http.MethodPost: + s.uiModelPull(w, r) + case r.URL.Path == "/gateway/ui-api/models/action" && r.Method == http.MethodPost: + s.uiModelAction(w, r) + case r.URL.Path == "/gateway/ui-api/jobs" && r.Method == http.MethodGet: + s.uiJobs(w, r) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/batches"): + s.uiBatchJobs(w, r, id) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/jobs/") && strings.HasSuffix(r.URL.Path, "/cancel") && r.Method == http.MethodPost: + jobID := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/gateway/ui-api/jobs/"), "/cancel") + if jobID == "" || !s.jobs.cancelJob(jobID) { + proxy.WriteJSONError(w, 409, "not_running", "job is not running") + return + } + s.log.Info("job cancellation requested", "request_id", jobID, "admin_subject", id.Subject, "admin_auth_type", id.AuthType) + writeJSON(w, 202, map[string]any{"status": "cancelling", "id": jobID}) + case r.URL.Path == "/gateway/ui-api/operations" && r.Method == http.MethodGet: + writeJSON(w, 200, map[string]any{"operations": s.ops.list()}) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/operations/") && strings.HasSuffix(r.URL.Path, "/cancel") && r.Method == http.MethodPost: + id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/gateway/ui-api/operations/"), "/cancel") + if !s.ops.cancelOperation(id) { + proxy.WriteJSONError(w, 409, "not_running", "operation is not running") + return + } + writeJSON(w, 202, map[string]any{"status": "cancelling"}) + case r.URL.Path == "/gateway/ui-api/policies" && r.Method == http.MethodGet: + overrides, err := s.policies.List(r.Context()) + if err != nil { + proxy.WriteJSONError(w, 503, "policy_store", err.Error()) + return + } + writeJSON(w, 200, map[string]any{"baseline": s.cfg.Scheduler.Policies, "overrides": overrides}) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/policies/"): + tenant, err := url.PathUnescape(strings.TrimPrefix(r.URL.Path, "/gateway/ui-api/policies/")) + if err != nil || tenant == "" || len(tenant) > 256 { + proxy.WriteJSONError(w, 400, "bad_tenant", "invalid tenant") + return + } + s.uiPolicy(w, r, tenant) + case r.URL.Path == "/gateway/ui-api/api-keys" && r.Method == http.MethodGet: + storage := "in-memory" + if s.auth.HasPersistentRuntimeStore() { + storage = "persistent-file" + } + writeJSON(w, 200, map[string]any{"keys": s.auth.APIKeys(), "storage": storage}) + case r.URL.Path == "/gateway/ui-api/api-keys" && r.Method == http.MethodPost: + s.uiAPIKeyCreate(w, r, id) + case strings.HasPrefix(r.URL.Path, "/gateway/ui-api/api-keys/") && r.Method == http.MethodDelete: + s.uiAPIKeyDelete(w, r, id) + case r.URL.Path == "/gateway/ui-api/config" && r.Method == http.MethodGet: + writeJSON(w, 200, s.redactedConfig()) + case r.URL.Path == "/gateway/ui-api/config" && r.Method == http.MethodPut: + s.uiConfigSave(w, r, id) + case r.URL.Path == "/gateway/ui-api/config" && r.Method == http.MethodDelete: + s.uiConfigReset(w, r, id) + case r.URL.Path == "/gateway/ui-api/storage" && r.Method == http.MethodGet: + writeJSON(w, 200, s.storageStatus()) + case r.URL.Path == "/gateway/ui-api/storage/flush" && r.Method == http.MethodPost: + s.uiStorageFlush(w, r) + case r.URL.Path == "/gateway/ui-api/storage/compact" && r.Method == http.MethodPost: + s.uiStorageCompact(w, r) + case r.URL.Path == "/gateway/ui-api/storage/backup" && r.Method == http.MethodGet: + s.uiStorageBackup(w, r) + default: + proxy.WriteJSONError(w, 404, "not_found", "unknown UI endpoint") + } +} + +func (s *Server) uiWarmModelsGet(w http.ResponseWriter, r *http.Request) { + if s.warm == nil { + writeJSON(w, 200, map[string]any{"enabled": false, "policies": map[string]any{}, "baseline": map[string]any{}, "actions": []any{}, "eviction_suggestions": []any{}}) + return + } + writeJSON(w, 200, s.warm.Status()) +} + +func (s *Server) uiWarmModelsSave(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if s.warm == nil { + proxy.WriteJSONError(w, 503, "warm_models_unavailable", "warm model manager is unavailable") + return + } + var in struct { + Policies map[string]config.WarmModelPolicy `json:"policies"` + } + if err := decodeJSON(r, &in, 256<<10); err != nil { + proxy.WriteJSONError(w, 400, "bad_warm_models", err.Error()) + return + } + if in.Policies == nil { + in.Policies = map[string]config.WarmModelPolicy{} + } + if err := s.warm.SetPolicies(in.Policies); err != nil { + proxy.WriteJSONError(w, 400, "bad_warm_models", err.Error()) + return + } + s.log.Info("warm model policies saved", "rules", len(in.Policies), "admin_subject", actor.Subject) + writeJSON(w, 200, s.warm.Status()) +} + +func (s *Server) uiWarmModelsReset(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if s.warm == nil { + proxy.WriteJSONError(w, 503, "warm_models_unavailable", "warm model manager is unavailable") + return + } + if err := s.warm.Reset(); err != nil { + proxy.WriteJSONError(w, 503, "warm_models_store", err.Error()) + return + } + s.log.Info("warm model policies reset", "admin_subject", actor.Subject) + writeJSON(w, 200, s.warm.Status()) +} + +func (s *Server) uiWarmModelsReconcile(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if s.warm == nil { + proxy.WriteJSONError(w, 503, "warm_models_unavailable", "warm model manager is unavailable") + return + } + s.warm.Reconcile(context.Background()) + s.log.Info("warm model reconciliation requested", "admin_subject", actor.Subject) + writeJSON(w, 202, s.warm.Status()) +} + +func (s *Server) uiAlertsGet(w http.ResponseWriter, r *http.Request) { + if s.alerts == nil { + writeJSON(w, 200, map[string]any{"enabled": false, "active": []any{}, "history": []any{}, "deliveries": []any{}}) + return + } + writeJSON(w, 200, s.alerts.Status()) +} + +func (s *Server) uiAlertsTest(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if s.alerts == nil { + proxy.WriteJSONError(w, 503, "alerts_unavailable", "alerts manager is unavailable") + return + } + var in struct { + Name string `json:"name"` + } + if err := decodeJSON(r, &in, 16<<10); err != nil { + proxy.WriteJSONError(w, 400, "bad_webhook", err.Error()) + return + } + if err := s.alerts.TestWebhook(strings.TrimSpace(in.Name)); err != nil { + proxy.WriteJSONError(w, 502, "webhook_test", err.Error()) + return + } + s.log.Info("alert webhook test sent", "webhook", in.Name, "admin_subject", actor.Subject) + writeJSON(w, 200, map[string]any{"sent": true}) +} + +func (s *Server) uiAutoTuneGet(w http.ResponseWriter, r *http.Request) { + if s.autoTune == nil { + writeJSON(w, 200, map[string]any{"enabled": false, "profiles": []any{}, "applied": map[string]any{}, "config": s.cfg.AutoTuning}) + return + } + writeJSON(w, 200, map[string]any{"enabled": s.cfg.AutoTuning.Enabled, "profiles": s.autoTune.List(), "applied": s.autoTune.Applied(), "config": s.cfg.AutoTuning}) +} + +func (s *Server) uiAutoTuneStart(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if s.autoTune == nil || !s.cfg.AutoTuning.Enabled { + proxy.WriteJSONError(w, 409, "autotune_disabled", "auto tuning is disabled") + return + } + var in autotune.StartRequest + if err := decodeJSON(r, &in, 64<<10); err != nil { + proxy.WriteJSONError(w, 400, "bad_autotune", err.Error()) + return + } + p, err := s.autoTune.Start(context.Background(), in) + if err != nil { + proxy.WriteJSONError(w, 400, "bad_autotune", err.Error()) + return + } + s.log.Info("auto-tune benchmark started", "profile_id", p.ID, "worker", p.Worker, "model", p.Model, "admin_subject", actor.Subject) + writeJSON(w, http.StatusAccepted, map[string]any{"profile": p}) +} + +func (s *Server) uiAutoTuneAction(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if s.autoTune == nil { + proxy.WriteJSONError(w, 404, "autotune_unavailable", "auto tuning is unavailable") + return + } + rest := strings.Trim(strings.TrimPrefix(r.URL.Path, "/gateway/ui-api/autotune/"), "/") + if rest == "reset" && r.Method == http.MethodPost { + var in struct { + Worker string `json:"worker"` + Model string `json:"model"` + } + if err := decodeJSON(r, &in, 16<<10); err != nil || strings.TrimSpace(in.Worker) == "" || strings.TrimSpace(in.Model) == "" { + proxy.WriteJSONError(w, 400, "bad_autotune_reset", "worker and model are required") + return + } + if err := s.autoTune.Reset(in.Worker, in.Model); err != nil { + proxy.WriteJSONError(w, 503, "autotune_store", err.Error()) + return + } + if err := s.workers.ResetModelConcurrency(in.Worker, in.Model); err != nil { + proxy.WriteJSONError(w, 400, "worker_config", err.Error()) + return + } + s.log.Info("auto-tune override reset", "worker", in.Worker, "model", in.Model, "admin_subject", actor.Subject) + writeJSON(w, 200, map[string]any{"status": "reset", "worker": in.Worker, "model": in.Model}) + return + } + parts := strings.Split(rest, "/") + if len(parts) != 2 || parts[0] == "" { + proxy.WriteJSONError(w, 404, "not_found", "unknown auto-tune action") + return + } + profileID, action := parts[0], parts[1] + switch { + case action == "cancel" && r.Method == http.MethodPost: + if !s.autoTune.Cancel(profileID) { + proxy.WriteJSONError(w, 409, "not_running", "benchmark is not running") + return + } + s.log.Info("auto-tune benchmark cancellation requested", "profile_id", profileID, "admin_subject", actor.Subject) + writeJSON(w, http.StatusAccepted, map[string]any{"status": "cancelling", "id": profileID}) + case action == "apply" && r.Method == http.MethodPost: + p, err := s.autoTune.Apply(profileID) + if err != nil { + proxy.WriteJSONError(w, 409, "autotune_apply", err.Error()) + return + } + if err := s.workers.SetModelConcurrency(p.Worker, p.Model, p.RecommendedConcurrency); err != nil { + _ = s.autoTune.Reset(p.Worker, p.Model) + proxy.WriteJSONError(w, 400, "worker_config", err.Error()) + return + } + s.log.Info("auto-tune recommendation applied", "profile_id", profileID, "worker", p.Worker, "model", p.Model, "concurrency", p.RecommendedConcurrency, "admin_subject", actor.Subject) + writeJSON(w, 200, map[string]any{"profile": p}) + default: + proxy.WriteJSONError(w, 404, "not_found", "unknown auto-tune action") + } +} + +func (s *Server) uiWorkerRuntime(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + rest := strings.TrimPrefix(r.URL.Path, "/gateway/ui-api/workers/") + parts := strings.Split(strings.Trim(rest, "/"), "/") + if len(parts) < 2 { + proxy.WriteJSONError(w, 404, "not_found", "unknown worker action") + return + } + name, err := url.PathUnescape(parts[0]) + if err != nil || name == "" { + proxy.WriteJSONError(w, 400, "bad_worker", "invalid worker") + return + } + if _, ok := s.workers.URLFor(name); !ok { + proxy.WriteJSONError(w, 404, "worker_not_found", "worker not found") + return + } + switch { + case len(parts) == 2 && parts[1] == "maintenance" && r.Method == http.MethodPost: + var in struct { + Mode string `json:"mode"` + } + if err := decodeJSON(r, &in, 16<<10); err != nil { + proxy.WriteJSONError(w, 400, "bad_worker_mode", err.Error()) + return + } + if err := s.workers.SetMaintenance(name, in.Mode); err != nil { + proxy.WriteJSONError(w, 400, "bad_worker_mode", err.Error()) + return + } + if s.workerStateStore != nil { + if strings.EqualFold(strings.TrimSpace(in.Mode), "active") { + err = s.workerStateStore.Delete(r.Context(), name) + } else { + err = s.workerStateStore.Put(r.Context(), name, strings.ToLower(strings.TrimSpace(in.Mode))) + } + if err != nil { + proxy.WriteJSONError(w, 503, "worker_state_store", err.Error()) + return + } + } + s.log.Info("worker maintenance changed", "worker", name, "mode", in.Mode, "admin_subject", actor.Subject) + if s.warm != nil { + s.warm.Wake() + } + writeJSON(w, 200, map[string]any{"worker": name, "mode": in.Mode}) + case len(parts) == 3 && parts[1] == "circuit" && parts[2] == "reset" && r.Method == http.MethodPost: + if err := s.workers.CircuitReset(name); err != nil { + proxy.WriteJSONError(w, 400, "circuit_reset", err.Error()) + return + } + s.log.Info("worker circuit reset", "worker", name, "admin_subject", actor.Subject) + if s.warm != nil { + s.warm.Wake() + } + writeJSON(w, 200, map[string]any{"worker": name, "circuit_state": "closed"}) + default: + proxy.WriteJSONError(w, 404, "not_found", "unknown worker action") + } +} + +func (s *Server) uiLiveStream(w http.ResponseWriter, r *http.Request) { + flusher, ok := w.(http.Flusher) + if !ok { + proxy.WriteJSONError(w, 500, "streaming_unavailable", "response writer does not support streaming") + return + } + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache, no-store") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("X-Accel-Buffering", "no") + w.Header().Set("X-Content-Type-Options", "nosniff") + + send := func() error { + b, err := json.Marshal(s.live.Snapshot()) + if err != nil { + return err + } + if _, err := fmt.Fprintf(w, "event: snapshot\ndata: %s\n\n", b); err != nil { + return err + } + flusher.Flush() + return nil + } + changed := s.live.Changed() + if err := send(); err != nil { + return + } + // Coalesce bursts of queue/progress changes. The browser animation runs at + // display refresh rate, so sending more than four metadata snapshots per + // second adds network/JSON work without making the pulse map smoother. + flushTicker := time.NewTicker(250 * time.Millisecond) + heartbeat := time.NewTicker(15 * time.Second) + defer flushTicker.Stop() + defer heartbeat.Stop() + dirty := false + for { + select { + case <-r.Context().Done(): + return + case <-changed: + changed = s.live.Changed() + dirty = true + case <-flushTicker.C: + if dirty { + if err := send(); err != nil { + return + } + dirty = false + } + case <-heartbeat.C: + if _, err := fmt.Fprint(w, ": heartbeat\n\n"); err != nil { + return + } + flusher.Flush() + } + } +} + +func (s *Server) uiInfrastructureStream(w http.ResponseWriter, r *http.Request) { + if s.infrastructure == nil { + proxy.WriteJSONError(w, 503, "infrastructure_unavailable", "infrastructure live view is unavailable") + return + } + flusher, ok := w.(http.Flusher) + if !ok { + proxy.WriteJSONError(w, 500, "streaming_unavailable", "response writer does not support streaming") + return + } + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache, no-store") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("X-Accel-Buffering", "no") + w.Header().Set("X-Content-Type-Options", "nosniff") + send := func() error { + b, err := json.Marshal(s.infrastructure.Snapshot()) + if err != nil { + return err + } + if _, err := fmt.Fprintf(w, "event: snapshot\ndata: %s\n\n", b); err != nil { + return err + } + flusher.Flush() + return nil + } + changed := s.infrastructure.Changed() + if err := send(); err != nil { + return + } + flushTicker := time.NewTicker(250 * time.Millisecond) + heartbeat := time.NewTicker(15 * time.Second) + defer flushTicker.Stop() + defer heartbeat.Stop() + dirty := false + for { + select { + case <-r.Context().Done(): + return + case <-changed: + changed = s.infrastructure.Changed() + dirty = true + case <-flushTicker.C: + if dirty { + if err := send(); err != nil { + return + } + dirty = false + } + case <-heartbeat.C: + if _, err := fmt.Fprint(w, ": heartbeat\n\n"); err != nil { + return + } + flusher.Flush() + } + } +} + +func (s *Server) uiAPIKeyCreate(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + var in struct { + Name string `json:"name"` + Tenant string `json:"tenant"` + Subject string `json:"subject"` + Application string `json:"application"` + Scopes []string `json:"scopes"` + AllowedModels []string `json:"allowed_models"` + DeniedModels []string `json:"denied_models"` + ServiceClass string `json:"service_class"` + } + if err := decodeJSON(r, &in, 64<<10); err != nil { + proxy.WriteJSONError(w, 400, "bad_api_key", err.Error()) + return + } + if cls := strings.TrimSpace(in.ServiceClass); cls != "" { + if _, ok := s.cfg.ServiceClasses.Classes[cls]; !ok { + proxy.WriteJSONError(w, 400, "bad_service_class", "unknown service class") + return + } + } + info, secret, err := s.auth.CreateAPIKey(auth.APIKeyCreate{Name: in.Name, Tenant: in.Tenant, Subject: in.Subject, Application: in.Application, Scopes: in.Scopes, AllowedModels: in.AllowedModels, DeniedModels: in.DeniedModels, ServiceClass: in.ServiceClass}) + if err != nil { + proxy.WriteJSONError(w, 400, "bad_api_key", err.Error()) + return + } + s.log.Info("API key created", "key_id", info.ID, "key_name", info.Name, "tenant", info.Tenant, "admin_subject", actor.Subject, "admin_auth_type", actor.AuthType) + writeJSON(w, http.StatusCreated, map[string]any{"key": info, "secret": secret, "warning": "This secret is shown only once. Only its SHA-256 hash is stored persistently."}) +} + +func (s *Server) uiAPIKeyDelete(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + raw := strings.TrimPrefix(r.URL.Path, "/gateway/ui-api/api-keys/") + id, err := url.PathUnescape(raw) + if err != nil || id == "" || strings.Contains(id, "/") || len(id) > 128 { + proxy.WriteJSONError(w, 400, "bad_api_key_id", "invalid API key id") + return + } + info, ok, err := s.auth.DeleteAPIKey(id) + if err != nil { + proxy.WriteJSONError(w, 503, "api_key_store", err.Error()) + return + } + if !ok { + proxy.WriteJSONError(w, 404, "api_key_not_found", "persistent API key not found") + return + } + s.log.Info("API key deleted", "key_id", info.ID, "key_name", info.Name, "tenant", info.Tenant, "admin_subject", actor.Subject, "admin_auth_type", actor.AuthType) + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) uiModelPull(w http.ResponseWriter, r *http.Request) { + var in struct{ Worker, Model string } + if err := decodeJSON(r, &in, 64<<10); err != nil || in.Worker == "" || in.Model == "" { + proxy.WriteJSONError(w, 400, "bad_request", "worker and model are required") + return + } + u, ok := s.workers.URLFor(in.Worker) + if !ok { + proxy.WriteJSONError(w, 404, "worker_not_found", "unknown worker") + return + } + op := s.ops.startPull(u, in.Worker, in.Model) + writeJSON(w, 202, op) +} +func (s *Server) uiModelAction(w http.ResponseWriter, r *http.Request) { + var in struct{ Action, Worker, Model string } + if err := decodeJSON(r, &in, 64<<10); err != nil || in.Worker == "" || in.Model == "" { + proxy.WriteJSONError(w, 400, "bad_request", "action, worker and model are required") + return + } + if in.Action != "stop" && in.Action != "delete" { + proxy.WriteJSONError(w, 400, "bad_action", "supported actions are stop and delete") + return + } + u, ok := s.workers.URLFor(in.Worker) + if !ok { + proxy.WriteJSONError(w, 404, "worker_not_found", "unknown worker") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute) + defer cancel() + if err := s.ops.modelAction(ctx, u, in.Action, in.Model); err != nil { + proxy.WriteJSONError(w, 502, "ollama_error", err.Error()) + return + } + writeJSON(w, 200, map[string]any{"status": "ok"}) +} + +func (s *Server) uiPlacementGet(w http.ResponseWriter, r *http.Request) { + placements := s.workers.PlacementSnapshots() + installed := make(map[string]map[string]bool, len(placements)) + loaded := make(map[string]map[string]bool, len(placements)) + modelSet := map[string]bool{} + invErrors := map[string]string{} + for _, p := range placements { + iset := map[string]bool{} + for _, model := range p.InstalledModels { + iset[model] = true + modelSet[model] = true + } + installed[p.Worker] = iset + lset := map[string]bool{} + for _, model := range p.LoadedModels { + lset[model] = true + modelSet[model] = true + } + loaded[p.Worker] = lset + if p.InventoryError != "" { + invErrors[p.Worker] = p.InventoryError + } + } + models := make([]string, 0, len(modelSet)) + for model := range modelSet { + models = append(models, model) + } + sort.Strings(models) + rows := make([]map[string]any, 0, len(models)) + for _, model := range models { + cells := map[string]any{} + for _, p := range placements { + decision, _ := s.workers.PlacementDecision(p.Worker, model) + cells[p.Worker] = map[string]any{ + "allowed": decision.Allowed, + "source": decision.Source, + "pattern": decision.Pattern, + "exact_override": decision.ExactOverride, + "installed": installed[p.Worker][model], + "loaded": loaded[p.Worker][model], + } + } + rows = append(rows, map[string]any{"model": model, "workers": cells}) + } + writeJSON(w, 200, map[string]any{"workers": placements, "models": rows, "inventory_errors": invErrors, "semantics": map[string]any{"rule_order": "most-specific-match; deny wins ties", "fallback_allow_all": true, "prefix_wildcard": "trailing * only"}}) +} + +func (s *Server) uiPlacementRule(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if s.placementStore == nil { + proxy.WriteJSONError(w, 503, "placement_store", "persistent model placement store unavailable") + return + } + rest := strings.TrimPrefix(r.URL.Path, "/gateway/ui-api/placement/") + isModelAction := strings.HasSuffix(rest, "/model") + if isModelAction { + rest = strings.TrimSuffix(rest, "/model") + } + workerName, err := url.PathUnescape(strings.Trim(rest, "/")) + if err != nil || workerName == "" || strings.Contains(workerName, "/") || len(workerName) > 256 { + proxy.WriteJSONError(w, 400, "bad_worker", "invalid worker name") + return + } + if _, ok := s.workers.URLFor(workerName); !ok { + proxy.WriteJSONError(w, 404, "worker_not_found", "unknown worker") + return + } + if isModelAction { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + s.uiPlacementModelAction(w, r, actor, workerName) + return + } + switch r.Method { + case http.MethodPut: + var rule config.ModelPlacementRule + if err := decodeJSON(r, &rule, 128<<10); err != nil { + proxy.WriteJSONError(w, 400, "bad_placement", err.Error()) + return + } + rule = normalizePlacementForUI(rule) + if err := config.ValidateModelPlacementRule(rule); err != nil { + proxy.WriteJSONError(w, 400, "bad_placement", err.Error()) + return + } + if err := s.placementStore.Put(r.Context(), workerName, rule); err != nil { + proxy.WriteJSONError(w, 503, "placement_store", err.Error()) + return + } + if err := s.workers.SetPlacement(workerName, rule, true); err != nil { + proxy.WriteJSONError(w, 500, "placement_runtime", err.Error()) + return + } + s.log.Info("model placement override saved", "worker", workerName, "mode", rule.Mode, "allowed", len(rule.AllowedModels), "denied", len(rule.DeniedModels), "admin_subject", actor.Subject) + if s.warm != nil { + s.warm.Wake() + } + writeJSON(w, 200, map[string]any{"worker": workerName, "rule": rule, "override": true, "applied": true}) + case http.MethodDelete: + if err := s.placementStore.Delete(r.Context(), workerName); err != nil { + proxy.WriteJSONError(w, 503, "placement_store", err.Error()) + return + } + if err := s.workers.ResetPlacement(workerName); err != nil { + proxy.WriteJSONError(w, 500, "placement_runtime", err.Error()) + return + } + s.log.Info("model placement override reset", "worker", workerName, "admin_subject", actor.Subject) + if s.warm != nil { + s.warm.Wake() + } + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func (s *Server) uiPlacementModelAction(w http.ResponseWriter, r *http.Request, actor auth.Identity, workerName string) { + var in struct { + Model string `json:"model"` + Action string `json:"action"` // allow | deny | inherit + } + if err := decodeJSON(r, &in, 32<<10); err != nil { + proxy.WriteJSONError(w, 400, "bad_placement_action", err.Error()) + return + } + in.Model = strings.TrimSpace(in.Model) + if in.Model == "" || strings.Contains(in.Model, "*") || len(in.Model) > 512 { + proxy.WriteJSONError(w, 400, "bad_model", "model must be a non-empty exact model name") + return + } + if in.Action != "allow" && in.Action != "deny" && in.Action != "inherit" { + proxy.WriteJSONError(w, 400, "bad_action", "action must be allow, deny, or inherit") + return + } + var baseline config.ModelPlacementRule + var effective config.ModelPlacementRule + for _, p := range s.workers.PlacementSnapshots() { + if p.Worker == workerName { + baseline, effective = p.Baseline, p.Effective + break + } + } + effective = normalizePlacementForUI(effective) + effective.AllowedModels = removePlacementExact(effective.AllowedModels, in.Model) + effective.DeniedModels = removePlacementExact(effective.DeniedModels, in.Model) + switch in.Action { + case "allow": + effective.AllowedModels = append(effective.AllowedModels, in.Model) + case "deny": + effective.DeniedModels = append(effective.DeniedModels, in.Model) + } + effective = normalizePlacementForUI(effective) + baseline = normalizePlacementForUI(baseline) + if placementRulesEqual(effective, baseline) { + if err := s.placementStore.Delete(r.Context(), workerName); err != nil { + proxy.WriteJSONError(w, 503, "placement_store", err.Error()) + return + } + if err := s.workers.ResetPlacement(workerName); err != nil { + proxy.WriteJSONError(w, 500, "placement_runtime", err.Error()) + return + } + } else { + if err := s.placementStore.Put(r.Context(), workerName, effective); err != nil { + proxy.WriteJSONError(w, 503, "placement_store", err.Error()) + return + } + if err := s.workers.SetPlacement(workerName, effective, true); err != nil { + proxy.WriteJSONError(w, 500, "placement_runtime", err.Error()) + return + } + } + s.log.Info("model placement exact rule changed", "worker", workerName, "model", in.Model, "action", in.Action, "admin_subject", actor.Subject) + if s.warm != nil { + s.warm.Wake() + } + decision, _ := s.workers.PlacementDecision(workerName, in.Model) + writeJSON(w, 200, map[string]any{"worker": workerName, "model": in.Model, "action": in.Action, "decision": decision, "rule": effective}) +} + +func normalizePlacementForUI(r config.ModelPlacementRule) config.ModelPlacementRule { + if strings.TrimSpace(r.Mode) == "" { + r.Mode = "allow_all" + } + r.Mode = strings.TrimSpace(r.Mode) + norm := func(in []string) []string { + out := make([]string, 0, len(in)) + seen := map[string]bool{} + for _, x := range in { + x = strings.TrimSpace(x) + if x == "" || seen[x] { + continue + } + seen[x] = true + out = append(out, x) + } + sort.Strings(out) + return out + } + r.AllowedModels = norm(r.AllowedModels) + r.DeniedModels = norm(r.DeniedModels) + return r +} + +func removePlacementExact(in []string, model string) []string { + out := in[:0] + for _, x := range in { + if strings.TrimSpace(x) != model { + out = append(out, x) + } + } + return out +} + +func placementRulesEqual(a, b config.ModelPlacementRule) bool { + a, b = normalizePlacementForUI(a), normalizePlacementForUI(b) + if a.Mode != b.Mode || len(a.AllowedModels) != len(b.AllowedModels) || len(a.DeniedModels) != len(b.DeniedModels) { + return false + } + for i := range a.AllowedModels { + if a.AllowedModels[i] != b.AllowedModels[i] { + return false + } + } + for i := range a.DeniedModels { + if a.DeniedModels[i] != b.DeniedModels[i] { + return false + } + } + return true +} +func (s *Server) uiJobs(w http.ResponseWriter, r *http.Request) { + entries := s.jobs.list() + byID := make(map[string]jobEntry, len(entries)) + for _, j := range entries { + byID[j.ID] = j + } + snap := s.live.Snapshot() + jobs := make([]jobView, 0, len(entries)) + for _, req := range snap.Requests { + j, ok := byID[req.ID] + if !ok { + continue + } + jobs = append(jobs, jobView{Request: req, Cancellable: true, Cancelling: j.Cancelling}) + } + writeJSON(w, http.StatusOK, map[string]any{"jobs": jobs, "scope": "in-memory-process"}) +} + +func (s *Server) uiPolicy(w http.ResponseWriter, r *http.Request, tenant string) { + switch r.Method { + case http.MethodPut: + var p config.TenantPolicy + if err := decodeJSON(r, &p, 64<<10); err != nil { + proxy.WriteJSONError(w, 400, "bad_policy", err.Error()) + return + } + if p.TenantWeight <= 0 || p.ActorWeight <= 0 || p.ActorCreditsPerMinute < 0 || p.ActorBurstCredits < 0 || p.TenantCreditsPerMinute < 0 || p.TenantBurstCredits < 0 { + proxy.WriteJSONError(w, 400, "bad_policy", "weights must be > 0 and credit values must be >= 0") + return + } + if err := s.policies.Put(r.Context(), tenant, p); err != nil { + proxy.WriteJSONError(w, 503, "policy_store", err.Error()) + return + } + writeJSON(w, 200, map[string]any{"tenant": tenant, "policy": p}) + case http.MethodDelete: + if err := s.policies.Delete(r.Context(), tenant); err != nil { + proxy.WriteJSONError(w, 503, "policy_store", err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func decodeJSON(r *http.Request, out any, maxBytes int64) error { + defer r.Body.Close() + dec := json.NewDecoder(io.LimitReader(r.Body, maxBytes)) + dec.DisallowUnknownFields() + return dec.Decode(out) +} + +func (s *Server) policyFor(ctx context.Context, tenant string) config.TenantPolicy { + if s.policies != nil { + if p, ok, err := s.policies.Get(ctx, tenant); err == nil && ok { + return s.normalizePolicy(p) + } + if _, exact := s.cfg.Scheduler.Policies[tenant]; !exact { + if p, ok, err := s.policies.Get(ctx, "*"); err == nil && ok { + return s.normalizePolicy(p) + } + } + } + return s.cfg.Policy(tenant) +} +func (s *Server) normalizePolicy(p config.TenantPolicy) config.TenantPolicy { + if p.TenantWeight <= 0 { + p.TenantWeight = s.cfg.Scheduler.DefaultTenantWeight + } + if p.ActorWeight <= 0 { + p.ActorWeight = s.cfg.Scheduler.DefaultActorWeight + } + return p +} + +func (s *Server) redactedConfig() map[string]any { + b, _ := json.Marshal(s.cfg) + var out map[string]any + _ = json.Unmarshal(b, &out) + out["model_aliases"] = s.aliasSnapshot() + out["model_access"] = s.modelAccessSnapshot() + if authMap, ok := out["auth"].(map[string]any); ok { + if keys, ok := authMap["api_keys"].([]any); ok { + for _, raw := range keys { + if key, ok := raw.(map[string]any); ok && key["key"] != nil { + key["key"] = "" + } + } + } + } + if uiMap, ok := out["ui"].(map[string]any); ok { + if _, ok := uiMap["session_secret"]; ok { + uiMap["session_secret"] = "" + } + if oidc, ok := uiMap["oidc"].(map[string]any); ok { + if _, ok := oidc["client_secret"]; ok { + oidc["client_secret"] = "" + } + } + } + if alertsMap, ok := out["alerts"].(map[string]any); ok { + if hooks, ok := alertsMap["webhooks"].([]any); ok { + for _, raw := range hooks { + if hook, ok := raw.(map[string]any); ok { + if secret, exists := hook["secret"]; exists && strings.TrimSpace(fmt.Sprint(secret)) != "" { + hook["secret"] = "" + } + } + } + } + } + if otelMap, ok := out["opentelemetry"].(map[string]any); ok { + if headers, ok := otelMap["headers"].(map[string]any); ok { + for k := range headers { + headers[k] = "" + } + } + } + if convMap, ok := out["conversations"].(map[string]any); ok { + if secret, exists := convMap["encryption_key"]; exists && strings.TrimSpace(fmt.Sprint(secret)) != "" { + convMap["encryption_key"] = "" + } + } + return out +} + +func (s *Server) restoreRedactedConfigSecrets(candidate *config.Config) { + if candidate == nil { + return + } + candidate.Auth.APIKeys = append([]config.APIKeyConfig(nil), s.cfg.Auth.APIKeys...) + if candidate.UI.SessionSecret == "" || candidate.UI.SessionSecret == "" { + candidate.UI.SessionSecret = s.cfg.UI.SessionSecret + } + if candidate.UI.OIDC.ClientSecret == "" || candidate.UI.OIDC.ClientSecret == "" { + candidate.UI.OIDC.ClientSecret = s.cfg.UI.OIDC.ClientSecret + } + if candidate.Conversations.EncryptionKey == "" || candidate.Conversations.EncryptionKey == "" { + candidate.Conversations.EncryptionKey = s.cfg.Conversations.EncryptionKey + } + for i := range candidate.Alerts.Webhooks { + if candidate.Alerts.Webhooks[i].Secret != "" && candidate.Alerts.Webhooks[i].Secret != "" { + continue + } + name := candidate.Alerts.Webhooks[i].Name + if i < len(s.cfg.Alerts.Webhooks) && s.cfg.Alerts.Webhooks[i].Name == name { + candidate.Alerts.Webhooks[i].Secret = s.cfg.Alerts.Webhooks[i].Secret + continue + } + for _, old := range s.cfg.Alerts.Webhooks { + if old.Name == name { + candidate.Alerts.Webhooks[i].Secret = old.Secret + break + } + } + } + if candidate.OpenTelemetry.Headers == nil { + candidate.OpenTelemetry.Headers = map[string]string{} + } + for k, v := range candidate.OpenTelemetry.Headers { + if v == "" { + if old, ok := s.cfg.OpenTelemetry.Headers[k]; ok { + candidate.OpenTelemetry.Headers[k] = old + } + } + } +} + +func (s *Server) uiConfigSave(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if s.configStore == nil { + proxy.WriteJSONError(w, 503, "config_store", "persistent configuration store unavailable") + return + } + var raw map[string]any + if err := decodeJSON(r, &raw, 4<<20); err != nil { + proxy.WriteJSONError(w, 400, "bad_config", err.Error()) + return + } + b, err := json.Marshal(raw) + if err != nil { + proxy.WriteJSONError(w, 400, "bad_config", err.Error()) + return + } + candidate, err := config.ParseBytes(b) + if err != nil { + // The UI never exposes plaintext bootstrap secrets. Reparse after + // restoring fields that are intentionally immutable through this editor. + var tmp config.Config + if json.Unmarshal(b, &tmp) == nil { + s.restoreRedactedConfigSecrets(&tmp) + b, _ = json.Marshal(tmp) + candidate, err = config.ParseBytes(b) + } + } + if err != nil { + proxy.WriteJSONError(w, 400, "bad_config", err.Error()) + return + } + // Secrets are never exposed by the JSON editor. Restore redacted values + // from the active bootstrap/effective config before persisting. + s.restoreRedactedConfigSecrets(candidate) + // Tests and programmatic embedders may construct Config without applying + // defaults; compare against a default-normalized bootstrap storage value. + expectedStorage := s.cfg.Storage + if expectedStorage.WorkerStateFile == "" { + expectedStorage.WorkerStateFile = "worker-state.json" + } + if expectedStorage.AutoTuneFile == "" { + expectedStorage.AutoTuneFile = "auto-tune.json" + } + if expectedStorage.WarmModelsFile == "" { + expectedStorage.WarmModelsFile = "warm-models.json" + } + if expectedStorage.AlertsFile == "" { + expectedStorage.AlertsFile = "alerts.json" + } + if expectedStorage.ConversationsFile == "" { + expectedStorage.ConversationsFile = "conversations.enc.json" + } + if expectedStorage.BatchJobsFile == "" { + expectedStorage.BatchJobsFile = "batch-jobs.json" + } + if expectedStorage.BatchJobsDir == "" { + expectedStorage.BatchJobsDir = "batch" + } + if candidate.Storage != expectedStorage { + proxy.WriteJSONError(w, 400, "storage_bootstrap_only", "storage settings are bootstrap-only; change them in the startup config") + return + } + if err := candidate.Validate(); err != nil { + proxy.WriteJSONError(w, 400, "bad_config", err.Error()) + return + } + if err := s.configStore.Save(candidate); err != nil { + proxy.WriteJSONError(w, 503, "config_store", err.Error()) + return + } + s.log.Info("persistent configuration saved", "path", s.configStore.Path(), "admin_subject", actor.Subject, "admin_auth_type", actor.AuthType) + writeJSON(w, 200, map[string]any{"saved": true, "path": s.configStore.Path(), "restart_required": true, "message": "Configuration persisted. Restart the gateway to activate all changes."}) +} + +func (s *Server) uiConfigReset(w http.ResponseWriter, r *http.Request, actor auth.Identity) { + if s.configStore == nil { + proxy.WriteJSONError(w, 503, "config_store", "persistent configuration store unavailable") + return + } + if err := s.configStore.Delete(); err != nil { + proxy.WriteJSONError(w, 503, "config_store", err.Error()) + return + } + s.log.Info("persistent configuration override deleted", "path", s.configStore.Path(), "admin_subject", actor.Subject, "admin_auth_type", actor.AuthType) + writeJSON(w, 200, map[string]any{"deleted": true, "restart_required": true, "message": "Persistent override removed. Restart to return to the bootstrap configuration."}) +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/internal/server/ui_test.go b/internal/server/ui_test.go new file mode 100644 index 0000000..a024c4f --- /dev/null +++ b/internal/server/ui_test.go @@ -0,0 +1,1099 @@ +package server + +import ( + "archive/zip" + "bufio" + "bytes" + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/batch" + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/cost" + "github.com/example/ollama-fair-gateway/internal/infrastructure" + "github.com/example/ollama-fair-gateway/internal/liveflow" + "github.com/example/ollama-fair-gateway/internal/metrics" + "github.com/example/ollama-fair-gateway/internal/policy" + px "github.com/example/ollama-fair-gateway/internal/proxy" + "github.com/example/ollama-fair-gateway/internal/quota" + "github.com/example/ollama-fair-gateway/internal/scheduler" + "github.com/example/ollama-fair-gateway/internal/session" + "github.com/example/ollama-fair-gateway/internal/state" + "github.com/example/ollama-fair-gateway/internal/usage" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +func newUITestServer(t *testing.T) (*Server, *session.Memory, func()) { + t.Helper() + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"models":[{"name":"qwen3:8b","model":"qwen3:8b","size":5000000000,"size_vram":4200000000,"context_length":32768}]}`) + case "/api/tags": + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"models":[{"name":"qwen3:8b","size":1234,"details":{"parameter_size":"8B","quantization_level":"Q4_K_M"}}]}`) + case "/api/chat": + // Simulate a long-running Ollama NDJSON stream. Cancelling the gateway + // job must close the upstream request context and end this handler. + w.Header().Set("Content-Type", "application/x-ndjson") + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + t := time.NewTicker(25 * time.Millisecond) + defer t.Stop() + for { + select { + case <-r.Context().Done(): + return + case <-t.C: + _, _ = io.WriteString(w, `{"message":{"content":"x"},"done":false}`+"\n") + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + } + } + default: + w.WriteHeader(http.StatusNotFound) + } + })) + cfg := &config.Config{ + Server: config.ServerConfig{MaxBodyBytes: 1 << 20, MaxRequestDuration: config.Duration(time.Minute)}, + Auth: config.AuthConfig{APIKeys: []config.APIKeyConfig{{Name: "admin", Key: "test-admin-key", Tenant: "ops", Subject: "admin", Scopes: []string{"gateway:admin"}}}}, + Scheduler: config.SchedulerConfig{GlobalConcurrency: 2, MaxQueue: 32, MaxQueuePerActor: 8, QueueTimeout: config.Duration(time.Second), DefaultTenantWeight: 1, DefaultActorWeight: 1, Policies: map[string]config.TenantPolicy{"*": {TenantWeight: 1, ActorWeight: 1, ActorCreditsPerMinute: 60, ActorBurstCredits: 120, TenantCreditsPerMinute: 300, TenantBurstCredits: 600}}, ComputePaths: []string{"/api/chat"}}, + Cost: config.CostConfig{Default: config.ModelRate{InputCreditsPer1K: 1, OutputCreditsPer1K: 3}, DefaultMaxOutputTokens: 16}, + Workers: []config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 2, HealthInterval: config.Duration(time.Hour), MemoryCapacityBytes: 256 << 30, VRAMCapacityBytes: 256 << 30}}, + Infrastructure: config.InfrastructureConfig{NodeName: "test-gateway", RefreshInterval: config.Duration(100 * time.Millisecond), MaxRequests: 100}, + UI: config.UIConfig{Enabled: true, Path: "/admin", Title: "Test Gateway", RecentEvents: 100}, + } + a, err := auth.New(context.Background(), cfg.Auth) + if err != nil { + backend.Close() + t.Fatal(err) + } + wp := worker.New(cfg.Workers, "w") + ctx, cancel := context.WithCancel(context.Background()) + wp.Start(ctx) + sched := scheduler.NewLocal(2, 32, 8) + live := liveflow.New(10*time.Second, 100) + infra := infrastructure.New(cfg.Infrastructure, live, sched, wp) + infra.Start(ctx) + rec, _ := usage.New("", 100, time.Second, nil) + sessions := session.NewMemory() + placementStore, err := state.NewModelPlacementStore(filepath.Join(t.TempDir(), "model-placement.json")) + if err != nil { + cancel() + backend.Close() + t.Fatal(err) + } + sv := New(cfg, Dependencies{Auth: a, Scheduler: sched, Quota: quota.Disabled{}, Estimator: cost.New(cfg.Cost), Workers: wp, Proxy: px.New(), Usage: rec, Metrics: metrics.New(), Live: live, Infrastructure: infra, Policies: policy.NewMemory(), Sessions: sessions, Logger: slog.Default(), PlacementStore: placementStore}) + return sv, sessions, func() { cancel(); backend.CloseClientConnections(); backend.Close() } +} + +func TestUIStaticAndAuthenticatedSession(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + resp, err := http.Get(front.URL + "/admin/") + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 || !strings.Contains(string(body), "Ollama Fair Gateway") || !strings.Contains(string(body), "Request Pulse Map") || !strings.Contains(string(body), "LLM Infrastructure Map") || !strings.Contains(string(body), "Model Placement Matrix") || !strings.Contains(string(body), "Durable Batch Jobs") { + t.Fatalf("static UI status=%d body=%s", resp.StatusCode, body) + } + if got := resp.Header.Get("Content-Security-Policy"); !strings.Contains(got, "frame-ancestors 'none'") { + t.Fatalf("missing CSP: %q", got) + } + + req, _ := http.NewRequest(http.MethodGet, front.URL+"/gateway/ui-api/session", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 || !strings.Contains(string(body), `"admin":true`) { + t.Fatalf("session status=%d body=%s", resp.StatusCode, body) + } +} + +func TestUICookieMutationRequiresCSRF(t *testing.T) { + sv, sessions, done := newUITestServer(t) + defer done() + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + id, err := sessions.Create(context.Background(), "test-admin-key", time.Hour) + if err != nil { + t.Fatal(err) + } + body := `{"tenant_weight":2,"actor_weight":1,"actor_credits_per_minute":100,"actor_burst_credits":200,"tenant_credits_per_minute":500,"tenant_burst_credits":1000}` + + req, _ := http.NewRequest(http.MethodPut, front.URL+"/gateway/ui-api/policies/team-a", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.AddCookie(&http.Cookie{Name: uiSessionCookie, Value: id}) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("without CSRF status=%d, want 403", resp.StatusCode) + } + + req, _ = http.NewRequest(http.MethodPut, front.URL+"/gateway/ui-api/policies/team-a", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-CSRF-Token", "csrf-test") + req.AddCookie(&http.Cookie{Name: uiSessionCookie, Value: id}) + req.AddCookie(&http.Cookie{Name: uiCSRFCookie, Value: "csrf-test"}) + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("with CSRF status=%d body=%s", resp.StatusCode, b) + } +} + +func TestUIRuntimeAPIKeyCreateListDelete(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + doAdmin := func(method, path, body string) (*http.Response, []byte) { + t.Helper() + req, _ := http.NewRequest(method, front.URL+path, strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer test-admin-key") + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return resp, b + } + + resp, b := doAdmin(http.MethodGet, "/gateway/ui-api/api-keys", "") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"source":"config"`) || strings.Contains(string(b), "test-admin-key") { + t.Fatalf("initial list status=%d body=%s", resp.StatusCode, b) + } + + resp, b = doAdmin(http.MethodPost, "/gateway/ui-api/api-keys", `{"name":"openwebui","tenant":"interactive","application":"openwebui","scopes":[]}`) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("create status=%d body=%s", resp.StatusCode, b) + } + var created struct { + Key struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"key"` + Secret string `json:"secret"` + } + if err := json.Unmarshal(b, &created); err != nil { + t.Fatal(err) + } + if created.Key.ID == "" || created.Key.Name != "openwebui" || !strings.HasPrefix(created.Secret, "ofg_") { + t.Fatalf("unexpected create response: %s", b) + } + + resp, b = doAdmin(http.MethodGet, "/gateway/ui-api/api-keys", "") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"source":"runtime"`) || strings.Contains(string(b), created.Secret) { + t.Fatalf("runtime list leaked or missing key status=%d body=%s", resp.StatusCode, b) + } + + req, _ := http.NewRequest(http.MethodGet, front.URL+"/api/tags", nil) + req.Header.Set("Authorization", "Bearer "+created.Secret) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + b, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), "qwen3:8b") { + t.Fatalf("runtime key auth status=%d body=%s", resp.StatusCode, b) + } + + resp, b = doAdmin(http.MethodDelete, "/gateway/ui-api/api-keys/"+created.Key.ID, "") + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("delete status=%d body=%s", resp.StatusCode, b) + } + + req, _ = http.NewRequest(http.MethodGet, front.URL+"/api/tags", nil) + req.Header.Set("Authorization", "Bearer "+created.Secret) + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("deleted key status=%d want 401", resp.StatusCode) + } +} + +func TestUIModelInventory(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + front := httptest.NewServer(sv.Handler()) + defer front.Close() + req, _ := http.NewRequest(http.MethodGet, front.URL+"/gateway/ui-api/models", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 || !strings.Contains(string(b), "qwen3:8b") { + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } +} + +func TestUILiveSnapshotAndStream(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + sv.live.Begin(liveflow.Request{ID: "live-1", Tenant: "team-a", Actor: "user-a", Model: "qwen3:8b", API: "ollama", Path: "/api/chat", EstimatedCredits: 2.5}) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + req, _ := http.NewRequest(http.MethodGet, front.URL+"/gateway/ui-api/live", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"id":"live-1"`) || !strings.Contains(string(b), `"state":"queued"`) { + t.Fatalf("snapshot status=%d body=%s", resp.StatusCode, b) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + req, _ = http.NewRequestWithContext(ctx, http.MethodGet, front.URL+"/gateway/ui-api/live/stream", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if got := resp.Header.Get("Content-Type"); !strings.Contains(got, "text/event-stream") { + t.Fatalf("content-type=%q", got) + } + reader := bufio.NewReader(resp.Body) + var block strings.Builder + deadline := time.After(2 * time.Second) + for !strings.Contains(block.String(), "\n\n") { + select { + case <-deadline: + t.Fatalf("timed out waiting for SSE block: %q", block.String()) + default: + line, err := reader.ReadString('\n') + if err != nil { + t.Fatal(err) + } + block.WriteString(line) + } + } + cancel() + if got := block.String(); !strings.Contains(got, "event: snapshot") || !strings.Contains(got, `"id":"live-1"`) { + t.Fatalf("unexpected SSE block: %s", got) + } +} + +func TestUIInfrastructureSnapshot(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + sv.live.Begin(liveflow.Request{ID: "infra-1", Tenant: "team-a", Actor: "app-a", Model: "qwen3:8b", Worker: "w", API: "ollama", Path: "/api/chat", EstimatedCredits: 1.5}) + time.Sleep(150 * time.Millisecond) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + req, _ := http.NewRequest(http.MethodGet, front.URL+"/gateway/ui-api/infrastructure", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"node_name":"test-gateway"`) || !strings.Contains(string(b), `"id":"infra-1"`) || !strings.Contains(string(b), `"size_vram":4200000000`) { + t.Fatalf("infrastructure status=%d body=%s", resp.StatusCode, b) + } +} + +func TestUIJobCancelAbortsRunningRequest(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + result := make(chan int, 1) + go func() { + req, _ := http.NewRequest(http.MethodPost, front.URL+"/api/chat", strings.NewReader(`{"model":"qwen3:8b","messages":[{"role":"user","content":"hello"}]}`)) + req.Header.Set("Authorization", "Bearer test-admin-key") + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + result <- 0 + return + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + result <- resp.StatusCode + }() + + var jobID string + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + req, _ := http.NewRequest(http.MethodGet, front.URL+"/gateway/ui-api/jobs", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + var body struct { + Jobs []struct { + ID string `json:"id"` + State string `json:"state"` + } `json:"jobs"` + } + _ = json.NewDecoder(resp.Body).Decode(&body) + resp.Body.Close() + if len(body.Jobs) > 0 { + jobID = body.Jobs[0].ID + if body.Jobs[0].State == "running" || body.Jobs[0].State == "routing" { + break + } + } + time.Sleep(20 * time.Millisecond) + } + if jobID == "" { + t.Fatal("job did not appear in UI API") + } + + req, _ := http.NewRequest(http.MethodPost, front.URL+"/gateway/ui-api/jobs/"+jobID+"/cancel", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("cancel status=%d body=%s", resp.StatusCode, b) + } + + select { + case status := <-result: + // Once Ollama has already sent HTTP 200 for a stream the gateway cannot + // change the wire status; cancellation is represented by terminating the + // stream and by internal status 499 in live/accounting metadata. + if status != http.StatusOK { + t.Fatalf("stream HTTP status=%d want 200", status) + } + case <-time.After(2 * time.Second): + t.Fatal("cancelled request did not terminate") + } + + snap := sv.live.Snapshot() + found := false + for _, r := range snap.Requests { + if r.ID == jobID { + found = true + if r.State != liveflow.StateCancelled || r.Status != 499 { + t.Fatalf("cancelled request state=%s status=%d", r.State, r.Status) + } + } + } + if !found { + t.Fatal("cancelled request missing from live snapshot") + } +} + +func TestUIJobCancelRemovesQueuedRequest(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + // Force one global slot so the second request stays in the fair queue. + sv.sched = scheduler.NewLocal(1, 32, 8) + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + startChat := func() <-chan int { + ch := make(chan int, 1) + go func() { + req, _ := http.NewRequest(http.MethodPost, front.URL+"/api/chat", strings.NewReader(`{"model":"qwen3:8b","messages":[{"role":"user","content":"hello"}]}`)) + req.Header.Set("Authorization", "Bearer test-admin-key") + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + ch <- 0 + return + } + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + ch <- resp.StatusCode + }() + return ch + } + cancel := func(id string) { + req, _ := http.NewRequest(http.MethodPost, front.URL+"/gateway/ui-api/jobs/"+id+"/cancel", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("cancel %s status=%d", id, resp.StatusCode) + } + } + listJobs := func() []jobView { + req, _ := http.NewRequest(http.MethodGet, front.URL+"/gateway/ui-api/jobs", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out struct { + Jobs []jobView `json:"jobs"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) + } + return out.Jobs + } + + first := startChat() + var firstID string + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + for _, j := range listJobs() { + if j.State == liveflow.StateRunning || j.State == liveflow.StateStreaming { + firstID = j.ID + break + } + } + if firstID != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + if firstID == "" { + t.Fatal("first request never became running") + } + + second := startChat() + var queuedID string + deadline = time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + for _, j := range listJobs() { + if j.ID != firstID && j.State == liveflow.StateQueued { + queuedID = j.ID + break + } + } + if queuedID != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + if queuedID == "" { + cancel(firstID) + t.Fatal("second request never entered queue") + } + + cancel(queuedID) + select { + case status := <-second: + if status != 499 { + t.Fatalf("queued cancellation status=%d want 499", status) + } + case <-time.After(2 * time.Second): + t.Fatal("queued request did not terminate after cancel") + } + + st := sv.sched.Stats(context.Background()) + if st.Queued != 0 { + t.Fatalf("queued=%d want 0 after cancellation", st.Queued) + } + + cancel(firstID) + select { + case <-first: + case <-time.After(2 * time.Second): + t.Fatal("first request did not terminate during cleanup") + } +} + +type testConfigStore struct { + saved *config.Config + deleted bool +} + +func (m *testConfigStore) Save(c *config.Config) error { x := *c; m.saved = &x; return nil } +func (m *testConfigStore) Delete() error { m.deleted = true; return nil } +func (m *testConfigStore) Path() string { return "/tmp/test-gateway-config.json" } + +func TestUIPersistentConfigSaveAndReset(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + sv.cfg.Storage = config.StorageConfig{DataDir: "./data", ConfigFile: "gateway-config.json", APIKeysFile: "api-keys.json", PoliciesFile: "policies.json", MetricsFile: "metrics.json", QuotaFile: "quota.json", WorkerPerformanceFile: "worker-performance.json", ModelPlacementFile: "model-placement.json", FlushInterval: config.Duration(10 * time.Second)} + sv.cfg.Alerts.Webhooks = []config.WebhookConfig{{Name: "ops", URL: "https://alerts.example.invalid/hook", Secret: "webhook-secret", Enabled: true}} + sv.cfg.OpenTelemetry.Headers = map[string]string{"Authorization": "Bearer otel-secret"} + sv.cfg.Conversations = config.ConversationsConfig{Enabled: true, EncryptionKey: "01234567890123456789012345678901", Retention: config.Duration(time.Hour), MaxEntries: 100, MaxContentBytes: 4096} + store := &testConfigStore{} + sv.configStore = store + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + raw := sv.redactedConfig() + rawBytes, _ := json.Marshal(raw) + if strings.Contains(string(rawBytes), "webhook-secret") || strings.Contains(string(rawBytes), "otel-secret") || strings.Contains(string(rawBytes), "01234567890123456789012345678901") { + t.Fatalf("redacted config leaked secret: %s", rawBytes) + } + costMap := raw["cost"].(map[string]any) + def := costMap["default"].(map[string]any) + def["output_credits_per_1k"] = 9.0 + b, _ := json.Marshal(raw) + req, _ := http.NewRequest(http.MethodPut, front.URL+"/gateway/ui-api/config", strings.NewReader(string(b))) + req.Header.Set("Authorization", "Bearer test-admin-key") + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("save status=%d body=%s", resp.StatusCode, body) + } + if store.saved == nil || store.saved.Cost.Default.OutputCreditsPer1K != 9 { + t.Fatalf("saved=%#v", store.saved) + } + if len(store.saved.Auth.APIKeys) != 1 || store.saved.Auth.APIKeys[0].Key != "test-admin-key" { + t.Fatalf("redaction overwrote bootstrap key: %#v", store.saved.Auth.APIKeys) + } + if len(store.saved.Alerts.Webhooks) != 1 || store.saved.Alerts.Webhooks[0].Secret != "webhook-secret" { + t.Fatalf("redaction overwrote webhook secret: %#v", store.saved.Alerts.Webhooks) + } + if store.saved.OpenTelemetry.Headers["Authorization"] != "Bearer otel-secret" { + t.Fatalf("redaction overwrote OTEL header: %#v", store.saved.OpenTelemetry.Headers) + } + if store.saved.Conversations.EncryptionKey != "01234567890123456789012345678901" { + t.Fatalf("redaction overwrote conversation key: %#v", store.saved.Conversations) + } + + req, _ = http.NewRequest(http.MethodDelete, front.URL+"/gateway/ui-api/config", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != 200 || !store.deleted { + t.Fatalf("reset status=%d deleted=%v", resp.StatusCode, store.deleted) + } +} + +func TestUIStorageFlushStatusAndBackup(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + dir := t.TempDir() + sv.cfg.Storage = config.StorageConfig{ + DataDir: dir, ConfigFile: "gateway-config.json", APIKeysFile: "api-keys.json", + PoliciesFile: "policies.json", MetricsFile: "metrics.json", QuotaFile: "quota.json", + WorkerPerformanceFile: "worker-performance.json", ModelPlacementFile: "model-placement.json", FlushInterval: config.Duration(time.Second), + } + sv.cfg.Usage.JournalDir = filepath.Join(dir, "usage") + rec, err := usage.NewWithRetention(sv.cfg.Usage.JournalDir, 32, time.Hour, usage.RetentionConfig{DetailDays: 1, DailyDays: 400, CompactionInterval: time.Hour}, nil) + if err != nil { + t.Fatal(err) + } + defer rec.Close() + sv.usage = rec + sv.configStore = state.NewConfigStore(filepath.Join(dir, "gateway-config.json")) + sv.metrics.Record("ollama", 200, time.Millisecond, time.Second, 10, 2, 1.25, 20, 30) + rec.Record(usage.Event{ID: "persist-1", Time: time.Now().UTC(), Tenant: "ops", Subject: "admin", Actor: "admin", Status: 200}) + rec.Record(usage.Event{ID: "persist-old", Time: time.Now().UTC().AddDate(0, 0, -3), Tenant: "ops", Subject: "admin", Actor: "admin", Model: "qwen3:8b", Status: 200}) + + front := httptest.NewServer(sv.Handler()) + defer front.Close() + do := func(method, path string) (*http.Response, []byte) { + t.Helper() + req, _ := http.NewRequest(method, front.URL+path, nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return resp, b + } + + resp, b := do(http.MethodPost, "/gateway/ui-api/storage/flush") + if resp.StatusCode != 200 { + t.Fatalf("flush status=%d body=%s", resp.StatusCode, b) + } + for _, name := range []string{"metrics.json", "worker-performance.json"} { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Fatalf("%s not persisted: %v", name, err) + } + } + matches, _ := filepath.Glob(filepath.Join(dir, "usage", "usage-*.jsonl")) + if len(matches) != 2 { + t.Fatalf("usage journals missing: %v", matches) + } + + resp, b = do(http.MethodGet, "/gateway/ui-api/storage") + if resp.StatusCode != 200 || !strings.Contains(string(b), `"mode":"local-persistent"`) || !strings.Contains(string(b), "worker performance") { + t.Fatalf("status=%d body=%s", resp.StatusCode, b) + } + + resp, b = do(http.MethodPost, "/gateway/ui-api/storage/compact") + if resp.StatusCode != 200 || !strings.Contains(string(b), `"compacted":true`) { + t.Fatalf("compact status=%d body=%s", resp.StatusCode, b) + } + daily, _ := filepath.Glob(filepath.Join(dir, "usage", "rollups", "daily", "rollup-daily-*.json")) + if len(daily) != 1 { + t.Fatalf("daily rollup missing after compaction: %v", daily) + } + resp, b = do(http.MethodGet, "/gateway/ui-api/usage/rollups?granularity=daily&dimension=model&name=qwen3:8b&limit=30") + if resp.StatusCode != 200 || !strings.Contains(string(b), `"requests":1`) { + t.Fatalf("rollup query status=%d body=%s", resp.StatusCode, b) + } + + resp, b = do(http.MethodGet, "/gateway/ui-api/storage/backup") + if resp.StatusCode != 200 || !strings.Contains(resp.Header.Get("Content-Type"), "application/zip") { + t.Fatalf("backup status=%d", resp.StatusCode) + } + zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b))) + if err != nil { + t.Fatal(err) + } + names := map[string]bool{} + for _, f := range zr.File { + names[f.Name] = true + } + if !names["state/metrics.json"] || !names["state/worker-performance.json"] || len(names) < 3 { + t.Fatalf("backup entries=%v", names) + } +} + +func TestUIModelPlacementPersistsAndAppliesImmediately(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + placementPath := filepath.Join(t.TempDir(), "model-placement.json") + store, err := state.NewModelPlacementStore(placementPath) + if err != nil { + t.Fatal(err) + } + sv.placementStore = store + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + put := func(body string) *http.Response { + req, _ := http.NewRequest(http.MethodPut, front.URL+"/gateway/ui-api/placement/w", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer test-admin-key") + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + return resp + } + resp := put(`{"mode":"whitelist","allowed_models":["qwen3:8b"],"denied_models":[]}`) + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("put status=%d body=%s", resp.StatusCode, body) + } + if d, _ := sv.workers.PlacementDecision("w", "qwen3:8b"); !d.Allowed { + t.Fatalf("qwen should be allowed: %#v", d) + } + if d, _ := sv.workers.PlacementDecision("w", "other:latest"); d.Allowed { + t.Fatalf("other should be blocked by whitelist: %#v", d) + } + + // A matrix-style exact deny should take effect without a restart. + req, _ := http.NewRequest(http.MethodPost, front.URL+"/gateway/ui-api/placement/w/model", strings.NewReader(`{"model":"qwen3:8b","action":"deny"}`)) + req.Header.Set("Authorization", "Bearer test-admin-key") + req.Header.Set("Content-Type", "application/json") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("model action status=%d body=%s", resp.StatusCode, body) + } + if d, _ := sv.workers.PlacementDecision("w", "qwen3:8b"); d.Allowed { + t.Fatalf("exact deny not applied: %#v", d) + } + + // Reload the persistent store to prove durability. + reloaded, err := state.NewModelPlacementStore(placementPath) + if err != nil { + t.Fatal(err) + } + if rule, ok, _ := reloaded.Get(context.Background(), "w"); !ok || len(rule.DeniedModels) != 1 { + t.Fatalf("persistent rule=%#v ok=%v", rule, ok) + } + + req, _ = http.NewRequest(http.MethodDelete, front.URL+"/gateway/ui-api/placement/w", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("reset status=%d", resp.StatusCode) + } + if d, _ := sv.workers.PlacementDecision("w", "other:latest"); !d.Allowed { + t.Fatalf("reset did not restore allow_all baseline: %#v", d) + } +} + +func TestUIBatchAdminControlAndOutput(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + cfg := config.BatchJobsConfig{Enabled: true, Retention: config.Duration(time.Hour), MaxJobs: 100, MaxConcurrent: 1, MaxInputBytes: 1 << 20} + dir := t.TempDir() + m, err := batch.New(cfg, filepath.Join(dir, "batch-jobs.json"), filepath.Join(dir, "batch")) + if err != nil { + t.Fatal(err) + } + sv.batchJobs = m + sv.cfg.BatchJobs = cfg + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + do := func(method, path string) (*http.Response, []byte) { + req, _ := http.NewRequest(method, front.URL+path, nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return resp, body + } + + queued, err := m.Create(batch.IdentitySnapshot{Tenant: "tenant-a", Subject: "alice", Actor: "alice", AuthType: "oidc"}, "/api/chat", "qwen3:8b", []byte(`{"model":"qwen3:8b"}`)) + if err != nil { + t.Fatal(err) + } + resp, body := do(http.MethodGet, "/gateway/ui-api/batches") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), queued.ID) || !strings.Contains(string(body), `"enabled":true`) { + t.Fatalf("list status=%d body=%s", resp.StatusCode, body) + } + resp, body = do(http.MethodPost, "/gateway/ui-api/batches/"+queued.ID+"/pause") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"state":"paused"`) { + t.Fatalf("pause status=%d body=%s", resp.StatusCode, body) + } + resp, body = do(http.MethodPost, "/gateway/ui-api/batches/"+queued.ID+"/resume") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"state":"queued"`) { + t.Fatalf("resume status=%d body=%s", resp.StatusCode, body) + } + resp, body = do(http.MethodPost, "/gateway/ui-api/batches/"+queued.ID+"/cancel") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"state":"cancelled"`) { + t.Fatalf("cancel status=%d body=%s", resp.StatusCode, body) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + m.Start(ctx, func(ctx context.Context, j batch.Job, in io.Reader, out io.Writer) batch.RunResult { + _, _ = io.WriteString(out, `{"ok":true}`) + return batch.RunResult{HTTPStatus: http.StatusOK, ResponseContentType: "application/json", RequestID: "req-batch-ui"} + }) + completed, err := m.Create(batch.IdentitySnapshot{Tenant: "tenant-b", Subject: "bob", Actor: "bob", AuthType: "oidc"}, "/api/chat", "qwen3:8b", []byte(`{"model":"qwen3:8b"}`)) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + j, ok := m.Get(completed.ID, "", "", true) + if ok && j.State == batch.StateCompleted { + break + } + time.Sleep(5 * time.Millisecond) + } + resp, body = do(http.MethodGet, "/gateway/ui-api/batches/"+completed.ID+"/output") + if resp.StatusCode != http.StatusOK || string(body) != `{"ok":true}` || resp.Header.Get("Content-Type") != "application/json" { + t.Fatalf("output status=%d content-type=%q body=%s", resp.StatusCode, resp.Header.Get("Content-Type"), body) + } +} + +func TestUIModelAliasRuntimeCRUDPersistsAndPublishesAtomically(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + store := &testConfigStore{} + sv.configStore = store + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + put := func(path, body string) *http.Response { + t.Helper() + req, _ := http.NewRequest(http.MethodPut, front.URL+path, strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer test-admin-key") + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + return resp + } + + resp := put("/gateway/ui-api/model-aliases/quick", `{"models":[" qwen3:8b "],"required_capabilities":[" completion "]}`) + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("put status=%d body=%s", resp.StatusCode, body) + } + got, ok := sv.aliasConfig("quick") + if !ok || len(got.Models) != 1 || got.Models[0] != "qwen3:8b" || len(got.RequiredCapabilities) != 1 || got.RequiredCapabilities[0] != "completion" { + t.Fatalf("runtime alias=%#v ok=%v", got, ok) + } + if store.saved == nil || store.saved.ModelAliases["quick"].Models[0] != "qwen3:8b" { + t.Fatalf("persistent alias missing: %#v", store.saved) + } + + req, _ := http.NewRequest(http.MethodGet, front.URL+"/gateway/ui-api/model-aliases", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"quick"`) || !strings.Contains(string(body), `"runtime":true`) { + t.Fatalf("get status=%d body=%s", resp.StatusCode, body) + } + + req, _ = http.NewRequest(http.MethodDelete, front.URL+"/gateway/ui-api/model-aliases/quick", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("delete status=%d body=%s", resp.StatusCode, body) + } + if _, ok := sv.aliasConfig("quick"); ok { + t.Fatal("deleted alias remains in runtime snapshot") + } + if store.saved == nil { + t.Fatal("delete was not persisted") + } + if _, ok := store.saved.ModelAliases["quick"]; ok { + t.Fatalf("deleted alias remains persisted: %#v", store.saved.ModelAliases) + } +} + +func TestUIModelAliasRejectsInvalidWithoutPublishing(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + sv.configStore = &testConfigStore{} + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + req, _ := http.NewRequest(http.MethodPut, front.URL+"/gateway/ui-api/model-aliases/broken", strings.NewReader(`{"models":[]}`)) + req.Header.Set("Authorization", "Bearer test-admin-key") + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status=%d want 400", resp.StatusCode) + } + if _, ok := sv.aliasConfig("broken"); ok { + t.Fatal("invalid alias was published") + } +} + +func TestUITenantModelAccessRuntimeCRUD(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + store := &testConfigStore{} + sv.configStore = store + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + putBody := `{"mode":"whitelist","allowed_models":[" qwen3:* "],"denied_models":["qwen3:secret*"]}` + req, _ := http.NewRequest(http.MethodPut, front.URL+"/gateway/ui-api/model-access/interns", strings.NewReader(putBody)) + req.Header.Set("Authorization", "Bearer test-admin-key") + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("put status=%d body=%s", resp.StatusCode, body) + } + id := auth.Identity{Tenant: "interns"} + if !sv.modelAllowed(id, "qwen3:8b") || sv.modelAllowed(id, "gemma3:12b") || sv.modelAllowed(id, "qwen3:secret-1") { + t.Fatalf("runtime ACL not enforced: %#v", sv.modelAccessSnapshot()) + } + if store.saved == nil || store.saved.ModelAccess.Tenants["interns"].Mode != "whitelist" { + t.Fatalf("persistent ACL missing: %#v", store.saved) + } + + req, _ = http.NewRequest(http.MethodGet, front.URL+"/gateway/ui-api/model-access", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"interns"`) || !strings.Contains(string(body), `"runtime":true`) { + t.Fatalf("get status=%d body=%s", resp.StatusCode, body) + } + + req, _ = http.NewRequest(http.MethodDelete, front.URL+"/gateway/ui-api/model-access/interns", nil) + req.Header.Set("Authorization", "Bearer test-admin-key") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("delete status=%d", resp.StatusCode) + } + if !sv.modelAllowed(id, "gemma3:12b") { + t.Fatal("tenant ACL delete did not revert to allow-all default") + } + if _, ok := sv.modelAccessSnapshot().Tenants["interns"]; ok { + t.Fatal("tenant ACL remains in runtime snapshot") + } +} + +func TestUITenantModelAccessRejectsInvalidPattern(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + sv.configStore = &testConfigStore{} + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + req, _ := http.NewRequest(http.MethodPut, front.URL+"/gateway/ui-api/model-access/interns", strings.NewReader(`{"mode":"whitelist","allowed_models":["bad*pattern"]}`)) + req.Header.Set("Authorization", "Bearer test-admin-key") + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status=%d want 400", resp.StatusCode) + } + if _, ok := sv.modelAccessSnapshot().Tenants["interns"]; ok { + t.Fatal("invalid tenant ACL was published") + } +} + +func TestPublicDashboardIsUnauthenticatedAndSanitized(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + sv.cfg.PublicDashboard = config.PublicDashboardConfig{ + Enabled: true, Path: "/status", Title: "Public Test", Subtitle: "Sanitized", + RefreshInterval: config.Duration(2 * time.Second), MaxLiveRequests: 16, + ShowWorkerNames: false, ShowModelNames: false, ShowResourceMetrics: true, + WorkerDisplayNames: map[string]string{"w": "GPU Node A"}, + } + sv.live.Begin(liveflow.Request{ID: "secret-request-id", Tenant: "super-secret-tenant", Actor: "user:alice@example.test", Application: "private-app", Model: "private-model:latest", Worker: "w", API: "openai", Path: "/v1/chat/completions", EstimatedCredits: 99}) + + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + resp, err := http.Get(front.URL + "/status/") + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "Read-only public status") { + t.Fatalf("public UI status=%d body=%s", resp.StatusCode, body) + } + if got := resp.Header.Get("Content-Security-Policy"); !strings.Contains(got, "form-action 'none'") { + t.Fatalf("public UI missing restrictive CSP: %q", got) + } + + resp, err = http.Get(front.URL + "/status/api/snapshot") + if err != nil { + t.Fatal(err) + } + body, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("snapshot status=%d body=%s", resp.StatusCode, body) + } + text := string(body) + for _, forbidden := range []string{ + "super-secret-tenant", "alice@example.test", "private-app", "secret-request-id", + `"tenant"`, `"actor"`, `"application"`, `"url"`, `"labels"`, `"last_error"`, `"telemetry_error"`, `"estimated_credits"`, + } { + if strings.Contains(text, forbidden) { + t.Fatalf("public snapshot leaked %q: %s", forbidden, text) + } + } + if !strings.Contains(text, `"name":"GPU Node A"`) { + t.Fatalf("configured public worker alias missing: %s", text) + } + if strings.Contains(text, "private-model:latest") || !strings.Contains(text, "Model ") { + t.Fatalf("model anonymization failed: %s", text) + } + if strings.Contains(text, `"id":"secret-request-id"`) || !strings.Contains(text, `"id":"REQ-`) { + t.Fatalf("request id was not anonymized: %s", text) + } +} + +func TestPublicDashboardDisabledIs404WithoutAuthChallenge(t *testing.T) { + sv, _, done := newUITestServer(t) + defer done() + sv.cfg.PublicDashboard = config.PublicDashboardConfig{Enabled: false, Path: "/status"} + front := httptest.NewServer(sv.Handler()) + defer front.Close() + + resp, err := http.Get(front.URL + "/status/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status=%d want 404", resp.StatusCode) + } + if got := resp.Header.Get("WWW-Authenticate"); got != "" { + t.Fatalf("disabled public dashboard should not enter auth middleware: %q", got) + } +} diff --git a/internal/session/store.go b/internal/session/store.go new file mode 100644 index 0000000..2a2271d --- /dev/null +++ b/internal/session/store.go @@ -0,0 +1,68 @@ +package session + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "sync" + "time" +) + +var ErrNotFound = errors.New("session not found") + +type Store interface { + Create(context.Context, string, time.Duration) (string, error) + Get(context.Context, string) (string, error) + Delete(context.Context, string) error + Health(context.Context) error +} + +type memoryEntry struct { + token string + exp time.Time +} +type Memory struct { + mu sync.Mutex + m map[string]memoryEntry +} + +func NewMemory() *Memory { return &Memory{m: map[string]memoryEntry{}} } +func (m *Memory) Create(_ context.Context, token string, ttl time.Duration) (string, error) { + id := newID() + m.mu.Lock() + m.m[id] = memoryEntry{token: token, exp: time.Now().Add(ttl)} + if len(m.m) > 4096 { + now := time.Now() + for k, v := range m.m { + if now.After(v.exp) { + delete(m.m, k) + } + } + } + m.mu.Unlock() + return id, nil +} +func (m *Memory) Get(_ context.Context, id string) (string, error) { + m.mu.Lock() + defer m.mu.Unlock() + e, ok := m.m[id] + if !ok || time.Now().After(e.exp) { + delete(m.m, id) + return "", ErrNotFound + } + return e.token, nil +} +func (m *Memory) Delete(_ context.Context, id string) error { + m.mu.Lock() + delete(m.m, id) + m.mu.Unlock() + return nil +} +func (m *Memory) Health(context.Context) error { return nil } + +func newID() string { + b := make([]byte, 32) + _, _ = rand.Read(b) + return base64.RawURLEncoding.EncodeToString(b) +} diff --git a/internal/state/apikeys.go b/internal/state/apikeys.go new file mode 100644 index 0000000..e1abcb9 --- /dev/null +++ b/internal/state/apikeys.go @@ -0,0 +1,93 @@ +package state + +import ( + "context" + "errors" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/example/ollama-fair-gateway/internal/auth" +) + +type APIKeyStore struct { + mu sync.Mutex + file AtomicJSON + records map[string]auth.StoredAPIKey +} + +type apiKeyFile struct { + Keys []auth.StoredAPIKey `json:"keys"` +} + +func NewAPIKeyStore(path string) (*APIKeyStore, error) { + s := &APIKeyStore{file: AtomicJSON{Path: path, Mode: 0600}, records: map[string]auth.StoredAPIKey{}} + var f apiKeyFile + if err := s.file.Load(&f); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + for _, r := range f.Keys { + if r.ID != "" { + s.records[r.ID] = r + } + } + return s, nil +} +func (s *APIKeyStore) Load() ([]auth.StoredAPIKey, error) { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]auth.StoredAPIKey, 0, len(s.records)) + for _, r := range s.records { + r.Scopes = append([]string(nil), r.Scopes...) + out = append(out, r) + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.Before(out[j].CreatedAt) }) + return out, nil +} +func (s *APIKeyStore) Put(r auth.StoredAPIKey) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.records[r.ID]; ok { + return errors.New("API key id already exists") + } + s.records[r.ID] = r + if err := s.saveLocked(); err != nil { + delete(s.records, r.ID) + return err + } + return nil +} +func (s *APIKeyStore) Delete(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + old, ok := s.records[id] + if !ok { + return nil + } + delete(s.records, id) + if err := s.saveLocked(); err != nil { + s.records[id] = old + return err + } + return nil +} +func (s *APIKeyStore) saveLocked() error { + a := make([]auth.StoredAPIKey, 0, len(s.records)) + for _, r := range s.records { + a = append(a, r) + } + sort.Slice(a, func(i, j int) bool { return a[i].CreatedAt.Before(a[j].CreatedAt) }) + return s.file.Save(apiKeyFile{Keys: a}) +} +func (s *APIKeyStore) Health(context.Context) error { + if err := os.MkdirAll(filepath.Dir(s.file.Path), 0750); err != nil { + return err + } + f, err := os.OpenFile(s.file.Path+".health", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return err + } + _ = f.Close() + return os.Remove(s.file.Path + ".health") +} diff --git a/internal/state/atomic.go b/internal/state/atomic.go new file mode 100644 index 0000000..2f2baa1 --- /dev/null +++ b/internal/state/atomic.go @@ -0,0 +1,110 @@ +package state + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +// AtomicJSON provides small, durable JSON state files. Writes are staged in +// the target directory, fsynced, chmodded, then atomically renamed. +type AtomicJSON struct { + Path string + Mode os.FileMode +} + +func (f AtomicJSON) Load(out any) error { + b, err := os.ReadFile(f.Path) + if errors.Is(err, os.ErrNotExist) { + return os.ErrNotExist + } + if err != nil { + return err + } + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("decode %s: %w", f.Path, err) + } + return nil +} + +func (f AtomicJSON) Save(v any) error { + if f.Path == "" { + return errors.New("state path is empty") + } + dir := filepath.Dir(f.Path) + if err := os.MkdirAll(dir, 0750); err != nil { + return err + } + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + tmp, err := os.CreateTemp(dir, ".state-*.tmp") + if err != nil { + return err + } + name := tmp.Name() + ok := false + defer func() { + _ = tmp.Close() + if !ok { + _ = os.Remove(name) + } + }() + mode := f.Mode + if mode == 0 { + mode = 0600 + } + if err := tmp.Chmod(mode); err != nil { + return err + } + if _, err := tmp.Write(b); err != nil { + return err + } + if err := tmp.Sync(); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + // os.Rename replaces an existing regular file on Unix. On Windows the + // destination may need to be removed first; keep a .bak so a failed + // replacement does not destroy the previous durable state. + if err := os.Rename(name, f.Path); err != nil { + bak := f.Path + ".bak" + _ = os.Remove(bak) + if _, statErr := os.Stat(f.Path); statErr == nil { + if rerr := os.Rename(f.Path, bak); rerr != nil { + return err + } + } + if rerr := os.Rename(name, f.Path); rerr != nil { + _ = os.Rename(bak, f.Path) + return rerr + } + _ = os.Remove(bak) + } + syncDir(dir) + ok = true + return nil +} + +func syncDir(dir string) { + d, err := os.Open(dir) + if err != nil { + return + } + _ = d.Sync() + _ = d.Close() +} + +func (f AtomicJSON) Delete() error { + err := os.Remove(f.Path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} diff --git a/internal/state/config.go b/internal/state/config.go new file mode 100644 index 0000000..31edf78 --- /dev/null +++ b/internal/state/config.go @@ -0,0 +1,77 @@ +package state + +import ( + "encoding/json" + "errors" + "os" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +const bootstrapSecret = "" + +type ConfigStore struct{ file AtomicJSON } + +func NewConfigStore(path string) *ConfigStore { + return &ConfigStore{file: AtomicJSON{Path: path, Mode: 0600}} +} +func (s *ConfigStore) Path() string { return s.file.Path } + +// LoadWithBootstrap restores secrets that deliberately remain owned by the +// startup configuration and are never copied into the persistent UI override. +func (s *ConfigStore) LoadWithBootstrap(base *config.Config) (*config.Config, error) { + b, err := os.ReadFile(s.file.Path) + if err != nil { + return nil, err + } + var c config.Config + if err := json.Unmarshal(b, &c); err != nil { + return nil, err + } + c.Auth.APIKeys = append([]config.APIKeyConfig(nil), base.Auth.APIKeys...) + if c.UI.SessionSecret == "" || c.UI.SessionSecret == bootstrapSecret { + c.UI.SessionSecret = base.UI.SessionSecret + } + if c.UI.OIDC.ClientSecret == "" || c.UI.OIDC.ClientSecret == bootstrapSecret { + c.UI.OIDC.ClientSecret = base.UI.OIDC.ClientSecret + } + merged, err := json.Marshal(c) + if err != nil { + return nil, err + } + return config.ParseBytes(merged) +} + +// Save writes a complete config override but strips bootstrap-owned secrets. +func (s *ConfigStore) Save(c *config.Config) error { + b, err := json.Marshal(c) + if err != nil { + return err + } + var clean config.Config + if err := json.Unmarshal(b, &clean); err != nil { + return err + } + for i := range clean.Auth.APIKeys { + clean.Auth.APIKeys[i].Key = bootstrapSecret + } + if clean.UI.SessionSecret != "" { + clean.UI.SessionSecret = bootstrapSecret + } + if clean.UI.OIDC.ClientSecret != "" { + clean.UI.OIDC.ClientSecret = bootstrapSecret + } + return s.file.Save(&clean) +} +func (s *ConfigStore) Delete() error { return s.file.Delete() } +func (s *ConfigStore) Exists() bool { _, err := os.Stat(s.file.Path); return err == nil } +func (s *ConfigStore) LoadIfExists(base *config.Config) (*config.Config, bool, error) { + c, err := s.LoadWithBootstrap(base) + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return c, true, nil +} diff --git a/internal/state/modelplacement.go b/internal/state/modelplacement.go new file mode 100644 index 0000000..f60fd95 --- /dev/null +++ b/internal/state/modelplacement.go @@ -0,0 +1,103 @@ +package state + +import ( + "context" + "errors" + "os" + "sync" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +// ModelPlacementStore persists only runtime overrides. Bootstrap rules remain +// part of config.json and can be restored per worker by deleting an override. +type ModelPlacementStore struct { + mu sync.RWMutex + file AtomicJSON + m map[string]config.ModelPlacementRule +} + +type modelPlacementFile struct { + Workers map[string]config.ModelPlacementRule `json:"workers"` +} + +func NewModelPlacementStore(path string) (*ModelPlacementStore, error) { + s := &ModelPlacementStore{file: AtomicJSON{Path: path, Mode: 0600}, m: map[string]config.ModelPlacementRule{}} + var f modelPlacementFile + if err := s.file.Load(&f); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + for name, rule := range f.Workers { + if err := config.ValidateModelPlacementRule(rule); err != nil { + return nil, err + } + s.m[name] = clonePlacementRule(rule) + } + return s, nil +} + +func (s *ModelPlacementStore) Get(_ context.Context, worker string) (config.ModelPlacementRule, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.m[worker] + return clonePlacementRule(r), ok, nil +} + +func (s *ModelPlacementStore) List(context.Context) (map[string]config.ModelPlacementRule, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := make(map[string]config.ModelPlacementRule, len(s.m)) + for k, v := range s.m { + out[k] = clonePlacementRule(v) + } + return out, nil +} + +func (s *ModelPlacementStore) Put(_ context.Context, worker string, r config.ModelPlacementRule) error { + if err := config.ValidateModelPlacementRule(r); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + old, had := s.m[worker] + s.m[worker] = clonePlacementRule(r) + if err := s.saveLocked(); err != nil { + if had { + s.m[worker] = old + } else { + delete(s.m, worker) + } + return err + } + return nil +} + +func (s *ModelPlacementStore) Delete(_ context.Context, worker string) error { + s.mu.Lock() + defer s.mu.Unlock() + old, had := s.m[worker] + delete(s.m, worker) + if err := s.saveLocked(); err != nil { + if had { + s.m[worker] = old + } + return err + } + return nil +} + +func (s *ModelPlacementStore) Health(context.Context) error { return nil } + +func (s *ModelPlacementStore) saveLocked() error { + out := make(map[string]config.ModelPlacementRule, len(s.m)) + for k, v := range s.m { + out[k] = clonePlacementRule(v) + } + return s.file.Save(modelPlacementFile{Workers: out}) +} + +func clonePlacementRule(r config.ModelPlacementRule) config.ModelPlacementRule { + r.AllowedModels = append([]string(nil), r.AllowedModels...) + r.DeniedModels = append([]string(nil), r.DeniedModels...) + return r +} diff --git a/internal/state/paths.go b/internal/state/paths.go new file mode 100644 index 0000000..711c25e --- /dev/null +++ b/internal/state/paths.go @@ -0,0 +1,46 @@ +package state + +import ( + "github.com/example/ollama-fair-gateway/internal/config" + "path/filepath" +) + +type Paths struct { + DataDir string + Config string + APIKeys string + Policies string + Metrics string + Quota string + WorkerPerformance string + ModelPlacement string + WorkerState string + AutoTune string + WarmModels string + Alerts string + Conversations string + BatchJobs string + BatchDir string + UsageDir string +} + +func Resolve(c config.StorageConfig) Paths { + return Paths{ + DataDir: c.DataDir, + Config: filepath.Join(c.DataDir, c.ConfigFile), + APIKeys: filepath.Join(c.DataDir, c.APIKeysFile), + Policies: filepath.Join(c.DataDir, c.PoliciesFile), + Metrics: filepath.Join(c.DataDir, c.MetricsFile), + Quota: filepath.Join(c.DataDir, c.QuotaFile), + WorkerPerformance: filepath.Join(c.DataDir, c.WorkerPerformanceFile), + ModelPlacement: filepath.Join(c.DataDir, c.ModelPlacementFile), + WorkerState: filepath.Join(c.DataDir, c.WorkerStateFile), + AutoTune: filepath.Join(c.DataDir, c.AutoTuneFile), + WarmModels: filepath.Join(c.DataDir, c.WarmModelsFile), + Alerts: filepath.Join(c.DataDir, c.AlertsFile), + Conversations: filepath.Join(c.DataDir, c.ConversationsFile), + BatchJobs: filepath.Join(c.DataDir, c.BatchJobsFile), + BatchDir: filepath.Join(c.DataDir, c.BatchJobsDir), + UsageDir: filepath.Join(c.DataDir, "usage"), + } +} diff --git a/internal/state/policies.go b/internal/state/policies.go new file mode 100644 index 0000000..0883249 --- /dev/null +++ b/internal/state/policies.go @@ -0,0 +1,83 @@ +package state + +import ( + "context" + "errors" + "os" + "sync" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +type PolicyStore struct { + mu sync.RWMutex + file AtomicJSON + m map[string]config.TenantPolicy +} + +type policyFile struct { + Policies map[string]config.TenantPolicy `json:"policies"` +} + +func NewPolicyStore(path string) (*PolicyStore, error) { + s := &PolicyStore{file: AtomicJSON{Path: path, Mode: 0600}, m: map[string]config.TenantPolicy{}} + var f policyFile + if err := s.file.Load(&f); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + for k, v := range f.Policies { + s.m[k] = v + } + return s, nil +} +func (s *PolicyStore) Get(_ context.Context, tenant string) (config.TenantPolicy, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + p, ok := s.m[tenant] + return p, ok, nil +} +func (s *PolicyStore) Put(_ context.Context, tenant string, p config.TenantPolicy) error { + s.mu.Lock() + defer s.mu.Unlock() + old, had := s.m[tenant] + s.m[tenant] = p + if err := s.saveLocked(); err != nil { + if had { + s.m[tenant] = old + } else { + delete(s.m, tenant) + } + return err + } + return nil +} +func (s *PolicyStore) Delete(_ context.Context, tenant string) error { + s.mu.Lock() + defer s.mu.Unlock() + old, had := s.m[tenant] + delete(s.m, tenant) + if err := s.saveLocked(); err != nil { + if had { + s.m[tenant] = old + } + return err + } + return nil +} +func (s *PolicyStore) List(context.Context) (map[string]config.TenantPolicy, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := make(map[string]config.TenantPolicy, len(s.m)) + for k, v := range s.m { + out[k] = v + } + return out, nil +} +func (s *PolicyStore) Health(context.Context) error { return nil } +func (s *PolicyStore) saveLocked() error { + out := make(map[string]config.TenantPolicy, len(s.m)) + for k, v := range s.m { + out[k] = v + } + return s.file.Save(policyFile{Policies: out}) +} diff --git a/internal/state/state_test.go b/internal/state/state_test.go new file mode 100644 index 0000000..24d16a4 --- /dev/null +++ b/internal/state/state_test.go @@ -0,0 +1,180 @@ +package state + +import ( + "bytes" + "context" + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/example/ollama-fair-gateway/internal/auth" + "github.com/example/ollama-fair-gateway/internal/config" +) + +func TestPersistentAPIKeySurvivesAuthenticatorRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "keys.json") + store, err := NewAPIKeyStore(path) + if err != nil { + t.Fatal(err) + } + a, err := auth.NewWithRuntimeStore(context.Background(), config.AuthConfig{}, store) + if err != nil { + t.Fatal(err) + } + info, secret, err := a.CreateAPIKey(auth.APIKeyCreate{Name: "openwebui", Tenant: "interactive", Application: "openwebui"}) + if err != nil { + t.Fatal(err) + } + if info.Source != "persistent" { + t.Fatalf("source=%q", info.Source) + } + + store2, err := NewAPIKeyStore(path) + if err != nil { + t.Fatal(err) + } + b, err := auth.NewWithRuntimeStore(context.Background(), config.AuthConfig{}, store2) + if err != nil { + t.Fatal(err) + } + r := httptest.NewRequest("GET", "http://gateway/api/tags", nil) + r.Header.Set("Authorization", "Bearer "+secret) + id, err := b.Authenticate(r) + if err != nil { + t.Fatal(err) + } + if id.Tenant != "interactive" || id.Application != "openwebui" { + t.Fatalf("identity=%#v", id) + } + if _, ok, err := b.DeleteAPIKey(info.ID); err != nil || !ok { + t.Fatalf("delete ok=%v err=%v", ok, err) + } + + store3, err := NewAPIKeyStore(path) + if err != nil { + t.Fatal(err) + } + c, err := auth.NewWithRuntimeStore(context.Background(), config.AuthConfig{}, store3) + if err != nil { + t.Fatal(err) + } + if _, err := c.Authenticate(r); err == nil { + t.Fatal("deleted key authenticated after restart") + } +} + +func TestPersistentPolicySurvivesRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "policies.json") + a, err := NewPolicyStore(path) + if err != nil { + t.Fatal(err) + } + want := config.TenantPolicy{TenantWeight: 2, ActorWeight: 3, TenantCreditsPerMinute: 0, ActorCreditsPerMinute: 120} + if err := a.Put(context.Background(), "team", want); err != nil { + t.Fatal(err) + } + b, err := NewPolicyStore(path) + if err != nil { + t.Fatal(err) + } + got, ok, err := b.Get(context.Background(), "team") + if err != nil || !ok { + t.Fatalf("get ok=%v err=%v", ok, err) + } + if got != want { + t.Fatalf("got=%#v want=%#v", got, want) + } +} + +func TestConfigStoreKeepsBootstrapSecretsOutOfPersistentFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "gateway-config.json") + base := &config.Config{ + Auth: config.AuthConfig{APIKeys: []config.APIKeyConfig{{Name: "admin", Key: "super-secret", Tenant: "ops"}}}, + Workers: []config.WorkerConfig{{Name: "w", URL: "http://127.0.0.1:11434", MaxConcurrent: 1}}, + UI: config.UIConfig{SessionSecret: "01234567890123456789012345678901", OIDC: config.UIOIDCConfig{ClientSecret: "oidc-secret"}}, + } + // Fill defaults/validation through the normal parser so the stored object is realistic. + b, _ := json.Marshal(base) + parsed, err := config.ParseBytes(b) + if err != nil { + t.Fatal(err) + } + store := NewConfigStore(path) + if err := store.Save(parsed); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(raw, []byte("super-secret")) || bytes.Contains(raw, []byte("oidc-secret")) || bytes.Contains(raw, []byte("01234567890123456789012345678901")) { + t.Fatalf("persistent config leaked bootstrap secret: %s", raw) + } + loaded, err := store.LoadWithBootstrap(parsed) + if err != nil { + t.Fatal(err) + } + if loaded.Auth.APIKeys[0].Key != "super-secret" || loaded.UI.SessionSecret != parsed.UI.SessionSecret || loaded.UI.OIDC.ClientSecret != "oidc-secret" { + t.Fatalf("secrets not restored: %#v", loaded.UI) + } +} + +func TestModelPlacementStoreSurvivesRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "model-placement.json") + a, err := NewModelPlacementStore(path) + if err != nil { + t.Fatal(err) + } + want := config.ModelPlacementRule{Mode: "whitelist", AllowedModels: []string{"qwen3:8b", "gemma4:*"}, DeniedModels: []string{"gemma4:e4b"}} + if err := a.Put(context.Background(), "rtx-4090", want); err != nil { + t.Fatal(err) + } + b, err := NewModelPlacementStore(path) + if err != nil { + t.Fatal(err) + } + got, ok, err := b.Get(context.Background(), "rtx-4090") + if err != nil || !ok { + t.Fatalf("get ok=%v err=%v", ok, err) + } + if got.Mode != want.Mode || len(got.AllowedModels) != 2 || len(got.DeniedModels) != 1 { + t.Fatalf("got=%#v want=%#v", got, want) + } + if err := b.Delete(context.Background(), "rtx-4090"); err != nil { + t.Fatal(err) + } + c, err := NewModelPlacementStore(path) + if err != nil { + t.Fatal(err) + } + if _, ok, _ := c.Get(context.Background(), "rtx-4090"); ok { + t.Fatal("deleted placement override survived restart") + } +} + +func TestWorkerRuntimeStorePersistence(t *testing.T) { + path := filepath.Join(t.TempDir(), "worker-state.json") + s, err := NewWorkerRuntimeStore(path) + if err != nil { + t.Fatal(err) + } + if err := s.Put(context.Background(), "node-2", "draining"); err != nil { + t.Fatal(err) + } + s2, err := NewWorkerRuntimeStore(path) + if err != nil { + t.Fatal(err) + } + m, err := s2.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if m["node-2"] != "draining" { + t.Fatalf("modes=%v", m) + } + if err := s2.Delete(context.Background(), "node-2"); err != nil { + t.Fatal(err) + } +} diff --git a/internal/state/workerstate.go b/internal/state/workerstate.go new file mode 100644 index 0000000..394b502 --- /dev/null +++ b/internal/state/workerstate.go @@ -0,0 +1,95 @@ +package state + +import ( + "context" + "errors" + "os" + "path/filepath" + "sort" + "sync" +) + +type WorkerRuntimeStore struct { + mu sync.Mutex + file AtomicJSON + modes map[string]string +} + +type workerRuntimeFile struct { + Workers map[string]string `json:"workers"` +} + +func NewWorkerRuntimeStore(path string) (*WorkerRuntimeStore, error) { + s := &WorkerRuntimeStore{file: AtomicJSON{Path: path, Mode: 0600}, modes: map[string]string{}} + var f workerRuntimeFile + if err := s.file.Load(&f); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + for k, v := range f.Workers { + if k != "" { + s.modes[k] = v + } + } + return s, nil +} +func (s *WorkerRuntimeStore) List(context.Context) (map[string]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + out := map[string]string{} + for k, v := range s.modes { + out[k] = v + } + return out, nil +} +func (s *WorkerRuntimeStore) Put(_ context.Context, name, mode string) error { + s.mu.Lock() + defer s.mu.Unlock() + old, had := s.modes[name] + s.modes[name] = mode + if err := s.saveLocked(); err != nil { + if had { + s.modes[name] = old + } else { + delete(s.modes, name) + } + return err + } + return nil +} +func (s *WorkerRuntimeStore) Delete(_ context.Context, name string) error { + s.mu.Lock() + defer s.mu.Unlock() + old, had := s.modes[name] + delete(s.modes, name) + if err := s.saveLocked(); err != nil { + if had { + s.modes[name] = old + } + return err + } + return nil +} +func (s *WorkerRuntimeStore) saveLocked() error { + keys := make([]string, 0, len(s.modes)) + for k := range s.modes { + keys = append(keys, k) + } + sort.Strings(keys) + m := map[string]string{} + for _, k := range keys { + m[k] = s.modes[k] + } + return s.file.Save(workerRuntimeFile{Workers: m}) +} +func (s *WorkerRuntimeStore) Health(context.Context) error { + if err := os.MkdirAll(filepath.Dir(s.file.Path), 0750); err != nil { + return err + } + f, err := os.OpenFile(s.file.Path+".health", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return err + } + _ = f.Close() + return os.Remove(s.file.Path + ".health") +} +func (s *WorkerRuntimeStore) Path() string { return s.file.Path } diff --git a/internal/telemetry/otel.go b/internal/telemetry/otel.go new file mode 100644 index 0000000..16d2c9c --- /dev/null +++ b/internal/telemetry/otel.go @@ -0,0 +1,376 @@ +package telemetry + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync/atomic" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +type Context struct { + TraceID string + RootSpanID string + ParentSpanID string + Sampled bool +} + +type Interval struct { + Name string + Start time.Time + End time.Time + Attrs map[string]any +} + +type Record struct { + Trace Context + RequestID string + API string + Path string + Tenant string + Actor string + Application string + ServiceClass string + Model string + Alias string + Worker string + Started time.Time + Finished time.Time + FirstByte time.Duration + Status int + PromptTokens int64 + OutputTokens int64 + CachedTokens int64 + Credits float64 + Error string + Intervals []Interval +} + +type Exporter struct { + cfg config.OpenTelemetryConfig + client *http.Client + ch chan []span + stop chan struct{} + done chan struct{} + closed atomic.Bool + dropped atomic.Uint64 + failed atomic.Uint64 + exported atomic.Uint64 +} + +type span struct { + TraceID string + SpanID string + ParentSpanID string + Name string + Kind int + Start time.Time + End time.Time + Attrs map[string]any + Error string +} + +func New(cfg config.OpenTelemetryConfig) *Exporter { + if !cfg.Enabled { + return nil + } + if cfg.BatchSize <= 0 { + cfg.BatchSize = 128 + } + if cfg.FlushInterval == 0 { + cfg.FlushInterval = config.Duration(2 * time.Second) + } + e := &Exporter{cfg: cfg, client: &http.Client{Timeout: 10 * time.Second}, ch: make(chan []span, cfg.BatchSize*4), stop: make(chan struct{}), done: make(chan struct{})} + go e.loop() + return e +} + +func (e *Exporter) NewTrace(traceparent string) Context { + if e == nil { + return Context{} + } + ratio := e.cfg.SampleRatio + if ratio <= 0 || ratio > 1 { + ratio = 1 + } + sampled := ratio >= 1 || randomFraction() < ratio + traceID, parent := parseTraceParent(traceparent) + if traceID == "" { + traceID = randomHex(16) + } + return Context{TraceID: traceID, RootSpanID: randomHex(8), ParentSpanID: parent, Sampled: sampled} +} +func (e *Exporter) TraceParent(c Context) string { + if e == nil || !c.Sampled { + return "" + } + return "00-" + c.TraceID + "-" + c.RootSpanID + "-01" +} +func (e *Exporter) Dropped() uint64 { + if e == nil { + return 0 + } + return e.dropped.Load() +} +func (e *Exporter) Failed() uint64 { + if e == nil { + return 0 + } + return e.failed.Load() +} +func (e *Exporter) Exported() uint64 { + if e == nil { + return 0 + } + return e.exported.Load() +} + +func (e *Exporter) Record(r Record) { + if e == nil || !r.Trace.Sampled || e.closed.Load() { + return + } + if r.Finished.IsZero() { + r.Finished = time.Now().UTC() + } + attrs := map[string]any{ + "gen_ai.operation.name": operationName(r.Path), + "gen_ai.provider.name": "ollama", + "gen_ai.request.model": r.Model, + "gen_ai.usage.input_tokens": r.PromptTokens, + "gen_ai.usage.output_tokens": r.OutputTokens, + "gen_ai.usage.cache_read.input_tokens": r.CachedTokens, + "http.request.method": "POST", + "http.response.status_code": r.Status, + "server.address": r.Worker, + "ollama.gateway.request_id": r.RequestID, + "ollama.gateway.api": r.API, + "ollama.gateway.tenant": r.Tenant, + "ollama.gateway.actor": r.Actor, + "ollama.gateway.application": r.Application, + "ollama.gateway.service_class": r.ServiceClass, + "ollama.gateway.credits": r.Credits, + } + if r.Alias != "" { + attrs["ollama.gateway.model_alias"] = r.Alias + } + if r.FirstByte > 0 { + attrs["gen_ai.response.time_to_first_chunk"] = r.FirstByte.Seconds() + } + root := span{TraceID: r.Trace.TraceID, SpanID: r.Trace.RootSpanID, ParentSpanID: r.Trace.ParentSpanID, Name: operationName(r.Path) + " " + r.Model, Kind: 2, Start: r.Started, End: r.Finished, Attrs: attrs, Error: r.Error} + spans := []span{root} + for _, iv := range r.Intervals { + if iv.Start.IsZero() || iv.End.IsZero() || iv.End.Before(iv.Start) { + continue + } + a := map[string]any{"ollama.gateway.request_id": r.RequestID} + for k, v := range iv.Attrs { + a[k] = v + } + spans = append(spans, span{TraceID: r.Trace.TraceID, SpanID: randomHex(8), ParentSpanID: r.Trace.RootSpanID, Name: iv.Name, Kind: 1, Start: iv.Start, End: iv.End, Attrs: a}) + } + select { + case e.ch <- spans: + default: + e.dropped.Add(uint64(len(spans))) + } +} + +func (e *Exporter) Close(ctx context.Context) error { + if e == nil || !e.closed.CompareAndSwap(false, true) { + return nil + } + close(e.stop) + select { + case <-e.done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (e *Exporter) loop() { + defer close(e.done) + tick := time.NewTicker(e.cfg.FlushInterval.Value()) + defer tick.Stop() + batch := make([]span, 0, e.cfg.BatchSize) + flush := func() { + if len(batch) == 0 { + return + } + if err := e.export(batch); err != nil { + e.failed.Add(uint64(len(batch))) + } else { + e.exported.Add(uint64(len(batch))) + } + batch = batch[:0] + } + for { + select { + case xs := <-e.ch: + batch = append(batch, xs...) + if len(batch) >= e.cfg.BatchSize { + flush() + } + case <-tick.C: + flush() + case <-e.stop: + for { + select { + case xs := <-e.ch: + batch = append(batch, xs...) + default: + flush() + return + } + } + } + } +} + +func (e *Exporter) export(spans []span) error { + endpoint := strings.TrimSpace(e.cfg.Endpoint) + if endpoint == "" { + return fmt.Errorf("empty OTLP endpoint") + } + u, err := url.Parse(endpoint) + if err != nil { + return err + } + if u.Path == "" || u.Path == "/" { + u.Path = "/v1/traces" + } else if !strings.HasSuffix(u.Path, "/v1/traces") { + u.Path = strings.TrimRight(u.Path, "/") + "/v1/traces" + } + payload := map[string]any{"resourceSpans": []any{map[string]any{"resource": map[string]any{"attributes": attrsToOTLP(map[string]any{"service.name": e.cfg.ServiceName, "service.version": e.cfg.ServiceVersion, "telemetry.sdk.name": "ollama-fair-gateway", "telemetry.sdk.language": "go"})}, "scopeSpans": []any{map[string]any{"scope": map[string]any{"name": "ollama-fair-gateway"}, "spans": spansToOTLP(spans)}}}}} + b, _ := json.Marshal(payload) + req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewReader(b)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + for k, v := range e.cfg.Headers { + req.Header.Set(k, v) + } + resp, err := e.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<20)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("OTLP HTTP %d", resp.StatusCode) + } + return nil +} + +func spansToOTLP(in []span) []any { + out := make([]any, 0, len(in)) + for _, s := range in { + x := map[string]any{"traceId": s.TraceID, "spanId": s.SpanID, "name": s.Name, "kind": s.Kind, "startTimeUnixNano": fmt.Sprintf("%d", s.Start.UnixNano()), "endTimeUnixNano": fmt.Sprintf("%d", s.End.UnixNano()), "attributes": attrsToOTLP(s.Attrs)} + if s.ParentSpanID != "" { + x["parentSpanId"] = s.ParentSpanID + } + if s.Error != "" { + x["status"] = map[string]any{"code": 2, "message": s.Error} + } + out = append(out, x) + } + return out +} +func attrsToOTLP(in map[string]any) []any { + keys := make([]string, 0, len(in)) + for k, v := range in { + if !empty(v) { + keys = append(keys, k) + } + } + sortStrings(keys) + out := make([]any, 0, len(keys)) + for _, k := range keys { + out = append(out, map[string]any{"key": k, "value": otlpValue(in[k])}) + } + return out +} +func otlpValue(v any) map[string]any { + switch x := v.(type) { + case string: + return map[string]any{"stringValue": x} + case bool: + return map[string]any{"boolValue": x} + case int: + return map[string]any{"intValue": fmt.Sprintf("%d", x)} + case int64: + return map[string]any{"intValue": fmt.Sprintf("%d", x)} + case uint64: + return map[string]any{"intValue": fmt.Sprintf("%d", x)} + case float64: + return map[string]any{"doubleValue": x} + default: + return map[string]any{"stringValue": fmt.Sprint(x)} + } +} +func empty(v any) bool { + switch x := v.(type) { + case string: + return x == "" + case int: + return x == 0 + case int64: + return x == 0 + case uint64: + return x == 0 + case float64: + return x == 0 + } + return false +} +func operationName(path string) string { + p := strings.ToLower(path) + switch { + case strings.Contains(p, "embed"): + return "embeddings" + case strings.Contains(p, "chat") || strings.Contains(p, "messages"): + return "chat" + default: + return "text_completion" + } +} +func randomHex(n int) string { b := make([]byte, n); _, _ = rand.Read(b); return hex.EncodeToString(b) } +func randomFraction() float64 { + b := make([]byte, 8) + _, _ = rand.Read(b) + var x uint64 + for _, z := range b { + x = x<<8 | uint64(z) + } + return float64(x>>11) / float64(uint64(1)<<53) +} +func parseTraceParent(v string) (string, string) { + parts := strings.Split(strings.TrimSpace(v), "-") + if len(parts) != 4 || len(parts[1]) != 32 || len(parts[2]) != 16 { + return "", "" + } + if _, e := hex.DecodeString(parts[1]); e != nil { + return "", "" + } + if _, e := hex.DecodeString(parts[2]); e != nil { + return "", "" + } + return strings.ToLower(parts[1]), strings.ToLower(parts[2]) +} +func sortStrings(x []string) { + for i := 1; i < len(x); i++ { + for j := i; j > 0 && x[j] < x[j-1]; j-- { + x[j], x[j-1] = x[j-1], x[j] + } + } +} diff --git a/internal/telemetry/otel_test.go b/internal/telemetry/otel_test.go new file mode 100644 index 0000000..39363ae --- /dev/null +++ b/internal/telemetry/otel_test.go @@ -0,0 +1,66 @@ +package telemetry + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +func TestOTLPHTTPJSONExportIsMetadataOnly(t *testing.T) { + bodyCh := make(chan []byte, 1) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/traces" { + t.Fatalf("path=%s", r.URL.Path) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Fatalf("content-type=%s", r.Header.Get("Content-Type")) + } + b, _ := io.ReadAll(r.Body) + bodyCh <- b + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + e := New(config.OpenTelemetryConfig{Enabled: true, Endpoint: ts.URL, ServiceName: "gateway-test", SampleRatio: 1, BatchSize: 1, FlushInterval: config.Duration(10 * time.Millisecond)}) + ctx := e.NewTrace("00-0123456789abcdef0123456789abcdef-0123456789abcdef-01") + start := time.Now().Add(-time.Second) + e.Record(Record{Trace: ctx, RequestID: "req-1", API: "openai", Path: "/v1/chat/completions", Tenant: "t", Actor: "u", ServiceClass: "interactive", Model: "qwen3:8b", Worker: "gpu", Started: start, Finished: time.Now(), FirstByte: 100 * time.Millisecond, Status: 200, PromptTokens: 12, OutputTokens: 7, Credits: 2.5, Intervals: []Interval{{Name: "gateway.queue", Start: start, End: start.Add(20 * time.Millisecond)}}}) + + var b []byte + select { + case b = <-bodyCh: + case <-time.After(2 * time.Second): + t.Fatal("no OTLP export") + } + if strings.Contains(string(b), "secret prompt") { + t.Fatal("content leaked") + } + var doc map[string]any + if err := json.Unmarshal(b, &doc); err != nil { + t.Fatal(err) + } + if len(doc["resourceSpans"].([]any)) != 1 { + t.Fatalf("bad resourceSpans: %s", b) + } + if !strings.Contains(string(b), "gen_ai.usage.input_tokens") || !strings.Contains(string(b), "gen_ai.response.time_to_first_chunk") { + t.Fatalf("missing GenAI attrs: %s", b) + } + if !strings.Contains(string(b), "0123456789abcdef0123456789abcdef") { + t.Fatalf("incoming trace id not propagated: %s", b) + } + cctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := e.Close(cctx); err != nil { + t.Fatal(err) + } + if e.Exported() == 0 { + t.Fatal("exported counter not incremented") + } +} diff --git a/internal/usage/recorder.go b/internal/usage/recorder.go new file mode 100644 index 0000000..98c74c7 --- /dev/null +++ b/internal/usage/recorder.go @@ -0,0 +1,420 @@ +package usage + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/cost" +) + +type Event struct { + ID string `json:"id"` + Time time.Time `json:"time"` + Tenant string `json:"tenant"` + Subject string `json:"subject"` + Actor string `json:"actor"` + Application string `json:"application,omitempty"` + ServiceClass string `json:"service_class,omitempty"` + AuthType string `json:"auth_type"` + ClientIP string `json:"client_ip,omitempty"` + API string `json:"api"` + Path string `json:"path"` + Model string `json:"model,omitempty"` + Worker string `json:"worker,omitempty"` + Status int `json:"status"` + QueueMS int64 `json:"queue_ms"` + ServiceMS int64 `json:"service_ms"` + EstimatedCredits float64 `json:"estimated_credits"` + ActualCredits float64 `json:"actual_credits"` + Usage cost.Usage `json:"usage"` + BytesIn int64 `json:"bytes_in"` + BytesOut int64 `json:"bytes_out"` +} + +type Summary struct { + Requests uint64 `json:"requests"` + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + Credits float64 `json:"credits"` + QueueMS int64 `json:"queue_ms"` + ServiceMS int64 `json:"service_ms"` + LastRequest time.Time `json:"last_request"` +} + +type Recorder struct { + dir string + journalCh chan Event + flush time.Duration + drop func() + mu sync.RWMutex + byActor map[string]Summary + byTenant map[string]Summary + global Summary + recent []Event + recentCap int + + retention RetentionConfig + compactMu sync.Mutex + retentionMu sync.RWMutex + retentionStatus RetentionStatus + rollupMu sync.RWMutex + dailyAgg map[string]RollupData + monthlyAgg map[string]RollupData + loaded bool + + closeOnce sync.Once + closeCh chan struct{} + flushCh chan chan error + wg sync.WaitGroup +} + +func New(dir string, buffer int, flush time.Duration, drop func()) (*Recorder, error) { + return NewWithRetention(dir, buffer, flush, RetentionConfig{}, drop) +} + +func NewWithRetention(dir string, buffer int, flush time.Duration, retention RetentionConfig, drop func()) (*Recorder, error) { + retention = normalizeRetention(retention) + r := &Recorder{dir: dir, journalCh: make(chan Event, buffer), flush: flush, drop: drop, byActor: map[string]Summary{}, byTenant: map[string]Summary{}, recentCap: 10000, closeCh: make(chan struct{}), flushCh: make(chan chan error), retention: retention, dailyAgg: map[string]RollupData{}, monthlyAgg: map[string]RollupData{}} + if dir != "" { + if err := os.MkdirAll(dir, 0750); err != nil { + return nil, err + } + // Compact before replay so startup does not load request-level history that + // is already outside the configured detail retention window. + if _, err := r.compactDisk(context.Background(), time.Now().UTC()); err != nil { + return nil, fmt.Errorf("usage retention startup compaction: %w", err) + } + if err := r.loadRollupsForReplay(); err != nil { + return nil, fmt.Errorf("replay usage rollups: %w", err) + } + if err := r.replayExisting(); err != nil { + return nil, err + } + r.loaded = true + r.wg.Add(2) + go r.journalLoop() + go r.retentionLoop() + } + return r, nil +} +func (r *Recorder) apply(e Event) { + actor := eventActor(e) + ak := e.Tenant + "\x00" + actor + r.byActor[ak] = add(r.byActor[ak], e) + r.byTenant[e.Tenant] = add(r.byTenant[e.Tenant], e) + r.global = add(r.global, e) + day := e.Time.UTC().Format("2006-01-02") + r.rollupMu.Lock() + d := r.dailyAgg[day] + if d.Tenants == nil { + d = newRollupData() + } + d.addEvent(e) + r.dailyAgg[day] = d + r.rollupMu.Unlock() + if r.recentCap > 0 { + if len(r.recent) >= r.recentCap { + copy(r.recent, r.recent[len(r.recent)-r.recentCap+1:]) + r.recent = r.recent[:r.recentCap-1] + } + r.recent = append(r.recent, e) + } +} + +func (r *Recorder) Record(e Event) { + r.mu.Lock() + r.apply(e) + r.mu.Unlock() + if r.dir != "" { + select { + case r.journalCh <- e: + default: + r.dropped() + } + } +} +func (r *Recorder) dropped() { + if r.drop != nil { + r.drop() + } +} +func eventActor(e Event) string { + if e.Actor != "" { + return e.Actor + } + return e.Subject +} + +func add(s Summary, e Event) Summary { + s.Requests++ + s.PromptTokens += e.Usage.PromptTokens + s.CompletionTokens += e.Usage.CompletionTokens + s.Credits += e.ActualCredits + s.QueueMS += e.QueueMS + s.ServiceMS += e.ServiceMS + s.LastRequest = e.Time + return s +} +func (r *Recorder) localActor(tenant, subject string) Summary { + r.mu.RLock() + defer r.mu.RUnlock() + return r.byActor[tenant+"\x00"+subject] +} +func (r *Recorder) localTenant(tenant string) Summary { + r.mu.RLock() + defer r.mu.RUnlock() + return r.byTenant[tenant] +} +func (r *Recorder) Actor(_ context.Context, tenant, subject string) Summary { + return r.localActor(tenant, subject) +} +func (r *Recorder) Tenant(_ context.Context, tenant string) Summary { + return r.localTenant(tenant) +} + +type NamedSummary struct { + Name string `json:"name"` + Summary Summary `json:"summary"` +} + +func (r *Recorder) SetRecentCapacity(n int) { + r.mu.Lock() + defer r.mu.Unlock() + if n < 0 { + n = 0 + } + r.recentCap = n + if n == 0 { + r.recent = nil + } else if len(r.recent) > n { + r.recent = append([]Event(nil), r.recent[len(r.recent)-n:]...) + } +} + +func (r *Recorder) Recent(limit int) []Event { + r.mu.RLock() + defer r.mu.RUnlock() + if limit <= 0 || limit > len(r.recent) { + limit = len(r.recent) + } + out := make([]Event, limit) + for i := 0; i < limit; i++ { + out[i] = r.recent[len(r.recent)-1-i] + } + return out +} + +func (r *Recorder) Global() Summary { + r.mu.RLock() + defer r.mu.RUnlock() + return r.global +} + +func (r *Recorder) LocalTenants() []NamedSummary { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]NamedSummary, 0, len(r.byTenant)) + for name, summary := range r.byTenant { + out = append(out, NamedSummary{Name: name, Summary: summary}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Summary.Credits == out[j].Summary.Credits { + return out[i].Name < out[j].Name + } + return out[i].Summary.Credits > out[j].Summary.Credits + }) + return out +} + +func (r *Recorder) LocalActors(tenant string) []NamedSummary { + r.mu.RLock() + defer r.mu.RUnlock() + prefix := tenant + "\x00" + out := []NamedSummary{} + for key, summary := range r.byActor { + if strings.HasPrefix(key, prefix) { + out = append(out, NamedSummary{Name: strings.TrimPrefix(key, prefix), Summary: summary}) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Summary.Credits == out[j].Summary.Credits { + return out[i].Name < out[j].Name + } + return out[i].Summary.Credits > out[j].Summary.Credits + }) + return out +} + +func (r *Recorder) journalLoop() { + defer r.wg.Done() + var f *os.File + var bw *bufio.Writer + day := "" + ticker := time.NewTicker(r.flush) + defer ticker.Stop() + open := func(now time.Time) error { + d := now.UTC().Format("2006-01-02") + if d == day && f != nil { + return nil + } + if bw != nil { + _ = bw.Flush() + } + if f != nil { + _ = f.Sync() + _ = f.Close() + } + path := filepath.Join(r.dir, "usage-"+d+".jsonl") + nf, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640) + if err != nil { + return err + } + f = nf + bw = bufio.NewWriterSize(f, 256<<10) + day = d + return nil + } + flushClose := func() { + if bw != nil { + _ = bw.Flush() + } + if f != nil { + _ = f.Sync() + _ = f.Close() + } + } + defer flushClose() + for { + select { + case e := <-r.journalCh: + if err := open(e.Time); err != nil { + r.dropped() + continue + } + b, _ := json.Marshal(e) + _, _ = bw.Write(b) + _ = bw.WriteByte('\n') + case <-ticker.C: + if bw != nil { + _ = bw.Flush() + } + if f != nil { + _ = f.Sync() + } + case ack := <-r.flushCh: + // A flush is a barrier for events that were already queued before the + // request. Since journalCh and flushCh are independent channels, drain + // the event queue explicitly before flushing the buffered writer. + for draining := true; draining; { + select { + case e := <-r.journalCh: + if err := open(e.Time); err != nil { + r.dropped() + continue + } + b, _ := json.Marshal(e) + _, _ = bw.Write(b) + _ = bw.WriteByte('\n') + default: + draining = false + } + } + var err error + if bw != nil { + err = bw.Flush() + } + if err == nil && f != nil { + err = f.Sync() + } + ack <- err + case <-r.closeCh: + for { + select { + case e := <-r.journalCh: + if err := open(e.Time); err == nil { + b, _ := json.Marshal(e) + _, _ = bw.Write(b) + _ = bw.WriteByte('\n') + } + default: + return + } + } + } + } +} +func (r *Recorder) replayExisting() error { + paths, err := filepath.Glob(filepath.Join(r.dir, "usage-*.jsonl")) + if err != nil { + return err + } + sort.Strings(paths) + for _, path := range paths { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("replay usage journal %s: %w", path, err) + } + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 64<<10), 2<<20) + for sc.Scan() { + var e Event + if err := json.Unmarshal(sc.Bytes(), &e); err != nil { + continue + } // tolerate a partial/crashed final line + r.apply(e) + } + err = sc.Err() + _ = f.Close() + if err != nil { + return fmt.Errorf("replay usage journal %s: %w", path, err) + } + } + return nil +} + +func (r *Recorder) Flush(ctx context.Context) error { + if r == nil || r.dir == "" { + return nil + } + ack := make(chan error, 1) + select { + case r.flushCh <- ack: + case <-ctx.Done(): + return ctx.Err() + } + select { + case err := <-ack: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r *Recorder) Close() { + if r == nil || r.dir == "" { + return + } + r.closeOnce.Do(func() { close(r.closeCh); r.wg.Wait() }) +} + +func (r *Recorder) Health(context.Context) error { + if r.dir == "" { + return nil + } + test := filepath.Join(r.dir, ".health") + f, err := os.OpenFile(test, os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("usage journal: %w", err) + } + _ = f.Close() + _ = os.Remove(test) + return nil +} diff --git a/internal/usage/recorder_test.go b/internal/usage/recorder_test.go new file mode 100644 index 0000000..50f6cb0 --- /dev/null +++ b/internal/usage/recorder_test.go @@ -0,0 +1,240 @@ +package usage + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/cost" +) + +func TestRecorderAggregatesByEffectiveActor(t *testing.T) { + r, err := New("", 8, time.Second, nil) + if err != nil { + t.Fatal(err) + } + r.Record(Event{Time: time.Now(), Tenant: "t", Subject: "shared-key-subject", Actor: "app:svc-a", ActualCredits: 2}) + if got := r.Actor(context.Background(), "t", "app:svc-a"); got.Requests != 1 || got.Credits != 2 { + t.Fatalf("actor summary=%#v", got) + } + if got := r.Actor(context.Background(), "t", "shared-key-subject"); got.Requests != 0 { + t.Fatalf("subject unexpectedly used as actor: %#v", got) + } +} + +func TestRecorderReplaysPersistentJournal(t *testing.T) { + dir := t.TempDir() + r, err := New(dir, 32, 10*time.Millisecond, nil) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + r.Record(Event{ID: "a", Time: now, Tenant: "team", Subject: "user", Actor: "user", ActualCredits: 3.5, Usage: cost.Usage{PromptTokens: 100, CompletionTokens: 20}}) + r.Close() + r2, err := New(dir, 32, 10*time.Millisecond, nil) + if err != nil { + t.Fatal(err) + } + defer r2.Close() + s := r2.Global() + if s.Requests != 1 || s.PromptTokens != 100 || s.CompletionTokens != 20 || s.Credits != 3.5 { + t.Fatalf("summary=%#v", s) + } + recent := r2.Recent(10) + if len(recent) != 1 || recent[0].ID != "a" { + t.Fatalf("recent=%#v", recent) + } +} + +func TestFlushPersistsBufferedEvent(t *testing.T) { + dir := t.TempDir() + r, err := New(dir, 8, time.Hour, nil) + if err != nil { + t.Fatal(err) + } + defer r.Close() + r.Record(Event{ID: "flush-1", Time: time.Now().UTC(), Tenant: "t", Subject: "s", Actor: "s", Status: 200}) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := r.Flush(ctx); err != nil { + t.Fatal(err) + } + paths, err := filepath.Glob(filepath.Join(dir, "usage-*.jsonl")) + if err != nil || len(paths) != 1 { + t.Fatalf("journal paths: %v %v", paths, err) + } + b, err := os.ReadFile(paths[0]) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(b, []byte(`"id":"flush-1"`)) { + t.Fatalf("event not flushed: %s", b) + } +} + +func TestRetentionCompactsDetailToDailyAndPreservesTotals(t *testing.T) { + dir := t.TempDir() + r, err := NewWithRetention(dir, 32, 10*time.Millisecond, RetentionConfig{DetailDays: 30, DailyDays: 400, CompactionInterval: time.Hour}, nil) + if err != nil { + t.Fatal(err) + } + base := time.Now().UTC().AddDate(0, 0, -5).Truncate(24 * time.Hour) + r.Record(Event{ID: "old-a", Time: base, Tenant: "team-a", Actor: "user-a", Application: "openwebui", Model: "qwen3:8b", Worker: "gpu-a", Status: 200, ActualCredits: 4.5, QueueMS: 12, ServiceMS: 500, Usage: cost.Usage{PromptTokens: 1000, CompletionTokens: 200, PromptEvalNS: int64(time.Second), EvalNS: int64(2 * time.Second)}}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := r.Flush(ctx); err != nil { + t.Fatal(err) + } + future := base.AddDate(0, 0, 40) + st, err := r.CompactAt(ctx, future) + if err != nil { + t.Fatal(err) + } + if st.LastRawCompacted != 1 { + t.Fatalf("raw compacted=%d", st.LastRawCompacted) + } + if paths, _ := filepath.Glob(filepath.Join(dir, "usage-*.jsonl")); len(paths) != 0 { + t.Fatalf("raw journals remain: %v", paths) + } + daily, _ := filepath.Glob(filepath.Join(dir, "rollups", "daily", "rollup-daily-*.json")) + if len(daily) != 1 { + t.Fatalf("daily rollups=%v", daily) + } + if got := r.Global(); got.Requests != 1 || got.PromptTokens != 1000 || got.Credits != 4.5 { + t.Fatalf("live summary after compaction=%#v", got) + } + r.Close() + + // A restart must reconstruct all-time totals from the rollup even though the + // request-level journal no longer exists. + r2, err := NewWithRetention(dir, 32, time.Second, RetentionConfig{DetailDays: 30, DailyDays: 400, CompactionInterval: time.Hour}, nil) + if err != nil { + t.Fatal(err) + } + defer r2.Close() + got := r2.Global() + if got.Requests != 1 || got.PromptTokens != 1000 || got.CompletionTokens != 200 || got.Credits != 4.5 { + t.Fatalf("replayed summary=%#v", got) + } + if recent := r2.Recent(10); len(recent) != 0 { + t.Fatalf("compacted detail unexpectedly replayed: %#v", recent) + } + pts := r2.Series("daily", "tenant", "team-a", 30) + if len(pts) != 1 || pts[0].Requests != 1 || pts[0].OutputTPS != 100 { + t.Fatalf("daily points=%#v", pts) + } +} + +func TestRetentionFoldsDailyIntoIdempotentMonthlyRollup(t *testing.T) { + dir := t.TempDir() + r, err := NewWithRetention(dir, 32, 10*time.Millisecond, RetentionConfig{DetailDays: 2, DailyDays: 4, CompactionInterval: time.Hour}, nil) + if err != nil { + t.Fatal(err) + } + base := time.Now().UTC().AddDate(0, 0, -1).Truncate(24 * time.Hour) + r.Record(Event{ID: "m-a", Time: base, Tenant: "t", Actor: "a", Model: "m", Worker: "w", Status: 500, ActualCredits: 2, Usage: cost.Usage{PromptTokens: 50, CompletionTokens: 10, EvalNS: int64(time.Second)}}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := r.Flush(ctx); err != nil { + t.Fatal(err) + } + future := base.AddDate(0, 0, 10) + if _, err := r.CompactAt(ctx, future); err != nil { + t.Fatal(err) + } + monthly, _ := filepath.Glob(filepath.Join(dir, "rollups", "monthly", "rollup-monthly-*.json")) + if len(monthly) != 1 { + t.Fatalf("monthly rollups=%v", monthly) + } + if daily, _ := filepath.Glob(filepath.Join(dir, "rollups", "daily", "*.json")); len(daily) != 0 { + t.Fatalf("daily remains=%v", daily) + } + // Running compaction again must not duplicate the monthly contribution. + if _, err := r.CompactAt(ctx, future); err != nil { + t.Fatal(err) + } + r.Close() + r2, err := NewWithRetention(dir, 32, time.Second, RetentionConfig{DetailDays: 2, DailyDays: 4, CompactionInterval: time.Hour}, nil) + if err != nil { + t.Fatal(err) + } + defer r2.Close() + if got := r2.Global(); got.Requests != 1 || got.Credits != 2 { + t.Fatalf("monthly replay duplicated: %#v", got) + } + pts := r2.Series("monthly", "global", "", 12) + if len(pts) != 1 || pts[0].Requests != 1 || pts[0].Errors != 1 { + t.Fatalf("monthly points=%#v", pts) + } +} + +func TestRetentionExpiresMonthlyRollupsAndUpdatesAllTimeTotals(t *testing.T) { + dir := t.TempDir() + cfg := RetentionConfig{DetailDays: 1, DailyDays: 2, MonthlyMonths: 2, CompactionInterval: time.Hour} + r, err := NewWithRetention(dir, 32, 10*time.Millisecond, cfg, nil) + if err != nil { + t.Fatal(err) + } + base := time.Date(2026, 1, 15, 12, 0, 0, 0, time.UTC) + r.Record(Event{ID: "jan", Time: base, Tenant: "team", Actor: "alice", Application: "ui", Model: "qwen3:8b", Worker: "gpu", Status: 200, ActualCredits: 5, Usage: cost.Usage{PromptTokens: 100, CompletionTokens: 20}}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := r.Flush(ctx); err != nil { + t.Fatal(err) + } + // By April, January is older than the configured two-month monthly window. + if _, err := r.CompactAt(ctx, time.Date(2026, 4, 20, 0, 0, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + if files, _ := filepath.Glob(filepath.Join(dir, "rollups", "monthly", "rollup-monthly-*.json")); len(files) != 0 { + t.Fatalf("expired monthly rollup still present: %v", files) + } + if got := r.Global(); got.Requests != 0 || got.PromptTokens != 0 || got.Credits != 0 { + t.Fatalf("expired monthly data still counted in all-time totals: %#v", got) + } + r.Close() + + r2, err := NewWithRetention(dir, 32, time.Second, cfg, nil) + if err != nil { + t.Fatal(err) + } + defer r2.Close() + if got := r2.Global(); got.Requests != 0 || got.Credits != 0 { + t.Fatalf("expired monthly data replayed after restart: %#v", got) + } +} + +func TestRetentionSeriesDimensions(t *testing.T) { + dir := t.TempDir() + r, err := NewWithRetention(dir, 32, 10*time.Millisecond, RetentionConfig{DetailDays: 1, DailyDays: 100, CompactionInterval: time.Hour}, nil) + if err != nil { + t.Fatal(err) + } + defer r.Close() + base := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + r.Record(Event{ID: "dims", Time: base, Tenant: "tenant-a", Actor: "actor-a", Application: "openwebui", Model: "gemma4:e4b", Worker: "rtx-4090", Status: 200, ActualCredits: 3, Usage: cost.Usage{PromptTokens: 200, CompletionTokens: 50, PromptEvalNS: int64(time.Second), EvalNS: int64(time.Second)}}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := r.Flush(ctx); err != nil { + t.Fatal(err) + } + if _, err := r.CompactAt(ctx, base.AddDate(0, 0, 10)); err != nil { + t.Fatal(err) + } + checks := []struct{ dim, name string }{ + {"tenant", "tenant-a"}, + {"actor", "tenant-a\x00actor-a"}, + {"application", "openwebui"}, + {"model", "gemma4:e4b"}, + {"worker", "rtx-4090"}, + } + for _, tc := range checks { + pts := r.Series("daily", tc.dim, tc.name, 10) + if len(pts) != 1 || pts[0].Requests != 1 || pts[0].Credits != 3 { + t.Fatalf("series %s/%q = %#v", tc.dim, tc.name, pts) + } + } +} diff --git a/internal/usage/retention.go b/internal/usage/retention.go new file mode 100644 index 0000000..dc041dc --- /dev/null +++ b/internal/usage/retention.go @@ -0,0 +1,693 @@ +package usage + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// RetentionConfig controls the tiered usage history. DetailDays keeps raw +// per-request JSONL journals. Older detail is converted to daily rollups. +// DailyDays is the maximum age of daily rollups; older days are folded into +// idempotent monthly rollups. MonthlyMonths=0 keeps monthly rollups forever. +type RetentionConfig struct { + DetailDays int + DailyDays int + MonthlyMonths int + CompactionInterval time.Duration +} + +type Aggregate struct { + Requests uint64 `json:"requests"` + Errors uint64 `json:"errors,omitempty"` + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + CachedPromptTokens int64 `json:"cached_prompt_tokens,omitempty"` + Credits float64 `json:"credits"` + QueueMS int64 `json:"queue_ms"` + ServiceMS int64 `json:"service_ms"` + PromptEvalNS int64 `json:"prompt_eval_ns,omitempty"` + EvalNS int64 `json:"eval_ns,omitempty"` + BytesIn int64 `json:"bytes_in,omitempty"` + BytesOut int64 `json:"bytes_out,omitempty"` + LastRequest time.Time `json:"last_request,omitempty"` +} + +func (a Aggregate) PromptTPS() float64 { + if a.PromptEvalNS <= 0 { + return 0 + } + return float64(a.PromptTokens) / (float64(a.PromptEvalNS) / 1e9) +} +func (a Aggregate) OutputTPS() float64 { + if a.EvalNS <= 0 { + return 0 + } + return float64(a.CompletionTokens) / (float64(a.EvalNS) / 1e9) +} +func (a *Aggregate) addEvent(e Event) { + a.Requests++ + if e.Status >= 400 { + a.Errors++ + } + a.PromptTokens += e.Usage.PromptTokens + a.CompletionTokens += e.Usage.CompletionTokens + a.CachedPromptTokens += e.Usage.CachedPromptTokens + a.Credits += e.ActualCredits + a.QueueMS += e.QueueMS + a.ServiceMS += e.ServiceMS + a.PromptEvalNS += e.Usage.PromptEvalNS + a.EvalNS += e.Usage.EvalNS + a.BytesIn += e.BytesIn + a.BytesOut += e.BytesOut + if e.Time.After(a.LastRequest) { + a.LastRequest = e.Time + } +} +func (a *Aggregate) merge(b Aggregate) { + a.Requests += b.Requests + a.Errors += b.Errors + a.PromptTokens += b.PromptTokens + a.CompletionTokens += b.CompletionTokens + a.CachedPromptTokens += b.CachedPromptTokens + a.Credits += b.Credits + a.QueueMS += b.QueueMS + a.ServiceMS += b.ServiceMS + a.PromptEvalNS += b.PromptEvalNS + a.EvalNS += b.EvalNS + a.BytesIn += b.BytesIn + a.BytesOut += b.BytesOut + if b.LastRequest.After(a.LastRequest) { + a.LastRequest = b.LastRequest + } +} + +type RollupData struct { + Global Aggregate `json:"global"` + Tenants map[string]Aggregate `json:"tenants,omitempty"` + Actors map[string]Aggregate `json:"actors,omitempty"` + Applications map[string]Aggregate `json:"applications,omitempty"` + Models map[string]Aggregate `json:"models,omitempty"` + Workers map[string]Aggregate `json:"workers,omitempty"` +} + +func newRollupData() RollupData { + return RollupData{Tenants: map[string]Aggregate{}, Actors: map[string]Aggregate{}, Applications: map[string]Aggregate{}, Models: map[string]Aggregate{}, Workers: map[string]Aggregate{}} +} +func addMap(m map[string]Aggregate, key string, e Event) { + if key == "" { + return + } + x := m[key] + x.addEvent(e) + m[key] = x +} +func mergeMap(dst map[string]Aggregate, src map[string]Aggregate) { + for k, v := range src { + x := dst[k] + x.merge(v) + dst[k] = x + } +} +func (d *RollupData) addEvent(e Event) { + d.Global.addEvent(e) + addMap(d.Tenants, e.Tenant, e) + actor := eventActor(e) + if e.Tenant != "" && actor != "" { + addMap(d.Actors, e.Tenant+"\x00"+actor, e) + } + addMap(d.Applications, e.Application, e) + addMap(d.Models, e.Model, e) + addMap(d.Workers, e.Worker, e) +} +func (d *RollupData) merge(o RollupData) { + d.Global.merge(o.Global) + mergeMap(d.Tenants, o.Tenants) + mergeMap(d.Actors, o.Actors) + mergeMap(d.Applications, o.Applications) + mergeMap(d.Models, o.Models) + mergeMap(d.Workers, o.Workers) +} + +type DailyRollup struct { + Version int `json:"version"` + Granularity string `json:"granularity"` + Day string `json:"day"` + GeneratedAt time.Time `json:"generated_at"` + Data RollupData `json:"data"` +} + +type MonthlyRollup struct { + Version int `json:"version"` + Granularity string `json:"granularity"` + Month string `json:"month"` + GeneratedAt time.Time `json:"generated_at"` + // Days makes the monthly update idempotent if the process crashes after + // writing the month but before removing the source daily file. + Days map[string]RollupData `json:"days"` +} + +func (m MonthlyRollup) Total() RollupData { + out := newRollupData() + keys := make([]string, 0, len(m.Days)) + for k := range m.Days { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + out.merge(m.Days[k]) + } + return out +} + +type RollupPoint struct { + Period string `json:"period"` + Requests uint64 `json:"requests"` + Errors uint64 `json:"errors"` + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + Credits float64 `json:"credits"` + QueueMS int64 `json:"queue_ms"` + ServiceMS int64 `json:"service_ms"` + PromptTPS float64 `json:"prompt_tps"` + OutputTPS float64 `json:"output_tps"` +} + +func point(period string, a Aggregate) RollupPoint { + return RollupPoint{Period: period, Requests: a.Requests, Errors: a.Errors, PromptTokens: a.PromptTokens, CompletionTokens: a.CompletionTokens, Credits: a.Credits, QueueMS: a.QueueMS, ServiceMS: a.ServiceMS, PromptTPS: a.PromptTPS(), OutputTPS: a.OutputTPS()} +} + +type RetentionStatus struct { + Enabled bool `json:"enabled"` + DetailDays int `json:"detail_days"` + DailyDays int `json:"daily_days"` + MonthlyMonths int `json:"monthly_months"` + CompactionInterval string `json:"compaction_interval"` + RawFiles int `json:"raw_files"` + RawBytes int64 `json:"raw_bytes"` + DailyFiles int `json:"daily_files"` + DailyBytes int64 `json:"daily_bytes"` + MonthlyFiles int `json:"monthly_files"` + MonthlyBytes int64 `json:"monthly_bytes"` + LastCompaction time.Time `json:"last_compaction,omitempty"` + LastError string `json:"last_error,omitempty"` + LastReclaimedBytes int64 `json:"last_reclaimed_bytes,omitempty"` + LastRawCompacted int `json:"last_raw_compacted,omitempty"` + LastDailyCompacted int `json:"last_daily_compacted,omitempty"` +} + +func normalizeRetention(c RetentionConfig) RetentionConfig { + if c.DetailDays <= 0 { + c.DetailDays = 30 + } + if c.DailyDays <= 0 { + c.DailyDays = 400 + } + if c.DailyDays < c.DetailDays { + c.DailyDays = c.DetailDays + } + if c.CompactionInterval <= 0 { + c.CompactionInterval = 6 * time.Hour + } + return c +} +func (r *Recorder) rollupDirs() (string, string) { + return filepath.Join(r.dir, "rollups", "daily"), filepath.Join(r.dir, "rollups", "monthly") +} + +func atomicJSON(path string, v any) error { + if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil { + return err + } + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + f, err := os.CreateTemp(filepath.Dir(path), ".tmp-*") + if err != nil { + return err + } + tmp := f.Name() + defer os.Remove(tmp) + if err = f.Chmod(0640); err == nil { + _, err = f.Write(b) + } + if err == nil { + err = f.Sync() + } + cerr := f.Close() + if err == nil { + err = cerr + } + if err != nil { + return err + } + if err = os.Rename(tmp, path); err != nil { + return err + } + if d, e := os.Open(filepath.Dir(path)); e == nil { + _ = d.Sync() + _ = d.Close() + } + return nil +} +func readJSON(path string, v any) error { + b, err := os.ReadFile(path) + if err != nil { + return err + } + return json.Unmarshal(b, v) +} + +func parseDayFromRaw(path string) (time.Time, bool) { + base := filepath.Base(path) + if !strings.HasPrefix(base, "usage-") || !strings.HasSuffix(base, ".jsonl") { + return time.Time{}, false + } + s := strings.TrimSuffix(strings.TrimPrefix(base, "usage-"), ".jsonl") + t, err := time.Parse("2006-01-02", s) + return t, err == nil +} +func parseDayFromDaily(path string) (time.Time, bool) { + base := filepath.Base(path) + if !strings.HasPrefix(base, "rollup-daily-") || !strings.HasSuffix(base, ".json") { + return time.Time{}, false + } + s := strings.TrimSuffix(strings.TrimPrefix(base, "rollup-daily-"), ".json") + t, err := time.Parse("2006-01-02", s) + return t, err == nil +} +func cutoffDays(now time.Time, days int) time.Time { + today := time.Date(now.UTC().Year(), now.UTC().Month(), now.UTC().Day(), 0, 0, 0, 0, time.UTC) + return today.AddDate(0, 0, -days+1) +} +func cutoffMonths(now time.Time, months int) time.Time { + n := now.UTC() + first := time.Date(n.Year(), n.Month(), 1, 0, 0, 0, 0, time.UTC) + return first.AddDate(0, -months+1, 0) +} + +func aggregateRaw(path string) (DailyRollup, error) { + day, ok := parseDayFromRaw(path) + if !ok { + return DailyRollup{}, fmt.Errorf("invalid journal filename %s", path) + } + d := newRollupData() + f, err := os.Open(path) + if err != nil { + return DailyRollup{}, err + } + defer f.Close() + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 64<<10), 2<<20) + for sc.Scan() { + var e Event + if json.Unmarshal(sc.Bytes(), &e) == nil { + d.addEvent(e) + } + } + if err := sc.Err(); err != nil { + return DailyRollup{}, err + } + return DailyRollup{Version: 1, Granularity: "daily", Day: day.Format("2006-01-02"), GeneratedAt: time.Now().UTC(), Data: d}, nil +} + +func (r *Recorder) compactDisk(ctx context.Context, now time.Time) (RetentionStatus, error) { + if r.dir == "" { + return RetentionStatus{}, nil + } + r.compactMu.Lock() + defer r.compactMu.Unlock() + c := r.retention + dailyDir, monthlyDir := r.rollupDirs() + _ = os.MkdirAll(dailyDir, 0750) + _ = os.MkdirAll(monthlyDir, 0750) + before := dirSize(r.dir) + stat := RetentionStatus{Enabled: true, DetailDays: c.DetailDays, DailyDays: c.DailyDays, MonthlyMonths: c.MonthlyMonths, CompactionInterval: c.CompactionInterval.String()} + detailCut := cutoffDays(now, c.DetailDays) + raws, _ := filepath.Glob(filepath.Join(r.dir, "usage-*.jsonl")) + sort.Strings(raws) + for _, p := range raws { + select { + case <-ctx.Done(): + return stat, ctx.Err() + default: + } + day, ok := parseDayFromRaw(p) + if !ok || !day.Before(detailCut) { + continue + } + roll, err := aggregateRaw(p) + if err != nil { + return stat, err + } + out := filepath.Join(dailyDir, "rollup-daily-"+roll.Day+".json") + if err := atomicJSON(out, roll); err != nil { + return stat, err + } + if err := os.Remove(p); err != nil { + return stat, err + } + stat.LastRawCompacted++ + } + dailyCut := cutoffDays(now, c.DailyDays) + ds, _ := filepath.Glob(filepath.Join(dailyDir, "rollup-daily-*.json")) + sort.Strings(ds) + for _, p := range ds { + select { + case <-ctx.Done(): + return stat, ctx.Err() + default: + } + day, ok := parseDayFromDaily(p) + if !ok || !day.Before(dailyCut) { + continue + } + var d DailyRollup + if err := readJSON(p, &d); err != nil { + return stat, err + } + month := day.Format("2006-01") + mp := filepath.Join(monthlyDir, "rollup-monthly-"+month+".json") + m := MonthlyRollup{Version: 1, Granularity: "monthly", Month: month, GeneratedAt: time.Now().UTC(), Days: map[string]RollupData{}} + if err := readJSON(mp, &m); err != nil && !errors.Is(err, os.ErrNotExist) { + return stat, err + } + if m.Days == nil { + m.Days = map[string]RollupData{} + } + m.Version = 1 + m.Granularity = "monthly" + m.Month = month + m.GeneratedAt = time.Now().UTC() + m.Days[d.Day] = d.Data + if err := atomicJSON(mp, m); err != nil { + return stat, err + } + if err := os.Remove(p); err != nil { + return stat, err + } + stat.LastDailyCompacted++ + } + if c.MonthlyMonths > 0 { + cut := cutoffMonths(now, c.MonthlyMonths) + ms, _ := filepath.Glob(filepath.Join(monthlyDir, "rollup-monthly-*.json")) + for _, p := range ms { + base := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(p), "rollup-monthly-"), ".json") + mt, err := time.Parse("2006-01", base) + if err != nil || !mt.Before(cut) { + continue + } + var expired MonthlyRollup + _ = readJSON(p, &expired) + if err := os.Remove(p); err != nil { + return stat, err + } + if r.loaded && expired.Days != nil { + r.removeAggregate(expired.Total()) + } + } + } + after := dirSize(r.dir) + if before > after { + stat.LastReclaimedBytes = before - after + } + stat.LastCompaction = time.Now().UTC() + r.setRetentionStatus(stat) + return stat, nil +} + +func dirSize(dir string) int64 { + var n int64 + _ = filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error { + if err == nil && info.Mode().IsRegular() { + n += info.Size() + } + return nil + }) + return n +} +func fileCounts(pattern string) (int, int64) { + ps, _ := filepath.Glob(pattern) + var n int64 + for _, p := range ps { + if s, e := os.Stat(p); e == nil && s.Mode().IsRegular() { + n += s.Size() + } + } + return len(ps), n +} +func (r *Recorder) setRetentionStatus(s RetentionStatus) { + r.retentionMu.Lock() + r.retentionStatus = s + r.retentionMu.Unlock() +} +func (r *Recorder) RetentionStatus() RetentionStatus { + r.retentionMu.RLock() + s := r.retentionStatus + r.retentionMu.RUnlock() + if r.dir == "" { + return s + } + d, m := r.rollupDirs() + s.RawFiles, s.RawBytes = fileCounts(filepath.Join(r.dir, "usage-*.jsonl")) + s.DailyFiles, s.DailyBytes = fileCounts(filepath.Join(d, "rollup-daily-*.json")) + s.MonthlyFiles, s.MonthlyBytes = fileCounts(filepath.Join(m, "rollup-monthly-*.json")) + return s +} +func (r *Recorder) Compact(ctx context.Context) (RetentionStatus, error) { + if r == nil || r.dir == "" { + return RetentionStatus{}, nil + } + if err := r.Flush(ctx); err != nil { + return r.RetentionStatus(), err + } + stat, err := r.compactDisk(ctx, time.Now().UTC()) + if err != nil { + stat.LastError = err.Error() + r.setRetentionStatus(stat) + return stat, err + } + // Refresh only the rollup index used for historical charts. All-time + // counters are already correct in memory and must not be applied twice. + _ = r.reloadRollupIndex() + return r.RetentionStatus(), nil +} +func (r *Recorder) retentionLoop() { + defer r.wg.Done() + t := time.NewTicker(r.retention.CompactionInterval) + defer t.Stop() + for { + select { + case <-r.closeCh: + return + case <-t.C: + ctx, c := context.WithTimeout(context.Background(), 30*time.Minute) + _, err := r.compactDisk(ctx, time.Now().UTC()) + if err == nil { + _ = r.reloadRollupIndex() + } + c() + if err != nil { + st := r.RetentionStatus() + st.LastError = err.Error() + r.setRetentionStatus(st) + } + } + } +} + +func (r *Recorder) loadRollupsForReplay() error { + dailyDir, monthlyDir := r.rollupDirs() + ms, _ := filepath.Glob(filepath.Join(monthlyDir, "rollup-monthly-*.json")) + sort.Strings(ms) + for _, p := range ms { + var m MonthlyRollup + if err := readJSON(p, &m); err != nil { + return err + } + d := m.Total() + r.applyAggregate(d) + r.monthlyAgg[m.Month] = d + } + ds, _ := filepath.Glob(filepath.Join(dailyDir, "rollup-daily-*.json")) + sort.Strings(ds) + for _, p := range ds { + var d DailyRollup + if err := readJSON(p, &d); err != nil { + return err + } + r.applyAggregate(d.Data) + r.dailyAgg[d.Day] = d.Data + } + return nil +} +func (r *Recorder) reloadRollupIndex() error { + daily := map[string]RollupData{} + monthly := map[string]RollupData{} + dd, md := r.rollupDirs() + ms, _ := filepath.Glob(filepath.Join(md, "rollup-monthly-*.json")) + for _, p := range ms { + var m MonthlyRollup + if readJSON(p, &m) == nil { + monthly[m.Month] = m.Total() + } + } + ds, _ := filepath.Glob(filepath.Join(dd, "rollup-daily-*.json")) + for _, p := range ds { + var d DailyRollup + if readJSON(p, &d) == nil { + daily[d.Day] = d.Data + } + } + // Preserve in-memory raw/detail days; replace only days that are now on disk rollups. + r.rollupMu.Lock() + for day, v := range r.dailyAgg { + if _, ok := daily[day]; !ok { + if t, e := time.Parse("2006-01-02", day); e == nil && !t.Before(cutoffDays(time.Now().UTC(), r.retention.DetailDays)) { + daily[day] = v + } + } + } + r.dailyAgg = daily + r.monthlyAgg = monthly + r.rollupMu.Unlock() + return nil +} +func (r *Recorder) applyAggregate(d RollupData) { + r.global = mergeSummary(r.global, d.Global) + for k, a := range d.Tenants { + r.byTenant[k] = mergeSummary(r.byTenant[k], a) + } + for k, a := range d.Actors { + r.byActor[k] = mergeSummary(r.byActor[k], a) + } +} +func (r *Recorder) removeAggregate(d RollupData) { + r.mu.Lock() + defer r.mu.Unlock() + r.global = subtractSummary(r.global, d.Global) + for k, a := range d.Tenants { + x := subtractSummary(r.byTenant[k], a) + if x.Requests == 0 { + delete(r.byTenant, k) + } else { + r.byTenant[k] = x + } + } + for k, a := range d.Actors { + x := subtractSummary(r.byActor[k], a) + if x.Requests == 0 { + delete(r.byActor, k) + } else { + r.byActor[k] = x + } + } +} +func subtractSummary(s Summary, a Aggregate) Summary { + if a.Requests >= s.Requests { + s.Requests = 0 + } else { + s.Requests -= a.Requests + } + s.PromptTokens -= a.PromptTokens + if s.PromptTokens < 0 { + s.PromptTokens = 0 + } + s.CompletionTokens -= a.CompletionTokens + if s.CompletionTokens < 0 { + s.CompletionTokens = 0 + } + s.Credits -= a.Credits + if s.Credits < 0 { + s.Credits = 0 + } + s.QueueMS -= a.QueueMS + if s.QueueMS < 0 { + s.QueueMS = 0 + } + s.ServiceMS -= a.ServiceMS + if s.ServiceMS < 0 { + s.ServiceMS = 0 + } + if !a.LastRequest.IsZero() && a.LastRequest.Equal(s.LastRequest) { + s.LastRequest = time.Time{} + } + return s +} +func mergeSummary(s Summary, a Aggregate) Summary { + s.Requests += a.Requests + s.PromptTokens += a.PromptTokens + s.CompletionTokens += a.CompletionTokens + s.Credits += a.Credits + s.QueueMS += a.QueueMS + s.ServiceMS += a.ServiceMS + if a.LastRequest.After(s.LastRequest) { + s.LastRequest = a.LastRequest + } + return s +} + +func chooseAggregate(d RollupData, dimension, name string) Aggregate { + switch dimension { + case "tenant": + return d.Tenants[name] + case "actor": + return d.Actors[name] + case "application": + return d.Applications[name] + case "model": + return d.Models[name] + case "worker": + return d.Workers[name] + default: + return d.Global + } +} +func (r *Recorder) Series(granularity, dimension, name string, limit int) []RollupPoint { + if limit <= 0 { + limit = 90 + } + if limit > 2000 { + limit = 2000 + } + r.rollupMu.RLock() + defer r.rollupMu.RUnlock() + src := r.dailyAgg + if granularity == "monthly" { + src = r.monthlyAgg + } + keys := make([]string, 0, len(src)) + for k := range src { + keys = append(keys, k) + } + sort.Strings(keys) + if len(keys) > limit { + keys = keys[len(keys)-limit:] + } + out := make([]RollupPoint, 0, len(keys)) + for _, k := range keys { + out = append(out, point(k, chooseAggregate(src[k], dimension, name))) + } + return out +} + +// CompactAt is intentionally exported for deterministic retention tests. +func (r *Recorder) CompactAt(ctx context.Context, now time.Time) (RetentionStatus, error) { + if err := r.Flush(ctx); err != nil { + return r.RetentionStatus(), err + } + s, e := r.compactDisk(ctx, now) + if e == nil { + _ = r.reloadRollupIndex() + } + return s, e +} diff --git a/internal/warm/manager.go b/internal/warm/manager.go new file mode 100644 index 0000000..4d92900 --- /dev/null +++ b/internal/warm/manager.go @@ -0,0 +1,608 @@ +package warm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "sort" + "strings" + "sync" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/state" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +type Action struct { + Time time.Time `json:"time"` + Type string `json:"type"` + Worker string `json:"worker"` + Model string `json:"model"` + Policy string `json:"policy,omitempty"` + Status string `json:"status"` + Message string `json:"message,omitempty"` +} + +type Suggestion struct { + Worker string `json:"worker"` + Model string `json:"model"` + Class string `json:"class"` + Reason string `json:"reason"` + LastUsed time.Time `json:"last_used,omitempty"` + VRAMPercent float64 `json:"vram_percent,omitempty"` +} + +type Status struct { + Enabled bool `json:"enabled"` + Override bool `json:"override"` + Baseline map[string]config.WarmModelPolicy `json:"baseline"` + Policies map[string]config.WarmModelPolicy `json:"policies"` + Actions []Action `json:"actions"` + Suggestions []Suggestion `json:"eviction_suggestions"` + LastReconcile time.Time `json:"last_reconcile,omitempty"` + LastError string `json:"last_error,omitempty"` +} + +type persistentFile struct { + Override bool `json:"override"` + Policies map[string]config.WarmModelPolicy `json:"policies,omitempty"` + Actions []Action `json:"actions,omitempty"` +} + +type Manager struct { + mu sync.RWMutex + + cfg config.WarmModelsConfig + pool *worker.Pool + file state.AtomicJSON + client *http.Client + baseline map[string]config.WarmModelPolicy + policies map[string]config.WarmModelPolicy + override bool + + lastUse map[string]time.Time + seenLoaded map[string]bool + inFlight map[string]bool + actions []Action + suggestions []Suggestion + lastReconcile time.Time + lastError string + wake chan struct{} + sem chan struct{} + wg sync.WaitGroup +} + +func New(cfg config.WarmModelsConfig, pool *worker.Pool, path string) (*Manager, error) { + m := &Manager{ + cfg: cfg, + pool: pool, + file: state.AtomicJSON{Path: path, Mode: 0600}, + client: &http.Client{Timeout: cfg.OperationTimeout.Value()}, + baseline: clonePolicies(cfg.Policies), + policies: clonePolicies(cfg.Policies), + lastUse: map[string]time.Time{}, + seenLoaded: map[string]bool{}, + inFlight: map[string]bool{}, + wake: make(chan struct{}, 1), + sem: make(chan struct{}, 2), + } + var pf persistentFile + if err := m.file.Load(&pf); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else if err == nil { + if pf.Override { + if err := config.ValidateWarmModelPolicies(pf.Policies, workerNames(pool)); err != nil { + return nil, fmt.Errorf("load warm model policies: %w", err) + } + m.override = true + m.policies = clonePolicies(pf.Policies) + } + m.actions = append([]Action(nil), pf.Actions...) + if len(m.actions) > 200 { + m.actions = m.actions[len(m.actions)-200:] + } + } + return m, nil +} + +func workerNames(p *worker.Pool) map[string]bool { + out := map[string]bool{} + if p == nil { + return out + } + for _, s := range p.Snapshots() { + out[s.Name] = true + } + return out +} + +func clonePolicies(in map[string]config.WarmModelPolicy) map[string]config.WarmModelPolicy { + out := make(map[string]config.WarmModelPolicy, len(in)) + for k, v := range in { + v.Workers = append([]string(nil), v.Workers...) + out[k] = v + } + return out +} + +func (m *Manager) Start(ctx context.Context) { + if m == nil || !m.cfg.Enabled { + return + } + go func() { + m.Reconcile(ctx) + t := time.NewTicker(m.cfg.ReconcileInterval.Value()) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + m.Reconcile(ctx) + case <-m.wake: + m.Reconcile(ctx) + } + } + }() +} + +// Wait blocks until currently scheduled preload/unload actions have finished. +// It is primarily useful for graceful shutdowns and deterministic tests. +func (m *Manager) Wait(ctx context.Context) error { + if m == nil { + return nil + } + done := make(chan struct{}) + go func() { m.wg.Wait(); close(done) }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (m *Manager) Wake() { + if m == nil { + return + } + select { + case m.wake <- struct{}{}: + default: + } +} + +func (m *Manager) Touch(workerName, model string) { + if m == nil || workerName == "" || model == "" { + return + } + m.mu.Lock() + m.lastUse[key(workerName, model)] = time.Now().UTC() + m.mu.Unlock() +} + +func (m *Manager) Status() Status { + if m == nil { + return Status{} + } + m.mu.RLock() + defer m.mu.RUnlock() + a := append([]Action(nil), m.actions...) + sort.Slice(a, func(i, j int) bool { return a[i].Time.After(a[j].Time) }) + if len(a) > 100 { + a = a[:100] + } + return Status{Enabled: m.cfg.Enabled, Override: m.override, Baseline: clonePolicies(m.baseline), Policies: clonePolicies(m.policies), Actions: a, Suggestions: append([]Suggestion(nil), m.suggestions...), LastReconcile: m.lastReconcile, LastError: m.lastError} +} + +func (m *Manager) SetPolicies(p map[string]config.WarmModelPolicy) error { + if m == nil { + return errors.New("warm manager unavailable") + } + if err := config.ValidateWarmModelPolicies(p, workerNames(m.pool)); err != nil { + return err + } + m.mu.Lock() + old, oldOverride := m.policies, m.override + m.policies, m.override = clonePolicies(p), true + err := m.saveLocked() + if err != nil { + m.policies, m.override = old, oldOverride + } + m.mu.Unlock() + if err == nil { + m.Wake() + } + return err +} + +func (m *Manager) Reset() error { + if m == nil { + return errors.New("warm manager unavailable") + } + m.mu.Lock() + old, oldOverride := m.policies, m.override + m.policies, m.override = clonePolicies(m.baseline), false + err := m.saveLocked() + if err != nil { + m.policies, m.override = old, oldOverride + } + m.mu.Unlock() + if err == nil { + m.Wake() + } + return err +} + +func (m *Manager) Reconcile(ctx context.Context) { + if m == nil || !m.cfg.Enabled || m.pool == nil { + return + } + placements := m.pool.PlacementSnapshots() + snaps := m.pool.Snapshots() + pByName := map[string]worker.PlacementSnapshot{} + for _, p := range placements { + pByName[p.Worker] = p + } + now := time.Now().UTC() + + m.mu.Lock() + policies := clonePolicies(m.policies) + for _, s := range snaps { + for _, lm := range s.LoadedModels { + model := loadedName(lm) + if model == "" { + continue + } + k := key(s.Name, model) + if !m.seenLoaded[k] { + m.seenLoaded[k] = true + if m.lastUse[k].IsZero() { + m.lastUse[k] = now + } + } + } + } + lastUse := make(map[string]time.Time, len(m.lastUse)) + for k, v := range m.lastUse { + lastUse[k] = v + } + m.mu.Unlock() + + models := installedModels(placements) + for _, model := range models { + pattern, pol, ok := selectPolicy(policies, model) + if !ok { + continue + } + eligible := eligibleWorkers(model, pol, snaps, pByName, m.pool) + if pol.Class == "hot" || (pol.Class == "warm" && pol.Preload) { + n := pol.Replicas + if n <= 0 { + n = 1 + } + if n > len(eligible) { + n = len(eligible) + } + for i := 0; i < n; i++ { + if !isLoaded(eligible[i], model) { + m.startAction(ctx, "preload", eligible[i].Name, model, pattern) + } + } + } + if pol.Class == "warm" || pol.Class == "cold" { + idle := pol.IdleTimeout.Value() + for _, s := range snaps { + if s.Maintenance != "active" || !policyTargetsWorker(pol, s.Name) || !isLoaded(s, model) || modelActive(s, model) > 0 { + continue + } + lu := lastUse[key(s.Name, model)] + if lu.IsZero() { + lu = now + } + if idle <= 0 || now.Sub(lu) >= idle { + m.startAction(ctx, "unload", s.Name, model, pattern) + } + } + } + } + + suggestions := evictionSuggestions(snaps, policies, lastUse) + m.mu.Lock() + m.suggestions = suggestions + m.lastReconcile = now + m.lastError = "" + m.mu.Unlock() +} + +func (m *Manager) startAction(parent context.Context, action, workerName, model, pattern string) { + k := action + "\x00" + workerName + "\x00" + model + m.mu.Lock() + if m.inFlight[k] { + m.mu.Unlock() + return + } + m.inFlight[k] = true + a := Action{Time: time.Now().UTC(), Type: action, Worker: workerName, Model: model, Policy: pattern, Status: "running"} + m.actions = append(m.actions, a) + m.trimActionsLocked() + _ = m.saveLocked() + m.mu.Unlock() + m.wg.Add(1) + go func() { + defer m.wg.Done() + m.sem <- struct{}{} + defer func() { <-m.sem }() + ctx, cancel := context.WithTimeout(parent, m.cfg.OperationTimeout.Value()) + defer cancel() + release, err := m.pool.BeginModelMaintenance(workerName, model) + if err == nil { + defer release() + err = m.modelAction(ctx, action, workerName, model) + } + m.mu.Lock() + delete(m.inFlight, k) + status, msg := "completed", "success" + if err != nil { + status, msg = "failed", err.Error() + m.lastError = err.Error() + } + for i := len(m.actions) - 1; i >= 0; i-- { + if m.actions[i].Type == action && m.actions[i].Worker == workerName && m.actions[i].Model == model && m.actions[i].Status == "running" { + m.actions[i].Status = status + m.actions[i].Message = msg + m.actions[i].Time = time.Now().UTC() + break + } + } + if action == "preload" && err == nil { + m.lastUse[key(workerName, model)] = time.Now().UTC() + } + m.trimActionsLocked() + _ = m.saveLocked() + m.mu.Unlock() + if err == nil { + m.Wake() + } + }() +} + +func (m *Manager) modelAction(ctx context.Context, action, workerName, model string) error { + base, ok := m.pool.URLFor(workerName) + if !ok { + return fmt.Errorf("unknown worker %q", workerName) + } + if mode, ok := m.pool.Maintenance(workerName); !ok || mode != "active" { + return fmt.Errorf("worker %s is not active", workerName) + } + if action == "preload" { + pd, ok := m.pool.PlacementDecision(workerName, model) + if !ok || !pd.Allowed { + return fmt.Errorf("model %s is blocked by placement on worker %s", model, workerName) + } + for _, ps := range m.pool.PlacementSnapshots() { + if ps.Worker != workerName || !ps.InventoryKnown { + continue + } + found := false + for _, installed := range ps.InstalledModels { + if sameModel(installed, model) { + found = true + break + } + } + if !found { + return fmt.Errorf("model %s is not installed on worker %s", model, workerName) + } + } + } + keep := any(-1) + if action == "unload" { + keep = 0 + } + body, _ := json.Marshal(map[string]any{"model": model, "prompt": "", "keep_alive": keep, "stream": false}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(base.String(), "/")+"/api/generate", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := m.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + return fmt.Errorf("Ollama HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))) + } + return nil +} + +func (m *Manager) saveLocked() error { + return m.file.Save(persistentFile{Override: m.override, Policies: clonePolicies(m.policies), Actions: append([]Action(nil), m.actions...)}) +} +func (m *Manager) trimActionsLocked() { + if len(m.actions) > 200 { + m.actions = append([]Action(nil), m.actions[len(m.actions)-200:]...) + } +} + +func loadedName(m worker.LoadedModel) string { + if strings.TrimSpace(m.Model) != "" { + return strings.TrimSpace(m.Model) + } + return strings.TrimSpace(m.Name) +} +func key(workerName, model string) string { return workerName + "\x00" + model } +func canonical(s string) string { return strings.TrimSuffix(strings.TrimSpace(s), ":latest") } +func sameModel(a, b string) bool { return a == b || canonical(a) == canonical(b) } +func isLoaded(s worker.Snapshot, model string) bool { + for _, lm := range s.LoadedModels { + if sameModel(loadedName(lm), model) { + return true + } + } + return false +} +func modelActive(s worker.Snapshot, model string) int { + if n := s.ModelActive[model]; n > 0 { + return n + } + for k, n := range s.ModelActive { + if sameModel(k, model) { + return n + } + } + return 0 +} + +func installedModels(ps []worker.PlacementSnapshot) []string { + set := map[string]bool{} + for _, p := range ps { + for _, m := range p.InstalledModels { + if strings.TrimSpace(m) != "" { + set[m] = true + } + } + } + out := make([]string, 0, len(set)) + for m := range set { + out = append(out, m) + } + sort.Strings(out) + return out +} + +func match(pattern, model string) (int, bool) { + pattern = strings.TrimSpace(pattern) + if pattern == "*" { + return 0, true + } + if strings.HasSuffix(pattern, "*") { + p := strings.TrimSuffix(pattern, "*") + if strings.HasPrefix(model, p) { + return len(p), true + } + return -1, false + } + if sameModel(pattern, model) { + return 100000 + len(pattern), true + } + return -1, false +} +func selectPolicy(ps map[string]config.WarmModelPolicy, model string) (string, config.WarmModelPolicy, bool) { + best := -1 + var bp string + var out config.WarmModelPolicy + for p, v := range ps { + if sp, ok := match(p, model); ok && sp > best { + best, bp, out = sp, p, v + } + } + return bp, out, best >= 0 +} + +func policyTargetsWorker(pol config.WarmModelPolicy, workerName string) bool { + if len(pol.Workers) == 0 { + return true + } + for _, w := range pol.Workers { + if w == workerName { + return true + } + } + return false +} + +func eligibleWorkers(model string, pol config.WarmModelPolicy, snaps []worker.Snapshot, placements map[string]worker.PlacementSnapshot, pool *worker.Pool) []worker.Snapshot { + allowedNames := map[string]bool{} + if len(pol.Workers) > 0 { + for _, w := range pol.Workers { + allowedNames[w] = true + } + } + out := []worker.Snapshot{} + for _, s := range snaps { + if !s.Healthy || s.Maintenance != "active" || s.CircuitState == "open" { + continue + } + if len(allowedNames) > 0 && !allowedNames[s.Name] { + continue + } + pd, ok := pool.PlacementDecision(s.Name, model) + if !ok || !pd.Allowed { + continue + } + p := placements[s.Name] + if p.InventoryKnown { + found := false + for _, m := range p.InstalledModels { + if sameModel(m, model) { + found = true + break + } + } + if !found { + continue + } + } + out = append(out, s) + } + sort.Slice(out, func(i, j int) bool { + li, lj := isLoaded(out[i], model), isLoaded(out[j], model) + if li != lj { + return li + } + ri := float64(out[i].Active) / float64(max(1, out[i].MaxConcurrent)) + rj := float64(out[j].Active) / float64(max(1, out[j].MaxConcurrent)) + if ri != rj { + return ri < rj + } + return out[i].Name < out[j].Name + }) + return out +} + +func evictionSuggestions(snaps []worker.Snapshot, policies map[string]config.WarmModelPolicy, last map[string]time.Time) []Suggestion { + out := []Suggestion{} + for _, s := range snaps { + if s.VRAMTotalBytes <= 0 || s.VRAMUsedBytes <= 0 { + continue + } + pct := 100 * float64(s.VRAMUsedBytes) / float64(s.VRAMTotalBytes) + if pct < 90 { + continue + } + for _, lm := range s.LoadedModels { + m := loadedName(lm) + _, p, ok := selectPolicy(policies, m) + if !ok || p.Class == "hot" || modelActive(s, m) > 0 { + continue + } + out = append(out, Suggestion{Worker: s.Name, Model: m, Class: p.Class, Reason: "VRAM pressure; inactive non-hot model", LastUsed: last[key(s.Name, m)], VRAMPercent: pct}) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Class != out[j].Class { + return out[i].Class == "cold" + } + return out[i].LastUsed.Before(out[j].LastUsed) + }) + if len(out) > 50 { + out = out[:50] + } + return out +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/internal/warm/manager_test.go b/internal/warm/manager_test.go new file mode 100644 index 0000000..88805c7 --- /dev/null +++ b/internal/warm/manager_test.go @@ -0,0 +1,179 @@ +package warm + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/worker" +) + +func TestHotPolicyPreloadsInstalledEligibleModel(t *testing.T) { + var mu sync.Mutex + var keep any + called := make(chan struct{}, 1) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "qwen3:8b", "model": "qwen3:8b"}}}) + case "/api/generate": + var in map[string]any + _ = json.NewDecoder(r.Body).Decode(&in) + mu.Lock() + keep = in["keep_alive"] + mu.Unlock() + select { + case called <- struct{}{}: + default: + } + _ = json.NewEncoder(w).Encode(map[string]any{"done": true}) + default: + http.NotFound(w, r) + } + })) + defer ts.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p := worker.New([]config.WorkerConfig{{Name: "w", URL: ts.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}}, "w") + p.Start(ctx) + m, err := New(config.WarmModelsConfig{Enabled: true, OperationTimeout: config.Duration(time.Second), Policies: map[string]config.WarmModelPolicy{"qwen3:*": {Class: "hot", Replicas: 1, IdleTimeout: config.Duration(time.Hour)}}}, p, t.TempDir()+"/warm.json") + if err != nil { + t.Fatal(err) + } + m.Reconcile(ctx) + select { + case <-called: + case <-time.After(2 * time.Second): + t.Fatal("preload was not issued") + } + waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second) + defer waitCancel() + if err := m.Wait(waitCtx); err != nil { + t.Fatal(err) + } + mu.Lock() + got := keep + mu.Unlock() + if n, ok := got.(float64); !ok || n != -1 { + t.Fatalf("keep_alive=%#v want -1", got) + } +} + +func TestColdPolicyUnloadsAfterIdle(t *testing.T) { + called := make(chan struct{}, 2) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "qwen3:8b", "model": "qwen3:8b"}}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "qwen3:8b", "model": "qwen3:8b"}}}) + case "/api/generate": + var in map[string]any + _ = json.NewDecoder(r.Body).Decode(&in) + if v, ok := in["keep_alive"].(float64); ok && v == 0 { + select { + case called <- struct{}{}: + default: + } + } + _ = json.NewEncoder(w).Encode(map[string]any{"done": true}) + default: + http.NotFound(w, r) + } + })) + defer ts.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p := worker.New([]config.WorkerConfig{{Name: "w", URL: ts.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}}, "w") + p.Start(ctx) + m, err := New(config.WarmModelsConfig{Enabled: true, OperationTimeout: config.Duration(time.Second), Policies: map[string]config.WarmModelPolicy{"qwen3:8b": {Class: "cold", Replicas: 1, IdleTimeout: config.Duration(10 * time.Millisecond)}}}, p, t.TempDir()+"/warm.json") + if err != nil { + t.Fatal(err) + } + m.Reconcile(ctx) // establishes last-use grace + time.Sleep(20 * time.Millisecond) + m.Reconcile(ctx) + select { + case <-called: + case <-time.After(2 * time.Second): + t.Fatal("unload was not issued") + } + waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second) + defer waitCancel() + if err := m.Wait(waitCtx); err != nil { + t.Fatal(err) + } +} + +func TestRuntimePoliciesPersist(t *testing.T) { + dir := t.TempDir() + path := dir + "/warm.json" + p := worker.New([]config.WorkerConfig{{Name: "w", URL: "http://127.0.0.1:1", MaxConcurrent: 1}}, "w") + base := config.WarmModelsConfig{Policies: map[string]config.WarmModelPolicy{"a:*": {Class: "warm", Replicas: 1, IdleTimeout: config.Duration(time.Minute)}}} + m, err := New(base, p, path) + if err != nil { + t.Fatal(err) + } + over := map[string]config.WarmModelPolicy{"b:*": {Class: "hot", Replicas: 1, Workers: []string{"w"}, IdleTimeout: config.Duration(time.Minute)}} + if err := m.SetPolicies(over); err != nil { + t.Fatal(err) + } + m2, err := New(base, p, path) + if err != nil { + t.Fatal(err) + } + st := m2.Status() + if !st.Override || st.Policies["b:*"].Class != "hot" { + t.Fatalf("status=%#v", st) + } + if err := m2.Reset(); err != nil { + t.Fatal(err) + } + if m2.Status().Override { + t.Fatal("override still active") + } +} + +func TestColdPolicyDoesNotUnloadDrainingWorker(t *testing.T) { + called := make(chan struct{}, 1) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "qwen3:8b", "model": "qwen3:8b"}}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "qwen3:8b", "model": "qwen3:8b"}}}) + case "/api/generate": + called <- struct{}{} + _ = json.NewEncoder(w).Encode(map[string]any{"done": true}) + default: + http.NotFound(w, r) + } + })) + defer ts.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p := worker.New([]config.WorkerConfig{{Name: "w", URL: ts.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour)}}, "w") + p.Start(ctx) + if err := p.SetMaintenance("w", "draining"); err != nil { + t.Fatal(err) + } + m, err := New(config.WarmModelsConfig{Enabled: true, OperationTimeout: config.Duration(time.Second), Policies: map[string]config.WarmModelPolicy{"qwen3:8b": {Class: "cold", Replicas: 1, IdleTimeout: config.Duration(time.Millisecond)}}}, p, t.TempDir()+"/warm.json") + if err != nil { + t.Fatal(err) + } + m.Reconcile(ctx) + time.Sleep(5 * time.Millisecond) + m.Reconcile(ctx) + select { + case <-called: + t.Fatal("warm manager issued model action while worker was draining") + case <-time.After(100 * time.Millisecond): + } +} diff --git a/internal/webui/assets/app.css b/internal/webui/assets/app.css new file mode 100644 index 0000000..6245c86 --- /dev/null +++ b/internal/webui/assets/app.css @@ -0,0 +1,101 @@ +:root{--bg:#eef2f6;--surface:#ffffff;--surface2:#f5f7fa;--surface3:#e8edf3;--border:#d6dde6;--border-strong:#c5cfda;--text:#253345;--muted:#66778a;--accent:#147fa5;--accent2:#6b5bb5;--danger:#b83d4b;--warn:#a56600;--ok:#1a805f;--good:#1a805f;--panel:#ffffff;--cyan-soft:#e1f1f6;--violet-soft:#eeeafb;--shadow:0 10px 28px rgba(39,57,77,.08);--radius:10px;--radius-sm:7px;--font-ui:"IBM Plex Sans Condensed","Roboto Condensed","Arial Narrow",Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;--font-mono:"IBM Plex Mono","JetBrains Mono","Cascadia Mono",SFMono-Regular,Consolas,"Liberation Mono",monospace;font-family:var(--font-ui);color-scheme:light} +*{box-sizing:border-box}html,body{margin:0;min-height:100%;background:var(--bg);color:var(--text)}body{font-size:14px}.hidden{display:none!important}button,input,select{font:inherit}button{color:inherit}.app-shell{min-height:100vh}.sidebar{position:fixed;inset:0 auto 0 0;width:248px;border-right:1px solid var(--border);background:linear-gradient(180deg,#0d1118,#0a0d12);padding:22px 14px;display:flex;flex-direction:column;z-index:20}.brand{display:flex;gap:11px;align-items:center;padding:0 8px 22px}.brand-mark{display:grid;place-items:center;width:37px;height:37px;border-radius:11px;background:linear-gradient(145deg,var(--accent),var(--accent2));color:#07100c;font-weight:900;letter-spacing:-1px}.brand strong{display:block;font-size:14px}.brand small{display:block;color:var(--muted);margin-top:2px}.brand-large{padding:0;margin-bottom:26px}.brand-large .brand-mark{width:44px;height:44px}.nav{display:flex;flex-direction:column;gap:4px}.nav button{background:transparent;border:0;border-radius:10px;text-align:left;padding:10px 12px;color:var(--muted);cursor:pointer;display:flex;align-items:center;gap:10px}.nav button span{width:18px;text-align:center;color:#738196}.nav button:hover{background:var(--surface2);color:var(--text)}.nav button.active{background:var(--surface3);color:var(--text);box-shadow:inset 0 0 0 1px rgba(255,255,255,.03)}.nav button.active span{color:var(--accent)}.sidebar-foot{margin-top:auto;padding:16px 8px 2px;border-top:1px solid var(--border)}.status-line{display:flex;align-items:center;gap:8px;color:var(--muted);font-size:12px;margin-bottom:10px}.dot{width:8px;height:8px;border-radius:50%;background:var(--warn);box-shadow:0 0 0 3px rgba(255,197,109,.1)}.status-line.ok .dot{background:var(--ok);box-shadow:0 0 0 3px rgba(117,230,181,.1)}.status-line.bad .dot{background:var(--danger);box-shadow:0 0 0 3px rgba(255,114,125,.1)}.text-button,.link-button{border:0;background:none;color:var(--muted);cursor:pointer;padding:0}.text-button:hover,.link-button:hover{color:var(--text)}.main{margin-left:248px;min-height:100vh;padding:26px 34px 60px}.topbar{display:flex;justify-content:space-between;align-items:center;margin-bottom:26px}.topbar h1{font-size:27px;letter-spacing:-.6px;margin:0 0 4px}.topbar p{margin:0;color:var(--muted)}.top-actions{display:flex;align-items:center;gap:10px}.identity-pill{padding:7px 12px;border:1px solid var(--border);border-radius:11px;background:var(--surface);text-align:right}.identity-pill span,.identity-pill small{display:block}.identity-pill small{color:var(--muted);font-size:11px;margin-top:2px}.icon-button{width:37px;height:37px;border:1px solid var(--border);border-radius:10px;background:var(--surface);cursor:pointer}.icon-button:hover{background:var(--surface2)}.page{display:none}.page.active{display:block}.kpi-grid{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));gap:12px;margin-bottom:14px}.kpi-grid.four{grid-template-columns:repeat(4,minmax(0,1fr))}.kpi{background:linear-gradient(160deg,var(--surface2),var(--surface));border:1px solid var(--border);border-radius:var(--radius);padding:17px 18px;min-width:0}.kpi>span,.metric-panel>span{display:block;color:var(--muted);font-size:12px;margin-bottom:9px}.kpi strong,.metric-panel strong{font-size:25px;letter-spacing:-.5px}.kpi small{display:block;color:#687688;margin-top:6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.grid{display:grid;gap:14px;margin-bottom:14px}.grid.two{grid-template-columns:1fr 1fr}.grid.three{grid-template-columns:repeat(3,1fr)}.grid.two-thirds{grid-template-columns:minmax(0,2fr) minmax(280px,1fr)}.panel{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:18px;box-shadow:0 1px 0 rgba(255,255,255,.02)}.panel-head{display:flex;align-items:flex-start;justify-content:space-between;gap:14px;margin-bottom:16px}.panel-head h2{margin:0;font-size:15px}.panel-head p{margin:4px 0 0;color:var(--muted);font-size:12px;line-height:1.45}.metric-panel{padding:20px}.badge{display:inline-flex;padding:4px 8px;border-radius:999px;background:rgba(117,230,181,.09);color:var(--accent);font-size:11px}.chart{height:245px;position:relative}.chart svg{width:100%;height:100%;display:block}.chart .axis{stroke:#283344;stroke-width:1}.chart .area{fill:url(#areaGradient)}.chart .line{stroke:var(--accent2);fill:none;stroke-width:2}.chart .credit-line{stroke:var(--accent);fill:none;stroke-width:2}.chart text{fill:#66768a;font-size:10px}.health-list{display:flex;flex-direction:column;gap:10px}.health-item{display:flex;align-items:center;gap:10px;padding:10px 0;border-bottom:1px solid var(--border)}.health-item:last-child{border-bottom:0}.health-item .grow{flex:1}.health-item strong{display:block;font-size:13px}.health-item small{color:var(--muted)}.state-dot{width:9px;height:9px;border-radius:50%;background:var(--danger)}.state-dot.ok{background:var(--ok)}.bar-list{display:flex;flex-direction:column;gap:13px}.bar-row .bar-meta{display:flex;justify-content:space-between;gap:10px;margin-bottom:6px;font-size:12px}.bar-row .bar-meta span:last-child{color:var(--muted)}.bar-track{height:5px;border-radius:10px;background:var(--surface3);overflow:hidden}.bar-fill{height:100%;background:linear-gradient(90deg,var(--accent2),var(--accent));border-radius:10px}.table-wrap{width:100%;overflow:auto}.table-wrap table{width:100%;border-collapse:collapse;min-width:720px}.table-wrap.compact table{min-width:500px}.table-wrap th{text-align:left;color:#6f7e91;font-size:10px;text-transform:uppercase;letter-spacing:.07em;font-weight:700;padding:8px 10px;border-bottom:1px solid var(--border)}.table-wrap td{padding:10px;border-bottom:1px solid rgba(37,46,59,.7);vertical-align:middle}.table-wrap tr:last-child td{border-bottom:0}.table-wrap tbody tr:hover{background:rgba(255,255,255,.018)}.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px}.muted{color:var(--muted)}.status-chip{display:inline-flex;padding:3px 7px;border-radius:999px;font-size:10px;background:var(--surface3);color:var(--muted)}.status-chip.ok{background:rgba(117,230,181,.09);color:var(--ok)}.status-chip.bad{background:rgba(255,114,125,.09);color:var(--danger)}.status-chip.warn{background:rgba(255,197,109,.09);color:var(--warn)}.worker-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(280px,1fr));gap:14px;margin-bottom:14px}.worker-card{background:linear-gradient(160deg,var(--surface2),var(--surface));border:1px solid var(--border);border-radius:var(--radius);padding:18px}.worker-title{display:flex;align-items:center;gap:9px;margin-bottom:16px}.worker-title strong{font-size:15px}.worker-title small{color:var(--muted);display:block;margin-top:3px;max-width:210px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.slot-row{display:flex;align-items:center;justify-content:space-between;margin-bottom:8px}.slot-bar{height:7px;background:var(--surface3);border-radius:10px;overflow:hidden}.slot-fill{height:100%;background:var(--accent2)}.model-tags{display:flex;flex-wrap:wrap;gap:6px;margin-top:14px}.model-tag{font-size:10px;padding:4px 7px;border-radius:7px;background:var(--surface3);color:#a5b2c3}.toolbar{display:flex;align-items:center;gap:10px;margin-bottom:14px}.toolbar .grow{flex:1}.search input{min-width:300px}input,select{border:1px solid var(--border);border-radius:9px;background:var(--surface2);color:var(--text);padding:9px 11px;outline:none}input:focus,select:focus{border-color:#426b91;box-shadow:0 0 0 3px rgba(119,184,255,.08)}.button{border:1px solid var(--border);border-radius:9px;background:var(--surface2);color:var(--text);padding:9px 13px;cursor:pointer}.button:hover{background:var(--surface3)}.button.primary{border-color:transparent;background:var(--accent);color:#08120e;font-weight:700}.button.danger{background:rgba(255,114,125,.08);color:var(--danger);border-color:rgba(255,114,125,.22)}.button.small{padding:5px 8px;font-size:11px}.button.wide{width:100%;display:block;text-align:center;text-decoration:none}.actions{display:flex;gap:6px}.flow-diagram{display:flex;flex-direction:column;align-items:center;gap:6px;padding:4px 0}.flow-diagram div{width:80%;text-align:center;padding:10px;border:1px solid var(--border);background:var(--surface2);border-radius:9px}.flow-diagram span{color:var(--muted)}.operation-list{display:flex;flex-direction:column;gap:10px}.operation{padding:13px;border:1px solid var(--border);border-radius:11px;background:var(--surface2)}.operation-top{display:flex;justify-content:space-between;gap:10px;margin-bottom:9px}.operation-top strong{font-size:13px}.operation-top small{display:block;color:var(--muted);margin-top:3px}.progress{height:6px;background:var(--surface3);border-radius:10px;overflow:hidden;margin:8px 0}.progress div{height:100%;background:var(--accent);border-radius:10px;transition:width .2s}.code-block{margin:0;max-height:70vh;overflow:auto;background:#080b10;border:1px solid #1d2530;border-radius:11px;padding:16px;color:#b8c6d8;font:12px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace}.login-shell{min-height:100vh;display:grid;place-items:center;padding:24px;background:radial-gradient(circle at 50% 20%,#122030 0,#0a0d12 42%)}.login-card{width:min(430px,100%);border:1px solid var(--border);background:rgba(16,21,29,.96);border-radius:20px;padding:30px;box-shadow:var(--shadow)}.login-card h1{font-size:26px;margin:0 0 8px}.login-card>p{line-height:1.6}.stack{display:flex;flex-direction:column;gap:12px}.stack label,.dialog-card label{display:flex;flex-direction:column;gap:6px;color:var(--muted);font-size:12px}.divider{display:flex;align-items:center;gap:10px;margin:18px 0;color:#5f6d80;font-size:11px}.divider:before,.divider:after{content:"";height:1px;background:var(--border);flex:1}.fineprint{font-size:11px;color:#657386!important;margin-top:18px}.callout{padding:10px 12px;border-radius:9px;margin-top:12px}.callout.danger{background:rgba(255,114,125,.08);color:var(--danger);border:1px solid rgba(255,114,125,.18)}.toast-root{position:fixed;right:20px;bottom:20px;z-index:100;display:flex;flex-direction:column;gap:8px}.toast{padding:10px 13px;border:1px solid var(--border);border-radius:10px;background:var(--surface3);box-shadow:var(--shadow);animation:toastIn .18s ease}.toast.bad{border-color:rgba(255,114,125,.3);color:var(--danger)}@keyframes toastIn{from{transform:translateY(5px);opacity:0}to{transform:none;opacity:1}}dialog{border:0;padding:0;background:transparent;color:var(--text)}dialog::backdrop{background:rgba(0,0,0,.62);backdrop-filter:blur(3px)}.dialog-card{width:min(620px,calc(100vw - 30px));background:var(--surface);border:1px solid var(--border);border-radius:16px;padding:20px;box-shadow:var(--shadow)}.dialog-card label{margin-bottom:13px}.dialog-card input,.dialog-card select{width:100%}.dialog-actions{display:flex;justify-content:flex-end;gap:8px;margin-top:18px}.form-grid{display:grid;grid-template-columns:1fr 1fr;gap:0 12px}.form-grid .wide{grid-column:1/-1}.empty{color:var(--muted);padding:20px 5px;text-align:center}.security-card h3{margin:0 0 10px;font-size:14px}.kv{display:grid;grid-template-columns:150px 1fr;gap:8px 14px}.kv div:nth-child(odd){color:var(--muted)}code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace} +@media(max-width:1180px){.kpi-grid{grid-template-columns:repeat(3,1fr)}.grid.two-thirds{grid-template-columns:1fr}.kpi-grid.four{grid-template-columns:repeat(2,1fr)}} +@media(max-width:780px){.sidebar{width:70px;padding:18px 9px}.brand{padding:0 7px 18px}.brand>div{display:none}.nav button{font-size:0;padding:11px;justify-content:center}.nav button span{font-size:16px}.sidebar-foot{padding:12px 4px}.status-line span:last-child,.text-button{display:none}.main{margin-left:70px;padding:20px 16px 45px}.identity-pill{display:none}.kpi-grid,.kpi-grid.four,.grid.two,.grid.three{grid-template-columns:1fr 1fr}.worker-grid{grid-template-columns:1fr}.search{flex:1}.search input{min-width:0;width:100%}} +@media(max-width:520px){.kpi-grid,.kpi-grid.four,.grid.two,.grid.three{grid-template-columns:1fr}.topbar h1{font-size:22px}.toolbar{flex-wrap:wrap}.form-grid{grid-template-columns:1fr}.form-grid .wide{grid-column:auto}} + +/* Live request pulse map */ +.live-flow-panel{padding-bottom:14px;overflow:hidden}.live-flow-head{align-items:center}.live-controls{display:flex;align-items:center;gap:8px;flex-wrap:wrap;justify-content:flex-end}.live-controls input{width:210px}.live-flow-stage{position:relative;height:590px;border:1px solid #1d2734;border-radius:13px;overflow:hidden;background:radial-gradient(circle at 50% 46%,rgba(35,61,78,.19),transparent 38%),linear-gradient(180deg,#080c12,#070a0f)}.live-flow-stage:before{content:"";position:absolute;inset:0;pointer-events:none;background-image:linear-gradient(rgba(119,184,255,.025) 1px,transparent 1px),linear-gradient(90deg,rgba(119,184,255,.025) 1px,transparent 1px);background-size:32px 32px;mask-image:linear-gradient(to bottom,rgba(0,0,0,.9),rgba(0,0,0,.35))}.live-flow-stage canvas{position:absolute;inset:0;width:100%;height:100%;display:block}.live-flow-empty{position:absolute;left:50%;top:50%;transform:translate(-50%,-50%);padding:9px 13px;border:1px solid rgba(142,156,175,.16);border-radius:999px;background:rgba(7,10,15,.72);color:#708096;font-size:12px;pointer-events:none;backdrop-filter:blur(5px)}.live-flow-tooltip{position:absolute;z-index:5;width:min(300px,calc(100% - 20px));padding:11px 12px;border:1px solid #304154;border-radius:10px;background:rgba(10,14,20,.94);box-shadow:0 10px 35px rgba(0,0,0,.38);pointer-events:none;font-size:11px;line-height:1.5}.live-flow-tooltip strong{display:block;margin-bottom:4px;color:#f3f7fc;font-size:12px}.live-flow-tooltip .mono{font-size:10px}.live-flow-legend{position:absolute;left:13px;bottom:12px;display:flex;gap:13px;align-items:center;flex-wrap:wrap;padding:7px 9px;border:1px solid rgba(142,156,175,.12);border-radius:9px;background:rgba(7,10,15,.68);backdrop-filter:blur(5px);color:#748397;font-size:10px;pointer-events:none}.live-flow-legend span{display:flex;align-items:center;gap:5px}.legend-dot{width:7px;height:7px;border-radius:50%;display:inline-block;box-shadow:0 0 8px currentColor}.legend-dot.queued{background:var(--warn);color:var(--warn)}.legend-dot.running{background:var(--accent2);color:var(--accent2)}.legend-dot.streaming{background:var(--accent);color:var(--accent)}.legend-dot.done{background:#8e9caf;color:#8e9caf}.live-detail-grid{margin-top:14px}.live-request-list,.live-worker-list{display:flex;flex-direction:column;gap:8px;max-height:430px;overflow:auto}.live-request{display:grid;grid-template-columns:auto minmax(0,1fr) auto;gap:10px;align-items:center;padding:10px 11px;border:1px solid var(--border);border-radius:10px;background:linear-gradient(120deg,rgba(255,255,255,.018),transparent)}.live-request-pulse{width:9px;height:9px;border-radius:50%;background:var(--accent2);box-shadow:0 0 11px currentColor}.live-request-pulse.queued{background:var(--warn);color:var(--warn)}.live-request-pulse.streaming{background:var(--accent);color:var(--accent);animation:liveBreath 1.2s ease-in-out infinite}.live-request-pulse.failed{background:var(--danger);color:var(--danger)}.live-request-main{min-width:0}.live-request-main strong{display:block;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:12px}.live-request-main small{display:block;color:var(--muted);overflow:hidden;text-overflow:ellipsis;white-space:nowrap;margin-top:3px}.live-request-meta{text-align:right}.live-request-meta strong,.live-request-meta small{display:block}.live-request-meta strong{font-size:11px}.live-request-meta small{color:var(--muted);font-size:10px;margin-top:3px}.live-worker{padding:11px;border:1px solid var(--border);border-radius:10px;background:var(--surface2)}.live-worker-top{display:flex;align-items:center;justify-content:space-between;gap:10px;margin-bottom:7px}.live-worker-top strong{font-size:12px}.live-worker-top small{color:var(--muted)}.live-worker-bar{height:4px;background:var(--surface3);border-radius:8px;overflow:hidden}.live-worker-bar div{height:100%;border-radius:8px;background:linear-gradient(90deg,var(--accent2),var(--accent));transition:width .25s ease}.live-worker-models{display:flex;flex-wrap:wrap;gap:5px;margin-top:8px}.live-worker-models span{font-size:9px;padding:3px 5px;border-radius:6px;background:#1a2330;color:#8392a5}@keyframes liveBreath{0%,100%{transform:scale(.8);opacity:.65}50%{transform:scale(1.25);opacity:1}} +@media(prefers-reduced-motion:reduce){.live-request-pulse.streaming{animation:none}} +@media(max-width:1000px){.live-flow-head{align-items:flex-start;flex-direction:column}.live-controls{justify-content:flex-start;width:100%}.live-controls input{flex:1;min-width:160px}.live-flow-stage{height:500px}} +@media(max-width:620px){.live-flow-stage{height:430px}.live-controls select,.live-controls input{width:100%;flex-basis:100%}.live-flow-legend{right:10px;gap:8px}.live-detail-grid{grid-template-columns:1fr!important}} + +/* Global LLM infrastructure map */ +.kpi-grid.five{grid-template-columns:repeat(5,minmax(0,1fr))}.cluster-map-panel{padding-bottom:14px;overflow:hidden}.cluster-map-stage{height:680px;background:radial-gradient(circle at 48% 48%,rgba(42,68,91,.22),transparent 42%),radial-gradient(circle at 78% 42%,rgba(100,71,152,.08),transparent 30%),linear-gradient(180deg,#070b10,#06090d)}.cluster-node-list{display:flex;flex-direction:column;gap:9px;max-height:470px;overflow:auto}.cluster-node{padding:12px;border:1px solid var(--border);border-radius:11px;background:linear-gradient(120deg,rgba(255,255,255,.02),transparent),var(--surface2)}.cluster-node-head{display:flex;align-items:center;justify-content:space-between;gap:12px;margin-bottom:10px}.cluster-node-head strong,.cluster-node-head small{display:block}.cluster-node-head strong{font-size:12px}.cluster-node-head small{margin-top:3px;color:var(--muted);font-size:10px;max-width:320px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.cluster-mini-grid{display:grid;grid-template-columns:repeat(4,1fr);gap:7px}.cluster-mini-grid span{padding:7px;border-radius:8px;background:rgba(255,255,255,.025);color:var(--muted);font-size:9px}.cluster-mini-grid b{display:block;margin-top:3px;color:var(--text);font-size:12px}.resource-line{display:flex;align-items:center;justify-content:space-between;gap:12px;margin-top:7px;color:var(--muted);font-size:9px}.resource-line b{color:#aebdcb;font-weight:600}.resource-bar{height:4px;margin-top:4px;border-radius:999px;background:var(--surface3);overflow:hidden}.resource-bar i{display:block;height:100%;border-radius:inherit;background:linear-gradient(90deg,var(--accent2),var(--accent))}.resource-bar.vram i{background:linear-gradient(90deg,#77b8ff,#b484ff)}.legend-dot.gateway{background:#77b8ff;color:#77b8ff}.cluster-map-stage .live-flow-tooltip{width:min(320px,calc(100% - 20px))} +@media(max-width:1180px){.kpi-grid.five{grid-template-columns:repeat(3,1fr)}} +@media(max-width:780px){.kpi-grid.five{grid-template-columns:repeat(2,1fr)}.cluster-map-stage{height:540px}} +@media(max-width:520px){.kpi-grid.five{grid-template-columns:1fr}.cluster-map-stage{height:460px}.cluster-mini-grid{grid-template-columns:repeat(2,1fr)}} +.resource-bar.gpu i{background:linear-gradient(90deg,#ffcf73,#ff9c65)} + +/* Runtime API key management */ +.security-card-head{display:flex;align-items:center;justify-content:space-between;gap:14px;margin:0 0 10px}.security-card-head p{margin:0;color:var(--muted);font-size:11px;line-height:1.45}.api-key-list{display:flex;flex-direction:column;gap:8px}.api-key-row{display:flex;align-items:center;gap:12px;padding:11px;border:1px solid var(--border);border-radius:10px;background:var(--surface2)}.api-key-title{display:flex;align-items:center;gap:8px;min-width:0}.api-key-title strong{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.api-key-row small{display:block;color:var(--muted);margin-top:3px}.api-key-meta{display:flex;gap:8px;flex-wrap:wrap;margin-top:7px;color:#718197;font-size:9px}.api-key-meta span{padding:3px 6px;border-radius:6px;background:rgba(255,255,255,.025)}.api-key-locked{font-size:10px;white-space:nowrap}.danger-button{border-color:rgba(255,114,125,.24)!important;color:var(--danger)!important}.danger-button:hover{background:rgba(255,114,125,.08)!important}.callout.warn-callout{background:rgba(255,197,109,.07);color:#d8b46f;border:1px solid rgba(255,197,109,.16)}.secret-row{display:flex;gap:8px;align-items:center}.secret-row input{flex:1;min-width:0}.secret-row .button{white-space:nowrap} +@media(max-width:620px){.security-card-head,.api-key-row,.secret-row{align-items:stretch;flex-direction:column}.api-key-row .button{width:100%}.api-key-locked{align-self:flex-start}} +.live-request-pulse.cancelled{background:#8e9caf;color:#8e9caf}.live-request-meta .button{margin-top:7px} +.cap-list{display:flex;flex-wrap:wrap;gap:4px;min-width:150px}.cap-badge{display:inline-flex;padding:3px 6px;border-radius:999px;background:rgba(119,184,255,.09);color:#91c8ff;font-size:10px}.cap-tools{background:rgba(117,230,181,.09);color:var(--ok)}.cap-vision{background:rgba(204,157,255,.1);color:#cea9ff}.cap-thinking{background:rgba(255,197,109,.1);color:var(--warn)}.telemetry-line{display:flex;flex-wrap:wrap;gap:6px;margin-top:12px}.telemetry-line span{font-size:10px;padding:4px 6px;border-radius:7px;background:rgba(119,184,255,.07);color:#9bb4ce}.perf-list{margin-top:12px;padding-top:10px;border-top:1px solid var(--border);display:flex;flex-direction:column;gap:6px}.perf-list div{display:flex;justify-content:space-between;gap:12px;font-size:10px}.perf-list span{color:var(--muted);overflow:hidden;text-overflow:ellipsis}.perf-list strong{color:var(--accent)}.warn-text{color:var(--warn)}.dialog-card label.check-label{display:flex;flex-direction:row;align-items:center;gap:8px;padding:8px 0;color:var(--text)}.check-label input{width:auto;margin:0}.dialog-card input:disabled{opacity:.5;cursor:not-allowed} + +/* Persistent config editor */ +.config-editor{display:block;width:100%;min-height:620px;resize:vertical;border:1px solid var(--border);outline:none;font:11px/1.55 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;white-space:pre;overflow:auto}.config-editor:not([readonly]){border-color:rgba(119,184,255,.45);box-shadow:0 0 0 3px rgba(119,184,255,.06)}.config-actions{justify-content:flex-end;margin-top:12px}.config-actions.hidden{display:none}.callout.ok-callout{background:rgba(117,230,181,.07);color:#8fd8b7;border:1px solid rgba(117,230,181,.16)} + +/* Tiered usage retention */ +.retention-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px}.retention-card{padding:16px;border:1px solid var(--border);border-radius:12px;background:var(--surface2);display:flex;flex-direction:column;gap:5px}.retention-card span,.retention-card small{color:var(--muted)}.retention-card strong{font-size:1.05rem}.compact-input{max-width:220px}@media(max-width:900px){.retention-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media(max-width:560px){.retention-grid{grid-template-columns:1fr}} + +/* Model placement / worker routing policy */ +.placement-head{align-items:center}.placement-toolbar{align-items:center;flex-wrap:wrap}.placement-toolbar input{min-width:220px}.placement-legend{display:flex;flex-wrap:wrap;gap:14px;margin:-4px 0 14px;color:var(--muted);font-size:10px}.placement-legend span{display:flex;align-items:center;gap:6px}.placement-dot{width:8px;height:8px;border-radius:50%;display:inline-block;background:#607083}.placement-dot.allowed{background:var(--ok)}.placement-dot.allowed-empty{border:1px solid var(--ok);background:transparent}.placement-dot.loaded{background:var(--accent2);box-shadow:0 0 0 3px rgba(119,184,255,.12)}.placement-dot.denied{background:var(--danger)}.placement-matrix table{min-width:max(860px,100%)}.placement-matrix th:not(:first-child),.placement-matrix td:not(:first-child){text-align:center;min-width:135px}.placement-model-cell strong{display:block;font-size:12px}.placement-model-cell small{display:block;margin-top:4px;color:var(--muted)}.placement-cell{width:100%;min-width:108px;border:1px solid var(--border);border-radius:9px;background:var(--surface2);padding:7px 8px;color:var(--muted);cursor:pointer;transition:border-color .12s,background .12s,transform .12s}.placement-cell:hover{transform:translateY(-1px);border-color:#46627e}.placement-cell.allowed.installed{color:var(--ok);background:rgba(117,230,181,.06);border-color:rgba(117,230,181,.18)}.placement-cell.allowed:not(.installed){color:#8ebfa8;border-style:dashed}.placement-cell.denied{color:var(--danger);background:rgba(255,114,125,.045);border-color:rgba(255,114,125,.16)}.placement-cell.loaded{box-shadow:inset 0 0 0 1px rgba(119,184,255,.35)}.placement-cell .placement-main{display:flex;align-items:center;justify-content:center;gap:5px;font-weight:700;font-size:11px}.placement-cell small{display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-top:3px;font-size:8px;color:inherit;opacity:.72}.placement-cell .override-mark{font-size:9px;color:var(--accent2)}.placement-help{font-size:11px;line-height:1.6;color:var(--muted)}.placement-worker-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(320px,1fr));gap:12px}.placement-worker-card{border:1px solid var(--border);background:var(--surface2);border-radius:12px;padding:14px}.placement-worker-card .worker-rule-head{display:flex;justify-content:space-between;align-items:flex-start;gap:10px}.placement-worker-card .worker-rule-head strong{font-size:14px}.placement-worker-card .worker-rule-head small{display:block;color:var(--muted);margin-top:3px}.placement-rule-meta{display:flex;gap:6px;flex-wrap:wrap;margin:12px 0}.placement-rule-list{display:grid;grid-template-columns:1fr 1fr;gap:10px}.placement-rule-box{border:1px solid rgba(37,46,59,.7);border-radius:9px;padding:9px;min-width:0}.placement-rule-box>span{display:block;color:var(--muted);font-size:9px;text-transform:uppercase;letter-spacing:.06em;margin-bottom:6px}.placement-rule-box code{display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:#aebdd0;font-size:10px;margin:3px 0}.placement-rule-box .empty-rule{color:#5f6d80;font-size:10px}.placement-dialog-card textarea{width:100%;resize:vertical;min-height:105px;border:1px solid var(--border);border-radius:9px;background:var(--surface2);color:var(--text);padding:10px 11px;outline:none;font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace}.placement-dialog-card textarea:focus{border-color:#426b91;box-shadow:0 0 0 3px rgba(119,184,255,.08)}.placement-presets{display:flex;align-items:center;gap:6px;flex-wrap:wrap;margin-top:4px;color:var(--muted);font-size:10px}.placement-dialog-card .dialog-actions .grow{flex:1}.placement-inventory-error{color:var(--warn);font-size:9px;margin-top:8px} +@media(max-width:780px){.placement-head{align-items:stretch;flex-direction:column}.placement-toolbar{width:100%}.placement-toolbar input{min-width:0;flex:1}.placement-worker-grid{grid-template-columns:1fr}.placement-rule-list{grid-template-columns:1fr}.placement-dialog-card .dialog-actions{flex-wrap:wrap}.placement-dialog-card .dialog-actions .grow{display:none}} +.checkbox-line{flex-direction:row!important;align-items:center!important;gap:8px!important}.checkbox-line input{width:auto!important}.model-tag input{width:auto;margin:0 4px 0 0}.state-dot.warn{background:var(--warn)}.state-dot.bad{background:var(--danger)} + +/* Policy simulator */ +.simulator-form{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;align-items:end;margin-bottom:18px}.simulator-form label{display:flex;flex-direction:column;gap:6px;font-size:.82rem}.simulator-form input,.simulator-form select{width:100%}.sim-capabilities{grid-column:span 2}.sim-submit{display:flex;align-items:end;height:100%}.sim-decision{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:1px;background:var(--border);border:1px solid var(--border);border-radius:12px;overflow:hidden;margin-bottom:14px}.sim-decision>div{background:var(--panel);padding:12px}.sim-decision small{display:block;color:var(--muted);margin-bottom:4px}.sim-decision.ok{border-color:rgba(60,190,120,.45)}.sim-decision.bad{border-color:rgba(230,90,90,.5)}.sim-grid{display:grid;grid-template-columns:1fr 1fr;gap:12px;margin-bottom:12px}.sim-card{border:1px solid var(--border);border-radius:12px;padding:12px;margin-bottom:12px}.sim-card h3{margin:0 0 10px}.sim-step{display:flex;justify-content:space-between;padding:7px 0;border-top:1px solid var(--border)}.sim-step.ok span{color:var(--good)}.sim-step.bad span{color:var(--danger)}.selected-row{background:color-mix(in srgb,var(--good) 8%,transparent)}.callout.danger{border-color:rgba(230,90,90,.45)} +@media(max-width:900px){.simulator-form{grid-template-columns:1fr 1fr}.sim-decision{grid-template-columns:1fr 1fr}.sim-grid{grid-template-columns:1fr}}@media(max-width:600px){.simulator-form{grid-template-columns:1fr}.sim-capabilities{grid-column:auto}.sim-decision{grid-template-columns:1fr}} + + +/* Checkpoint 24 — SCMDB-inspired professional light control-plane theme. */ +html,body{background:var(--bg);color:var(--text)} +body{font-family:var(--font-ui);font-size:13px;line-height:1.45;background-image:linear-gradient(rgba(111,126,145,.035) 1px,transparent 1px),linear-gradient(90deg,rgba(111,126,145,.025) 1px,transparent 1px);background-size:32px 32px} +button,input,select,textarea{font-family:var(--font-ui)} +.mono,.code-block,.config-editor,.placement-dialog-card textarea,.placement-rule-box code{font-family:var(--font-mono)} +.sidebar{background:linear-gradient(180deg,#edf1f5 0%,#e7ecf2 100%);border-right-color:#ccd5df;box-shadow:2px 0 10px rgba(49,67,87,.035)} +.brand{gap:10px}.brand-mark{border:1px solid #a9d2df;border-radius:7px;background:#f7fcfd;color:#147fa5;box-shadow:inset 0 0 0 1px rgba(255,255,255,.8);font-family:var(--font-mono);font-size:12px;letter-spacing:.02em}.brand strong{font-family:var(--font-mono);font-size:12px;letter-spacing:.02em}.brand small{font-family:var(--font-mono);font-size:9px;text-transform:uppercase;letter-spacing:.12em;color:#8090a2} +.nav{gap:2px}.nav button{position:relative;border-radius:6px;padding:9px 10px;color:#5e7084;font-family:var(--font-mono);font-size:11px;letter-spacing:.01em;transition:background .12s ease,color .12s ease,border-color .12s ease}.nav button span{color:#8998aa}.nav button:hover{background:#f7f9fb;color:#2d4156}.nav button.active{background:#dcecf2;color:#166f8f;box-shadow:inset 3px 0 0 #159dcc,inset 0 0 0 1px rgba(20,127,165,.08)}.nav button.active span{color:#147fa5}.sidebar-foot{border-top-color:#cfd7e0}.status-line{font-family:var(--font-mono);font-size:10px}.text-button,.link-button{color:#607286}.text-button:hover,.link-button:hover{color:#147fa5} +.main{background:linear-gradient(180deg,rgba(255,255,255,.44),rgba(238,242,246,.14));padding-top:24px}.topbar{padding-bottom:17px;border-bottom:1px solid #d7dee7;margin-bottom:20px}.topbar h1{font-family:var(--font-mono);font-size:21px;line-height:1.2;letter-spacing:.015em;font-weight:700;color:#293a4d}.topbar p{font-family:var(--font-mono);font-size:9px;letter-spacing:.09em;text-transform:uppercase;color:#8190a1}.identity-pill{border-radius:7px;background:#f8fafc;border-color:#d2dae4;box-shadow:0 1px 1px rgba(39,57,77,.03)}.identity-pill span{font-family:var(--font-mono);font-size:11px}.identity-pill small{font-family:var(--font-mono);font-size:9px}.icon-button{border-radius:7px;background:#fff;border-color:#cfd8e2;color:#506479}.icon-button:hover{background:#edf5f8;color:#147fa5;border-color:#b8d5df} +.kpi,.worker-card{background:#fff;border-color:#d6dde6;border-radius:8px;box-shadow:0 1px 2px rgba(39,57,77,.035)}.kpi{padding:14px 15px}.kpi>span,.metric-panel>span{font-family:var(--font-mono);font-size:9px;text-transform:uppercase;letter-spacing:.08em;color:#78899b;margin-bottom:7px}.kpi strong,.metric-panel strong{font-family:var(--font-mono);font-size:22px;color:#2a4358;letter-spacing:-.02em}.kpi small{font-family:var(--font-mono);font-size:9px;color:#8493a4} +.panel{background:#fff;border-color:#d6dde6;border-radius:8px;box-shadow:0 1px 2px rgba(39,57,77,.035);padding:16px}.panel-head{margin-bottom:13px}.panel-head h2{font-family:var(--font-mono);font-size:11px;text-transform:uppercase;letter-spacing:.055em;color:#33485d}.panel-head p{font-size:11px;color:#76879a}.badge{border:1px solid #bbdce6;background:#e7f5f8;color:#147fa5;border-radius:5px;font-family:var(--font-mono);font-size:9px;text-transform:uppercase;letter-spacing:.06em} +.chart .axis{stroke:#d4dce5}.chart text{fill:#76879a;font-family:var(--font-mono);font-size:9px}.chart .line{stroke:#147fa5}.chart .credit-line{stroke:#1a805f} +.health-item{border-bottom-color:#e2e7ed}.health-item strong,.bar-row .bar-meta,.slot-row{font-family:var(--font-mono)}.bar-track,.slot-bar,.progress,.live-worker-bar,.resource-bar{background:#e8edf3}.bar-fill,.live-worker-bar div{background:linear-gradient(90deg,#147fa5,#1a805f)} +.table-wrap{border:1px solid #dbe2ea;border-radius:7px;background:#fff}.table-wrap table{background:#fff}.table-wrap th{position:sticky;top:0;z-index:1;background:#f1f4f7;color:#65778a;font-family:var(--font-mono);font-size:9px;letter-spacing:.075em;padding:8px 10px;border-bottom-color:#d5dde6}.table-wrap td{border-bottom-color:#e3e8ee}.table-wrap tbody tr:nth-child(even){background:#fbfcfd}.table-wrap tbody tr:hover{background:#eef6f9}.status-chip{border:1px solid #d6dee7;background:#edf1f5;color:#607286;border-radius:5px;font-family:var(--font-mono);font-size:9px}.status-chip.ok{border-color:#b9d7cb;background:#e9f4ef;color:#1a805f}.status-chip.bad{border-color:#e2bfc4;background:#faeef0;color:#b83d4b}.status-chip.warn{border-color:#e4cfaa;background:#faf4e8;color:#a56600} +.worker-title strong{font-family:var(--font-mono);font-size:12px}.worker-title small{color:#74879a}.model-tag{border:1px solid #d7dee6;border-radius:5px;background:#f0f4f7;color:#566b80;font-family:var(--font-mono);font-size:9px}.telemetry-line span{border:1px solid #d1e2e8;background:#edf6f8;color:#416c7d} +input,select,textarea{border-color:#cbd5df;border-radius:6px;background:#fff;color:#253345;box-shadow:inset 0 1px 1px rgba(39,57,77,.025)}input::placeholder,textarea::placeholder{color:#97a4b2}input:focus,select:focus,textarea:focus,.placement-dialog-card textarea:focus{border-color:#6aacc2;box-shadow:0 0 0 3px rgba(20,127,165,.10)} +.button{border-radius:6px;background:#fff;border-color:#cbd5df;color:#40566c;font-family:var(--font-mono);font-size:10px;letter-spacing:.015em;box-shadow:0 1px 1px rgba(39,57,77,.025)}.button:hover{background:#eef5f8;border-color:#b9d3dd;color:#147fa5}.button.primary{background:#147fa5;border-color:#147fa5;color:#fff;box-shadow:0 1px 2px rgba(20,127,165,.18)}.button.primary:hover{background:#0f7498;border-color:#0f7498;color:#fff}.button.danger,.danger-button{background:#fff5f6;color:#b83d4b;border-color:#e4bcc2}.button.danger:hover,.danger-button:hover{background:#fbecef!important}.actions{align-items:center} +.flow-diagram div,.operation,.api-key-row,.retention-card,.placement-worker-card{background:#f7f9fb;border-color:#d9e0e8;border-radius:7px}.flow-diagram span{color:#8997a6}.operation-top strong,.api-key-title strong{font-family:var(--font-mono)} +.code-block,.config-editor{background:#f7f9fc;color:#30465b;border-color:#cfd8e2;border-radius:7px}.config-editor:not([readonly]){border-color:#76aec1;box-shadow:0 0 0 3px rgba(20,127,165,.08)} +.login-shell{background:radial-gradient(circle at 50% 18%,#ffffff 0,#eef4f7 42%,#e7edf3 100%)}.login-card{background:rgba(255,255,255,.98);border-color:#cfd8e2;border-radius:12px;box-shadow:0 18px 50px rgba(43,64,85,.14)}.login-card h1{font-family:var(--font-mono);font-size:21px;letter-spacing:.01em}.fineprint{color:#8190a1} +dialog::backdrop{background:rgba(39,52,66,.24);backdrop-filter:blur(2px)}.dialog-card{background:#fff;border-color:#cbd5df;border-radius:10px;box-shadow:0 22px 55px rgba(39,52,66,.18)}.dialog-card label,.stack label{color:#65778a}.toast{background:#fff;border-color:#cbd5df;border-radius:7px;color:#253345} +.callout.warn-callout{background:#fbf5e9;color:#8f5b00;border-color:#e8d3ad}.callout.ok-callout{background:#edf6f1;color:#166b50;border-color:#c3ddd2}.callout.danger{background:#fcf0f2;color:#a53542;border-color:#e5c0c5} +.api-key-meta{color:#728397}.api-key-meta span,.cluster-mini-grid span{background:#edf1f5}.cap-badge{border:1px solid #c9dce4;background:#edf6f8;color:#1a6681}.cap-tools{border-color:#c5ddd3;background:#eef7f3;color:#1a805f}.cap-vision{border-color:#d8cff0;background:#f3f0fa;color:#6b5bb5}.cap-thinking{border-color:#e7d7b8;background:#faf5eb;color:#a56600}.perf-list strong{color:#166f8f} +.placement-cell{border-radius:6px;background:#f7f9fb;color:#66778a}.placement-cell:hover{border-color:#82a9b8;background:#f0f7f9}.placement-cell.allowed.installed{color:#166b50;background:#edf6f1;border-color:#c3ddd2}.placement-cell.allowed:not(.installed){color:#4f7f6c}.placement-cell.denied{color:#a83b48;background:#fcf0f2;border-color:#e3c0c5}.placement-cell.loaded{box-shadow:inset 0 0 0 1px rgba(20,127,165,.34)}.placement-cell .override-mark{color:#147fa5}.placement-rule-box{border-color:#d8e0e8;background:#fff}.placement-rule-box code{color:#425a70}.placement-rule-box .empty-rule{color:#8b99a8}.placement-dot{background:#8998a9}.placement-dot.loaded{background:#147fa5;box-shadow:0 0 0 3px rgba(20,127,165,.10)} +.live-flow-stage,.cluster-map-stage{border-color:#cbd5df;border-radius:8px;background:radial-gradient(circle at 50% 46%,rgba(20,127,165,.06),transparent 38%),linear-gradient(180deg,#f9fbfc,#eef3f6)}.cluster-map-stage{background:radial-gradient(circle at 48% 48%,rgba(20,127,165,.055),transparent 42%),radial-gradient(circle at 78% 42%,rgba(107,91,181,.04),transparent 30%),linear-gradient(180deg,#f9fbfc,#edf2f6)}.live-flow-stage:before{background-image:linear-gradient(rgba(89,113,134,.09) 1px,transparent 1px),linear-gradient(90deg,rgba(89,113,134,.09) 1px,transparent 1px);mask-image:linear-gradient(to bottom,rgba(0,0,0,.75),rgba(0,0,0,.12))}.live-flow-empty{border-color:#d1dae3;background:rgba(255,255,255,.88);color:#738496}.live-flow-tooltip{border-color:#bfcbd6;background:rgba(255,255,255,.97);box-shadow:0 10px 28px rgba(39,57,77,.14);color:#33485d}.live-flow-tooltip strong{color:#253345}.live-flow-legend{border-color:#ccd6df;background:rgba(255,255,255,.88);color:#66778a}.live-request{background:linear-gradient(120deg,#fafcfd,#f5f8fa);border-color:#dbe2e9}.live-worker{background:#f7f9fb}.live-worker-models span{background:#eaf0f5;color:#607286}.cluster-node{background:linear-gradient(120deg,#fbfcfd,#f5f8fa);border-color:#d8e0e8}.cluster-mini-grid b{color:#2c4055}.resource-line b{color:#445b70}.resource-bar.vram i{background:linear-gradient(90deg,#147fa5,#6b5bb5)}.legend-dot.gateway{background:#147fa5;color:#147fa5} +.resource-bar.gpu i{background:linear-gradient(90deg,#c58a20,#d96e38)} +.sim-decision{border-radius:7px}.sim-card{background:#fff;border-color:#d8e0e8;border-radius:7px}.selected-row{background:color-mix(in srgb,var(--good) 7%,#fff)} +@media(max-width:780px){.sidebar{background:#e9eef3}.nav button.active{box-shadow:inset 3px 0 0 #159dcc}.main{background:var(--bg)}} + +/* Checkpoint 25: sidebar overflow hardening for shorter viewports */ +.sidebar{padding:18px 12px 14px;overflow:hidden} +.brand{flex:0 0 auto;padding:0 8px 18px} +.nav{flex:1 1 auto;min-height:0;overflow-y:auto;overflow-x:hidden;padding:2px 4px 8px 0;scrollbar-gutter:stable} +.nav::-webkit-scrollbar{width:8px} +.nav::-webkit-scrollbar-track{background:transparent} +.nav::-webkit-scrollbar-thumb{background:#c7d4df;border-radius:999px;border:2px solid transparent;background-clip:padding-box} +.nav::-webkit-scrollbar-thumb:hover{background:#aebfd0;border:2px solid transparent;background-clip:padding-box} +.sidebar-foot{flex:0 0 auto;margin-top:10px;padding:12px 8px 2px;background:linear-gradient(180deg,rgba(231,236,242,0),rgba(231,236,242,.92) 22%,rgba(231,236,242,.98) 100%)} + +@media (max-height: 900px){ + .sidebar{padding:14px 10px 12px} + .brand{padding-bottom:14px} + .nav button{padding:8px 10px;font-size:10px} + .sidebar-foot{padding-top:10px} + .status-line{margin-bottom:8px} +} + +@media (max-height: 760px){ + .brand{gap:9px;padding-bottom:12px} + .brand strong{font-size:13px} + .brand small{font-size:10px} + .brand-mark{width:34px;height:34px;border-radius:10px} + .nav button{padding:7px 9px;font-size:10px} +} + +@media (max-height: 680px){ + .brand small{display:none} + .nav button{padding:6px 9px} + .sidebar-foot{padding-top:8px} +} diff --git a/internal/webui/assets/app.js b/internal/webui/assets/app.js new file mode 100644 index 0000000..dc1774e --- /dev/null +++ b/internal/webui/assets/app.js @@ -0,0 +1,214 @@ +'use strict'; +const S={bootstrap:null,session:null,overview:null,recent:[],rollups:null,models:[],aliases:null,aliasEditing:null,placement:null,placementEditingWorker:null,policies:null,config:null,configDirty:false,apiKeys:null,storage:null,autotune:null,autotuneSelected:null,warm:null,warmEditing:null,alerts:null,simulation:null,jobs:[],batches:null,operations:[],timer:null,page:'overview',live:{requests:[],connected:false,version:0,generatedAt:null,counts:null,truncated:false},cluster:{gateways:[],workers:[],requests:[],counts:{},queue:0,running:0,connected:false,mode:'in-memory',version:0,generatedAt:null},liveAbort:null,clusterAbort:null,liveRAF:null,livePaused:window.matchMedia?.('(prefers-reduced-motion: reduce)').matches||false,livePauseTime:0,liveHit:[],clusterHit:[]}; +const $=(q,r=document)=>r.querySelector(q), $$=(q,r=document)=>[...r.querySelectorAll(q)]; +const esc=v=>String(v??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); +const num=(v,d=0)=>Number(v||0).toLocaleString('de-DE',{maximumFractionDigits:d}); +const credits=v=>Number(v||0).toLocaleString('de-DE',{maximumFractionDigits:2}); +const bytes=v=>{v=Number(v||0);if(!v)return '0 B';const u=['B','KB','MB','GB','TB'];let i=0;while(v>=1024&&i{v=Number(v||0);return v<1000?`${num(v)} ms`:`${(v/1000).toLocaleString('de-DE',{maximumFractionDigits:2})} s`}; +const when=v=>v?new Date(v).toLocaleString('de-DE'):'–'; +const infinity=v=>Number(v||0)<=0?'∞':num(v,2); +const capsHTML=a=>(a||[]).length?(a||[]).map(c=>`${esc(c)}`).join(''):'unbekannt'; +const pct1=v=>Number(v||0).toLocaleString('de-DE',{maximumFractionDigits:1})+' %'; +const isAdmin=()=>!!S.session?.admin; +function authHeaders(){const h={'Accept':'application/json'};const key=sessionStorage.getItem('ofgCredential');if(key)h['Authorization']='Bearer '+key;const csrf=getCookie('ofg_csrf');if(csrf)h['X-CSRF-Token']=csrf;return h} +function getCookie(n){return document.cookie.split(';').map(v=>v.trim()).find(v=>v.startsWith(n+'='))?.slice(n.length+1)||''} +async function api(path,opt={}){const headers={...authHeaders(),...(opt.headers||{})};if(opt.body&&!headers['Content-Type'])headers['Content-Type']='application/json';const r=await fetch(path,{...opt,headers,credentials:'same-origin'});let data=null;const ct=r.headers.get('content-type')||'';if(ct.includes('json'))data=await r.json().catch(()=>null);else data=await r.text();if(!r.ok){const e=new Error(data?.error?.message||data?.message||`HTTP ${r.status}`);e.status=r.status;throw e}return data} +function toast(msg,bad=false){const x=document.createElement('div');x.className='toast'+(bad?' bad':'');x.textContent=msg;$('#toast-root').appendChild(x);setTimeout(()=>x.remove(),3200)} +async function copyText(v){if(navigator.clipboard?.writeText){try{await navigator.clipboard.writeText(v);return true}catch{}}const x=document.createElement('textarea');x.value=v;x.setAttribute('readonly','');x.style.position='fixed';x.style.opacity='0';document.body.appendChild(x);x.select();let ok=false;try{ok=document.execCommand('copy')}catch{}x.remove();return ok} +async function init(){try{S.bootstrap=await fetch('/gateway/ui-api/bootstrap',{credentials:'same-origin'}).then(r=>r.json());document.title=S.bootstrap.title||'Ollama Fair Gateway';$('#brand-title').textContent=S.bootstrap.title;$('#login-title').textContent=S.bootstrap.title;if(S.bootstrap.oidc_enabled){$('#oidc-login').classList.remove('hidden');$('#login-divider').classList.remove('hidden');$('#oidc-login').href=S.bootstrap.oidc_login_url}await establishSession()}catch(e){showLogin(e.message)}} +async function establishSession(){try{S.session=await api('/gateway/ui-api/session');showApp();await refreshAll(true)}catch(e){if(e.status===401)showLogin();else showLogin(e.message)}} +function showLogin(err=''){clearInterval(S.timer);stopLiveStream();stopClusterStream();$('#app').classList.add('hidden');$('#login').classList.remove('hidden');$('#login-error').textContent=err;$('#login-error').classList.toggle('hidden',!err)} +function showApp(){$('#login').classList.add('hidden');$('#app').classList.remove('hidden');$('#identity-name').textContent=S.session.subject||S.session.application||'–';$('#identity-tenant').textContent=`${S.session.tenant} · ${S.session.auth_type}`;$$('[data-admin]').forEach(x=>x.classList.toggle('hidden',!isAdmin()));const requested=location.hash.replace(/^#/,'');if(!isAdmin()&&S.page!=='usage')goto('usage');else if(requested&&$('#page-'+requested))goto(requested);clearInterval(S.timer);S.timer=setInterval(()=>refreshAll(false),3500);if(isAdmin()){startLiveStream();startClusterStream();startLiveAnimation()}} +$('#key-login').addEventListener('submit',async e=>{e.preventDefault();const v=$('#login-key').value.trim();if(!v)return;sessionStorage.setItem('ofgCredential',v);await establishSession()}); +$('#logout').onclick=async()=>{sessionStorage.removeItem('ofgCredential');try{await fetch((S.bootstrap?.ui_path||'/admin')+'/logout',{method:'POST',headers:authHeaders(),credentials:'same-origin'})}catch{}location.reload()}; +$$('#nav button').forEach(b=>b.onclick=()=>goto(b.dataset.page));$$('[data-goto]').forEach(b=>b.onclick=()=>goto(b.dataset.goto));$('#refresh').onclick=()=>refreshAll(true); +function goto(p){S.page=p;if(location.hash!=='#'+p)history.replaceState(null,'','#'+p);$$('#nav button').forEach(b=>b.classList.toggle('active',b.dataset.page===p));$$('.page').forEach(x=>x.classList.toggle('active',x.id==='page-'+p));if(p==='usage'&&isAdmin()&&S.rollups===null)loadRollups().catch(e=>toast(e.message,true));const titles={overview:['Übersicht','Live-Zustand des Gateways'],live:['Live Flow','Animierte Request-Pulse durch Queue, Scheduler und Worker'],infrastructure:['Infrastruktur','Globale LLM-Topologie über Gateways, Worker und Modelle'],workers:['Worker','Gesundheit, Slots und Modell-Affinity'],models:['Modelle','Inventar und Modellverwaltung'],routing:['Model Placement','Harte Modell-zu-Worker-Routingregeln'],fairness:['Fairness & Quotas','Gewichte, Credit-Buckets und persistente Overrides'],autotune:['Auto-Tuning','Concurrency-Benchmarks und OpenTelemetry-Status'],warm:['Warm Models','Residency, Preload und Idle-Eviction für Modelle'],alerts:['Alerts','Betriebssignale und signierte Webhooks'],usage:['Nutzung','Tokens, Credits und Request-Journal'],jobs:['Jobs','Aktive Inferenz-Requests überwachen und abbrechen'],batches:['Batch Jobs','Durable Hintergrundjobs überwachen, pausieren und fortsetzen'],operations:['Operationen','Laufende und abgeschlossene Admin-Aktionen'],security:['Sicherheit','OIDC, API-Keys und Netzwerk-Bypass'],storage:['Persistenz','Durabler Control-Plane-State, Usage und Backups'],config:['Konfiguration','Persistente, validierte Gateway-Konfiguration']};$('#page-title').textContent=titles[p][0];$('#page-subtitle').textContent=titles[p][1];render()} +async function refreshAll(loud){try{const jobs=[api('/gateway/v1/usage/me').then(x=>S.myUsage=x)];if(isAdmin()){jobs.push(api('/gateway/ui-api/overview').then(x=>S.overview=x),api('/gateway/ui-api/recent?limit=1000').then(x=>S.recent=x.events||[]),api('/gateway/ui-api/policies').then(x=>S.policies=x),api('/gateway/ui-api/jobs').then(x=>S.jobs=x.jobs||[]),api('/gateway/ui-api/batches').then(x=>S.batches=x),api('/gateway/ui-api/operations').then(x=>S.operations=x.operations||[]));if(['models','workers','overview'].includes(S.page)||!S.models.length)jobs.push(api('/gateway/ui-api/models').then(x=>S.models=x.inventories||[]));if(S.page==='models'||S.aliases===null)jobs.push(api('/gateway/ui-api/model-aliases').then(x=>S.aliases=x));if(S.page==='routing'||S.placement===null)jobs.push(api('/gateway/ui-api/placement').then(x=>S.placement=x));if(['security','config'].includes(S.page)&&!S.config)jobs.push(api('/gateway/ui-api/config').then(x=>S.config=x));if(S.page==='security'||S.apiKeys===null)jobs.push(api('/gateway/ui-api/api-keys').then(x=>S.apiKeys=x.keys||[]));if(S.page==='autotune'||S.autotune===null)jobs.push(api('/gateway/ui-api/autotune').then(x=>S.autotune=x));if(S.page==='warm'||S.warm===null)jobs.push(api('/gateway/ui-api/warm-models').then(x=>S.warm=x));if(S.page==='alerts'||S.alerts===null)jobs.push(api('/gateway/ui-api/alerts').then(x=>S.alerts=x));if(S.page==='storage'||S.storage===null)jobs.push(api('/gateway/ui-api/storage').then(x=>S.storage=x));if(S.page==='usage')jobs.push(loadRollups())}await Promise.all(jobs);setConnection(true);render();if(loud)toast('Aktualisiert')}catch(e){setConnection(false,e.message);if(e.status===401)return showLogin('Sitzung abgelaufen oder Credential ungültig.');if(loud)toast(e.message,true)}} +function setConnection(ok,msg=''){$('#connection-state').className='status-line '+(ok?'ok':'bad');$('#connection-state span:last-child').textContent=ok?'Gateway verbunden':(msg||'Verbindung gestört')} +function render(){renderUsageMine();if(!isAdmin())return;if(!S.overview)return;renderOverview();renderLivePanel();renderClusterPanel();renderWorkers();renderModels();renderPlacement();renderFairness();renderModelAccess();renderAutoTune();renderWarm();renderAlerts();renderPolicySimulator();renderUsage();renderJobs();renderBatches();renderOperations();renderStorage();if(S.config){renderSecurity();const ed=$('#config-json');if(ed&&!S.configDirty)ed.value=JSON.stringify(S.config,null,2)}} +function renderOverview(){const o=S.overview, g=o.usage||{}, sch=o.scheduler||{}, ws=o.workers||[];$('#kpi-queue').textContent=num(sch.queued);$('#kpi-running').textContent=num(sch.running);$('#kpi-requests').textContent=num(g.requests);$('#kpi-credits').textContent=credits(g.credits);$('#kpi-tokens').textContent=num((g.prompt_tokens||0)+(g.completion_tokens||0));$('#kpi-workers').textContent=`${ws.filter(w=>w.healthy).length}/${ws.length}`;renderTraffic();$('#system-health').innerHTML=ws.map(w=>`
${esc(w.name)}${w.active}/${w.max_concurrent} Slots · ${w.models?.length||0} geladen
${w.healthy?'healthy':'down'}
`).join('')||'
Keine Worker
';const tops=groupRecent('model');renderBars('#top-models',tops,'credits');$('#recent-mini').innerHTML=tableRecent(S.recent.slice(0,6),true)} +function groupRecent(field){const m=new Map();for(const e of S.recent){const k=e[field]||'(none)',x=m.get(k)||{name:k,credits:0,requests:0};x.credits+=Number(e.actual_credits||0);x.requests++;m.set(k,x)}return [...m.values()].sort((a,b)=>b.credits-a.credits).slice(0,7)} +function renderBars(sel,items,metric){const max=Math.max(1,...items.map(x=>Number(x[metric]||0)));$(sel).innerHTML=items.length?items.map(x=>`
${esc(x.name)}${metric==='credits'?credits(x[metric])+' cr':num(x[metric])}
`).join(''):'
Noch keine Daten
'} +function renderTraffic(){const ev=[...S.recent].reverse().slice(-240);if(!ev.length){$('#traffic-chart').innerHTML='
Noch keine Requests
';return}const buckets=new Map();for(const e of ev){const d=new Date(e.time),k=new Date(Math.floor(d.getTime()/60000)*60000).getTime(),x=buckets.get(k)||{t:k,r:0,c:0};x.r++;x.c+=Number(e.actual_credits||0);buckets.set(k,x)}const a=[...buckets.values()].sort((x,y)=>x.t-y.t);const W=800,H=230,p=28,maxR=Math.max(1,...a.map(x=>x.r)),maxC=Math.max(1,...a.map(x=>x.c));const pts=(key,max)=>a.map((x,i)=>`${p+(W-2*p)*(a.length===1?.5:i/(a.length-1))},${H-p-(H-2*p)*x[key]/max}`).join(' ');const labels=[a[0],a[Math.floor((a.length-1)/2)],a[a.length-1]].filter((x,i,z)=>z.indexOf(x)===i);$('#traffic-chart').innerHTML=`${labels.map(x=>{const i=a.indexOf(x),xx=p+(W-2*p)*(a.length===1?.5:i/(a.length-1));return `${new Date(x.t).toLocaleTimeString('de-DE',{hour:'2-digit',minute:'2-digit'})}`}).join('')}`} +function renderWorkers(){const ws=S.overview.workers||[];$('#worker-cards').innerHTML=ws.map(w=>{const perf=(w.performance||[]).filter(x=>Number(x.samples)>0).sort((a,b)=>Number(b.output_tps||0)-Number(a.output_tps||0)).slice(0,3);const tele=[w.gpu_utilization_percent?`GPU ${pct1(w.gpu_utilization_percent)}`:'',w.vram_total_bytes?`VRAM ${bytes(w.vram_used_bytes)} / ${bytes(w.vram_total_bytes)}`:'',w.gpu_temperature_c?`${num(w.gpu_temperature_c,1)} °C`:'',w.gpu_power_watts?`${num(w.gpu_power_watts,1)} W`:''].filter(Boolean);const maint=w.maintenance||'active',circuit=w.circuit_state||'closed';const actions=maint==='active'?``:``;return `
${esc(w.name)}${esc(w.url)}
${esc(maint)}CB ${esc(circuit)}
Slots${w.active} / ${w.max_concurrent}
${tele.length?`
${tele.map(x=>`${x}`).join('')}
`:''}
${(w.models||[]).slice(0,8).map(m=>`${esc(m)}`).join('')}${w.models?.length>8?`+${w.models.length-8}`:''}
${perf.length?`
${perf.map(x=>`
${esc(x.model)}${num(x.output_tps,1)} tok/s
`).join('')}
`:''}
${actions}${circuit!=='closed'?``:''}
${w.last_circuit_error?`
Circuit: ${esc(w.last_circuit_error)}
`:''}${w.telemetry_error?`
Telemetry: ${esc(w.telemetry_error)}
`:''}${w.last_error?`
${esc(w.last_error)}
`:''}
`}).join('');$('#worker-table').innerHTML=table(['Worker','Status','Maintenance','Circuit','Slots','GPU','VRAM','Letzter Check'],ws.map(w=>[esc(w.name),`${w.healthy?'Healthy':'Down'}`,esc(w.maintenance||'active'),esc(w.circuit_state||'closed'),`${w.active} / ${w.max_concurrent}`,w.gpu_utilization_percent?pct1(w.gpu_utilization_percent):'–',w.vram_total_bytes?`${bytes(w.vram_used_bytes)} / ${bytes(w.vram_total_bytes)}`:'–',when(w.last_check)]))} +function flatModels(){return S.models.flatMap(inv=>(inv.models||[]).map(m=>({...m,worker:inv.worker,worker_error:inv.error})))} +function aliasData(){return S.aliases?.aliases||S.overview?.model_aliases||{}} +function renderModels(){let ms=flatModels(),f=($('#model-filter')?.value||'').toLowerCase();if(f)ms=ms.filter(m=>`${m.name} ${m.worker} ${m.details?.family||''} ${m.details?.parameter_size||''} ${(m.capabilities||[]).join(' ')}`.toLowerCase().includes(f));const aliases=aliasData();const aliasRows=Object.entries(aliases).sort((a,b)=>a[0].localeCompare(b[0])).map(([name,a])=>[`${esc(name)}`,esc((a.models||[]).join(' → ')),esc((a.required_capabilities||[]).join(', ')||'–'),a.visible===false?'versteckt':'sichtbar',`
`]);$('#alias-table').innerHTML=aliasRows.length?table(['Alias','Fallback-Reihenfolge','Capabilities','Discovery','Aktionen'],aliasRows):'
Keine Model-Aliases konfiguriert.
';$('#model-table').innerHTML=table(['Modell','Worker','Capabilities','Modell-Max','Konfig.','Geladen','Parameter','Quant.','Größe','Status','Aktionen'],ms.map(m=>[`${esc(m.name)}
${esc((m.digest||'').slice(0,14))}${m.metadata_error?`
Metadaten nicht verfügbar`:''}`,esc(m.worker),`
${capsHTML(m.capabilities)}
`,m.context_length?`${num(m.context_length)} tok`:'–',m.configured_context_length?`${num(m.configured_context_length)} tok`:'–',m.loaded_context_length?`${num(m.loaded_context_length)} tok`:'–',esc(m.details?.parameter_size||'–'),esc(m.details?.quantization_level||'–'),bytes(m.size),`${m.loaded?'loaded':'stored'}`,`
`]))} +function openAliasDialog(name=''){const a=aliasData()[name]||{};S.aliasEditing=name;$('#alias-name').value=name;$('#alias-name').readOnly=!!name;$('#alias-models').value=(a.models||[]).join('\n');$('#alias-capabilities').value=(a.required_capabilities||[]).join('\n');$('#alias-visible').checked=a.visible!==false;$('#alias-dialog').showModal()} +async function reloadAliases(){S.aliases=await api('/gateway/ui-api/model-aliases');S.overview=await api('/gateway/ui-api/overview');renderModels();renderPolicySimulator()} +$('#new-alias')?.addEventListener('click',()=>openAliasDialog('')); +$('#alias-table')?.addEventListener('click',async e=>{const edit=e.target.closest('[data-alias-edit]');if(edit){openAliasDialog(edit.dataset.aliasEdit);return}const del=e.target.closest('[data-alias-delete]');if(!del)return;const name=del.dataset.aliasDelete;if(!confirm(`Model Alias ${name} wirklich löschen?`))return;try{await api('/gateway/ui-api/model-aliases/'+encodeURIComponent(name),{method:'DELETE'});await reloadAliases();toast('Alias gelöscht')}catch(err){toast(err.message,true)}}); +$('#alias-form')?.addEventListener('submit',async e=>{e.preventDefault();const name=$('#alias-name').value.trim(),payload={models:lines($('#alias-models').value),required_capabilities:lines($('#alias-capabilities').value),visible:$('#alias-visible').checked};if(!name||!payload.models.length){toast('Alias-Name und mindestens ein Fallback-Modell sind erforderlich.',true);return}try{await api('/gateway/ui-api/model-aliases/'+encodeURIComponent(name),{method:'PUT',body:JSON.stringify(payload)});$('#alias-dialog').close();await reloadAliases();toast('Alias gespeichert und aktiviert')}catch(err){toast(err.message,true)}}); +function placementRule(worker){return (S.placement?.workers||[]).find(x=>x.worker===worker)} +function placementRows(){return S.placement?.models||[]} +function placementCell(row,worker){return row?.workers?.[worker]||{allowed:false,installed:false,loaded:false,source:'unknown'}} +function placementSourceLabel(c){if(c.source==='allow_rule')return c.pattern?`allow ${c.pattern}`:'allow';if(c.source==='deny_rule')return c.pattern?`deny ${c.pattern}`:'deny';if(c.source==='whitelist_default')return 'Whitelist';if(c.source==='allow_all_default')return 'Allow all';return c.source||'–'} +function renderPlacement(){if(!S.placement)return;const workers=S.placement.workers||[],rows=placementRows();const routable=r=>workers.some(w=>{const c=placementCell(r,w.worker);return c.allowed&&(c.installed||!w.inventory_known)}),unroutable=rows.filter(r=>!routable(r)).length;$('#placement-workers').textContent=num(workers.length);$('#placement-models').textContent=num(rows.length);$('#placement-overrides').textContent=num(workers.filter(w=>w.override).length);$('#placement-unroutable').textContent=num(unroutable);let list=rows.slice(),f=($('#placement-filter')?.value||'').trim().toLowerCase(),view=$('#placement-view')?.value||'all';if(f)list=list.filter(r=>r.model.toLowerCase().includes(f));if(view==='routable')list=list.filter(routable);else if(view==='unroutable')list=list.filter(r=>!routable(r));else if(view==='multi')list=list.filter(r=>workers.filter(w=>{const c=placementCell(r,w.worker);return c.allowed&&(c.installed||!w.inventory_known)}).length>1);const headers=['Modell',...workers.map(w=>`${esc(w.worker)}${w.override?' ↳':''}`)],tableRows=list.map(r=>{const destinations=workers.filter(w=>{const c=placementCell(r,w.worker);return c.allowed&&c.installed}).length;return [`
${esc(r.model)}${destinations?destinations+' aktive Ziel'+(destinations===1?'':'e'):'kein installiertes Ziel'}
`,...workers.map(w=>placementCellHTML(r,w))]});$('#placement-matrix').innerHTML=table(headers,tableRows);$('#placement-worker-cards').innerHTML=workers.length?workers.map(placementWorkerCard).join(''):'
Keine Worker
'} +function placementCellHTML(row,w){const c=placementCell(row,w.worker),klass=`placement-cell ${c.allowed?'allowed':'denied'} ${c.installed?'installed':''} ${c.loaded?'loaded':''}`,icon=c.loaded?'●':c.allowed?(c.installed?'✓':'○'):'⛔',title=`${row.model} → ${w.worker}\n${c.allowed?'erlaubt':'gesperrt'} · ${c.installed?'installiert':'nicht installiert'}${c.loaded?' · geladen':''}\nQuelle: ${placementSourceLabel(c)}\nKlick: ${c.exact_override?'exakte Ausnahme entfernen':c.allowed?'explizit sperren':'explizit erlauben'}`;return ``} +function placementWorkerCard(w){const e=w.effective||{},base=w.baseline||{},allow=e.allowed_models||[],deny=e.denied_models||[],mode=e.mode||'allow_all';return `
${esc(w.worker)}${w.inventory_known?'Inventar bekannt':'Inventar momentan unbekannt'}
${mode==='whitelist'?'Whitelist':'Allow all'}${w.override?'persistent override':'config default'}${JSON.stringify(e)!==JSON.stringify(base)?'abweichend':''}
Allow${allow.length?allow.slice(0,8).map(x=>`${esc(x)}`).join(''):'
keine expliziten Regeln
'}${allow.length>8?`+${allow.length-8}`:''}
Deny${deny.length?deny.slice(0,8).map(x=>`${esc(x)}`).join(''):'
keine expliziten Regeln
'}${deny.length>8?`+${deny.length-8}`:''}
${S.placement?.inventory_errors?.[w.worker]?`
Inventory: ${esc(S.placement.inventory_errors[w.worker])}
`:''}
`} +async function loadPlacement(){S.placement=await api('/gateway/ui-api/placement');renderPlacement();return S.placement} +function openPlacementDialog(worker){const w=placementRule(worker);if(!w)return;S.placementEditingWorker=worker;const e=w.effective||{};$('#placement-dialog-worker').textContent=worker+(w.override?' · persistenter Override':' · Config-Default');$('#placement-mode').value=e.mode||'allow_all';$('#placement-allowed').value=(e.allowed_models||[]).join('\n');$('#placement-denied').value=(e.denied_models||[]).join('\n');$('#placement-reset').classList.toggle('hidden',!w.override);$('#placement-dialog').showModal()} +function lines(v){return [...new Set(String(v||'').split(/\r?\n|,/).map(x=>x.trim()).filter(Boolean))]} +function installedForWorker(worker){return placementRows().filter(r=>placementCell(r,worker).installed).map(r=>r.model).sort()} +async function savePlacementRule(worker,rule){await api('/gateway/ui-api/placement/'+encodeURIComponent(worker),{method:'PUT',body:JSON.stringify(rule)});await loadPlacement();toast(`Routing-Regel für ${worker} gespeichert`)} +async function placementExactAction(worker,model,action){await api('/gateway/ui-api/placement/'+encodeURIComponent(worker)+'/model',{method:'POST',body:JSON.stringify({model,action})});await loadPlacement();toast(`${model} → ${worker}: ${action==='inherit'?'exakte Regel entfernt':action==='allow'?'erlaubt':'gesperrt'}`)} +$('#placement-filter')?.addEventListener('input',renderPlacement);$('#placement-view')?.addEventListener('change',renderPlacement);$('#refresh-placement')?.addEventListener('click',async()=>{try{await loadPlacement();toast('Placement aktualisiert')}catch(e){toast(e.message,true)}}); +$('#placement-matrix')?.addEventListener('click',async e=>{const b=e.target.closest('[data-placement-cell]');if(!b)return;const action=b.dataset.exact==='1'&&b.dataset.workerOverride==='1'?'inherit':b.dataset.allowed==='1'?'deny':'allow';try{b.disabled=true;await placementExactAction(b.dataset.worker,b.dataset.model,action)}catch(err){toast(err.message,true)}finally{b.disabled=false}}); +$('#placement-worker-cards')?.addEventListener('click',e=>{const b=e.target.closest('[data-placement-edit]');if(b)openPlacementDialog(b.dataset.placementEdit)}); +$('#placement-form')?.addEventListener('submit',async e=>{e.preventDefault();const worker=S.placementEditingWorker;if(!worker)return;const rule={mode:$('#placement-mode').value,allowed_models:lines($('#placement-allowed').value),denied_models:lines($('#placement-denied').value)};try{await savePlacementRule(worker,rule);$('#placement-dialog').close()}catch(err){toast(err.message,true)}}); +$('#placement-reset')?.addEventListener('click',async()=>{const worker=S.placementEditingWorker;if(!worker||!confirm(`Placement-Override für ${worker} löschen und auf config.json zurücksetzen?`))return;try{await api('/gateway/ui-api/placement/'+encodeURIComponent(worker),{method:'DELETE'});await loadPlacement();$('#placement-dialog').close();toast('Config-Default wiederhergestellt')}catch(e){toast(e.message,true)}}); +$$('[data-placement-preset]').forEach(b=>b.addEventListener('click',()=>{const worker=S.placementEditingWorker;if(!worker)return;switch(b.dataset.placementPreset){case'all':$('#placement-mode').value='allow_all';$('#placement-allowed').value='';$('#placement-denied').value='';break;case'installed':{const w=placementRule(worker),installed=installedForWorker(worker);if(!w?.inventory_known&&!installed.length){toast('Inventar dieses Workers ist unbekannt; Preset wurde nicht angewendet.',true);return}$('#placement-mode').value='whitelist';$('#placement-allowed').value=installed.join('\n');$('#placement-denied').value='';break;}case'none':$('#placement-mode').value='whitelist';$('#placement-allowed').value='';$('#placement-denied').value='';break}})); +function renderModelAccess(){const el=$('#model-access-table');if(!el)return;const ma=S.overview?.model_access||{},def=ma.default||{},tenants=ma.tenants||{};const rows=[['* default',esc(def.mode||'allow_all'),esc((def.allowed_models||[]).join(', ')||'–'),esc((def.denied_models||[]).join(', ')||'–'),'Config-Basis'],...Object.entries(tenants).sort((a,b)=>a[0].localeCompare(b[0])).map(([tenant,r])=>[esc(tenant),esc(r.mode||'allow_all'),esc((r.allowed_models||[]).join(', ')||'–'),esc((r.denied_models||[]).join(', ')||'–'),`
`])];el.innerHTML=table(['Tenant','Modus','Allow','Deny','Aktionen'],rows)} +function openModelAccessDialog(tenant=''){const tenants=S.overview?.model_access?.tenants||{},r=tenants[tenant]||{};$('#model-access-tenant').value=tenant;$('#model-access-tenant').readOnly=!!tenant;$('#model-access-mode').value=r.mode||'allow_all';$('#model-access-allowed').value=(r.allowed_models||[]).join('\n');$('#model-access-denied').value=(r.denied_models||[]).join('\n');$('#model-access-dialog').showModal()} +async function reloadModelAccess(){S.overview=await api('/gateway/ui-api/overview');if(S.config)S.config=await api('/gateway/ui-api/config');renderModelAccess();renderPolicySimulator()} +$('#new-model-access')?.addEventListener('click',()=>openModelAccessDialog('')); +$('#model-access-table')?.addEventListener('click',async e=>{const edit=e.target.closest('[data-model-access-edit]');if(edit){openModelAccessDialog(edit.dataset.modelAccessEdit);return}const del=e.target.closest('[data-model-access-delete]');if(!del)return;const tenant=del.dataset.modelAccessDelete;if(!confirm(`Tenant Model-ACL für ${tenant} auf die Default-Regel zurücksetzen?`))return;try{await api('/gateway/ui-api/model-access/'+encodeURIComponent(tenant),{method:'DELETE'});await reloadModelAccess();toast('Tenant Model-ACL zurückgesetzt')}catch(err){toast(err.message,true)}}); +$('#model-access-form')?.addEventListener('submit',async e=>{e.preventDefault();const tenant=$('#model-access-tenant').value.trim(),payload={mode:$('#model-access-mode').value,allowed_models:lines($('#model-access-allowed').value),denied_models:lines($('#model-access-denied').value)};if(!tenant){toast('Tenant ist erforderlich.',true);return}try{await api('/gateway/ui-api/model-access/'+encodeURIComponent(tenant),{method:'PUT',body:JSON.stringify(payload)});$('#model-access-dialog').close();await reloadModelAccess();toast('Tenant Model-ACL gespeichert und aktiviert')}catch(err){toast(err.message,true)}}); +function renderFairness(){const c=S.overview.scheduler_config||{};$('#fair-concurrency').textContent=num(c.global_concurrency);$('#fair-maxqueue').textContent=num(c.max_queue);$('#fair-actorqueue').textContent=num(c.max_queue_per_actor);const p=S.policies||{baseline:{},overrides:{}};const names=[...new Set([...Object.keys(p.baseline||{}),...Object.keys(p.overrides||{})])].sort();$('#policy-table').innerHTML=table(['Tenant','Quelle','Tenant W.','Actor W.','Actor cr/min','Actor Burst','Tenant cr/min','Tenant Burst','Aktionen'],names.map(n=>{const ov=p.overrides?.[n],x=ov||p.baseline[n];return[esc(n),`${ov?'persistent':'config'}`,num(x.tenant_weight,2),num(x.actor_weight,2),infinity(x.actor_credits_per_minute),infinity(x.actor_burst_credits),infinity(x.tenant_credits_per_minute),infinity(x.tenant_burst_credits),`
${ov?``:''}
`] }));const sc=S.overview.service_classes||{},stats=S.overview.scheduler?.classes||{};const cls=Object.keys(sc.classes||{}).sort();$('#service-class-table')&&($('#service-class-table').innerHTML=table(['Klasse','Default','Weight','Max Queue','Concurrency','Queued','Running'],cls.map(n=>{const x=sc.classes[n]||{},st=stats[n]||{};return[esc(n),n===sc.default?'default':'',num(x.weight,2),esc(x.max_queue_wait||'–'),Number(x.max_concurrent||0)>0?num(x.max_concurrent):'∞',num(st.queued),num(st.running)]})));renderBars('#tenant-usage',(S.overview.tenants||[]).map(x=>({name:x.name,credits:x.summary.credits})),'credits')} +$('#new-policy').onclick=()=>openPolicy('');function syncUnlimited(prefix,on){for(const id of prefix==='actor'?['#policy-acpm','#policy-ab']:['#policy-tcpm','#policy-tb']){$(id).disabled=on;if(on)$(id).value=0}}function openPolicy(n){const p=S.policies?.overrides?.[n]||S.policies?.baseline?.[n]||{};$('#policy-tenant').value=n;$('#policy-tenant').readOnly=!!n;for(const [id,k] of [['#policy-tw','tenant_weight'],['#policy-aw','actor_weight'],['#policy-acpm','actor_credits_per_minute'],['#policy-ab','actor_burst_credits'],['#policy-tcpm','tenant_credits_per_minute'],['#policy-tb','tenant_burst_credits']])$(id).value=p[k]??'';const au=Number(p.actor_credits_per_minute||0)<=0&&Number(p.actor_burst_credits||0)<=0,tu=Number(p.tenant_credits_per_minute||0)<=0&&Number(p.tenant_burst_credits||0)<=0;$('#policy-actor-unlimited').checked=au;$('#policy-tenant-unlimited').checked=tu;syncUnlimited('actor',au);syncUnlimited('tenant',tu);$('#policy-dialog').showModal()} +$('#policy-actor-unlimited')?.addEventListener('change',e=>syncUnlimited('actor',e.target.checked));$('#policy-tenant-unlimited')?.addEventListener('change',e=>syncUnlimited('tenant',e.target.checked));$('#policy-form').addEventListener('submit',async e=>{e.preventDefault();const n=$('#policy-tenant').value.trim(),v=id=>Number($(id).value||0),policy={tenant_weight:v('#policy-tw'),actor_weight:v('#policy-aw'),actor_credits_per_minute:$('#policy-actor-unlimited').checked?0:v('#policy-acpm'),actor_burst_credits:$('#policy-actor-unlimited').checked?0:v('#policy-ab'),tenant_credits_per_minute:$('#policy-tenant-unlimited').checked?0:v('#policy-tcpm'),tenant_burst_credits:$('#policy-tenant-unlimited').checked?0:v('#policy-tb')};try{await api('/gateway/ui-api/policies/'+encodeURIComponent(n),{method:'PUT',body:JSON.stringify(policy)});$('#policy-dialog').close();toast('Policy gespeichert');S.policies=await api('/gateway/ui-api/policies');renderFairness()}catch(x){toast(x.message,true)}});async function deletePolicy(n){if(!confirm(`Runtime-Override für ${n} entfernen?`))return;try{await api('/gateway/ui-api/policies/'+encodeURIComponent(n),{method:'DELETE'});S.policies=await api('/gateway/ui-api/policies');renderFairness();toast('Override entfernt')}catch(e){toast(e.message,true)}} +function autotuneAppliedCount(a){return Object.values(a||{}).reduce((n,m)=>n+Object.keys(m||{}).length,0)} +function renderAutoTune(){const el=$('#autotune-table');if(!el)return;const a=S.autotune||{enabled:false,profiles:[],applied:{},config:{}},profiles=a.profiles||[],ot=S.overview?.opentelemetry||{};$('#autotune-enabled').textContent=a.enabled?'aktiv':'aus';$('#autotune-profiles').textContent=num(profiles.length);$('#autotune-applied').textContent=num(autotuneAppliedCount(a.applied));$('#otel-exported').textContent=num(ot.exported_spans||0);$('#otel-status').textContent=ot.enabled?`${num(ot.failed_spans||0)} failed · ${num(ot.dropped_spans||0)} dropped`:'deaktiviert';const oc=$('#otel-card');if(oc)oc.innerHTML=`
Status
${ot.enabled?'aktiv':'aus'}
Endpoint
${esc(ot.endpoint||'–')}
Service
${esc(ot.service_name||'–')}
Sampling
${num(Number(ot.sample_ratio||0)*100,1)}%
Exportiert
${num(ot.exported_spans||0)} Spans
Fehlgeschlagen
${num(ot.failed_spans||0)}
Gedroppt
${num(ot.dropped_spans||0)}
`;const ws=(S.overview?.workers||[]).map(w=>w.name),wsel=$('#autotune-worker'),msel=$('#autotune-model');if(wsel&&document.activeElement!==wsel){const old=wsel.value;wsel.innerHTML=ws.map(n=>``).join('');if(ws.includes(old))wsel.value=old}const selectedWorker=wsel?.value||ws[0],models=[...new Set((S.models||[]).filter(x=>!selectedWorker||x.worker===selectedWorker).flatMap(x=>(x.models||[]).map(m=>m.model||m.name)).filter(Boolean))].sort();if(msel&&document.activeElement!==msel){const old=msel.value;msel.innerHTML=models.map(n=>``).join('');if(models.includes(old))msel.value=old}el.innerHTML=table(['Start','Worker / Modell','Status','Empfehlung','Bestes Level','Aktionen'],profiles.map(p=>{const best=(p.levels||[]).slice().sort((x,y)=>(y.score||0)-(x.score||0))[0],actions=p.status==='running'||p.status==='queued'?``:p.status==='completed'?` `:``;return[when(p.started_at),`${esc(p.worker)}
${esc(p.model)}`,`${esc(p.status)}${p.error?`
${esc(p.error)}`:''}`,p.recommended_concurrency?`${num(p.recommended_concurrency)}${p.applied?' · applied':''}`:'–',best?`${num(best.aggregate_output_tps,1)} tok/s · TTFT p95 ${num(best.ttft_p95_ms,0)} ms`:'–',actions]}));if(!S.autotuneSelected&&profiles.length)S.autotuneSelected=profiles[0].id;renderAutoTuneLevels()} +function renderAutoTuneLevels(){const el=$('#autotune-levels');if(!el)return;const p=(S.autotune?.profiles||[]).find(x=>x.id===S.autotuneSelected);if(!p){el.innerHTML='
Kein Profil ausgewählt
';return}$('#autotune-detail-title').textContent=`${p.worker} · ${p.model} · Empfehlung ${p.recommended_concurrency||'–'}`;el.innerHTML=table(['Concurrency','Erfolg','TTFT p50','TTFT p95','Service Ø','Output tok/s Ø','Aggregate tok/s','Peak VRAM','Peak GPU','Score'],(p.levels||[]).map(x=>[num(x.concurrency),`${num(x.successful)}/${num(x.requests)}`,`${num(x.ttft_p50_ms,0)} ms`,`${num(x.ttft_p95_ms,0)} ms`,`${num(x.mean_service_ms,0)} ms`,num(x.mean_output_tps,1),num(x.aggregate_output_tps,1),x.peak_vram_bytes?bytes(x.peak_vram_bytes):'–',x.peak_gpu_percent?num(x.peak_gpu_percent,1)+'%':'–',num(x.score,3)]))} +function warmAllModels(){return[...new Set((S.models||[]).flatMap(x=>(x.models||[]).map(m=>m.model||m.name)).filter(Boolean))].sort()} +function warmWorkers(){return(S.overview?.workers||[]).map(w=>w.name)} +function clearWarmForm(){S.warmEditing=null;$('#warm-pattern').value='';$('#warm-class').value='warm';$('#warm-replicas').value='1';$('#warm-idle').value='30m';$('#warm-preload').checked=false;$$('[data-warm-worker]').forEach(x=>x.checked=false)} +function editWarmRule(pattern){const p=S.warm?.policies?.[pattern];if(!p)return;S.warmEditing=pattern;$('#warm-pattern').value=pattern;$('#warm-class').value=p.class||'warm';$('#warm-replicas').value=Number(p.replicas||1);$('#warm-idle').value=p.idle_timeout||((p.class==='cold')?'5m':'30m');$('#warm-preload').checked=!!p.preload;const set=new Set(p.workers||[]);$$('[data-warm-worker]').forEach(x=>x.checked=set.has(x.value));$('#warm-pattern').focus()} +function renderWarm(){const x=S.warm;if(!x||!$('#warm-policy-table'))return;const ps=x.policies||{},actions=x.actions||[],sugs=x.eviction_suggestions||[];$('#warm-enabled').textContent=x.enabled?'aktiv':'aus';$('#warm-policy-count').textContent=num(Object.keys(ps).length);$('#warm-running').textContent=num(actions.filter(a=>a.status==='running').length);$('#warm-suggestions').textContent=num(sugs.length);const dl=$('#warm-model-list');if(dl)dl.innerHTML=warmAllModels().map(m=>``).join('');const ww=$('#warm-workers');if(ww&&document.activeElement?.closest?.('#warm-policy-form')!==ww){const checked=new Set($$('[data-warm-worker]').filter(x=>x.checked).map(x=>x.value));ww.innerHTML=warmWorkers().map(w=>``).join('')||'Keine Worker'}const rows=Object.keys(ps).sort().map(k=>{const p=ps[k],workers=(p.workers||[]).length?(p.workers||[]).join(', '):'auto';return[`${esc(k)}`,`${esc(p.class||'warm')}`,num(p.replicas||1),esc(workers),p.preload?'ja':'–',esc(p.idle_timeout||'–'),`
`]});$('#warm-policy-table').innerHTML=table(['Modell / Pattern','Klasse','Replicas','Worker','Preload','Idle','Aktionen'],rows);$('#warm-suggestion-list').innerHTML=sugs.length?sugs.map(v=>`
${esc(v.model)}${esc(v.worker)} · ${esc(v.class)} · ${num(v.vram_percent,1)}% VRAM${v.last_used?' · last '+esc(when(v.last_used)):''}
Evict candidate
`).join(''):'
Keine Eviction-Empfehlungen
';$('#warm-actions-table').innerHTML=table(['Zeit','Aktion','Worker','Modell','Policy','Status','Meldung'],actions.slice(0,100).map(a=>[when(a.time),esc(a.type),esc(a.worker),esc(a.model),esc(a.policy||'–'),`${esc(a.status)}`,esc(a.message||'–')]))} +async function reloadWarm(){S.warm=await api('/gateway/ui-api/warm-models');renderWarm()} +function renderAlerts(){const x=S.alerts;if(!x||!$('#alerts-active-list'))return;const active=x.active||[],wh=x.webhooks||[],hist=x.history||[],del=x.deliveries||[];$('#alerts-enabled').textContent=x.enabled?'aktiv':'aus';$('#alerts-active').textContent=num(active.length);$('#alerts-webhooks').textContent=num(wh.filter(w=>w.enabled).length);$('#alerts-last').textContent=x.last_evaluate?new Date(x.last_evaluate).toLocaleTimeString('de-DE'):'–';$('#alerts-active-list').innerHTML=active.length?active.map(a=>`
${esc(a.type)}${esc(a.message)} · seit ${esc(when(a.started_at))}
${esc(a.severity)}
`).join(''):'
Keine aktiven Alerts
';$('#alerts-webhook-list').innerHTML=wh.length?wh.map(w=>`
${esc(w.name||'webhook')}${esc(w.url)} · ${w.signed?'HMAC signiert':'unsigned'}
${w.enabled?``:''}
`).join(''):'
Keine Webhooks konfiguriert
';$('#alerts-history-table').innerHTML=table(['Zeit','Status','Typ','Ziel','Meldung'],hist.slice(0,200).map(a=>[when(a.updated_at),`${esc(a.state)}`,esc(a.type),esc(a.worker||a.tenant||'–'),esc(a.message)]));$('#alerts-delivery-table').innerHTML=table(['Zeit','Event','Webhook','Versuch','HTTP','Fehler'],del.slice(0,100).map(d=>[when(d.time),esc(d.event_id),esc(d.webhook),num(d.attempt||1),d.status_code?num(d.status_code):'–',esc(d.error||'–')]))} +async function reloadAlerts(){S.alerts=await api('/gateway/ui-api/alerts');renderAlerts()} +async function reloadAutoTune(){S.autotune=await api('/gateway/ui-api/autotune');renderAutoTune()} +$('#autotune-worker')?.addEventListener('change',renderAutoTune); +$('#refresh-autotune')?.addEventListener('click',()=>reloadAutoTune().catch(e=>toast(e.message,true))); +$('#warm-policy-form')?.addEventListener('submit',async e=>{e.preventDefault();const pattern=$('#warm-pattern').value.trim();if(!pattern)return;const policies=structuredClone(S.warm?.policies||{});if(S.warmEditing&&S.warmEditing!==pattern)delete policies[S.warmEditing];policies[pattern]={class:$('#warm-class').value,replicas:Number($('#warm-replicas').value||1),preload:$('#warm-preload').checked,idle_timeout:$('#warm-idle').value.trim()||($('#warm-class').value==='cold'?'5m':'30m'),workers:$$('[data-warm-worker]').filter(x=>x.checked).map(x=>x.value)};try{S.warm=await api('/gateway/ui-api/warm-models',{method:'PUT',body:JSON.stringify({policies})});clearWarmForm();renderWarm();toast('Warm-Policy gespeichert')}catch(err){toast(err.message,true)}}); +$('#warm-form-clear')?.addEventListener('click',clearWarmForm); +$('#warm-policy-table')?.addEventListener('click',async e=>{const ed=e.target.closest('[data-warm-edit]');if(ed){editWarmRule(ed.dataset.warmEdit);return}const del=e.target.closest('[data-warm-delete]');if(del){const pattern=del.dataset.warmDelete;if(!confirm(`Warm-Policy ${pattern} löschen?`))return;const policies=structuredClone(S.warm?.policies||{});delete policies[pattern];try{S.warm=await api('/gateway/ui-api/warm-models',{method:'PUT',body:JSON.stringify({policies})});renderWarm();toast('Warm-Policy gelöscht')}catch(err){toast(err.message,true)}}}); +$('#warm-reset')?.addEventListener('click',async()=>{if(!confirm('Alle Runtime-Warm-Policies auf Config-Default zurücksetzen?'))return;try{S.warm=await api('/gateway/ui-api/warm-models',{method:'DELETE'});clearWarmForm();renderWarm();toast('Warm-Policies zurückgesetzt')}catch(err){toast(err.message,true)}}); +$('#warm-reconcile')?.addEventListener('click',async()=>{try{S.warm=await api('/gateway/ui-api/warm-models/reconcile',{method:'POST'});renderWarm();toast('Residency-Abgleich gestartet')}catch(err){toast(err.message,true)}}); +$('#alerts-webhook-list')?.addEventListener('click',async e=>{const b=e.target.closest('[data-alert-test]');if(!b)return;try{await api('/gateway/ui-api/alerts/test',{method:'POST',body:JSON.stringify({name:b.dataset.alertTest})});toast('Test-Webhook gesendet');setTimeout(()=>reloadAlerts().catch(()=>{}),500)}catch(err){toast(err.message,true)}}); +$('#autotune-form')?.addEventListener('submit',async e=>{e.preventDefault();const body={worker:$('#autotune-worker').value,model:$('#autotune-model').value,max_concurrency:Number($('#autotune-max-concurrency').value||0),samples_per_level:Number($('#autotune-samples').value||0),max_tokens:Number($('#autotune-max-tokens').value||0)};if(!body.worker||!body.model)return toast('Worker und Modell auswählen.',true);if(!confirm(`Benchmark für ${body.model} auf ${body.worker} starten? Dies erzeugt echte GPU-Last.`))return;try{const out=await api('/gateway/ui-api/autotune/start',{method:'POST',body:JSON.stringify(body)});S.autotuneSelected=out.profile?.id;toast('Auto-Tuning gestartet');await reloadAutoTune()}catch(err){toast(err.message,true)}}); +$('#autotune-table')?.addEventListener('click',async e=>{const show=e.target.closest('[data-autotune-show]');if(show){S.autotuneSelected=show.dataset.autotuneShow;renderAutoTuneLevels();return}const cancel=e.target.closest('[data-autotune-cancel]');if(cancel){try{await api('/gateway/ui-api/autotune/'+encodeURIComponent(cancel.dataset.autotuneCancel)+'/cancel',{method:'POST'});toast('Benchmark-Abbruch angefordert');await reloadAutoTune()}catch(err){toast(err.message,true)}return}const apply=e.target.closest('[data-autotune-apply]');if(apply){try{await api('/gateway/ui-api/autotune/'+encodeURIComponent(apply.dataset.autotuneApply)+'/apply',{method:'POST'});toast('Empfohlene Concurrency persistent angewendet');await reloadAutoTune();await refreshAll(false)}catch(err){toast(err.message,true)}}}); +function renderUsageMine(){const u=S.myUsage||{};$('#my-requests').textContent=num(u.requests);$('#my-credits').textContent=credits(u.credits);$('#my-prompt').textContent=num(u.prompt_tokens);$('#my-completion').textContent=num(u.completion_tokens)} +function renderUsage(){let a=S.recent.slice(0,Number($('#usage-limit').value||250)),q=($('#usage-filter').value||'').toLowerCase(),st=$('#usage-status').value;if(q)a=a.filter(e=>`${e.tenant} ${e.actor} ${e.subject} ${e.model} ${e.path} ${e.worker}`.toLowerCase().includes(q));if(st)a=a.filter(e=>String(e.status).startsWith(st));$('#usage-table').innerHTML=tableRecent(a,false);renderRollups()} +async function loadRollups(){const g=$('#rollup-granularity')?.value||'daily',d=$('#rollup-dimension')?.value||'global',n=$('#rollup-name')?.value?.trim()||'';if(d!=='global'&&!n){S.rollups={points:[],granularity:g,dimension:d};renderRollups();return}const q=new URLSearchParams({granularity:g,dimension:d,limit:g==='daily'?'120':'120'});if(n)q.set('name',n);S.rollups=await api('/gateway/ui-api/usage/rollups?'+q.toString());renderRollups()} +function renderRollups(){const el=$('#rollup-table');if(!el)return;const a=S.rollups?.points||[];el.innerHTML=table(['Periode','Requests','Fehler','Prompt','Completion','Credits','Ø Queue','Ø Service','Prompt tok/s','Output tok/s'],a.slice().reverse().map(x=>[esc(x.period),num(x.requests),num(x.errors),num(x.prompt_tokens),num(x.completion_tokens),credits(x.credits),x.requests?ms(x.queue_ms/x.requests):'–',x.requests?ms(x.service_ms/x.requests):'–',x.prompt_tps?num(x.prompt_tps,1):'–',x.output_tps?num(x.output_tps,1):'–']))} +$('#usage-filter').addEventListener('input',renderUsage);$('#usage-status').addEventListener('change',renderUsage);$('#usage-limit').addEventListener('change',renderUsage);$('#rollup-granularity')?.addEventListener('change',()=>{S.rollups=null;loadRollups().catch(e=>toast(e.message,true))});$('#rollup-dimension')?.addEventListener('change',()=>{const d=$('#rollup-dimension').value;$('#rollup-name').classList.toggle('hidden',d==='global');S.rollups=null;if(d==='global')loadRollups().catch(e=>toast(e.message,true))});$('#rollup-name')?.addEventListener('change',()=>{S.rollups=null;loadRollups().catch(e=>toast(e.message,true))}); +function tableRecent(a,mini){const heads=mini?['Zeit','Modell','Status','Latenz']:['Zeit','Tenant / Actor','API / Pfad','Modell / Worker','Status','Queue','Service','Tokens','Credits'];const rows=a.map(e=>mini?[new Date(e.time).toLocaleTimeString('de-DE'),esc(e.model||'–'),status(e.status),ms(e.service_ms)]:[when(e.time),`${esc(e.tenant)}
${esc(e.actor||e.subject)}`,`${esc(e.api)}
${esc(e.path)}`,`${esc(e.model||'–')}
${esc(e.worker||'–')}`,status(e.status),ms(e.queue_ms),ms(e.service_ms),`${num(e.usage?.prompt_tokens)} / ${num(e.usage?.completion_tokens)}`,credits(e.actual_credits)]);return table(heads,rows)}function status(s){const c=s>=200&&s<400?'ok':s>=500?'bad':'warn';return `${s||'–'}`} +function renderJobs(){const el=$('#jobs-table');if(!el)return;const a=(S.jobs||[]).slice().sort((x,y)=>new Date(x.queued_at||x.updated_at)-new Date(y.queued_at||y.updated_at));el.innerHTML=table(['Request','Tenant / Actor','Modell / Worker','Status','Laufzeit','Tokens','Aktion'],a.map(j=>{const active=!['completed','failed','cancelled'].includes(j.state),dur=j.started_at?liveAge(j,'started_at'):liveAge(j,'queued_at'),tok=Number(j.prompt_tokens||j.estimated_prompt_tokens||0)+Number(j.completion_tokens||0),cancel=j.cancelling?'cancelling':active&&j.cancellable?``:'–';return [`${esc(j.id)}
${esc(j.api||'')} · ${esc(j.path||'')}`,`${esc(j.tenant)}
${esc(j.actor||'–')}`,`${esc(j.model||'–')}
${esc(j.worker||'wartet auf Routing')}`,`${esc(j.state)}`,`${dur.toFixed(dur<10?1:0)} s`,num(tok),cancel]}))} +function renderBatches(){const el=$('#batches-table'),meta=$('#batches-meta');if(!el||!meta)return;const b=S.batches;if(!b){meta.textContent='Lade Batch-Status …';el.innerHTML='';return}if(!b.enabled){meta.className='callout warn-callout';meta.textContent='Durable Batch Jobs sind deaktiviert. Aktiviere batch_jobs.enabled und die Service Class batch in der Konfiguration.';el.innerHTML='
Batch-System deaktiviert
';return}meta.className='callout';meta.textContent=`Retention ${b.retention||'–'} · maximal ${num(b.max_jobs)} Jobs · ${num(b.max_concurrent)} parallel · Input-Limit ${bytes(b.max_input_bytes)}`;const a=(b.jobs||[]).slice().sort((x,y)=>new Date(y.created_at)-new Date(x.created_at));el.innerHTML=table(['Batch','Tenant / Actor','Request','Status','Versuche','Zeit','Aktionen'],a.map(j=>{const state=j.state||'',statusClass=state==='completed'?'ok':state==='failed'||state==='cancelled'?'bad':state==='running'||state==='pausing'||state==='cancelling'?'warn':'';const actions=[];if(state==='queued'||state==='running')actions.push(``);if(state==='paused')actions.push(``);if(['queued','running','paused','pausing'].includes(state))actions.push(``);if(j.output_ref)actions.push(``);const times=`${when(j.created_at)}${j.finished_at?`
fertig ${when(j.finished_at)}`:j.started_at?`
gestartet ${when(j.started_at)}`:''}`;const err=j.error?`
${esc(j.error)}`:'';return [`${esc(j.id)}
${esc(j.service_class||'batch')}`,`${esc(j.identity?.tenant||'–')}
${esc(j.identity?.actor||j.identity?.subject||'–')}`,`${esc(j.model||'–')}
${esc(j.path||'–')}`,`${esc(state)}${err}`,num(j.attempts),times,actions.join(' ')||'–']}))} +async function batchAction(id,action){try{await api('/gateway/ui-api/batches/'+encodeURIComponent(id)+'/'+action,{method:'POST'});toast(`Batch ${action}: ${id}`);S.batches=await api('/gateway/ui-api/batches');renderBatches()}catch(e){toast(e.message,true)}} +async function downloadBatchOutput(id){try{const r=await fetch('/gateway/ui-api/batches/'+encodeURIComponent(id)+'/output',{headers:authHeaders(),credentials:'same-origin'});if(!r.ok){let msg=`HTTP ${r.status}`;try{const x=await r.json();msg=x?.error?.message||x?.message||msg}catch{}throw new Error(msg)}const blob=await r.blob(),u=URL.createObjectURL(blob),a=document.createElement('a');a.href=u;a.download=id+'.response';document.body.appendChild(a);a.click();a.remove();setTimeout(()=>URL.revokeObjectURL(u),1000)}catch(e){toast(e.message,true)}} +async function cancelJob(id){if(!confirm(`Request ${id} wirklich abbrechen?`))return;try{await api('/gateway/ui-api/jobs/'+encodeURIComponent(id)+'/cancel',{method:'POST'});toast('Job-Abbruch angefordert');S.jobs=(await api('/gateway/ui-api/jobs')).jobs||[];renderJobs()}catch(e){toast(e.message,true)}} +$('#refresh-jobs')?.addEventListener('click',async()=>{try{S.jobs=(await api('/gateway/ui-api/jobs')).jobs||[];renderJobs()}catch(e){toast(e.message,true)}}); +$('#refresh-batches')?.addEventListener('click',async()=>{try{S.batches=await api('/gateway/ui-api/batches');renderBatches()}catch(e){toast(e.message,true)}}); +function renderOperations(){const a=S.operations;$('#operations-list').innerHTML=a.length?a.map(o=>`
${esc(o.type)} · ${esc(o.model)}${esc(o.worker)} · ${when(o.started_at)}
${esc(o.status)}
${esc(o.message||'')} ${o.total?`· ${bytes(o.completed)} / ${bytes(o.total)}`:''}
${o.error?`
${esc(o.error)}
`:''}${['running','queued'].includes(o.status)?`
`:''}
`).join(''):'
Keine Operationen
'} +async function cancelOperation(id){try{await api('/gateway/ui-api/operations/'+id+'/cancel',{method:'POST'});toast('Abbruch angefordert');await refreshAll(false)}catch(e){toast(e.message,true)}} +function renderStorage(){const st=S.storage;if(!st)return;const rt=st.usage?.retention||{};$('#storage-total').textContent=bytes(st.total_bytes);$('#storage-usage-files').textContent=num((rt.raw_files??st.usage?.files)||0);$('#storage-config-override').textContent=st.config_override_active?'aktiv':'bootstrap';$('#storage-flush-interval').textContent=st.flush_interval||'–';$('#storage-summary').innerHTML=`${esc(st.mode||'local-persistent')} · ${esc(st.data_dir||'–')} · Detail ${bytes(rt.raw_bytes||st.usage?.size_bytes||0)} · Rollups ${bytes((rt.daily_bytes||0)+(rt.monthly_bytes||0))}`;const rows=(st.files||[]).map(f=>[esc(f.name),esc(f.kind),`${esc(f.path)}`,f.exists?'persistent':'noch leer',bytes(f.size_bytes||0),f.modified_at?esc(when(f.modified_at)):'–']);$('#storage-table').innerHTML=table(['Store','Typ','Pfad','Status','Größe','Geändert'],rows);$('#storage-volatile').innerHTML=(st.volatile||[]).map(x=>`${esc(x)}`).join('')||'–';const monthly=Number(rt.monthly_months||0)<=0?'unbegrenzt':`${num(rt.monthly_months)} Monate`;$('#retention-summary').innerHTML=`
Request-Details${num(rt.detail_days||0)} Tage${num(rt.raw_files||0)} Dateien · ${bytes(rt.raw_bytes||0)}
Tagesaggregate${num(rt.daily_days||0)} Tage${num(rt.daily_files||0)} Dateien · ${bytes(rt.daily_bytes||0)}
Monatsaggregate${monthly}${num(rt.monthly_files||0)} Dateien · ${bytes(rt.monthly_bytes||0)}
Letzte Kompaktierung${rt.last_compaction?esc(when(rt.last_compaction)):'–'}${rt.last_reclaimed_bytes?bytes(rt.last_reclaimed_bytes)+' freigegeben':'kein Speicher freigegeben'}${rt.last_error?' · '+esc(rt.last_error):''}
`;$('#storage-flush').onclick=async()=>{try{const out=await api('/gateway/ui-api/storage/flush',{method:'POST'});S.storage=out.storage;renderStorage();toast('Persistenter State geflusht')}catch(e){toast(e.message,true)}};$('#storage-compact').onclick=async()=>{if(!confirm('Usage-Journale jetzt gemäß Retention-Regeln kompaktieren?'))return;try{const out=await api('/gateway/ui-api/storage/compact',{method:'POST'});S.storage=out.storage;S.rollups=null;renderStorage();toast(`Kompaktierung abgeschlossen · ${bytes(out.retention?.last_reclaimed_bytes||0)} freigegeben`)}catch(e){toast(e.message,true)}};$('#storage-backup').onclick=async()=>{try{const r=await fetch('/gateway/ui-api/storage/backup',{headers:authHeaders(),credentials:'same-origin'});if(!r.ok){let msg=`HTTP ${r.status}`;try{const x=await r.json();msg=x?.error?.message||x?.error||msg}catch{}throw new Error(msg)}const blob=await r.blob(),cd=r.headers.get('content-disposition')||'',m=/filename="?([^";]+)"?/i.exec(cd),name=m?.[1]||'ollama-gateway-backup.zip',u=URL.createObjectURL(blob),a=document.createElement('a');a.href=u;a.download=name;document.body.appendChild(a);a.click();a.remove();setTimeout(()=>URL.revokeObjectURL(u),1000);toast('Backup erstellt')}catch(e){toast(e.message,true)}}} + +function simulatorModels(){const set=new Set();(S.models||[]).forEach(inv=>(inv.models||[]).forEach(m=>set.add(m.model||m.name)));Object.keys(S.overview?.model_aliases||{}).forEach(x=>set.add(x));return [...set].filter(Boolean).sort()} +function simulatorTenants(){const set=new Set();Object.keys(S.overview?.tenants||{}).forEach(x=>set.add(x));Object.keys(S.policies?.effective||S.policies?.policies||{}).filter(x=>x!=='*').forEach(x=>set.add(x));(S.apiKeys||[]).forEach(k=>k.tenant&&set.add(k.tenant));return [...set].sort()} +function renderPolicySimulator(){const form=$('#policy-simulator-form');if(!form)return;$('#sim-tenant-list').innerHTML=simulatorTenants().map(x=>``).join('');$('#sim-model-list').innerHTML=simulatorModels().map(x=>``).join('');const tenant=$('#sim-tenant')?.value||'';const keys=(S.apiKeys||[]).filter(k=>!tenant||k.tenant===tenant);const keySel=$('#sim-api-key');const old=keySel?.value||'';if(keySel){keySel.innerHTML=''+keys.map(k=>``).join('');if([...keySel.options].some(o=>o.value===old))keySel.value=old}const sc=$('#sim-service-class');if(sc){const oldsc=sc.value;sc.innerHTML=''+Object.keys(S.overview?.service_classes?.classes||{}).sort().map(x=>``).join('');if([...sc.options].some(o=>o.value===oldsc))sc.value=oldsc}if(!S.simulation)return;const r=S.simulation,ok=r.decision==='would_route';const access=r.access||{};const alias=(r.alias_candidates||[]).map(x=>`
${esc(x.model)}${x.routable?'routbar':esc(x.reason||'nicht routbar')}
`).join('');const workers=(r.workers||[]).map((w,i)=>`${esc(w.worker)}${w.worker===r.selected_worker?' selected':''}${w.eligible?'✓':'✗'}${w.available_now?' · frei':w.eligible?' · ausgelastet':''}${w.placement?.allowed?'✓ '+esc(w.placement.source||''):'✗ '+esc(w.placement?.source||'')}${w.inventory_known?(w.installed?'installiert':'fehlt'):'unbekannt'}${w.loaded?' · loaded':''}${w.active||0}/${w.max_concurrent||0} · model ${w.model_active||0}/${w.model_limit||0}${w.output_tps?num(w.output_tps,1)+' tok/s':'–'}${w.vram_percent?num(w.vram_percent,1)+' %':'–'}${w.eligible?num(w.score,1):esc(w.reason||'–')}`).join('');const errors=(r.errors||[]).length?`
Hinweise: ${esc(r.errors.join(' · '))}
`:'';$('#policy-simulator-result').innerHTML=`
Entscheidung${esc(r.decision)}
Auflösung${esc(r.requested_model)}${r.resolved_model&&r.resolved_model!==r.requested_model?' → '+esc(r.resolved_model):''}
Worker${esc(r.selected_worker||'–')}
Estimated Cost${num(r.estimated_credits||0,2)} cr

Model Access

Tenant ACL
${access.tenant_allowed?'✓ erlaubt':'✗ gesperrt'}
API-Key ACL
${access.key_applied?(access.key_allowed?'✓ erlaubt':'✗ gesperrt'):'nicht gesetzt'}
Gesamt
${access.allowed?'✓ PASS':'✗ DENY'}

Preflight

Capabilities
${r.capabilities_ok?'✓':'✗'} ${esc((r.capabilities_required||[]).join(', ')||'keine')}
Context
${r.context_ok?'✓':'✗'} ${num(r.context_requested||0)} / effektiv ${r.context_effective_max?num(r.context_effective_max):'?'} · Modell-Max ${r.context_length?num(r.context_length):'?'}
QoS
${esc(r.service_class||'–')} · weight ${num(r.service_class_config?.weight||0,1)}
${r.alias?`

Alias ${esc(r.alias)}

${alias||'
Keine Kandidaten
'}
`:''}${errors}
${workers||''}
WorkerEligibilityPlacementInventarSlotsOutputVRAMScore / Grund
Keine Worker
`} +$('#sim-tenant')?.addEventListener('input',()=>{S.simulation=null;renderPolicySimulator()}); +$('#policy-simulator-form')?.addEventListener('submit',async e=>{e.preventDefault();const keySel=$('#sim-api-key'),opt=keySel?.selectedOptions?.[0];const body={tenant:$('#sim-tenant').value.trim(),api_key_id:keySel?.value||'',api_key_name:!keySel?.value?(opt?.dataset?.name||''):'',model:$('#sim-model').value.trim(),service_class:$('#sim-service-class').value,input_tokens:Number($('#sim-input-tokens').value||0),output_tokens:Number($('#sim-output-tokens').value||0),required_capabilities:($('#sim-capabilities').value||'').split(',').map(x=>x.trim()).filter(Boolean)};try{S.simulation=await api('/gateway/ui-api/policy-simulator',{method:'POST',body:JSON.stringify(body)});renderPolicySimulator()}catch(err){toast(err.message,true)}}); + +function renderSecurity(){const c=S.config,auth=c.auth||{},oidc=auth.oidc||{},keys=S.apiKeys||[];const keyRows=keys.length?keys.map(k=>`
${esc(k.name)}${esc(k.source||'config')}
${esc(k.tenant)} · ${esc(k.application||k.subject||'–')}
${esc(k.key_hint||'Secret redacted')}${k.service_class?`QoS: ${esc(k.service_class)}`:''}${(k.scopes||[]).length?`${esc(k.scopes.join(', '))}`:'keine Scopes'}${(k.allowed_models||[]).length?`Allow: ${esc(k.allowed_models.join(', '))}`:''}${(k.denied_models||[]).length?`Deny: ${esc(k.denied_models.join(', '))}`:''}${k.created_at?`${esc(when(k.created_at))}`:''}
${k.deletable?``:'aus Konfiguration'}
`).join(''):'
Keine API-Keys
';const cards=[['OIDC',`
Status
${oidc.enabled?'aktiv':'inaktiv'}
Issuer
${esc(oidc.issuer||'–')}
Audience
${esc(oidc.audience||'–')}
Admin Groups
${esc((oidc.admin_groups||[]).join(', ')||'–')}
`],['API Keys',`

UI-Keys werden persistent gespeichert; im State liegt ausschließlich ihr SHA-256-Hash.

${keyRows}
`],['Persistenz',`
Data directory
${esc(c.storage?.data_dir||'–')}
Config
${esc(c.storage?.config_file||'–')}
API keys
${esc(c.storage?.api_keys_file||'–')}
Policies
${esc(c.storage?.policies_file||'–')}
Metrics
${esc(c.storage?.metrics_file||'–')}
Quota
${esc(c.storage?.quota_file||'–')}
Worker performance
${esc(c.storage?.worker_performance_file||'–')}
Model placement
${esc(c.storage?.model_placement_file||'–')}
Worker state
${esc(c.storage?.worker_state_file||'–')}
Usage
${esc(c.usage?.journal_dir||'–')}
`],['IP Bypass',auth.ip_bypass?.length?auth.ip_bypass.map(x=>`
${esc((x.cidrs||[]).join(', '))}${esc(x.tenant)} · ${esc(x.subject||'')}
`).join(''):'
Kein IP-Bypass
'],['Trusted Proxies',`
${(auth.trusted_proxies||[]).map(x=>`${esc(x)}`).join('')||'Keine'}
`]];$('#security-grid').innerHTML=cards.map(([t,b])=>`

${t}

${b}
`).join('');$('#new-api-key')?.addEventListener('click',openAPIKeyDialog)} + +function openAPIKeyDialog(){const f=$('#api-key-form');f.reset();$('#api-key-tenant').value=S.session?.tenant||'';const sel=$('#api-key-service-class'),cls=Object.keys(S.overview?.service_classes?.classes||{}).sort();if(sel)sel.innerHTML=''+cls.map(n=>``).join('');$('#api-key-dialog').showModal()} +$('#api-key-form')?.addEventListener('submit',async e=>{e.preventDefault();const splitRules=v=>v.split(/[\n,]+/).map(x=>x.trim()).filter(Boolean);const scopes=$('#api-key-scopes').value.split(',').map(x=>x.trim()).filter(Boolean);const body={name:$('#api-key-name').value.trim(),tenant:$('#api-key-tenant').value.trim(),subject:$('#api-key-subject').value.trim(),application:$('#api-key-application').value.trim(),scopes,allowed_models:splitRules($('#api-key-allowed-models').value),denied_models:splitRules($('#api-key-denied-models').value),service_class:$('#api-key-service-class')?.value||''};if(!body.name||!body.tenant)return toast('Name und Tenant sind erforderlich.',true);try{const out=await api('/gateway/ui-api/api-keys',{method:'POST',body:JSON.stringify(body)});$('#api-key-dialog').close();$('#created-key-name').textContent=out.key?.name||body.name;$('#created-key-secret').value=out.secret||'';$('#created-key-dialog').showModal();S.apiKeys=(await api('/gateway/ui-api/api-keys')).keys||[];renderSecurity();toast('API-Key erstellt')}catch(err){toast(err.message,true)}}); +$('#copy-created-key')?.addEventListener('click',async()=>{const v=$('#created-key-secret').value;if(!v)return;if(await copyText(v))toast('API-Key kopiert');else toast('Kopieren nicht möglich – bitte den Key manuell markieren.',true)}); +$('#created-key-dialog')?.addEventListener('close',()=>{$('#created-key-secret').value='' }); +$('#security-grid')?.addEventListener('click',async e=>{const b=e.target.closest('[data-api-key-delete]');if(!b)return;const name=b.dataset.apiKeyName||'diesen API-Key';if(!confirm(`API-Key „${name}“ wirklich löschen?\n\nDer Key wird sofort ungültig.`))return;try{await api('/gateway/ui-api/api-keys/'+encodeURIComponent(b.dataset.apiKeyDelete),{method:'DELETE'});S.apiKeys=(await api('/gateway/ui-api/api-keys')).keys||[];renderSecurity();toast('API-Key gelöscht')}catch(err){toast(err.message,true)}}); +$('#copy-config').onclick=async()=>{if(await copyText($('#config-json').value))toast('Konfiguration kopiert');else toast('Kopieren nicht möglich.',true)}; +function setConfigEditing(on){const ed=$('#config-json');if(!ed)return;ed.readOnly=!on;S.configDirty=on;$('#config-actions')?.classList.toggle('hidden',!on);$('#edit-config').textContent=on?'Bearbeitung aktiv':'Bearbeiten';if(on){ed.focus();$('#config-status').textContent='JSON wird vor dem Speichern vollständig validiert. Aktivierung aller Änderungen erfolgt nach einem Gateway-Neustart.'}else{$('#config-status').textContent='Effektive Konfiguration. Änderungen an der Storage-Sektion sind nur in der Bootstrap-Datei möglich.'}} +$('#edit-config')?.addEventListener('click',()=>setConfigEditing($('#config-json').readOnly)); +$('#cancel-config')?.addEventListener('click',()=>{S.configDirty=false;$('#config-json').value=JSON.stringify(S.config,null,2);setConfigEditing(false)}); +$('#save-config')?.addEventListener('click',async()=>{let obj;try{obj=JSON.parse($('#config-json').value)}catch(e){return toast('Ungültiges JSON: '+e.message,true)};if(!confirm('Konfiguration persistent speichern? Die Änderungen werden vollständig nach dem nächsten Gateway-Neustart aktiv.'))return;try{const out=await api('/gateway/ui-api/config',{method:'PUT',body:JSON.stringify(obj)});S.configDirty=false;setConfigEditing(false);$('#config-status').className='callout ok-callout';$('#config-status').textContent='Persistent gespeichert. Gateway neu starten, um alle Änderungen zu aktivieren.';toast(out.message||'Konfiguration gespeichert')}catch(e){toast(e.message,true)}}); +$('#reset-config')?.addEventListener('click',async()=>{if(!confirm('Persistenten Konfigurations-Override löschen? Nach dem nächsten Neustart wird wieder die Bootstrap-config.json verwendet.'))return;try{const out=await api('/gateway/ui-api/config',{method:'DELETE'});toast(out.message||'Persistenter Override gelöscht');$('#config-status').className='callout warn-callout';$('#config-status').textContent='Override gelöscht. Nach Neustart gilt wieder die Bootstrap-Konfiguration.'}catch(e){toast(e.message,true)}}); +function table(headers,rows){if(!rows.length)return '
Keine Daten
';return `${headers.map(x=>``).join('')}${rows.map(r=>`${r.map(c=>``).join('')}`).join('')}
${x}
${c??''}
`} +document.addEventListener('click',e=>{const b=e.target.closest('button');if(!b)return;if(b.dataset.modelAction)modelAction(b.dataset.modelAction,b.dataset.worker,b.dataset.model);else if(b.dataset.policyEdit!==undefined)openPolicy(b.dataset.policyEdit);else if(b.dataset.policyDelete!==undefined)deletePolicy(b.dataset.policyDelete);else if(b.dataset.jobCancel)cancelJob(b.dataset.jobCancel);else if(b.dataset.batchAction)batchAction(b.dataset.batchId,b.dataset.batchAction);else if(b.dataset.batchOutput)downloadBatchOutput(b.dataset.batchOutput);else if(b.dataset.operationCancel)cancelOperation(b.dataset.operationCancel)}); + + +// ---- Local in-memory infrastructure map ------------------------------------- +function stopClusterStream(){if(S.clusterAbort){S.clusterAbort.abort();S.clusterAbort=null}S.cluster.connected=false;updateClusterConnection()} +function startClusterStream(){if(!isAdmin()||S.clusterAbort)return;const ctl=new AbortController();S.clusterAbort=ctl;void clusterStreamLoop(ctl.signal)} +async function clusterStreamLoop(signal){let retry=700;while(!signal.aborted){try{await clusterStreamOnce(signal);retry=700}catch(e){if(signal.aborted)return;S.cluster.connected=false;updateClusterConnection(e.message);if(e.status===401){showLogin('Sitzung abgelaufen oder Credential ungültig.');return}try{handleClusterSnapshot(await api('/gateway/ui-api/infrastructure'))}catch{}await liveSleep(retry,signal);retry=Math.min(5000,retry*1.7)}}} +async function clusterStreamOnce(signal){const headers=authHeaders();headers.Accept='text/event-stream';const r=await fetch('/gateway/ui-api/infrastructure/stream',{headers,credentials:'same-origin',signal});if(!r.ok){const e=new Error(`Infrastructure-Stream HTTP ${r.status}`);e.status=r.status;throw e}if(!r.body)throw new Error('Infrastructure-Stream nicht verfügbar');S.cluster.connected=true;updateClusterConnection();const reader=r.body.getReader(),decoder=new TextDecoder();let buf='';while(!signal.aborted){const {value,done}=await reader.read();if(done)throw new Error('Infrastructure-Stream beendet');buf+=decoder.decode(value,{stream:true}).replace(/\r\n/g,'\n');let i;while((i=buf.indexOf('\n\n'))>=0){const raw=buf.slice(0,i);buf=buf.slice(i+2);let event='message';const data=[];for(const line of raw.split('\n')){if(line.startsWith('event:'))event=line.slice(6).trim();else if(line.startsWith('data:'))data.push(line.slice(5).trimStart())}if(event==='snapshot'&&data.length){try{handleClusterSnapshot(JSON.parse(data.join('\n')))}catch{}}}}} +function handleClusterSnapshot(x){S.cluster={...S.cluster,...x,gateways:Array.isArray(x?.gateways)?x.gateways:[],workers:Array.isArray(x?.workers)?x.workers:[],requests:Array.isArray(x?.requests)?x.requests:[],counts:x?.counts||{},connected:true};updateClusterConnection();renderClusterPanel()} +function updateClusterConnection(msg=''){const el=$('#cluster-stream-state');if(!el)return;el.className='status-chip '+(S.cluster.connected?'ok':'warn');el.textContent=S.cluster.connected?'In-Memory live':(msg?'Reconnect':'verbinde …')} +function clusterFilteredRequests(includeTerminal=true){const q=($('#cluster-search')?.value||'').trim().toLowerCase();return (S.cluster.requests||[]).filter(r=>(includeTerminal||liveActive(r))&&(!q||`${r.gateway_name||''} ${r.tenant||''} ${r.actor||''} ${r.worker||''} ${r.model||''}`.toLowerCase().includes(q)))} +function pct(v,total){return total>0?Math.max(0,Math.min(100,100*Number(v||0)/Number(total))):0} +function renderClusterPanel(){if(!isAdmin())return;const c=S.cluster.counts||{};$('#cluster-gateways')&&($('#cluster-gateways').textContent=num(c.gateways??S.cluster.gateways.length));$('#cluster-workers')&&($('#cluster-workers').textContent=num(c.workers??S.cluster.workers.length));$('#cluster-active')&&($('#cluster-active').textContent=num(c.active??clusterFilteredRequests(false).length));$('#cluster-queue')&&($('#cluster-queue').textContent=num(S.cluster.queue||0));$('#cluster-models')&&($('#cluster-models').textContent=num(c.models||0));$('#cluster-map-empty')?.classList.toggle('hidden',(S.cluster.gateways?.length||0)>0);renderClusterGateways();renderClusterWorkers();updateClusterConnection()} +function renderClusterGateways(){const el=$('#cluster-gateway-list');if(!el)return;const gs=S.cluster.gateways||[];el.innerHTML=gs.length?gs.map(g=>{const age=Math.max(0,(Date.now()-new Date(g.last_seen).getTime())/1000);return `
${esc(g.node_name)}${esc(g.hostname||g.node_id)}
${age<3?'live':age.toFixed(0)+'s'}
Queue ${num(g.queued)}Running ${num(g.running)}Requests ${num(g.live_active)}Worker ${num(g.workers)}
`}).join(''):'
Keine Gateway-Nodes
'} +function renderClusterWorkers(){const el=$('#cluster-worker-list');if(!el)return;const ws=S.cluster.workers||[];el.innerHTML=ws.length?ws.map(w=>{const memUsed=Number(w.memory_used_bytes||w.resident_bytes||0),memTotal=Number(w.memory_total_bytes||w.memory_capacity_bytes||0),vrUsed=Number(w.vram_used_bytes||w.vram_bytes||0),vrTotal=Number(w.vram_total_bytes||w.vram_capacity_bytes||0),vp=pct(vrUsed,vrTotal),mp=pct(memUsed,memTotal),models=(w.loaded_models||[]).slice(0,5),memLabel=w.memory_used_bytes?'System RAM':'Model resident',vrLabel=w.vram_used_bytes?'VRAM used':'VRAM footprint',gpu=Number(w.gpu_utilization_percent||0);return `
${esc(w.name)}${esc((w.gateways||[]).join(', ')||w.url)}${w.telemetry_source?` · ${esc(w.telemetry_source)}`:''}
${w.active}/${w.max_concurrent}
${memLabel}${bytes(memUsed)}${memTotal?' / '+bytes(memTotal):''}
${vrLabel}${bytes(vrUsed)}${vrTotal?' / '+bytes(vrTotal):''}
${gpu?`
GPU utilization${num(gpu,1)}%
`:''}
${models.map(m=>`${esc(m.name||m.model)} · ${bytes(m.size_vram)}`).join('')||'idle'}
${w.telemetry_error?`Telemetry: ${esc(w.telemetry_error)}`:''}
`}).join(''):'
Keine Worker
'} +function clusterLayout(reqs,W,H){const gs=(S.cluster.gateways||[]).slice(0,8),ws=(S.cluster.workers||[]).slice(0,12),margin=65,gatewayPos=new Map(),queuePos=new Map(),workerPos=new Map(),modelPos=new Map(),tenantPos=new Map();const tenants=[...new Set(reqs.map(r=>r.tenant).filter(Boolean))].slice(0,9);tenants.forEach((n,i)=>tenantPos.set(n,{x:Math.max(55,W*.055),y:margin+(H-2*margin)*(tenants.length===1?.5:i/Math.max(1,tenants.length-1))}));gs.forEach((g,i)=>{const y=margin+(H-2*margin)*(gs.length===1?.5:i/Math.max(1,gs.length-1));gatewayPos.set(g.node_name,{x:W*.27,y});queuePos.set(g.node_name,{x:W*.44,y})});ws.forEach((w,i)=>{const y=margin+(H-2*margin)*(ws.length===1?.5:i/Math.max(1,ws.length-1));const key=w.name+'\0'+w.url,lm=w.loaded_models||[];workerPos.set(key,{x:W*.72,y,w});for(const [j,m] of lm.slice(0,4).entries()){const off=(j-(Math.min(4,lm.length)-1)/2)*24;modelPos.set(key+'\0'+(m.name||m.model),{x:W*.94,y:Math.max(36,Math.min(H-36,y+off)),m})}});return{tenantPos,gatewayPos,queuePos,workerPos,modelPos,gateways:gs,workers:ws}} +function clusterWorkerPoint(l,r){for(const [k,p] of l.workerPos){if(p.w.name===r.worker)return {key:k,p}}return null} +function clusterRoute(l,r,H){const tp=l.tenantPos.get(r.tenant)||{x:55,y:40+(liveHash(r.tenant)%(Math.max(80,H-80)))},gp=l.gatewayPos.get(r.gateway_name)||{x:220,y:H*.5},qp=l.queuePos.get(r.gateway_name)||{x:360,y:gp.y},wp=clusterWorkerPoint(l,r),w=wp?.p||{x:qp.x+220,y:H*.5},m=l.modelPos.get((wp?.key||'')+'\0'+r.model)||w,j=((liveHash(r.actor)%17)-8);return[Object.assign(curve({x:tp.x+6,y:tp.y+j},gp,.45,j*.25),{w:1}),Object.assign(curve(gp,qp,.5,j*.1),{w:.55}),Object.assign(curve(qp,w,.5,(w.y-qp.y)*-.08+j),{w:1.35}),Object.assign(curve(w,m,.5,j*.1),{w:.55})]} +function drawClusterMap(tms){const canvas=$('#cluster-map-canvas'),stage=$('#cluster-map-stage');if(!canvas||!stage)return;const rect=stage.getBoundingClientRect();if(rect.width<20||rect.height<20)return;const dpr=Math.min(2,window.devicePixelRatio||1),cw=Math.round(rect.width*dpr),ch=Math.round(rect.height*dpr);if(canvas.width!==cw||canvas.height!==ch){canvas.width=cw;canvas.height=ch}const ctx=canvas.getContext('2d');ctx.setTransform(dpr,0,0,dpr,0,0);const W=rect.width,H=rect.height;ctx.clearRect(0,0,W,H);const all=clusterFilteredRequests(true).slice(0,320),active=all.filter(liveActive),l=clusterLayout(active,W,H),time=tms/1000;ctx.lineCap='round';ctx.lineJoin='round';for(const g of l.gateways){const gp=l.gatewayPos.get(g.node_name),qp=l.queuePos.get(g.node_name);ctx.strokeStyle='rgba(26,128,95,.22)';ctx.lineWidth=1.3;drawCurve(ctx,curve(gp,qp,.5,0));for(const [k,p] of l.workerPos){if((p.w.gateways||[]).includes(g.node_name)){ctx.strokeStyle='rgba(20,127,165,.18)';ctx.lineWidth=1;drawCurve(ctx,curve(qp,p,.5,(p.y-qp.y)*-.08))}}}for(const [key,p] of l.modelPos){const parts=key.split('\0'),wp=l.workerPos.get(parts[0]+'\0'+parts[1]);if(wp){ctx.strokeStyle='rgba(20,127,165,.12)';ctx.lineWidth=.8;drawCurve(ctx,curve(wp,p,.5,0))}}for(const r of active){const tp=l.tenantPos.get(r.tenant),gp=l.gatewayPos.get(r.gateway_name);if(tp&&gp){ctx.strokeStyle=liveColor(r.tenant,.07);ctx.lineWidth=.7;drawCurve(ctx,curve(tp,gp,.45,0))}}drawClusterNodes(ctx,l,active,W,H);S.clusterHit=[];for(const r of all)drawClusterRequest(ctx,l,r,H,time)} +function drawClusterNodes(ctx,l,active,W,H){ctx.save();ctx.textBaseline='middle';for(const [n,p] of l.tenantPos){ctx.fillStyle=liveColor(n,.8);ctx.beginPath();ctx.arc(p.x,p.y,3.8,0,Math.PI*2);ctx.fill();ctx.fillStyle='rgba(55,74,92,.82)';ctx.textAlign='left';ctx.font='9px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(n.slice(0,15),p.x+8,p.y)}for(const g of l.gateways){const p=l.gatewayPos.get(g.node_name),q=l.queuePos.get(g.node_name),live=active.filter(r=>r.gateway_name===g.node_name).length;ctx.fillStyle='rgba(255,255,255,.97)';ctx.strokeStyle='rgba(26,128,95,.58)';ctx.lineWidth=1.4;ctx.beginPath();ctx.arc(p.x,p.y,14+Math.min(6,Math.sqrt(live)*2),0,Math.PI*2);ctx.fill();ctx.stroke();ctx.fillStyle='rgba(37,51,69,.94)';ctx.textAlign='center';ctx.font='700 9px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(g.node_name.slice(0,16),p.x,p.y-2);ctx.fillStyle='rgba(102,119,138,.90)';ctx.font='8px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(`${live} live`,p.x,p.y+10);const qr=8+Math.min(10,Math.sqrt(Number(g.queued||0))*2.5);ctx.fillStyle='rgba(247,249,251,.98)';ctx.strokeStyle=g.queued?'rgba(165,102,0,.65)':'rgba(102,119,138,.30)';ctx.beginPath();ctx.arc(q.x,q.y,qr,0,Math.PI*2);ctx.fill();ctx.stroke();ctx.fillStyle='rgba(45,63,81,.88)';ctx.font='700 8px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(String(g.queued||0),q.x,q.y)}for(const [k,p] of l.workerPos){const w=p.w,util=Math.min(1,Number(w.active||0)/Math.max(1,Number(w.max_concurrent||1))),vr=pct(w.vram_used_bytes||w.vram_bytes,w.vram_total_bytes||w.vram_capacity_bytes),gpu=Math.min(100,Number(w.gpu_utilization_percent||0));ctx.fillStyle='rgba(255,255,255,.97)';ctx.strokeStyle=w.healthy?'rgba(20,127,165,.58)':'rgba(184,61,75,.66)';ctx.lineWidth=1.3;ctx.beginPath();ctx.arc(p.x,p.y,13,0,Math.PI*2);ctx.fill();ctx.stroke();ctx.strokeStyle='rgba(26,128,95,.76)';ctx.lineWidth=2;ctx.beginPath();ctx.arc(p.x,p.y,17,-Math.PI/2,-Math.PI/2+Math.PI*2*util);ctx.stroke();if(w.vram_total_bytes||w.vram_capacity_bytes){ctx.strokeStyle='rgba(107,91,181,.76)';ctx.beginPath();ctx.arc(p.x,p.y,20,-Math.PI/2,-Math.PI/2+Math.PI*2*(vr/100));ctx.stroke()}if(gpu){ctx.strokeStyle='rgba(165,102,0,.78)';ctx.beginPath();ctx.arc(p.x,p.y,23,-Math.PI/2,-Math.PI/2+Math.PI*2*(gpu/100));ctx.stroke()}ctx.fillStyle='rgba(37,51,69,.92)';ctx.textAlign='center';ctx.font='600 9px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(w.name.slice(0,16),p.x,p.y+29);ctx.fillStyle='rgba(102,119,138,.88)';ctx.font='8px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(`${w.active}/${w.max_concurrent} slots`,p.x,p.y+40)}for(const [k,p] of l.modelPos){ctx.fillStyle='rgba(107,91,181,.62)';ctx.beginPath();ctx.arc(p.x,p.y,3,0,Math.PI*2);ctx.fill();ctx.fillStyle='rgba(92,108,126,.86)';ctx.textAlign='right';ctx.font='8px ui-monospace,SFMono-Regular,monospace';ctx.fillText((p.m.name||p.m.model||'').slice(0,18),p.x-7,p.y-7)}ctx.restore()} +function drawClusterRequest(ctx,l,r,H,time){const route=clusterRoute(l,r,H),seed=(liveHash(r.id)%1000)/1000;ctx.save();ctx.strokeStyle=liveColor(r.tenant,.15);ctx.lineWidth=.6+Math.min(2,Math.sqrt(Math.max(.05,Number(r.estimated_credits||.1)))*.35);for(const x of route)drawCurve(ctx,x);let pts=[];if(r.state==='queued'){const age=liveAge(r),p=Math.min(.46,.03+age*.22);pts=[{p:p>=.45?.43+.015*Math.sin(time*2.2+seed*6.28):p,a:1}]}else if(r.state==='routing'){pts=[{p:.46+Math.min(.4,liveAge(r,'updated_at')*.7),a:1}]}else if(r.state==='running'){for(let k=0;k<2;k++)pts.push({p:.48+.5*((time*.2+seed+k*.47)%1),a:.9-k*.2})}else if(r.state==='streaming'){for(let k=0;k<3;k++)pts.push({p:1-((time*.28+seed+k*.3)%1),a:1-k*.2})}else{const age=liveAge(r,'finished_at'),fade=Math.max(0,1-age/10);pts=[{p:r.state==='failed'?.7:Math.max(.02,1-age*.15),a:fade}]}for(const x of pts){const p=routePoint(route,x.p),col=r.state==='failed'?`rgba(184,61,75,${x.a})`:r.state==='queued'?`rgba(165,102,0,${x.a})`:r.state==='streaming'?`rgba(26,128,95,${x.a})`:liveColor(r.tenant,x.a,44),rad=2.2+Math.min(3.5,Math.sqrt(Math.max(.1,Number(r.estimated_credits||.1))));ctx.shadowColor=col;ctx.shadowBlur=14;ctx.fillStyle=col;ctx.beginPath();ctx.arc(p.x,p.y,rad,0,Math.PI*2);ctx.fill();ctx.shadowBlur=0;S.clusterHit.push({x:p.x,y:p.y,r,rad:rad+6})}ctx.restore()} +function clusterPointerMove(e){const canvas=$('#cluster-map-canvas'),tt=$('#cluster-map-tooltip');if(!canvas||!tt)return;const b=canvas.getBoundingClientRect(),x=e.clientX-b.left,y=e.clientY-b.top;let best=null,d=Infinity;for(const h of S.clusterHit){const x2=Math.hypot(h.x-x,h.y-y);if(x2${esc(r.model||r.path||'Request')}
${esc(r.tenant)} · ${esc(r.actor||'–')}
${esc(r.gateway_name||'Gateway')} → ${esc(r.worker||'–')}
${esc(r.state)} · ${ms(r.queue_ms||0)} Queue
${esc(r.id)}
`;tt.classList.remove('hidden');tt.style.left=Math.min(sw-310,Math.max(8,x+14))+'px';tt.style.top=Math.min(sh-120,Math.max(8,y+14))+'px'} + +// ---- Live request pulse map ------------------------------------------------- +const LIVE_TERMINAL=new Set(['completed','failed','cancelled']); +const liveActive=r=>!LIVE_TERMINAL.has(r.state); +const liveAge=(r,field='queued_at')=>Math.max(0,(Date.now()-new Date(r[field]||r.updated_at||Date.now()).getTime())/1000); +const liveHash=v=>{let h=2166136261;for(const c of String(v||'')){h^=c.charCodeAt(0);h=Math.imul(h,16777619)}return h>>>0}; +const liveColor=(v,a=1,l=44)=>`hsla(${(liveHash(v)%290)+30},68%,${Math.min(l,52)}%,${a})`; + +function stopLiveStream(){if(S.liveAbort){S.liveAbort.abort();S.liveAbort=null}S.live.connected=false;updateLiveConnection()} +function startLiveStream(){if(!isAdmin()||S.liveAbort)return;const ctl=new AbortController();S.liveAbort=ctl;void liveStreamLoop(ctl.signal)} +async function liveStreamLoop(signal){let retry=650;while(!signal.aborted){try{await liveStreamOnce(signal);retry=650}catch(e){if(signal.aborted)return;S.live.connected=false;updateLiveConnection(e.message);if(e.status===401){showLogin('Sitzung abgelaufen oder Credential ungültig.');return}try{const snap=await api('/gateway/ui-api/live');handleLiveSnapshot(snap)}catch{}await liveSleep(retry,signal);retry=Math.min(5000,retry*1.7)}}} +async function liveStreamOnce(signal){const headers=authHeaders();headers.Accept='text/event-stream';const r=await fetch('/gateway/ui-api/live/stream',{headers,credentials:'same-origin',signal});if(!r.ok){const e=new Error(`Live-Stream HTTP ${r.status}`);e.status=r.status;throw e}if(!r.body)throw new Error('Live-Stream nicht verfügbar');S.live.connected=true;updateLiveConnection();const reader=r.body.getReader(),decoder=new TextDecoder();let buf='';while(!signal.aborted){const {value,done}=await reader.read();if(done)throw new Error('Live-Stream beendet');buf+=decoder.decode(value,{stream:true}).replace(/\r\n/g,'\n');let i;while((i=buf.indexOf('\n\n'))>=0){const raw=buf.slice(0,i);buf=buf.slice(i+2);let event='message';const data=[];for(const line of raw.split('\n')){if(line.startsWith('event:'))event=line.slice(6).trim();else if(line.startsWith('data:'))data.push(line.slice(5).trimStart())}if(event==='snapshot'&&data.length){try{handleLiveSnapshot(JSON.parse(data.join('\n')))}catch{}}}}} +function liveSleep(ms,signal){return new Promise(resolve=>{const t=setTimeout(resolve,ms);signal?.addEventListener('abort',()=>{clearTimeout(t);resolve()},{once:true})})} +function handleLiveSnapshot(snap){S.live.requests=Array.isArray(snap?.requests)?snap.requests:[];S.live.version=Number(snap?.version||0);S.live.generatedAt=snap?.generated_at||null;S.live.counts=snap?.counts||null;S.live.truncated=!!snap?.truncated;S.live.connected=true;updateLiveConnection();renderLivePanel()} +function updateLiveConnection(msg=''){const el=$('#live-stream-state');if(!el)return;el.className='status-chip '+(S.live.connected?'ok':'warn');el.textContent=S.live.connected?'SSE live':(msg?'Reconnect':'verbinde …')} +function liveFiltered(includeTerminal=true){const state=$('#live-state-filter')?.value||'',q=($('#live-search')?.value||'').trim().toLowerCase();return S.live.requests.filter(r=>(includeTerminal||liveActive(r))&&(!state||r.state===state)&&(!q||`${r.tenant} ${r.actor} ${r.application||''} ${r.model||''} ${r.worker||''} ${r.path||''}`.toLowerCase().includes(q)))} +function renderLivePanel(){if(!isAdmin())return;const all=S.live.requests,active=all.filter(liveActive),queued=active.filter(r=>r.state==='queued'),streaming=active.filter(r=>r.state==='streaming'),c=S.live.counts||{};const tokens=active.reduce((n,r)=>n+Number(r.prompt_tokens||r.estimated_prompt_tokens||0)+Number(r.completion_tokens||0),0);if($('#live-active')){$('#live-active').textContent=num(c.active??active.length);$('#live-queued').textContent=num(c.queued??queued.length);$('#live-streaming').textContent=num(c.streaming??streaming.length);$('#live-tokens').textContent=num(tokens);$('#live-flow-empty').classList.toggle('hidden',liveFiltered(false).length>0)}renderLiveRequestList();renderLiveWorkerList();updateLiveConnection()} +function renderLiveRequestList(){const el=$('#live-request-list');if(!el)return;const a=liveFiltered(true).sort((x,y)=>Number(liveActive(y))-Number(liveActive(x))||new Date(y.updated_at)-new Date(x.updated_at)).slice(0,40);el.innerHTML=a.length?a.map(r=>{const dur=r.started_at?liveAge(r,'started_at'):liveAge(r,'queued_at'),tok=Number(r.prompt_tokens||r.estimated_prompt_tokens||0)+Number(r.completion_tokens||0);return `
${esc(r.model||r.path||'request')}${esc(r.tenant)} · ${esc(r.actor||'–')} ${r.worker?`→ ${esc(r.worker)}`:''}
${esc(r.state)}${dur.toFixed(dur<10?1:0)} s · ${num(tok)} tok${liveActive(r)?``:''}
`}).join(''):'
Keine Requests für diesen Filter
'} +function renderLiveWorkerList(){const el=$('#live-worker-list');if(!el)return;const active=liveFiltered(false),configured=S.overview?.workers||[],names=[...new Set([...configured.map(w=>w.name),...active.map(r=>r.worker).filter(Boolean)])];const max=Math.max(1,...names.map(n=>active.filter(r=>r.worker===n).length));el.innerHTML=names.length?names.map(n=>{const rs=active.filter(r=>r.worker===n),w=configured.find(x=>x.name===n),models=[...new Set(rs.map(r=>r.model).filter(Boolean))].slice(0,6);return `
${esc(n)}${w?`${w.active}/${w.max_concurrent} Slots`:''}
${rs.length} live
${models.map(m=>`${esc(m)}`).join('')||'idle'}
`}).join(''):'
Keine Worker
'} + +function startLiveAnimation(){if(S.liveRAF)return;const loop=t=>{S.liveRAF=requestAnimationFrame(loop);if(S.page==='live')drawLiveFlow(S.livePaused?(S.livePauseTime||t):t);else if(S.page==='infrastructure')drawClusterMap(S.livePaused?(S.livePauseTime||t):t)};S.liveRAF=requestAnimationFrame(loop);for(const b of [$('#live-motion'),$('#cluster-motion')].filter(Boolean))b.textContent=S.livePaused?'Animation fortsetzen':'Animation pausieren'} +function toggleLiveMotion(){S.livePaused=!S.livePaused;if(S.livePaused)S.livePauseTime=performance.now();else S.livePauseTime=0;for(const b of [$('#live-motion'),$('#cluster-motion')].filter(Boolean))b.textContent=S.livePaused?'Animation fortsetzen':'Animation pausieren'} +function cubic(a,b,c,d,t){const u=1-t;return{x:u*u*u*a.x+3*u*u*t*b.x+3*u*t*t*c.x+t*t*t*d.x,y:u*u*u*a.y+3*u*u*t*b.y+3*u*t*t*c.y+t*t*t*d.y}} +function curve(a,d,bias=.5,bend=0){const dx=d.x-a.x;return{a,b:{x:a.x+dx*bias,y:a.y+bend},c:{x:d.x-dx*bias,y:d.y-bend},d}} +function routePoint(route,p){p=Math.max(0,Math.min(.999999,p));const weights=route.map(s=>s.w||1),sum=weights.reduce((a,b)=>a+b,0);let x=p*sum;for(let i=0;ib[1]-a[1]).map(x=>x[0]);const visibleTenants=tenants.slice(0,8),tenantPos=new Map();visibleTenants.forEach((n,i)=>tenantPos.set(n,{x:Math.max(70,W*.075),y:marginY+(H-2*marginY)*(visibleTenants.length===1?.5:i/Math.max(1,visibleTenants.length-1))}));const admission={x:W*.31,y:H*.50},queue={x:W*.50,y:H*.50};const configured=S.overview?.workers||[],workerNames=[...new Set([...configured.map(w=>w.name),...reqs.map(r=>r.worker).filter(Boolean)])].slice(0,8),workerPos=new Map();workerNames.forEach((n,i)=>workerPos.set(n,{x:W*.78,y:marginY+(H-2*marginY)*(workerNames.length===1?.5:i/Math.max(1,workerNames.length-1))}));const modelPos=new Map();for(const w of workerNames){const models=[...new Set(reqs.filter(r=>r.worker===w).map(r=>r.model).filter(Boolean))].slice(0,4),wp=workerPos.get(w);models.forEach((m,i)=>{const off=(i-(models.length-1)/2)*26;modelPos.set(`${w}\0${m}`,{x:W*.94,y:Math.max(42,Math.min(H-42,wp.y+off))})})}return{tenantPos,visibleTenants,admission,queue,workerPos,workerNames,modelPos}} +function tenantLane(layout,r,H){const p=layout.tenantPos.get(r.tenant);if(p)return p;const lanes=[...layout.tenantPos.values()];if(lanes.length)return lanes[liveHash(r.tenant)%lanes.length];return{x:70,y:H*.5}} +function requestRoute(layout,r,H){const tp=tenantLane(layout,r,H),a=layout.admission,q=layout.queue,w=layout.workerPos.get(r.worker)||{x:q.x+140,y:q.y+(liveHash(r.model)%160)-80},m=layout.modelPos.get(`${r.worker}\0${r.model}`)||w;const jitter=((liveHash(r.actor)%21)-10)*1.1;return[Object.assign(curve({x:tp.x+8,y:tp.y+jitter},a,.45,jitter*.35),{w:1.25}),Object.assign(curve(a,q,.5,jitter*.15),{w:.75}),Object.assign(curve(q,w,.52,(w.y-q.y)*-.08+jitter),{w:1.45}),Object.assign(curve(w,m,.5,jitter*.1),{w:.55})]} +function drawLiveFlow(tms){const canvas=$('#live-flow-canvas'),stage=$('#live-flow-stage');if(!canvas||!stage)return;const rect=stage.getBoundingClientRect();if(rect.width<20||rect.height<20)return;const dpr=Math.min(2,window.devicePixelRatio||1),cw=Math.round(rect.width*dpr),ch=Math.round(rect.height*dpr);if(canvas.width!==cw||canvas.height!==ch){canvas.width=cw;canvas.height=ch}const ctx=canvas.getContext('2d');ctx.setTransform(dpr,0,0,dpr,0,0);const W=rect.width,H=rect.height;ctx.clearRect(0,0,W,H);const all=liveFiltered(true).slice(0,220),layout=liveLayout(all,W,H),active=all.filter(liveActive),time=tms/1000; + // topology trunks + ctx.lineCap='round';ctx.lineJoin='round';ctx.shadowBlur=0;ctx.strokeStyle='rgba(92,117,140,.22)';ctx.lineWidth=1; + for(const [name,p] of layout.tenantPos){drawCurve(ctx,curve({x:p.x+7,y:p.y},layout.admission,.46,0))} + ctx.strokeStyle='rgba(26,128,95,.22)';ctx.lineWidth=1.3;drawCurve(ctx,curve(layout.admission,layout.queue,.5,0)); + for(const [name,p] of layout.workerPos){ctx.strokeStyle='rgba(20,127,165,.20)';ctx.lineWidth=1;drawCurve(ctx,curve(layout.queue,p,.52,(p.y-layout.queue.y)*-.08))} + for(const [key,p] of layout.modelPos){const worker=key.split('\0')[0],wp=layout.workerPos.get(worker);if(wp){ctx.strokeStyle='rgba(20,127,165,.14)';ctx.lineWidth=.8;drawCurve(ctx,curve(wp,p,.5,0))}} + drawLiveNodes(ctx,layout,W,H,active); + S.liveHit=[]; + for(const r of all){drawLiveRequest(ctx,layout,r,W,H,time)} +} +function drawLiveNodes(ctx,l,W,H,active){ctx.save();ctx.textBaseline='middle';ctx.font='600 10px ui-monospace,SFMono-Regular,Consolas,monospace';for(const [n,p] of l.tenantPos){const c=active.filter(r=>r.tenant===n).length;ctx.fillStyle=liveColor(n,.85);ctx.beginPath();ctx.arc(p.x,p.y,4.2,0,Math.PI*2);ctx.fill();ctx.shadowBlur=0;ctx.fillStyle='rgba(48,67,85,.90)';ctx.textAlign='left';ctx.fillText(n.slice(0,18),p.x+10,p.y-5);ctx.fillStyle='rgba(102,119,138,.88)';ctx.font='9px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(`${c} live`,p.x+10,p.y+8);ctx.font='600 10px ui-monospace,SFMono-Regular,Consolas,monospace'}drawHub(ctx,l.admission,'ADMISSION',active.length);drawHub(ctx,l.queue,'FAIR QUEUE',active.filter(r=>r.state==='queued').length);for(const [n,p] of l.workerPos){const c=active.filter(r=>r.worker===n).length;ctx.strokeStyle=c?'rgba(26,128,95,.58)':'rgba(20,127,165,.30)';ctx.fillStyle='rgba(255,255,255,.96)';ctx.lineWidth=1.2;ctx.beginPath();ctx.arc(p.x,p.y,10,0,Math.PI*2);ctx.fill();ctx.stroke();ctx.fillStyle='rgba(42,59,77,.92)';ctx.textAlign='center';ctx.font='600 9px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(n.slice(0,16),p.x,p.y+19);ctx.fillStyle='rgba(102,119,138,.90)';ctx.fillText(`${c} live`,p.x,p.y+31)}for(const [key,p] of l.modelPos){const m=key.split('\0')[1];ctx.fillStyle='rgba(20,127,165,.55)';ctx.beginPath();ctx.arc(p.x,p.y,2.5,0,Math.PI*2);ctx.fill();ctx.fillStyle='rgba(88,105,123,.82)';ctx.textAlign='right';ctx.font='8px ui-monospace,SFMono-Regular,monospace';ctx.fillText((m||'').slice(0,18),p.x-7,p.y-7)}ctx.restore()} +function drawHub(ctx,p,label,count){ctx.save();const radius=13+Math.min(7,Math.sqrt(count||0)*2);ctx.strokeStyle='rgba(26,128,95,.42)';ctx.fillStyle='rgba(255,255,255,.97)';ctx.lineWidth=1.2;ctx.beginPath();ctx.arc(p.x,p.y,radius,0,Math.PI*2);ctx.fill();ctx.stroke();ctx.fillStyle='rgba(37,51,69,.94)';ctx.textAlign='center';ctx.font='700 9px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(label,p.x,p.y-2);ctx.fillStyle='rgba(102,119,138,.90)';ctx.font='9px ui-monospace,SFMono-Regular,Consolas,monospace';ctx.fillText(`${count||0}`,p.x,p.y+10);ctx.restore()} +function drawLiveRequest(ctx,layout,r,W,H,time){const route=requestRoute(layout,r,H),base=liveColor(r.tenant,.20),width=.55+Math.min(2.6,Math.sqrt(Math.max(.05,Number(r.estimated_credits||.1)))*.4);ctx.save();ctx.strokeStyle=base;ctx.lineWidth=width;for(const s of route)drawCurve(ctx,s);const seed=(liveHash(r.id)%1000)/1000;let pts=[];if(r.state==='queued'){const age=liveAge(r),p=Math.min(.48,.05+age*.22);pts=[{p:p>=.47?.455+.018*Math.sin(time*2.2+seed*6.28):p,a:.95}]}else if(r.state==='routing'){const age=liveAge(r,'updated_at');pts=[{p:.48+Math.min(.43,age*.75),a:1}]}else if(r.state==='running'){for(let k=0;k<2;k++)pts.push({p:.48+(.50*((time*.22+seed+k*.48)%1)),a:.85-k*.18})}else if(r.state==='streaming'){for(let k=0;k<3;k++)pts.push({p:1-((time*.30+seed+k*.29)%1),a:1-k*.20});pts.push({p:.98,a:.75})}else{const age=liveAge(r,'finished_at'),fade=Math.max(0,1-age/10),p=r.state==='failed'?.72:Math.max(.02,1-age*.16);pts=[{p,a:fade}]} + for(const x of pts){const p=routePoint(route,x.p),terminal=LIVE_TERMINAL.has(r.state),col=r.state==='failed'?`rgba(184,61,75,${x.a})`:r.state==='queued'?`rgba(165,102,0,${x.a})`:r.state==='streaming'?`rgba(26,128,95,${x.a})`:liveColor(r.tenant,x.a,44);const rad=2.2+Math.min(4,Math.sqrt(Math.max(.1,Number(r.estimated_credits||.1))));ctx.shadowColor=col;ctx.shadowBlur=terminal?8:16;ctx.fillStyle=col;ctx.beginPath();ctx.arc(p.x,p.y,rad,0,Math.PI*2);ctx.fill();ctx.shadowBlur=0;S.liveHit.push({x:p.x,y:p.y,r,rad:rad+6})} + ctx.restore()} +function showLiveTooltip(r,x,y){const tt=$('#live-flow-tooltip'),stage=$('#live-flow-stage');if(!tt||!stage)return;const tok=Number(r.prompt_tokens||r.estimated_prompt_tokens||0)+Number(r.completion_tokens||0);tt.innerHTML=`${esc(r.model||r.path||'Request')}
${esc(r.tenant)} · ${esc(r.actor||'–')}
${esc(r.id)}
${esc(r.state)} ${r.worker?`→ ${esc(r.worker)}`:''}
${num(tok)} Tokens · ${credits(r.estimated_credits)} cr est.
`;tt.classList.remove('hidden');const sw=stage.clientWidth,sh=stage.clientHeight;tt.style.left=Math.min(sw-310,Math.max(8,x+14))+'px';tt.style.top=Math.min(sh-120,Math.max(8,y+14))+'px'} +function livePointerMove(e){const canvas=$('#live-flow-canvas'),tt=$('#live-flow-tooltip');if(!canvas||!tt)return;const b=canvas.getBoundingClientRect(),x=e.clientX-b.left,y=e.clientY-b.top;let best=null,dist=Infinity;for(const h of S.liveHit){const d=Math.hypot(h.x-x,h.y-y);if(d$('#cluster-map-tooltip')?.classList.add('hidden')); + +$('#live-state-filter')?.addEventListener('change',renderLivePanel); +$('#live-search')?.addEventListener('input',renderLivePanel); +$('#live-motion')?.addEventListener('click',toggleLiveMotion); +$('#live-flow-canvas')?.addEventListener('pointermove',livePointerMove); +$('#live-flow-canvas')?.addEventListener('pointerleave',()=>$('#live-flow-tooltip')?.classList.add('hidden')); +document.addEventListener('click',e=>{const b=e.target.closest('[data-close-dialog]');if(!b)return;document.getElementById(b.dataset.closeDialog)?.close()}); + +$('#worker-cards')?.addEventListener('click',async e=>{const b=e.target.closest('[data-worker-mode]');if(b){try{await api('/gateway/ui-api/workers/'+encodeURIComponent(b.dataset.worker)+'/maintenance',{method:'POST',body:JSON.stringify({mode:b.dataset.workerMode})});toast(`Worker ${b.dataset.worker}: ${b.dataset.workerMode}`);await refreshAll(false)}catch(err){toast(err.message,true)}return}const c=e.target.closest('[data-circuit-reset]');if(c){try{await api('/gateway/ui-api/workers/'+encodeURIComponent(c.dataset.circuitReset)+'/circuit/reset',{method:'POST'});toast('Circuit Breaker zurückgesetzt');await refreshAll(false)}catch(err){toast(err.message,true)}}}); + +init(); diff --git a/internal/webui/assets/index.html b/internal/webui/assets/index.html new file mode 100644 index 0000000..b44702d --- /dev/null +++ b/internal/webui/assets/index.html @@ -0,0 +1,422 @@ + + + + + + + + Ollama Fair Gateway + + + +
+ + + + + +
+

Model Alias

Fallbacks werden in Reihenfolge geprüft; Capability-Anforderungen gelten zusätzlich vor dem Routing.

+ + + + +
Speichern wirkt ohne Gateway-Neustart. Der vollständige Alias-Snapshot wird atomar in der persistenten Gateway-Konfiguration aktualisiert.
+
+
+ + + +
+

Modell pullen

Der Download läuft als Gateway-Operation weiter.

+ + +
+
+ + + +
+

Worker Placement bearbeiten

–

+ +
Patterns: exakter Name oder ein abschließendes *, z. B. gemma4:*. Exakte Regeln können breitere Prefix-Regeln übersteuern.
+ + +
Preset:
+
+
+ + + +
+

API-Key erstellen

Der Schlüssel wird nur einmal angezeigt; anschließend wird ausschließlich sein SHA-256-Hash persistent gespeichert.

+
+ + + + + + + + +
+
UI-Keys bleiben über Neustarts erhalten. Das Secret selbst wird nie persistent gespeichert.
+
+
+ + + +
+

API-Key erstellt

– dieses Secret wird nicht erneut angezeigt.

+ +
Kopiere den Schlüssel jetzt an einen sicheren Ort. Im Gateway bleibt nur sein Hash gespeichert.
+
+
+ + + +
+

Tenant Model Access

Exakte Modellnamen und ein abschließendes * werden unterstützt; bei gleicher Spezifität gewinnt deny.

+ + + + +
Die Tenant-Regel ist die äußere Sicherheitsgrenze. Eine API-Key-ACL kann sie weiter einschränken, aber niemals erweitern.
+
+
+ + + +
+

Policy Override

Gewichte müssen größer 0 sein. Credit-Werte 0 bedeuten unbegrenzt.

+
+ + + + + + + + + +
+
+
+ + + + + diff --git a/internal/webui/webui.go b/internal/webui/webui.go new file mode 100644 index 0000000..7b9310a --- /dev/null +++ b/internal/webui/webui.go @@ -0,0 +1,15 @@ +package webui + +import ( + "embed" + "io/fs" + "net/http" +) + +//go:embed assets/* +var embedded embed.FS + +func Handler() http.Handler { + sub, _ := fs.Sub(embedded, "assets") + return http.FileServer(http.FS(sub)) +} diff --git a/internal/webui/webui_test.go b/internal/webui/webui_test.go new file mode 100644 index 0000000..649b2cb --- /dev/null +++ b/internal/webui/webui_test.go @@ -0,0 +1,191 @@ +package webui + +import ( + "regexp" + "strings" + "testing" +) + +func appJS(t *testing.T) string { + t.Helper() + b, err := embedded.ReadFile("assets/app.js") + if err != nil { + t.Fatalf("read embedded app.js: %v", err) + } + return string(b) +} + +func TestRenderDispatcherFunctionsExist(t *testing.T) { + js := appJS(t) + start := strings.Index(js, "function render(){") + if start < 0 { + t.Fatal("render dispatcher not found") + } + end := strings.Index(js[start:], "}\n") + if end < 0 { + t.Fatal("render dispatcher end not found") + } + body := js[start : start+end+1] + calls := regexp.MustCompile(`\b(render[A-Z][A-Za-z0-9_]*)\(`).FindAllStringSubmatch(body, -1) + if len(calls) == 0 { + t.Fatal("render dispatcher contains no render function calls") + } + seen := map[string]bool{} + for _, call := range calls { + name := call[1] + if seen[name] { + continue + } + seen[name] = true + if !strings.Contains(js, "function "+name+"(") { + t.Errorf("render dispatcher calls %s but app.js does not define it", name) + } + } +} + +func TestPlacementUIHelpersExist(t *testing.T) { + js := appJS(t) + for _, name := range []string{ + "placementRule", + "placementRows", + "placementCell", + "placementSourceLabel", + "renderPlacement", + "placementCellHTML", + "placementWorkerCard", + "loadPlacement", + "openPlacementDialog", + "lines", + "installedForWorker", + "savePlacementRule", + "placementExactAction", + } { + if !strings.Contains(js, "function "+name+"(") { + t.Errorf("missing placement UI helper %s", name) + } + } +} + +func appCSS(t *testing.T) string { + t.Helper() + b, err := embedded.ReadFile("assets/app.css") + if err != nil { + t.Fatalf("read embedded app.css: %v", err) + } + return string(b) +} + +func indexHTML(t *testing.T) string { + t.Helper() + b, err := embedded.ReadFile("assets/index.html") + if err != nil { + t.Fatalf("read embedded index.html: %v", err) + } + return string(b) +} + +func TestProfessionalLightThemeIsEmbedded(t *testing.T) { + css := appCSS(t) + for _, want := range []string{ + "color-scheme:light", + "--bg:#eef2f6", + "--surface:#ffffff", + "--text:#253345", + "--font-mono:", + "Checkpoint 24 — SCMDB-inspired professional light control-plane theme", + ".live-flow-stage,.cluster-map-stage", + } { + if !strings.Contains(css, want) { + t.Errorf("light theme marker %q missing from app.css", want) + } + } + if strings.Contains(indexHTML(t), `name="color-scheme" content="dark light"`) { + t.Fatal("index still advertises dark color scheme") + } + if !strings.Contains(indexHTML(t), `name="color-scheme" content="light"`) { + t.Fatal("index does not advertise light color scheme") + } +} + +func TestCanvasThemeUsesLightNodeSurfaces(t *testing.T) { + js := appJS(t) + for _, want := range []string{ + "rgba(255,255,255,.97)", + "rgba(37,51,69,.94)", + "const liveColor=(v,a=1,l=44)", + } { + if !strings.Contains(js, want) { + t.Errorf("light canvas marker %q missing from app.js", want) + } + } + for _, legacy := range []string{ + "rgba(10,18,25,.96)", + "rgba(11,20,27,.94)", + } { + if strings.Contains(js, legacy) { + t.Errorf("legacy dark canvas color %q still present", legacy) + } + } +} + +func TestPolicyDialogCloseControlsBypassFormValidation(t *testing.T) { + html := indexHTML(t) + if strings.Contains(html, `
`) { + t.Fatal("policy dialog still uses method=dialog; required fields can block cancel controls") + } + for _, want := range []string{ + ``, + ``, + ``, + } { + if !strings.Contains(html, want) { + t.Errorf("policy dialog control missing or unsafe: %s", want) + } + } + js := appJS(t) + if !strings.Contains(js, `document.getElementById(b.dataset.closeDialog)?.close()`) { + t.Fatal("generic data-close-dialog handler missing") + } + if strings.Contains(js, `if(e.submitter?.value==='cancel')return;e.preventDefault();`) { + t.Fatal("policy submit handler still depends on cancel submit semantics") + } +} + +func TestDialogCloseTargetsExist(t *testing.T) { + html := indexHTML(t) + dialogIDs := map[string]bool{} + for _, m := range regexp.MustCompile(``) + block := pattern.FindString(html) + if block == "" { + t.Fatalf("dialog %s not found", dialogID) + } + if !strings.Contains(block, `required`) { + t.Fatalf("test assumption failed: dialog %s has no required field", dialogID) + } + if !strings.Contains(block, `type="button"`) || !strings.Contains(block, `data-close-dialog="`+dialogID+`"`) { + t.Errorf("dialog %s lacks a validation-independent close control", dialogID) + } + } +} + +func TestModelTableExposesDistinctContextWindows(t *testing.T) { + js := appJS(t) + for _, want := range []string{"Modell-Max", "Konfig.", "Geladen", "configured_context_length", "loaded_context_length"} { + if !strings.Contains(js, want) { + t.Errorf("model context UI marker %q missing", want) + } + } +} diff --git a/internal/worker/persistence.go b/internal/worker/persistence.go new file mode 100644 index 0000000..7f8c87e --- /dev/null +++ b/internal/worker/persistence.go @@ -0,0 +1,94 @@ +package worker + +import ( + "context" + "errors" + "os" + "time" + + persiststate "github.com/example/ollama-fair-gateway/internal/state" +) + +type PersistentPerformanceState struct { + Version int `json:"version"` + SavedAt time.Time `json:"saved_at"` + Workers map[string]map[string]ModelPerformance `json:"workers"` +} + +func (p *Pool) SnapshotPerformance() PersistentPerformanceState { + out := PersistentPerformanceState{Version: 1, SavedAt: time.Now().UTC(), Workers: map[string]map[string]ModelPerformance{}} + for _, w := range p.workers { + w.mu.RLock() + models := make(map[string]ModelPerformance, len(w.performance)) + for model, perf := range w.performance { + models[model] = ModelPerformance{Model: model, PromptTPS: perf.PromptTPS, OutputTPS: perf.OutputTPS, Samples: perf.Samples} + } + w.mu.RUnlock() + if len(models) > 0 { + out.Workers[w.cfg.Name] = models + } + } + return out +} + +func (p *Pool) RestorePerformance(s PersistentPerformanceState) { + if s.Version != 1 { + return + } + for workerName, models := range s.Workers { + w := p.byName[workerName] + if w == nil { + continue + } + w.mu.Lock() + for model, perf := range models { + if model == "" || perf.Samples <= 0 { + continue + } + w.performance[canonicalModel(model)] = performanceState{PromptTPS: perf.PromptTPS, OutputTPS: perf.OutputTPS, Samples: perf.Samples} + } + w.mu.Unlock() + } +} + +func (p *Pool) LoadPerformance(path string) error { + var s PersistentPerformanceState + err := (persiststate.AtomicJSON{Path: path, Mode: 0640}).Load(&s) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + p.RestorePerformance(s) + return nil +} + +func (p *Pool) SavePerformance(path string) error { + return (persiststate.AtomicJSON{Path: path, Mode: 0640}).Save(p.SnapshotPerformance()) +} + +func (p *Pool) StartPerformancePersistence(ctx context.Context, path string, interval time.Duration, onError func(error)) { + if interval < time.Second { + interval = 10 * time.Second + } + go func() { + t := time.NewTicker(interval) + defer t.Stop() + defer func() { + if err := p.SavePerformance(path); err != nil && onError != nil { + onError(err) + } + }() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := p.SavePerformance(path); err != nil && onError != nil { + onError(err) + } + } + } + }() +} diff --git a/internal/worker/persistence_test.go b/internal/worker/persistence_test.go new file mode 100644 index 0000000..806d5fd --- /dev/null +++ b/internal/worker/persistence_test.go @@ -0,0 +1,32 @@ +package worker + +import ( + "path/filepath" + "testing" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +func TestPerformancePersistenceRoundTrip(t *testing.T) { + cfg := []config.WorkerConfig{{Name: "gpu-a", URL: "http://127.0.0.1:11434", MaxConcurrent: 2}} + p := New(cfg, "") + p.RestorePerformance(PersistentPerformanceState{Version: 1, Workers: map[string]map[string]ModelPerformance{ + "gpu-a": {"qwen3:8b": {Model: "qwen3:8b", PromptTPS: 123.5, OutputTPS: 77.25, Samples: 9}}, + }}) + path := filepath.Join(t.TempDir(), "worker-performance.json") + if err := p.SavePerformance(path); err != nil { + t.Fatal(err) + } + q := New(cfg, "") + if err := q.LoadPerformance(path); err != nil { + t.Fatal(err) + } + s := q.Snapshots() + if len(s) != 1 || len(s[0].Performance) != 1 { + t.Fatalf("unexpected performance snapshot: %#v", s) + } + got := s[0].Performance[0] + if got.Model != "qwen3:8b" || got.PromptTPS != 123.5 || got.OutputTPS != 77.25 || got.Samples != 9 { + t.Fatalf("unexpected restored performance: %#v", got) + } +} diff --git a/internal/worker/pool.go b/internal/worker/pool.go new file mode 100644 index 0000000..0b213f8 --- /dev/null +++ b/internal/worker/pool.go @@ -0,0 +1,2060 @@ +package worker + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" + "github.com/example/ollama-fair-gateway/internal/hoststats" +) + +var ( + ErrModelPlacementBlocked = errors.New("model blocked by placement policy") + ErrModelNotInstalled = errors.New("model not installed on eligible worker") +) + +type state struct { + cfg config.WorkerConfig + url *url.URL + active atomic.Int64 + healthy atomic.Bool + mu sync.RWMutex + models map[string]bool + installed map[string]bool + installedKnown bool + installedModels []string + inventoryError string + modelActive map[string]int + modelMaintenance map[string]bool + metadata map[string]metadataEntry + performance map[string]performanceState + baselinePlacement config.ModelPlacementRule + baselineModelConcurrency map[string]int + placement config.ModelPlacementRule + placementOverride bool + loadedModels []LoadedModel + telemetry ResourceTelemetry + lastError string + lastCheck time.Time + maintenance string + circuitState string + circuitFailures int + circuitOpenUntil time.Time + halfOpenInFlight bool + lastCircuitError string +} + +type LoadedModel struct { + Name string `json:"name"` + Model string `json:"model,omitempty"` + Size int64 `json:"size,omitempty"` + SizeVRAM int64 `json:"size_vram,omitempty"` + ContextLength int64 `json:"context_length,omitempty"` + ExpiresAt time.Time `json:"expires_at,omitempty"` +} + +type ResourceTelemetry struct { + MemoryUsedBytes int64 `json:"memory_used_bytes,omitempty"` + MemoryTotalBytes int64 `json:"memory_total_bytes,omitempty"` + VRAMUsedBytes int64 `json:"vram_used_bytes,omitempty"` + VRAMTotalBytes int64 `json:"vram_total_bytes,omitempty"` + GPUUtilizationPct float64 `json:"gpu_utilization_percent,omitempty"` + GPUTemperatureC float64 `json:"gpu_temperature_c,omitempty"` + GPUPowerWatts float64 `json:"gpu_power_watts,omitempty"` + Source string `json:"source,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` + Error string `json:"error,omitempty"` +} + +// externalTelemetry uses pointers so an omitted field does not accidentally +// overwrite a previously collected local value with the JSON zero value. +type externalTelemetry struct { + MemoryUsedBytes *int64 `json:"memory_used_bytes"` + MemoryTotalBytes *int64 `json:"memory_total_bytes"` + VRAMUsedBytes *int64 `json:"vram_used_bytes"` + VRAMTotalBytes *int64 `json:"vram_total_bytes"` + GPUUtilizationPct *float64 `json:"gpu_utilization_percent"` + GPUTemperatureC *float64 `json:"gpu_temperature_c"` + GPUPowerWatts *float64 `json:"gpu_power_watts"` + Source string `json:"source,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` + Error string `json:"error,omitempty"` +} + +type ModelMetadata struct { + Model string `json:"model"` + Capabilities []string `json:"capabilities,omitempty"` + ContextLength int64 `json:"context_length,omitempty"` + ConfiguredContextLength int64 `json:"configured_context_length,omitempty"` + Details ModelDetails `json:"details,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` + Error string `json:"error,omitempty"` +} + +type ContextWindow struct { + Worker string `json:"worker"` + Model string `json:"model"` + ModelMaxTokens int64 `json:"model_max_tokens,omitempty"` + ConfiguredTokens int64 `json:"configured_tokens,omitempty"` + LoadedTokens int64 `json:"loaded_tokens,omitempty"` + WorkerDefaultTokens int64 `json:"worker_default_tokens,omitempty"` + WorkerLimitTokens int64 `json:"worker_limit_tokens,omitempty"` + EffectiveTokens int64 `json:"effective_tokens,omitempty"` + EffectiveSource string `json:"effective_source,omitempty"` + MetadataError string `json:"metadata_error,omitempty"` +} + +type metadataEntry struct { + Data ModelMetadata + FetchedAt time.Time +} + +type ModelPerformance struct { + Model string `json:"model"` + PromptTPS float64 `json:"prompt_tps,omitempty"` + OutputTPS float64 `json:"output_tps,omitempty"` + Samples int64 `json:"samples"` +} + +type performanceState struct { + PromptTPS float64 + OutputTPS float64 + Samples int64 +} + +type Snapshot struct { + Name string `json:"name"` + URL string `json:"url"` + Healthy bool `json:"healthy"` + Active int64 `json:"active"` + MaxConcurrent int `json:"max_concurrent"` + Models []string `json:"models"` + LoadedModels []LoadedModel `json:"loaded_models,omitempty"` + ResidentBytes int64 `json:"resident_bytes,omitempty"` + VRAMBytes int64 `json:"vram_bytes,omitempty"` + MemoryCapacityBytes int64 `json:"memory_capacity_bytes,omitempty"` + VRAMCapacityBytes int64 `json:"vram_capacity_bytes,omitempty"` + MemoryUsedBytes int64 `json:"memory_used_bytes,omitempty"` + MemoryTotalBytes int64 `json:"memory_total_bytes,omitempty"` + VRAMUsedBytes int64 `json:"vram_used_bytes,omitempty"` + VRAMTotalBytes int64 `json:"vram_total_bytes,omitempty"` + GPUUtilizationPct float64 `json:"gpu_utilization_percent,omitempty"` + GPUTemperatureC float64 `json:"gpu_temperature_c,omitempty"` + GPUPowerWatts float64 `json:"gpu_power_watts,omitempty"` + ModelActive map[string]int `json:"model_active,omitempty"` + ModelLimits map[string]int `json:"model_limits,omitempty"` + Performance []ModelPerformance `json:"performance,omitempty"` + ModelPlacement config.ModelPlacementRule `json:"model_placement"` + PlacementOverride bool `json:"placement_override,omitempty"` + Maintenance string `json:"maintenance"` + AcceptingNew bool `json:"accepting_new"` + CircuitState string `json:"circuit_state"` + CircuitFailures int `json:"circuit_failures,omitempty"` + CircuitOpenUntil time.Time `json:"circuit_open_until,omitempty"` + LastCircuitError string `json:"last_circuit_error,omitempty"` + TelemetrySource string `json:"telemetry_source,omitempty"` + TelemetryError string `json:"telemetry_error,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + LastError string `json:"last_error,omitempty"` + LastCheck time.Time `json:"last_check"` +} + +type PlacementDecision struct { + Allowed bool `json:"allowed"` + Source string `json:"source"` + Pattern string `json:"pattern,omitempty"` + ExactOverride bool `json:"exact_override,omitempty"` +} + +type PlacementSnapshot struct { + Worker string `json:"worker"` + Baseline config.ModelPlacementRule `json:"baseline"` + Effective config.ModelPlacementRule `json:"effective"` + Override bool `json:"override"` + InventoryKnown bool `json:"inventory_known"` + InstalledModels []string `json:"installed_models,omitempty"` + LoadedModels []string `json:"loaded_models,omitempty"` + InventoryError string `json:"inventory_error,omitempty"` +} + +type Pool struct { + workers []*state + byName map[string]*state + client *http.Client + notify chan struct{} + control string + routing config.RoutingConfig + capCfg config.ModelCapabilitiesConfig + reliability config.ReliabilityConfig +} + +type Lease struct { + State *state + model string + once sync.Once + pool *Pool +} + +func (l *Lease) Name() string { return l.State.cfg.Name } +func (l *Lease) URL() *url.URL { return l.State.url } +func (l *Lease) Release() { + if l == nil { + return + } + l.once.Do(func() { + l.State.abandonCircuitProbe() + l.State.active.Add(-1) + if l.model != "" { + l.State.mu.Lock() + if l.State.modelActive[l.model] > 1 { + l.State.modelActive[l.model]-- + } else { + delete(l.State.modelActive, l.model) + } + l.State.mu.Unlock() + } + l.pool.signal() + }) +} + +func New(cfgs []config.WorkerConfig, control string) *Pool { + p := &Pool{byName: map[string]*state{}, notify: make(chan struct{}, 1), control: control, client: &http.Client{Timeout: 4 * time.Second}} + p.SetRoutingConfig(config.RoutingConfig{}) + p.SetModelCapabilitiesConfig(config.ModelCapabilitiesConfig{}) + for _, c := range cfgs { + u, _ := url.Parse(strings.TrimRight(c.URL, "/")) + placement := normalizePlacementRule(c.ModelPlacement) + baselineConc := map[string]int{} + for k, v := range c.ModelConcurrency { + baselineConc[k] = v + } + s := &state{cfg: c, url: u, models: map[string]bool{}, installed: map[string]bool{}, modelActive: map[string]int{}, modelMaintenance: map[string]bool{}, metadata: map[string]metadataEntry{}, performance: map[string]performanceState{}, baselinePlacement: placement, baselineModelConcurrency: baselineConc, placement: placement, maintenance: "active", circuitState: "closed"} + s.healthy.Store(true) + p.workers = append(p.workers, s) + p.byName[c.Name] = s + } + return p +} + +func (p *Pool) SetRoutingConfig(c config.RoutingConfig) { + if c.LoadedBonus == 0 { + c.LoadedBonus = 60 + } + if c.InstalledBonus == 0 { + c.InstalledBonus = 30 + } + if c.ThroughputBonus == 0 { + c.ThroughputBonus = 20 + } + if c.VRAMPressurePenalty == 0 { + c.VRAMPressurePenalty = 35 + } + if c.GPUUtilizationPenalty == 0 { + c.GPUUtilizationPenalty = 10 + } + if c.AvoidVRAMPercent == 0 { + c.AvoidVRAMPercent = 97 + } + p.routing = c +} + +func (p *Pool) SetModelCapabilitiesConfig(c config.ModelCapabilitiesConfig) { + if c.Mode == "" { + c.Mode = "enforce" + } + if c.CacheTTL == 0 { + c.CacheTTL = config.Duration(10 * time.Minute) + } + if c.ContextGuard == "" { + c.ContextGuard = "reject" + } + if c.Context.MaxRequestedTokens == 0 { + c.Context.MaxRequestedTokens = 32768 + } + if c.Context.DefaultWorkerTokens == 0 { + c.Context.DefaultWorkerTokens = 4096 + } + if c.Context.EstimationMarginPercent == 0 { + c.Context.EstimationMarginPercent = 15 + } + if c.Context.VisionReserveTokensPerImage == 0 { + c.Context.VisionReserveTokensPerImage = 2048 + } + p.capCfg = c +} + +func (p *Pool) SetReliabilityConfig(c config.ReliabilityConfig) { + if c.FailureThreshold <= 0 { + c.FailureThreshold = 3 + } + if c.OpenDuration == 0 { + c.OpenDuration = config.Duration(30 * time.Second) + } + p.reliability = c +} + +func (p *Pool) Start(ctx context.Context) { + for _, w := range p.workers { + p.refresh(ctx, w) + go p.healthLoop(ctx, w) + } +} +func (p *Pool) Health(context.Context) error { + for _, w := range p.workers { + if w.healthy.Load() { + return nil + } + } + return fmt.Errorf("no healthy Ollama workers") +} +func (p *Pool) signal() { + select { + case p.notify <- struct{}{}: + default: + } +} + +func (p *Pool) Acquire(ctx context.Context, model string) (*Lease, error) { + return p.AcquireExcluding(ctx, model, nil) +} + +func (p *Pool) AcquireExcluding(ctx context.Context, model string, excluded map[string]bool) (*Lease, error) { + return p.AcquireAllowedExcluding(ctx, model, nil, excluded, 0) +} + +// AcquireAllowed limits worker selection to an admission-approved set. A nil +// set preserves the legacy behavior. requestedContext is used only to avoid a +// misleading loaded-model bonus when an explicit num_ctx would require Ollama +// to reload the model with a larger KV cache. +func (p *Pool) AcquireAllowed(ctx context.Context, model string, allowed map[string]bool, requestedContext int64) (*Lease, error) { + return p.AcquireAllowedExcluding(ctx, model, allowed, nil, requestedContext) +} + +func (p *Pool) AcquireAllowedExcluding(ctx context.Context, model string, allowed, excluded map[string]bool, requestedContext int64) (*Lease, error) { + for { + candidates := p.candidatesExcludingAllowed(model, excluded, allowed, requestedContext) + for _, w := range candidates { + for { + cur := w.active.Load() + if cur >= int64(w.cfg.MaxConcurrent) { + break + } + if !w.active.CompareAndSwap(cur, cur+1) { + continue + } + if !w.claimCircuitProbe(p.reliability) { + w.active.Add(-1) + continue + } + if model != "" && !w.acquireModelSlot(model) { + w.abandonCircuitProbe() + w.active.Add(-1) + break + } + return &Lease{State: w, model: canonicalModel(model), pool: p}, nil + } + } + if len(candidates) == 0 { + return nil, p.noCandidateError(model) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-p.notify: + case <-time.After(50 * time.Millisecond): + } + } +} +func (p *Pool) Control() (*url.URL, string, error) { + if p.control != "" { + if w := p.byName[p.control]; w != nil && w.healthy.Load() { + return w.url, w.cfg.Name, nil + } + } + for _, w := range p.workers { + if w.healthy.Load() { + return w.url, w.cfg.Name, nil + } + } + return nil, "", fmt.Errorf("no healthy control worker") +} +func (p *Pool) ControlForModel(model string) (*url.URL, string, error) { + if model != "" { + candidates := p.candidates(model) + if len(candidates) == 0 { + return nil, "", p.noCandidateError(model) + } + return candidates[0].url, candidates[0].cfg.Name, nil + } + return p.Control() +} + +func (p *Pool) noCandidateError(model string) error { + healthy, allowed, knownInventories := 0, 0, 0 + for _, w := range p.workers { + if !w.healthy.Load() { + continue + } + healthy++ + if model != "" && !w.placementDecision(model).Allowed { + continue + } + allowed++ + w.mu.RLock() + if w.installedKnown { + knownInventories++ + } + w.mu.RUnlock() + } + if healthy == 0 { + return fmt.Errorf("no healthy Ollama workers") + } + if model != "" && allowed == 0 { + return fmt.Errorf("%w: model %q is blocked on all healthy workers", ErrModelPlacementBlocked, model) + } + if model != "" && allowed > 0 && knownInventories == allowed { + return fmt.Errorf("%w: model %q is not installed on any eligible worker", ErrModelNotInstalled, model) + } + return fmt.Errorf("no eligible Ollama worker for model %q", model) +} + +func (p *Pool) Snapshots() []Snapshot { + out := make([]Snapshot, 0, len(p.workers)) + for _, w := range p.workers { + w.mu.RLock() + models := make([]string, 0, len(w.models)) + for m := range w.models { + models = append(models, m) + } + sort.Strings(models) + loaded := append([]LoadedModel(nil), w.loadedModels...) + tel := w.telemetry + var resident, vram int64 + for _, m := range loaded { + resident += m.Size + vram += m.SizeVRAM + } + memoryTotal := tel.MemoryTotalBytes + if memoryTotal <= 0 { + memoryTotal = w.cfg.MemoryCapacityBytes + } + vramTotal := tel.VRAMTotalBytes + if vramTotal <= 0 { + vramTotal = w.cfg.VRAMCapacityBytes + } + activeByModel := make(map[string]int, len(w.modelActive)) + for k, v := range w.modelActive { + activeByModel[k] = v + } + limits := make(map[string]int, len(w.cfg.ModelConcurrency)) + for k, v := range w.cfg.ModelConcurrency { + limits[k] = v + } + perf := performanceSnapshotLocked(w) + maintenance := w.maintenance + circuitState := w.circuitState + if circuitState == "open" && !w.circuitOpenUntil.IsZero() && !time.Now().Before(w.circuitOpenUntil) { + circuitState = "half_open" + } + accepting := maintenance == "active" && circuitState != "open" + x := Snapshot{Name: w.cfg.Name, URL: w.url.String(), Healthy: w.healthy.Load(), Active: w.active.Load(), MaxConcurrent: w.cfg.MaxConcurrent, Models: models, LoadedModels: loaded, ResidentBytes: resident, VRAMBytes: vram, MemoryCapacityBytes: w.cfg.MemoryCapacityBytes, VRAMCapacityBytes: w.cfg.VRAMCapacityBytes, MemoryUsedBytes: tel.MemoryUsedBytes, MemoryTotalBytes: memoryTotal, VRAMUsedBytes: tel.VRAMUsedBytes, VRAMTotalBytes: vramTotal, GPUUtilizationPct: tel.GPUUtilizationPct, GPUTemperatureC: tel.GPUTemperatureC, GPUPowerWatts: tel.GPUPowerWatts, ModelActive: activeByModel, ModelLimits: limits, Performance: perf, ModelPlacement: clonePlacementRule(w.placement), PlacementOverride: w.placementOverride, Maintenance: maintenance, AcceptingNew: accepting, CircuitState: circuitState, CircuitFailures: w.circuitFailures, CircuitOpenUntil: w.circuitOpenUntil, LastCircuitError: w.lastCircuitError, TelemetrySource: tel.Source, TelemetryError: tel.Error, Labels: w.cfg.Labels, LastError: w.lastError, LastCheck: w.lastCheck} + w.mu.RUnlock() + out = append(out, x) + } + return out +} +func (p *Pool) candidates(model string) []*state { return p.candidatesExcluding(model, nil) } +func (p *Pool) candidatesExcluding(model string, excluded map[string]bool) []*state { + return p.candidatesExcludingAllowed(model, excluded, nil, 0) +} +func (p *Pool) candidatesExcludingAllowed(model string, excluded, allowed map[string]bool, requestedContext int64) []*state { + eligible := make([]*state, 0, len(p.workers)) + maxTPS := 0.0 + for _, w := range p.workers { + if !w.healthy.Load() || (excluded != nil && excluded[w.cfg.Name]) || (allowed != nil && !allowed[w.cfg.Name]) || !w.acceptingNew(p.reliability) { + continue + } + if model != "" && !w.placementDecision(model).Allowed { + continue + } + if model != "" && w.modelMaintenanceActive(model) { + continue + } + eligible = append(eligible, w) + if t := w.outputTPS(model); t > maxTPS { + maxTPS = t + } + } + out := eligible + if model != "" { + installed := make([]*state, 0, len(eligible)) + unknown := make([]*state, 0, len(eligible)) + for _, w := range eligible { + known, has := w.inventoryState(model) + if !known { + unknown = append(unknown, w) + continue + } + if has { + installed = append(installed, w) + } + } + if len(installed) > 0 { + out = installed + } else if len(unknown) > 0 { + out = unknown + } else { + out = nil + } + } + sort.SliceStable(out, func(i, j int) bool { + return p.scoreForContext(out[i], model, maxTPS, requestedContext) < p.scoreForContext(out[j], model, maxTPS, requestedContext) + }) + return out +} + +func clonePlacementRule(r config.ModelPlacementRule) config.ModelPlacementRule { + r.AllowedModels = append([]string(nil), r.AllowedModels...) + r.DeniedModels = append([]string(nil), r.DeniedModels...) + return r +} + +func normalizePlacementRule(r config.ModelPlacementRule) config.ModelPlacementRule { + if strings.TrimSpace(r.Mode) == "" { + r.Mode = "allow_all" + } + r.Mode = strings.TrimSpace(r.Mode) + norm := func(in []string) []string { + out := make([]string, 0, len(in)) + seen := map[string]bool{} + for _, x := range in { + x = strings.TrimSpace(x) + if x == "" || seen[x] { + continue + } + seen[x] = true + out = append(out, x) + } + sort.Strings(out) + return out + } + r.AllowedModels = norm(r.AllowedModels) + r.DeniedModels = norm(r.DeniedModels) + return r +} + +func modelPatternSpecificity(pattern, model string) (int, bool, bool) { + pattern = strings.TrimSpace(pattern) + model = strings.TrimSpace(model) + plain := canonicalModel(model) + if pattern == "" { + return 0, false, false + } + if pattern == model || pattern == plain { + return 100000 + len(pattern), true, true + } + if pattern == "*" { + return 0, true, false + } + if strings.HasSuffix(pattern, "*") { + prefix := strings.TrimSuffix(pattern, "*") + if strings.HasPrefix(model, prefix) || strings.HasPrefix(plain, prefix) { + return len(prefix), true, false + } + } + return 0, false, false +} + +func evaluatePlacement(r config.ModelPlacementRule, model string) PlacementDecision { + r = normalizePlacementRule(r) + bestAllow, bestDeny := -1, -1 + allowPattern, denyPattern := "", "" + allowExact, denyExact := false, false + for _, p := range r.AllowedModels { + if spec, ok, exact := modelPatternSpecificity(p, model); ok && spec > bestAllow { + bestAllow, allowPattern, allowExact = spec, p, exact + } + } + for _, p := range r.DeniedModels { + if spec, ok, exact := modelPatternSpecificity(p, model); ok && spec > bestDeny { + bestDeny, denyPattern, denyExact = spec, p, exact + } + } + if bestAllow >= 0 || bestDeny >= 0 { + if bestAllow > bestDeny { + return PlacementDecision{Allowed: true, Source: "allow_rule", Pattern: allowPattern, ExactOverride: allowExact} + } + return PlacementDecision{Allowed: false, Source: "deny_rule", Pattern: denyPattern, ExactOverride: denyExact} + } + if r.Mode == "whitelist" { + return PlacementDecision{Allowed: false, Source: "whitelist_default"} + } + return PlacementDecision{Allowed: true, Source: "allow_all_default"} +} + +func (w *state) placementDecision(model string) PlacementDecision { + w.mu.RLock() + r := clonePlacementRule(w.placement) + w.mu.RUnlock() + return evaluatePlacement(r, model) +} + +func (w *state) inventoryState(model string) (known, has bool) { + if model == "" { + return true, true + } + plain := canonicalModel(model) + w.mu.RLock() + known = w.installedKnown + has = w.installed[model] || w.installed[plain] || w.models[model] || w.models[plain] + w.mu.RUnlock() + return known, has +} + +func (p *Pool) PlacementDecision(workerName, model string) (PlacementDecision, bool) { + w := p.byName[workerName] + if w == nil { + return PlacementDecision{}, false + } + return w.placementDecision(model), true +} + +func (p *Pool) PlacementSnapshots() []PlacementSnapshot { + out := make([]PlacementSnapshot, 0, len(p.workers)) + for _, w := range p.workers { + w.mu.RLock() + loaded := make([]string, 0, len(w.loadedModels)) + seenLoaded := map[string]bool{} + for _, m := range w.loadedModels { + name := strings.TrimSpace(m.Model) + if name == "" { + name = strings.TrimSpace(m.Name) + } + if name != "" && !seenLoaded[name] { + seenLoaded[name] = true + loaded = append(loaded, name) + } + } + sort.Strings(loaded) + out = append(out, PlacementSnapshot{Worker: w.cfg.Name, Baseline: clonePlacementRule(w.baselinePlacement), Effective: clonePlacementRule(w.placement), Override: w.placementOverride, InventoryKnown: w.installedKnown, InstalledModels: append([]string(nil), w.installedModels...), LoadedModels: loaded, InventoryError: w.inventoryError}) + w.mu.RUnlock() + } + sort.Slice(out, func(i, j int) bool { return out[i].Worker < out[j].Worker }) + return out +} + +func (p *Pool) SetPlacement(workerName string, rule config.ModelPlacementRule, override bool) error { + if err := config.ValidateModelPlacementRule(rule); err != nil { + return err + } + w := p.byName[workerName] + if w == nil { + return fmt.Errorf("unknown worker %q", workerName) + } + rule = normalizePlacementRule(rule) + w.mu.Lock() + w.placement = rule + w.placementOverride = override + w.mu.Unlock() + p.signal() + return nil +} + +func (p *Pool) ResetPlacement(workerName string) error { + w := p.byName[workerName] + if w == nil { + return fmt.Errorf("unknown worker %q", workerName) + } + w.mu.Lock() + w.placement = clonePlacementRule(w.baselinePlacement) + w.placementOverride = false + w.mu.Unlock() + p.signal() + return nil +} + +func normalizeMaintenance(mode string) (string, error) { + mode = strings.ToLower(strings.TrimSpace(mode)) + if mode == "" { + mode = "active" + } + switch mode { + case "active", "draining", "disabled": + return mode, nil + } + return "", fmt.Errorf("maintenance mode must be active, draining, or disabled") +} + +func (p *Pool) SetMaintenance(workerName, mode string) error { + w := p.byName[workerName] + if w == nil { + return fmt.Errorf("unknown worker %q", workerName) + } + m, err := normalizeMaintenance(mode) + if err != nil { + return err + } + w.mu.Lock() + w.maintenance = m + w.mu.Unlock() + p.signal() + return nil +} + +func (p *Pool) Maintenance(workerName string) (string, bool) { + w := p.byName[workerName] + if w == nil { + return "", false + } + w.mu.RLock() + m := w.maintenance + w.mu.RUnlock() + return m, true +} + +func (w *state) acceptingNew(rel config.ReliabilityConfig) bool { + w.mu.RLock() + defer w.mu.RUnlock() + if w.maintenance != "active" { + return false + } + if !rel.Enabled { + return true + } + if w.circuitState == "open" && time.Now().Before(w.circuitOpenUntil) { + return false + } + if w.circuitState == "half_open" && w.halfOpenInFlight { + return false + } + return true +} + +func (w *state) claimCircuitProbe(rel config.ReliabilityConfig) bool { + if !rel.Enabled { + return true + } + w.mu.Lock() + defer w.mu.Unlock() + if w.maintenance != "active" { + return false + } + now := time.Now() + if w.circuitState == "open" { + if now.Before(w.circuitOpenUntil) { + return false + } + if w.halfOpenInFlight { + return false + } + w.circuitState = "half_open" + w.halfOpenInFlight = true + return true + } + if w.circuitState == "half_open" { + if w.halfOpenInFlight { + return false + } + w.halfOpenInFlight = true + return true + } + return true +} + +func (w *state) abandonCircuitProbe() { + w.mu.Lock() + if w.circuitState == "half_open" && w.halfOpenInFlight { + w.halfOpenInFlight = false + } + w.mu.Unlock() +} + +func (p *Pool) ReportResult(workerName string, failed bool, errText string) bool { + w := p.byName[workerName] + if w == nil || !p.reliability.Enabled { + return false + } + w.mu.Lock() + defer w.mu.Unlock() + if !failed { + w.circuitFailures = 0 + w.circuitState = "closed" + w.circuitOpenUntil = time.Time{} + w.halfOpenInFlight = false + w.lastCircuitError = "" + return false + } + wasOpen := w.circuitState == "open" + w.circuitFailures++ + w.lastCircuitError = errText + if w.circuitState == "half_open" || w.circuitFailures >= p.reliability.FailureThreshold { + w.circuitState = "open" + w.circuitOpenUntil = time.Now().Add(p.reliability.OpenDuration.Value()) + } + w.halfOpenInFlight = false + p.signal() + return !wasOpen && w.circuitState == "open" +} + +func (p *Pool) CircuitReset(workerName string) error { + w := p.byName[workerName] + if w == nil { + return fmt.Errorf("unknown worker %q", workerName) + } + w.mu.Lock() + w.circuitState = "closed" + w.circuitFailures = 0 + w.circuitOpenUntil = time.Time{} + w.halfOpenInFlight = false + w.lastCircuitError = "" + w.mu.Unlock() + p.signal() + return nil +} + +func (p *Pool) CanRoute(model string) bool { return len(p.candidates(model)) > 0 } + +func (p *Pool) SetModelConcurrency(workerName, model string, limit int) error { + w := p.byName[workerName] + if w == nil { + return fmt.Errorf("unknown worker %q", workerName) + } + model = strings.TrimSpace(model) + if model == "" || limit <= 0 { + return fmt.Errorf("model and positive limit are required") + } + w.mu.Lock() + if w.cfg.ModelConcurrency == nil { + w.cfg.ModelConcurrency = map[string]int{} + } + w.cfg.ModelConcurrency[model] = minInt(limit, w.cfg.MaxConcurrent) + w.mu.Unlock() + p.signal() + return nil +} + +func (p *Pool) ResetModelConcurrency(workerName, model string) error { + w := p.byName[workerName] + if w == nil { + return fmt.Errorf("unknown worker %q", workerName) + } + model = strings.TrimSpace(model) + if model == "" { + return fmt.Errorf("model required") + } + w.mu.Lock() + if v, ok := w.baselineModelConcurrency[model]; ok { + if w.cfg.ModelConcurrency == nil { + w.cfg.ModelConcurrency = map[string]int{} + } + w.cfg.ModelConcurrency[model] = v + } else { + delete(w.cfg.ModelConcurrency, model) + } + w.mu.Unlock() + p.signal() + return nil +} + +func (p *Pool) WorkerConfig(workerName string) (config.WorkerConfig, bool) { + w := p.byName[workerName] + if w == nil { + return config.WorkerConfig{}, false + } + w.mu.RLock() + defer w.mu.RUnlock() + c := w.cfg + c.ModelConcurrency = map[string]int{} + for k, v := range w.cfg.ModelConcurrency { + c.ModelConcurrency[k] = v + } + c.ContextLimits = map[string]int64{} + for k, v := range w.cfg.ContextLimits { + c.ContextLimits[k] = v + } + return c, true +} + +func canonicalModel(model string) string { + model = strings.TrimSpace(model) + if model == "" { + return "" + } + return strings.TrimSuffix(model, ":latest") +} + +func (w *state) hasModel(model string) bool { + if model == "" { + return true + } + plain := canonicalModel(model) + w.mu.RLock() + ok := w.installed[model] || w.installed[plain] || w.models[model] || w.models[plain] + w.mu.RUnlock() + return ok +} + +func (w *state) loadedContext(model string) int64 { + plain := canonicalModel(model) + w.mu.RLock() + defer w.mu.RUnlock() + for _, m := range w.loadedModels { + id := strings.TrimSpace(m.Model) + if id == "" { + id = strings.TrimSpace(m.Name) + } + if id == model || canonicalModel(id) == plain { + return m.ContextLength + } + } + return 0 +} + +func (w *state) contextLimit(model string) int64 { + w.mu.RLock() + limits := make(map[string]int64, len(w.cfg.ContextLimits)) + for pattern, limit := range w.cfg.ContextLimits { + limits[pattern] = limit + } + w.mu.RUnlock() + bestSpec := -1 + var best int64 + for pattern, limit := range limits { + if spec, ok, _ := modelPatternSpecificity(pattern, model); ok && spec > bestSpec { + bestSpec, best = spec, limit + } + } + return best +} + +func minPositive(values ...int64) int64 { + var out int64 + for _, v := range values { + if v <= 0 { + continue + } + if out == 0 || v < out { + out = v + } + } + return out +} + +func (p *Pool) contextWindowForWorker(ctx context.Context, w *state, model string) ContextWindow { + workerDefault := w.cfg.DefaultContextTokens + if workerDefault == 0 { + workerDefault = p.capCfg.Context.DefaultWorkerTokens + } + x := ContextWindow{Worker: w.cfg.Name, Model: model, WorkerDefaultTokens: workerDefault, WorkerLimitTokens: w.contextLimit(model)} + meta, err := p.metadataForWorker(ctx, w, model) + if err != nil { + x.MetadataError = err.Error() + } else { + x.ModelMaxTokens = meta.ContextLength + x.ConfiguredTokens = meta.ConfiguredContextLength + } + x.LoadedTokens = w.loadedContext(model) + // The loaded context is the strongest evidence because it is what Ollama is + // actually using right now. Otherwise prefer a Modelfile num_ctx, then the + // operator/default worker context. -1 means explicitly fall back to the + // theoretical model maximum. + switch { + case x.LoadedTokens > 0: + x.EffectiveTokens, x.EffectiveSource = x.LoadedTokens, "loaded" + case x.ConfiguredTokens > 0: + x.EffectiveTokens, x.EffectiveSource = x.ConfiguredTokens, "modelfile" + case x.WorkerDefaultTokens > 0: + x.EffectiveTokens, x.EffectiveSource = x.WorkerDefaultTokens, "worker_default" + case x.WorkerDefaultTokens == -1 && x.ModelMaxTokens > 0: + x.EffectiveTokens, x.EffectiveSource = x.ModelMaxTokens, "model_max" + } + x.EffectiveTokens = minPositive(x.EffectiveTokens, x.ModelMaxTokens, x.WorkerLimitTokens) + if x.EffectiveTokens == 0 && x.EffectiveSource != "" { + x.EffectiveSource = "" + } + return x +} + +// ContextWindows returns the effective context evidence for every currently +// routable worker that owns the model. It is intentionally independent of +// concurrency so admission can reject an impossible context before waiting in +// the fair scheduler queue. +func (p *Pool) ContextWindows(ctx context.Context, model string) []ContextWindow { + candidates := p.candidates(model) + out := make([]ContextWindow, 0, len(candidates)) + for _, w := range candidates { + out = append(out, p.contextWindowForWorker(ctx, w, model)) + } + return out +} + +func (w *state) modelLimit(model string) int { + raw := strings.TrimSpace(model) + plain := canonicalModel(raw) + limit := w.cfg.MaxConcurrent + best := "" + for pattern, n := range w.cfg.ModelConcurrency { + if pattern == raw || pattern == plain { + return minInt(n, w.cfg.MaxConcurrent) + } + if pattern == "*" && best == "" { + best = pattern + limit = n + continue + } + if strings.HasSuffix(pattern, "*") { + prefix := strings.TrimSuffix(pattern, "*") + if (strings.HasPrefix(raw, prefix) || strings.HasPrefix(plain, prefix)) && len(pattern) > len(best) { + best = pattern + limit = n + } + } + } + if limit <= 0 { + limit = w.cfg.MaxConcurrent + } + return minInt(limit, w.cfg.MaxConcurrent) +} + +func (w *state) acquireModelSlot(model string) bool { + key := canonicalModel(model) + limit := w.modelLimit(model) + w.mu.Lock() + defer w.mu.Unlock() + if w.modelMaintenance[key] || w.modelActive[key] >= limit { + return false + } + w.modelActive[key]++ + return true +} + +func (w *state) modelMaintenanceActive(model string) bool { + key := canonicalModel(model) + w.mu.RLock() + busy := w.modelMaintenance[key] + w.mu.RUnlock() + return busy +} + +// BeginModelMaintenance reserves a worker/model pair for a short control-plane +// operation such as preload or unload. New inference leases for the same model +// are excluded until the returned release function is called. This closes the +// race where an idle-unload could otherwise start exactly as inference arrives. +func (p *Pool) BeginModelMaintenance(workerName, model string) (func(), error) { + w := p.byName[workerName] + if w == nil { + return nil, fmt.Errorf("unknown worker %q", workerName) + } + key := canonicalModel(model) + if key == "" { + return nil, errors.New("model required") + } + if !w.healthy.Load() { + return nil, fmt.Errorf("worker %s is unhealthy", workerName) + } + w.mu.Lock() + if w.maintenance != "active" { + w.mu.Unlock() + return nil, fmt.Errorf("worker %s is %s", workerName, w.maintenance) + } + if w.circuitState != "" && w.circuitState != "closed" { + w.mu.Unlock() + return nil, fmt.Errorf("worker %s circuit is %s", workerName, w.circuitState) + } + if w.modelMaintenance[key] { + w.mu.Unlock() + return nil, fmt.Errorf("model %s already has a maintenance operation on worker %s", model, workerName) + } + if w.modelActive[key] > 0 { + w.mu.Unlock() + return nil, fmt.Errorf("model %s is active on worker %s", model, workerName) + } + w.modelMaintenance[key] = true + w.mu.Unlock() + p.signal() + var once sync.Once + return func() { + once.Do(func() { + w.mu.Lock() + delete(w.modelMaintenance, key) + w.mu.Unlock() + p.signal() + }) + }, nil +} + +func (w *state) outputTPS(model string) float64 { + w.mu.RLock() + defer w.mu.RUnlock() + if p, ok := w.performance[canonicalModel(model)]; ok { + return p.OutputTPS + } + return 0 +} + +func (p *Pool) score(w *state, model string, maxTPS float64) float64 { + return p.scoreForContext(w, model, maxTPS, 0) +} + +func (p *Pool) scoreForContext(w *state, model string, maxTPS float64, requestedContext int64) float64 { + active := float64(w.active.Load()) / float64(max(1, w.cfg.MaxConcurrent)) + s := active * 100 + w.mu.RLock() + plain := canonicalModel(model) + loaded := w.models[model] || w.models[plain] + installed := w.installed[model] || w.installed[plain] + loadedContext := int64(0) + if requestedContext > 0 && loaded { + for _, m := range w.loadedModels { + id := strings.TrimSpace(m.Model) + if id == "" { + id = strings.TrimSpace(m.Name) + } + if id == model || canonicalModel(id) == plain { + loadedContext = m.ContextLength + break + } + } + } + tel := w.telemetry + modelActive := w.modelActive[plain] + perf := w.performance[plain] + w.mu.RUnlock() + if requestedContext > 0 && loadedContext > 0 && loadedContext < requestedContext { + // An explicit larger num_ctx can still be routed here, but Ollama must + // reload/resize the model, so do not pretend the current loaded state is + // a warm-context advantage. + loaded = false + installed = true + } + if loaded { + s -= p.routing.LoadedBonus + } else if installed { + s -= p.routing.InstalledBonus + } + if lim := w.modelLimit(model); lim > 0 { + s += 25 * float64(modelActive) / float64(lim) + } + if maxTPS > 0 && perf.OutputTPS > 0 { + s -= p.routing.ThroughputBonus * math.Min(1, perf.OutputTPS/maxTPS) + } + vramTotal := tel.VRAMTotalBytes + if vramTotal <= 0 { + vramTotal = w.cfg.VRAMCapacityBytes + } + if vramTotal > 0 && tel.VRAMUsedBytes > 0 { + pct := 100 * float64(tel.VRAMUsedBytes) / float64(vramTotal) + s += p.routing.VRAMPressurePenalty * math.Min(1, pct/100) + if !loaded && p.routing.AvoidVRAMPercent > 0 && pct >= p.routing.AvoidVRAMPercent { + s += 500 + } + } + if tel.GPUUtilizationPct > 0 { + s += p.routing.GPUUtilizationPenalty * math.Min(1, tel.GPUUtilizationPct/100) + } + return s +} + +// RoutingExplanation describes why a worker is or is not eligible for a model. +// It is intended for the admin policy simulator and never acquires a worker slot. +type RoutingExplanation struct { + Worker string `json:"worker"` + Eligible bool `json:"eligible"` + Reason string `json:"reason,omitempty"` + Healthy bool `json:"healthy"` + Maintenance string `json:"maintenance"` + CircuitState string `json:"circuit_state"` + Placement PlacementDecision `json:"placement"` + InventoryKnown bool `json:"inventory_known"` + Installed bool `json:"installed"` + Loaded bool `json:"loaded"` + Active int64 `json:"active"` + MaxConcurrent int `json:"max_concurrent"` + ModelActive int `json:"model_active"` + ModelLimit int `json:"model_limit"` + OutputTPS float64 `json:"output_tps,omitempty"` + VRAMPercent float64 `json:"vram_percent,omitempty"` + GPUUtilizationPercent float64 `json:"gpu_utilization_percent,omitempty"` + Score float64 `json:"score,omitempty"` + ScoreComponents map[string]float64 `json:"score_components,omitempty"` +} + +// ExplainRouting snapshots the exact eligibility gates and adaptive score used +// for a model without reserving scheduler or worker capacity. +func (p *Pool) ExplainRouting(model string) []RoutingExplanation { + return p.ExplainRoutingAllowed(model, nil, 0) +} + +// ExplainRoutingAllowed additionally applies an admission-approved context +// worker set so the policy simulator can mirror real context-aware routing. +func (p *Pool) ExplainRoutingAllowed(model string, allowed map[string]bool, requestedContext int64) []RoutingExplanation { + plain := canonicalModel(model) + maxTPS := 0.0 + for _, w := range p.workers { + if t := w.outputTPS(model); t > maxTPS { + maxTPS = t + } + } + out := make([]RoutingExplanation, 0, len(p.workers)) + for _, w := range p.workers { + w.mu.RLock() + tel := w.telemetry + loaded := w.models[model] || w.models[plain] + installed := w.installed[model] || w.installed[plain] || loaded + loadedContext := int64(0) + if requestedContext > 0 && loaded { + for _, lm := range w.loadedModels { + id := strings.TrimSpace(lm.Model) + if id == "" { + id = strings.TrimSpace(lm.Name) + } + if id == model || canonicalModel(id) == plain { + loadedContext = lm.ContextLength + break + } + } + } + known := w.installedKnown + modelActive := w.modelActive[plain] + perf := w.performance[plain] + maintenance := w.maintenance + circuit := w.circuitState + openUntil := w.circuitOpenUntil + w.mu.RUnlock() + if requestedContext > 0 && loadedContext > 0 && loadedContext < requestedContext { + loaded = false + installed = true + } + if circuit == "open" && !openUntil.IsZero() && !time.Now().Before(openUntil) { + circuit = "half_open" + } + pl := w.placementDecision(model) + e := RoutingExplanation{Worker: w.cfg.Name, Healthy: w.healthy.Load(), Maintenance: maintenance, CircuitState: circuit, Placement: pl, InventoryKnown: known, Installed: installed, Loaded: loaded, Active: w.active.Load(), MaxConcurrent: w.cfg.MaxConcurrent, ModelActive: modelActive, ModelLimit: w.modelLimit(model), OutputTPS: perf.OutputTPS, GPUUtilizationPercent: tel.GPUUtilizationPct} + total := tel.VRAMTotalBytes + if total <= 0 { + total = w.cfg.VRAMCapacityBytes + } + if total > 0 && tel.VRAMUsedBytes > 0 { + e.VRAMPercent = 100 * float64(tel.VRAMUsedBytes) / float64(total) + } + switch { + case allowed != nil && !allowed[w.cfg.Name]: + e.Reason = "context_window" + case !e.Healthy: + e.Reason = "worker_unhealthy" + case maintenance != "active": + e.Reason = "worker_" + maintenance + case circuit == "open": + e.Reason = "circuit_open" + case !pl.Allowed: + e.Reason = "placement_denied" + case known && !installed: + e.Reason = "model_not_installed" + case e.Active >= int64(max(1, w.cfg.MaxConcurrent)): + e.Reason = "worker_concurrency_full" + case e.ModelActive >= e.ModelLimit: + e.Reason = "model_concurrency_full" + default: + e.Eligible = true + } + if e.Eligible { + components := map[string]float64{} + components["worker_load"] = float64(e.Active) / float64(max(1, w.cfg.MaxConcurrent)) * 100 + if loaded { + components["loaded_bonus"] = -p.routing.LoadedBonus + } else if installed { + components["installed_bonus"] = -p.routing.InstalledBonus + } + if e.ModelLimit > 0 { + components["model_load"] = 25 * float64(e.ModelActive) / float64(e.ModelLimit) + } + if maxTPS > 0 && perf.OutputTPS > 0 { + components["throughput_bonus"] = -p.routing.ThroughputBonus * math.Min(1, perf.OutputTPS/maxTPS) + } + if e.VRAMPercent > 0 { + components["vram_pressure"] = p.routing.VRAMPressurePenalty * math.Min(1, e.VRAMPercent/100) + if !loaded && p.routing.AvoidVRAMPercent > 0 && e.VRAMPercent >= p.routing.AvoidVRAMPercent { + components["vram_avoid"] = 500 + } + } + if tel.GPUUtilizationPct > 0 { + components["gpu_pressure"] = p.routing.GPUUtilizationPenalty * math.Min(1, tel.GPUUtilizationPct/100) + } + for _, v := range components { + e.Score += v + } + e.ScoreComponents = components + } + out = append(out, e) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Eligible != out[j].Eligible { + return out[i].Eligible + } + if out[i].Eligible && out[i].Score != out[j].Score { + return out[i].Score < out[j].Score + } + return out[i].Worker < out[j].Worker + }) + return out +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +func (p *Pool) healthLoop(ctx context.Context, w *state) { + t := time.NewTicker(w.cfg.HealthInterval.Value()) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + p.refresh(ctx, w) + } + } +} +func (p *Pool) refresh(ctx context.Context, w *state) { + cctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(cctx, http.MethodGet, w.url.String()+"/api/ps", nil) + resp, err := p.client.Do(req) + if err != nil { + p.setHealth(w, false, err.Error(), nil, nil) + return + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + p.setHealth(w, false, fmt.Sprintf("HTTP %d", resp.StatusCode), nil, nil) + return + } + b, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + p.setHealth(w, false, err.Error(), nil, nil) + return + } + var doc struct { + Models []LoadedModel `json:"models"` + } + if err := json.Unmarshal(b, &doc); err != nil { + p.setHealth(w, false, err.Error(), nil, nil) + return + } + models := map[string]bool{} + for _, m := range doc.Models { + if m.Name != "" { + models[m.Name] = true + models[strings.TrimSuffix(m.Name, ":latest")] = true + } + if m.Model != "" { + models[m.Model] = true + models[strings.TrimSuffix(m.Model, ":latest")] = true + } + } + p.setHealth(w, true, "", models, doc.Models) + p.refreshInstalled(ctx, w) + p.refreshTelemetry(ctx, w) +} +func (p *Pool) refreshInstalled(ctx context.Context, w *state) { + cctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(cctx, http.MethodGet, w.url.String()+"/api/tags", nil) + resp, err := p.client.Do(req) + if err != nil { + w.mu.Lock() + w.inventoryError = err.Error() + w.mu.Unlock() + return + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + w.mu.Lock() + w.inventoryError = fmt.Sprintf("HTTP %d", resp.StatusCode) + w.mu.Unlock() + return + } + var doc struct { + Models []ModelInfo `json:"models"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&doc); err != nil { + w.mu.Lock() + w.inventoryError = err.Error() + w.mu.Unlock() + return + } + installed := make(map[string]bool, len(doc.Models)*2) + display := make([]string, 0, len(doc.Models)) + seenDisplay := map[string]bool{} + for _, m := range doc.Models { + preferred := strings.TrimSpace(m.Model) + if preferred == "" { + preferred = strings.TrimSpace(m.Name) + } + if preferred != "" && !seenDisplay[preferred] { + seenDisplay[preferred] = true + display = append(display, preferred) + } + for _, id := range []string{m.Model, m.Name} { + id = strings.TrimSpace(id) + if id == "" { + continue + } + installed[id] = true + installed[strings.TrimSuffix(id, ":latest")] = true + } + } + sort.Strings(display) + w.mu.Lock() + w.installed = installed + w.installedKnown = true + w.installedModels = display + w.inventoryError = "" + w.mu.Unlock() +} + +func (p *Pool) refreshTelemetry(ctx context.Context, w *state) { + if !w.cfg.LocalSystemStats && !w.cfg.NVIDIASMI && w.cfg.TelemetryURL == "" { + return + } + t := ResourceTelemetry{UpdatedAt: time.Now().UTC()} + if w.cfg.LocalSystemStats { + cctx, cancel := context.WithTimeout(ctx, 2*time.Second) + m, err := hoststats.ReadMemory(cctx) + cancel() + if err != nil { + t.Error = err.Error() + } else { + t.MemoryTotalBytes = m.TotalBytes + t.MemoryUsedBytes = m.UsedBytes + t.Source = "local-system" + } + } + if w.cfg.NVIDIASMI { + cctx, cancel := context.WithTimeout(ctx, 2*time.Second) + n, err := hoststats.ReadNVIDIA(cctx, w.cfg.NVIDIAGPU) + cancel() + if err != nil { + if t.Error == "" { + t.Error = err.Error() + } else { + t.Error += "; " + err.Error() + } + } else { + t.VRAMUsedBytes = n.MemoryUsedBytes + t.VRAMTotalBytes = n.MemoryTotalBytes + t.GPUUtilizationPct = n.UtilizationPercent + t.GPUTemperatureC = n.TemperatureC + t.GPUPowerWatts = n.PowerWatts + if t.Source == "" { + t.Source = "nvidia-smi" + } else { + t.Source += "+nvidia-smi" + } + } + } + if w.cfg.TelemetryURL != "" { + cctx, cancel := context.WithTimeout(ctx, 2*time.Second) + req, _ := http.NewRequestWithContext(cctx, http.MethodGet, w.cfg.TelemetryURL, nil) + resp, err := p.client.Do(req) + if err != nil { + appendTelemetryError(&t, err.Error()) + } else { + func() { + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + appendTelemetryError(&t, fmt.Sprintf("telemetry HTTP %d", resp.StatusCode)) + return + } + var ext externalTelemetry + if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&ext); err != nil { + appendTelemetryError(&t, err.Error()) + return + } + if err := validateExternalTelemetryTimestamp(time.Now().UTC(), ext.UpdatedAt, telemetryMaxAge(w.cfg.HealthInterval.Value())); err != nil { + appendTelemetryError(&t, err.Error()) + return + } + mergeExternalTelemetry(&t, ext) + }() + } + cancel() + } + w.mu.Lock() + w.telemetry = t + w.mu.Unlock() +} + +func telemetryMaxAge(healthInterval time.Duration) time.Duration { + maxAge := 6 * healthInterval + if maxAge < 30*time.Second { + maxAge = 30 * time.Second + } + return maxAge +} + +func validateExternalTelemetryTimestamp(now time.Time, updatedAt *time.Time, maxAge time.Duration) error { + // updated_at was optional before checkpoint 21. Preserve compatibility with + // existing exporters, while the shipped agent always supplies it. + if updatedAt == nil { + return nil + } + if maxAge <= 0 { + maxAge = 30 * time.Second + } + age := now.Sub(updatedAt.UTC()) + if age < -30*time.Second { + return fmt.Errorf("telemetry timestamp is %.0fs in the future", -age.Seconds()) + } + if age > maxAge { + return fmt.Errorf("telemetry sample is stale: age %s exceeds %s", age.Round(time.Second), maxAge) + } + return nil +} + +func mergeExternalTelemetry(t *ResourceTelemetry, ext externalTelemetry) { + if ext.MemoryTotalBytes != nil { + t.MemoryTotalBytes = *ext.MemoryTotalBytes + } + if ext.MemoryUsedBytes != nil { + t.MemoryUsedBytes = *ext.MemoryUsedBytes + } + if ext.VRAMTotalBytes != nil { + t.VRAMTotalBytes = *ext.VRAMTotalBytes + } + if ext.VRAMUsedBytes != nil { + t.VRAMUsedBytes = *ext.VRAMUsedBytes + } + if ext.GPUUtilizationPct != nil { + t.GPUUtilizationPct = *ext.GPUUtilizationPct + } + if ext.GPUTemperatureC != nil { + t.GPUTemperatureC = *ext.GPUTemperatureC + } + if ext.GPUPowerWatts != nil { + t.GPUPowerWatts = *ext.GPUPowerWatts + } + source := "telemetry-url" + if extSource := strings.TrimSpace(ext.Source); extSource != "" { + source += ":" + extSource + } + if t.Source == "" { + t.Source = source + } else { + t.Source += "+" + source + } + if strings.TrimSpace(ext.Error) != "" { + appendTelemetryError(t, ext.Error) + } +} + +func appendTelemetryError(t *ResourceTelemetry, message string) { + message = strings.TrimSpace(message) + if message == "" { + return + } + if t.Error == "" { + t.Error = message + } else { + t.Error += "; " + message + } +} + +// Observe updates an in-memory EWMA of model throughput for adaptive routing. +// Exact Ollama eval durations are preferred; wall time is only used when an +// OpenAI-compatible response did not expose token evaluation timing. +func (p *Pool) Observe(workerName, model string, promptTokens, completionTokens, promptEvalNS, evalNS int64, service time.Duration) { + w := p.byName[workerName] + model = canonicalModel(model) + if w == nil || model == "" { + return + } + promptTPS, outputTPS := 0.0, 0.0 + if promptTokens > 0 && promptEvalNS > 0 { + promptTPS = float64(promptTokens) / (float64(promptEvalNS) / 1e9) + } + if completionTokens > 0 { + den := time.Duration(evalNS) + if den <= 0 { + den = service + } + if den > 0 { + outputTPS = float64(completionTokens) / den.Seconds() + } + } + if promptTPS <= 0 && outputTPS <= 0 { + return + } + w.mu.Lock() + ps := w.performance[model] + const alpha = 0.25 + if ps.Samples == 0 { + ps.PromptTPS, ps.OutputTPS = promptTPS, outputTPS + } else { + if promptTPS > 0 { + ps.PromptTPS = alpha*promptTPS + (1-alpha)*ps.PromptTPS + } + if outputTPS > 0 { + ps.OutputTPS = alpha*outputTPS + (1-alpha)*ps.OutputTPS + } + } + ps.Samples++ + w.performance[model] = ps + w.mu.Unlock() +} + +func performanceSnapshotLocked(w *state) []ModelPerformance { + out := make([]ModelPerformance, 0, len(w.performance)) + for model, p := range w.performance { + out = append(out, ModelPerformance{Model: model, PromptTPS: p.PromptTPS, OutputTPS: p.OutputTPS, Samples: p.Samples}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Model < out[j].Model }) + return out +} + +func (p *Pool) setHealth(w *state, ok bool, errText string, models map[string]bool, loaded []LoadedModel) { + was := w.healthy.Swap(ok) + w.mu.Lock() + w.lastCheck = time.Now() + w.lastError = errText + if models != nil { + w.models = models + w.loadedModels = append([]LoadedModel(nil), loaded...) + } + w.mu.Unlock() + if was != ok { + p.signal() + } +} +func newID() string { b := make([]byte, 12); _, _ = rand.Read(b); return hex.EncodeToString(b) } + +type ModelDetails struct { + ParentModel string `json:"parent_model,omitempty"` + Format string `json:"format,omitempty"` + Family string `json:"family,omitempty"` + Families []string `json:"families,omitempty"` + ParameterSize string `json:"parameter_size,omitempty"` + QuantizationLevel string `json:"quantization_level,omitempty"` +} + +type ModelInfo struct { + Name string `json:"name"` + Model string `json:"model,omitempty"` + ModifiedAt time.Time `json:"modified_at,omitempty"` + Size int64 `json:"size,omitempty"` + Digest string `json:"digest,omitempty"` + Details ModelDetails `json:"details"` + Loaded bool `json:"loaded"` + Capabilities []string `json:"capabilities,omitempty"` + ContextLength int64 `json:"context_length,omitempty"` + ConfiguredContextLength int64 `json:"configured_context_length,omitempty"` + LoadedContextLength int64 `json:"loaded_context_length,omitempty"` + MetadataError string `json:"metadata_error,omitempty"` +} + +// Metadata returns cached /api/show metadata for a model from a healthy +// worker that has the model installed. The second return value is the worker +// used for discovery. +func (p *Pool) Metadata(ctx context.Context, model string) (ModelMetadata, string, error) { + if p.capCfg.Mode == "off" || strings.TrimSpace(model) == "" { + return ModelMetadata{Model: model}, "", nil + } + for _, w := range p.candidates(model) { + if !w.hasModel(model) { + continue + } + m, err := p.metadataForWorker(ctx, w, model) + return m, w.cfg.Name, err + } + return ModelMetadata{Model: model}, "", fmt.Errorf("model %q is not installed on a healthy worker", model) +} + +func (p *Pool) metadataForWorker(ctx context.Context, w *state, model string) (ModelMetadata, error) { + key := canonicalModel(model) + w.mu.RLock() + entry, ok := w.metadata[key] + w.mu.RUnlock() + if ok { + ttl := p.capCfg.CacheTTL.Value() + if entry.Data.Error != "" && ttl > 15*time.Second { + ttl = 15 * time.Second + } + if ttl > 0 && time.Since(entry.FetchedAt) < ttl { + if entry.Data.Error != "" { + return entry.Data, errors.New(entry.Data.Error) + } + return entry.Data, nil + } + } + cctx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + body, _ := json.Marshal(map[string]any{"model": model, "verbose": false}) + req, _ := http.NewRequestWithContext(cctx, http.MethodPost, w.url.String()+"/api/show", strings.NewReader(string(body))) + req.Header.Set("Content-Type", "application/json") + resp, err := p.client.Do(req) + if err != nil { + m := ModelMetadata{Model: model, Error: err.Error(), UpdatedAt: time.Now().UTC()} + w.mu.Lock() + w.metadata[key] = metadataEntry{Data: m, FetchedAt: time.Now()} + w.mu.Unlock() + return m, err + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<10)) + err := fmt.Errorf("/api/show HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))) + m := ModelMetadata{Model: model, Error: err.Error(), UpdatedAt: time.Now().UTC()} + w.mu.Lock() + w.metadata[key] = metadataEntry{Data: m, FetchedAt: time.Now()} + w.mu.Unlock() + return m, err + } + var doc struct { + Capabilities []string `json:"capabilities"` + Details ModelDetails `json:"details"` + ModelInfo map[string]any `json:"model_info"` + Parameters json.RawMessage `json:"parameters"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&doc); err != nil { + return ModelMetadata{Model: model}, err + } + caps := make([]string, 0, len(doc.Capabilities)) + seen := map[string]bool{} + for _, c := range doc.Capabilities { + c = strings.ToLower(strings.TrimSpace(c)) + if c != "" && !seen[c] { + seen[c] = true + caps = append(caps, c) + } + } + sort.Strings(caps) + var contextLength int64 + for k, v := range doc.ModelInfo { + if k != "context_length" && !strings.HasSuffix(k, ".context_length") { + continue + } + var n int64 + switch x := v.(type) { + case float64: + n = int64(x) + case int64: + n = x + case json.Number: + n, _ = x.Int64() + } + if n > contextLength { + contextLength = n + } + } + m := ModelMetadata{Model: model, Capabilities: caps, ContextLength: contextLength, ConfiguredContextLength: parseNumCtxParameter(doc.Parameters), Details: doc.Details, UpdatedAt: time.Now().UTC()} + w.mu.Lock() + w.metadata[key] = metadataEntry{Data: m, FetchedAt: time.Now()} + w.mu.Unlock() + return m, nil +} + +func parseNumCtxParameter(raw json.RawMessage) int64 { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { + return 0 + } + var text string + if json.Unmarshal(raw, &text) == nil { + for _, line := range strings.Split(text, "\n") { + fields := strings.Fields(strings.TrimSpace(line)) + if len(fields) < 2 || !strings.EqualFold(fields[0], "num_ctx") { + continue + } + n, err := strconv.ParseInt(strings.Trim(fields[1], "\"'"), 10, 64) + if err == nil && n > 0 { + return n + } + } + return 0 + } + var obj map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if dec.Decode(&obj) != nil { + return 0 + } + v, ok := obj["num_ctx"] + if !ok { + return 0 + } + switch x := v.(type) { + case json.Number: + n, _ := x.Int64() + if n > 0 { + return n + } + case float64: + if x > 0 { + return int64(x) + } + case string: + n, _ := strconv.ParseInt(strings.TrimSpace(x), 10, 64) + if n > 0 { + return n + } + } + return 0 +} + +func HasCapability(m ModelMetadata, capability string) bool { + capability = strings.ToLower(strings.TrimSpace(capability)) + for _, c := range m.Capabilities { + if strings.EqualFold(c, capability) { + return true + } + } + return false +} + +type Inventory struct { + Worker string `json:"worker"` + URL string `json:"url"` + Models []ModelInfo `json:"models"` + Error string `json:"error,omitempty"` +} + +func (p *Pool) URLFor(name string) (*url.URL, bool) { + w := p.byName[name] + if w == nil { + return nil, false + } + u := *w.url + return &u, true +} + +// TagModel is the native Ollama /api/tags model shape used for client-facing +// discovery. It intentionally excludes gateway-only fields such as Loaded. +type TagModel struct { + Name string `json:"name"` + Model string `json:"model"` + ModifiedAt time.Time `json:"modified_at,omitempty"` + Size int64 `json:"size,omitempty"` + Digest string `json:"digest,omitempty"` + Details ModelDetails `json:"details"` +} + +// Tags queries every healthy worker, merges installed models and normalizes the +// modern Ollama discovery contract so clients can rely on both name and model. +// A failed worker does not hide models served by the remaining healthy workers. +func (p *Pool) Tags(ctx context.Context) ([]TagModel, []string) { + inventories := p.Inventories(ctx) + byID := make(map[string]TagModel) + errs := make([]string, 0) + for _, inv := range inventories { + if inv.Error != "" { + errs = append(errs, inv.Worker+": "+inv.Error) + continue + } + for _, m := range inv.Models { + name := strings.TrimSpace(m.Name) + model := strings.TrimSpace(m.Model) + if model == "" { + model = name + } + if name == "" { + name = model + } + if model == "" { + continue + } + if decision, ok := p.PlacementDecision(inv.Worker, model); ok && !decision.Allowed { + continue + } + cur, exists := byID[model] + if !exists || m.ModifiedAt.After(cur.ModifiedAt) { + byID[model] = TagModel{Name: name, Model: model, ModifiedAt: m.ModifiedAt, Size: m.Size, Digest: m.Digest, Details: m.Details} + } + } + } + out := make([]TagModel, 0, len(byID)) + for _, m := range byID { + out = append(out, m) + } + sort.Slice(out, func(i, j int) bool { return out[i].Model < out[j].Model }) + sort.Strings(errs) + return out, errs +} + +// Loaded returns a de-duplicated native Ollama /api/ps view across workers. +func (p *Pool) Loaded() []LoadedModel { + byID := make(map[string]LoadedModel) + for _, snap := range p.Snapshots() { + if !snap.Healthy { + continue + } + for _, m := range snap.LoadedModels { + id := strings.TrimSpace(m.Model) + if id == "" { + id = strings.TrimSpace(m.Name) + } + if id == "" { + continue + } + if m.Model == "" { + m.Model = id + } + if m.Name == "" { + m.Name = id + } + if cur, ok := byID[id]; !ok || m.SizeVRAM > cur.SizeVRAM { + byID[id] = m + } + } + } + out := make([]LoadedModel, 0, len(byID)) + for _, m := range byID { + out = append(out, m) + } + sort.Slice(out, func(i, j int) bool { return out[i].Model < out[j].Model }) + return out +} + +func (p *Pool) Inventories(ctx context.Context) []Inventory { + out := make([]Inventory, len(p.workers)) + var wg sync.WaitGroup + for i, w := range p.workers { + wg.Add(1) + go func(i int, w *state) { + defer wg.Done() + inv := Inventory{Worker: w.cfg.Name, URL: w.url.String()} + cctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(cctx, http.MethodGet, w.url.String()+"/api/tags", nil) + resp, err := p.client.Do(req) + if err != nil { + inv.Error = err.Error() + out[i] = inv + return + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + inv.Error = fmt.Sprintf("HTTP %d", resp.StatusCode) + out[i] = inv + return + } + var doc struct { + Models []ModelInfo `json:"models"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&doc); err != nil { + inv.Error = err.Error() + out[i] = inv + return + } + installed := make(map[string]bool, len(doc.Models)*2) + for _, m := range doc.Models { + for _, id := range []string{m.Model, m.Name} { + id = strings.TrimSpace(id) + if id != "" { + installed[id] = true + installed[strings.TrimSuffix(id, ":latest")] = true + } + } + } + w.mu.Lock() + w.installed = installed + w.installedKnown = true + display := make([]string, 0, len(doc.Models)) + seenDisplay := map[string]bool{} + for _, m := range doc.Models { + name := strings.TrimSpace(m.Model) + if name == "" { + name = strings.TrimSpace(m.Name) + } + if name != "" && !seenDisplay[name] { + seenDisplay[name] = true + display = append(display, name) + } + } + sort.Strings(display) + w.installedModels = display + w.inventoryError = "" + w.mu.Unlock() + w.mu.RLock() + loaded := make(map[string]bool, len(w.models)) + loadedCtx := make(map[string]int64, len(w.loadedModels)*2) + for k, v := range w.models { + loaded[k] = v + } + for _, lm := range w.loadedModels { + id := strings.TrimSpace(lm.Model) + if id == "" { + id = strings.TrimSpace(lm.Name) + } + if id != "" && lm.ContextLength > 0 { + loadedCtx[id] = lm.ContextLength + loadedCtx[canonicalModel(id)] = lm.ContextLength + } + } + w.mu.RUnlock() + for j := range doc.Models { + name := doc.Models[j].Name + if name == "" { + name = doc.Models[j].Model + } + doc.Models[j].Loaded = loaded[name] || loaded[strings.TrimSuffix(name, ":latest")] + doc.Models[j].LoadedContextLength = loadedCtx[name] + if doc.Models[j].LoadedContextLength == 0 { + doc.Models[j].LoadedContextLength = loadedCtx[canonicalModel(name)] + } + } + if p.capCfg.Mode != "off" { + sem := make(chan struct{}, 4) + var mwg sync.WaitGroup + for j := range doc.Models { + j := j + model := doc.Models[j].Model + if model == "" { + model = doc.Models[j].Name + } + mwg.Add(1) + go func() { + defer mwg.Done() + select { + case sem <- struct{}{}: + case <-ctx.Done(): + return + } + defer func() { <-sem }() + m, err := p.metadataForWorker(ctx, w, model) + if err != nil { + doc.Models[j].MetadataError = err.Error() + return + } + doc.Models[j].Capabilities = append([]string(nil), m.Capabilities...) + doc.Models[j].ContextLength = m.ContextLength + doc.Models[j].ConfiguredContextLength = m.ConfiguredContextLength + }() + } + mwg.Wait() + } + sort.Slice(doc.Models, func(a, b int) bool { return doc.Models[a].Name < doc.Models[b].Name }) + inv.Models = doc.Models + out[i] = inv + }(i, w) + } + wg.Wait() + return out +} diff --git a/internal/worker/pool_test.go b/internal/worker/pool_test.go new file mode 100644 index 0000000..c4d600e --- /dev/null +++ b/internal/worker/pool_test.go @@ -0,0 +1,387 @@ +package worker + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/example/ollama-fair-gateway/internal/config" +) + +func TestMetadataCapabilitiesAndContextCached(t *testing.T) { + var showCalls atomic.Int64 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "qwen:latest", "model": "qwen:latest"}}}) + case "/api/show": + showCalls.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{ + "capabilities": []string{"completion", "tools", "thinking"}, + "details": map[string]any{"family": "qwen"}, + "model_info": map[string]any{"qwen.context_length": 32768}, + }) + default: + http.NotFound(w, r) + } + })) + defer backend.Close() + + p := New([]config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 2, HealthInterval: config.Duration(time.Hour)}}, "w") + p.SetModelCapabilitiesConfig(config.ModelCapabilitiesConfig{Mode: "enforce", CacheTTL: config.Duration(time.Hour), ContextGuard: "reject"}) + p.Start(context.Background()) + m, worker, err := p.Metadata(context.Background(), "qwen:latest") + if err != nil { + t.Fatal(err) + } + if worker != "w" || m.ContextLength != 32768 || !HasCapability(m, "tools") { + t.Fatalf("unexpected metadata: %#v worker=%s", m, worker) + } + if _, _, err := p.Metadata(context.Background(), "qwen:latest"); err != nil { + t.Fatal(err) + } + if showCalls.Load() != 1 { + t.Fatalf("expected one cached /api/show call, got %d", showCalls.Load()) + } +} + +func TestPerModelConcurrency(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "large:latest", "model": "large:latest"}}}) + default: + http.NotFound(w, r) + } + })) + defer backend.Close() + p := New([]config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 4, ModelConcurrency: map[string]int{"large:*": 1}, HealthInterval: config.Duration(time.Hour)}}, "w") + p.Start(context.Background()) + l1, err := p.Acquire(context.Background(), "large:latest") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond) + defer cancel() + if _, err := p.Acquire(ctx, "large:latest"); err == nil { + t.Fatal("expected second large request to wait for model slot") + } + l1.Release() + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + l2, err := p.Acquire(ctx2, "large:latest") + if err != nil { + t.Fatal(err) + } + l2.Release() +} + +func TestObserveLearnsThroughput(t *testing.T) { + p := New([]config.WorkerConfig{{Name: "w", URL: "http://127.0.0.1:11434", MaxConcurrent: 1}}, "w") + p.Observe("w", "m:latest", 100, 50, int64(time.Second), int64(2*time.Second), 3*time.Second) + s := p.Snapshots()[0] + if len(s.Performance) != 1 { + t.Fatalf("performance missing: %#v", s.Performance) + } + if s.Performance[0].PromptTPS < 99 || s.Performance[0].OutputTPS < 24 { + t.Fatalf("unexpected performance: %#v", s.Performance[0]) + } +} + +func TestAdaptiveRoutingPrefersFasterEquivalentWorker(t *testing.T) { + p := New([]config.WorkerConfig{ + {Name: "slow", URL: "http://127.0.0.1:11434", MaxConcurrent: 2}, + {Name: "fast", URL: "http://127.0.0.1:11435", MaxConcurrent: 2}, + }, "slow") + p.SetRoutingConfig(config.RoutingConfig{LoadedBonus: 1, InstalledBonus: 1, ThroughputBonus: 80, VRAMPressurePenalty: 1, GPUUtilizationPenalty: 1, AvoidVRAMPercent: 99}) + for _, w := range p.workers { + w.mu.Lock() + w.installed["m"] = true + w.models["m"] = true + w.mu.Unlock() + } + p.Observe("slow", "m", 100, 100, int64(time.Second), int64(4*time.Second), 4*time.Second) // 25 tok/s + p.Observe("fast", "m", 100, 100, int64(time.Second), int64(time.Second), time.Second) // 100 tok/s + c := p.candidates("m") + if len(c) != 2 || c[0].cfg.Name != "fast" { + t.Fatalf("expected faster worker first, got %#v", []string{c[0].cfg.Name, c[1].cfg.Name}) + } +} + +func TestModelPlacementFiltersWorkersBeforeAdaptiveRouting(t *testing.T) { + p := New([]config.WorkerConfig{ + {Name: "node1", URL: "http://127.0.0.1:11434", MaxConcurrent: 2, ModelPlacement: config.ModelPlacementRule{Mode: "whitelist", AllowedModels: []string{"model-a", "model-b"}}}, + {Name: "node2", URL: "http://127.0.0.1:11435", MaxConcurrent: 2, ModelPlacement: config.ModelPlacementRule{Mode: "whitelist", AllowedModels: []string{"model-b"}}}, + }, "node1") + for _, w := range p.workers { + w.mu.Lock() + w.installedKnown = true + w.installed["model-a"] = true + w.installed["model-b"] = true + w.mu.Unlock() + } + c := p.candidates("model-a") + if len(c) != 1 || c[0].cfg.Name != "node1" { + t.Fatalf("model-a candidates=%v", workerNames(c)) + } + c = p.candidates("model-b") + if len(c) != 2 { + t.Fatalf("model-b candidates=%v", workerNames(c)) + } + if err := p.SetPlacement("node1", config.ModelPlacementRule{Mode: "whitelist", AllowedModels: []string{"model-a"}}, true); err != nil { + t.Fatal(err) + } + c = p.candidates("model-b") + if len(c) != 1 || c[0].cfg.Name != "node2" { + t.Fatalf("model-b after override candidates=%v", workerNames(c)) + } +} + +func TestPlacementSpecificityAllowsExactExceptionToPrefixDeny(t *testing.T) { + r := config.ModelPlacementRule{Mode: "allow_all", DeniedModels: []string{"gemma4:*"}, AllowedModels: []string{"gemma4:latest"}} + if d := evaluatePlacement(r, "gemma4:latest"); !d.Allowed || d.Pattern != "gemma4:latest" || !d.ExactOverride { + t.Fatalf("latest decision=%#v", d) + } + if d := evaluatePlacement(r, "gemma4:e4b"); d.Allowed || d.Pattern != "gemma4:*" { + t.Fatalf("e4b decision=%#v", d) + } +} + +func TestPlacementKnownInventoryDoesNotRouteToWorkerWithoutModel(t *testing.T) { + p := New([]config.WorkerConfig{ + {Name: "a", URL: "http://127.0.0.1:11434", MaxConcurrent: 1}, + {Name: "b", URL: "http://127.0.0.1:11435", MaxConcurrent: 1}, + }, "a") + for _, w := range p.workers { + w.mu.Lock() + w.installedKnown = true + w.mu.Unlock() + } + p.byName["a"].mu.Lock() + p.byName["a"].installed["m"] = true + p.byName["a"].mu.Unlock() + c := p.candidates("m") + if len(c) != 1 || c[0].cfg.Name != "a" { + t.Fatalf("candidates=%v", workerNames(c)) + } + if got := p.candidates("missing"); len(got) != 0 { + t.Fatalf("missing model candidates=%v", workerNames(got)) + } +} + +func workerNames(in []*state) []string { + out := make([]string, 0, len(in)) + for _, w := range in { + out = append(out, w.cfg.Name) + } + return out +} + +func TestTagsExcludeModelsOnlyAvailableOnPlacementBlockedWorkers(t *testing.T) { + backend := func(model string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": model, "model": model}}}) + case "/api/show": + _ = json.NewEncoder(w).Encode(map[string]any{"capabilities": []string{"completion"}}) + default: + http.NotFound(w, r) + } + })) + } + a := backend("model-a:latest") + defer a.Close() + b := backend("model-b:latest") + defer b.Close() + p := New([]config.WorkerConfig{ + {Name: "a", URL: a.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour), ModelPlacement: config.ModelPlacementRule{Mode: "allow_all"}}, + {Name: "b", URL: b.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour), ModelPlacement: config.ModelPlacementRule{Mode: "whitelist", AllowedModels: []string{"model-a:*"}}}, + }, "a") + p.Start(context.Background()) + tags, errs := p.Tags(context.Background()) + if len(errs) != 0 { + t.Fatalf("errs=%v", errs) + } + if len(tags) != 1 || tags[0].Model != "model-a:latest" { + t.Fatalf("tags=%#v", tags) + } +} + +func TestMaintenanceAndCircuitBreakerExcludeWorker(t *testing.T) { + p := New([]config.WorkerConfig{{Name: "w", URL: "http://127.0.0.1:11434", MaxConcurrent: 1}}, "w") + p.SetReliabilityConfig(config.ReliabilityConfig{Enabled: true, FailureThreshold: 1, OpenDuration: config.Duration(time.Hour), RetryAttempts: 2}) + w := p.byName["w"] + w.mu.Lock() + w.installedKnown = true + w.installed["m"] = true + w.mu.Unlock() + if !p.CanRoute("m") { + t.Fatal("worker should initially route") + } + if err := p.SetMaintenance("w", "draining"); err != nil { + t.Fatal(err) + } + if p.CanRoute("m") { + t.Fatal("draining worker must not accept new work") + } + if err := p.SetMaintenance("w", "active"); err != nil { + t.Fatal(err) + } + p.ReportResult("w", true, "boom") + if p.CanRoute("m") { + t.Fatal("open circuit must exclude worker") + } + snap := p.Snapshots()[0] + if snap.CircuitState != "open" || snap.CircuitFailures != 1 { + t.Fatalf("snapshot=%#v", snap) + } + if err := p.CircuitReset("w"); err != nil { + t.Fatal(err) + } + if !p.CanRoute("m") { + t.Fatal("reset circuit should route") + } +} + +func TestModelMaintenanceBlocksInferenceAcquire(t *testing.T) { + p := New([]config.WorkerConfig{{Name: "w", URL: "http://127.0.0.1:11434", MaxConcurrent: 1}}, "w") + release, err := p.BeginModelMaintenance("w", "qwen3:8b") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond) + defer cancel() + if lease, err := p.Acquire(ctx, "qwen3:8b"); err == nil { + lease.Release() + t.Fatal("inference acquired worker while model maintenance was active") + } + release() + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + lease, err := p.Acquire(ctx2, "qwen3:8b") + if err != nil { + t.Fatalf("acquire after maintenance release: %v", err) + } + lease.Release() +} + +func TestContextWindowsPreferLoadedThenModelfileContext(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "qwen:latest", "model": "qwen:latest", "context_length": 8192}}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "qwen:latest", "model": "qwen:latest"}}}) + case "/api/show": + _ = json.NewEncoder(w).Encode(map[string]any{ + "capabilities": []string{"completion"}, + "model_info": map[string]any{"qwen.context_length": 131072}, + "parameters": "num_ctx 16384\ntemperature 0.7", + }) + default: + http.NotFound(w, r) + } + })) + defer backend.Close() + p := New([]config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour), ContextLimits: map[string]int64{"qwen:*": 32768}}}, "w") + p.SetModelCapabilitiesConfig(config.ModelCapabilitiesConfig{Mode: "enforce", CacheTTL: config.Duration(time.Hour), ContextGuard: "reject", Context: config.ContextPolicyConfig{DefaultWorkerTokens: 4096, MaxRequestedTokens: 32768}}) + p.Start(context.Background()) + windows := p.ContextWindows(context.Background(), "qwen:latest") + if len(windows) != 1 { + t.Fatalf("windows=%#v", windows) + } + x := windows[0] + if x.ModelMaxTokens != 131072 || x.ConfiguredTokens != 16384 || x.LoadedTokens != 8192 || x.EffectiveTokens != 8192 || x.EffectiveSource != "loaded" || x.WorkerLimitTokens != 32768 { + t.Fatalf("unexpected loaded context window: %#v", x) + } + inv := p.Inventories(context.Background()) + if len(inv) != 1 || len(inv[0].Models) != 1 || inv[0].Models[0].ContextLength != 131072 || inv[0].Models[0].ConfiguredContextLength != 16384 || inv[0].Models[0].LoadedContextLength != 8192 { + t.Fatalf("inventory context fields missing: %#v", inv) + } + p.byName["w"].mu.Lock() + p.byName["w"].loadedModels = nil + p.byName["w"].models = map[string]bool{} + p.byName["w"].mu.Unlock() + windows = p.ContextWindows(context.Background(), "qwen:latest") + if len(windows) != 1 || windows[0].EffectiveTokens != 16384 || windows[0].EffectiveSource != "modelfile" { + t.Fatalf("unexpected unloaded context window: %#v", windows) + } +} + +func TestAcquireAllowedContextDoesNotPreferUndersizedLoadedContext(t *testing.T) { + p := New([]config.WorkerConfig{ + {Name: "smallctx", URL: "http://127.0.0.1:11434", MaxConcurrent: 1}, + {Name: "largectx", URL: "http://127.0.0.1:11435", MaxConcurrent: 1}, + }, "smallctx") + p.SetRoutingConfig(config.RoutingConfig{LoadedBonus: 100, InstalledBonus: 10, ThroughputBonus: 1, VRAMPressurePenalty: 1, GPUUtilizationPenalty: 1, AvoidVRAMPercent: 99}) + for _, w := range p.workers { + w.mu.Lock() + w.installedKnown = true + w.installed["m"] = true + w.models["m"] = true + ctx := int64(4096) + if w.cfg.Name == "largectx" { + ctx = 16384 + } + w.loadedModels = []LoadedModel{{Name: "m", Model: "m", ContextLength: ctx}} + w.mu.Unlock() + } + lease, err := p.AcquireAllowed(context.Background(), "m", map[string]bool{"smallctx": true, "largectx": true}, 8192) + if err != nil { + t.Fatal(err) + } + defer lease.Release() + if lease.Name() != "largectx" { + t.Fatalf("selected %s, want largectx", lease.Name()) + } +} + +func TestContextWindowsUsesPerWorkerDefaultContext(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/ps": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{}}) + case "/api/tags": + _ = json.NewEncoder(w).Encode(map[string]any{"models": []any{map[string]any{"name": "m", "model": "m"}}}) + case "/api/show": + _ = json.NewEncoder(w).Encode(map[string]any{"capabilities": []string{"completion"}, "model_info": map[string]any{"m.context_length": 131072}}) + default: + http.NotFound(w, r) + } + })) + defer backend.Close() + p := New([]config.WorkerConfig{{Name: "w", URL: backend.URL, MaxConcurrent: 1, HealthInterval: config.Duration(time.Hour), DefaultContextTokens: 12288}}, "w") + p.SetModelCapabilitiesConfig(config.ModelCapabilitiesConfig{Mode: "enforce", CacheTTL: config.Duration(time.Hour), ContextGuard: "reject", Context: config.ContextPolicyConfig{DefaultWorkerTokens: 4096, MaxRequestedTokens: 32768}}) + p.Start(context.Background()) + windows := p.ContextWindows(context.Background(), "m") + if len(windows) != 1 || windows[0].EffectiveTokens != 12288 || windows[0].EffectiveSource != "worker_default" { + t.Fatalf("windows=%#v", windows) + } +} + +func TestParseNumCtxParameterStringAndObject(t *testing.T) { + for _, tc := range []struct { + raw string + want int64 + }{ + {`"num_ctx 8192\ntemperature 0.7"`, 8192}, + {`{"num_ctx":16384}`, 16384}, + {`{"num_ctx":"32768"}`, 32768}, + {`"temperature 0.7"`, 0}, + } { + if got := parseNumCtxParameter(json.RawMessage(tc.raw)); got != tc.want { + t.Fatalf("raw=%s got=%d want=%d", tc.raw, got, tc.want) + } + } +} diff --git a/internal/worker/telemetry_test.go b/internal/worker/telemetry_test.go new file mode 100644 index 0000000..d203e13 --- /dev/null +++ b/internal/worker/telemetry_test.go @@ -0,0 +1,64 @@ +package worker + +import ( + "testing" + "time" +) + +func int64p(v int64) *int64 { return &v } +func float64p(v float64) *float64 { return &v } + +func TestMergeExternalTelemetryPreservesOmittedFieldsAndAcceptsZero(t *testing.T) { + base := ResourceTelemetry{MemoryTotalBytes: 100, MemoryUsedBytes: 50, VRAMTotalBytes: 200, VRAMUsedBytes: 80, GPUUtilizationPct: 75, GPUTemperatureC: 60, GPUPowerWatts: 120, Source: "local-system", Error: "local warning"} + mergeExternalTelemetry(&base, externalTelemetry{ + VRAMUsedBytes: int64p(0), + GPUUtilizationPct: float64p(0), + Source: "host-memory+amdgpu-sysfs", + Error: "agent warning", + }) + if base.MemoryTotalBytes != 100 || base.MemoryUsedBytes != 50 || base.VRAMTotalBytes != 200 { + t.Fatalf("omitted fields were overwritten: %+v", base) + } + if base.VRAMUsedBytes != 0 || base.GPUUtilizationPct != 0 { + t.Fatalf("explicit zero fields were not applied: %+v", base) + } + if base.GPUTemperatureC != 60 || base.GPUPowerWatts != 120 { + t.Fatalf("omitted GPU fields were overwritten: %+v", base) + } + if base.Source != "local-system+telemetry-url:host-memory+amdgpu-sysfs" { + t.Fatalf("unexpected source %q", base.Source) + } + if base.Error != "local warning; agent warning" { + t.Fatalf("unexpected error %q", base.Error) + } +} + +func timep(v time.Time) *time.Time { return &v } + +func TestValidateExternalTelemetryTimestamp(t *testing.T) { + now := time.Date(2026, 9, 8, 18, 0, 0, 0, time.UTC) + if err := validateExternalTelemetryTimestamp(now, nil, 30*time.Second); err != nil { + t.Fatalf("missing legacy timestamp should remain compatible: %v", err) + } + fresh := now.Add(-10 * time.Second) + if err := validateExternalTelemetryTimestamp(now, timep(fresh), 30*time.Second); err != nil { + t.Fatalf("fresh timestamp rejected: %v", err) + } + stale := now.Add(-31 * time.Second) + if err := validateExternalTelemetryTimestamp(now, timep(stale), 30*time.Second); err == nil { + t.Fatal("expected stale timestamp rejection") + } + future := now.Add(31 * time.Second) + if err := validateExternalTelemetryTimestamp(now, timep(future), 30*time.Second); err == nil { + t.Fatal("expected future timestamp rejection") + } +} + +func TestTelemetryMaxAge(t *testing.T) { + if got := telemetryMaxAge(5 * time.Second); got != 30*time.Second { + t.Fatalf("got %s", got) + } + if got := telemetryMaxAge(20 * time.Second); got != 2*time.Minute { + t.Fatalf("got %s", got) + } +} diff --git a/scripts/ha-readiness.sh b/scripts/ha-readiness.sh new file mode 100644 index 0000000..0709180 --- /dev/null +++ b/scripts/ha-readiness.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env sh +set -eu + +BASE_URL=${BASE_URL:-http://127.0.0.1:8080} +MODEL=${MODEL:-qwen3:8b} +REQUESTS=${REQUESTS:-500} +WARMUP=${WARMUP:-20} +CONCURRENCIES=${CONCURRENCIES:-"1 4 16 32 64"} +OUT_DIR=${OUT_DIR:-"ha-readiness-$(date +%Y%m%d-%H%M%S)"} +TIMEOUT=${TIMEOUT:-2m} +STREAM=${STREAM:-false} +SERVICE_CLASS=${SERVICE_CLASS:-} +CAPTURE_METRICS=${CAPTURE_METRICS:-true} +METRICS_URL=${METRICS_URL:-"${BASE_URL%/}/metrics"} +GATEWAY_PID=${GATEWAY_PID:-} +CAPTURE_RESOURCES=${CAPTURE_RESOURCES:-auto} +CAPTURE_SUSTAINED_RESOURCES=${CAPTURE_SUSTAINED_RESOURCES:-auto} +RESOURCE_SAMPLE_INTERVAL=${RESOURCE_SAMPLE_INTERVAL:-250ms} +RESOURCE_SAMPLE_MAX_DURATION=${RESOURCE_SAMPLE_MAX_DURATION:-15m} + +if [ "$CAPTURE_RESOURCES" = "auto" ]; then + if [ -n "$GATEWAY_PID" ]; then + CAPTURE_RESOURCES=true + else + CAPTURE_RESOURCES=false + fi +fi +if [ "$CAPTURE_RESOURCES" = "true" ] && [ -z "$GATEWAY_PID" ]; then + echo "GATEWAY_PID is required when CAPTURE_RESOURCES=true" >&2 + exit 2 +fi + +if [ "$CAPTURE_SUSTAINED_RESOURCES" = "auto" ]; then + if [ -n "$GATEWAY_PID" ]; then + CAPTURE_SUSTAINED_RESOURCES=true + else + CAPTURE_SUSTAINED_RESOURCES=false + fi +fi +if [ "$CAPTURE_SUSTAINED_RESOURCES" = "true" ] && [ -z "$GATEWAY_PID" ]; then + echo "GATEWAY_PID is required when CAPTURE_SUSTAINED_RESOURCES=true" >&2 + exit 2 +fi + +mkdir -p "$OUT_DIR" +SAMPLER_BIN="" +SAMPLER_BG_PID="" +SAMPLER_STOP_FILE="" + +cleanup() { + if [ -n "$SAMPLER_STOP_FILE" ]; then + : > "$SAMPLER_STOP_FILE" 2>/dev/null || true + fi + if [ -n "$SAMPLER_BG_PID" ]; then + wait "$SAMPLER_BG_PID" 2>/dev/null || true + fi + if [ -n "$SAMPLER_BIN" ]; then + rm -f "$SAMPLER_BIN" + fi + if [ -n "$SAMPLER_STOP_FILE" ]; then + rm -f "$SAMPLER_STOP_FILE" + fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +if [ "$CAPTURE_SUSTAINED_RESOURCES" = "true" ]; then + SAMPLER_BIN="$OUT_DIR/.ha-sampler" + CGO_ENABLED=0 go build -trimpath -o "$SAMPLER_BIN" ./cmd/ha-sampler +fi + +echo "HA readiness sweep" +echo "base_url=$BASE_URL model=$MODEL requests=$REQUESTS warmup=$WARMUP stream=$STREAM" +echo "concurrency=$CONCURRENCIES" +echo "capture_metrics=$CAPTURE_METRICS metrics_url=$METRICS_URL" +echo "capture_resources=$CAPTURE_RESOURCES gateway_pid=${GATEWAY_PID:-unset}" +echo "capture_sustained_resources=$CAPTURE_SUSTAINED_RESOURCES sample_interval=$RESOURCE_SAMPLE_INTERVAL" +echo "output=$OUT_DIR" + +capture_metrics() { + dst=$1 + if [ "$CAPTURE_METRICS" != "true" ]; then + return 0 + fi + if ! command -v curl >/dev/null 2>&1; then + echo "curl is required when CAPTURE_METRICS=true" >&2 + exit 2 + fi + if [ -n "${GATEWAY_BENCH_API_KEY:-}" ]; then + curl -fsS --max-time 30 -H "Authorization: Bearer ${GATEWAY_BENCH_API_KEY}" "$METRICS_URL" > "$dst" + else + curl -fsS --max-time 30 "$METRICS_URL" > "$dst" + fi +} + +capture_resources() { + dst=$1 + if [ "$CAPTURE_RESOURCES" != "true" ]; then + return 0 + fi + go run ./cmd/ha-snapshot -pid "$GATEWAY_PID" -json-out "$dst" +} + +start_resource_sampler() { + c=$1 + if [ "$CAPTURE_SUSTAINED_RESOURCES" != "true" ]; then + return 0 + fi + SAMPLER_STOP_FILE="$OUT_DIR/.resource-sampler-stop-c$c" + rm -f "$SAMPLER_STOP_FILE" + "$SAMPLER_BIN" \ + -pid "$GATEWAY_PID" \ + -interval "$RESOURCE_SAMPLE_INTERVAL" \ + -max-duration "$RESOURCE_SAMPLE_MAX_DURATION" \ + -stop-file "$SAMPLER_STOP_FILE" \ + -json-out "$OUT_DIR/resources-samples-c$c.json" \ + >"$OUT_DIR/resource-sampler-c$c.log" 2>&1 & + SAMPLER_BG_PID=$! + # Give the already-built sampler enough time to capture its initial baseline. + sleep 0.1 + if ! kill -0 "$SAMPLER_BG_PID" 2>/dev/null; then + wait "$SAMPLER_BG_PID" || true + echo "resource sampler failed to start for concurrency $c" >&2 + cat "$OUT_DIR/resource-sampler-c$c.log" >&2 || true + exit 1 + fi +} + +stop_resource_sampler() { + c=$1 + if [ "$CAPTURE_SUSTAINED_RESOURCES" != "true" ]; then + return 0 + fi + : > "$SAMPLER_STOP_FILE" + sampler_rc=0 + wait "$SAMPLER_BG_PID" || sampler_rc=$? + rm -f "$SAMPLER_STOP_FILE" + SAMPLER_STOP_FILE="" + SAMPLER_BG_PID="" + if [ "$sampler_rc" -ne 0 ]; then + echo "resource sampler failed for concurrency $c (exit $sampler_rc)" >&2 + cat "$OUT_DIR/resource-sampler-c$c.log" >&2 || true + exit "$sampler_rc" + fi +} + +for c in $CONCURRENCIES; do + echo "== concurrency $c ==" + capture_metrics "$OUT_DIR/metrics-before-c$c.prom" + capture_resources "$OUT_DIR/resources-before-c$c.json" + start_resource_sampler "$c" + + set -- go run ./cmd/bench \ + -base-url "$BASE_URL" \ + -model "$MODEL" \ + -concurrency "$c" \ + -requests "$REQUESTS" \ + -warmup "$WARMUP" \ + -timeout "$TIMEOUT" \ + -stream="$STREAM" \ + -json-out "$OUT_DIR/concurrency-$c.json" + if [ -n "$SERVICE_CLASS" ]; then + set -- "$@" -service-class "$SERVICE_CLASS" + fi + bench_rc=0 + "$@" || bench_rc=$? + stop_resource_sampler "$c" + if [ "$bench_rc" -ne 0 ]; then + echo "benchmark failed for concurrency $c (exit $bench_rc)" >&2 + exit "$bench_rc" + fi + + capture_resources "$OUT_DIR/resources-after-c$c.json" + capture_metrics "$OUT_DIR/metrics-after-c$c.prom" +done + +if [ "$CAPTURE_METRICS" = "true" ]; then + go run ./cmd/ha-report \ + -input "$OUT_DIR" \ + -json-out "$OUT_DIR/report.json" \ + -markdown-out "$OUT_DIR/report.md" +else + echo "metrics capture disabled; no reconciled HA evidence report generated" +fi + +echo "completed: $OUT_DIR" diff --git a/scripts/ollama-env.sh b/scripts/ollama-env.sh new file mode 100644 index 0000000..d67419d --- /dev/null +++ b/scripts/ollama-env.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Starting point for the 256 GB M5 Ultra. Benchmark 1/2/4 parallel requests +# with your actual model and context length before raising these values. +export OLLAMA_NUM_PARALLEL="${OLLAMA_NUM_PARALLEL:-4}" +export OLLAMA_MAX_LOADED_MODELS="${OLLAMA_MAX_LOADED_MODELS:-2}" +export OLLAMA_MAX_QUEUE="${OLLAMA_MAX_QUEUE:-64}" +export OLLAMA_CONTEXT_LENGTH="${OLLAMA_CONTEXT_LENGTH:-32768}" +export OLLAMA_KEEP_ALIVE="${OLLAMA_KEEP_ALIVE:-30m}" +exec ollama serve diff --git a/scripts/production-preflight.sh b/scripts/production-preflight.sh new file mode 100644 index 0000000..0d01cd9 --- /dev/null +++ b/scripts/production-preflight.sh @@ -0,0 +1,54 @@ +#!/bin/sh +set -eu + +CONFIG=${GATEWAY_CONFIG:-./gateway-config.json} +COMPOSE_FILE=${COMPOSE_FILE:-docker-compose.production.yml} +PUBLISH_ADDRESS=${GATEWAY_PUBLISH_ADDRESS:-127.0.0.1} +PUBLISH_PORT=${GATEWAY_PUBLISH_PORT:-9080} +ALLOW_NON_LOOPBACK_BIND=${ALLOW_NON_LOOPBACK_BIND:-0} + +fail() { + printf 'production preflight: ERROR: %s\n' "$*" >&2 + exit 1 +} + +warn() { + printf 'production preflight: WARN: %s\n' "$*" >&2 +} + +command -v docker >/dev/null 2>&1 || fail 'docker is not available' +docker compose version >/dev/null 2>&1 || fail 'docker compose is not available' +[ -r "$CONFIG" ] || fail "bootstrap config is not readable: $CONFIG" + +case "$CONFIG" in + *config.example.json|*/config.example.json) + fail 'GATEWAY_CONFIG points at config.example.json; select the production bootstrap config explicitly' + ;; +esac + +case "$PUBLISH_ADDRESS" in + 127.0.0.1|::1|localhost) + ;; + 0.0.0.0|::|'[::]') + [ "$ALLOW_NON_LOOPBACK_BIND" = 1 ] || fail "gateway would be published on all interfaces ($PUBLISH_ADDRESS:$PUBLISH_PORT); bind to loopback or set ALLOW_NON_LOOPBACK_BIND=1 only with an external firewall/ACL" + warn "non-loopback wildcard publishing explicitly allowed: $PUBLISH_ADDRESS:$PUBLISH_PORT" + ;; + *) + [ "$ALLOW_NON_LOOPBACK_BIND" = 1 ] || fail "gateway would be published on non-loopback address $PUBLISH_ADDRESS:$PUBLISH_PORT; use loopback for a host-local reverse proxy, or set ALLOW_NON_LOOPBACK_BIND=1 after restricting the host firewall to the real proxy" + warn "non-loopback publishing explicitly allowed: $PUBLISH_ADDRESS:$PUBLISH_PORT" + ;; +esac + +export GATEWAY_CONFIG="$CONFIG" GATEWAY_PUBLISH_ADDRESS="$PUBLISH_ADDRESS" GATEWAY_PUBLISH_PORT="$PUBLISH_PORT" + +docker compose -f "$COMPOSE_FILE" config >/dev/null + +printf 'production preflight: compose syntax OK\n' +printf 'production preflight: published endpoint %s:%s -> container :8080\n' "$PUBLISH_ADDRESS" "$PUBLISH_PORT" + +# Runs the candidate image with the exact bootstrap/state mounts and runtime user. +# The gateway preflight output is secret-redacted by design. +docker compose -f "$COMPOSE_FILE" run --rm --no-deps gateway \ + -config /etc/ollama-gateway/config.json -check-config + +printf 'production preflight: OK\n' diff --git a/scripts/production-preflight_test.sh b/scripts/production-preflight_test.sh new file mode 100644 index 0000000..8924350 --- /dev/null +++ b/scripts/production-preflight_test.sh @@ -0,0 +1,70 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT HUP INT TERM +mkdir -p "$TMP/bin" + +cat > "$TMP/bin/docker" <<'EOF' +#!/bin/sh +set -eu +[ "$1" = compose ] || exit 64 +shift +# Accept optional -f . +if [ "${1:-}" = -f ]; then shift 2; fi +case "${1:-}" in + version) exit 0 ;; + config) exit 0 ;; + run) + cat < "$TMP/gateway-config.json" <<'EOF' +{} +EOF +cat > "$TMP/config.example.json" <<'EOF' +{} +EOF + +PATH="$TMP/bin:$PATH" \ +GATEWAY_CONFIG="$TMP/gateway-config.json" \ +GATEWAY_PUBLISH_ADDRESS=127.0.0.1 \ +COMPOSE_FILE="$ROOT/docker-compose.production.yml" \ +"$ROOT/scripts/production-preflight.sh" > "$TMP/ok.out" 2> "$TMP/ok.err" +grep -q 'production preflight: OK' "$TMP/ok.out" + +if PATH="$TMP/bin:$PATH" \ + GATEWAY_CONFIG="$TMP/config.example.json" \ + COMPOSE_FILE="$ROOT/docker-compose.production.yml" \ + "$ROOT/scripts/production-preflight.sh" >/dev/null 2>&1; then + echo 'expected config.example.json rejection' >&2 + exit 1 +fi + +if PATH="$TMP/bin:$PATH" \ + GATEWAY_CONFIG="$TMP/gateway-config.json" \ + GATEWAY_PUBLISH_ADDRESS=0.0.0.0 \ + COMPOSE_FILE="$ROOT/docker-compose.production.yml" \ + "$ROOT/scripts/production-preflight.sh" >/dev/null 2>&1; then + echo 'expected wildcard bind rejection' >&2 + exit 1 +fi + +PATH="$TMP/bin:$PATH" \ +GATEWAY_CONFIG="$TMP/gateway-config.json" \ +GATEWAY_PUBLISH_ADDRESS=10.2.10.20 \ +ALLOW_NON_LOOPBACK_BIND=1 \ +COMPOSE_FILE="$ROOT/docker-compose.production.yml" \ +"$ROOT/scripts/production-preflight.sh" > "$TMP/remote.out" 2> "$TMP/remote.err" +grep -q 'production preflight: OK' "$TMP/remote.out" +grep -q 'non-loopback publishing explicitly allowed' "$TMP/remote.err" + +echo 'production preflight tests: PASS'