diff --git a/proxy/internal/middleware/builtin/llm_router/middleware.go b/proxy/internal/middleware/builtin/llm_router/middleware.go index f76d24a4c..cf7a1843a 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware.go @@ -188,6 +188,25 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar } } + // GET /v1/models/{id} carries no body, so no model reaches the router in + // metadata — but the path names one, and answering it confirms a model + // exists and is reachable. Authorise it against the model table like any + // other per-model request, then mark it non-inference so it still skips + // the token pre-flight it would otherwise charge nothing against. + if detail, isDetail := modelDetailID(reqPath); isDetail { + route, outcome := m.matchRoute(detail, surface, reqPath, in.UserGroups) + switch outcome { + case matchOutcomeFound: + out := m.allowWithRoute(route, surface, in.UserGroups) + out.Metadata = append(out.Metadata, middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"}) + return out, nil + case matchOutcomeUnauthorised: + return denyNoAuthorisedRoute(surface, detail), nil + default: + return denyUnknownModel(surface, detail), nil + } + } + model, ok := lookupMetadata(in.Metadata, middleware.KeyLLMModel) if !ok || model == "" { // Non-inference endpoints (model listing) carry no model but still @@ -322,20 +341,36 @@ func (m *Middleware) matchRoute(model, vendor, reqPath string, userGroups []stri // the access log with rejections at every session start. const connectionWarmPath = "/api/hello" -// isModelLessPath reports whether reqPath is a known non-inference endpoint -// that legitimately carries no model in its request (model listing and the -// connection-warming probe). These must route to an upstream rather than -// deny, so model enumeration works end to end. // modelListingPath is the endpoint clients read at startup to populate // their model picker. Its response is a list the proxy can bound; the // per-model "/v1/models/{id}" lookup returns a single object and is left // alone. const modelListingPath = "/v1/models" +// isModelLessPath reports whether reqPath is a known non-inference endpoint +// that legitimately carries no model at all: the model listing and the +// connection-warming probe. These must route to an upstream rather than +// deny, so model enumeration works end to end. The per-model +// "/v1/models/{id}" lookup is deliberately excluded — it names a model, so +// it is authorised against the model table instead (see modelDetailID). func isModelLessPath(reqPath string) bool { - return reqPath == modelListingPath || - strings.HasPrefix(reqPath, modelListingPath+"/") || - reqPath == connectionWarmPath + return reqPath == modelListingPath || reqPath == connectionWarmPath +} + +// modelDetailID returns the model id named by a "/v1/models/{id}" lookup. +// reqPath comes from url.URL.Path, which is already percent-decoded, so an +// id carrying a "/" (a self-hosted "Qwen/Qwen2.5-0.5B-Instruct" sent as +// "Qwen%2FQwen2.5-...") arrives whole and everything after the prefix is the +// id, separators included. +func modelDetailID(reqPath string) (string, bool) { + if !strings.HasPrefix(reqPath, modelListingPath+"/") { + return "", false + } + id := strings.TrimPrefix(reqPath, modelListingPath+"/") + if id == "" { + return "", false + } + return id, true } // isBedrockModelLessPath reports whether reqPath is a Bedrock @@ -343,6 +378,14 @@ func isModelLessPath(reqPath string) bool { // namespace. Clients read these at startup to resolve a configured profile // to its underlying model. They carry no model of their own, so they route // by path to a Bedrock provider rather than through the model table. +// +// On native AWS these live on the control plane ("bedrock.") while a +// provider's upstream is normally the runtime host ("bedrock-runtime."), +// so forwarding yields a 404 there. That is deliberate: a client has one base +// URL, so pointing it straight at the runtime host 404s identically, and +// forwarding keeps the proxy transparent instead of inventing a policy denial +// the client would never otherwise see. Operators whose Bedrock upstream is a +// gateway that does serve the lookup get a working answer. func isBedrockModelLessPath(reqPath string) bool { native, _ := splitBedrockNamespace(reqPath) return native == "/inference-profiles" || strings.HasPrefix(native, "/inference-profiles/") diff --git a/proxy/internal/middleware/builtin/llm_router/middleware_test.go b/proxy/internal/middleware/builtin/llm_router/middleware_test.go index f1b0e4583..f83e7ddf9 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware_test.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware_test.go @@ -980,3 +980,58 @@ func TestRouter_ModelListingCarriesAuthorisedModels(t *testing.T) { "the single-object lookup has no data array to filter") }) } + +// TestRouter_ModelDetailHonoursAllowlist pins that GET /v1/models/{id} is +// authorised against the model table. It carries no body model, so treating +// it as a model-less endpoint would let a caller confirm a model the route +// does not list — the listing itself is bounded to the allowlist, so the +// detail lookup must be too. +func TestRouter_ModelDetailHonoursAllowlist(t *testing.T) { + enumerated := ProviderRoute{ + ID: "anthropic-prod", + Models: []string{"claude-sonnet-5"}, + AllowedGroupIDs: []string{defaultTestGroup}, + UpstreamScheme: "https", + UpstreamHost: "api.anthropic.com", + } + + t.Run("allowlisted model routes and skips metering", func(t *testing.T) { + mw := New(Config{Providers: []ProviderRoute{enumerated}}) + out, err := mw.Invoke(context.Background(), newModellessInput("/v1/models/claude-sonnet-5")) + require.NoError(t, err) + assert.Equal(t, middleware.DecisionAllow, out.Decision) + require.NotNil(t, out.Mutations) + require.NotNil(t, out.Mutations.RewriteUpstream) + assert.Equal(t, "api.anthropic.com", out.Mutations.RewriteUpstream.Host) + + nonInference, _ := metaValue(t, out.Metadata, middleware.KeyLLMNonInference) + assert.Equal(t, "true", nonInference, "a detail lookup spends no tokens") + }) + + t.Run("model outside the allowlist denies", func(t *testing.T) { + mw := New(Config{Providers: []ProviderRoute{enumerated}}) + out, err := mw.Invoke(context.Background(), newModellessInput("/v1/models/claude-opus-5")) + require.NoError(t, err) + assert.Equal(t, middleware.DecisionDeny, out.Decision, + "a model no route lists must not be confirmed by the detail lookup") + }) + + t.Run("dated id matches its undated registration", func(t *testing.T) { + mw := New(Config{Providers: []ProviderRoute{enumerated}}) + out, err := mw.Invoke(context.Background(), newModellessInput("/v1/models/claude-sonnet-5-20250929")) + require.NoError(t, err) + assert.Equal(t, middleware.DecisionAllow, out.Decision, + "a pinned release of an allowlisted family stays reachable") + }) + + t.Run("catch-all route still answers every lookup", func(t *testing.T) { + catchAll := enumerated + catchAll.Models = nil + mw := New(Config{Providers: []ProviderRoute{catchAll}}) + + out, err := mw.Invoke(context.Background(), newModellessInput("/v1/models/anything-at-all")) + require.NoError(t, err) + assert.Equal(t, middleware.DecisionAllow, out.Decision, + "a gateway that enumerates nothing cannot refuse a lookup") + }) +}