diff --git a/proxy/internal/middleware/builtin/llm_router/middleware.go b/proxy/internal/middleware/builtin/llm_router/middleware.go index 93d648fcd..5fbdce837 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware.go @@ -179,13 +179,13 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar // exists and is reachable. Authorise it against the model table like any // other per-model request, then mark it non-inference so it still skips // the token pre-flight it would otherwise charge nothing against. - if detail, isDetail := modelDetailID(reqPath); isDetail { + if detail, isDetail := modelDetailID(reqPath); isDetail && isNonInferenceMethod(in.Method) { route, outcome := m.matchRoute(detail, surface, reqPath, in.UserGroups) return m.decide(route, outcome, surface, detail, in.UserGroups, markNonInference), nil } if model == "" { - return m.routeModelless(reqPath, surface, in.UserGroups), nil + return m.routeModelless(reqPath, surface, in.Method, in.UserGroups), nil } route, outcome := m.matchRoute(model, surface, reqPath, in.UserGroups) @@ -223,8 +223,8 @@ func (m *Middleware) decide( // lookup. They still need rewriting from the synth placeholder to a real // upstream — clients such as Codex call GET /v1/models at startup to enumerate // availability and read a 403 as "model unavailable". -func (m *Middleware) routeModelless(reqPath, surface string, userGroups []string) *middleware.Output { - route, outcome := m.matchModelless(reqPath, userGroups) +func (m *Middleware) routeModelless(reqPath, surface, method string, userGroups []string) *middleware.Output { + route, outcome := m.matchModelless(reqPath, method, userGroups) switch outcome { case matchOutcomeFound: out := m.allowWithRoute(route, surface, userGroups) @@ -250,6 +250,17 @@ func (m *Middleware) routeModelless(reqPath, surface string, userGroups []string } } +// isNonInferenceMethod reports whether a request method is one the +// non-inference endpoints actually use: the listing and the per-model lookup +// are GET, the connection-warming probe is HEAD or GET. The method is the only +// thing separating "GET /v1/models/{id}" from a POST to the same path carrying +// an inference body, and the non-inference mark exempts a request from the +// token pre-flight — so anything else falls through to normal per-model +// routing, which denies when the request names no model. +func isNonInferenceMethod(method string) bool { + return method == http.MethodGet || method == http.MethodHead +} + // markNonInference tags an allow as a request that spends no tokens, so the // limit check skips the management pre-flight it would charge nothing against. func markNonInference(out *middleware.Output) { @@ -534,7 +545,10 @@ func (m *Middleware) matchPathRoute(reqPath, model string, userGroups []string, // declaration order), matchOutcomeUnauthorised when no provider authorises // the caller, or matchOutcomeUnknownModel when the path isn't a recognised // model-less endpoint. -func (m *Middleware) matchModelless(reqPath string, userGroups []string) (ProviderRoute, matchOutcome) { +func (m *Middleware) matchModelless(reqPath, method string, userGroups []string) (ProviderRoute, matchOutcome) { + if !isNonInferenceMethod(method) { + return ProviderRoute{}, matchOutcomeUnknownModel + } var eligible func(ProviderRoute) bool switch { case isBedrockModelLessPath(reqPath): diff --git a/proxy/internal/middleware/builtin/llm_router/middleware_test.go b/proxy/internal/middleware/builtin/llm_router/middleware_test.go index f83e7ddf9..a0ce265f5 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware_test.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware_test.go @@ -173,8 +173,12 @@ func TestRouter_MissingModel(t *testing.T) { // from which a model could be parsed). UserGroups matches defaultTestGroup. func newModellessInput(reqURL string) *middleware.Input { return &middleware.Input{ - Slot: middleware.SlotOnRequest, - URL: reqURL, + Slot: middleware.SlotOnRequest, + URL: reqURL, + // The non-inference endpoints are read requests; the method is what + // separates them from an inference body posted to the same path, so + // state it rather than leaning on the zero value. + Method: http.MethodGet, UserGroups: []string{defaultTestGroup}, } } @@ -1035,3 +1039,49 @@ func TestRouter_ModelDetailHonoursAllowlist(t *testing.T) { "a gateway that enumerates nothing cannot refuse a lookup") }) } + +// TestRouter_NonInferenceRequiresReadMethod pins that the non-inference mark — +// which exempts a request from the token pre-flight — is reachable only by the +// read methods these endpoints actually use. A POST to the same path could +// carry an inference body, so it must not buy the exemption; it falls through +// to normal per-model routing instead, which denies when no model is named. +func TestRouter_NonInferenceRequiresReadMethod(t *testing.T) { + route := ProviderRoute{ + ID: "gateway", + AllowedGroupIDs: []string{defaultTestGroup}, + UpstreamScheme: "https", + UpstreamHost: "gateway.example.com", + } + + for _, path := range []string{"/v1/models", "/v1/models/claude-sonnet-5", "/api/hello"} { + t.Run("POST "+path, func(t *testing.T) { + mw := New(Config{Providers: []ProviderRoute{route}}) + + in := newModellessInput(path) + in.Method = http.MethodPost + + out, err := mw.Invoke(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, middleware.DecisionDeny, out.Decision, + "a write to a non-inference path must not route unmetered") + + nonInference, _ := metaValue(t, out.Metadata, middleware.KeyLLMNonInference) + assert.NotEqual(t, "true", nonInference, + "only a read method may skip the token pre-flight") + }) + } + + t.Run("HEAD keeps the warm probe working", func(t *testing.T) { + mw := New(Config{Providers: []ProviderRoute{route}}) + + in := newModellessInput(connectionWarmPath) + in.Method = http.MethodHead + + out, err := mw.Invoke(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, middleware.DecisionAllow, out.Decision) + + nonInference, _ := metaValue(t, out.Metadata, middleware.KeyLLMNonInference) + assert.Equal(t, "true", nonInference) + }) +}