From d928bcb6302ed9b51cb953a065ec31e316426595 Mon Sep 17 00:00:00 2001 From: mlsmaycon Date: Tue, 11 Aug 2026 02:47:19 +0000 Subject: [PATCH] [proxy] Exempt non-inference endpoints from the model allowlist gate GET /v1/models carries no model, and management's per-model allowlist fails closed on an undetermined one, so gateway model discovery denied with model_blocked for every account that enables a model allowlist. The client treats a failed discovery as silent and falls back to its built-in list, so the operator sees an empty picker with no error to chase. The router already classifies these paths and authorises the route against the caller's groups before allowing them, so mark them non-inference there and let the limits gate skip a pre-flight that has no model to evaluate and no tokens to book. The marker comes from the router's own path classification, never from client input, so an inference request cannot set it to escape the allowlist. --- .../builtin/llm_limit_check/middleware.go | 9 ++++++ .../llm_limit_check/middleware_test.go | 32 +++++++++++++++++++ .../builtin/llm_router/middleware.go | 5 ++- .../builtin/llm_router/middleware_test.go | 7 ++++ proxy/internal/middleware/keys.go | 8 +++++ 5 files changed, 60 insertions(+), 1 deletion(-) diff --git a/proxy/internal/middleware/builtin/llm_limit_check/middleware.go b/proxy/internal/middleware/builtin/llm_limit_check/middleware.go index 42ac56b9b..c910d57da 100644 --- a/proxy/internal/middleware/builtin/llm_limit_check/middleware.go +++ b/proxy/internal/middleware/builtin/llm_limit_check/middleware.go @@ -84,6 +84,15 @@ func (m *Middleware) Invoke(ctx context.Context, in *middleware.Input) (*middlew return allowNoAttribution(), nil } + // Model-listing and other non-inference endpoints carry no model, and + // management's per-model allowlist fails closed on an empty one. The + // router has already authorised the route against the caller's groups + // and the request consumes no tokens, so gating it on a model that + // cannot exist would only break gateway model discovery. + if lookupKV(in.Metadata, middleware.KeyLLMNonInference) == "true" { + return allowNoAttribution(), nil + } + providerID := lookupKV(in.Metadata, middleware.KeyLLMResolvedProviderID) if providerID == "" { // llm_router didn't emit a resolved provider id — usually diff --git a/proxy/internal/middleware/builtin/llm_limit_check/middleware_test.go b/proxy/internal/middleware/builtin/llm_limit_check/middleware_test.go index 87aa8e9e9..7754998ee 100644 --- a/proxy/internal/middleware/builtin/llm_limit_check/middleware_test.go +++ b/proxy/internal/middleware/builtin/llm_limit_check/middleware_test.go @@ -224,3 +224,35 @@ func TestMetadataKeys_Allowlist(t *testing.T) { } assert.ElementsMatch(t, want, keys) } + +// TestInvoke_NonInferenceSkipsPreflight covers gateway model discovery: +// GET /v1/models carries no model, and management's per-model allowlist +// fails closed on an empty one, so a pre-flight would deny discovery for +// exactly the accounts that use the model allowlist. The router marks the +// request non-inference after authorising the route, and the gate must +// then allow without calling management at all. +func TestInvoke_NonInferenceSkipsPreflight(t *testing.T) { + mgmt := &fakeMgmt{ + checkResp: &proto.CheckLLMPolicyLimitsResponse{ + Decision: "deny", + DenyCode: "llm_policy.model_blocked", + }, + } + m := New(mgmt, nil) + + out := runInvoke(t, m, &middleware.Input{ + AccountID: "acc-1", + UserID: "user-bob", + UserGroups: []string{"grp-engineers"}, + Metadata: []middleware.KV{ + {Key: middleware.KeyLLMResolvedProviderID, Value: "prov-1"}, + {Key: middleware.KeyLLMNonInference, Value: "true"}, + }, + }) + + assert.Equal(t, middleware.DecisionAllow, out.Decision, "model-less endpoints must not be gated on a model") + assert.Nil(t, mgmt.checkReq, "no pre-flight may be sent for a non-inference request") + + assert.Empty(t, lookupKV(out.Metadata, middleware.KeyLLMSelectedPolicyID), + "no policy is attributed when nothing was metered") +} diff --git a/proxy/internal/middleware/builtin/llm_router/middleware.go b/proxy/internal/middleware/builtin/llm_router/middleware.go index 2d987eef6..d19a4d941 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware.go @@ -109,6 +109,7 @@ func (m *Middleware) MetadataKeys() []string { middleware.KeyLLMAuthorisingGroups, middleware.KeyLLMPolicyDecision, middleware.KeyLLMPolicyReason, + middleware.KeyLLMNonInference, } } @@ -193,7 +194,9 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar route, outcome := m.matchModelless(requestPath(in.URL), in.UserGroups) switch outcome { case matchOutcomeFound: - return m.allowWithRoute(route, in.UserGroups), nil + out := m.allowWithRoute(route, in.UserGroups) + out.Metadata = append(out.Metadata, middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"}) + return out, nil case matchOutcomeUnauthorised: // A recognised model-less endpoint exists but no provider // authorises the caller — deny as an authorisation failure diff --git a/proxy/internal/middleware/builtin/llm_router/middleware_test.go b/proxy/internal/middleware/builtin/llm_router/middleware_test.go index 425c383c1..754174254 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware_test.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware_test.go @@ -60,6 +60,7 @@ func TestMiddlewareIdentity(t *testing.T) { []string{ middleware.KeyLLMResolvedProviderID, middleware.KeyLLMAuthorisingGroups, + middleware.KeyLLMNonInference, middleware.KeyLLMPolicyDecision, middleware.KeyLLMPolicyReason, }, @@ -197,6 +198,12 @@ func TestRouter_ModelLessPath_RoutesToAuthorisedProvider(t *testing.T) { provider, _ := metaValue(t, out.Metadata, middleware.KeyLLMResolvedProviderID) assert.Equal(t, "openai-prod", provider, "resolved provider must be the authorised route") + + // The limits gate reads this to tell "no model applies here" from + // "the model could not be determined", which fails closed. + nonInference, ok := metaValue(t, out.Metadata, middleware.KeyLLMNonInference) + require.True(t, ok, "model-less allow must mark the request non-inference") + assert.Equal(t, "true", nonInference) } func TestRouter_ModelLessPath_MultiProviderDeclarationOrder(t *testing.T) { diff --git a/proxy/internal/middleware/keys.go b/proxy/internal/middleware/keys.go index 336bed19f..53233c925 100644 --- a/proxy/internal/middleware/keys.go +++ b/proxy/internal/middleware/keys.go @@ -66,6 +66,14 @@ const ( // downstream gateways' spend logs. KeyLLMAuthorisingGroups = "llm.authorising_groups" + // LLM non-inference marker (emitted by llm_router on the allow path + // for endpoints that legitimately carry no model, such as model + // listing). The router still authorises these against the caller's + // groups; the marker only tells the limits gate that a per-model + // allowlist has nothing to evaluate, so an empty model must not be + // read as an undetermined one. Never derived from client input. + KeyLLMNonInference = "llm.non_inference" + // LLM policy attribution (emitted by llm_limit_check on the allow // path). Names the policy that paid for this request and the // dimension counters the post-flight llm_limit_record middleware