diff --git a/proxy/internal/middleware/builtin/llm_limit_check/middleware.go b/proxy/internal/middleware/builtin/llm_limit_check/middleware.go index 42ac56b9b..c910d57da 100644 --- a/proxy/internal/middleware/builtin/llm_limit_check/middleware.go +++ b/proxy/internal/middleware/builtin/llm_limit_check/middleware.go @@ -84,6 +84,15 @@ func (m *Middleware) Invoke(ctx context.Context, in *middleware.Input) (*middlew return allowNoAttribution(), nil } + // Model-listing and other non-inference endpoints carry no model, and + // management's per-model allowlist fails closed on an empty one. The + // router has already authorised the route against the caller's groups + // and the request consumes no tokens, so gating it on a model that + // cannot exist would only break gateway model discovery. + if lookupKV(in.Metadata, middleware.KeyLLMNonInference) == "true" { + return allowNoAttribution(), nil + } + providerID := lookupKV(in.Metadata, middleware.KeyLLMResolvedProviderID) if providerID == "" { // llm_router didn't emit a resolved provider id — usually diff --git a/proxy/internal/middleware/builtin/llm_limit_check/middleware_test.go b/proxy/internal/middleware/builtin/llm_limit_check/middleware_test.go index 87aa8e9e9..7754998ee 100644 --- a/proxy/internal/middleware/builtin/llm_limit_check/middleware_test.go +++ b/proxy/internal/middleware/builtin/llm_limit_check/middleware_test.go @@ -224,3 +224,35 @@ func TestMetadataKeys_Allowlist(t *testing.T) { } assert.ElementsMatch(t, want, keys) } + +// TestInvoke_NonInferenceSkipsPreflight covers gateway model discovery: +// GET /v1/models carries no model, and management's per-model allowlist +// fails closed on an empty one, so a pre-flight would deny discovery for +// exactly the accounts that use the model allowlist. The router marks the +// request non-inference after authorising the route, and the gate must +// then allow without calling management at all. +func TestInvoke_NonInferenceSkipsPreflight(t *testing.T) { + mgmt := &fakeMgmt{ + checkResp: &proto.CheckLLMPolicyLimitsResponse{ + Decision: "deny", + DenyCode: "llm_policy.model_blocked", + }, + } + m := New(mgmt, nil) + + out := runInvoke(t, m, &middleware.Input{ + AccountID: "acc-1", + UserID: "user-bob", + UserGroups: []string{"grp-engineers"}, + Metadata: []middleware.KV{ + {Key: middleware.KeyLLMResolvedProviderID, Value: "prov-1"}, + {Key: middleware.KeyLLMNonInference, Value: "true"}, + }, + }) + + assert.Equal(t, middleware.DecisionAllow, out.Decision, "model-less endpoints must not be gated on a model") + assert.Nil(t, mgmt.checkReq, "no pre-flight may be sent for a non-inference request") + + assert.Empty(t, lookupKV(out.Metadata, middleware.KeyLLMSelectedPolicyID), + "no policy is attributed when nothing was metered") +} diff --git a/proxy/internal/middleware/builtin/llm_router/middleware.go b/proxy/internal/middleware/builtin/llm_router/middleware.go index 2d987eef6..d19a4d941 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware.go @@ -109,6 +109,7 @@ func (m *Middleware) MetadataKeys() []string { middleware.KeyLLMAuthorisingGroups, middleware.KeyLLMPolicyDecision, middleware.KeyLLMPolicyReason, + middleware.KeyLLMNonInference, } } @@ -193,7 +194,9 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar route, outcome := m.matchModelless(requestPath(in.URL), in.UserGroups) switch outcome { case matchOutcomeFound: - return m.allowWithRoute(route, in.UserGroups), nil + out := m.allowWithRoute(route, in.UserGroups) + out.Metadata = append(out.Metadata, middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"}) + return out, nil case matchOutcomeUnauthorised: // A recognised model-less endpoint exists but no provider // authorises the caller — deny as an authorisation failure diff --git a/proxy/internal/middleware/builtin/llm_router/middleware_test.go b/proxy/internal/middleware/builtin/llm_router/middleware_test.go index 425c383c1..754174254 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware_test.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware_test.go @@ -60,6 +60,7 @@ func TestMiddlewareIdentity(t *testing.T) { []string{ middleware.KeyLLMResolvedProviderID, middleware.KeyLLMAuthorisingGroups, + middleware.KeyLLMNonInference, middleware.KeyLLMPolicyDecision, middleware.KeyLLMPolicyReason, }, @@ -197,6 +198,12 @@ func TestRouter_ModelLessPath_RoutesToAuthorisedProvider(t *testing.T) { provider, _ := metaValue(t, out.Metadata, middleware.KeyLLMResolvedProviderID) assert.Equal(t, "openai-prod", provider, "resolved provider must be the authorised route") + + // The limits gate reads this to tell "no model applies here" from + // "the model could not be determined", which fails closed. + nonInference, ok := metaValue(t, out.Metadata, middleware.KeyLLMNonInference) + require.True(t, ok, "model-less allow must mark the request non-inference") + assert.Equal(t, "true", nonInference) } func TestRouter_ModelLessPath_MultiProviderDeclarationOrder(t *testing.T) { diff --git a/proxy/internal/middleware/keys.go b/proxy/internal/middleware/keys.go index 336bed19f..53233c925 100644 --- a/proxy/internal/middleware/keys.go +++ b/proxy/internal/middleware/keys.go @@ -66,6 +66,14 @@ const ( // downstream gateways' spend logs. KeyLLMAuthorisingGroups = "llm.authorising_groups" + // LLM non-inference marker (emitted by llm_router on the allow path + // for endpoints that legitimately carry no model, such as model + // listing). The router still authorises these against the caller's + // groups; the marker only tells the limits gate that a per-model + // allowlist has nothing to evaluate, so an empty model must not be + // read as an undetermined one. Never derived from client input. + KeyLLMNonInference = "llm.non_inference" + // LLM policy attribution (emitted by llm_limit_check on the allow // path). Names the policy that paid for this request and the // dimension counters the post-flight llm_limit_record middleware