diff --git a/proxy/internal/middleware/builtin/llm_router/bedrock_route_test.go b/proxy/internal/middleware/builtin/llm_router/bedrock_route_test.go index 8c7810724..badd358c5 100644 --- a/proxy/internal/middleware/builtin/llm_router/bedrock_route_test.go +++ b/proxy/internal/middleware/builtin/llm_router/bedrock_route_test.go @@ -57,3 +57,61 @@ func TestRouter_BedrockCountTokensRoutes(t *testing.T) { require.NotNil(t, out.Mutations.RewriteUpstream) assert.Equal(t, "bedrock-runtime.eu-central-1.amazonaws.com", out.Mutations.RewriteUpstream.Host) } + +// TestRouter_BedrockInferenceProfilesRoutes covers the startup lookups a +// client makes to resolve a configured inference profile. They carry no +// model, so before they were recognised they denied and wrote a policy +// rejection into the access log on every session start. +func TestRouter_BedrockInferenceProfilesRoutes(t *testing.T) { + bedrock := ProviderRoute{ + ID: "bedrock-prod", + Bedrock: true, + AllowedGroupIDs: []string{defaultTestGroup}, + UpstreamScheme: "https", + UpstreamHost: "bedrock-runtime.eu-central-1.amazonaws.com", + } + openai := ProviderRoute{ + ID: "openai-prod", + Models: []string{"gpt-4o"}, + AllowedGroupIDs: []string{defaultTestGroup}, + UpstreamScheme: "https", + UpstreamHost: "api.openai.com", + } + mw := New(Config{Providers: []ProviderRoute{openai, bedrock}}) + + for _, path := range []string{ + "/inference-profiles?type=SYSTEM_DEFINED", + "/inference-profiles/us.anthropic.claude-sonnet-5", + } { + out, err := mw.Invoke(context.Background(), newModellessInput(path)) + require.NoError(t, err) + require.NotNil(t, out) + assert.Equal(t, middleware.DecisionAllow, out.Decision, "%s must route", path) + require.NotNil(t, out.Mutations) + require.NotNil(t, out.Mutations.RewriteUpstream) + assert.Equal(t, "bedrock-runtime.eu-central-1.amazonaws.com", out.Mutations.RewriteUpstream.Host, + "%s must reach the Bedrock provider, not the first authorised one", path) + + nonInference, _ := metaValue(t, out.Metadata, middleware.KeyLLMNonInference) + assert.Equal(t, "true", nonInference, "%s carries no model to gate on", path) + } +} + +// TestRouter_BedrockNamespacedInferenceProfilesStripsPrefix pins that the +// optional gateway namespace is removed before the request goes upstream. +func TestRouter_BedrockNamespacedInferenceProfilesStripsPrefix(t *testing.T) { + mw := New(Config{Providers: []ProviderRoute{{ + ID: "bedrock-prod", + Bedrock: true, + AllowedGroupIDs: []string{defaultTestGroup}, + UpstreamScheme: "https", + UpstreamHost: "bedrock-runtime.eu-central-1.amazonaws.com", + }}}) + + out, err := mw.Invoke(context.Background(), newModellessInput("/bedrock/inference-profiles")) + require.NoError(t, err) + require.NotNil(t, out.Mutations) + require.NotNil(t, out.Mutations.RewriteUpstream) + assert.Equal(t, "/bedrock", out.Mutations.RewriteUpstream.StripPathPrefix, + "the namespace prefix must not reach the real Bedrock endpoint") +} diff --git a/proxy/internal/middleware/builtin/llm_router/middleware.go b/proxy/internal/middleware/builtin/llm_router/middleware.go index b6eba92f4..c38e00a43 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware.go @@ -194,11 +194,14 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar // need rewriting from the synth placeholder to a real upstream; // clients such as Codex call GET /v1/models at startup to enumerate // availability and read a 403 as "model unavailable". - route, outcome := m.matchModelless(requestPath(in.URL), in.UserGroups) + route, outcome := m.matchModelless(reqPath, in.UserGroups) switch outcome { case matchOutcomeFound: out := m.allowWithRoute(route, surface, in.UserGroups) out.Metadata = append(out.Metadata, middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"}) + if _, hadPrefix := splitBedrockNamespace(reqPath); hadPrefix && out.Mutations != nil && out.Mutations.RewriteUpstream != nil { + out.Mutations.RewriteUpstream.StripPathPrefix = bedrockNamespacePrefix + } return out, nil case matchOutcomeUnauthorised: // A recognised model-less endpoint exists but no provider @@ -305,14 +308,24 @@ func (m *Middleware) matchRoute(model, vendor, reqPath string, userGroups []stri return best, matchOutcomeFound } -// isModelLessPath reports whether reqPath is a known OpenAI-shaped -// non-inference endpoint that legitimately carries no model in its -// request (the model-listing endpoints). These must route to an upstream -// rather than deny, so model enumeration works end to end. +// isModelLessPath reports whether reqPath is a known non-inference endpoint +// that legitimately carries no model in its request (the model-listing +// endpoints). These must route to an upstream rather than deny, so model +// enumeration works end to end. func isModelLessPath(reqPath string) bool { return reqPath == "/v1/models" || strings.HasPrefix(reqPath, "/v1/models/") } +// isBedrockModelLessPath reports whether reqPath is a Bedrock +// inference-profile lookup, optionally behind the "/bedrock" gateway +// namespace. Clients read these at startup to resolve a configured profile +// to its underlying model. They carry no model of their own, so they route +// by path to a Bedrock provider rather than through the model table. +func isBedrockModelLessPath(reqPath string) bool { + native, _ := splitBedrockNamespace(reqPath) + return native == "/inference-profiles" || strings.HasPrefix(native, "/inference-profiles/") +} + // isVertexPath reports whether reqPath is a Google Vertex AI publisher // endpoint: /v1/projects/{project}/locations/{region}/publishers/{publisher}/ // models/{model}:{action}. The model + vendor live in the path, so these @@ -444,18 +457,22 @@ func (m *Middleware) matchPathRoute(reqPath, model string, userGroups []string, // the caller, or matchOutcomeUnknownModel when the path isn't a recognised // model-less endpoint. func (m *Middleware) matchModelless(reqPath string, userGroups []string) (ProviderRoute, matchOutcome) { - if !isModelLessPath(reqPath) { - return ProviderRoute{}, matchOutcomeUnknownModel - } - var candidates []ProviderRoute - for _, route := range m.cfg.Providers { + var eligible func(ProviderRoute) bool + switch { + case isBedrockModelLessPath(reqPath): + eligible = func(r ProviderRoute) bool { return r.Bedrock } + case isModelLessPath(reqPath): // Vertex/Bedrock are path-routed and don't serve OpenAI-style // model-listing endpoints; including them here could rewrite a // GET /v1/models to an upstream that 404s it. - if route.Vertex || route.Bedrock { - continue - } - if routeAuthorisesGroups(route, userGroups) { + eligible = func(r ProviderRoute) bool { return !r.Vertex && !r.Bedrock } + default: + return ProviderRoute{}, matchOutcomeUnknownModel + } + + var candidates []ProviderRoute + for _, route := range m.cfg.Providers { + if eligible(route) && routeAuthorisesGroups(route, userGroups) { candidates = append(candidates, route) } }