mirror of
https://github.com/netbirdio/netbird.git
synced 2026-08-24 16:41:30 +02:00
[proxy] Route Bedrock inference-profile lookups
A client resolving a configured inference profile calls GET /inference-profiles at startup. The path carries no model and was not recognised as non-inference, so it denied as not-routable and wrote a policy rejection into the access log on every session start, which is the log operators read to find real policy problems. Recognise the path and match it against a Bedrock provider specifically: sending it to whichever provider happened to be authorised first would rewrite it to an upstream that 404s it. The optional gateway namespace is stripped the same way the runtime paths strip it.
This commit is contained in:
@@ -57,3 +57,61 @@ func TestRouter_BedrockCountTokensRoutes(t *testing.T) {
|
||||
require.NotNil(t, out.Mutations.RewriteUpstream)
|
||||
assert.Equal(t, "bedrock-runtime.eu-central-1.amazonaws.com", out.Mutations.RewriteUpstream.Host)
|
||||
}
|
||||
|
||||
// TestRouter_BedrockInferenceProfilesRoutes covers the startup lookups a
|
||||
// client makes to resolve a configured inference profile. They carry no
|
||||
// model, so before they were recognised they denied and wrote a policy
|
||||
// rejection into the access log on every session start.
|
||||
func TestRouter_BedrockInferenceProfilesRoutes(t *testing.T) {
|
||||
bedrock := ProviderRoute{
|
||||
ID: "bedrock-prod",
|
||||
Bedrock: true,
|
||||
AllowedGroupIDs: []string{defaultTestGroup},
|
||||
UpstreamScheme: "https",
|
||||
UpstreamHost: "bedrock-runtime.eu-central-1.amazonaws.com",
|
||||
}
|
||||
openai := ProviderRoute{
|
||||
ID: "openai-prod",
|
||||
Models: []string{"gpt-4o"},
|
||||
AllowedGroupIDs: []string{defaultTestGroup},
|
||||
UpstreamScheme: "https",
|
||||
UpstreamHost: "api.openai.com",
|
||||
}
|
||||
mw := New(Config{Providers: []ProviderRoute{openai, bedrock}})
|
||||
|
||||
for _, path := range []string{
|
||||
"/inference-profiles?type=SYSTEM_DEFINED",
|
||||
"/inference-profiles/us.anthropic.claude-sonnet-5",
|
||||
} {
|
||||
out, err := mw.Invoke(context.Background(), newModellessInput(path))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "%s must route", path)
|
||||
require.NotNil(t, out.Mutations)
|
||||
require.NotNil(t, out.Mutations.RewriteUpstream)
|
||||
assert.Equal(t, "bedrock-runtime.eu-central-1.amazonaws.com", out.Mutations.RewriteUpstream.Host,
|
||||
"%s must reach the Bedrock provider, not the first authorised one", path)
|
||||
|
||||
nonInference, _ := metaValue(t, out.Metadata, middleware.KeyLLMNonInference)
|
||||
assert.Equal(t, "true", nonInference, "%s carries no model to gate on", path)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRouter_BedrockNamespacedInferenceProfilesStripsPrefix pins that the
|
||||
// optional gateway namespace is removed before the request goes upstream.
|
||||
func TestRouter_BedrockNamespacedInferenceProfilesStripsPrefix(t *testing.T) {
|
||||
mw := New(Config{Providers: []ProviderRoute{{
|
||||
ID: "bedrock-prod",
|
||||
Bedrock: true,
|
||||
AllowedGroupIDs: []string{defaultTestGroup},
|
||||
UpstreamScheme: "https",
|
||||
UpstreamHost: "bedrock-runtime.eu-central-1.amazonaws.com",
|
||||
}}})
|
||||
|
||||
out, err := mw.Invoke(context.Background(), newModellessInput("/bedrock/inference-profiles"))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out.Mutations)
|
||||
require.NotNil(t, out.Mutations.RewriteUpstream)
|
||||
assert.Equal(t, "/bedrock", out.Mutations.RewriteUpstream.StripPathPrefix,
|
||||
"the namespace prefix must not reach the real Bedrock endpoint")
|
||||
}
|
||||
|
||||
@@ -194,11 +194,14 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar
|
||||
// need rewriting from the synth placeholder to a real upstream;
|
||||
// clients such as Codex call GET /v1/models at startup to enumerate
|
||||
// availability and read a 403 as "model unavailable".
|
||||
route, outcome := m.matchModelless(requestPath(in.URL), in.UserGroups)
|
||||
route, outcome := m.matchModelless(reqPath, in.UserGroups)
|
||||
switch outcome {
|
||||
case matchOutcomeFound:
|
||||
out := m.allowWithRoute(route, surface, in.UserGroups)
|
||||
out.Metadata = append(out.Metadata, middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"})
|
||||
if _, hadPrefix := splitBedrockNamespace(reqPath); hadPrefix && out.Mutations != nil && out.Mutations.RewriteUpstream != nil {
|
||||
out.Mutations.RewriteUpstream.StripPathPrefix = bedrockNamespacePrefix
|
||||
}
|
||||
return out, nil
|
||||
case matchOutcomeUnauthorised:
|
||||
// A recognised model-less endpoint exists but no provider
|
||||
@@ -305,14 +308,24 @@ func (m *Middleware) matchRoute(model, vendor, reqPath string, userGroups []stri
|
||||
return best, matchOutcomeFound
|
||||
}
|
||||
|
||||
// isModelLessPath reports whether reqPath is a known OpenAI-shaped
|
||||
// non-inference endpoint that legitimately carries no model in its
|
||||
// request (the model-listing endpoints). These must route to an upstream
|
||||
// rather than deny, so model enumeration works end to end.
|
||||
// isModelLessPath reports whether reqPath is a known non-inference endpoint
|
||||
// that legitimately carries no model in its request (the model-listing
|
||||
// endpoints). These must route to an upstream rather than deny, so model
|
||||
// enumeration works end to end.
|
||||
func isModelLessPath(reqPath string) bool {
|
||||
return reqPath == "/v1/models" || strings.HasPrefix(reqPath, "/v1/models/")
|
||||
}
|
||||
|
||||
// isBedrockModelLessPath reports whether reqPath is a Bedrock
|
||||
// inference-profile lookup, optionally behind the "/bedrock" gateway
|
||||
// namespace. Clients read these at startup to resolve a configured profile
|
||||
// to its underlying model. They carry no model of their own, so they route
|
||||
// by path to a Bedrock provider rather than through the model table.
|
||||
func isBedrockModelLessPath(reqPath string) bool {
|
||||
native, _ := splitBedrockNamespace(reqPath)
|
||||
return native == "/inference-profiles" || strings.HasPrefix(native, "/inference-profiles/")
|
||||
}
|
||||
|
||||
// isVertexPath reports whether reqPath is a Google Vertex AI publisher
|
||||
// endpoint: /v1/projects/{project}/locations/{region}/publishers/{publisher}/
|
||||
// models/{model}:{action}. The model + vendor live in the path, so these
|
||||
@@ -444,18 +457,22 @@ func (m *Middleware) matchPathRoute(reqPath, model string, userGroups []string,
|
||||
// the caller, or matchOutcomeUnknownModel when the path isn't a recognised
|
||||
// model-less endpoint.
|
||||
func (m *Middleware) matchModelless(reqPath string, userGroups []string) (ProviderRoute, matchOutcome) {
|
||||
if !isModelLessPath(reqPath) {
|
||||
return ProviderRoute{}, matchOutcomeUnknownModel
|
||||
}
|
||||
var candidates []ProviderRoute
|
||||
for _, route := range m.cfg.Providers {
|
||||
var eligible func(ProviderRoute) bool
|
||||
switch {
|
||||
case isBedrockModelLessPath(reqPath):
|
||||
eligible = func(r ProviderRoute) bool { return r.Bedrock }
|
||||
case isModelLessPath(reqPath):
|
||||
// Vertex/Bedrock are path-routed and don't serve OpenAI-style
|
||||
// model-listing endpoints; including them here could rewrite a
|
||||
// GET /v1/models to an upstream that 404s it.
|
||||
if route.Vertex || route.Bedrock {
|
||||
continue
|
||||
}
|
||||
if routeAuthorisesGroups(route, userGroups) {
|
||||
eligible = func(r ProviderRoute) bool { return !r.Vertex && !r.Bedrock }
|
||||
default:
|
||||
return ProviderRoute{}, matchOutcomeUnknownModel
|
||||
}
|
||||
|
||||
var candidates []ProviderRoute
|
||||
for _, route := range m.cfg.Providers {
|
||||
if eligible(route) && routeAuthorisesGroups(route, userGroups) {
|
||||
candidates = append(candidates, route)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user