mirror of
https://github.com/netbirdio/netbird.git
synced 2026-10-10 07:29:06 +02:00
[proxy] Route Bedrock inference-profile lookups
A client resolving a configured inference profile calls GET /inference-profiles at startup. The path carries no model and was not recognised as non-inference, so it denied as not-routable and wrote a policy rejection into the access log on every session start, which is the log operators read to find real policy problems. Recognise the path and match it against a Bedrock provider specifically: sending it to whichever provider happened to be authorised first would rewrite it to an upstream that 404s it. The optional gateway namespace is stripped the same way the runtime paths strip it.
This commit is contained in:
@@ -57,3 +57,61 @@ func TestRouter_BedrockCountTokensRoutes(t *testing.T) {
|
|||||||
require.NotNil(t, out.Mutations.RewriteUpstream)
|
require.NotNil(t, out.Mutations.RewriteUpstream)
|
||||||
assert.Equal(t, "bedrock-runtime.eu-central-1.amazonaws.com", out.Mutations.RewriteUpstream.Host)
|
assert.Equal(t, "bedrock-runtime.eu-central-1.amazonaws.com", out.Mutations.RewriteUpstream.Host)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRouter_BedrockInferenceProfilesRoutes covers the startup lookups a
|
||||||
|
// client makes to resolve a configured inference profile. They carry no
|
||||||
|
// model, so before they were recognised they denied and wrote a policy
|
||||||
|
// rejection into the access log on every session start.
|
||||||
|
func TestRouter_BedrockInferenceProfilesRoutes(t *testing.T) {
|
||||||
|
bedrock := ProviderRoute{
|
||||||
|
ID: "bedrock-prod",
|
||||||
|
Bedrock: true,
|
||||||
|
AllowedGroupIDs: []string{defaultTestGroup},
|
||||||
|
UpstreamScheme: "https",
|
||||||
|
UpstreamHost: "bedrock-runtime.eu-central-1.amazonaws.com",
|
||||||
|
}
|
||||||
|
openai := ProviderRoute{
|
||||||
|
ID: "openai-prod",
|
||||||
|
Models: []string{"gpt-4o"},
|
||||||
|
AllowedGroupIDs: []string{defaultTestGroup},
|
||||||
|
UpstreamScheme: "https",
|
||||||
|
UpstreamHost: "api.openai.com",
|
||||||
|
}
|
||||||
|
mw := New(Config{Providers: []ProviderRoute{openai, bedrock}})
|
||||||
|
|
||||||
|
for _, path := range []string{
|
||||||
|
"/inference-profiles?type=SYSTEM_DEFINED",
|
||||||
|
"/inference-profiles/us.anthropic.claude-sonnet-5",
|
||||||
|
} {
|
||||||
|
out, err := mw.Invoke(context.Background(), newModellessInput(path))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, out)
|
||||||
|
assert.Equal(t, middleware.DecisionAllow, out.Decision, "%s must route", path)
|
||||||
|
require.NotNil(t, out.Mutations)
|
||||||
|
require.NotNil(t, out.Mutations.RewriteUpstream)
|
||||||
|
assert.Equal(t, "bedrock-runtime.eu-central-1.amazonaws.com", out.Mutations.RewriteUpstream.Host,
|
||||||
|
"%s must reach the Bedrock provider, not the first authorised one", path)
|
||||||
|
|
||||||
|
nonInference, _ := metaValue(t, out.Metadata, middleware.KeyLLMNonInference)
|
||||||
|
assert.Equal(t, "true", nonInference, "%s carries no model to gate on", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRouter_BedrockNamespacedInferenceProfilesStripsPrefix pins that the
|
||||||
|
// optional gateway namespace is removed before the request goes upstream.
|
||||||
|
func TestRouter_BedrockNamespacedInferenceProfilesStripsPrefix(t *testing.T) {
|
||||||
|
mw := New(Config{Providers: []ProviderRoute{{
|
||||||
|
ID: "bedrock-prod",
|
||||||
|
Bedrock: true,
|
||||||
|
AllowedGroupIDs: []string{defaultTestGroup},
|
||||||
|
UpstreamScheme: "https",
|
||||||
|
UpstreamHost: "bedrock-runtime.eu-central-1.amazonaws.com",
|
||||||
|
}}})
|
||||||
|
|
||||||
|
out, err := mw.Invoke(context.Background(), newModellessInput("/bedrock/inference-profiles"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, out.Mutations)
|
||||||
|
require.NotNil(t, out.Mutations.RewriteUpstream)
|
||||||
|
assert.Equal(t, "/bedrock", out.Mutations.RewriteUpstream.StripPathPrefix,
|
||||||
|
"the namespace prefix must not reach the real Bedrock endpoint")
|
||||||
|
}
|
||||||
|
|||||||
@@ -194,11 +194,14 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar
|
|||||||
// need rewriting from the synth placeholder to a real upstream;
|
// need rewriting from the synth placeholder to a real upstream;
|
||||||
// clients such as Codex call GET /v1/models at startup to enumerate
|
// clients such as Codex call GET /v1/models at startup to enumerate
|
||||||
// availability and read a 403 as "model unavailable".
|
// availability and read a 403 as "model unavailable".
|
||||||
route, outcome := m.matchModelless(requestPath(in.URL), in.UserGroups)
|
route, outcome := m.matchModelless(reqPath, in.UserGroups)
|
||||||
switch outcome {
|
switch outcome {
|
||||||
case matchOutcomeFound:
|
case matchOutcomeFound:
|
||||||
out := m.allowWithRoute(route, surface, in.UserGroups)
|
out := m.allowWithRoute(route, surface, in.UserGroups)
|
||||||
out.Metadata = append(out.Metadata, middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"})
|
out.Metadata = append(out.Metadata, middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"})
|
||||||
|
if _, hadPrefix := splitBedrockNamespace(reqPath); hadPrefix && out.Mutations != nil && out.Mutations.RewriteUpstream != nil {
|
||||||
|
out.Mutations.RewriteUpstream.StripPathPrefix = bedrockNamespacePrefix
|
||||||
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
case matchOutcomeUnauthorised:
|
case matchOutcomeUnauthorised:
|
||||||
// A recognised model-less endpoint exists but no provider
|
// A recognised model-less endpoint exists but no provider
|
||||||
@@ -305,14 +308,24 @@ func (m *Middleware) matchRoute(model, vendor, reqPath string, userGroups []stri
|
|||||||
return best, matchOutcomeFound
|
return best, matchOutcomeFound
|
||||||
}
|
}
|
||||||
|
|
||||||
// isModelLessPath reports whether reqPath is a known OpenAI-shaped
|
// isModelLessPath reports whether reqPath is a known non-inference endpoint
|
||||||
// non-inference endpoint that legitimately carries no model in its
|
// that legitimately carries no model in its request (the model-listing
|
||||||
// request (the model-listing endpoints). These must route to an upstream
|
// endpoints). These must route to an upstream rather than deny, so model
|
||||||
// rather than deny, so model enumeration works end to end.
|
// enumeration works end to end.
|
||||||
func isModelLessPath(reqPath string) bool {
|
func isModelLessPath(reqPath string) bool {
|
||||||
return reqPath == "/v1/models" || strings.HasPrefix(reqPath, "/v1/models/")
|
return reqPath == "/v1/models" || strings.HasPrefix(reqPath, "/v1/models/")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isBedrockModelLessPath reports whether reqPath is a Bedrock
|
||||||
|
// inference-profile lookup, optionally behind the "/bedrock" gateway
|
||||||
|
// namespace. Clients read these at startup to resolve a configured profile
|
||||||
|
// to its underlying model. They carry no model of their own, so they route
|
||||||
|
// by path to a Bedrock provider rather than through the model table.
|
||||||
|
func isBedrockModelLessPath(reqPath string) bool {
|
||||||
|
native, _ := splitBedrockNamespace(reqPath)
|
||||||
|
return native == "/inference-profiles" || strings.HasPrefix(native, "/inference-profiles/")
|
||||||
|
}
|
||||||
|
|
||||||
// isVertexPath reports whether reqPath is a Google Vertex AI publisher
|
// isVertexPath reports whether reqPath is a Google Vertex AI publisher
|
||||||
// endpoint: /v1/projects/{project}/locations/{region}/publishers/{publisher}/
|
// endpoint: /v1/projects/{project}/locations/{region}/publishers/{publisher}/
|
||||||
// models/{model}:{action}. The model + vendor live in the path, so these
|
// models/{model}:{action}. The model + vendor live in the path, so these
|
||||||
@@ -444,18 +457,22 @@ func (m *Middleware) matchPathRoute(reqPath, model string, userGroups []string,
|
|||||||
// the caller, or matchOutcomeUnknownModel when the path isn't a recognised
|
// the caller, or matchOutcomeUnknownModel when the path isn't a recognised
|
||||||
// model-less endpoint.
|
// model-less endpoint.
|
||||||
func (m *Middleware) matchModelless(reqPath string, userGroups []string) (ProviderRoute, matchOutcome) {
|
func (m *Middleware) matchModelless(reqPath string, userGroups []string) (ProviderRoute, matchOutcome) {
|
||||||
if !isModelLessPath(reqPath) {
|
var eligible func(ProviderRoute) bool
|
||||||
return ProviderRoute{}, matchOutcomeUnknownModel
|
switch {
|
||||||
}
|
case isBedrockModelLessPath(reqPath):
|
||||||
var candidates []ProviderRoute
|
eligible = func(r ProviderRoute) bool { return r.Bedrock }
|
||||||
for _, route := range m.cfg.Providers {
|
case isModelLessPath(reqPath):
|
||||||
// Vertex/Bedrock are path-routed and don't serve OpenAI-style
|
// Vertex/Bedrock are path-routed and don't serve OpenAI-style
|
||||||
// model-listing endpoints; including them here could rewrite a
|
// model-listing endpoints; including them here could rewrite a
|
||||||
// GET /v1/models to an upstream that 404s it.
|
// GET /v1/models to an upstream that 404s it.
|
||||||
if route.Vertex || route.Bedrock {
|
eligible = func(r ProviderRoute) bool { return !r.Vertex && !r.Bedrock }
|
||||||
continue
|
default:
|
||||||
}
|
return ProviderRoute{}, matchOutcomeUnknownModel
|
||||||
if routeAuthorisesGroups(route, userGroups) {
|
}
|
||||||
|
|
||||||
|
var candidates []ProviderRoute
|
||||||
|
for _, route := range m.cfg.Providers {
|
||||||
|
if eligible(route) && routeAuthorisesGroups(route, userGroups) {
|
||||||
candidates = append(candidates, route)
|
candidates = append(candidates, route)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user