mirror of
https://github.com/netbirdio/netbird.git
synced 2026-08-24 16:41:30 +02:00
[proxy] Forward the Anthropic connection-warming probe
Clients send HEAD /api/hello before their first inference request to open the upstream connection early. The path carries no model, so it denied as not-routable and each session start left a policy rejection in the access log for a request that was never a policy question. Treat it as a model-less endpoint. Forwarding it warms the connection the first real request will use, which is what the probe is for.
This commit is contained in:
@@ -308,12 +308,20 @@ func (m *Middleware) matchRoute(model, vendor, reqPath string, userGroups []stri
|
||||
return best, matchOutcomeFound
|
||||
}
|
||||
|
||||
// connectionWarmPath is the probe Anthropic clients send before their first
|
||||
// inference request to open the upstream connection early. Forwarding it
|
||||
// warms the connection the request will actually use; denying it only fills
|
||||
// the access log with rejections at every session start.
|
||||
const connectionWarmPath = "/api/hello"
|
||||
|
||||
// isModelLessPath reports whether reqPath is a known non-inference endpoint
|
||||
// that legitimately carries no model in its request (the model-listing
|
||||
// endpoints). These must route to an upstream rather than deny, so model
|
||||
// enumeration works end to end.
|
||||
// that legitimately carries no model in its request (model listing and the
|
||||
// connection-warming probe). These must route to an upstream rather than
|
||||
// deny, so model enumeration works end to end.
|
||||
func isModelLessPath(reqPath string) bool {
|
||||
return reqPath == "/v1/models" || strings.HasPrefix(reqPath, "/v1/models/")
|
||||
return reqPath == "/v1/models" ||
|
||||
strings.HasPrefix(reqPath, "/v1/models/") ||
|
||||
reqPath == connectionWarmPath
|
||||
}
|
||||
|
||||
// isBedrockModelLessPath reports whether reqPath is a Bedrock
|
||||
|
||||
@@ -2,6 +2,7 @@ package llm_router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -905,3 +906,32 @@ func TestRouter_DatedAnthropicModelRoutes(t *testing.T) {
|
||||
require.NotNil(t, out.Mutations.RewriteUpstream)
|
||||
assert.Equal(t, "api.anthropic.com", out.Mutations.RewriteUpstream.Host)
|
||||
}
|
||||
|
||||
// TestRouter_ConnectionWarmProbeRoutes covers the HEAD /api/hello probe an
|
||||
// Anthropic client sends before its first request. Forwarding it warms the
|
||||
// connection that request will use; denying it only wrote a rejection into
|
||||
// the access log at every session start.
|
||||
func TestRouter_ConnectionWarmProbeRoutes(t *testing.T) {
|
||||
mw := New(Config{Providers: []ProviderRoute{{
|
||||
ID: "anthropic-prod",
|
||||
Vendor: "anthropic",
|
||||
Models: []string{"claude-sonnet-5"},
|
||||
AllowedGroupIDs: []string{defaultTestGroup},
|
||||
UpstreamScheme: "https",
|
||||
UpstreamHost: "api.anthropic.com",
|
||||
}}})
|
||||
|
||||
in := newModellessInput("/api/hello")
|
||||
in.Method = http.MethodHead
|
||||
|
||||
out, err := mw.Invoke(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "the warm-up probe must reach the upstream")
|
||||
require.NotNil(t, out.Mutations)
|
||||
require.NotNil(t, out.Mutations.RewriteUpstream)
|
||||
assert.Equal(t, "api.anthropic.com", out.Mutations.RewriteUpstream.Host)
|
||||
|
||||
nonInference, _ := metaValue(t, out.Metadata, middleware.KeyLLMNonInference)
|
||||
assert.Equal(t, "true", nonInference, "the probe carries no model to gate on")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user