mirror of
https://github.com/netbirdio/netbird.git
synced 2026-08-24 16:41:30 +02:00
[proxy] Select the LLM parser by request path before provider_id
Gateway catalog entries pin provider_id "openai", and the same record serves Claude Code on /v1/messages. The parser preferred the pinned id over the path, so an Anthropic body was read with the OpenAI parser: on a streaming response the input tokens ride message_start nested under message, which that parser never reads, so input counted as zero. Both cache buckets were dropped, and pricing resolved against the openai surface where no claude-* model exists, skipping cost entirely. Detect from the path first and keep provider_id as the fallback for upstreams whose path carries no surface. The Kimi entry already leaves ParserID empty to work around this; the fallback ordering makes that unnecessary.
This commit is contained in:
@@ -72,9 +72,9 @@ func (middlewareImpl) Close() error { return nil }
|
||||
|
||||
// Invoke detects the LLM provider, parses request facts, and emits
|
||||
// metadata. Always returns DecisionAllow; never errors. Provider
|
||||
// selection prefers the configured providerID (synthesiser-stamped on
|
||||
// agent-network targets) so requests routed to a custom upstream URL
|
||||
// still resolve. Falls back to URL sniffing when no providerID is set.
|
||||
// selection prefers the request path, falling back to the configured
|
||||
// providerID (synthesiser-stamped on agent-network targets) so requests
|
||||
// routed to a custom upstream URL still resolve.
|
||||
func (m middlewareImpl) Invoke(_ context.Context, in *middleware.Input) (*middleware.Output, error) {
|
||||
out := &middleware.Output{Decision: middleware.DecisionAllow}
|
||||
if in == nil {
|
||||
@@ -92,9 +92,14 @@ func (m middlewareImpl) Invoke(_ context.Context, in *middleware.Input) (*middle
|
||||
return m.invokeBedrock(in, br), nil
|
||||
}
|
||||
|
||||
parser, ok := llm.ParserByName(m.providerID)
|
||||
// A path that names an API surface wins over the configured providerID:
|
||||
// a gateway record pinned to "openai" still serves Claude Code on
|
||||
// /v1/messages, and reading that body with the OpenAI parser loses the
|
||||
// Anthropic usage block and prices the request on the wrong surface.
|
||||
// providerID stays the fallback for upstreams whose path says nothing.
|
||||
parser, ok := llm.DetectParser(extractPath(in.URL))
|
||||
if !ok {
|
||||
parser, ok = llm.DetectParser(extractPath(in.URL))
|
||||
parser, ok = llm.ParserByName(m.providerID)
|
||||
}
|
||||
if !ok {
|
||||
return out, nil
|
||||
|
||||
@@ -230,6 +230,31 @@ func TestInvoke_ProviderIDConfigBypassesURLSniff(t *testing.T) {
|
||||
assert.Equal(t, "gpt-4o-mini", model)
|
||||
}
|
||||
|
||||
func TestInvoke_PathSurfaceBeatsProviderIDConfig(t *testing.T) {
|
||||
// Gateway records (LiteLLM, Portkey, OpenRouter) pin provider_id
|
||||
// "openai", but the same record serves Claude Code on /v1/messages.
|
||||
// Parsing that body as OpenAI reads no usage off the Anthropic
|
||||
// response and prices the request on a surface where no claude-*
|
||||
// model exists, so the path has to win.
|
||||
mw, err := Factory{}.New([]byte(`{"provider_id":"openai"}`))
|
||||
require.NoError(t, err, "factory must accept provider_id config")
|
||||
|
||||
out, err := mw.Invoke(context.Background(), &middleware.Input{
|
||||
URL: "/v1/messages",
|
||||
Body: []byte(`{"model":"claude-sonnet-5","stream":true,"messages":[{"role":"user","content":"Hi"}]}`),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
|
||||
provider, ok := metaValue(t, out.Metadata, middleware.KeyLLMProvider)
|
||||
require.True(t, ok, "provider must be emitted")
|
||||
assert.Equal(t, "anthropic", provider, "the /v1/messages path selects the Anthropic surface")
|
||||
|
||||
model, ok := metaValue(t, out.Metadata, middleware.KeyLLMModel)
|
||||
require.True(t, ok, "model must be extracted")
|
||||
assert.Equal(t, "claude-sonnet-5", model)
|
||||
}
|
||||
|
||||
func TestInvoke_UnknownProviderIDFallsBackToURL(t *testing.T) {
|
||||
mw, err := Factory{}.New([]byte(`{"provider_id":"not-a-real-parser"}`))
|
||||
require.NoError(t, err, "factory must accept any provider_id string")
|
||||
|
||||
Reference in New Issue
Block a user