mirror of
https://github.com/netbirdio/netbird.git
synced 2026-10-09 06:59:08 +02:00
[proxy] Select the LLM parser by request path before provider_id
Gateway catalog entries pin provider_id "openai", and the same record serves Claude Code on /v1/messages. The parser preferred the pinned id over the path, so an Anthropic body was read with the OpenAI parser: on a streaming response the input tokens ride message_start nested under message, which that parser never reads, so input counted as zero. Both cache buckets were dropped, and pricing resolved against the openai surface where no claude-* model exists, skipping cost entirely. Detect from the path first and keep provider_id as the fallback for upstreams whose path carries no surface. The Kimi entry already leaves ParserID empty to work around this; the fallback ordering makes that unnecessary.
This commit is contained in:
@@ -72,9 +72,9 @@ func (middlewareImpl) Close() error { return nil }
|
|||||||
|
|
||||||
// Invoke detects the LLM provider, parses request facts, and emits
|
// Invoke detects the LLM provider, parses request facts, and emits
|
||||||
// metadata. Always returns DecisionAllow; never errors. Provider
|
// metadata. Always returns DecisionAllow; never errors. Provider
|
||||||
// selection prefers the configured providerID (synthesiser-stamped on
|
// selection prefers the request path, falling back to the configured
|
||||||
// agent-network targets) so requests routed to a custom upstream URL
|
// providerID (synthesiser-stamped on agent-network targets) so requests
|
||||||
// still resolve. Falls back to URL sniffing when no providerID is set.
|
// routed to a custom upstream URL still resolve.
|
||||||
func (m middlewareImpl) Invoke(_ context.Context, in *middleware.Input) (*middleware.Output, error) {
|
func (m middlewareImpl) Invoke(_ context.Context, in *middleware.Input) (*middleware.Output, error) {
|
||||||
out := &middleware.Output{Decision: middleware.DecisionAllow}
|
out := &middleware.Output{Decision: middleware.DecisionAllow}
|
||||||
if in == nil {
|
if in == nil {
|
||||||
@@ -92,9 +92,14 @@ func (m middlewareImpl) Invoke(_ context.Context, in *middleware.Input) (*middle
|
|||||||
return m.invokeBedrock(in, br), nil
|
return m.invokeBedrock(in, br), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
parser, ok := llm.ParserByName(m.providerID)
|
// A path that names an API surface wins over the configured providerID:
|
||||||
|
// a gateway record pinned to "openai" still serves Claude Code on
|
||||||
|
// /v1/messages, and reading that body with the OpenAI parser loses the
|
||||||
|
// Anthropic usage block and prices the request on the wrong surface.
|
||||||
|
// providerID stays the fallback for upstreams whose path says nothing.
|
||||||
|
parser, ok := llm.DetectParser(extractPath(in.URL))
|
||||||
if !ok {
|
if !ok {
|
||||||
parser, ok = llm.DetectParser(extractPath(in.URL))
|
parser, ok = llm.ParserByName(m.providerID)
|
||||||
}
|
}
|
||||||
if !ok {
|
if !ok {
|
||||||
return out, nil
|
return out, nil
|
||||||
|
|||||||
@@ -230,6 +230,31 @@ func TestInvoke_ProviderIDConfigBypassesURLSniff(t *testing.T) {
|
|||||||
assert.Equal(t, "gpt-4o-mini", model)
|
assert.Equal(t, "gpt-4o-mini", model)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInvoke_PathSurfaceBeatsProviderIDConfig(t *testing.T) {
|
||||||
|
// Gateway records (LiteLLM, Portkey, OpenRouter) pin provider_id
|
||||||
|
// "openai", but the same record serves Claude Code on /v1/messages.
|
||||||
|
// Parsing that body as OpenAI reads no usage off the Anthropic
|
||||||
|
// response and prices the request on a surface where no claude-*
|
||||||
|
// model exists, so the path has to win.
|
||||||
|
mw, err := Factory{}.New([]byte(`{"provider_id":"openai"}`))
|
||||||
|
require.NoError(t, err, "factory must accept provider_id config")
|
||||||
|
|
||||||
|
out, err := mw.Invoke(context.Background(), &middleware.Input{
|
||||||
|
URL: "/v1/messages",
|
||||||
|
Body: []byte(`{"model":"claude-sonnet-5","stream":true,"messages":[{"role":"user","content":"Hi"}]}`),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, out)
|
||||||
|
|
||||||
|
provider, ok := metaValue(t, out.Metadata, middleware.KeyLLMProvider)
|
||||||
|
require.True(t, ok, "provider must be emitted")
|
||||||
|
assert.Equal(t, "anthropic", provider, "the /v1/messages path selects the Anthropic surface")
|
||||||
|
|
||||||
|
model, ok := metaValue(t, out.Metadata, middleware.KeyLLMModel)
|
||||||
|
require.True(t, ok, "model must be extracted")
|
||||||
|
assert.Equal(t, "claude-sonnet-5", model)
|
||||||
|
}
|
||||||
|
|
||||||
func TestInvoke_UnknownProviderIDFallsBackToURL(t *testing.T) {
|
func TestInvoke_UnknownProviderIDFallsBackToURL(t *testing.T) {
|
||||||
mw, err := Factory{}.New([]byte(`{"provider_id":"not-a-real-parser"}`))
|
mw, err := Factory{}.New([]byte(`{"provider_id":"not-a-real-parser"}`))
|
||||||
require.NoError(t, err, "factory must accept any provider_id string")
|
require.NoError(t, err, "factory must accept any provider_id string")
|
||||||
|
|||||||
Reference in New Issue
Block a user