[proxy] Select the LLM parser by request path before provider_id

Gateway catalog entries pin provider_id "openai", and the same record
serves Claude Code on /v1/messages. The parser preferred the pinned id
over the path, so an Anthropic body was read with the OpenAI parser: on a
streaming response the input tokens ride message_start nested under
message, which that parser never reads, so input counted as zero. Both
cache buckets were dropped, and pricing resolved against the openai
surface where no claude-* model exists, skipping cost entirely.

Detect from the path first and keep provider_id as the fallback for
upstreams whose path carries no surface. The Kimi entry already leaves
ParserID empty to work around this; the fallback ordering makes that
unnecessary.
This commit is contained in:
mlsmaycon
2026-08-11 02:45:23 +00:00
committed by GitHub
parent 9ad1d9a39a
commit ed7eec7c3e
2 changed files with 35 additions and 5 deletions

View File

@@ -72,9 +72,9 @@ func (middlewareImpl) Close() error { return nil }
// Invoke detects the LLM provider, parses request facts, and emits
// metadata. Always returns DecisionAllow; never errors. Provider
// selection prefers the configured providerID (synthesiser-stamped on
// agent-network targets) so requests routed to a custom upstream URL
// still resolve. Falls back to URL sniffing when no providerID is set.
// selection prefers the request path, falling back to the configured
// providerID (synthesiser-stamped on agent-network targets) so requests
// routed to a custom upstream URL still resolve.
func (m middlewareImpl) Invoke(_ context.Context, in *middleware.Input) (*middleware.Output, error) {
out := &middleware.Output{Decision: middleware.DecisionAllow}
if in == nil {
@@ -92,9 +92,14 @@ func (m middlewareImpl) Invoke(_ context.Context, in *middleware.Input) (*middle
return m.invokeBedrock(in, br), nil
}
parser, ok := llm.ParserByName(m.providerID)
// A path that names an API surface wins over the configured providerID:
// a gateway record pinned to "openai" still serves Claude Code on
// /v1/messages, and reading that body with the OpenAI parser loses the
// Anthropic usage block and prices the request on the wrong surface.
// providerID stays the fallback for upstreams whose path says nothing.
parser, ok := llm.DetectParser(extractPath(in.URL))
if !ok {
parser, ok = llm.DetectParser(extractPath(in.URL))
parser, ok = llm.ParserByName(m.providerID)
}
if !ok {
return out, nil

View File

@@ -230,6 +230,31 @@ func TestInvoke_ProviderIDConfigBypassesURLSniff(t *testing.T) {
assert.Equal(t, "gpt-4o-mini", model)
}
func TestInvoke_PathSurfaceBeatsProviderIDConfig(t *testing.T) {
// Gateway records (LiteLLM, Portkey, OpenRouter) pin provider_id
// "openai", but the same record serves Claude Code on /v1/messages.
// Parsing that body as OpenAI reads no usage off the Anthropic
// response and prices the request on a surface where no claude-*
// model exists, so the path has to win.
mw, err := Factory{}.New([]byte(`{"provider_id":"openai"}`))
require.NoError(t, err, "factory must accept provider_id config")
out, err := mw.Invoke(context.Background(), &middleware.Input{
URL: "/v1/messages",
Body: []byte(`{"model":"claude-sonnet-5","stream":true,"messages":[{"role":"user","content":"Hi"}]}`),
})
require.NoError(t, err)
require.NotNil(t, out)
provider, ok := metaValue(t, out.Metadata, middleware.KeyLLMProvider)
require.True(t, ok, "provider must be emitted")
assert.Equal(t, "anthropic", provider, "the /v1/messages path selects the Anthropic surface")
model, ok := metaValue(t, out.Metadata, middleware.KeyLLMModel)
require.True(t, ok, "model must be extracted")
assert.Equal(t, "claude-sonnet-5", model)
}
func TestInvoke_UnknownProviderIDFallsBackToURL(t *testing.T) {
mw, err := Factory{}.New([]byte(`{"provider_id":"not-a-real-parser"}`))
require.NoError(t, err, "factory must accept any provider_id string")