From 919e558614710e852fc7a40e02ce470aac705c6e Mon Sep 17 00:00:00 2001 From: mlsmaycon Date: Tue, 11 Aug 2026 02:45:23 +0000 Subject: [PATCH] [proxy] Select the LLM parser by request path before provider_id Gateway catalog entries pin provider_id "openai", and the same record serves Claude Code on /v1/messages. The parser preferred the pinned id over the path, so an Anthropic body was read with the OpenAI parser: on a streaming response the input tokens ride message_start nested under message, which that parser never reads, so input counted as zero. Both cache buckets were dropped, and pricing resolved against the openai surface where no claude-* model exists, skipping cost entirely. Detect from the path first and keep provider_id as the fallback for upstreams whose path carries no surface. The Kimi entry already leaves ParserID empty to work around this; the fallback ordering makes that unnecessary. --- .../builtin/llm_request_parser/middleware.go | 15 +++++++---- .../llm_request_parser/middleware_test.go | 25 +++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/proxy/internal/middleware/builtin/llm_request_parser/middleware.go b/proxy/internal/middleware/builtin/llm_request_parser/middleware.go index b4d1e16d4..7f6e80a73 100644 --- a/proxy/internal/middleware/builtin/llm_request_parser/middleware.go +++ b/proxy/internal/middleware/builtin/llm_request_parser/middleware.go @@ -72,9 +72,9 @@ func (middlewareImpl) Close() error { return nil } // Invoke detects the LLM provider, parses request facts, and emits // metadata. Always returns DecisionAllow; never errors. Provider -// selection prefers the configured providerID (synthesiser-stamped on -// agent-network targets) so requests routed to a custom upstream URL -// still resolve. Falls back to URL sniffing when no providerID is set. +// selection prefers the request path, falling back to the configured +// providerID (synthesiser-stamped on agent-network targets) so requests +// routed to a custom upstream URL still resolve. func (m middlewareImpl) Invoke(_ context.Context, in *middleware.Input) (*middleware.Output, error) { out := &middleware.Output{Decision: middleware.DecisionAllow} if in == nil { @@ -92,9 +92,14 @@ func (m middlewareImpl) Invoke(_ context.Context, in *middleware.Input) (*middle return m.invokeBedrock(in, br), nil } - parser, ok := llm.ParserByName(m.providerID) + // A path that names an API surface wins over the configured providerID: + // a gateway record pinned to "openai" still serves Claude Code on + // /v1/messages, and reading that body with the OpenAI parser loses the + // Anthropic usage block and prices the request on the wrong surface. + // providerID stays the fallback for upstreams whose path says nothing. + parser, ok := llm.DetectParser(extractPath(in.URL)) if !ok { - parser, ok = llm.DetectParser(extractPath(in.URL)) + parser, ok = llm.ParserByName(m.providerID) } if !ok { return out, nil diff --git a/proxy/internal/middleware/builtin/llm_request_parser/middleware_test.go b/proxy/internal/middleware/builtin/llm_request_parser/middleware_test.go index bc185b295..7afe545b9 100644 --- a/proxy/internal/middleware/builtin/llm_request_parser/middleware_test.go +++ b/proxy/internal/middleware/builtin/llm_request_parser/middleware_test.go @@ -230,6 +230,31 @@ func TestInvoke_ProviderIDConfigBypassesURLSniff(t *testing.T) { assert.Equal(t, "gpt-4o-mini", model) } +func TestInvoke_PathSurfaceBeatsProviderIDConfig(t *testing.T) { + // Gateway records (LiteLLM, Portkey, OpenRouter) pin provider_id + // "openai", but the same record serves Claude Code on /v1/messages. + // Parsing that body as OpenAI reads no usage off the Anthropic + // response and prices the request on a surface where no claude-* + // model exists, so the path has to win. + mw, err := Factory{}.New([]byte(`{"provider_id":"openai"}`)) + require.NoError(t, err, "factory must accept provider_id config") + + out, err := mw.Invoke(context.Background(), &middleware.Input{ + URL: "/v1/messages", + Body: []byte(`{"model":"claude-sonnet-5","stream":true,"messages":[{"role":"user","content":"Hi"}]}`), + }) + require.NoError(t, err) + require.NotNil(t, out) + + provider, ok := metaValue(t, out.Metadata, middleware.KeyLLMProvider) + require.True(t, ok, "provider must be emitted") + assert.Equal(t, "anthropic", provider, "the /v1/messages path selects the Anthropic surface") + + model, ok := metaValue(t, out.Metadata, middleware.KeyLLMModel) + require.True(t, ok, "model must be extracted") + assert.Equal(t, "claude-sonnet-5", model) +} + func TestInvoke_UnknownProviderIDFallsBackToURL(t *testing.T) { mw, err := Factory{}.New([]byte(`{"provider_id":"not-a-real-parser"}`)) require.NoError(t, err, "factory must accept any provider_id string")