mirror of
https://github.com/netbirdio/netbird.git
synced 2026-08-28 18:41:30 +02:00
3fac269506
Field report: a Bedrock Claude Sonnet 4.6 request showing 3 input / 1514 output tokens displayed a cost of $0.1372 instead of the expected $0.0227. Root cause analysis: the calculation is correct — the request's first call also wrote a ~30.5k-token prompt cache (cache_creation_input_tokens, billed at 1.25x input per AWS pricing), which is folded into total_tokens and the cost but not visible next to the input/output counts. Lock the pipeline down with tests so any real calculation regression fails loudly: - Add a provider cost matrix test driving the real proxy pipeline (llm_request_parser -> llm_response_parser -> cost_meter) with realistic wire fixtures for every metered surface (OpenAI JSON/SSE incl. cached subset, Anthropic JSON/SSE incl. cache buckets, Bedrock InvokeModel and Converse in both buffered and streaming form, Vertex path-routed, Kimi Anthropic-shape, unpriced gateway-prefixed ids) against the embedded default pricing table, asserting exact USD amounts derived from the published per-million prices. Covers the reported scenario byte-for-byte ($0.022719 bare, $0.137199 with the 30,528-token cache write) and would catch a per-token-instead-of-per-1k-chunk regression as a 1000x blowup. - Pin the management catalog (dashboard-displayed prices) to the proxy's embedded pricing table so the two can never drift apart silently. - Validate cost end-to-end in the live e2e provider matrix: each provider's ingested access-log row must carry a cost_usd matching the vendor's published per-1k rates applied to the row's token counts (cache-aware for the additive Anthropic/Bedrock buckets, zero for gateway-prefixed model ids the meter deliberately skips). - Fix a real under-billing bug the audit surfaced: the Bedrock Converse shapes report prompt-cache usage as camelCase cacheReadInputTokens / cacheWriteInputTokens, which neither the buffered parser nor the converse-stream metadata handler read - cached Converse traffic was metered without its cache buckets. Parse both fields into the same Usage buckets as the InvokeModel snake_case fields.
81 lines
4.2 KiB
Go
81 lines
4.2 KiB
Go
package llm
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestBedrockParser_ParseResponse_Invoke(t *testing.T) {
|
|
body := []byte(`{"usage":{"input_tokens":13,"output_tokens":5,"cache_read_input_tokens":2,"cache_creation_input_tokens":4}}`)
|
|
u, err := BedrockParser{}.ParseResponse(200, "application/json", body)
|
|
require.NoError(t, err)
|
|
require.Equal(t, int64(13), u.InputTokens, "invoke input tokens")
|
|
require.Equal(t, int64(5), u.OutputTokens, "invoke output tokens")
|
|
require.Equal(t, int64(2), u.CachedInputTokens, "invoke cache-read tokens")
|
|
require.Equal(t, int64(4), u.CacheCreationTokens, "invoke cache-creation tokens")
|
|
require.Equal(t, int64(13+5+2+4), u.TotalTokens, "invoke total is additive")
|
|
}
|
|
|
|
func TestBedrockParser_ParseResponse_Converse(t *testing.T) {
|
|
body := []byte(`{"output":{"message":{"content":[{"text":"pong"}]}},"usage":{"inputTokens":11,"outputTokens":3,"totalTokens":14}}`)
|
|
u, err := BedrockParser{}.ParseResponse(200, "application/json", body)
|
|
require.NoError(t, err)
|
|
require.Equal(t, int64(11), u.InputTokens, "converse camelCase input tokens")
|
|
require.Equal(t, int64(3), u.OutputTokens, "converse camelCase output tokens")
|
|
require.Equal(t, int64(14), u.TotalTokens, "converse uses provider total")
|
|
}
|
|
|
|
// TestBedrockParser_ParseResponse_ConverseCacheBuckets proves the Converse
|
|
// camelCase cache fields (cacheReadInputTokens / cacheWriteInputTokens) land
|
|
// in the same Usage buckets as the InvokeModel snake_case fields — the cost
|
|
// meter bills them, so dropping them silently under-counts cached requests.
|
|
func TestBedrockParser_ParseResponse_ConverseCacheBuckets(t *testing.T) {
|
|
body := []byte(`{"usage":{"inputTokens":11,"outputTokens":3,"cacheReadInputTokens":7,"cacheWriteInputTokens":9}}`)
|
|
u, err := BedrockParser{}.ParseResponse(200, "application/json", body)
|
|
require.NoError(t, err)
|
|
require.Equal(t, int64(11), u.InputTokens, "converse input tokens")
|
|
require.Equal(t, int64(3), u.OutputTokens, "converse output tokens")
|
|
require.Equal(t, int64(7), u.CachedInputTokens, "converse cache-read tokens")
|
|
require.Equal(t, int64(9), u.CacheCreationTokens, "converse cache-write tokens")
|
|
require.Equal(t, int64(11+3+7+9), u.TotalTokens, "total backfill is additive when the provider omits totalTokens")
|
|
}
|
|
|
|
func TestBedrockParser_ParseResponse_StreamingUnsupported(t *testing.T) {
|
|
_, err := BedrockParser{}.ParseResponse(200, "application/vnd.amazon.eventstream", []byte("binary"))
|
|
require.ErrorIs(t, err, ErrStreamingUnsupported, "event-stream must route to the streaming accumulator")
|
|
}
|
|
|
|
func TestBedrockParser_ParseResponse_NonSuccess(t *testing.T) {
|
|
_, err := BedrockParser{}.ParseResponse(404, "application/json", []byte(`{"message":"gated"}`))
|
|
require.ErrorIs(t, err, ErrNotLLMResponse, "non-200 is not an LLM response")
|
|
}
|
|
|
|
func TestBedrockParser_ExtractCompletion(t *testing.T) {
|
|
invoke := BedrockParser{}.ExtractCompletion(200, "application/json", []byte(`{"content":[{"text":"a"},{"text":"b"}]}`))
|
|
require.Equal(t, "a\nb", invoke, "invoke completion joins content parts")
|
|
|
|
converse := BedrockParser{}.ExtractCompletion(200, "application/json", []byte(`{"output":{"message":{"content":[{"text":"x"}]}}}`))
|
|
require.Equal(t, "x", converse, "converse completion reads output.message.content")
|
|
}
|
|
|
|
func TestBedrockParser_ExtractPrompt(t *testing.T) {
|
|
invoke := BedrockParser{}.ExtractPrompt([]byte(`{"messages":[{"role":"user","content":"hi"}]}`))
|
|
require.Equal(t, "user: hi", invoke, "invoke prompt reads anthropic content string")
|
|
|
|
converse := BedrockParser{}.ExtractPrompt([]byte(`{"messages":[{"role":"user","content":[{"text":"hello"}]}]}`))
|
|
require.Equal(t, "user: hello", converse, "converse prompt reads content parts")
|
|
}
|
|
|
|
func TestBedrockParser_DetectFromURL(t *testing.T) {
|
|
require.True(t, BedrockParser{}.DetectFromURL("/model/eu.anthropic.claude/invoke"), "invoke path")
|
|
require.True(t, BedrockParser{}.DetectFromURL("/model/x/converse-stream"), "converse-stream path")
|
|
require.False(t, BedrockParser{}.DetectFromURL("/v1/chat/completions"), "openai path is not bedrock")
|
|
}
|
|
|
|
func TestBedrockParser_RegisteredByName(t *testing.T) {
|
|
p, ok := ParserByName(ProviderNameBedrock)
|
|
require.True(t, ok, "bedrock parser is registered")
|
|
require.Equal(t, ProviderNameBedrock, p.ProviderName())
|
|
}
|