diff --git a/proxy/internal/accesslog/logger.go b/proxy/internal/accesslog/logger.go index d47c71ca4..ed6058b70 100644 --- a/proxy/internal/accesslog/logger.go +++ b/proxy/internal/accesslog/logger.go @@ -221,14 +221,17 @@ func (l *Logger) allowDenyLog(serviceID types.ServiceID, reason string) bool { // proxy/internal/middleware/keys.go — only the dimensions management needs to // record a usage row (provider / model / tokens / cost / groups). var usageMetadataKeys = map[string]struct{}{ - "llm.provider": {}, - "llm.model": {}, - "llm.resolved_provider_id": {}, - "llm.input_tokens": {}, - "llm.output_tokens": {}, - "llm.total_tokens": {}, - "cost.usd_total": {}, - "llm.authorising_groups": {}, + "llm.provider": {}, + "llm.model": {}, + "llm.resolved_provider_id": {}, + "llm.input_tokens": {}, + "llm.output_tokens": {}, + "llm.total_tokens": {}, + "llm.cached_input_tokens": {}, + "llm.cache_creation_tokens": {}, + "cost.usd_total": {}, + "cost.usd_cache": {}, + "llm.authorising_groups": {}, } // stripAgentNetworkEntryForUsage returns the entry reduced to what's needed to diff --git a/proxy/internal/llm/pricing/pricing.go b/proxy/internal/llm/pricing/pricing.go index 09afec5ff..5320245d5 100644 --- a/proxy/internal/llm/pricing/pricing.go +++ b/proxy/internal/llm/pricing/pricing.go @@ -128,6 +128,20 @@ type Table struct { // - Other providers: cached and cacheCreation are ignored; cost is // inTokens*InputPer1K + outTokens*OutputPer1K. func (t *Table) Cost(provider, model string, inTokens, outTokens, cachedInput, cacheCreation int64) (float64, bool) { + c, ok := t.Costs(provider, model, inTokens, outTokens, cachedInput, cacheCreation) + return c.TotalUSD, ok +} + +// Costs is a per-request cost split. CacheUSD is the portion of TotalUSD billed for +// prompt-cache buckets and is always <= TotalUSD. +type Costs struct { + TotalUSD float64 + CacheUSD float64 +} + +// Costs returns the estimated USD cost split for the given token counts, with +// the same semantics as Cost. +func (t *Table) Costs(provider, model string, inTokens, outTokens, cachedInput, cacheCreation int64) (Costs, bool) { // Clamp negatives to zero before any pricing math so a malformed // upstream count can never produce a negative cost. if inTokens < 0 { @@ -143,15 +157,15 @@ func (t *Table) Cost(provider, model string, inTokens, outTokens, cachedInput, c cacheCreation = 0 } if t == nil { - return 0, false + return Costs{}, false } byModel, ok := t.entries[provider] if !ok { - return 0, false + return Costs{}, false } entry, ok := byModel[model] if !ok { - return 0, false + return Costs{}, false } output := (float64(outTokens) / 1000.0) * entry.OutputPer1K switch provider { @@ -168,7 +182,7 @@ func (t *Table) Cost(provider, model string, inTokens, outTokens, cachedInput, c } nonCached := float64(inTokens-clamped) / 1000.0 * entry.InputPer1K cached := float64(clamped) / 1000.0 * cachedRate - return nonCached + cached + output, true + return Costs{TotalUSD: nonCached + cached + output, CacheUSD: cached}, true case "anthropic", "bedrock": // Bedrock-Anthropic returns the same additive cache buckets as // first-party Anthropic; non-Anthropic Bedrock models simply report @@ -184,10 +198,10 @@ func (t *Table) Cost(provider, model string, inTokens, outTokens, cachedInput, c input := float64(inTokens) / 1000.0 * entry.InputPer1K read := float64(cachedInput) / 1000.0 * readRate create := float64(cacheCreation) / 1000.0 * createRate - return input + read + create + output, true + return Costs{TotalUSD: input + read + create + output, CacheUSD: read + create}, true default: input := float64(inTokens) / 1000.0 * entry.InputPer1K - return input + output, true + return Costs{TotalUSD: input + output}, true } } diff --git a/proxy/internal/middleware/builtin/cost_meter/middleware.go b/proxy/internal/middleware/builtin/cost_meter/middleware.go index 4da620310..d0d715d9e 100644 --- a/proxy/internal/middleware/builtin/cost_meter/middleware.go +++ b/proxy/internal/middleware/builtin/cost_meter/middleware.go @@ -33,6 +33,7 @@ const ( var metadataKeys = []string{ middleware.KeyCostUSDTotal, + middleware.KeyCostUSDCache, middleware.KeyCostSkipped, } @@ -140,14 +141,15 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar } table := m.loader.Get() - cost, ok := table.Cost(provider, model, inTokens, outTokens, cachedTokens, cacheCreationTokens) + costs, ok := table.Costs(provider, model, inTokens, outTokens, cachedTokens, cacheCreationTokens) if !ok { out.Metadata = skip(skipUnknownModel) return out, nil } out.Metadata = []middleware.KV{ - {Key: middleware.KeyCostUSDTotal, Value: fmt.Sprintf("%.6f", cost)}, + {Key: middleware.KeyCostUSDTotal, Value: fmt.Sprintf("%.6f", costs.TotalUSD)}, + {Key: middleware.KeyCostUSDCache, Value: fmt.Sprintf("%.6f", costs.CacheUSD)}, } return out, nil } diff --git a/proxy/internal/middleware/builtin/cost_meter/middleware_test.go b/proxy/internal/middleware/builtin/cost_meter/middleware_test.go index d1c161cab..37811be40 100644 --- a/proxy/internal/middleware/builtin/cost_meter/middleware_test.go +++ b/proxy/internal/middleware/builtin/cost_meter/middleware_test.go @@ -67,7 +67,7 @@ func TestMiddleware_StaticSurface(t *testing.T) { assert.NoError(t, mw.Close(), "Close on stateless middleware is a no-op") keys := mw.MetadataKeys() - expected := []string{middleware.KeyCostUSDTotal, middleware.KeyCostSkipped} + expected := []string{middleware.KeyCostUSDTotal, middleware.KeyCostUSDCache, middleware.KeyCostSkipped} assert.Equal(t, expected, keys, "metadata key allowlist must match the spec") } @@ -359,6 +359,11 @@ func TestInvoke_OpenAICachedSubsetDiscount(t *testing.T) { // 250 non-cached at 0.0025/1k + 750 cached at 0.00125/1k + 500 output at 0.01/1k. assert.Equal(t, "0.006563", value, "cached subset must be billed at the discount rate, non-cached at the full rate; never double-billed") + + cache, ok := metaValue(t, out.Metadata, middleware.KeyCostUSDCache) + require.True(t, ok, "cost.usd_cache must be emitted alongside cost.usd_total") + // 750 cached at 0.00125/1k = 0.0009375, rendered 0.000937 by %.6f on the binary float. + assert.Equal(t, "0.000937", cache, "cache cost is the discounted cost of the cached subset") } // TestInvoke_AnthropicCacheBucketsAdditive proves the Anthropic @@ -387,6 +392,11 @@ func TestInvoke_AnthropicCacheBucketsAdditive(t *testing.T) { // = 0.000768 + 0.0002304 + 0.00192 + 0.003 = 0.0059184 → "0.005918" with 6-decimal format. assert.Equal(t, "0.005918", value, "each Anthropic input bucket must bill at its own rate — cache_read cheap, cache_creation expensive, regular input mid") + + cache, ok := metaValue(t, out.Metadata, middleware.KeyCostUSDCache) + require.True(t, ok, "cost.usd_cache must be emitted alongside cost.usd_total") + // 768 cache_read * 0.0003 + 512 cache_creation * 0.00375 = 0.0021504 → "0.002150". + assert.Equal(t, "0.002150", cache, "cache cost sums the read and creation buckets") } // TestInvoke_CachedTokensAbsentFallsBackToBaseFormula covers the diff --git a/proxy/internal/middleware/keys.go b/proxy/internal/middleware/keys.go index 9c584ad82..c031b0364 100644 --- a/proxy/internal/middleware/keys.go +++ b/proxy/internal/middleware/keys.go @@ -77,6 +77,8 @@ const ( // Cost metering (emitted by cost_meter). KeyCostUSDTotal = "cost.usd_total" + // KeyCostUSDCache is the portion of cost.usd_total billed for prompt-cache buckets (cache read/creation, or OpenAI's cached input subset). + KeyCostUSDCache = "cost.usd_cache" KeyCostSkipped = "cost.skipped" // Framework-emitted error markers. Use the mw..* prefix to