mirror of
https://github.com/netbirdio/netbird.git
synced 2026-09-01 04:21:29 +02:00
[proxy] split prompt-cache cost out of the metered total
Pricing now returns the cache portion of the request cost next to the total (cache read + creation for Anthropic-shape providers, the discounted cached subset for OpenAI). cost_meter emits it as cost.usd_cache, and the usage-only access-log strip keeps the cache token counts and cache cost so management can persist them.
This commit is contained in:
@@ -221,14 +221,17 @@ func (l *Logger) allowDenyLog(serviceID types.ServiceID, reason string) bool {
|
||||
// proxy/internal/middleware/keys.go — only the dimensions management needs to
|
||||
// record a usage row (provider / model / tokens / cost / groups).
|
||||
var usageMetadataKeys = map[string]struct{}{
|
||||
"llm.provider": {},
|
||||
"llm.model": {},
|
||||
"llm.resolved_provider_id": {},
|
||||
"llm.input_tokens": {},
|
||||
"llm.output_tokens": {},
|
||||
"llm.total_tokens": {},
|
||||
"cost.usd_total": {},
|
||||
"llm.authorising_groups": {},
|
||||
"llm.provider": {},
|
||||
"llm.model": {},
|
||||
"llm.resolved_provider_id": {},
|
||||
"llm.input_tokens": {},
|
||||
"llm.output_tokens": {},
|
||||
"llm.total_tokens": {},
|
||||
"llm.cached_input_tokens": {},
|
||||
"llm.cache_creation_tokens": {},
|
||||
"cost.usd_total": {},
|
||||
"cost.usd_cache": {},
|
||||
"llm.authorising_groups": {},
|
||||
}
|
||||
|
||||
// stripAgentNetworkEntryForUsage returns the entry reduced to what's needed to
|
||||
|
||||
@@ -128,6 +128,20 @@ type Table struct {
|
||||
// - Other providers: cached and cacheCreation are ignored; cost is
|
||||
// inTokens*InputPer1K + outTokens*OutputPer1K.
|
||||
func (t *Table) Cost(provider, model string, inTokens, outTokens, cachedInput, cacheCreation int64) (float64, bool) {
|
||||
c, ok := t.Costs(provider, model, inTokens, outTokens, cachedInput, cacheCreation)
|
||||
return c.TotalUSD, ok
|
||||
}
|
||||
|
||||
// Costs is a per-request cost split. CacheUSD is the portion of TotalUSD billed for
|
||||
// prompt-cache buckets and is always <= TotalUSD.
|
||||
type Costs struct {
|
||||
TotalUSD float64
|
||||
CacheUSD float64
|
||||
}
|
||||
|
||||
// Costs returns the estimated USD cost split for the given token counts, with
|
||||
// the same semantics as Cost.
|
||||
func (t *Table) Costs(provider, model string, inTokens, outTokens, cachedInput, cacheCreation int64) (Costs, bool) {
|
||||
// Clamp negatives to zero before any pricing math so a malformed
|
||||
// upstream count can never produce a negative cost.
|
||||
if inTokens < 0 {
|
||||
@@ -143,15 +157,15 @@ func (t *Table) Cost(provider, model string, inTokens, outTokens, cachedInput, c
|
||||
cacheCreation = 0
|
||||
}
|
||||
if t == nil {
|
||||
return 0, false
|
||||
return Costs{}, false
|
||||
}
|
||||
byModel, ok := t.entries[provider]
|
||||
if !ok {
|
||||
return 0, false
|
||||
return Costs{}, false
|
||||
}
|
||||
entry, ok := byModel[model]
|
||||
if !ok {
|
||||
return 0, false
|
||||
return Costs{}, false
|
||||
}
|
||||
output := (float64(outTokens) / 1000.0) * entry.OutputPer1K
|
||||
switch provider {
|
||||
@@ -168,7 +182,7 @@ func (t *Table) Cost(provider, model string, inTokens, outTokens, cachedInput, c
|
||||
}
|
||||
nonCached := float64(inTokens-clamped) / 1000.0 * entry.InputPer1K
|
||||
cached := float64(clamped) / 1000.0 * cachedRate
|
||||
return nonCached + cached + output, true
|
||||
return Costs{TotalUSD: nonCached + cached + output, CacheUSD: cached}, true
|
||||
case "anthropic", "bedrock":
|
||||
// Bedrock-Anthropic returns the same additive cache buckets as
|
||||
// first-party Anthropic; non-Anthropic Bedrock models simply report
|
||||
@@ -184,10 +198,10 @@ func (t *Table) Cost(provider, model string, inTokens, outTokens, cachedInput, c
|
||||
input := float64(inTokens) / 1000.0 * entry.InputPer1K
|
||||
read := float64(cachedInput) / 1000.0 * readRate
|
||||
create := float64(cacheCreation) / 1000.0 * createRate
|
||||
return input + read + create + output, true
|
||||
return Costs{TotalUSD: input + read + create + output, CacheUSD: read + create}, true
|
||||
default:
|
||||
input := float64(inTokens) / 1000.0 * entry.InputPer1K
|
||||
return input + output, true
|
||||
return Costs{TotalUSD: input + output}, true
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ const (
|
||||
|
||||
var metadataKeys = []string{
|
||||
middleware.KeyCostUSDTotal,
|
||||
middleware.KeyCostUSDCache,
|
||||
middleware.KeyCostSkipped,
|
||||
}
|
||||
|
||||
@@ -140,14 +141,15 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar
|
||||
}
|
||||
|
||||
table := m.loader.Get()
|
||||
cost, ok := table.Cost(provider, model, inTokens, outTokens, cachedTokens, cacheCreationTokens)
|
||||
costs, ok := table.Costs(provider, model, inTokens, outTokens, cachedTokens, cacheCreationTokens)
|
||||
if !ok {
|
||||
out.Metadata = skip(skipUnknownModel)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
out.Metadata = []middleware.KV{
|
||||
{Key: middleware.KeyCostUSDTotal, Value: fmt.Sprintf("%.6f", cost)},
|
||||
{Key: middleware.KeyCostUSDTotal, Value: fmt.Sprintf("%.6f", costs.TotalUSD)},
|
||||
{Key: middleware.KeyCostUSDCache, Value: fmt.Sprintf("%.6f", costs.CacheUSD)},
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -67,7 +67,7 @@ func TestMiddleware_StaticSurface(t *testing.T) {
|
||||
assert.NoError(t, mw.Close(), "Close on stateless middleware is a no-op")
|
||||
|
||||
keys := mw.MetadataKeys()
|
||||
expected := []string{middleware.KeyCostUSDTotal, middleware.KeyCostSkipped}
|
||||
expected := []string{middleware.KeyCostUSDTotal, middleware.KeyCostUSDCache, middleware.KeyCostSkipped}
|
||||
assert.Equal(t, expected, keys, "metadata key allowlist must match the spec")
|
||||
}
|
||||
|
||||
@@ -359,6 +359,11 @@ func TestInvoke_OpenAICachedSubsetDiscount(t *testing.T) {
|
||||
// 250 non-cached at 0.0025/1k + 750 cached at 0.00125/1k + 500 output at 0.01/1k.
|
||||
assert.Equal(t, "0.006563", value,
|
||||
"cached subset must be billed at the discount rate, non-cached at the full rate; never double-billed")
|
||||
|
||||
cache, ok := metaValue(t, out.Metadata, middleware.KeyCostUSDCache)
|
||||
require.True(t, ok, "cost.usd_cache must be emitted alongside cost.usd_total")
|
||||
// 750 cached at 0.00125/1k = 0.0009375, rendered 0.000937 by %.6f on the binary float.
|
||||
assert.Equal(t, "0.000937", cache, "cache cost is the discounted cost of the cached subset")
|
||||
}
|
||||
|
||||
// TestInvoke_AnthropicCacheBucketsAdditive proves the Anthropic
|
||||
@@ -387,6 +392,11 @@ func TestInvoke_AnthropicCacheBucketsAdditive(t *testing.T) {
|
||||
// = 0.000768 + 0.0002304 + 0.00192 + 0.003 = 0.0059184 → "0.005918" with 6-decimal format.
|
||||
assert.Equal(t, "0.005918", value,
|
||||
"each Anthropic input bucket must bill at its own rate — cache_read cheap, cache_creation expensive, regular input mid")
|
||||
|
||||
cache, ok := metaValue(t, out.Metadata, middleware.KeyCostUSDCache)
|
||||
require.True(t, ok, "cost.usd_cache must be emitted alongside cost.usd_total")
|
||||
// 768 cache_read * 0.0003 + 512 cache_creation * 0.00375 = 0.0021504 → "0.002150".
|
||||
assert.Equal(t, "0.002150", cache, "cache cost sums the read and creation buckets")
|
||||
}
|
||||
|
||||
// TestInvoke_CachedTokensAbsentFallsBackToBaseFormula covers the
|
||||
|
||||
@@ -77,6 +77,8 @@ const (
|
||||
|
||||
// Cost metering (emitted by cost_meter).
|
||||
KeyCostUSDTotal = "cost.usd_total"
|
||||
// KeyCostUSDCache is the portion of cost.usd_total billed for prompt-cache buckets (cache read/creation, or OpenAI's cached input subset).
|
||||
KeyCostUSDCache = "cost.usd_cache"
|
||||
KeyCostSkipped = "cost.skipped"
|
||||
|
||||
// Framework-emitted error markers. Use the mw.<id>.* prefix to
|
||||
|
||||
Reference in New Issue
Block a user