[management, proxy] Management-owned LLM pricing: file-backed defaults + (#6965)

proxy/internal/middleware/builtin/cost_meter/middleware.go

@@ -1,7 +1,9 @@
 // Package cost_meter implements the SlotOnResponse middleware that
 // converts token-usage metadata emitted by llm_response_parser into a
-// per-request USD cost estimate. The middleware uses the shared pricing
-// loader so operator pricing overrides apply to the chain.
+// per-request USD cost estimate. Pricing arrives from management inside
+// the middleware config: a per-provider-record table (the operator's
+// stored prices, matched via llm.resolved_provider_id) consulted first,
+// then the surface-keyed defaults table.
 package cost_meter
 
 import (
@@ -17,7 +19,9 @@ import (
 const ID = "cost_meter"
 
 // Version is the implementation version emitted via the spec merge.
-const Version = "1.0.0"
+// 1.1.0: pricing is config-delivered (defaults + per-provider-record
+// entries) instead of proxy-embedded.
+const Version = "1.1.0"
 
 // Skip reasons emitted under KeyCostSkipped. The set is closed; the
 // dashboard surfaces these verbatim.
@@ -42,19 +46,21 @@ var metadataKeys = []string{
 }
 
 // Middleware computes a per-response cost estimate from the token
-// counts emitted upstream by llm_response_parser.
+// counts emitted upstream by llm_response_parser. Both tables are
+// immutable — a pricing change arrives as a mapping push that rebuilds
+// the chain with a fresh instance.
 type Middleware struct {
-	loader *pricing.Loader
-	// cancel stops this instance's pricing-reload goroutine. Non-nil only
-	// when the loader watches an override file; Close calls it so a chain
-	// rebuild doesn't leak a poll goroutine per retired instance.
-	cancel context.CancelFunc
+	// defaults is the surface-keyed table (llm.provider x llm.model).
+	defaults *pricing.Table
+	// perRecord is keyed by provider record id (llm.resolved_provider_id)
+	// then normalized model id; entries arrive fully materialized from
+	// management. Consulted before defaults. May be nil.
+	perRecord map[string]map[string]pricing.Entry
 }
 
-// newMiddleware constructs a Middleware bound to the given pricing loader.
-// cancel may be nil (defaults-only loader with no reloader to stop).
-func newMiddleware(loader *pricing.Loader, cancel context.CancelFunc) *Middleware {
-	return &Middleware{loader: loader, cancel: cancel}
+// newMiddleware constructs a Middleware over the given pricing tables.
+func newMiddleware(defaults *pricing.Table, perRecord map[string]map[string]pricing.Entry) *Middleware {
+	return &Middleware{defaults: defaults, perRecord: perRecord}
 }
 
 // ID returns the registry identifier.
@@ -79,16 +85,9 @@ func (m *Middleware) MetadataKeys() []string {
 // response.
 func (m *Middleware) MutationsSupported() bool { return false }
 
-// Close stops this instance's pricing-reload goroutine, if any. Called by
-// the chain when a rebuild retires the instance, so the mtime-poll loop
-// doesn't outlive the chain it belonged to. Safe to call on a nil receiver
-// and on an instance with no reloader.
-func (m *Middleware) Close() error {
-	if m != nil && m.cancel != nil {
-		m.cancel()
-	}
-	return nil
-}
+// Close releases resources owned by the middleware. Stateless — the
+// pricing tables are plain maps owned by this instance.
+func (m *Middleware) Close() error { return nil }
 
 // Invoke reads provider, model, and token metadata, looks up pricing,
 // and emits either KeyCostUSDTotal or KeyCostSkipped. The decision is
@@ -144,8 +143,7 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar
 		return out, nil
 	}
 
-	table := m.loader.Get()
-	costs, ok := table.Costs(provider, model, inTokens, outTokens, cachedTokens, cacheCreationTokens)
+	costs, ok := m.lookupCosts(in.Metadata, provider, model, inTokens, outTokens, cachedTokens, cacheCreationTokens)
 	if !ok {
 		out.Metadata = skip(skipUnknownModel)
 		return out, nil
@@ -164,6 +162,26 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar
 	return out, nil
 }
 
+// lookupCosts resolves the price for this request and computes the cost
+// split. Resolution order:
+//
+//  1. Per-provider-record entry: the operator's stored price for the
+//     provider route that served the request, keyed by the
+//     llm.resolved_provider_id metadata llm_router stamped on the allow
+//     path. Absent metadata (e.g. no router in the chain) skips this tier.
+//  2. Surface defaults: the catalog-derived table keyed by llm.provider.
+//
+// The surface always selects the cache formula — a per-record entry for an
+// Anthropic route still bills its cache buckets additively.
+func (m *Middleware) lookupCosts(md []middleware.KV, surface, model string, inTokens, outTokens, cachedTokens, cacheCreationTokens int64) (pricing.Costs, bool) {
+	if recordID := lookupKV(md, middleware.KeyLLMResolvedProviderID); recordID != "" {
+		if entry, ok := m.perRecord[recordID][model]; ok {
+			return pricing.EntryCosts(entry, surface, inTokens, outTokens, cachedTokens, cacheCreationTokens), true
+		}
+	}
+	return m.defaults.Costs(surface, model, inTokens, outTokens, cachedTokens, cacheCreationTokens)
+}
+
 // usd renders a cost as the fixed-precision string every cost.usd_* key
 // carries, so the per-bucket values and the aggregates round identically.
 //