mirror of
https://github.com/netbirdio/netbird.git
synced 2026-09-23 15:19:08 +02:00
[proxy,management] Conform the Agent Network endpoint to the LLM gateway protocol (#7154)
[proxy,management] Conform the Agent Network endpoint to the LLM gateway protocol Reviewed the proxy against Claude Code's published gateway contract. The transport layer already held up; fourteen gaps sat one layer up, in the model catalog and in the non-inference endpoints clients call. Two of them cost money. The catalog carried no claude-opus-5 or claude-sonnet-5, so an operator could not authorise the models coding agents default to — those requests denied as not-routable, or priced at zero where a catch-all carried them. And gateway records pin ParserID "openai" while the same record serves /v1/messages, so Anthropic responses were read with the OpenAI parser, which never looks at message_start where input tokens live: input metered as roughly zero on every stream and cost was skipped entirely. The rest fix requests refused for structural rather than policy reasons: model discovery denied for every account with a model allowlist, token counting denied on Bedrock and mis-parsed on Vertex, startup probes refused and written into the access log at every session start, and denials rendered in a shape no LLM client parses. Two changes are additive by design — the deny body keeps every field it had and adds the vendor's error object alongside, and body-level identity injection is now gated on the request's dialect so it stops sending OpenAI-shape fields into Anthropic bodies that reject them. The end-to-end work turned up one more: the discovery filter treated any slash in a model id as a gateway prefix, which would have dropped every self-hosted "Qwen/..." model from the picker.
This commit is contained in:
Maycon Santos
@@ -23,9 +23,10 @@ import (
|
||||
// model the client asks for. The proxy prices off the REQUEST model, not the
|
||||
// upstream response model, so a made-up model id billed at operator rates lets
|
||||
// these tests assert exact costs without a real vendor key.
|
||||
// Sourced from the harness so the counts can't drift from the mock's config.
|
||||
const (
|
||||
vllmPromptTokens = 11
|
||||
vllmCompletionTokens = 2
|
||||
vllmPromptTokens = harness.VLLMChatInputTokens
|
||||
vllmCompletionTokens = harness.VLLMChatOutputTokens
|
||||
)
|
||||
|
||||
// pricedEnv is a connected single-provider agent-network deployment pointed at
|
||||
@@ -162,30 +163,85 @@ func chatOnce(t *testing.T, ctx context.Context, env pricedEnv, model, sessionID
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
if !waitBeforeRetry(ctx, 5*time.Second) {
|
||||
break
|
||||
}
|
||||
}
|
||||
require.Equal(t, 200, code,
|
||||
"chat for %s must return 200; body: %s\n=== proxy logs ===\n%s", model, body, env.proxy.Logs(context.Background()))
|
||||
return body
|
||||
}
|
||||
|
||||
// findAccessLogBySession polls the access-log page for the row carrying sessionID.
|
||||
func findAccessLogBySession(t *testing.T, ctx context.Context, sessionID string) api.AgentNetworkAccessLog {
|
||||
t.Helper()
|
||||
var row api.AgentNetworkAccessLog
|
||||
require.Eventually(t, func() bool {
|
||||
logs, lerr := srv.ListAccessLogs(ctx)
|
||||
if lerr != nil {
|
||||
return false
|
||||
}
|
||||
for _, r := range logs.Data {
|
||||
if r.SessionId != nil && *r.SessionId == sessionID {
|
||||
row = r
|
||||
return true
|
||||
// accessLogIngestWindow is how long a single request's access-log row is given
|
||||
// to appear before the caller gives up on it.
|
||||
const accessLogIngestWindow = 30 * time.Second
|
||||
|
||||
// accessLogPollInterval is how long the lookup waits between pages. Ingest is
|
||||
// asynchronous, so the row lands somewhere inside the window rather than on
|
||||
// any particular poll.
|
||||
const accessLogPollInterval = 2 * time.Second
|
||||
|
||||
// lookupAccessLogBySession polls the access-log page for the row carrying
|
||||
// sessionID and reports whether it arrived within the window. It never fails
|
||||
// the test: callers that can recover — by firing a fresh request under a new
|
||||
// session — need to see the miss rather than die on it.
|
||||
func lookupAccessLogBySession(ctx context.Context, sessionID string, within time.Duration) (api.AgentNetworkAccessLog, bool) {
|
||||
deadline := time.Now().Add(within)
|
||||
for {
|
||||
// Each poll is bounded by what is left of the window rather than by the
|
||||
// caller's context: a single stalled request would otherwise hold the
|
||||
// loop open long past the ingest window it is meant to enforce, and the
|
||||
// caller would read the delay as a missing row.
|
||||
if logs, lerr := listAccessLogsBy(ctx, deadline); lerr == nil {
|
||||
for _, r := range logs.Data {
|
||||
if r.SessionId != nil && *r.SessionId == sessionID {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}, 30*time.Second, 2*time.Second, "session id %q must be recorded in an access-log row", sessionID)
|
||||
// The wait is bounded by the window as well, so the answer arrives when
|
||||
// the caller's budget runs out rather than a poll interval later: a
|
||||
// full interval slept past the deadline reports "no row" up to two
|
||||
// seconds late, which reads as a slower lookup than the one asked for.
|
||||
wait := time.Until(deadline)
|
||||
if wait > accessLogPollInterval {
|
||||
wait = accessLogPollInterval
|
||||
}
|
||||
if wait <= 0 {
|
||||
return api.AgentNetworkAccessLog{}, false
|
||||
}
|
||||
timer := time.NewTimer(wait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return api.AgentNetworkAccessLog{}, false
|
||||
case <-timer.C:
|
||||
}
|
||||
// Checked after the wait rather than before the request: a poll issued
|
||||
// past the deadline carries no budget and would fail on arrival.
|
||||
if !time.Now().Before(deadline) {
|
||||
return api.AgentNetworkAccessLog{}, false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// listAccessLogsBy fetches one access-log page under a context that expires at
|
||||
// deadline, so no single call can outlive the window its caller is polling
|
||||
// within. The parent's cancellation still applies: the child inherits it.
|
||||
func listAccessLogsBy(ctx context.Context, deadline time.Time) (api.AgentNetworkAccessLogsResponse, error) {
|
||||
reqCtx, cancel := context.WithDeadline(ctx, deadline)
|
||||
defer cancel()
|
||||
return srv.ListAccessLogs(reqCtx)
|
||||
}
|
||||
|
||||
// findAccessLogBySession polls the access-log page for the row carrying
|
||||
// sessionID, failing the test if it never lands. Use it for a request whose row
|
||||
// must exist; where a missing row is a recoverable race, use
|
||||
// lookupAccessLogBySession and retry.
|
||||
func findAccessLogBySession(t *testing.T, ctx context.Context, sessionID string) api.AgentNetworkAccessLog {
|
||||
t.Helper()
|
||||
row, ok := lookupAccessLogBySession(ctx, sessionID, accessLogIngestWindow)
|
||||
require.True(t, ok, "session id %q must be recorded in an access-log row", sessionID)
|
||||
return row
|
||||
}
|
||||
|
||||
@@ -319,6 +375,11 @@ func TestPriceChangeUpdatesRecordedCost(t *testing.T) {
|
||||
outRateA = 0.020
|
||||
inRateB = 0.050 // 5x / 4x the original, so a repriced row is unmistakable
|
||||
outRateB = 0.080
|
||||
// Per-attempt ingest wait, shorter than the default so a request that
|
||||
// produces no row costs one retry rather than most of the budget, and an
|
||||
// overall deadline long enough to hold several attempts.
|
||||
repriceIngestWindow = 20 * time.Second
|
||||
repriceDeadline = 180 * time.Second
|
||||
)
|
||||
|
||||
env := provisionPricedProvider(t, ctx, "reprice", []api.AgentNetworkProviderModel{
|
||||
@@ -353,27 +414,61 @@ func TestPriceChangeUpdatesRecordedCost(t *testing.T) {
|
||||
// reading its cost, so an un-ingested row is never mistaken for "still rate A".
|
||||
// The expected new input cost is unmistakably higher than rate A, so a
|
||||
// lingering old-rate row can't satisfy the check.
|
||||
//
|
||||
// Every way an iteration can come up short — the request failing, its row not
|
||||
// landing, or the row still carrying rate A — is a symptom of the same
|
||||
// in-flight rebuild, so each one retries under a fresh session rather than
|
||||
// ending the test. Only the outer deadline is fatal.
|
||||
wantInputB := float64(vllmPromptTokens) / 1000 * inRateB
|
||||
var repriced api.AgentNetworkAccessLog
|
||||
var lastSession string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
// The cost last read, kept separately: repriced is the zero value on every
|
||||
// path that gives up, so reporting its cost would say "$0.000000" whether
|
||||
// the rows were still at rate A or no row was ever read.
|
||||
var lastCost float64
|
||||
var sawRow bool
|
||||
deadline := time.Now().Add(repriceDeadline)
|
||||
// Everything inside the loop runs under the deadline rather than the
|
||||
// test's own context. An attempt started just before it would otherwise
|
||||
// run well past it: the chat container is capped at 90s of its own and the
|
||||
// row lookup at another 20s, so the loop could report a repricing failure
|
||||
// nearly two minutes after the window it was given had closed.
|
||||
repriceCtx, cancelReprice := context.WithDeadline(ctx, deadline)
|
||||
defer cancelReprice()
|
||||
for time.Now().Before(deadline) {
|
||||
lastSession = fmt.Sprintf("e2e-session-reprice-b-%d", time.Now().UnixNano())
|
||||
code, _, cerr := env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireChat, customModel, "Reply with exactly: pong", lastSession)
|
||||
code, _, cerr := env.client.Chat(repriceCtx, env.endpoint, env.proxyIP, harness.WireChat, customModel, "Reply with exactly: pong", lastSession)
|
||||
if cerr != nil || code != 200 {
|
||||
time.Sleep(5 * time.Second)
|
||||
if !waitBeforeRetry(repriceCtx, 5*time.Second) {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
row, ok := lookupAccessLogBySession(repriceCtx, lastSession, repriceIngestWindow)
|
||||
if !ok {
|
||||
// No row for this request. The proxy now publishes a rebuilt chain
|
||||
// before the route that reaches it, so a request can no longer be
|
||||
// served unattributed mid-update; this retry covers the ingest
|
||||
// window alone. Fire another one under a fresh session.
|
||||
t.Logf("no access-log row for session %q within %s; retrying under a fresh session", lastSession, repriceIngestWindow)
|
||||
continue
|
||||
}
|
||||
row := findAccessLogBySession(t, ctx, lastSession)
|
||||
if inDelta(row.InputCostUsd, wantInputB, 1e-6) {
|
||||
repriced = row
|
||||
break
|
||||
}
|
||||
// Still priced at the old rate — the push hasn't landed yet; retry.
|
||||
time.Sleep(5 * time.Second)
|
||||
lastCost, sawRow = row.InputCostUsd, true
|
||||
if !waitBeforeRetry(repriceCtx, 5*time.Second) {
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, repriced.Id, "a request after the price change must be priced at the new rate B; last input_cost_usd=$%.6f, wanted $%.6f\n=== proxy logs ===\n%s",
|
||||
repriced.InputCostUsd, wantInputB, env.proxy.Logs(context.Background()))
|
||||
lastSeen := "no row was ever read"
|
||||
if sawRow {
|
||||
lastSeen = fmt.Sprintf("last input_cost_usd=$%.6f", lastCost)
|
||||
}
|
||||
require.NotEmpty(t, repriced.Id, "a request after the price change must be priced at the new rate B; %s, wanted $%.6f\n=== proxy logs ===\n%s",
|
||||
lastSeen, wantInputB, env.proxy.Logs(context.Background()))
|
||||
|
||||
assertOpenAICostAtRates(t, repriced, inRateB, outRateB)
|
||||
verifyUsageRowForSession(t, lastSession, inRateB, outRateB)
|
||||
@@ -630,3 +725,47 @@ func inDelta(a, b, tol float64) bool {
|
||||
}
|
||||
return d <= tol
|
||||
}
|
||||
|
||||
// TestCustomDatedModelKeepsItsOwnPrice covers the review fix that anchored the
|
||||
// release-date fallback to Claude ids. Pricing looks every model up through
|
||||
// that helper, so while it matched a bare trailing date any operator id ending
|
||||
// in eight digits inherited the rate of its undated sibling — a silent
|
||||
// mis-bill on models NetBird knows nothing about.
|
||||
func TestCustomDatedModelKeepsItsOwnPrice(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
const (
|
||||
baseModel = "internal-llm"
|
||||
datedModel = "internal-llm-20250101"
|
||||
baseIn = 0.010
|
||||
baseOut = 0.020
|
||||
// An order of magnitude apart, so a row billed at the wrong entry is
|
||||
// unmistakable rather than a rounding argument.
|
||||
datedIn = 0.100
|
||||
datedOut = 0.200
|
||||
)
|
||||
|
||||
env := provisionPricedProvider(t, ctx, "customdated", []api.AgentNetworkProviderModel{
|
||||
{Id: baseModel, InputPer1k: baseIn, OutputPer1k: baseOut},
|
||||
{Id: datedModel, InputPer1k: datedIn, OutputPer1k: datedOut},
|
||||
})
|
||||
|
||||
t.Run("the undated id bills at its own rate", func(t *testing.T) {
|
||||
session := fmt.Sprintf("e2e-session-customdated-base-%d", time.Now().UnixNano())
|
||||
chatOnce(t, ctx, env, baseModel, session)
|
||||
assertOpenAICostAtRates(t, findAccessLogBySession(t, ctx, session), baseIn, baseOut)
|
||||
})
|
||||
|
||||
t.Run("the dated id keeps its own rate", func(t *testing.T) {
|
||||
session := fmt.Sprintf("e2e-session-customdated-dated-%d", time.Now().UnixNano())
|
||||
chatOnce(t, ctx, env, datedModel, session)
|
||||
row := findAccessLogBySession(t, ctx, session)
|
||||
assertOpenAICostAtRates(t, row, datedIn, datedOut)
|
||||
|
||||
// Spelled out because it is the regression: inheriting the sibling's
|
||||
// rate would bill this request at a tenth of its price.
|
||||
assert.Greater(t, row.InputCostUsd, float64(vllmPromptTokens)/1000*baseIn*2,
|
||||
"a custom dated id must not inherit the undated entry's rate")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// Models each catalog surface is registered with in the matrix below. They
|
||||
// differ per provider so the router's choice is unambiguous: a request that
|
||||
// lands on the wrong provider record fails the surface assertion instead of
|
||||
// passing by coincidence.
|
||||
const (
|
||||
matrixAnthropicModel = "claude-sonnet-5"
|
||||
matrixBedrockModel = "anthropic.claude-sonnet-5"
|
||||
// matrixBedrockPathModel is what a Bedrock SDK client puts in the URL: a
|
||||
// cross-region inference profile with a release date and version suffix.
|
||||
// The proxy must normalise it back to matrixBedrockModel to route and price.
|
||||
matrixBedrockPathModel = "us.anthropic.claude-sonnet-5-20250101-v1:0"
|
||||
// matrixVertexModel differs from the Anthropic record's model on purpose:
|
||||
// a shared id would leave two routes claiming it and make which one serves
|
||||
// /v1/messages depend on declaration order.
|
||||
matrixVertexModel = "claude-haiku-4-5"
|
||||
matrixVertexProject = "e2e-project"
|
||||
matrixVertexRegion = "us-east5"
|
||||
)
|
||||
|
||||
// gatewayEnv is a connected client plus a set of provider records, all pointed
|
||||
// at one mock upstream, so several wire shapes can be driven over a single
|
||||
// tunnel.
|
||||
type gatewayEnv struct {
|
||||
endpoint string
|
||||
proxyIP string
|
||||
client *harness.Client
|
||||
proxy *harness.Proxy
|
||||
vllm *harness.VLLM
|
||||
// providerIDs maps the catalog id to the created provider record id.
|
||||
providerIDs map[string]string
|
||||
}
|
||||
|
||||
// provisionGatewayMatrix brings up one mock upstream and one provider record
|
||||
// per catalog surface, all authorised for the same group by a single policy.
|
||||
// Sharing one proxy and client keeps the wire-shape cases to one tunnel setup;
|
||||
// each case still creates its own session id so its access-log row is findable.
|
||||
func provisionGatewayMatrix(t *testing.T, ctx context.Context) gatewayEnv {
|
||||
t.Helper()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-gw-matrix"})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-gw-matrix-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
// The mock ignores auth, so a dummy credential satisfies each catalog
|
||||
// entry's auth template. Vertex is the exception: its api_key is a GCP
|
||||
// service-account keyfile the proxy mints an OAuth token from, and a dummy
|
||||
// one cannot mint. That is deliberate — the Vertex case below asserts on
|
||||
// routing, which happens before the token mint.
|
||||
dummyKey := "sk-gw-e2e"
|
||||
dummyKeyfile := "keyfile::" + "e2e-not-a-real-service-account-key"
|
||||
|
||||
specs := []struct {
|
||||
name string
|
||||
catalogID string
|
||||
apiKey string
|
||||
models []api.AgentNetworkProviderModel
|
||||
}{
|
||||
{
|
||||
name: "openai", catalogID: "openai_api", apiKey: dummyKey,
|
||||
models: []api.AgentNetworkProviderModel{{Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002}},
|
||||
},
|
||||
{
|
||||
name: "anthropic", catalogID: "anthropic_api", apiKey: dummyKey,
|
||||
models: []api.AgentNetworkProviderModel{{Id: matrixAnthropicModel, InputPer1k: 0.003, OutputPer1k: 0.015}},
|
||||
},
|
||||
{
|
||||
name: "bedrock", catalogID: "bedrock_api", apiKey: dummyKey,
|
||||
models: []api.AgentNetworkProviderModel{{Id: matrixBedrockModel, InputPer1k: 0.003, OutputPer1k: 0.015}},
|
||||
},
|
||||
{
|
||||
name: "vertex", catalogID: "vertex_ai_api", apiKey: dummyKeyfile,
|
||||
models: []api.AgentNetworkProviderModel{{Id: matrixVertexModel, InputPer1k: 0.001, OutputPer1k: 0.005}},
|
||||
},
|
||||
}
|
||||
|
||||
providerIDs := make(map[string]string, len(specs))
|
||||
ids := make([]string, 0, len(specs))
|
||||
for _, spec := range specs {
|
||||
key := spec.apiKey
|
||||
models := spec.models
|
||||
prov, perr := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-gw-" + spec.name,
|
||||
ProviderId: spec.catalogID,
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &key,
|
||||
Enabled: ptr(true),
|
||||
Models: &models,
|
||||
})
|
||||
require.NoError(t, perr, "create %s provider", spec.name)
|
||||
id := prov.Id
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), id) })
|
||||
providerIDs[spec.catalogID] = id
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
// Uncapped token limit: never blocks the handful of tokens driven here, but
|
||||
// switches on usage metering so consumption and cost land in the row.
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-gw-matrix",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: ids,
|
||||
Limits: &api.AgentNetworkPolicyLimits{
|
||||
TokenLimit: api.AgentNetworkPolicyTokenLimit{
|
||||
Enabled: true,
|
||||
GroupCap: 10_000_000,
|
||||
UserCap: 10_000_000,
|
||||
WindowSeconds: 60,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
endpoint, proxyIP, cl, px := connectClient(t, ctx, "gw-matrix", sk.Key)
|
||||
return gatewayEnv{
|
||||
endpoint: endpoint,
|
||||
proxyIP: proxyIP,
|
||||
client: cl,
|
||||
proxy: px,
|
||||
vllm: vllm,
|
||||
providerIDs: providerIDs,
|
||||
}
|
||||
}
|
||||
|
||||
// connectClient starts a proxy and a tunnel client for the shared account and
|
||||
// waits until the client can reach the proxy peer, returning the endpoint and
|
||||
// the proxy's tunnel IP to pin requests to.
|
||||
func connectClient(t *testing.T, ctx context.Context, name, setupKey string) (string, string, *harness.Client, *harness.Proxy) {
|
||||
t.Helper()
|
||||
|
||||
settings, err := srv.GetSettings(ctx)
|
||||
require.NoError(t, err, "read settings")
|
||||
require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
|
||||
|
||||
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-"+name+"-proxy")
|
||||
require.NoError(t, err, "mint proxy token")
|
||||
px, err := harness.StartProxy(ctx, srv, proxyToken)
|
||||
require.NoError(t, err, "start proxy")
|
||||
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, setupKey)
|
||||
require.NoError(t, err, "start client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
|
||||
// The probe resolves the endpoint and its first packet wakes the lazy proxy
|
||||
// peer, so WaitProxyPeer then observes it connected.
|
||||
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
|
||||
require.NoError(t, err, "resolve endpoint to proxy IP")
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
return settings.Endpoint, proxyIP, cl, px
|
||||
}
|
||||
|
||||
// callUntil retries an HTTP call through the tunnel until it returns one of the
|
||||
// wanted statuses or the deadline passes, absorbing the DNS and tunnel jitter
|
||||
// the first call through a fresh tunnel can hit. The last status and body are
|
||||
// returned either way so the caller can assert with real detail.
|
||||
func callUntil(t *testing.T, call func() (int, string, error), want ...int) (int, string) {
|
||||
t.Helper()
|
||||
wanted := make(map[int]struct{}, len(want))
|
||||
for _, w := range want {
|
||||
wanted[w] = struct{}{}
|
||||
}
|
||||
|
||||
var code int
|
||||
var body string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
c, b, err := call()
|
||||
if err == nil {
|
||||
code, body = c, b
|
||||
if _, ok := wanted[code]; ok {
|
||||
return code, body
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
return code, body
|
||||
}
|
||||
|
||||
// TestGatewayProtocolProviderMatrix drives one request per wire shape over a
|
||||
// single tunnel, with a provider record per catalog surface behind it. It is
|
||||
// the regression net for the routing and parser-selection changes: each case
|
||||
// asserts the surface the request was metered under and the token counts that
|
||||
// surface's own usage block carries, so a request parsed by the wrong provider's
|
||||
// parser meters zero and fails rather than passing on a coincidence.
|
||||
func TestGatewayProtocolProviderMatrix(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionGatewayMatrix(t, ctx)
|
||||
diag := func() string {
|
||||
return fmt.Sprintf("\n=== upstream logs ===\n%s\n=== proxy logs ===\n%s",
|
||||
env.vllm.Logs(context.Background()), env.proxy.Logs(context.Background()))
|
||||
}
|
||||
|
||||
t.Run("openai chat completions", func(t *testing.T) {
|
||||
session := "e2e-gw-openai"
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireChat, harness.VLLMModel, "ping", session)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "openai chat must succeed; body: %s%s", body, diag())
|
||||
require.Contains(t, body, "chat.completion", "body must be an OpenAI completion; got: %s", body)
|
||||
|
||||
row := findAccessLogBySession(t, ctx, session)
|
||||
require.NotNil(t, row.Provider)
|
||||
assert.Equal(t, "openai", *row.Provider, "the OpenAI chat path must meter under the openai surface")
|
||||
assert.Equal(t, int64(harness.VLLMChatInputTokens), row.InputTokens, "OpenAI usage block must be read")
|
||||
assert.Equal(t, int64(harness.VLLMChatOutputTokens), row.OutputTokens)
|
||||
})
|
||||
|
||||
t.Run("anthropic messages", func(t *testing.T) {
|
||||
session := "e2e-gw-anthropic"
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages, matrixAnthropicModel, "ping", session)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "anthropic messages must succeed; body: %s%s", body, diag())
|
||||
|
||||
row := findAccessLogBySession(t, ctx, session)
|
||||
require.NotNil(t, row.Provider)
|
||||
assert.Equal(t, "anthropic", *row.Provider, "the /v1/messages path must meter under the anthropic surface")
|
||||
// These counts only appear if the Anthropic parser read the response:
|
||||
// its usage fields are named differently from the OpenAI block.
|
||||
assert.Equal(t, int64(harness.VLLMMessagesInputTokens), row.InputTokens,
|
||||
"Anthropic input_tokens must be read; zero here means the wrong parser ran")
|
||||
assert.Equal(t, int64(harness.VLLMMessagesOutputTokens), row.OutputTokens)
|
||||
assert.Positive(t, row.CachedInputTokens, "the Anthropic cache-read bucket must be recorded")
|
||||
assert.Positive(t, row.CostUsd, "a metered request must carry a cost")
|
||||
require.NotNil(t, row.ResolvedProviderId)
|
||||
assert.Equal(t, env.providerIDs["anthropic_api"], *row.ResolvedProviderId,
|
||||
"a vendor-tagged request must not cross to another provider's record")
|
||||
})
|
||||
|
||||
t.Run("bedrock invoke normalises the path model", func(t *testing.T) {
|
||||
session := "e2e-gw-bedrock"
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Bedrock(ctx, env.endpoint, env.proxyIP, matrixBedrockPathModel, "ping", session)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "bedrock invoke must succeed; body: %s%s", body, diag())
|
||||
|
||||
row := findAccessLogBySession(t, ctx, session)
|
||||
require.NotNil(t, row.Provider)
|
||||
assert.Equal(t, "bedrock", *row.Provider, "a native Bedrock path must meter under the bedrock surface")
|
||||
require.NotNil(t, row.Model)
|
||||
assert.Equal(t, matrixBedrockModel, *row.Model,
|
||||
"the inference-profile prefix, release date and version suffix must be normalised away")
|
||||
assert.Equal(t, int64(harness.VLLMMessagesInputTokens), row.InputTokens)
|
||||
})
|
||||
|
||||
t.Run("anthropic token counting", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.PostJSON(ctx, env.endpoint, env.proxyIP, "/v1/messages/count_tokens",
|
||||
fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":"ping"}]}`, matrixAnthropicModel),
|
||||
[]string{"anthropic-version: 2023-06-01"})
|
||||
}, 200)
|
||||
assert.Equal(t, 200, code, "token counting must route rather than deny; body: %s%s", body, diag())
|
||||
})
|
||||
|
||||
t.Run("bedrock token counting", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.PostJSON(ctx, env.endpoint, env.proxyIP,
|
||||
"/model/"+matrixBedrockPathModel+"/count-tokens",
|
||||
`{"input":{"converse":{"messages":[{"role":"user","content":[{"text":"ping"}]}]}}}`, nil)
|
||||
}, 200)
|
||||
assert.Equal(t, 200, code,
|
||||
"the Bedrock count-tokens action must route; denying it pushes counting onto the billable inference path; body: %s%s",
|
||||
body, diag())
|
||||
})
|
||||
|
||||
t.Run("vertex token counting reaches its provider", func(t *testing.T) {
|
||||
// The dummy service-account key cannot mint an OAuth token, so the
|
||||
// request stops at the upstream credential. Both outcomes render as
|
||||
// 403, so the deny code is what distinguishes them: upstream_auth_failed
|
||||
// means the path resolved to the Vertex route and only the credential
|
||||
// failed, while model_not_routable would mean the method segment was
|
||||
// swallowed into the model id and no route ever claimed it.
|
||||
path := fmt.Sprintf("/v1/projects/%s/locations/%s/publishers/anthropic/models/%s/count-tokens:rawPredict",
|
||||
matrixVertexProject, matrixVertexRegion, matrixVertexModel)
|
||||
_, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.PostJSON(ctx, env.endpoint, env.proxyIP, path,
|
||||
`{"anthropic_version":"vertex-2023-10-16","messages":[{"role":"user","content":"ping"}]}`, nil)
|
||||
}, 403)
|
||||
assert.NotContains(t, body, "model_not_routable",
|
||||
"the count-tokens method segment must not be parsed as part of the model id; body: %s%s", body, diag())
|
||||
assert.Contains(t, body, "llm_policy.upstream_auth_failed",
|
||||
"the request must reach the Vertex route and fail only at the credential; body: %s%s", body, diag())
|
||||
})
|
||||
|
||||
t.Run("connection warming probe", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Get(ctx, env.endpoint, env.proxyIP, "/api/hello", nil)
|
||||
}, 200)
|
||||
assert.NotEqual(t, 403, code,
|
||||
"the warm-up probe carries no model and must not be refused as unroutable; body: %s%s", body, diag())
|
||||
})
|
||||
|
||||
t.Run("unknown model denies in the caller's error shape", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages,
|
||||
"claude-not-a-real-model-9", "ping", "e2e-gw-unknown")
|
||||
}, 403)
|
||||
require.Equal(t, 403, code, "a model no provider claims must still be refused; body: %s%s", body, diag())
|
||||
|
||||
// The NetBird fields stay where they were for existing consumers.
|
||||
assert.Contains(t, body, "llm_policy.model_not_routable", "the deny code must be preserved")
|
||||
// And the vendor's own envelope rides alongside, so the client can show
|
||||
// the reason instead of an unexplained API error.
|
||||
assert.Contains(t, body, `"type":"error"`, "an Anthropic caller must get the Anthropic error envelope")
|
||||
assert.Contains(t, body, "permission_error", "403 must map to the vendor's permission error type")
|
||||
})
|
||||
}
|
||||
|
||||
// TestModelDiscoveryWithModelAllowlist covers gateway model discovery on an
|
||||
// account that restricts models, which is the configuration that broke: the
|
||||
// listing carries no model, and the per-model allowlist fails closed on an
|
||||
// undetermined one, so discovery denied for exactly the accounts using the
|
||||
// feature. It also asserts the allowlist still refuses a model outside it, so
|
||||
// the exemption cannot be read as a way around the gate.
|
||||
func TestModelDiscoveryWithModelAllowlist(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-gw-discovery"})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-gw-discovery-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
// One provider enumerating a single model, while the upstream's own listing
|
||||
// advertises two. The proxy must serve the shorter list.
|
||||
dummyKey := "sk-discovery-e2e"
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-gw-discovery",
|
||||
ProviderId: "openai_api",
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &dummyKey,
|
||||
Enabled: ptr(true),
|
||||
Models: &[]api.AgentNetworkProviderModel{
|
||||
{Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
// The model allowlist is what makes this a regression test: without a
|
||||
// guardrail enabled, discovery was never gated in the first place.
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = "e2e-gw-discovery-allowlist"
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{harness.VLLMModel}
|
||||
guard, err := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, err, "create guardrail")
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) })
|
||||
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-gw-discovery",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{guard.Id},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
endpoint, proxyIP, cl, px := connectClient(t, ctx, "gw-discovery", sk.Key)
|
||||
diag := func() string {
|
||||
return fmt.Sprintf("\n=== upstream logs ===\n%s\n=== proxy logs ===\n%s",
|
||||
vllm.Logs(context.Background()), px.Logs(context.Background()))
|
||||
}
|
||||
|
||||
t.Run("listing is served and bounded by policy", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return cl.Get(ctx, endpoint, proxyIP, "/v1/models?limit=1000", nil)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code,
|
||||
"discovery must not be refused because the request carries no model; body: %s%s", body, diag())
|
||||
|
||||
assert.Contains(t, body, harness.VLLMModel, "the authorised model must reach the picker")
|
||||
assert.NotContains(t, body, harness.VLLMUnlistedModel,
|
||||
"a model the policy does not authorise must not be offered; body: %s", body)
|
||||
})
|
||||
|
||||
t.Run("allowlist still refuses a model outside it", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return cl.Chat(ctx, endpoint, proxyIP, harness.WireChat,
|
||||
harness.VLLMUnlistedModel, "ping", "e2e-gw-discovery-blocked")
|
||||
}, 403)
|
||||
require.Equal(t, 403, code,
|
||||
"exempting model-less endpoints must not exempt inference; body: %s%s", body, diag())
|
||||
assert.True(t,
|
||||
strings.Contains(body, "llm_policy.model_blocked") || strings.Contains(body, "llm_policy.model_not_routable"),
|
||||
"the refusal must name a model policy code; body: %s", body)
|
||||
})
|
||||
|
||||
t.Run("allowlisted model still routes", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return cl.Chat(ctx, endpoint, proxyIP, harness.WireChat,
|
||||
harness.VLLMModel, "ping", "e2e-gw-discovery-allowed")
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "the allowlisted model must still be served; body: %s%s", body, diag())
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// The cases in this file cover behaviour that arrived from code review, after
|
||||
// the gateway-protocol end-to-end tests were written. Each had unit coverage
|
||||
// only; none needed a new harness capability, which is why they belong here
|
||||
// rather than on a manual checklist.
|
||||
|
||||
// TestNonInferenceEndpointsAreAuthorised covers the two review findings on the
|
||||
// endpoints that carry no body: the per-model lookup must be authorised
|
||||
// against the same allowlist that bounds the listing beside it, and only a read
|
||||
// method may claim the non-inference exemption that skips the token pre-flight.
|
||||
func TestNonInferenceEndpointsAreAuthorised(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionDiscoveryProvider(t, ctx)
|
||||
|
||||
t.Run("lookup of an authorised model succeeds", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Get(ctx, env.endpoint, env.proxyIP, "/v1/models/"+harness.VLLMModel, nil)
|
||||
}, 200)
|
||||
assert.Equal(t, 200, code, "an allowlisted model must remain reachable; body: %s", body)
|
||||
})
|
||||
|
||||
t.Run("lookup of an unauthorised model is refused", func(t *testing.T) {
|
||||
code, body, err := env.client.Get(ctx, env.endpoint, env.proxyIP, "/v1/models/"+harness.VLLMUnlistedModel, nil)
|
||||
require.NoError(t, err, "request must reach the proxy")
|
||||
assert.Equal(t, 403, code,
|
||||
"a model the policy does not authorise must not be confirmed by the detail lookup; body: %s", body)
|
||||
})
|
||||
|
||||
// A write must not claim the exemption that lets the listing skip the token
|
||||
// pre-flight. The body names no model on purpose: that is what a request
|
||||
// probing for the exemption looks like, and it is the case the method gate
|
||||
// exists to refuse. (A POST that does name a model is a different thing —
|
||||
// it routes and meters as the inference request it is.)
|
||||
for _, path := range []string{"/v1/models", "/v1/models/" + harness.VLLMModel, "/api/hello"} {
|
||||
t.Run("write to "+path+" is refused", func(t *testing.T) {
|
||||
code, body, err := env.client.PostJSON(ctx, env.endpoint, env.proxyIP, path,
|
||||
`{"messages":[{"role":"user","content":"hi"}]}`, nil)
|
||||
require.NoError(t, err, "request must reach the proxy")
|
||||
assert.NotEqual(t, 200, code,
|
||||
"a write to a non-inference path must not be served unmetered; body: %s", body)
|
||||
})
|
||||
}
|
||||
|
||||
// A request carrying the sub-agent attribution headers must still be served
|
||||
// and metered normally. Asserting the ids themselves is not possible yet:
|
||||
// the parser lifts them onto the request's metadata, but nothing persists
|
||||
// them, so they have no queryable surface to check against.
|
||||
t.Run("sub-agent headers do not disturb the request", func(t *testing.T) {
|
||||
sessionID := fmt.Sprintf("e2e-session-agentid-%d", time.Now().UnixNano())
|
||||
code, body, err := env.client.PostJSON(ctx, env.endpoint, env.proxyIP, "/v1/chat/completions",
|
||||
fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":"Reply with exactly: pong"}]}`, harness.VLLMModel),
|
||||
[]string{
|
||||
"x-session-id: " + sessionID,
|
||||
"x-claude-code-agent-id: agent-child-7",
|
||||
"x-claude-code-parent-agent-id: agent-root-1",
|
||||
})
|
||||
require.NoError(t, err, "request must reach the proxy")
|
||||
require.Equal(t, 200, code, "the request must succeed; body: %s", body)
|
||||
|
||||
row := findAccessLogBySession(t, ctx, sessionID)
|
||||
assert.Positive(t, row.InputTokens, "the request must still be metered normally")
|
||||
})
|
||||
}
|
||||
|
||||
// TestDatedModelIdRouting covers both halves of the dated-id rule that review
|
||||
// tightened: a dated id still reaches an undated registration, but a route
|
||||
// pinned to one dated build must never serve a different one.
|
||||
func TestDatedModelIdRouting(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
const (
|
||||
undated = "claude-sonnet-9"
|
||||
datedA = "claude-sonnet-9-20250101"
|
||||
datedB = "claude-sonnet-9-20250202"
|
||||
)
|
||||
|
||||
t.Run("a dated id reaches its undated registration", func(t *testing.T) {
|
||||
env := provisionModelProvider(t, ctx, "dated-undated", "anthropic_api", undated)
|
||||
|
||||
sessionID := fmt.Sprintf("e2e-session-dated-%d", time.Now().UnixNano())
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages, datedA, "Reply with exactly: pong", sessionID)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "a pinned release of a registered family must route; body: %s", body)
|
||||
|
||||
row := findAccessLogBySession(t, ctx, sessionID)
|
||||
assert.Positive(t, row.InputTokens, "the dated request must price at the registered rate, not zero")
|
||||
})
|
||||
|
||||
t.Run("a route pinned to one dated build refuses another", func(t *testing.T) {
|
||||
env := provisionModelProvider(t, ctx, "dated-pinned", "anthropic_api", datedA)
|
||||
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages, datedA, "Reply with exactly: pong", "")
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "the exact dated id must still route; body: %s", body)
|
||||
|
||||
code, body, err := env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages, datedB, "Reply with exactly: pong", "")
|
||||
require.NoError(t, err, "request must reach the proxy")
|
||||
assert.Equal(t, 403, code,
|
||||
"a provider pinned to one dated build must not serve another; body: %s", body)
|
||||
})
|
||||
}
|
||||
|
||||
// TestBedrockInferenceProfilesReachTheUpstream covers the startup lookup a
|
||||
// Bedrock client makes. The proxy forwards it to the configured upstream rather
|
||||
// than denying it, so what comes back is the upstream's answer — never a
|
||||
// NetBird policy rejection.
|
||||
func TestBedrockInferenceProfilesReachTheUpstream(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionModelProvider(t, ctx, "infprofiles", "bedrock_api", "anthropic.claude-sonnet-5")
|
||||
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Get(ctx, env.endpoint, env.proxyIP, "/inference-profiles", nil)
|
||||
}, 200)
|
||||
|
||||
assert.Equal(t, 200, code, "the lookup must reach the upstream; body: %s", body)
|
||||
assert.NotContains(t, body, "llm_policy.",
|
||||
"the proxy must not answer a control-plane lookup with a policy denial")
|
||||
assert.Contains(t, body, "inferenceProfileSummaries",
|
||||
"the upstream's own answer must come back untouched")
|
||||
}
|
||||
|
||||
// provisionDiscoveryProvider brings up one mock-backed provider enumerating a
|
||||
// single model, with an allowlist guardrail in effect, plus a connected client.
|
||||
func provisionDiscoveryProvider(t *testing.T, ctx context.Context) pricedEnv {
|
||||
t.Helper()
|
||||
env := provisionModelProvider(t, ctx, "noninference", "openai_api", harness.VLLMModel)
|
||||
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = "e2e-noninference-allowlist-" + fmt.Sprint(time.Now().UnixNano())
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{harness.VLLMModel}
|
||||
guard, err := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, err, "create guardrail")
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) })
|
||||
|
||||
enabled := true
|
||||
_, err = srv.UpdatePolicy(ctx, env.policyID, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-noninference",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{env.groupID},
|
||||
DestinationProviderIds: []string{env.providerID},
|
||||
GuardrailIds: &[]string{guard.Id},
|
||||
})
|
||||
require.NoError(t, err, "attach guardrail to policy")
|
||||
return env
|
||||
}
|
||||
|
||||
// provisionModelProvider brings up the mock, one provider under the given
|
||||
// catalog id enumerating exactly one model, an authorising policy, and a
|
||||
// connected proxy + client.
|
||||
func provisionModelProvider(t *testing.T, ctx context.Context, name, catalogID, model string) pricedEnv {
|
||||
t.Helper()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
suffix := strings.ToLower(name)
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-gwr-" + suffix})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-gwr-" + suffix + "-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
dummyKey := "sk-gwr-e2e"
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-gwr-" + suffix,
|
||||
ProviderId: catalogID,
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &dummyKey,
|
||||
Enabled: ptr(true),
|
||||
Models: &[]api.AgentNetworkProviderModel{
|
||||
{Id: model, InputPer1k: 0.001, OutputPer1k: 0.002},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-gwr-" + suffix,
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
Limits: &api.AgentNetworkPolicyLimits{
|
||||
TokenLimit: api.AgentNetworkPolicyTokenLimit{
|
||||
Enabled: true,
|
||||
GroupCap: 10_000_000,
|
||||
UserCap: 10_000_000,
|
||||
WindowSeconds: 60,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
endpoint, proxyIP, cl, px := connectClient(t, ctx, "gwr-"+suffix, sk.Key)
|
||||
return pricedEnv{
|
||||
providerID: prov.Id,
|
||||
groupID: grp.Id,
|
||||
policyID: pol.Id,
|
||||
upstream: vllm.URL,
|
||||
endpoint: endpoint,
|
||||
proxyIP: proxyIP,
|
||||
client: cl,
|
||||
proxy: px,
|
||||
}
|
||||
}
|
||||
@@ -54,3 +54,19 @@ func run(m *testing.M) int {
|
||||
|
||||
return m.Run()
|
||||
}
|
||||
|
||||
// waitBeforeRetry pauses between attempts of a polling loop and reports
|
||||
// whether the caller should keep going. A cancelled context ends the loop
|
||||
// where a plain sleep would keep retrying against it: every call fails
|
||||
// instantly once ctx is done, so the loop would spend its whole remaining
|
||||
// window sleeping between failures nobody is waiting for any more.
|
||||
func waitBeforeRetry(ctx context.Context, d time.Duration) bool {
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// streamedModel is priced high enough that a mis-metered request is obvious in
|
||||
// the recorded cost, and named so it cannot collide with another test's route.
|
||||
const streamedModel = "e2e-streamed-model"
|
||||
|
||||
const (
|
||||
streamInRate = 0.010
|
||||
streamOutRate = 0.020
|
||||
// The cache-read bucket is priced separately from input, so a run that
|
||||
// folded the two together fails the per-bucket assertions below.
|
||||
streamCacheReadRate = 0.001
|
||||
)
|
||||
|
||||
// TestStreamingResponseMetersInputTokens is the end-to-end guard for the
|
||||
// metering bug this endpoint's gateway-protocol work fixed.
|
||||
//
|
||||
// On a streamed answer the input-token count exists only in the opening
|
||||
// message_start event; every later frame reports output. A response read with
|
||||
// the wrong vendor's parser — the shape a gateway record produces when it names
|
||||
// one API surface and serves another — never looks at that event, so input
|
||||
// metered as zero and the bulk of the bill silently vanished. Nothing in the
|
||||
// suite sent stream: true before this test, so the whole branch went unrun.
|
||||
//
|
||||
// The provider points at the mock's streaming listener, which answers every
|
||||
// request as SSE with token counts that differ from the buffered surface. That
|
||||
// difference is the point: passing these assertions is only possible if the
|
||||
// stream accumulator ran.
|
||||
func TestStreamingResponseMetersInputTokens(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionStreamingProvider(t, ctx, "anthropic_api")
|
||||
|
||||
sessionID := fmt.Sprintf("e2e-session-stream-%d", time.Now().UnixNano())
|
||||
code, body := chatStreamUntil(t, ctx, env, harness.WireMessages, streamedModel, sessionID)
|
||||
require.Equal(t, 200, code, "streamed chat must succeed; body: %s", body)
|
||||
assert.Contains(t, body, "message_start",
|
||||
"the client must receive the event stream itself, not a buffered rewrite of it")
|
||||
|
||||
row := findAccessLogBySession(t, ctx, sessionID)
|
||||
|
||||
assert.Equal(t, harness.VLLMStreamInputTokens, int(row.InputTokens),
|
||||
"input tokens live in message_start; zero here is the bug this test exists for")
|
||||
assert.Equal(t, harness.VLLMStreamOutputTokens, int(row.OutputTokens),
|
||||
"output tokens ride message_delta and supersede the message_start seed")
|
||||
assert.Equal(t, harness.VLLMStreamCacheReadTokens, int(row.CachedInputTokens),
|
||||
"the Anthropic cache bucket rides message_start too, and only its own parser reads it")
|
||||
|
||||
// The Anthropic surface bills cache reads additively, so the input bucket
|
||||
// prices the full input count rather than a remainder.
|
||||
wantInput := float64(harness.VLLMStreamInputTokens) / 1000 * streamInRate
|
||||
wantOutput := float64(harness.VLLMStreamOutputTokens) / 1000 * streamOutRate
|
||||
wantCacheRead := float64(harness.VLLMStreamCacheReadTokens) / 1000 * streamCacheReadRate
|
||||
assert.InDelta(t, wantInput, row.InputCostUsd, 1e-6, "input cost must price the streamed input tokens")
|
||||
assert.InDelta(t, wantOutput, row.OutputCostUsd, 1e-6, "output cost must price the streamed output tokens")
|
||||
// The total, not merely a positive number: input and output alone are
|
||||
// positive, so a cache bucket parsed and then never billed would pass any
|
||||
// weaker assertion. The gap is 7e-6, well outside the delta.
|
||||
assert.InDelta(t, wantInput+wantOutput+wantCacheRead, row.CostUsd, 1e-6,
|
||||
"the recorded cost must be every bucket the surface bills, cache reads included")
|
||||
}
|
||||
|
||||
// TestStreamingOnGatewayTypedProvider drives the same streamed Anthropic call
|
||||
// through a provider record whose catalog id names the OpenAI surface — the
|
||||
// exact misconfiguration that hid the bug, since gateway records commonly pin
|
||||
// one parser while the upstream serves another shape entirely.
|
||||
//
|
||||
// The router must choose the parser from the request path rather than the
|
||||
// record's provider id, or the Anthropic usage block goes unread and input
|
||||
// meters at zero all over again.
|
||||
func TestStreamingOnGatewayTypedProvider(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionStreamingProvider(t, ctx, "openai_api")
|
||||
|
||||
sessionID := fmt.Sprintf("e2e-session-stream-gw-%d", time.Now().UnixNano())
|
||||
code, body := chatStreamUntil(t, ctx, env, harness.WireMessages, streamedModel, sessionID)
|
||||
require.Equal(t, 200, code, "streamed chat through a gateway record must succeed; body: %s", body)
|
||||
|
||||
row := findAccessLogBySession(t, ctx, sessionID)
|
||||
|
||||
assert.Equal(t, harness.VLLMStreamInputTokens, int(row.InputTokens),
|
||||
"a record typed openai_api must still read the Anthropic usage block it is actually serving")
|
||||
assert.Equal(t, harness.VLLMStreamOutputTokens, int(row.OutputTokens),
|
||||
"output tokens must survive the surface mismatch too")
|
||||
assert.InDelta(t, float64(harness.VLLMStreamInputTokens)/1000*streamInRate, row.InputCostUsd, 1e-6,
|
||||
"the request must be priced on the surface it spoke, not the one the record names")
|
||||
}
|
||||
|
||||
// provisionStreamingProvider brings up the mock, one provider pointed at its
|
||||
// streaming listener under the given catalog id, a policy authorising it, and a
|
||||
// connected proxy + client.
|
||||
func provisionStreamingProvider(t *testing.T, ctx context.Context, catalogID string) pricedEnv {
|
||||
t.Helper()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
name := "stream-" + catalogID
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-" + name})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-" + name + "-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
// Deleting the group does not delete the key it auto-joins, so the key
|
||||
// needs a cleanup of its own.
|
||||
t.Cleanup(func() { _ = srv.API().SetupKeys.Delete(context.Background(), sk.Id) })
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
dummyKey := "sk-stream-e2e"
|
||||
cacheRead := streamCacheReadRate
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: name,
|
||||
ProviderId: catalogID,
|
||||
UpstreamUrl: vllm.StreamURL,
|
||||
ApiKey: &dummyKey,
|
||||
Enabled: ptr(true),
|
||||
Models: &[]api.AgentNetworkProviderModel{{
|
||||
Id: streamedModel,
|
||||
InputPer1k: streamInRate,
|
||||
OutputPer1k: streamOutRate,
|
||||
CacheReadPer1k: &cacheRead,
|
||||
}},
|
||||
})
|
||||
require.NoError(t, err, "create provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-" + name,
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
Limits: &api.AgentNetworkPolicyLimits{
|
||||
TokenLimit: api.AgentNetworkPolicyTokenLimit{
|
||||
Enabled: true,
|
||||
GroupCap: 10_000_000,
|
||||
UserCap: 10_000_000,
|
||||
WindowSeconds: 60,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
endpoint, proxyIP, cl, px := connectClient(t, ctx, name, sk.Key)
|
||||
return pricedEnv{
|
||||
providerID: prov.Id,
|
||||
groupID: grp.Id,
|
||||
policyID: pol.Id,
|
||||
upstream: vllm.StreamURL,
|
||||
endpoint: endpoint,
|
||||
proxyIP: proxyIP,
|
||||
client: cl,
|
||||
proxy: px,
|
||||
}
|
||||
}
|
||||
|
||||
// chatStreamUntil drives one streamed chat, retrying to absorb the tunnel and
|
||||
// DNS jitter a first call through a fresh peer can hit.
|
||||
func chatStreamUntil(t *testing.T, ctx context.Context, env pricedEnv, kind, model, sessionID string) (int, string) {
|
||||
t.Helper()
|
||||
var code int
|
||||
var body string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
c, b, cerr := env.client.ChatStream(ctx, env.endpoint, env.proxyIP, kind, model, "Reply with exactly: pong", sessionID)
|
||||
if cerr == nil {
|
||||
code, body = c, b
|
||||
if code == 200 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !waitBeforeRetry(ctx, 5*time.Second) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if code != 200 {
|
||||
t.Logf("=== proxy logs ===\n%s", env.proxy.Logs(context.Background()))
|
||||
}
|
||||
return code, body
|
||||
}
|
||||
+48
-5
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -292,6 +293,27 @@ func (cl *Client) ChatPrefixed(ctx context.Context, endpoint, proxyIP, pathPrefi
|
||||
return cl.post(ctx, endpoint, proxyIP, pathPrefix+path, body, withSessionID(headers, sessionID))
|
||||
}
|
||||
|
||||
// ChatStream is Chat with "stream": true in the request body, so the proxy's
|
||||
// request parser marks the call as streaming and its response parser takes the
|
||||
// SSE accumulator rather than the buffered-body path. Pair it with a provider
|
||||
// pointed at VLLM.StreamURL, which answers every request as an event stream.
|
||||
func (cl *Client) ChatStream(ctx context.Context, endpoint, proxyIP, kind, model, prompt, sessionID string) (int, string, error) {
|
||||
var path, body string
|
||||
var headers []string
|
||||
switch kind {
|
||||
case WireMessages:
|
||||
path = "/v1/messages"
|
||||
headers = []string{"anthropic-version: 2023-06-01"}
|
||||
body = fmt.Sprintf(`{"model":%q,"max_tokens":2048,"stream":true,"messages":[{"role":"user","content":%q}]}`, model, prompt)
|
||||
default:
|
||||
path = "/v1/chat/completions"
|
||||
// include_usage is what makes a real OpenAI stream emit its final usage
|
||||
// frame; without it the last chunk carries no tokens at all.
|
||||
body = fmt.Sprintf(`{"model":%q,"stream":true,"stream_options":{"include_usage":true},"messages":[{"role":"user","content":%q}]}`, model, prompt)
|
||||
}
|
||||
return cl.post(ctx, endpoint, proxyIP, path, body, withSessionID(headers, sessionID))
|
||||
}
|
||||
|
||||
// Vertex issues an Anthropic-on-Vertex rawPredict POST over the tunnel. Unlike
|
||||
// Chat, the model is carried in the request path (project/region/model), so the
|
||||
// proxy routes by path and mints the service-account OAuth token; the body uses
|
||||
@@ -322,10 +344,29 @@ func withSessionID(headers []string, sessionID string) []string {
|
||||
return append(headers, "x-session-id: "+sessionID)
|
||||
}
|
||||
|
||||
// post runs curl in a throwaway container sharing the client's network
|
||||
// namespace so the request traverses the WireGuard tunnel, pinning the endpoint
|
||||
// to the proxy IP. It returns the HTTP status and response body.
|
||||
// Get issues a GET to the agent-network endpoint over the client's tunnel.
|
||||
// Model discovery and the connection-warming probe are read-only endpoints
|
||||
// that carry no body, so they can't go through the chat helpers.
|
||||
func (cl *Client) Get(ctx context.Context, endpoint, proxyIP, path string, extraHeaders []string) (int, string, error) {
|
||||
return cl.do(ctx, http.MethodGet, endpoint, proxyIP, path, "", extraHeaders)
|
||||
}
|
||||
|
||||
// PostJSON issues an arbitrary JSON POST over the client's tunnel, for wire
|
||||
// shapes the typed helpers don't cover (token counting, say).
|
||||
func (cl *Client) PostJSON(ctx context.Context, endpoint, proxyIP, path, body string, extraHeaders []string) (int, string, error) {
|
||||
return cl.do(ctx, http.MethodPost, endpoint, proxyIP, path, body, extraHeaders)
|
||||
}
|
||||
|
||||
// post issues a JSON POST. Retained as the shorthand the chat helpers use.
|
||||
func (cl *Client) post(ctx context.Context, endpoint, proxyIP, path, body string, extraHeaders []string) (int, string, error) {
|
||||
return cl.do(ctx, http.MethodPost, endpoint, proxyIP, path, body, extraHeaders)
|
||||
}
|
||||
|
||||
// do runs curl in a throwaway container sharing the client's network
|
||||
// namespace so the request traverses the WireGuard tunnel, pinning the endpoint
|
||||
// to the proxy IP. It returns the HTTP status and response body. An empty body
|
||||
// sends no payload, which is what a GET needs.
|
||||
func (cl *Client) do(ctx context.Context, method, endpoint, proxyIP, path, body string, extraHeaders []string) (int, string, error) {
|
||||
url := "https://" + endpoint + path
|
||||
args := []string{
|
||||
"run", "--rm",
|
||||
@@ -334,13 +375,15 @@ func (cl *Client) post(ctx context.Context, endpoint, proxyIP, path, body string
|
||||
"-sk", "--connect-timeout", "5", "--max-time", "90",
|
||||
"--resolve", endpoint + ":443:" + proxyIP,
|
||||
"-o", "/dev/stderr", "-w", "%{http_code}",
|
||||
"-X", "POST", url,
|
||||
"-X", method, url,
|
||||
"-H", "Content-Type: application/json",
|
||||
}
|
||||
for _, h := range extraHeaders {
|
||||
args = append(args, "-H", h)
|
||||
}
|
||||
args = append(args, "--data", body)
|
||||
if body != "" {
|
||||
args = append(args, "--data", body)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "docker", args...)
|
||||
// -w writes the status code to stdout; -o /dev/stderr writes the body to
|
||||
// stderr so we can capture both separately.
|
||||
|
||||
+127
-8
@@ -18,18 +18,63 @@ const (
|
||||
vllmImage = "nginx:alpine"
|
||||
vllmAlias = "vllm"
|
||||
vllmPort = "8000/tcp"
|
||||
// vllmStreamPort serves the same wire shapes as an SSE stream. See the
|
||||
// nginx config for why streaming lives on its own listener.
|
||||
vllmStreamPort = "8001/tcp"
|
||||
// VLLMModel is the served model id the mock advertises and echoes back. It
|
||||
// matches a real small model commonly served by vLLM so the provider's
|
||||
// enumerated model and the client's request line up.
|
||||
VLLMModel = "Qwen/Qwen2.5-0.5B-Instruct"
|
||||
// VLLMUnlistedModel is a second id the mock's model listing advertises but
|
||||
// no test provider enumerates, so a filtered listing is observably shorter
|
||||
// than the upstream's own.
|
||||
VLLMUnlistedModel = "Qwen/Qwen2.5-7B-Instruct"
|
||||
)
|
||||
|
||||
// Token counts the mock reports per wire shape. Tests assert on these rather
|
||||
// than on "> 0" so a response parsed with the wrong provider's parser (which
|
||||
// would read a different field, or none) fails loudly instead of passing on
|
||||
// a coincidental non-zero.
|
||||
const (
|
||||
// VLLMChatInputTokens / VLLMChatOutputTokens ride the OpenAI usage block.
|
||||
VLLMChatInputTokens = 11
|
||||
VLLMChatOutputTokens = 2
|
||||
// VLLMMessagesInputTokens / VLLMMessagesOutputTokens ride the Anthropic
|
||||
// usage block, whose field names the OpenAI parser cannot read.
|
||||
VLLMMessagesInputTokens = 17
|
||||
VLLMMessagesOutputTokens = 3
|
||||
)
|
||||
|
||||
// Token counts the streaming surface reports. They differ from the
|
||||
// non-streaming ones on purpose: a test that asserts these numbers proves the
|
||||
// SSE accumulator ran, rather than a buffered JSON body having been parsed.
|
||||
//
|
||||
// Input and cache-read arrive on message_start; output arrives on
|
||||
// message_delta and supersedes the seed value message_start carries. Any
|
||||
// parser that cannot read message_start reports zero input tokens — which is
|
||||
// exactly the bug these counts exist to catch.
|
||||
const (
|
||||
VLLMStreamInputTokens = 29
|
||||
VLLMStreamOutputTokens = 5
|
||||
VLLMStreamCacheReadTokens = 7
|
||||
)
|
||||
|
||||
// vllmNginxConf emulates a vLLM OpenAI-compatible server over plain HTTP (vLLM's
|
||||
// default: no TLS, port 8000). It answers /v1/models with a one-model list and
|
||||
// any chat/completions path with a canned OpenAI-shaped chat completion carrying
|
||||
// a non-zero usage block, so the proxy's OpenAI parser records real token
|
||||
// consumption. Running actual vLLM in CI is infeasible (GPU + multi-GB model
|
||||
// default: no TLS, port 8000), and additionally answers the wire shapes the
|
||||
// other catalog surfaces speak so one mock can stand in for every provider the
|
||||
// proxy routes to. Running actual vLLM in CI is infeasible (GPU + multi-GB model
|
||||
// download), so this stands in for the wire contract the proxy depends on.
|
||||
//
|
||||
// Each shape answers with its own vendor's usage block, so a response parsed
|
||||
// under the wrong surface meters zero rather than passing by accident:
|
||||
//
|
||||
// - /v1/chat/completions (and any unmatched path): OpenAI chat completion.
|
||||
// - /v1/messages: Anthropic Messages, snake_case usage plus a cache bucket.
|
||||
// - /model/{id}/invoke: Bedrock InvokeModel, which carries the Anthropic body.
|
||||
// - the token-counting endpoints: a count, with no usage block at all.
|
||||
//
|
||||
// The model listing advertises two models so a policy that authorises one
|
||||
// produces an observably shorter list than the upstream's own.
|
||||
const vllmNginxConf = `pid /tmp/nginx.pid;
|
||||
events {}
|
||||
http {
|
||||
@@ -37,13 +82,75 @@ http {
|
||||
listen 8000;
|
||||
location = /v1/models {
|
||||
default_type application/json;
|
||||
return 200 '{"object":"list","data":[{"id":"Qwen/Qwen2.5-0.5B-Instruct","object":"model","owned_by":"vllm"}]}';
|
||||
return 200 '{"object":"list","data":[{"id":"Qwen/Qwen2.5-0.5B-Instruct","object":"model","owned_by":"vllm"},{"id":"Qwen/Qwen2.5-7B-Instruct","object":"model","owned_by":"vllm"}]}';
|
||||
}
|
||||
location = /v1/messages {
|
||||
default_type application/json;
|
||||
return 200 '{"id":"msg_e2e","type":"message","role":"assistant","model":"claude-sonnet-5","content":[{"type":"text","text":"pong"}],"stop_reason":"end_turn","usage":{"input_tokens":17,"output_tokens":3,"cache_read_input_tokens":5}}';
|
||||
}
|
||||
location = /v1/messages/count_tokens {
|
||||
default_type application/json;
|
||||
return 200 '{"input_tokens":7}';
|
||||
}
|
||||
location ~ ^/model/.+/invoke$ {
|
||||
default_type application/json;
|
||||
return 200 '{"id":"msg_e2e_bedrock","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"stop_reason":"end_turn","usage":{"input_tokens":17,"output_tokens":3,"cache_read_input_tokens":5}}';
|
||||
}
|
||||
location ~ ^/model/.+/count-tokens$ {
|
||||
default_type application/json;
|
||||
return 200 '{"inputTokens":9}';
|
||||
}
|
||||
location = /api/hello {
|
||||
return 200;
|
||||
}
|
||||
location = /inference-profiles {
|
||||
default_type application/json;
|
||||
return 200 '{"inferenceProfileSummaries":[{"inferenceProfileId":"us.anthropic.claude-sonnet-5","status":"ACTIVE"}]}';
|
||||
}
|
||||
location / {
|
||||
default_type application/json;
|
||||
return 200 '{"id":"chatcmpl-e2e-vllm","object":"chat.completion","created":1700000000,"model":"Qwen/Qwen2.5-0.5B-Instruct","choices":[{"index":0,"message":{"role":"assistant","content":"pong"},"finish_reason":"stop"}],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}';
|
||||
}
|
||||
}
|
||||
|
||||
# The streaming surface, on its own port so the response content type is a
|
||||
# property of the listener rather than of a per-request branch: nginx sets
|
||||
# Content-Type from default_type, which cannot be varied inside an "if", and
|
||||
# a second Content-Type via add_header would leave the proxy reading the
|
||||
# wrong one. A provider record pointed at this port streams every answer.
|
||||
#
|
||||
# Input and cache-read tokens ride message_start, output rides message_delta
|
||||
# — the split that makes a stream different from a buffered body, and the
|
||||
# reason a parser that ignores message_start meters input as zero.
|
||||
server {
|
||||
listen 8001;
|
||||
location = /v1/messages {
|
||||
default_type text/event-stream;
|
||||
return 200 'event: message_start
|
||||
data: {"type":"message_start","message":{"id":"msg_e2e_stream","type":"message","role":"assistant","model":"claude-sonnet-5","content":[],"usage":{"input_tokens":29,"output_tokens":1,"cache_read_input_tokens":7}}}
|
||||
|
||||
event: content_block_delta
|
||||
data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}
|
||||
|
||||
event: message_delta
|
||||
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}
|
||||
|
||||
event: message_stop
|
||||
data: {"type":"message_stop"}
|
||||
|
||||
';
|
||||
}
|
||||
location / {
|
||||
default_type text/event-stream;
|
||||
return 200 'data: {"choices":[{"delta":{"content":"pong"}}]}
|
||||
|
||||
data: {"choices":[],"usage":{"prompt_tokens":29,"completion_tokens":5,"total_tokens":34}}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
';
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
@@ -55,6 +162,10 @@ type VLLM struct {
|
||||
workDir string
|
||||
// URL is the upstream URL the vllm provider points at (http://<alias>:8000).
|
||||
URL string
|
||||
// StreamURL is the same mock's streaming listener. A provider pointed here
|
||||
// answers every request as SSE, so the proxy's streaming accumulator runs
|
||||
// instead of its buffered-body parser.
|
||||
StreamURL string
|
||||
}
|
||||
|
||||
// StartVLLM runs the mock vLLM server on the shared network over plain HTTP.
|
||||
@@ -73,14 +184,17 @@ func StartVLLM(ctx context.Context, c *Combined) (*VLLM, error) {
|
||||
|
||||
req := testcontainers.ContainerRequest{
|
||||
Image: vllmImage,
|
||||
ExposedPorts: []string{vllmPort},
|
||||
ExposedPorts: []string{vllmPort, vllmStreamPort},
|
||||
Networks: []string{c.network.Name},
|
||||
NetworkAliases: map[string][]string{c.network.Name: {vllmAlias}},
|
||||
Cmd: []string{"nginx", "-c", "/conf/nginx.conf", "-g", "daemon off;"},
|
||||
HostConfigModifier: func(hc *container.HostConfig) {
|
||||
hc.Binds = append(hc.Binds, workDir+":/conf:ro")
|
||||
},
|
||||
WaitingFor: wait.ForListeningPort(vllmPort).WithStartupTimeout(60 * time.Second),
|
||||
WaitingFor: wait.ForAll(
|
||||
wait.ForListeningPort(vllmPort),
|
||||
wait.ForListeningPort(vllmStreamPort),
|
||||
).WithStartupTimeout(60 * time.Second),
|
||||
}
|
||||
|
||||
ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
@@ -92,7 +206,12 @@ func StartVLLM(ctx context.Context, c *Combined) (*VLLM, error) {
|
||||
return nil, fmt.Errorf("start vllm container: %w", err)
|
||||
}
|
||||
|
||||
return &VLLM{container: ctr, workDir: workDir, URL: "http://" + vllmAlias + ":8000"}, nil
|
||||
return &VLLM{
|
||||
container: ctr,
|
||||
workDir: workDir,
|
||||
URL: "http://" + vllmAlias + ":8000",
|
||||
StreamURL: "http://" + vllmAlias + ":8001",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Logs returns the vLLM container logs, for diagnostics on failure.
|
||||
|
||||
Reference in New Issue
Block a user