mirror of
https://github.com/netbirdio/netbird.git
synced 2026-10-27 06:49:07 +01:00
[proxy,management] Conform the Agent Network endpoint to the LLM gateway protocol (#7154)
[proxy,management] Conform the Agent Network endpoint to the LLM gateway protocol Reviewed the proxy against Claude Code's published gateway contract. The transport layer already held up; fourteen gaps sat one layer up, in the model catalog and in the non-inference endpoints clients call. Two of them cost money. The catalog carried no claude-opus-5 or claude-sonnet-5, so an operator could not authorise the models coding agents default to — those requests denied as not-routable, or priced at zero where a catch-all carried them. And gateway records pin ParserID "openai" while the same record serves /v1/messages, so Anthropic responses were read with the OpenAI parser, which never looks at message_start where input tokens live: input metered as roughly zero on every stream and cost was skipped entirely. The rest fix requests refused for structural rather than policy reasons: model discovery denied for every account with a model allowlist, token counting denied on Bedrock and mis-parsed on Vertex, startup probes refused and written into the access log at every session start, and denials rendered in a shape no LLM client parses. Two changes are additive by design — the deny body keeps every field it had and adds the vendor's error object alongside, and body-level identity injection is now gated on the request's dialect so it stops sending OpenAI-shape fields into Anthropic bodies that reject them. The end-to-end work turned up one more: the discovery filter treated any slash in a model id as a gateway prefix, which would have dropped every self-hosted "Qwen/..." model from the picker.
This commit is contained in:
@@ -84,6 +84,15 @@ func (m *Middleware) Invoke(ctx context.Context, in *middleware.Input) (*middlew
|
||||
return allowNoAttribution(), nil
|
||||
}
|
||||
|
||||
// Model-listing and other non-inference endpoints carry no model, and
|
||||
// management's per-model allowlist fails closed on an empty one. The
|
||||
// router has already authorised the route against the caller's groups
|
||||
// and the request consumes no tokens, so gating it on a model that
|
||||
// cannot exist would only break gateway model discovery.
|
||||
if lookupKV(in.Metadata, middleware.KeyLLMNonInference) == "true" {
|
||||
return allowNoAttribution(), nil
|
||||
}
|
||||
|
||||
providerID := lookupKV(in.Metadata, middleware.KeyLLMResolvedProviderID)
|
||||
if providerID == "" {
|
||||
// llm_router didn't emit a resolved provider id — usually
|
||||
@@ -117,7 +126,7 @@ func (m *Middleware) Invoke(ctx context.Context, in *middleware.Input) (*middlew
|
||||
}
|
||||
|
||||
if resp.GetDecision() == "deny" {
|
||||
return denyFromManagement(resp), nil
|
||||
return denyFromManagement(resp, lookupKV(in.Metadata, middleware.KeyLLMProvider)), nil
|
||||
}
|
||||
return allowFromManagement(resp), nil
|
||||
}
|
||||
@@ -161,7 +170,7 @@ func allowFromManagement(resp *proto.CheckLLMPolicyLimitsResponse) *middleware.O
|
||||
// envelope. The deny code surfaces verbatim through the framework's
|
||||
// fixed JSON template; arbitrary middleware bytes can't reach the
|
||||
// wire.
|
||||
func denyFromManagement(resp *proto.CheckLLMPolicyLimitsResponse) *middleware.Output {
|
||||
func denyFromManagement(resp *proto.CheckLLMPolicyLimitsResponse, surface string) *middleware.Output {
|
||||
code := resp.GetDenyCode()
|
||||
if code == "" {
|
||||
code = "llm_policy.cap_exceeded"
|
||||
@@ -176,6 +185,7 @@ func denyFromManagement(resp *proto.CheckLLMPolicyLimitsResponse) *middleware.Ou
|
||||
DenyReason: &middleware.DenyReason{
|
||||
Code: code,
|
||||
Message: denyMessageForCode(code),
|
||||
Surface: surface,
|
||||
},
|
||||
Metadata: []middleware.KV{
|
||||
{Key: middleware.KeyLLMPolicyDecision, Value: "deny"},
|
||||
|
||||
@@ -224,3 +224,35 @@ func TestMetadataKeys_Allowlist(t *testing.T) {
|
||||
}
|
||||
assert.ElementsMatch(t, want, keys)
|
||||
}
|
||||
|
||||
// TestInvoke_NonInferenceSkipsPreflight covers gateway model discovery:
|
||||
// GET /v1/models carries no model, and management's per-model allowlist
|
||||
// fails closed on an empty one, so a pre-flight would deny discovery for
|
||||
// exactly the accounts that use the model allowlist. The router marks the
|
||||
// request non-inference after authorising the route, and the gate must
|
||||
// then allow without calling management at all.
|
||||
func TestInvoke_NonInferenceSkipsPreflight(t *testing.T) {
|
||||
mgmt := &fakeMgmt{
|
||||
checkResp: &proto.CheckLLMPolicyLimitsResponse{
|
||||
Decision: "deny",
|
||||
DenyCode: "llm_policy.model_blocked",
|
||||
},
|
||||
}
|
||||
m := New(mgmt, nil)
|
||||
|
||||
out := runInvoke(t, m, &middleware.Input{
|
||||
AccountID: "acc-1",
|
||||
UserID: "user-bob",
|
||||
UserGroups: []string{"grp-engineers"},
|
||||
Metadata: []middleware.KV{
|
||||
{Key: middleware.KeyLLMResolvedProviderID, Value: "prov-1"},
|
||||
{Key: middleware.KeyLLMNonInference, Value: "true"},
|
||||
},
|
||||
})
|
||||
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "model-less endpoints must not be gated on a model")
|
||||
assert.Nil(t, mgmt.checkReq, "no pre-flight may be sent for a non-inference request")
|
||||
|
||||
assert.Empty(t, lookupKV(out.Metadata, middleware.KeyLLMSelectedPolicyID),
|
||||
"no policy is attributed when nothing was metered")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user