mirror of
https://github.com/netbirdio/netbird.git
synced 2026-09-20 05:39:07 +02:00
[management,proxy] Serve guardrail allowlists of declared model ids (#7389)
After #7221, guardrail allowlists built from a path-style provider's declared model ids (Bedrock, Vertex) stopped working: the raw region/version form was compared against the parser's canonical id, so the agent config advertised an empty model list and requests for the allowlisted model were refused. Make every allowlist compare provider-aware, keyed on the destination provider's catalog id: the agent config, the policy gate, and the synthesized proxy allowlists match an entry on both its verbatim and canonical form — Bedrock's strip only under bedrock_api, Vertex's only under vertex_ai_api, verbatim everywhere else, so a plain provider's suffixed entries never widen. The router's claim compare learns the Vertex @version strip. New e2e, realstore, and unit tests reproduce both regressions and pin the fix.
This commit is contained in:
Maycon Santos
@@ -0,0 +1,143 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// joinGroup places the PAT's own user into the group so caller-scoped answers
|
||||
// (GET /api/agent-network/agent-config) see the policies sourced from it, and
|
||||
// restores the previous auto-groups on cleanup. Self-service updates of one's
|
||||
// own auto_groups are permitted for every role, so this needs no second user.
|
||||
func joinGroup(t *testing.T, ctx context.Context, groupID string) {
|
||||
t.Helper()
|
||||
me, err := srv.API().Users.Current(ctx)
|
||||
require.NoError(t, err, "read current user")
|
||||
before := append([]string(nil), me.AutoGroups...)
|
||||
_, err = srv.API().Users.Update(ctx, me.Id, api.PutApiUsersUserIdJSONRequestBody{
|
||||
Role: me.Role,
|
||||
IsBlocked: me.IsBlocked,
|
||||
AutoGroups: append(append([]string(nil), before...), groupID),
|
||||
})
|
||||
require.NoError(t, err, "add the caller to the policy source group")
|
||||
t.Cleanup(func() {
|
||||
_, _ = srv.API().Users.Update(context.Background(), me.Id, api.PutApiUsersUserIdJSONRequestBody{
|
||||
Role: me.Role,
|
||||
IsBlocked: me.IsBlocked,
|
||||
AutoGroups: before,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// configProvider returns the agent-config entry for the named provider, nil
|
||||
// when the answer does not offer it. The suite shares one account, so other
|
||||
// tests' fixtures may add unrelated providers to the caller's answer.
|
||||
func configProvider(cfg api.AgentNetworkAgentConfig, name string) *api.AgentNetworkAgentConfigProvider {
|
||||
for i := range cfg.Providers {
|
||||
if cfg.Providers[i].Name == name {
|
||||
return &cfg.Providers[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestAgentConfigAllowlistOfDeclaredModels reproduces the post-#7221 field
|
||||
// report: a provider carrying a declared model set plus a policy guardrail
|
||||
// whose allowlist holds those same declared ids must advertise the models on
|
||||
// GET /api/agent-network/agent-config — the guardrail was built FROM the
|
||||
// provider's model list (the dashboard's allowlist picker persists the
|
||||
// declared ids verbatim), so nothing about the setup excludes them.
|
||||
//
|
||||
// The plain case passes today. The path-style case (Bedrock; Vertex has the
|
||||
// same shape) fails: the declared id is compared through the proxy parser's
|
||||
// canonical form (region prefix and version suffix stripped) while the
|
||||
// allowlist entry is not, so the raw-vs-raw pair never intersects and the
|
||||
// caller sees an empty model list. The same one-sided normalization sits in
|
||||
// policyPermitsModel, so the proxy also denies the model at request time —
|
||||
// the guardrail meant to allow exactly this model turns it off end to end.
|
||||
func TestAgentConfigAllowlistOfDeclaredModels(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
catalogID string
|
||||
upstream string
|
||||
declared string
|
||||
}{
|
||||
{
|
||||
name: "plain-declared-id",
|
||||
catalogID: "openai_api",
|
||||
upstream: "https://api.openai.com",
|
||||
declared: "gpt-4o-mini",
|
||||
},
|
||||
{
|
||||
// The operator declares the id AWS issues — region-prefixed
|
||||
// inference profile with a version suffix — and the allowlist
|
||||
// picker copies it as-is.
|
||||
name: "bedrock-declared-id",
|
||||
catalogID: "bedrock_api",
|
||||
upstream: "https://bedrock-runtime.eu-central-1.amazonaws.com",
|
||||
declared: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-agentcfg-" + tc.name})
|
||||
require.NoError(t, err, "create source group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
joinGroup(t, ctx, grp.Id)
|
||||
|
||||
providerName := "e2e-agentcfg-" + tc.name
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: providerName,
|
||||
ProviderId: tc.catalogID,
|
||||
UpstreamUrl: tc.upstream,
|
||||
ApiKey: ptr("sk-dummy-e2e-key"),
|
||||
Enabled: ptr(true),
|
||||
Models: &[]api.AgentNetworkProviderModel{{Id: tc.declared, InputPer1k: 0.001, OutputPer1k: 0.002}},
|
||||
})
|
||||
require.NoError(t, err, "create provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
// Allowlist exactly the declared model, the way the dashboard
|
||||
// builds a guardrail from the provider's model list.
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = "e2e-agentcfg-" + tc.name
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{tc.declared}
|
||||
guard, err := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, err, "create guardrail")
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) })
|
||||
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-agentcfg-" + tc.name,
|
||||
Enabled: ptr(true),
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{guard.Id},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
cfg, err := srv.GetAgentConfig(ctx)
|
||||
require.NoError(t, err, "read the caller-scoped agent config")
|
||||
require.True(t, cfg.Configured, "the account endpoint is bootstrapped by TestMain")
|
||||
|
||||
entry := configProvider(cfg, providerName)
|
||||
require.NotNil(t, entry, "the policy authorizes the caller for the provider, so it must be offered")
|
||||
assert.False(t, entry.AllModelsAllowed, "an allowlist guardrail restricts the provider")
|
||||
assert.Equal(t, []string{tc.declared}, entry.Models,
|
||||
"the allowlist holds the provider's own declared id, so that model must be advertised")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// TestModelAllowlistOfDeclaredIDsServed drives the setup an operator actually
|
||||
// builds for a path-routed provider: the models are declared in the form the
|
||||
// vendor issues (Bedrock's region-prefixed, versioned inference-profile id;
|
||||
// Vertex's model@version), and the guardrail allowlist is built from that
|
||||
// declared list — the dashboard's allowlist picker persists the declared ids
|
||||
// verbatim. A request for the declared model must be served end to end, and a
|
||||
// model outside the allowlist must still be denied.
|
||||
//
|
||||
// TestModelAllowlistEnforced never caught this because it registers and
|
||||
// allowlists the pre-normalized catalog form (see the catalogModel comment
|
||||
// there and the one in providerRequest: "register the normalized form here or
|
||||
// routing fails as model_not_routable") — the harness encoded the
|
||||
// canonicalization workaround instead of the shape operators configure.
|
||||
func TestModelAllowlistOfDeclaredIDsServed(t *testing.T) {
|
||||
var providers []providerCase
|
||||
for _, pc := range availableProviders() {
|
||||
if pc.kind == harness.WireBedrock || pc.kind == harness.WireVertex {
|
||||
providers = append(providers, pc)
|
||||
}
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
t.Skip("no path-routed provider keys set (AWS_BEARER_TOKEN_BEDROCK / GOOGLE_VERTEX_*); source ~/.llm-keys")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-declared-allowlist"})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-declared-allowlist-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
t.Cleanup(func() { _ = srv.API().SetupKeys.Delete(context.Background(), sk.Id) })
|
||||
|
||||
// Providers declaring the raw vendor-issued model id — NOT the normalized
|
||||
// catalog form providerRequest would register.
|
||||
ids := make([]string, 0, len(providers))
|
||||
declared := make([]string, 0, len(providers))
|
||||
for _, pc := range providers {
|
||||
req := providerRequest(pc)
|
||||
req.Models = &[]api.AgentNetworkProviderModel{{Id: pc.model, InputPer1k: 0.001, OutputPer1k: 0.002}}
|
||||
prov, perr := srv.CreateProvider(ctx, req)
|
||||
require.NoError(t, perr, "create provider %s", pc.name)
|
||||
id := prov.Id
|
||||
ids = append(ids, id)
|
||||
declared = append(declared, pc.model)
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), id) })
|
||||
}
|
||||
|
||||
// Guardrail allowlisting the declared ids verbatim, the way the dashboard
|
||||
// builds an allowlist from the providers' model lists.
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = "e2e-declared-allowlist"
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = declared
|
||||
guard, err := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, err, "create guardrail")
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) })
|
||||
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-declared-allowlist",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: ids,
|
||||
GuardrailIds: &[]string{guard.Id},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
settings, err := srv.GetSettings(ctx)
|
||||
require.NoError(t, err, "read settings for endpoint")
|
||||
require.NotEmpty(t, settings.Endpoint, "agent-network endpoint must be assigned")
|
||||
|
||||
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-proxy-declared-allowlist")
|
||||
require.NoError(t, err, "mint proxy token via CLI")
|
||||
px, err := harness.StartProxy(ctx, srv, proxyToken)
|
||||
require.NoError(t, err, "start proxy")
|
||||
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, sk.Key)
|
||||
require.NoError(t, err, "start client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
|
||||
// Probe first: the GET resolves the endpoint (DNS error fails) and its first packet wakes the lazy proxy peer, so WaitProxyPeer sees it connected; any HTTP status counts.
|
||||
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
|
||||
require.NoError(t, err, "resolve agent-network endpoint to proxy IP")
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
|
||||
for _, pc := range providers {
|
||||
pc := pc
|
||||
t.Run(pc.name, func(t *testing.T) {
|
||||
// The model the operator declared and allowlisted is served end to
|
||||
// end: the route must claim it and the guardrail must permit it,
|
||||
// both through the canonicalization the parser applies at request
|
||||
// time — whatever id form the operator configured.
|
||||
assert.Equal(t, 200, sendModel(ctx, t, cl, settings.Endpoint, proxyIP, pc, pc.model),
|
||||
"the declared and allowlisted model must be served for %s", pc.name)
|
||||
// A model outside the allowlist stays denied.
|
||||
assert.Equal(t, 403, sendModel(ctx, t, cl, settings.Endpoint, proxyIP, pc, disallowedModel(pc)),
|
||||
"model outside the allowlist must be denied for %s", pc.name)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user