[management] validate the proxy cluster an agent network bootstraps onto

The agent network gateway service is synthesised as private: agents reach
it over the WireGuard tunnel and are authorised by ValidateTunnelPeer
against the enabled policies' source groups, and its only target is the
cluster itself with DirectUpstream. Only a proxy running embedded in a
netbird client can serve that, which management already reports per
cluster as the `private` capability.

CreateSettings accepted any hostname as proxy_address, so a labeled
bootstrap could pin the account to a cluster that cannot serve its
gateway — another account's BYOP cluster, or one whose proxies are all
centralised. The endpoint assigned at bootstrap is immutable, so the
account is then stuck with a dead gateway until someone deletes and
re-bootstraps the settings row.

Validate the cluster before allocating an endpoint beneath it: a cluster
whose live proxies have reported their capabilities must be one the
account may route through and must be private-capable. A cluster nothing
is connected to is left alone, so claiming an address ahead of the
proxy's first connection keeps working — the same address-first order the
dedicated (self-addressed) path documents, and the one the e2e suite and
self-hosted setups follow.

The e2e coverage drives the real thing: one combined server and two
proxies in the same cluster — a centralised one that makes the cluster
live but unusable, then an embedded one that makes it usable — so both
the refusal and the acceptance are exercised against the same account and
cluster address, with the domains endpoint (the list the dashboard picks
from) as the barrier between starting a proxy and asserting on it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude
2026-09-01 22:56:07 +00:00
parent 11733fd718
commit 083906b84d
6 changed files with 382 additions and 0 deletions
@@ -0,0 +1,122 @@
//go:build e2e
package agentnetwork
import (
"context"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/e2e/harness"
"github.com/netbirdio/netbird/shared/management/http/api"
)
// TestSettingsBootstrapValidatesProxyCluster covers the bootstrap-time check
// on the picked cluster, end to end against a real proxy.
//
// The synthesised gateway service is always private: agents reach it over the
// WireGuard tunnel and are authorised by their peer identity. Only a proxy
// running embedded in a netbird client (`netbird proxy --private`) can serve
// that, and management reports it per cluster as the `private` capability —
// the same supports_private flag the dashboard reads to decide which clusters
// it may offer. The endpoint assigned at bootstrap is immutable, so pinning
// to a cluster that cannot serve it has to be refused up front rather than
// leaving the account with a dead gateway.
//
// One combined server, two proxies in the same cluster: the centralised one
// makes the cluster live-but-unusable, the embedded one added afterwards
// makes it usable (the capability is any-true across the cluster's live
// proxies), so both the refusal and the acceptance are exercised against the
// same account and the same cluster address.
func TestSettingsBootstrapValidatesProxyCluster(t *testing.T) {
ctx := context.Background()
fresh, err := harnessStartFresh(ctx, t)
require.NoError(t, err, "start dedicated combined server")
proxyToken, err := fresh.CreateProxyTokenCLI(ctx, "e2e-cluster-validation")
require.NoError(t, err, "mint proxy token via CLI")
const cluster = harness.AgentNetworkCluster
// A centralised proxy: connected and serving the cluster, but not
// embedded in a netbird client, so it cannot authenticate tunnel peers.
central, err := harness.StartProxy(ctx, fresh, proxyToken, map[string]string{
"NB_PROXY_PRIVATE": "false",
})
require.NoError(t, err, "start centralised proxy")
t.Cleanup(func() { _ = central.Terminate(context.Background()) })
waitClusterPrivate(ctx, t, fresh, cluster, false)
_, err = fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{
ProxyAddress: ptr(cluster),
})
require.Error(t, err, "bootstrap onto a cluster with no embedded proxy must be refused")
requireClientError(t, err)
assert.Contains(t, err.Error(), "embedded proxy",
"the refusal must name what the cluster is missing: %v", err)
after, err := fresh.GetSettings(ctx)
require.NoError(t, err, "settings must still read after a refused bootstrap")
assert.Empty(t, after.Endpoint, "a refused bootstrap must not assign an endpoint")
assert.Empty(t, after.ProxyAddress, "a refused bootstrap must not pin a cluster")
// Add an embedded proxy to the same cluster: now it can serve a private
// service, and the very same request must go through.
embedded, err := harness.StartProxy(ctx, fresh, proxyToken)
require.NoError(t, err, "start embedded proxy")
t.Cleanup(func() { _ = embedded.Terminate(context.Background()) })
waitClusterPrivate(ctx, t, fresh, cluster, true)
bootstrapped, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{
ProxyAddress: ptr(cluster),
})
require.NoError(t, err, "bootstrap onto a private-capable cluster must succeed")
assert.Equal(t, cluster, bootstrapped.ProxyAddress, "the pinned cluster is the requested one")
assert.True(t, strings.HasSuffix(bootstrapped.Endpoint, "."+cluster),
"the endpoint must hang one label beneath the cluster: %s", bootstrapped.Endpoint)
}
// waitClusterPrivate polls the domains endpoint — the list the dashboard picks
// its bootstrap cluster from — until the free domain for clusterAddr reports
// supports_private == want. A proxy's capabilities land when it registers, so
// this is the barrier between starting a proxy and asserting on what
// management thinks its cluster can do.
func waitClusterPrivate(ctx context.Context, t *testing.T, c *harness.Combined, clusterAddr string, want bool) {
t.Helper()
deadline := time.Now().Add(90 * time.Second)
var last string
for time.Now().Before(deadline) {
domains, err := c.API().ReverseProxyDomains.List(ctx)
if err != nil {
last = "list domains: " + err.Error()
} else {
last = "cluster not listed"
for _, d := range domains {
if d.Domain != clusterAddr {
continue
}
if d.SupportsPrivate == nil {
last = "supports_private not reported yet"
break
}
if *d.SupportsPrivate == want {
return
}
last = "supports_private is not the expected value"
break
}
}
if !waitBeforeRetry(ctx, 2*time.Second) {
break
}
}
t.Fatalf("cluster %s never reported supports_private=%v: %s", clusterAddr, want, last)
}
@@ -9,6 +9,7 @@ import (
"runtime"
"strings"
"testing"
"time"
"go.uber.org/mock/gomock"
"github.com/gorilla/mux"
@@ -17,6 +18,7 @@ import (
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
rpproxy "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
"github.com/netbirdio/netbird/management/server/account"
nbcontext "github.com/netbirdio/netbird/management/server/context"
"github.com/netbirdio/netbird/management/server/permissions"
@@ -29,6 +31,9 @@ import (
const (
testAccountID = "acc-1"
testUserID = "user-bob"
// testClusterAddress is the shared proxy cluster the settings tests pin
// their gateway to; the fixture seeds a connected embedded proxy for it.
testClusterAddress = "eu.proxy.netbird.io"
)
// agentNetworkHandlerFixture builds a real agentnetwork.Manager with
@@ -75,6 +80,12 @@ func newAgentNetworkHandlerFixture(t *testing.T) *agentNetworkHandlerFixture {
manager := agentnetwork.NewManager(st, perms, accounts, nil)
h := &handler{manager: manager}
// The labeled bootstrap validates its proxy_address against the live
// clusters, so seed the shared cluster these tests pin to as a real,
// private-capable one — the wire-shape assertions then run through the
// validated path rather than the "nothing connected yet" carve-out.
seedSharedEmbeddedCluster(t, st, testClusterAddress)
router := mux.NewRouter()
router.HandleFunc("/agent-network/providers", h.createProvider).Methods("POST")
router.HandleFunc("/agent-network/providers/{providerId}", h.getProvider).Methods("GET")
@@ -268,3 +279,21 @@ func TestConsumptionHandler_PopulatedAccountListsRows(t *testing.T) {
assert.Equal(t, groupRow.WindowStartUtc, userRow.WindowStartUtc,
"rows recorded in the same window must share the aligned window_start_utc")
}
// seedSharedEmbeddedCluster registers a connected, NetBird-operated proxy
// running embedded in a netbird client (the `private` capability) so
// clusterAddr is a cluster any account may pin its agent-network gateway to.
func seedSharedEmbeddedCluster(t *testing.T, st store.Store, clusterAddr string) {
t.Helper()
private := true
now := time.Now().UTC()
require.NoError(t, st.SaveProxy(context.Background(), &rpproxy.Proxy{
ID: "shared-proxy-" + clusterAddr,
SessionID: "shared-session",
ClusterAddress: clusterAddr,
LastSeen: now,
ConnectedAt: &now,
Status: rpproxy.StatusConnected,
Capabilities: rpproxy.Capabilities{Private: &private},
}), "seeding the shared proxy cluster must succeed")
}
@@ -860,18 +860,108 @@ func (m *managerImpl) bootstrapSelfAddressed(ctx context.Context, settings *type
return nil
}
// validateGatewayCluster rejects a labeled bootstrap pinned to a cluster that
// cannot serve the account's gateway.
//
// The synthesised gateway service is unconditionally private
// (buildAccountService): agents reach it over the WireGuard tunnel and are
// authorised by ValidateTunnelPeer against the policies' source groups, and
// its single target is the cluster itself with DirectUpstream. Only a proxy
// running embedded in a netbird client (`netbird proxy`) can serve that — a
// centralised proxy has no tunnel identity to authenticate against and no
// WireGuard endpoint to be reached on. Management reports that per cluster as
// the `private` capability, the same flag the dashboard renders as
// supports_private when it gates NetBird-only services.
//
// Without this check the bootstrap happily pins to any hostname the caller
// names, including a cluster the account cannot use or one with no embedded
// proxy — and the endpoint it allocates is immutable, so the account is left
// with a dead gateway that only a DeleteSettings/re-bootstrap can undo.
//
// Only a cluster management can actually judge is rejected: one whose live
// proxies have reported their capabilities. A cluster nothing is connected to
// is left alone, because pinning ahead of the proxy's first connection is a
// legitimate order (the dedicated path claims an address the same way, before
// any proxy declares it).
func (m *managerImpl) validateGatewayCluster(ctx context.Context, accountID, clusterAddr string) error {
private := m.store.GetClusterSupportsPrivate(ctx, clusterAddr)
if private == nil {
// No live proxy in the cluster reported its capabilities: either
// nothing is connected there yet, or the proxies predate capability
// reporting. Nothing to judge — let the pin through.
return nil
}
available, err := m.accountClusterAddresses(ctx, accountID)
if err != nil {
return err
}
if !slices.Contains(available, clusterAddr) {
// Live, but not a cluster this account may route through: another
// account's BYOP cluster. Its proxies filter foreign mappings out on
// delivery, so the pin would be dead on arrival.
return status.Errorf(status.InvalidArgument,
"proxy cluster %s is not available to this account", clusterAddr)
}
if !*private {
return status.Errorf(status.InvalidArgument,
"proxy cluster %s cannot serve the agent network gateway: the gateway is reachable only from connected peers, "+
"which needs at least one connected embedded proxy (netbird proxy) in the cluster", clusterAddr)
}
return nil
}
// accountClusterAddresses lists the active proxy cluster addresses the account
// may pin its gateway to: its own (BYOP) clusters plus the shared ones. This
// mirrors the free-domain allow list the dashboard offers as cluster choices,
// so the API accepts exactly what the UI can present. Addresses are stored as
// the proxy declared them; they are normalised here so the comparison against
// a normalised proxy_address is not defeated by case.
func (m *managerImpl) accountClusterAddresses(ctx context.Context, accountID string) ([]string, error) {
byop, err := m.store.GetActiveProxyClusterAddressesForAccount(ctx, accountID)
if err != nil {
return nil, fmt.Errorf("list account proxy clusters: %w", err)
}
shared, err := m.store.GetActiveProxyClusterAddresses(ctx)
if err != nil {
return nil, fmt.Errorf("list shared proxy clusters: %w", err)
}
addresses := make([]string, 0, len(byop)+len(shared))
for _, addr := range slices.Concat(byop, shared) {
normalized, err := types.NormalizeHostname(addr)
if err != nil {
// A cluster address the proxy declared in a shape we cannot
// normalise is not one an endpoint can be allocated beneath.
log.WithContext(ctx).Debugf("skipping unusable proxy cluster address %q: %s", addr, err)
continue
}
addresses = append(addresses, normalized)
}
return addresses, nil
}
// bootstrapLabeled allocates a labeled endpoint one label beneath the given
// cluster address: Domain = <label>.<proxyAddress>, served by whichever proxy
// declares the parent. Labels are adjective-noun tuples; a candidate is
// checked by read and the domain unique index stays the authority, so a
// concurrent allocation of the same tuple surfaces as a unique violation and
// another tuple is drawn.
//
// Unlike the self-addressed path this pins to a cluster that must already
// exist, so the cluster is validated before an endpoint is allocated beneath
// it.
func (m *managerImpl) bootstrapLabeled(ctx context.Context, settings *types.Settings, proxyAddress string) error {
parent, err := types.NormalizeHostname(proxyAddress)
if err != nil {
return status.Errorf(status.InvalidArgument, "invalid proxy_address: %s", err)
}
if err := m.validateGatewayCluster(ctx, settings.AccountID, parent); err != nil {
return err
}
for attempt := 1; attempt <= maxDomainAllocationAttempts; attempt++ {
m.labelRngMu.Lock()
label := labelgen.PickTuple(m.labelRng)
@@ -5,12 +5,14 @@ import (
"runtime"
"strings"
"testing"
"time"
"go.uber.org/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
"github.com/netbirdio/netbird/management/server/account"
"github.com/netbirdio/netbird/management/server/permissions"
"github.com/netbirdio/netbird/management/server/permissions/modules"
@@ -64,6 +66,33 @@ func (f *bootstrapFixture) createSettings(ctx context.Context, accountID, userID
return f.manager.CreateSettings(ctx, userID, types.DefaultSettings(accountID), proxyAddress, endpoint)
}
// seedProxy registers a connected proxy in clusterAddr so the labeled
// bootstrap path has a real cluster to validate against. accountID empty
// makes it a shared (NetBird-operated) cluster; private mirrors the
// capability an embedded `netbird proxy` reports, nil an unreported one.
func (f *bootstrapFixture) seedProxy(t *testing.T, proxyID, accountID, clusterAddr string, private *bool) {
t.Helper()
p := &proxy.Proxy{
ID: proxyID,
ClusterAddress: clusterAddr,
Status: proxy.StatusConnected,
LastSeen: time.Now().UTC(),
Capabilities: proxy.Capabilities{Private: private},
}
if accountID != "" {
p.AccountID = &accountID
}
require.NoError(t, f.store.SaveProxy(context.Background(), p), "seeding a proxy must succeed")
}
// seedEmbeddedCluster is the common case: a shared cluster with a connected
// embedded proxy, which is what the labeled bootstrap requires.
func (f *bootstrapFixture) seedEmbeddedCluster(t *testing.T, clusterAddr string) {
t.Helper()
private := true
f.seedProxy(t, "proxy-"+clusterAddr, "", clusterAddr, &private)
}
// TestCreateSettingsRequiresPermission pins the gate: bootstrap assigns the
// account's immutable endpoint, a settings write requiring the settings
// Create permission — and a denial leaves no row behind.
@@ -88,6 +117,7 @@ func TestCreateSettingsRequiresPermission(t *testing.T) {
func TestCreateSettingsLabeled(t *testing.T) {
ctx := context.Background()
f := newBootstrapFixture(t)
f.seedEmbeddedCluster(t, "cluster1.example.com")
f.expectPermission("account1", "user1", modules.AgentNetworkSettings, operations.Create, true)
created, err := f.createSettings(ctx, "account1", "user1", "Cluster1.Example.com", "")
@@ -161,6 +191,7 @@ func TestCreateSettingsIdentityFieldValidation(t *testing.T) {
func TestCreateSettingsConflictsOnSecondBootstrap(t *testing.T) {
ctx := context.Background()
f := newBootstrapFixture(t)
f.seedEmbeddedCluster(t, "cluster1.example.com")
f.expectPermission("account1", "user1", modules.AgentNetworkSettings, operations.Create, true)
first, err := f.createSettings(ctx, "account1", "user1", "cluster1.example.com", "")
@@ -223,3 +254,91 @@ func TestCreateProviderHasNoSettingsSideEffects(t *testing.T) {
_, err = f.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, "account1")
assert.Error(t, err, "provider create must not conjure a settings row")
}
// TestCreateSettingsAllowsUnjudgeableCluster pins the address-first carve-out:
// a cluster nothing is connected to yet cannot be judged, so the pin is
// allowed — the same order the dedicated path documents (claim the address,
// connect the proxy after). A stale cluster whose proxies have aged out of
// the active window reads the same way.
func TestCreateSettingsAllowsUnjudgeableCluster(t *testing.T) {
ctx := context.Background()
private := true
cases := map[string]func(f *bootstrapFixture, t *testing.T){
"no proxy at all": func(*bootstrapFixture, *testing.T) {},
"heartbeat aged out": func(f *bootstrapFixture, t *testing.T) {
require.NoError(t, f.store.SaveProxy(ctx, &proxy.Proxy{
ID: "proxy-stale",
ClusterAddress: "future.example.com",
Status: proxy.StatusConnected,
LastSeen: time.Now().UTC().Add(-time.Hour),
Capabilities: proxy.Capabilities{Private: &private},
}), "seeding a stale proxy must succeed")
},
}
for name, seed := range cases {
t.Run(name, func(t *testing.T) {
f := newBootstrapFixture(t)
seed(f, t)
f.expectPermission("account1", "user1", modules.AgentNetworkSettings, operations.Create, true)
created, err := f.createSettings(ctx, "account1", "user1", "future.example.com", "")
require.NoError(t, err, "a cluster with nothing live in it must stay pinnable")
assert.Equal(t, "future.example.com", created.ProxyAddress)
})
}
}
// TestCreateSettingsRequiresPrivateCluster pins the capability gate: the
// synthesised gateway service is always private, so a live cluster whose
// proxies are not embedded in a netbird client cannot serve it and must not
// become the account's immutable endpoint.
func TestCreateSettingsRequiresPrivateCluster(t *testing.T) {
ctx := context.Background()
f := newBootstrapFixture(t)
notPrivate := false
f.seedProxy(t, "proxy1", "", "central.example.com", &notPrivate)
f.expectPermission("account1", "user1", modules.AgentNetworkSettings, operations.Create, true)
_, err := f.createSettings(ctx, "account1", "user1", "central.example.com", "")
require.Error(t, err, "a cluster without an embedded proxy must be rejected")
var sErr *status.Error
require.ErrorAs(t, err, &sErr)
assert.Equal(t, status.InvalidArgument, sErr.Type(), "rejection must be a validation error")
assert.Contains(t, err.Error(), "embedded proxy", "the error must name what the cluster is missing")
_, err = f.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, "account1")
assert.Error(t, err, "no row may be left behind by a rejected bootstrap")
}
// TestCreateSettingsRejectsForeignCluster pins tenant isolation on the pin: an
// account-owned (BYOP) cluster belongs to the account that runs it, and is not
// a cluster another account may hang its gateway beneath — even though it is
// private-capable.
func TestCreateSettingsRejectsForeignCluster(t *testing.T) {
ctx := context.Background()
f := newBootstrapFixture(t)
private := true
f.seedProxy(t, "proxy1", "account2", "byop.account2.example.com", &private)
f.expectPermission("account1", "user1", modules.AgentNetworkSettings, operations.Create, true)
_, err := f.createSettings(ctx, "account1", "user1", "byop.account2.example.com", "")
require.Error(t, err, "another account's BYOP cluster must be rejected")
var sErr *status.Error
require.ErrorAs(t, err, &sErr)
assert.Equal(t, status.InvalidArgument, sErr.Type(), "rejection must be a validation error")
}
// TestCreateSettingsAcceptsOwnPrivateCluster pins the BYOP happy path: the
// account's own cluster with a connected embedded proxy is a valid pin.
func TestCreateSettingsAcceptsOwnPrivateCluster(t *testing.T) {
ctx := context.Background()
f := newBootstrapFixture(t)
private := true
f.seedProxy(t, "proxy1", "account1", "byop.account1.example.com", &private)
f.expectPermission("account1", "user1", modules.AgentNetworkSettings, operations.Create, true)
created, err := f.createSettings(ctx, "account1", "user1", "byop.account1.example.com", "")
require.NoError(t, err, "the account's own private cluster must be accepted")
assert.Equal(t, "byop.account1.example.com", created.ProxyAddress)
}
@@ -89,6 +89,7 @@ func TestAgentNetwork_UpdateSettings_PreservesImmutableAndTogglesCollection(t *t
// Bootstrap is an explicit settings create; providers have no settings
// side effects anymore.
seedEmbeddedProxyCluster(t, am.Store, clusterAddr)
before, err := mgr.CreateSettings(ctx, adminUserID, agenttypes.DefaultSettings(accountID), clusterAddr, "")
require.NoError(t, err, "CreateSettings must bootstrap the row")
require.Equal(t, clusterAddr, before.ProxyAddress, "proxy address pinned at bootstrap")
@@ -13,6 +13,7 @@ import (
networkmap "github.com/netbirdio/netbird/management/internals/controllers/network_map"
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
agenttypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
rpproxy "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
nbpeer "github.com/netbirdio/netbird/management/server/peer"
"github.com/netbirdio/netbird/management/server/permissions"
"github.com/netbirdio/netbird/management/server/store"
@@ -92,6 +93,7 @@ func TestAgentNetwork_ProviderCRUD_FansOutToProxyAndClientPeers(t *testing.T) {
// UpdateAccountPeers, which is the path under test.
agentMgr := agentnetwork.NewManager(am.Store, permissions.NewManager(am.Store), am, nil)
seedEmbeddedProxyCluster(t, am.Store, clusterAddr)
_, err = agentMgr.CreateSettings(ctx, adminUserID, agenttypes.DefaultSettings(accountID), clusterAddr, "")
require.NoError(t, err, "CreateSettings must bootstrap the endpoint")
// The bootstrap itself reconciles and queues updates on both channels;
@@ -218,3 +220,22 @@ func synthZoneRData(sync *nbproto.SyncResponse, clusterAddr, fqdn string) string
}
return ""
}
// seedEmbeddedProxyCluster registers a connected proxy running embedded in a
// netbird client for clusterAddr, matching what a real deployment looks like
// when the account bootstraps: the agent-network gateway service is always
// private, so its cluster has to be one that can serve private services.
func seedEmbeddedProxyCluster(t *testing.T, st store.Store, clusterAddr string) {
t.Helper()
private := true
now := time.Now().UTC()
require.NoError(t, st.SaveProxy(context.Background(), &rpproxy.Proxy{
ID: "agent-net-proxy-" + clusterAddr,
SessionID: "agent-net-session",
ClusterAddress: clusterAddr,
LastSeen: now,
ConnectedAt: &now,
Status: rpproxy.StatusConnected,
Capabilities: rpproxy.Capabilities{Private: &private},
}), "seeding the proxy cluster must succeed")
}