[management] Withdraw a cluster address claim that is lost after the write

A cluster address is claimed two ways: an account-scoped proxy row, and an
agent network gateway pin on the address. Each side checked the other
before writing — IsClusterAddressAvailable before SaveProxy,
HasForeignAccountProxyAtHost before the settings insert — but check and
write are separate autocommit statements, so two concurrent claimants could
each pass their check and both commit, leaving a pin no proxy will ever
serve next to the proxy row that displaces it.

Both sides now re-read after they write. Manager.Connect re-asks
availability once the proxy row is committed and, if the address is no
longer free or the answer is inconclusive, deletes its own row and returns
ErrClusterAddressUnavailable, which the connect path reports as
AlreadyExists exactly as the pre-write check would have. bootstrapLabeled
re-asks ownership once the settings row is committed and withdraws the pin
on the same terms. Because both write before they re-read, of two
concurrent claimants at least one re-reads after the other has committed
and backs off — on sqlite, postgres and mysql alike, since each statement
sees every commit before it. Both may back off, which costs a retry;
neither keeps a claim the other holds.

No lock spans the proxies and settings tables portably, and a claims table
would be more machinery than the property needs, so the re-read is the
whole mechanism. DeleteProxy is session-guarded like DisconnectProxy, so a
stale session withdrawing itself cannot take out a newer session's row.

Reported by CodeRabbit on #7402 (CWE-362).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Sa3DsBDP3VciAi4PPG17L6

management/internals/modules/agentnetwork/manager.go

@@ -1097,8 +1097,7 @@ func (m *managerImpl) validateGatewayCluster(ctx context.Context, accountID, clu
 		return fmt.Errorf("check proxy cluster ownership: %w", err)
 	}
 	if foreign {
-		return status.Errorf(status.InvalidArgument,
-			"proxy cluster %s is not available to this account", clusterAddr)
+		return errForeignCluster(clusterAddr)
 	}
 
 	declared, err := m.accountClusterSpellings(ctx, accountID, clusterAddr)
@@ -1227,12 +1226,55 @@ func (m *managerImpl) bootstrapLabeled(ctx context.Context, settings *types.Sett
 			}
 			return fmt.Errorf("create agent network settings: %w", err)
 		}
-		return nil
+		return m.confirmGatewayClusterOwnership(ctx, settings)
 	}
 
 	return fmt.Errorf("allocate agent network endpoint for account %s: %d attempts exhausted", settings.AccountID, maxDomainAllocationAttempts)
 }
 
+// confirmGatewayClusterOwnership re-asks, once the settings row is committed,
+// whether another account's proxy declares the pinned cluster, and withdraws
+// the row if one does.
+//
+// validateGatewayCluster answered that before the insert, but the two are
+// separate statements: a foreign proxy can register at the host in between,
+// and its own availability check — run before its row is written — would not
+// have seen this pin yet either. Re-reading after the write closes that
+// window from this side, and Manager.Connect does the same from the proxy's:
+// both claimants write before they re-read, so of two concurrent claims at
+// least one re-reads after the other has committed and backs off. Each
+// statement runs autocommit, so that re-read sees every commit before it on
+// sqlite, postgres and mysql alike. Both may back off, which costs the caller
+// a retry; neither keeps a claim the other holds, which is the invariant.
+// No lock spans the proxies and settings tables portably, and a claims table
+// would be more machinery than the property needs.
+//
+// Only ownership is re-asked. The capability check is about what the cluster
+// can do, not who holds it, and does not race a claim.
+func (m *managerImpl) confirmGatewayClusterOwnership(ctx context.Context, settings *types.Settings) error {
+	foreign, err := m.store.HasForeignAccountProxyAtHost(ctx, settings.ProxyAddress, settings.AccountID)
+	if err == nil && !foreign {
+		return nil
+	}
+
+	if delErr := m.store.DeleteAgentNetworkSettings(ctx, settings.AccountID); delErr != nil {
+		log.WithContext(ctx).Errorf("failed to withdraw agent network settings for account %s after losing the claim on %s: %v",
+			settings.AccountID, settings.ProxyAddress, delErr)
+	}
+	if err != nil {
+		return fmt.Errorf("confirm proxy cluster ownership: %w", err)
+	}
+	log.WithContext(ctx).Warnf("proxy cluster %s was claimed by another account while account %s bootstrapped onto it, withdrawing the pin",
+		settings.ProxyAddress, settings.AccountID)
+	return errForeignCluster(settings.ProxyAddress)
+}
+
+// errForeignCluster is the refusal for a cluster another account's proxy
+// declares, worded the same whether it is caught before or after the insert.
+func errForeignCluster(clusterAddr string) error {
+	return status.Errorf(status.InvalidArgument, "proxy cluster %s is not available to this account", clusterAddr)
+}
+
 // isUniqueConstraintError reports whether err is a database unique-constraint
 // violation, matched on the driver message because CreateAgentNetworkSettings
 // deliberately returns the driver error unwrapped.

management/internals/modules/agentnetwork/settings_bootstrap_test.go

@@ -7,9 +7,9 @@ import (
 	"testing"
 	"time"
 
-	"go.uber.org/mock/gomock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/mock/gomock"
 
 	"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
 	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
@@ -35,6 +35,16 @@ type bootstrapFixture struct {
 }
 
 func newBootstrapFixture(t *testing.T) *bootstrapFixture {
+	t.Helper()
+	return newBootstrapFixtureWith(t, func(st store.Store) store.Store { return st })
+}
+
+// newBootstrapFixtureWith hands the manager the real store as seen through
+// wrap, while the fixture keeps the unwrapped store for seeding and
+// assertions. It exists for cases that need something to happen between two
+// of the manager's store calls — a competing claim landing mid-bootstrap —
+// which a real store cannot be made to do on cue.
+func newBootstrapFixtureWith(t *testing.T, wrap func(store.Store) store.Store) *bootstrapFixture {
 	t.Helper()
 	if runtime.GOOS == "windows" {
 		t.Skip("sqlite store not properly supported on Windows yet")
@@ -55,7 +65,7 @@ func newBootstrapFixture(t *testing.T) *bootstrapFixture {
 
 	vendor := &stubLister{}
 	return &bootstrapFixture{
-		manager: NewManager(st, perms, accounts, nil, WithModelLister(vendor)),
+		manager: NewManager(wrap(st), perms, accounts, nil, WithModelLister(vendor)),
 		store:   st,
 		perms:   perms,
 		vendor:  vendor,
@@ -403,6 +413,66 @@ func TestCreateSettingsRejectsHostAnotherAccountClaims(t *testing.T) {
 	assert.Error(t, err, "no row may be left behind by a rejected bootstrap")
 }
 
+// claimingStore is a store.Store on which another account's proxy registers
+// at the host being pinned in the moment the settings row is written — the
+// interleaving a concurrent proxy connect produces when it passes its own
+// availability check before this bootstrap's row exists, so neither side's
+// pre-write check sees the other.
+type claimingStore struct {
+	store.Store
+	t       *testing.T
+	claim   *proxy.Proxy
+	claimed bool
+}
+
+func (s *claimingStore) CreateAgentNetworkSettings(ctx context.Context, settings *types.Settings) error {
+	if !s.claimed {
+		s.claimed = true
+		require.NoError(s.t, s.Store.SaveProxy(ctx, s.claim), "the competing claim must land")
+	}
+	return s.Store.CreateAgentNetworkSettings(ctx, settings)
+}
+
+// TestCreateSettingsWithdrawsPinClaimedDuringBootstrap covers the window
+// between validateGatewayCluster and the insert: a foreign proxy that claims
+// the host in that window is seen by the ownership re-read after the write,
+// and the pin is withdrawn rather than left standing on a cluster that will
+// never serve it. The refusal reads exactly as it would have had the
+// pre-write check caught the claim.
+func TestCreateSettingsWithdrawsPinClaimedDuringBootstrap(t *testing.T) {
+	ctx := context.Background()
+	const host = "shared.example.com"
+
+	f := newBootstrapFixtureWith(t, func(st store.Store) store.Store {
+		return &claimingStore{Store: st, t: t, claim: &proxy.Proxy{
+			ID:             "foreign",
+			ClusterAddress: host,
+			Status:         proxy.StatusConnected,
+			LastSeen:       time.Now().UTC(),
+			AccountID:      ptrTo("account2"),
+			Capabilities:   proxy.Capabilities{Private: ptrTo(true)},
+		}}
+	})
+	// A shared embedded cluster, so the pre-write validation passes on its
+	// own merits and only the claim landing mid-bootstrap can refuse it.
+	f.seedEmbeddedCluster(t, host)
+	f.expectPermission("account1", "user1", modules.AgentNetworkSettings, operations.Create, true)
+
+	_, err := f.createSettings(ctx, "account1", "user1", host, "")
+	require.Error(t, err, "a host claimed by another account mid-bootstrap must be refused")
+	var sErr *status.Error
+	require.ErrorAs(t, err, &sErr)
+	assert.Equal(t, status.InvalidArgument, sErr.Type())
+	assert.Contains(t, err.Error(), "not available to this account")
+
+	_, err = f.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, "account1")
+	assert.Error(t, err, "the pin written before the claim was seen must be withdrawn")
+
+	foreign, err := f.store.HasForeignAccountProxyAtHost(ctx, host, "account1")
+	require.NoError(t, err)
+	assert.True(t, foreign, "the competing claim, having landed first, keeps the host")
+}
+
 // TestCreateSettingsAcceptsSharedClusterAlongsideOwnProxy pins the other side
 // of that ordering: a shared (NetBird-operated) proxy is not foreign, so
 // asking the ownership question first must not refuse the cluster most