[client, proxy] cancel context before stopping engine on embedded client (#6397)

- Engine.Start takes syncMsgMux with a deferred unlock (engine.go:445) and parks in receiveSignalEvents → WaitStreamConnected (engine.go:1762), which only wakes on
  signal-stream connect or client-context cancellation.
  - When signal never connects, the 30s startup timeout fires and embed.Client.Start's rollback (embed.go:281) called client.Stop() → Engine.Stop, which blocks acquiring
  syncMsgMux (engine.go:318). The cancel() that would unpark Start was deferred until Start returned — permanent cycle. RemovePeer calls (g43/g385) then queue behind the
  lifecycle mutex.
  - Notably, embed.Client.Stop and the daemon's cleanupConnection both cancel before stopping — the startup rollback was the only path that didn't.
  - Engine.Start takes syncMsgMux with a deferred unlock (engine.go:445) and parks in receiveSignalEvents → WaitStreamConnected (engine.go:1762), which only wakes on
  signal-stream connect or client-context cancellation.
  - When signal never connects, the 30s startup timeout fires and embed.Client.Start's rollback (embed.go:281) called client.Stop() → Engine.Stop, which blocks acquiring
  syncMsgMux (engine.go:318). The cancel() that would unpark Start was deferred until Start returned — permanent cycle. RemovePeer calls (g43/g385) then queue behind the
  lifecycle mutex.
  - Notably, embed.Client.Stop and the daemon's cleanupConnection both cancel before stopping — the startup rollback was the only path that didn't.

client/embed/embed.go

@@ -279,6 +279,10 @@ func (c *Client) Start(startCtx context.Context) error {
 
 	select {
 	case <-startCtx.Done():
+		// Cancel the client context before stopping: Engine.Start blocks on the
+		// signal stream while holding the engine mutex and only unblocks on
+		// cancellation. Stopping first would deadlock on that mutex.
+		cancel()
 		if stopErr := client.Stop(); stopErr != nil {
 			return fmt.Errorf("stop error after context done. Stop error: %w. Context done: %w", stopErr, startCtx.Err())
 		}

client/embed/embed_test.go

@@ -0,0 +1,168 @@
+package embed
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+
+	"github.com/netbirdio/netbird/management/internals/controllers/network_map/controller"
+	"github.com/netbirdio/netbird/management/internals/controllers/network_map/update_channel"
+	"github.com/netbirdio/netbird/management/internals/modules/peers"
+	"github.com/netbirdio/netbird/management/internals/modules/peers/ephemeral/manager"
+	"github.com/netbirdio/netbird/management/internals/server/config"
+	nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc"
+	mgmt "github.com/netbirdio/netbird/management/server"
+	"github.com/netbirdio/netbird/management/server/activity"
+	nbcache "github.com/netbirdio/netbird/management/server/cache"
+	"github.com/netbirdio/netbird/management/server/groups"
+	"github.com/netbirdio/netbird/management/server/integrations/integrated_validator/validator"
+	"github.com/netbirdio/netbird/management/server/integrations/port_forwarding"
+	"github.com/netbirdio/netbird/management/server/job"
+	"github.com/netbirdio/netbird/management/server/permissions"
+	"github.com/netbirdio/netbird/management/server/settings"
+	"github.com/netbirdio/netbird/management/server/store"
+	"github.com/netbirdio/netbird/management/server/telemetry"
+	"github.com/netbirdio/netbird/management/server/types"
+	mgmtProto "github.com/netbirdio/netbird/shared/management/proto"
+	"github.com/netbirdio/netbird/util"
+)
+
+const testSetupKey = "A2C8E62B-38F5-4553-B31E-DD66C696CEBB"
+
+// TestClientStartTimeoutRollback reproduces a deadlock between Engine.Start and
+// Engine.Stop. The signal endpoint accepts gRPC connections but never serves the
+// SignalExchange service, so Engine.Start parks in WaitStreamConnected while
+// holding the engine mutex. When the Start context expires, the rollback path
+// calls ConnectClient.Stop, which must not block forever acquiring that mutex.
+func TestClientStartTimeoutRollback(t *testing.T) {
+	signalAddr := startBlackholeSignal(t)
+	mgmAddr := startManagement(t, signalAddr)
+
+	wgPort := 0
+	client, err := New(Options{
+		DeviceName:    "embed-rollback-test",
+		SetupKey:      testSetupKey,
+		ManagementURL: "http://" + mgmAddr,
+		WireguardPort: &wgPort,
+	})
+	require.NoError(t, err, "embed client creation must succeed")
+
+	startCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	startErr := make(chan error, 1)
+	go func() {
+		startErr <- client.Start(startCtx)
+	}()
+
+	select {
+	case err := <-startErr:
+		require.ErrorIs(t, err, context.DeadlineExceeded)
+	case <-time.After(60 * time.Second):
+		t.Fatal("client.Start did not return after its context expired: Engine.Stop deadlocked against Engine.Start waiting for the signal stream")
+	}
+}
+
+// startBlackholeSignal starts a gRPC server without the SignalExchange service
+// registered. Connections succeed, but the signal stream can never be
+// established, which keeps Engine.Start parked in WaitStreamConnected.
+func startBlackholeSignal(t *testing.T) string {
+	t.Helper()
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	require.NoError(t, err)
+
+	s := grpc.NewServer()
+	go func() {
+		if err := s.Serve(lis); err != nil {
+			t.Error(err)
+		}
+	}()
+	t.Cleanup(s.Stop)
+
+	return lis.Addr().String()
+}
+
+func startManagement(t *testing.T, signalAddr string) string {
+	t.Helper()
+
+	cfg := &config.Config{
+		Stuns:      []*config.Host{},
+		TURNConfig: &config.TURNConfig{},
+		Relay: &config.Relay{
+			Addresses:      []string{"127.0.0.1:1234"},
+			CredentialsTTL: util.Duration{Duration: time.Hour},
+			Secret:         "222222222222222222",
+		},
+		Signal: &config.Host{
+			Proto: "http",
+			URI:   signalAddr,
+		},
+		Datadir:    t.TempDir(),
+		HttpConfig: nil,
+	}
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	require.NoError(t, err)
+
+	s := grpc.NewServer()
+
+	testStore, cleanUp, err := store.NewTestStoreFromSQL(context.Background(), "../testdata/store.sql", cfg.Datadir)
+	require.NoError(t, err)
+	t.Cleanup(cleanUp)
+
+	eventStore := &activity.InMemoryEventStore{}
+
+	permissionsManager := permissions.NewManager(testStore)
+	peersManager := peers.NewManager(testStore, permissionsManager)
+	jobManager := job.NewJobManager(nil, testStore, peersManager)
+
+	cacheStore, err := nbcache.NewStore(context.Background(), 100*time.Millisecond, 300*time.Millisecond, 100)
+	require.NoError(t, err)
+
+	iv, err := validator.NewIntegratedValidator(context.Background(), peersManager, nil, eventStore, cacheStore)
+	require.NoError(t, err)
+	metrics, err := telemetry.NewDefaultAppMetrics(context.Background())
+	require.NoError(t, err)
+
+	ctrl := gomock.NewController(t)
+	t.Cleanup(ctrl.Finish)
+	settingsMockManager := settings.NewMockManager(ctrl)
+	settingsMockManager.EXPECT().
+		GetSettings(gomock.Any(), gomock.Any(), gomock.Any()).
+		Return(&types.Settings{}, nil).
+		AnyTimes()
+	settingsMockManager.EXPECT().
+		GetExtraSettings(gomock.Any(), gomock.Any()).
+		Return(&types.ExtraSettings{}, nil).
+		AnyTimes()
+
+	groupsManager := groups.NewManagerMock()
+
+	updateManager := update_channel.NewPeersUpdateManager(metrics)
+	requestBuffer := mgmt.NewAccountRequestBuffer(context.Background(), testStore)
+	networkMapController := controller.NewController(context.Background(), testStore, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.selfhosted", port_forwarding.NewControllerMock(), manager.NewEphemeralManager(testStore, peersManager), cfg)
+	accountManager, err := mgmt.BuildManager(context.Background(), cfg, testStore, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false, cacheStore)
+	require.NoError(t, err)
+
+	secretsManager, err := nbgrpc.NewTimeBasedAuthSecretsManager(updateManager, cfg.TURNConfig, cfg.Relay, settingsMockManager, groupsManager)
+	require.NoError(t, err)
+
+	mgmtServer, err := nbgrpc.NewServer(cfg, accountManager, settingsMockManager, jobManager, secretsManager, nil, nil, &mgmt.MockIntegratedValidator{}, networkMapController, nil, nil)
+	require.NoError(t, err)
+	mgmtProto.RegisterManagementServiceServer(s, mgmtServer)
+
+	go func() {
+		if err := s.Serve(lis); err != nil {
+			t.Error(err)
+		}
+	}()
+	t.Cleanup(s.Stop)
+
+	return lis.Addr().String()
+}

management/internals/modules/reverseproxy/service/manager/l4_port_test.go

@@ -488,6 +488,195 @@ func TestUpdate_AllowsPortChange(t *testing.T) {
 	assert.Equal(t, uint16(54321), updated.ListenPort, "explicit port change should be applied")
 }
 
+func TestUpdate_PreservesPortWhenCustomPortsNotSupported(t *testing.T) {
+	mgr, testStore, _ := setupL4Test(t, boolPtr(false))
+	ctx := context.Background()
+
+	existing := seedService(t, testStore, "tcp-svc", "tcp", testCluster, testCluster, 12345)
+
+	updated := &rpservice.Service{
+		ID:           existing.ID,
+		AccountID:    testAccountID,
+		Name:         "tcp-svc-renamed",
+		Mode:         "tcp",
+		Domain:       testCluster,
+		ProxyCluster: testCluster,
+		ListenPort:   0,
+		Enabled:      true,
+		Targets: []*rpservice.Target{
+			{AccountID: testAccountID, TargetId: testPeerID, TargetType: rpservice.TargetTypePeer, Protocol: "tcp", Port: 9090, Enabled: true},
+		},
+	}
+
+	_, err := mgr.persistServiceUpdate(ctx, testAccountID, updated)
+	require.NoError(t, err, "update must not be rejected by the custom-port capability check")
+	assert.Equal(t, uint16(12345), updated.ListenPort, "existing listen port should be preserved on unsupported cluster")
+}
+
+func TestUpdate_PreservesPortWhenCustomPortsUnknown(t *testing.T) {
+	mgr, testStore, _ := setupL4Test(t, nil)
+	ctx := context.Background()
+
+	existing := seedService(t, testStore, "tcp-svc", "tcp", testCluster, testCluster, 12345)
+
+	updated := &rpservice.Service{
+		ID:           existing.ID,
+		AccountID:    testAccountID,
+		Name:         "tcp-svc-renamed",
+		Mode:         "tcp",
+		Domain:       testCluster,
+		ProxyCluster: testCluster,
+		ListenPort:   0,
+		Enabled:      true,
+		Targets: []*rpservice.Target{
+			{AccountID: testAccountID, TargetId: testPeerID, TargetType: rpservice.TargetTypePeer, Protocol: "tcp", Port: 9090, Enabled: true},
+		},
+	}
+
+	_, err := mgr.persistServiceUpdate(ctx, testAccountID, updated)
+	require.NoError(t, err, "update must not be rejected when cluster capability is unknown")
+	assert.Equal(t, uint16(12345), updated.ListenPort, "existing listen port should be preserved when capability is unknown")
+}
+
+func TestUpdate_RejectsPortChangeWhenCustomPortsNotSupported(t *testing.T) {
+	mgr, testStore, _ := setupL4Test(t, boolPtr(false))
+	ctx := context.Background()
+
+	existing := seedService(t, testStore, "tcp-svc", "tcp", testCluster, testCluster, 12345)
+
+	updated := &rpservice.Service{
+		ID:           existing.ID,
+		AccountID:    testAccountID,
+		Name:         "tcp-svc",
+		Mode:         "tcp",
+		Domain:       testCluster,
+		ProxyCluster: testCluster,
+		ListenPort:   54321,
+		Enabled:      true,
+		Targets: []*rpservice.Target{
+			{AccountID: testAccountID, TargetId: testPeerID, TargetType: rpservice.TargetTypePeer, Protocol: "tcp", Port: 9090, Enabled: true},
+		},
+	}
+
+	_, err := mgr.persistServiceUpdate(ctx, testAccountID, updated)
+	require.Error(t, err, "explicit port change on update must be rejected on unsupported clusters")
+	assert.Contains(t, err.Error(), "custom ports not supported on target cluster")
+}
+
+func TestUpdate_TLSPortChangeAllowedWhenNotSupported(t *testing.T) {
+	mgr, testStore, _ := setupL4Test(t, boolPtr(false))
+	ctx := context.Background()
+
+	existing := seedService(t, testStore, "tls-svc", "tls", "app.example.com", testCluster, 443)
+
+	updated := &rpservice.Service{
+		ID:           existing.ID,
+		AccountID:    testAccountID,
+		Name:         "tls-svc",
+		Mode:         "tls",
+		Domain:       "app.example.com",
+		ProxyCluster: testCluster,
+		ListenPort:   9999,
+		Enabled:      true,
+		Targets: []*rpservice.Target{
+			{AccountID: testAccountID, TargetId: testPeerID, TargetType: rpservice.TargetTypePeer, Protocol: "tcp", Port: 8443, Enabled: true},
+		},
+	}
+
+	_, err := mgr.persistServiceUpdate(ctx, testAccountID, updated)
+	require.NoError(t, err, "TLS port change uses SNI routing and is exempt from the custom-port check")
+	assert.Equal(t, uint16(9999), updated.ListenPort, "TLS port change should be applied")
+}
+
+func TestValidateL4PortDiffOnClusterDiff(t *testing.T) {
+	tests := []struct {
+		name        string
+		mode        string
+		customPorts *bool
+		newPort     uint16
+		oldPort     uint16
+		wantErr     bool
+	}{
+		{"tcp port change unsupported", "tcp", boolPtr(false), 54321, 12345, true},
+		{"tcp port change unknown capability", "tcp", nil, 54321, 12345, true},
+		{"udp port change unsupported", "udp", boolPtr(false), 54321, 12345, true},
+		{"tcp first port assignment unsupported", "tcp", boolPtr(false), 54321, 0, true},
+		{"tcp port change supported", "tcp", boolPtr(true), 54321, 12345, false},
+		{"tcp port unchanged unsupported", "tcp", boolPtr(false), 12345, 12345, false},
+		{"tcp zero port unsupported", "tcp", boolPtr(false), 0, 12345, false},
+		{"tls port change unsupported", "tls", boolPtr(false), 9999, 443, false},
+		{"http mode ignored", "http", boolPtr(false), 54321, 12345, false},
+		{"empty mode ignored", "", boolPtr(false), 54321, 12345, false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			newSvc := &rpservice.Service{Mode: tc.mode, ListenPort: tc.newPort, ProxyCluster: testCluster}
+			oldSvc := &rpservice.Service{Mode: tc.mode, ListenPort: tc.oldPort, ProxyCluster: testCluster}
+
+			err := validateL4PortDiffOnClusterDiff(tc.customPorts, newSvc, oldSvc)
+			if tc.wantErr {
+				assert.Error(t, err, "port diff should be rejected for %s", tc.name)
+			} else {
+				assert.NoError(t, err, "port diff should be allowed for %s", tc.name)
+			}
+		})
+	}
+}
+
+func TestUpdate_PortConflictRejected(t *testing.T) {
+	mgr, testStore, _ := setupL4Test(t, boolPtr(true))
+	ctx := context.Background()
+
+	seedService(t, testStore, "tcp-a", "tcp", "tcp-a."+testCluster, testCluster, 5432)
+	svcB := seedService(t, testStore, "tcp-b", "tcp", "tcp-b."+testCluster, testCluster, 6543)
+
+	updated := &rpservice.Service{
+		ID:           svcB.ID,
+		AccountID:    testAccountID,
+		Name:         "tcp-b",
+		Mode:         "tcp",
+		Domain:       "tcp-b." + testCluster,
+		ProxyCluster: testCluster,
+		ListenPort:   5432,
+		Enabled:      true,
+		Targets: []*rpservice.Target{
+			{AccountID: testAccountID, TargetId: testPeerID, TargetType: rpservice.TargetTypePeer, Protocol: "tcp", Port: 9090, Enabled: true},
+		},
+	}
+
+	_, err := mgr.persistServiceUpdate(ctx, testAccountID, updated)
+	require.Error(t, err, "updating to a port held by another service should be rejected")
+	assert.Contains(t, err.Error(), "already in use")
+}
+
+func TestUpdate_AutoAssignsWhenNoPort(t *testing.T) {
+	mgr, testStore, _ := setupL4Test(t, boolPtr(false))
+	ctx := context.Background()
+
+	existing := seedService(t, testStore, "tcp-svc", "tcp", testCluster, testCluster, 0)
+
+	updated := &rpservice.Service{
+		ID:           existing.ID,
+		AccountID:    testAccountID,
+		Name:         "tcp-svc",
+		Mode:         "tcp",
+		Domain:       testCluster,
+		ProxyCluster: testCluster,
+		ListenPort:   0,
+		Enabled:      true,
+		Targets: []*rpservice.Target{
+			{AccountID: testAccountID, TargetId: testPeerID, TargetType: rpservice.TargetTypePeer, Protocol: "tcp", Port: 9090, Enabled: true},
+		},
+	}
+
+	_, err := mgr.persistServiceUpdate(ctx, testAccountID, updated)
+	require.NoError(t, err)
+	assert.True(t, updated.ListenPort >= autoAssignPortMin && updated.ListenPort <= autoAssignPortMax,
+		"auto-assigned port %d should be in range [%d, %d]", updated.ListenPort, autoAssignPortMin, autoAssignPortMax)
+	assert.True(t, updated.PortAutoAssigned, "PortAutoAssigned should be set when update triggers auto-assignment")
+}
+
 func TestCreateServiceFromPeer_TCP(t *testing.T) {
 	mgr, _, _ := setupL4Test(t, boolPtr(false))
 	ctx := context.Background()

management/internals/modules/reverseproxy/service/manager/manager.go

@@ -338,7 +338,7 @@ func (m *Manager) persistNewService(ctx context.Context, accountID string, svc *
 			}
 		}
 
-		if err := m.ensureL4Port(ctx, transaction, svc, customPorts); err != nil {
+		if err := m.ensureL4Port(ctx, transaction, svc, customPorts, false); err != nil {
 			return err
 		}
 
@@ -367,11 +367,11 @@ func (m *Manager) clusterCustomPorts(ctx context.Context, svc *service.Service)
 
 // ensureL4Port auto-assigns a listen port when needed and validates cluster support.
 // customPorts must be pre-computed via clusterCustomPorts before entering a transaction.
-func (m *Manager) ensureL4Port(ctx context.Context, tx store.Store, svc *service.Service, customPorts *bool) error {
+func (m *Manager) ensureL4Port(ctx context.Context, tx store.Store, svc *service.Service, customPorts *bool, serviceUpdate bool) error {
 	if !service.IsL4Protocol(svc.Mode) {
 		return nil
 	}
-	if service.IsPortBasedProtocol(svc.Mode) && svc.ListenPort > 0 && (customPorts == nil || !*customPorts) {
+	if service.IsPortBasedProtocol(svc.Mode) && svc.ListenPort > 0 && !serviceUpdate && (customPorts == nil || !*customPorts) {
 		if svc.Source != service.SourceEphemeral {
 			return status.Errorf(status.InvalidArgument, "custom ports not supported on cluster %s", svc.ProxyCluster)
 		}
@@ -465,7 +465,7 @@ func (m *Manager) persistNewEphemeralService(ctx context.Context, accountID, pee
 			return err
 		}
 
-		if err := m.ensureL4Port(ctx, transaction, svc, customPorts); err != nil {
+		if err := m.ensureL4Port(ctx, transaction, svc, customPorts, false); err != nil {
 			return err
 		}
 
@@ -651,12 +651,22 @@ func (m *Manager) executeServiceUpdate(ctx context.Context, transaction store.St
 	m.preserveListenPort(service, existingService)
 	updateInfo.serviceEnabledChanged = existingService.Enabled != service.Enabled
 
-	if err := m.ensureL4Port(ctx, transaction, service, customPorts); err != nil {
+	// if the service is being updated, and we decide in the future to allow mode update,
+	// we should reconsider the currently assigned port if not 0 for clusters that don't support custom ports
+	if err := validateL4PortDiffOnClusterDiff(customPorts, service, existingService); err != nil {
 		return err
 	}
+
+	if err := m.ensureL4Port(ctx, transaction, service, customPorts, true); err != nil {
+		return err
+	}
+
+	// we can try carrying the previous service port into a new cluster, if this becomes a problem for multiple users,
+	// we should reconsider adding another check
 	if err := m.checkPortConflict(ctx, transaction, service); err != nil {
 		return err
 	}
+
 	if err := transaction.UpdateService(ctx, service); err != nil {
 		return fmt.Errorf("update service: %w", err)
 	}
@@ -664,6 +674,21 @@ func (m *Manager) executeServiceUpdate(ctx context.Context, transaction store.St
 	return nil
 }
 
+// validateL4PortDiffOnClusterDiff checks if custom L4 ports are configured and validates port changes across clusters.
+// It ensures no port changes if custom ports are unsupported for a given cluster and protocol mode.
+// Returns an error if validation fails, otherwise returns nil.
+func validateL4PortDiffOnClusterDiff(customPorts *bool, newSVC, oldSVC *service.Service) error {
+	if !service.IsPortBasedProtocol(newSVC.Mode) || (customPorts != nil && *customPorts) {
+		return nil
+	}
+
+	if newSVC.ListenPort != 0 && newSVC.ListenPort != oldSVC.ListenPort {
+		return status.Errorf(status.InvalidArgument, "custom ports not supported on target cluster %s", newSVC.ProxyCluster)
+	}
+
+	return nil
+}
+
 // handleDomainChange validates the new domain is free inside the transaction
 // and applies the pre-resolved cluster (computed outside the tx by
 // resolveEffectiveCluster). It must NOT call clusterDeriver here: that talks

management/internals/shared/grpc/conversion.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/go-version"
+	nbversion "github.com/netbirdio/netbird/version"
 	log "github.com/sirupsen/logrus"
 	goproto "google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -28,6 +30,23 @@ import (
 	"github.com/netbirdio/netbird/shared/sshauth"
 )
 
+const (
+	// deprecatedRemotePeersVersion is the version of Netbird that introduced the NetworkMap.RemotePeers field, deprecated in favor of RemotePeers.
+	deprecatedRemotePeersVersion = "0.29.3"
+)
+
+// precomputedDeprecatedRemotePeersConstraint is the parsed ">= 0.29.3" constraint,
+// built once at init since the bound is a compile-time constant.
+var precomputedDeprecatedRemotePeersConstraint version.Constraints
+
+func init() {
+	constraint, err := version.NewConstraint(">= " + deprecatedRemotePeersVersion)
+	if err != nil {
+		panic("parse deprecated remote peers version constraint: " + err.Error())
+	}
+	precomputedDeprecatedRemotePeersConstraint = constraint
+}
+
 func toNetbirdConfig(config *nbconfig.Config, turnCredentials *Token, relayToken *Token, extraSettings *types.ExtraSettings) *proto.NetbirdConfig {
 	if config == nil {
 		return nil
@@ -155,7 +174,11 @@ func ToSyncResponse(ctx context.Context, config *nbconfig.Config, httpConfig *nb
 
 	remotePeers := make([]*proto.RemotePeerConfig, 0, len(networkMap.Peers)+len(networkMap.OfflinePeers))
 	remotePeers = appendRemotePeerConfig(remotePeers, networkMap.Peers, dnsName, includeIPv6)
-	response.RemotePeers = remotePeers
+
+	if !shouldSkipSendingDeprecatedRemotePeers(peer.Meta.WtVersion) {
+		response.RemotePeers = remotePeers
+	}
+
 	response.NetworkMap.RemotePeers = remotePeers
 	response.RemotePeersIsEmpty = len(remotePeers) == 0
 	response.NetworkMap.RemotePeersIsEmpty = response.RemotePeersIsEmpty
@@ -246,6 +269,19 @@ func buildAuthorizedUsersProto(ctx context.Context, authorizedUsers map[string]m
 	return hashedUsers, machineUsers
 }
 
+func shouldSkipSendingDeprecatedRemotePeers(peerVersion string) bool {
+	if nbversion.IsDevelopmentVersion(peerVersion) {
+		return true
+	}
+
+	peerNBVersion, err := version.NewVersion(peerVersion)
+	if err != nil {
+		return false
+	}
+
+	return precomputedDeprecatedRemotePeersConstraint.Check(peerNBVersion)
+}
+
 func appendRemotePeerConfig(dst []*proto.RemotePeerConfig, peers []*nbpeer.Peer, dnsName string, includeIPv6 bool) []*proto.RemotePeerConfig {
 	for _, rPeer := range peers {
 		allowedIPs := []string{rPeer.IP.String() + "/32"}
@@ -363,7 +399,6 @@ func toProtocolFirewallRules(rules []*types.FirewallRule, includeIPv6, useSource
 	return result
 }
 
-
 // populateSourcePrefixes sets SourcePrefixes on fwRule and returns any
 // additional rules needed (e.g. a v6 wildcard clone when the peer IP is unspecified).
 func populateSourcePrefixes(fwRule *proto.FirewallRule, rule *types.FirewallRule, includeIPv6 bool) []*proto.FirewallRule {

management/internals/shared/grpc/conversion_test.go

@@ -202,6 +202,42 @@ func TestBuildJWTConfig_Audiences(t *testing.T) {
 	}
 }
 
+// TestShouldSkipSendingDeprecatedRemotePeers covers the version gate that
+// stops populating the deprecated top-level SyncResponse.RemotePeers field for
+// peers new enough to read RemotePeers off the NetworkMap. Development builds
+// are treated as latest and skip the field. The gate otherwise fails safe: a
+// release version older than the boundary, or one that can't be parsed (empty,
+// garbage, prereleases of the boundary) still receives the deprecated field so
+// older/unknown clients keep working.
+func TestShouldSkipSendingDeprecatedRemotePeers(t *testing.T) {
+	tests := []struct {
+		name        string
+		peerVersion string
+		wantSkip    bool
+	}{
+		{"exact boundary skips", "0.29.3", true},
+		{"newer patch skips", "0.29.4", true},
+		{"newer minor skips", "0.30.0", true},
+		{"newer major skips", "1.0.0", true},
+		{"v-prefixed newer skips", "v0.30.0", true},
+		{"development build skips", "development", true},
+		{"development build with commit skips", "development-abc123def456-dirty", true},
+		{"older patch keeps field", "0.29.2", false},
+		{"older minor keeps field", "0.28.0", false},
+		{"prerelease of boundary keeps field", "0.29.3-SNAPSHOT", false},
+		{"tagged dev prerelease keeps field", "v0.31.1-dev", false},
+		{"empty version keeps field", "", false},
+		{"garbage version keeps field", "not-a-version", false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := shouldSkipSendingDeprecatedRemotePeers(tc.peerVersion)
+			assert.Equal(t, tc.wantSkip, got, "skip decision for peer version %q", tc.peerVersion)
+		})
+	}
+}
+
 // TestEncodeSessionExpiresAt pins the wire encoding the client's
 // applySessionDeadline depends on:
 //

shared/management/client/client_test.go

@@ -322,15 +322,21 @@ func TestClient_Sync(t *testing.T) {
 		if resp.GetNetbirdConfig() == nil {
 			t.Error("expecting non nil NetbirdConfig got nil")
 		}
-		if len(resp.GetRemotePeers()) != 1 {
-			t.Errorf("expecting RemotePeers size %d got %d", 1, len(resp.GetRemotePeers()))
+		// we test network map peers from 0.29.3 and dev builds
+		if len(resp.GetRemotePeers()) != 0 {
+			t.Error("expecting top-level RemotePeers to be empty for v0.29.3+ clients")
+		}
+		networkMap := resp.GetNetworkMap()
+		if len(networkMap.GetRemotePeers()) != 1 {
+			t.Errorf("expecting RemotePeers size %d got %d", 1, len(networkMap.GetRemotePeers()))
 			return
 		}
-		if resp.GetRemotePeersIsEmpty() == true {
+
+		if networkMap.GetRemotePeersIsEmpty() {
 			t.Error("expecting RemotePeers property to be false, got true")
 		}
-		if resp.GetRemotePeers()[0].GetWgPubKey() != remoteKey.PublicKey().String() {
-			t.Errorf("expecting RemotePeer public key %s got %s", remoteKey.PublicKey().String(), resp.GetRemotePeers()[0].GetWgPubKey())
+		if networkMap.GetRemotePeers()[0].GetWgPubKey() != remoteKey.PublicKey().String() {
+			t.Errorf("expecting RemotePeer public key %s got %s", remoteKey.PublicKey().String(), networkMap.GetRemotePeers()[0].GetWgPubKey())
 		}
 	case <-time.After(3 * time.Second):
 		t.Error("timeout waiting for test to finish")