[client] Restore residual state in foreground mode before login (#6707)

* Improved residual state restoration during foreground startup and
foreground login, ensuring consistent recovery with stale states.
* Foreground flows now initialize advanced routing so stale routes 
are bypassed during login.

client/cmd/login.go

@@ -17,7 +17,9 @@ import (
 	"github.com/netbirdio/netbird/client/internal"
 	"github.com/netbirdio/netbird/client/internal/auth"
 	"github.com/netbirdio/netbird/client/internal/profilemanager"
+	nbnet "github.com/netbirdio/netbird/client/net"
 	"github.com/netbirdio/netbird/client/proto"
+	"github.com/netbirdio/netbird/client/server"
 	"github.com/netbirdio/netbird/client/system"
 	"github.com/netbirdio/netbird/util"
 )
@@ -331,6 +333,14 @@ func doForegroundLogin(ctx context.Context, cmd *cobra.Command, setupKey string,
 		return fmt.Errorf("read config file %s: %v", configFilePath, err)
 	}
 
+	// Mirror runInForegroundMode: recover residual state (DNS, firewall,
+	// ssh config, legacy routing) from a previous unclean shutdown and
+	// enable advanced routing before dialing management.
+	if err := server.RestoreResidualState(ctx, profilemanager.NewServiceManager(configFilePath).GetStatePath()); err != nil {
+		log.Warnf("failed to restore residual state: %v", err)
+	}
+	nbnet.Init()
+
 	err = foregroundLogin(ctx, cmd, config, setupKey, activeProf.ID)
 	if err != nil {
 		return fmt.Errorf("foreground login failed: %v", err)

client/cmd/up.go

@@ -22,6 +22,8 @@ import (
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/internal/profilemanager"
 	"github.com/netbirdio/netbird/client/proto"
+	nbnet "github.com/netbirdio/netbird/client/net"
+	"github.com/netbirdio/netbird/client/server"
 	"github.com/netbirdio/netbird/client/system"
 	"github.com/netbirdio/netbird/shared/management/domain"
 	"github.com/netbirdio/netbird/util"
@@ -229,6 +231,24 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command, activeProf *pr
 
 	_, _ = profilemanager.UpdateOldManagementURL(ctx, config, configFilePath)
 
+	// Restore residual state left by a previous run that did not shut down
+	// cleanly, mirroring what the daemon does before connecting: it recovers
+	// DNS config (a stale resolv.conf takeover can make the management
+	// hostname unresolvable), firewall rules, ssh config and legacy routing.
+	// Route cleanup itself happens at engine start; nbnet.Init() below lets
+	// the management dial bypass a leftover fwmark rule until then.
+	// Foreground mode is particularly exposed in containers: a crashed
+	// container restarts inside the same (pod) network namespace, so stale
+	// state survives while the process does not.
+	if err := server.RestoreResidualState(ctx, profilemanager.NewServiceManager(configPath).GetStatePath()); err != nil {
+		log.Warnf("failed to restore residual state: %v", err)
+	}
+
+	// Enable advanced routing (as the daemon does on startup) so the
+	// management dial bypasses a leftover fwmark rule instead of being
+	// shunted into a stale routing table.
+	nbnet.Init()
+
 	err = foregroundLogin(ctx, cmd, config, providedSetupKey, activeProf.ID)
 	if err != nil {
 		return fmt.Errorf("foreground login failed: %v", err)

client/internal/engine.go

@@ -2605,13 +2605,14 @@ func (e *Engine) updateForwardRules(rules []*mgmProto.ForwardingRule) ([]firewal
 
 func (e *Engine) toExcludedLazyPeers(rules []firewallManager.ForwardRule, peers []*mgmProto.RemotePeerConfig) map[string]bool {
 	excludedPeers := make(map[string]bool)
+
+	// Ingress forward targets: inbound forwarded traffic is initiated remotely and
+	// cannot wake a lazy connection, so the peer routing the target must stay
+	// permanently connected. AllowedIPs are already parsed on the peer conn, so
+	// reuse those typed prefixes instead of re-parsing the network map strings.
 	for _, r := range rules {
-		ip := r.TranslatedAddress
 		for _, p := range peers {
-			for _, allowedIP := range p.GetAllowedIps() {
-				if allowedIP != ip.String() {
-					continue
-				}
+			if e.peerRoutesAddr(p, r.TranslatedAddress) {
 				log.Infof("exclude forwarder peer from lazy connection: %s", p.GetWgPubKey())
 				excludedPeers[p.GetWgPubKey()] = true
 			}
@@ -2621,6 +2622,27 @@ func (e *Engine) toExcludedLazyPeers(rules []firewallManager.ForwardRule, peers
 	return excludedPeers
 }
 
+// peerRoutesAddr reports whether the peer is a router for addr, matched against
+// the peer's already-parsed AllowedIPs from the store (the same typed value the
+// lazy manager consumes) rather than re-parsing the network map strings.
+func (e *Engine) peerRoutesAddr(p *mgmProto.RemotePeerConfig, addr netip.Addr) bool {
+	prefixes, ok := e.peerStore.AllowedIPs(p.GetWgPubKey())
+	if !ok {
+		return false
+	}
+	return prefixesContain(prefixes, addr)
+}
+
+// prefixesContain reports whether addr falls within any of the prefixes.
+func prefixesContain(prefixes []netip.Prefix, addr netip.Addr) bool {
+	for _, prefix := range prefixes {
+		if prefix.Contains(addr) {
+			return true
+		}
+	}
+	return false
+}
+
 // isChecksEqual checks if two slices of checks are equal.
 func isChecksEqual(checks1, checks2 []*mgmProto.Checks) bool {
 	normalize := func(checks []*mgmProto.Checks) []string {

client/internal/engine_lazy_exclude_test.go

@@ -0,0 +1,87 @@
+package internal
+
+import (
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	firewallManager "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/peerstore"
+	mgmProto "github.com/netbirdio/netbird/shared/management/proto"
+)
+
+func TestPrefixesContain(t *testing.T) {
+	tests := []struct {
+		name     string
+		prefixes []string
+		addr     string
+		want     bool
+	}{
+		{name: "own overlay /32 matches", prefixes: []string{"100.110.8.145/32"}, addr: "100.110.8.145", want: true},
+		{name: "addr inside routed subnet", prefixes: []string{"10.121.0.0/16"}, addr: "10.121.208.4", want: true},
+		{name: "addr outside subnet", prefixes: []string{"10.121.0.0/16"}, addr: "10.122.0.1", want: false},
+		{name: "different /32", prefixes: []string{"100.110.8.145/32"}, addr: "100.110.8.146", want: false},
+		{name: "ipv6 /128 matches", prefixes: []string{"fd00::1/128"}, addr: "fd00::1", want: true},
+		{name: "no prefixes", prefixes: nil, addr: "10.121.208.4", want: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			prefixes := make([]netip.Prefix, 0, len(tt.prefixes))
+			for _, p := range tt.prefixes {
+				prefixes = append(prefixes, netip.MustParsePrefix(p))
+			}
+			require.Equal(t, tt.want, prefixesContain(prefixes, netip.MustParseAddr(tt.addr)))
+		})
+	}
+}
+
+// TestToExcludedLazyPeers_ForwardTarget guards a regression: the forward-target
+// peer (the peer routing a ForwardRule.TranslatedAddress) must be excluded from
+// lazy connections, matched via the peer's already-parsed AllowedIPs.
+func TestToExcludedLazyPeers_ForwardTarget(t *testing.T) {
+	const targetPeerKey = "cccccccccccccccccccccccccccccccccccccccccc0="
+	const otherPeerKey = "dddddddddddddddddddddddddddddddddddddddddd0="
+
+	store := peerstore.NewConnStore()
+	store.AddPeerConn(targetPeerKey, newTestConn(t, targetPeerKey, "100.110.8.145/32"))
+	store.AddPeerConn(otherPeerKey, newTestConn(t, otherPeerKey, "100.110.9.10/32"))
+
+	e := &Engine{peerStore: store}
+
+	peers := []*mgmProto.RemotePeerConfig{
+		{WgPubKey: targetPeerKey, AllowedIps: []string{"100.110.8.145/32"}},
+		{WgPubKey: otherPeerKey, AllowedIps: []string{"100.110.9.10/32"}},
+	}
+	rules := []firewallManager.ForwardRule{
+		{TranslatedAddress: netip.MustParseAddr("100.110.8.145")},
+	}
+
+	excluded := e.toExcludedLazyPeers(rules, peers)
+
+	require.True(t, excluded[targetPeerKey], "forward-target peer must be excluded from lazy connections")
+	require.False(t, excluded[otherPeerKey], "non-target peer must not be excluded")
+	require.Len(t, excluded, 1)
+}
+
+func TestToExcludedLazyPeers_NoRules(t *testing.T) {
+	e := &Engine{peerStore: peerstore.NewConnStore()}
+
+	peers := []*mgmProto.RemotePeerConfig{
+		{WgPubKey: "peer-a", AllowedIps: []string{"100.110.8.145/32"}},
+	}
+
+	require.Empty(t, e.toExcludedLazyPeers(nil, peers))
+}
+
+func newTestConn(t *testing.T, key, allowedIP string) *peer.Conn {
+	t.Helper()
+	conn, err := peer.NewConn(peer.ConnConfig{
+		Key:      key,
+		WgConfig: peer.WgConfig{AllowedIps: []netip.Prefix{netip.MustParsePrefix(allowedIP)}},
+	}, peer.ServiceDependencies{})
+	require.NoError(t, err)
+	return conn
+}

client/internal/peer/conn.go

@@ -203,7 +203,6 @@ func NewConn(config ConnConfig, services ServiceDependencies) (*Conn, error) {
 		statusICE:          worker.NewAtomicStatus(),
 		dumpState:          dumpState,
 		endpointUpdater:    NewEndpointUpdater(connLog, config.WgConfig, isController(config)),
-		wgWatcher:          NewWGWatcher(connLog, config.WgConfig.WgInterface, config.Key, dumpState),
 		metricsRecorder:    services.MetricsRecorder,
 	}
 
@@ -671,11 +670,12 @@ func (conn *Conn) onGuardEvent() {
 	}
 }
 
-func (conn *Conn) onWGDisconnected() {
+func (conn *Conn) onWGDisconnected(watcherCtx context.Context) {
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
 
-	if conn.ctx.Err() != nil {
+	// watcherCtx guards against a stale watcher tearing down a connection that already superseded it.
+	if conn.ctx.Err() != nil || watcherCtx.Err() != nil {
 		return
 	}
 
@@ -833,25 +833,39 @@ func (conn *Conn) isConnectedOnAllWay() (status guard.ConnStatus) {
 	})
 }
 
+// enableWgWatcherIfNeeded starts a fresh watcher instance per connection attempt, so its
+// lifecycle stays bound to conn.mu and enable/disable can't race an old goroutine's shutdown.
+// Caller must hold conn.mu.
 func (conn *Conn) enableWgWatcherIfNeeded(enabledTime time.Time) {
-	if !conn.wgWatcher.PrepareInitialHandshake() {
+	if conn.wgWatcher != nil {
 		return
 	}
 
+	watcher := NewWGWatcher(conn.Log, conn.config.WgConfig.WgInterface, conn.config.Key, conn.dumpState)
+	watcher.PrepareInitialHandshake()
+
 	wgWatcherCtx, wgWatcherCancel := context.WithCancel(conn.ctx)
+	conn.wgWatcher = watcher
 	conn.wgWatcherCancel = wgWatcherCancel
+
 	conn.wgWatcherWg.Add(1)
 	go func() {
 		defer conn.wgWatcherWg.Done()
-		conn.wgWatcher.EnableWgWatcher(wgWatcherCtx, enabledTime, conn.onWGDisconnected, conn.onWGHandshakeSuccess, conn.onWGCheckSuccess)
+		onDisconnected := func() { conn.onWGDisconnected(wgWatcherCtx) }
+		watcher.EnableWgWatcher(wgWatcherCtx, enabledTime, onDisconnected, conn.onWGHandshakeSuccess, conn.onWGCheckSuccess)
 	}()
 }
 
+// disableWgWatcherIfNeeded cancels and drops the watcher once no transport is active. It never
+// waits for the goroutine: the timeout path reentrantly calls back here under conn.mu, so
+// blocking would deadlock. Caller must hold conn.mu.
 func (conn *Conn) disableWgWatcherIfNeeded() {
-	if conn.currentConnPriority == conntype.None && conn.wgWatcherCancel != nil {
-		conn.wgWatcherCancel()
-		conn.wgWatcherCancel = nil
+	if conn.currentConnPriority != conntype.None || conn.wgWatcher == nil {
+		return
 	}
+	conn.wgWatcherCancel()
+	conn.wgWatcher = nil
+	conn.wgWatcherCancel = nil
 }
 
 func (conn *Conn) newProxy(remoteConn net.Conn) (wgproxy.Proxy, error) {
@@ -874,7 +888,9 @@ func (conn *Conn) resetEndpoint() {
 		return
 	}
 	conn.Log.Infof("reset wg endpoint")
-	conn.wgWatcher.Reset()
+	if conn.wgWatcher != nil {
+		conn.wgWatcher.Reset()
+	}
 	if err := conn.endpointUpdater.RemoveEndpointAddress(); err != nil {
 		conn.Log.Warnf("failed to remove endpoint address before update: %v", err)
 	}

client/internal/peer/conn_test.go

@@ -339,20 +339,20 @@ func TestConn_onWGDisconnected_EscalatesToRosenpassReset(t *testing.T) {
 	conn := newWGTimeoutTestConn(true, &disconnected)
 
 	for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
-		conn.onWGDisconnected()
+		conn.onWGDisconnected(conn.ctx)
 	}
 	assert.Empty(t, disconnected, "escalation must not fire below the threshold")
 
-	conn.onWGDisconnected()
+	conn.onWGDisconnected(conn.ctx)
 	assert.Equal(t, []string{conn.config.WgConfig.RemoteKey}, disconnected,
 		"reaching the threshold must report the peer disconnected once")
 
 	for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
-		conn.onWGDisconnected()
+		conn.onWGDisconnected(conn.ctx)
 	}
 	assert.Len(t, disconnected, 1, "escalation must restart counting after firing")
 
-	conn.onWGDisconnected()
+	conn.onWGDisconnected(conn.ctx)
 	assert.Len(t, disconnected, 2, "continued timeouts must escalate again")
 }
 
@@ -364,12 +364,12 @@ func TestConn_onWGDisconnected_CheckSuccessResetsEscalation(t *testing.T) {
 	conn := newWGTimeoutTestConn(true, &disconnected)
 
 	for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
-		conn.onWGDisconnected()
+		conn.onWGDisconnected(conn.ctx)
 	}
 	conn.onWGCheckSuccess()
 
 	for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
-		conn.onWGDisconnected()
+		conn.onWGDisconnected(conn.ctx)
 	}
 	assert.Empty(t, disconnected, "handshake success must reset the timeout count")
 }
@@ -382,7 +382,7 @@ func TestConn_onWGDisconnected_NoEscalationWithoutRosenpass(t *testing.T) {
 	conn := newWGTimeoutTestConn(false, &disconnected)
 
 	for i := 0; i < wgTimeoutEscalationThreshold*3; i++ {
-		conn.onWGDisconnected()
+		conn.onWGDisconnected(conn.ctx)
 	}
 	assert.Empty(t, disconnected, "escalation must be limited to rosenpass connections")
 }

client/internal/peer/wg_watcher.go

@@ -3,7 +3,6 @@ package peer
 import (
 	"context"
 	"fmt"
-	"sync"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -24,14 +23,14 @@ type WGInterfaceStater interface {
 	GetStats() (map[string]configurer.WGStats, error)
 }
 
+// WGWatcher is single-shot: one instance per connection attempt, run once, then discarded.
+// Lifecycle is owned by Conn under conn.mu, so it keeps no "enabled" state to go stale.
 type WGWatcher struct {
 	log           *log.Entry
 	wgIfaceStater WGInterfaceStater
 	peerKey       string
 	stateDump     *stateDump
 
-	enabled   bool
-	muEnabled sync.Mutex
 	// initialHandshake is not thread-safe; never call PrepareInitialHandshake and EnableWgWatcher concurrently.
 	initialHandshake time.Time
 
@@ -48,25 +47,14 @@ func NewWGWatcher(log *log.Entry, wgIfaceStater WGInterfaceStater, peerKey strin
 	}
 }
 
-// PrepareInitialHandshake reserves the watcher and reads the peer's current WireGuard
-// handshake time. It must be called before the peer is (re)configured on the WireGuard
-// interface, so the captured baseline reflects the state prior to this connection attempt
-// instead of racing with that configuration. Returns ok=false if the watcher is already
-// running, in which case EnableWgWatcher must not be called.
-func (w *WGWatcher) PrepareInitialHandshake() (ok bool) {
-	w.muEnabled.Lock()
-	if w.enabled {
-		w.muEnabled.Unlock()
-		return false
-	}
-
+// PrepareInitialHandshake reads the peer's current WireGuard handshake time. It must be
+// called before the peer is (re)configured on the WireGuard interface, so the captured
+// baseline reflects the state prior to this connection attempt instead of racing with
+// that configuration.
+func (w *WGWatcher) PrepareInitialHandshake() {
 	w.log.Debugf("enable WireGuard watcher")
-	w.enabled = true
-	w.muEnabled.Unlock()
-
 	handshake, _ := w.wgState()
 	w.initialHandshake = handshake
-	return true
 }
 
 // EnableWgWatcher runs the WireGuard watcher loop using the handshake baseline captured by
@@ -76,10 +64,6 @@ func (w *WGWatcher) PrepareInitialHandshake() (ok bool) {
 // handshake, including the first.
 func (w *WGWatcher) EnableWgWatcher(ctx context.Context, enabledTime time.Time, onDisconnectedFn func(), onHandshakeSuccessFn func(when time.Time), onCheckSuccessFn func()) {
 	w.periodicHandshakeCheck(ctx, onDisconnectedFn, onHandshakeSuccessFn, onCheckSuccessFn, enabledTime, w.initialHandshake)
-
-	w.muEnabled.Lock()
-	w.enabled = false
-	w.muEnabled.Unlock()
 }
 
 // Reset signals the watcher that the WireGuard peer has been reset and a new
@@ -105,6 +89,7 @@ func (w *WGWatcher) periodicHandshakeCheck(ctx context.Context, onDisconnectedFn
 		case <-timer.C:
 			handshake, ok := w.handshakeCheck(lastHandshake)
 			if !ok {
+				// early ctx cancel check return
 				if ctx.Err() != nil {
 					return
 				}
@@ -153,9 +138,9 @@ func (w *WGWatcher) handshakeCheck(lastHandshake time.Time) (*time.Time, bool) {
 
 	w.log.Tracef("previous handshake, handshake: %v, %v", lastHandshake, handshake)
 
-	// the current know handshake did not change
+	// the current known handshake did not change
 	if handshake.Equal(lastHandshake) {
-		w.log.Warnf("WireGuard handshake timed out: %v", handshake)
+		w.log.Warnf("WireGuard handshake not updated: %v", handshake)
 		return nil, false
 	}
 

client/internal/peer/wg_watcher_test.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/require"
 
 	"github.com/netbirdio/netbird/client/iface/configurer"
 )
@@ -62,7 +61,7 @@ func TestWGWatcher_CheckSuccessCallback(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	require.True(t, watcher.PrepareInitialHandshake())
+	watcher.PrepareInitialHandshake()
 
 	firstHandshake := make(chan struct{}, 1)
 	checkSuccess := make(chan struct{}, 1)
@@ -101,8 +100,7 @@ func TestWGWatcher_EnableWgWatcher(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	ok := watcher.PrepareInitialHandshake()
-	require.True(t, ok, "watcher should not be enabled yet")
+	watcher.PrepareInitialHandshake()
 
 	onDisconnected := make(chan struct{}, 1)
 	go watcher.EnableWgWatcher(ctx, time.Now(), func() {
@@ -132,8 +130,7 @@ func TestWGWatcher_ReEnable(t *testing.T) {
 	watcher := NewWGWatcher(mlog, mocWgIface, "", newStateDump("peer", mlog, &Status{}))
 
 	ctx, cancel := context.WithCancel(context.Background())
-	ok := watcher.PrepareInitialHandshake()
-	require.True(t, ok, "watcher should not be enabled yet")
+	watcher.PrepareInitialHandshake()
 
 	wg := &sync.WaitGroup{}
 	wg.Add(1)
@@ -149,8 +146,7 @@ func TestWGWatcher_ReEnable(t *testing.T) {
 	ctx, cancel = context.WithCancel(context.Background())
 	defer cancel()
 
-	ok = watcher.PrepareInitialHandshake()
-	require.True(t, ok, "watcher should be re-enabled after the previous run stopped")
+	watcher.PrepareInitialHandshake()
 
 	onDisconnected := make(chan struct{}, 1)
 	go watcher.EnableWgWatcher(ctx, time.Now(), func() {

client/server/server.go

@@ -181,7 +181,7 @@ func (s *Server) Start() error {
 		log.Warnf("failed to redirect stderr: %v", err)
 	}
 
-	if err := restoreResidualState(s.rootCtx, s.profileManager.GetStatePath()); err != nil {
+	if err := RestoreResidualState(s.rootCtx, s.profileManager.GetStatePath()); err != nil {
 		log.Warnf(errRestoreResidualState, err)
 	}
 
@@ -551,7 +551,7 @@ func (s *Server) Login(callerCtx context.Context, msg *proto.LoginRequest) (*pro
 	s.actCancel = cancel
 	s.mutex.Unlock()
 
-	if err := restoreResidualState(s.rootCtx, s.profileManager.GetStatePath()); err != nil {
+	if err := RestoreResidualState(s.rootCtx, s.profileManager.GetStatePath()); err != nil {
 		log.Warnf(errRestoreResidualState, err)
 	}
 
@@ -858,7 +858,7 @@ func (s *Server) Up(callerCtx context.Context, msg *proto.UpRequest) (*proto.UpR
 
 		return s.waitForUp(callerCtx)
 	}
-	if err := restoreResidualState(callerCtx, s.profileManager.GetStatePath()); err != nil {
+	if err := RestoreResidualState(callerCtx, s.profileManager.GetStatePath()); err != nil {
 		log.Warnf(errRestoreResidualState, err)
 	}
 

client/server/state.go

@@ -46,7 +46,7 @@ func (s *Server) CleanState(ctx context.Context, req *proto.CleanStateRequest) (
 
 	if req.All {
 		// Reuse existing cleanup logic for all states
-		if err := restoreResidualState(ctx, statePath); err != nil {
+		if err := RestoreResidualState(ctx, statePath); err != nil {
 			return nil, status.Errorf(codes.Internal, "failed to clean all states: %v", err)
 		}
 
@@ -113,9 +113,9 @@ func (s *Server) DeleteState(ctx context.Context, req *proto.DeleteStateRequest)
 	}, nil
 }
 
-// restoreResidualState checks if the client was not shut down in a clean way and restores residual if required.
+// RestoreResidualState checks if the client was not shut down in a clean way and restores residual if required.
 // Otherwise, we might not be able to connect to the management server to retrieve new config.
-func restoreResidualState(ctx context.Context, statePath string) error {
+func RestoreResidualState(ctx context.Context, statePath string) error {
 	if statePath == "" {
 		return nil
 	}

management/internals/modules/peers/manager.go

@@ -226,6 +226,30 @@ func (m *managerImpl) CreateProxyPeer(ctx context.Context, accountID string, pee
 		return nil
 	}
 
+	// Dedupe stale embedded peer records for the same (account, cluster).
+	// The proxy generates a fresh WireGuard keypair on every startup
+	// (proxy/internal/roundtrip/netbird.go), so without this sweep the
+	// prior embedded peer would linger forever — holding its CGNAT IP
+	// allocation, polluting other peers' rosters, and (most visibly)
+	// leaving the synth DNS pointing at the dead address. The
+	// (account, cluster) tuple identifies "the embedded peer for this
+	// proxy instance at this cluster"; any record matching that tuple
+	// with a different pubkey is by definition stale and must go.
+	staleIDs, err := m.findStaleEmbeddedProxyPeers(ctx, accountID, cluster, peerKey)
+	if err != nil {
+		return fmt.Errorf("scan for stale embedded proxy peers: %w", err)
+	}
+	if len(staleIDs) > 0 {
+		// userID="" + checkConnected=false: the deletion is initiated
+		// by management itself on behalf of the freshly-registering
+		// proxy, not by an end user; the stale peer may still be
+		// marked Connected from its prior session, but its session is
+		// dead by definition (its key no longer exists).
+		if err := m.DeletePeers(ctx, accountID, staleIDs, "", false); err != nil {
+			return fmt.Errorf("delete stale embedded proxy peers %v: %w", staleIDs, err)
+		}
+	}
+
 	name := fmt.Sprintf("proxy-%s", xid.New().String())
 	newPeer := &peer.Peer{
 		Ephemeral: true,
@@ -251,3 +275,29 @@ func (m *managerImpl) CreateProxyPeer(ctx context.Context, accountID string, pee
 
 	return nil
 }
+
+// findStaleEmbeddedProxyPeers returns the peer IDs of embedded proxy peer
+// records in accountID that target the same cluster but carry a different
+// WireGuard pubkey than the freshly-registering one. Used by CreateProxyPeer
+// to garbage-collect stale records left behind when the proxy restarts with a
+// regenerated keypair.
+func (m *managerImpl) findStaleEmbeddedProxyPeers(ctx context.Context, accountID, cluster, newKey string) ([]string, error) {
+	account, err := m.store.GetAccount(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	var stale []string
+	for _, p := range account.Peers {
+		if p == nil || !p.ProxyMeta.Embedded {
+			continue
+		}
+		if p.ProxyMeta.Cluster != cluster {
+			continue
+		}
+		if p.Key == newKey {
+			continue
+		}
+		stale = append(stale, p.ID)
+	}
+	return stale, nil
+}

management/server/agentnetwork_proxypeer_restart_test.go

@@ -0,0 +1,199 @@
+package server
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/management/internals/modules/peers"
+	"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
+	agenttypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
+	nbpeer "github.com/netbirdio/netbird/management/server/peer"
+	"github.com/netbirdio/netbird/management/server/permissions"
+	"github.com/netbirdio/netbird/management/server/store"
+	"github.com/netbirdio/netbird/management/server/types"
+)
+
+// TestAgentNetwork_ProxyRestart_PropagatesNewPeerAndDropsStale is the no-mock
+// regression guard for the bug the user reported: restarting the proxy creates
+// a fresh embedded peer with a NEW WireGuard public key (the proxy generates
+// the keypair on every startup at proxy/internal/roundtrip/netbird.go:312).
+// The PRIOR embedded peer record is never deleted on management, so the
+// account accumulates a stale peer holding a stale CGNAT IP. Other peers
+// in the account either keep routing to the dead IP, or — if synth DNS
+// picks the wrong record — never see the new IP at all.
+//
+// What this test exercises (no mocks):
+//   - real SQLite test store
+//   - real DefaultAccountManager, network-map controller, peer-update channels
+//   - real peers.Manager.CreateProxyPeer path (the very method the proxy
+//     invokes over gRPC on every startup)
+//   - real agentnetwork.Manager + synth chain so the client receives a
+//     concrete DNS record that must point at the LATEST proxy peer.
+//
+// Pre-fix expected behavior (red): two embedded peers exist after the
+// "restart"; the synth DNS record points at the stale one; the client
+// receives an update reflecting the new peer but the old one lingers.
+// Post-fix expected behavior (green): exactly one embedded peer exists
+// after restart (with the new key) AND the client's network map carries
+// the synth DNS pointing at that new peer's CGNAT IP.
+func TestAgentNetwork_ProxyRestart_PropagatesNewPeerAndDropsStale(t *testing.T) {
+	am, updateManager, err := createManager(t)
+	require.NoError(t, err, "createManager must succeed")
+	ctx := context.Background()
+
+	const (
+		accountID   = "an-restart-acct"
+		adminUserID = "an-restart-admin"
+		groupAID    = "an-restart-grp-A"
+		clusterAddr = "eu.proxy.netbird.io"
+		clientKey   = "BhRPtynAAYRDy08+q4HTMsos8fs4plTP4NOSh7C1ry8="
+		// Two different proxy pubkeys — the "before" and "after" of a
+		// proxy-process restart with fresh-keypair generation.
+		proxyKey1 = "Aaaaa1aaaaYRDy08+q4HTMsos8fs4plTP4NOSh7C1ry8="
+		proxyKey2 = "Bbbbb2bbbbYRDy08+q4HTMsos8fs4plTP4NOSh7C1ry8="
+	)
+
+	// --- Account scaffold ---
+	account := newAccountWithId(ctx, accountID, adminUserID, "an-restart.test", "", "", false)
+	require.NoError(t, am.Store.SaveAccount(ctx, account))
+
+	clientPeer := &nbpeer.Peer{
+		Key:      clientKey,
+		Name:     "an-restart-client",
+		DNSLabel: "an-restart-client",
+		Meta:     nbpeer.PeerSystemMeta{Hostname: "an-restart-client", GoOS: "linux", WtVersion: "development"},
+	}
+	addedClient, _, _, _, err := am.AddPeer(ctx, "", "", adminUserID, clientPeer, false)
+	require.NoError(t, err, "AddPeer for client must succeed")
+	require.NoError(t, am.MarkPeerConnected(ctx, clientKey, accountID, time.Now().UnixNano(), &types.NetworkMap{}),
+		"MarkPeerConnected for the client peer must succeed (affected-peer fan-out skips disconnected peers)")
+
+	// Place the client in group A so the synth policy reaches it.
+	account, err = am.Store.GetAccount(ctx, accountID)
+	require.NoError(t, err)
+	account.Groups[groupAID] = &types.Group{ID: groupAID, Name: "groupA", Peers: []string{addedClient.ID}}
+	require.NoError(t, am.Store.SaveAccount(ctx, account), "SaveAccount must persist group A")
+
+	// --- Real peers + agent-network managers ---
+	permMgr := permissions.NewManager(am.Store)
+	peersMgr := peers.NewManager(am.Store, permMgr)
+	peersMgr.SetAccountManager(am)
+	peersMgr.SetNetworkMapController(am.networkMapController)
+	agentMgr := agentnetwork.NewManager(am.Store, permMgr, am, nil)
+
+	// Subscribe BEFORE any state-mutating call so we don't lose the update
+	// that contains the synth DNS record.
+	clientCh := updateManager.CreateChannel(ctx, addedClient.ID)
+	t.Cleanup(func() { updateManager.CloseChannel(ctx, addedClient.ID) })
+	drain(clientCh)
+
+	// --- First proxy startup: register peer key K1, then mark it
+	// connected. In production the proxy follows CreateProxyPeer with the
+	// regular sync stream which lands on MarkPeerConnected; the synth DNS
+	// path filters out peers that aren't Connected (types/account.go:323),
+	// so without this step no DNS record would be emitted.
+	require.NoError(t, peersMgr.CreateProxyPeer(ctx, accountID, proxyKey1, clusterAddr),
+		"first CreateProxyPeer (proxy startup) must succeed")
+
+	peer1ID, err := am.Store.GetPeerIDByKey(ctx, store.LockingStrengthNone, proxyKey1)
+	require.NoError(t, err, "proxy peer for K1 must be persisted after CreateProxyPeer")
+	require.NotEmpty(t, peer1ID)
+
+	require.NoError(t, am.MarkPeerConnected(ctx, proxyKey1, accountID, time.Now().UnixNano(), &types.NetworkMap{}),
+		"MarkPeerConnected for K1 must succeed")
+
+	account, err = am.Store.GetAccount(ctx, accountID)
+	require.NoError(t, err)
+	proxyIP1 := account.Peers[peer1ID].IP.String()
+	require.NotEmpty(t, proxyIP1, "K1 must have an assigned overlay IP")
+
+	// --- Provider + policy. CreateProvider / CreatePolicy trigger the
+	// agentnetwork reconcile which runs UpdateAccountPeers; the resulting
+	// NetworkMap delivered to the client carries the synth DNS record
+	// pointing at K1's IP. ---
+	provider, err := agentMgr.CreateProvider(ctx, adminUserID, &agenttypes.Provider{
+		AccountID:   accountID,
+		ProviderID:  "openai_api",
+		Name:        "openai-test",
+		UpstreamURL: "https://api.openai.com",
+		APIKey:      "sk-test-key",
+		Enabled:     true,
+		Models:      []agenttypes.ProviderModel{{ID: "gpt-5.4"}},
+	}, clusterAddr)
+	require.NoError(t, err, "CreateProvider must succeed")
+
+	_, err = agentMgr.CreatePolicy(ctx, adminUserID, &agenttypes.Policy{
+		AccountID:              accountID,
+		Name:                   "p1",
+		Enabled:                true,
+		SourceGroups:           []string{groupAID},
+		DestinationProviderIDs: []string{provider.ID},
+	})
+	require.NoError(t, err, "CreatePolicy must succeed")
+
+	settings, err := am.Store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, accountID)
+	require.NoError(t, err)
+	fqdn := settings.Endpoint()
+
+	rdata1 := awaitZoneRData(clientCh, clusterAddr, fqdn, true)
+	require.Equal(t, proxyIP1, rdata1,
+		"client must receive a synth DNS record pointing at K1's overlay IP after the synth path runs")
+	drain(clientCh)
+
+	// --- Proxy restart: NEW keypair K2, same account, same cluster ---
+	require.NoError(t, peersMgr.CreateProxyPeer(ctx, accountID, proxyKey2, clusterAddr),
+		"second CreateProxyPeer (proxy restart with fresh keypair) must succeed")
+
+	peer2ID, err := am.Store.GetPeerIDByKey(ctx, store.LockingStrengthNone, proxyKey2)
+	require.NoError(t, err, "proxy peer for K2 must be persisted after restart")
+	require.NotEmpty(t, peer2ID)
+
+	require.NoError(t, am.MarkPeerConnected(ctx, proxyKey2, accountID, time.Now().UnixNano(), &types.NetworkMap{}),
+		"MarkPeerConnected for K2 must succeed")
+
+	// In production the agent's sync stream pulls a fresh NetworkMap as
+	// part of its normal reconcile cadence; in this isolated test
+	// MarkPeerConnected's affected-peer fan-out can race the channel-side
+	// buffer in a way that swallows the synth-DNS-bearing update before
+	// our await reads it. Trigger an explicit account-wide fan-out so the
+	// assertion below tests what production actually delivers, not the
+	// in-test buffer race.
+	am.UpdateAccountPeers(ctx, accountID, types.UpdateReason{Resource: types.UpdateResourcePeer, Operation: types.UpdateOperationUpdate})
+
+	account, err = am.Store.GetAccount(ctx, accountID)
+	require.NoError(t, err)
+	proxyIP2 := account.Peers[peer2ID].IP.String()
+	require.NotEmpty(t, proxyIP2, "K2 must have an assigned overlay IP")
+	require.NotEqual(t, proxyIP1, proxyIP2, "K2 must get a different overlay IP than K1 (sanity)")
+
+	// CRITICAL ASSERTION 1: K1 must no longer be in the store. The SqlStore
+	// returns ("", nil) for a missing key rather than NotFound, so assert
+	// on the returned ID being empty.
+	staleID, err := am.Store.GetPeerIDByKey(ctx, store.LockingStrengthNone, proxyKey1)
+	require.NoError(t, err, "GetPeerIDByKey for a missing peer must not error")
+	assert.Empty(t, staleID,
+		"stale embedded proxy peer K1 must be removed when a new embedded peer registers for the same (account, cluster); pre-fix this assertion fails because management never cleans up the prior peer record")
+
+	// CRITICAL ASSERTION 2: exactly one embedded proxy peer remains, and it
+	// is K2.
+	account, err = am.Store.GetAccount(ctx, accountID)
+	require.NoError(t, err)
+	embeddedKeys := []string{}
+	for _, p := range account.Peers {
+		if p.ProxyMeta.Embedded {
+			embeddedKeys = append(embeddedKeys, p.Key)
+		}
+	}
+	assert.Equal(t, []string{proxyKey2}, embeddedKeys,
+		"after a proxy restart exactly one embedded proxy peer should remain — the one with the new key K2")
+
+	// CRITICAL ASSERTION 3: the synth DNS record the client receives now
+	// points at K2's IP, not K1's.
+	rdata2 := awaitZoneRData(clientCh, clusterAddr, fqdn, true)
+	assert.Equal(t, proxyIP2, rdata2,
+		"after proxy restart, the client's synth DNS record must point at the NEW embedded peer's IP, not the stale K1 IP")
+}