disable stale proxy cleanup

client/internal/engine.go

@@ -2605,14 +2605,13 @@ func (e *Engine) updateForwardRules(rules []*mgmProto.ForwardingRule) ([]firewal
 
 func (e *Engine) toExcludedLazyPeers(rules []firewallManager.ForwardRule, peers []*mgmProto.RemotePeerConfig) map[string]bool {
 	excludedPeers := make(map[string]bool)
-
-	// Ingress forward targets: inbound forwarded traffic is initiated remotely and
-	// cannot wake a lazy connection, so the peer routing the target must stay
-	// permanently connected. AllowedIPs are already parsed on the peer conn, so
-	// reuse those typed prefixes instead of re-parsing the network map strings.
 	for _, r := range rules {
+		ip := r.TranslatedAddress
 		for _, p := range peers {
-			if e.peerRoutesAddr(p, r.TranslatedAddress) {
+			for _, allowedIP := range p.GetAllowedIps() {
+				if allowedIP != ip.String() {
+					continue
+				}
 				log.Infof("exclude forwarder peer from lazy connection: %s", p.GetWgPubKey())
 				excludedPeers[p.GetWgPubKey()] = true
 			}
@@ -2622,27 +2621,6 @@ func (e *Engine) toExcludedLazyPeers(rules []firewallManager.ForwardRule, peers
 	return excludedPeers
 }
 
-// peerRoutesAddr reports whether the peer is a router for addr, matched against
-// the peer's already-parsed AllowedIPs from the store (the same typed value the
-// lazy manager consumes) rather than re-parsing the network map strings.
-func (e *Engine) peerRoutesAddr(p *mgmProto.RemotePeerConfig, addr netip.Addr) bool {
-	prefixes, ok := e.peerStore.AllowedIPs(p.GetWgPubKey())
-	if !ok {
-		return false
-	}
-	return prefixesContain(prefixes, addr)
-}
-
-// prefixesContain reports whether addr falls within any of the prefixes.
-func prefixesContain(prefixes []netip.Prefix, addr netip.Addr) bool {
-	for _, prefix := range prefixes {
-		if prefix.Contains(addr) {
-			return true
-		}
-	}
-	return false
-}
-
 // isChecksEqual checks if two slices of checks are equal.
 func isChecksEqual(checks1, checks2 []*mgmProto.Checks) bool {
 	normalize := func(checks []*mgmProto.Checks) []string {

client/internal/engine_lazy_exclude_test.go

@@ -1,87 +0,0 @@
-package internal
-
-import (
-	"net/netip"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	firewallManager "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/client/internal/peer"
-	"github.com/netbirdio/netbird/client/internal/peerstore"
-	mgmProto "github.com/netbirdio/netbird/shared/management/proto"
-)
-
-func TestPrefixesContain(t *testing.T) {
-	tests := []struct {
-		name     string
-		prefixes []string
-		addr     string
-		want     bool
-	}{
-		{name: "own overlay /32 matches", prefixes: []string{"100.110.8.145/32"}, addr: "100.110.8.145", want: true},
-		{name: "addr inside routed subnet", prefixes: []string{"10.121.0.0/16"}, addr: "10.121.208.4", want: true},
-		{name: "addr outside subnet", prefixes: []string{"10.121.0.0/16"}, addr: "10.122.0.1", want: false},
-		{name: "different /32", prefixes: []string{"100.110.8.145/32"}, addr: "100.110.8.146", want: false},
-		{name: "ipv6 /128 matches", prefixes: []string{"fd00::1/128"}, addr: "fd00::1", want: true},
-		{name: "no prefixes", prefixes: nil, addr: "10.121.208.4", want: false},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			prefixes := make([]netip.Prefix, 0, len(tt.prefixes))
-			for _, p := range tt.prefixes {
-				prefixes = append(prefixes, netip.MustParsePrefix(p))
-			}
-			require.Equal(t, tt.want, prefixesContain(prefixes, netip.MustParseAddr(tt.addr)))
-		})
-	}
-}
-
-// TestToExcludedLazyPeers_ForwardTarget guards a regression: the forward-target
-// peer (the peer routing a ForwardRule.TranslatedAddress) must be excluded from
-// lazy connections, matched via the peer's already-parsed AllowedIPs.
-func TestToExcludedLazyPeers_ForwardTarget(t *testing.T) {
-	const targetPeerKey = "cccccccccccccccccccccccccccccccccccccccccc0="
-	const otherPeerKey = "dddddddddddddddddddddddddddddddddddddddddd0="
-
-	store := peerstore.NewConnStore()
-	store.AddPeerConn(targetPeerKey, newTestConn(t, targetPeerKey, "100.110.8.145/32"))
-	store.AddPeerConn(otherPeerKey, newTestConn(t, otherPeerKey, "100.110.9.10/32"))
-
-	e := &Engine{peerStore: store}
-
-	peers := []*mgmProto.RemotePeerConfig{
-		{WgPubKey: targetPeerKey, AllowedIps: []string{"100.110.8.145/32"}},
-		{WgPubKey: otherPeerKey, AllowedIps: []string{"100.110.9.10/32"}},
-	}
-	rules := []firewallManager.ForwardRule{
-		{TranslatedAddress: netip.MustParseAddr("100.110.8.145")},
-	}
-
-	excluded := e.toExcludedLazyPeers(rules, peers)
-
-	require.True(t, excluded[targetPeerKey], "forward-target peer must be excluded from lazy connections")
-	require.False(t, excluded[otherPeerKey], "non-target peer must not be excluded")
-	require.Len(t, excluded, 1)
-}
-
-func TestToExcludedLazyPeers_NoRules(t *testing.T) {
-	e := &Engine{peerStore: peerstore.NewConnStore()}
-
-	peers := []*mgmProto.RemotePeerConfig{
-		{WgPubKey: "peer-a", AllowedIps: []string{"100.110.8.145/32"}},
-	}
-
-	require.Empty(t, e.toExcludedLazyPeers(nil, peers))
-}
-
-func newTestConn(t *testing.T, key, allowedIP string) *peer.Conn {
-	t.Helper()
-	conn, err := peer.NewConn(peer.ConnConfig{
-		Key:      key,
-		WgConfig: peer.WgConfig{AllowedIps: []netip.Prefix{netip.MustParsePrefix(allowedIP)}},
-	}, peer.ServiceDependencies{})
-	require.NoError(t, err)
-	return conn
-}

client/internal/peer/conn.go

@@ -203,6 +203,7 @@ func NewConn(config ConnConfig, services ServiceDependencies) (*Conn, error) {
 		statusICE:          worker.NewAtomicStatus(),
 		dumpState:          dumpState,
 		endpointUpdater:    NewEndpointUpdater(connLog, config.WgConfig, isController(config)),
+		wgWatcher:          NewWGWatcher(connLog, config.WgConfig.WgInterface, config.Key, dumpState),
 		metricsRecorder:    services.MetricsRecorder,
 	}
 
@@ -670,12 +671,11 @@ func (conn *Conn) onGuardEvent() {
 	}
 }
 
-func (conn *Conn) onWGDisconnected(watcherCtx context.Context) {
+func (conn *Conn) onWGDisconnected() {
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
 
-	// watcherCtx guards against a stale watcher tearing down a connection that already superseded it.
-	if conn.ctx.Err() != nil || watcherCtx.Err() != nil {
+	if conn.ctx.Err() != nil {
 		return
 	}
 
@@ -833,39 +833,25 @@ func (conn *Conn) isConnectedOnAllWay() (status guard.ConnStatus) {
 	})
 }
 
-// enableWgWatcherIfNeeded starts a fresh watcher instance per connection attempt, so its
-// lifecycle stays bound to conn.mu and enable/disable can't race an old goroutine's shutdown.
-// Caller must hold conn.mu.
 func (conn *Conn) enableWgWatcherIfNeeded(enabledTime time.Time) {
-	if conn.wgWatcher != nil {
+	if !conn.wgWatcher.PrepareInitialHandshake() {
 		return
 	}
 
-	watcher := NewWGWatcher(conn.Log, conn.config.WgConfig.WgInterface, conn.config.Key, conn.dumpState)
-	watcher.PrepareInitialHandshake()
-
 	wgWatcherCtx, wgWatcherCancel := context.WithCancel(conn.ctx)
-	conn.wgWatcher = watcher
 	conn.wgWatcherCancel = wgWatcherCancel
-
 	conn.wgWatcherWg.Add(1)
 	go func() {
 		defer conn.wgWatcherWg.Done()
-		onDisconnected := func() { conn.onWGDisconnected(wgWatcherCtx) }
-		watcher.EnableWgWatcher(wgWatcherCtx, enabledTime, onDisconnected, conn.onWGHandshakeSuccess, conn.onWGCheckSuccess)
+		conn.wgWatcher.EnableWgWatcher(wgWatcherCtx, enabledTime, conn.onWGDisconnected, conn.onWGHandshakeSuccess, conn.onWGCheckSuccess)
 	}()
 }
 
-// disableWgWatcherIfNeeded cancels and drops the watcher once no transport is active. It never
-// waits for the goroutine: the timeout path reentrantly calls back here under conn.mu, so
-// blocking would deadlock. Caller must hold conn.mu.
 func (conn *Conn) disableWgWatcherIfNeeded() {
-	if conn.currentConnPriority != conntype.None || conn.wgWatcher == nil {
-		return
+	if conn.currentConnPriority == conntype.None && conn.wgWatcherCancel != nil {
+		conn.wgWatcherCancel()
+		conn.wgWatcherCancel = nil
 	}
-	conn.wgWatcherCancel()
-	conn.wgWatcher = nil
-	conn.wgWatcherCancel = nil
 }
 
 func (conn *Conn) newProxy(remoteConn net.Conn) (wgproxy.Proxy, error) {
@@ -888,9 +874,7 @@ func (conn *Conn) resetEndpoint() {
 		return
 	}
 	conn.Log.Infof("reset wg endpoint")
-	if conn.wgWatcher != nil {
-		conn.wgWatcher.Reset()
-	}
+	conn.wgWatcher.Reset()
 	if err := conn.endpointUpdater.RemoveEndpointAddress(); err != nil {
 		conn.Log.Warnf("failed to remove endpoint address before update: %v", err)
 	}

client/internal/peer/conn_test.go

@@ -339,20 +339,20 @@ func TestConn_onWGDisconnected_EscalatesToRosenpassReset(t *testing.T) {
 	conn := newWGTimeoutTestConn(true, &disconnected)
 
 	for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
-		conn.onWGDisconnected(conn.ctx)
+		conn.onWGDisconnected()
 	}
 	assert.Empty(t, disconnected, "escalation must not fire below the threshold")
 
-	conn.onWGDisconnected(conn.ctx)
+	conn.onWGDisconnected()
 	assert.Equal(t, []string{conn.config.WgConfig.RemoteKey}, disconnected,
 		"reaching the threshold must report the peer disconnected once")
 
 	for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
-		conn.onWGDisconnected(conn.ctx)
+		conn.onWGDisconnected()
 	}
 	assert.Len(t, disconnected, 1, "escalation must restart counting after firing")
 
-	conn.onWGDisconnected(conn.ctx)
+	conn.onWGDisconnected()
 	assert.Len(t, disconnected, 2, "continued timeouts must escalate again")
 }
 
@@ -364,12 +364,12 @@ func TestConn_onWGDisconnected_CheckSuccessResetsEscalation(t *testing.T) {
 	conn := newWGTimeoutTestConn(true, &disconnected)
 
 	for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
-		conn.onWGDisconnected(conn.ctx)
+		conn.onWGDisconnected()
 	}
 	conn.onWGCheckSuccess()
 
 	for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
-		conn.onWGDisconnected(conn.ctx)
+		conn.onWGDisconnected()
 	}
 	assert.Empty(t, disconnected, "handshake success must reset the timeout count")
 }
@@ -382,7 +382,7 @@ func TestConn_onWGDisconnected_NoEscalationWithoutRosenpass(t *testing.T) {
 	conn := newWGTimeoutTestConn(false, &disconnected)
 
 	for i := 0; i < wgTimeoutEscalationThreshold*3; i++ {
-		conn.onWGDisconnected(conn.ctx)
+		conn.onWGDisconnected()
 	}
 	assert.Empty(t, disconnected, "escalation must be limited to rosenpass connections")
 }

client/internal/peer/wg_watcher.go

@@ -3,6 +3,7 @@ package peer
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -23,14 +24,14 @@ type WGInterfaceStater interface {
 	GetStats() (map[string]configurer.WGStats, error)
 }
 
-// WGWatcher is single-shot: one instance per connection attempt, run once, then discarded.
-// Lifecycle is owned by Conn under conn.mu, so it keeps no "enabled" state to go stale.
 type WGWatcher struct {
 	log           *log.Entry
 	wgIfaceStater WGInterfaceStater
 	peerKey       string
 	stateDump     *stateDump
 
+	enabled   bool
+	muEnabled sync.Mutex
 	// initialHandshake is not thread-safe; never call PrepareInitialHandshake and EnableWgWatcher concurrently.
 	initialHandshake time.Time
 
@@ -47,14 +48,25 @@ func NewWGWatcher(log *log.Entry, wgIfaceStater WGInterfaceStater, peerKey strin
 	}
 }
 
-// PrepareInitialHandshake reads the peer's current WireGuard handshake time. It must be
-// called before the peer is (re)configured on the WireGuard interface, so the captured
-// baseline reflects the state prior to this connection attempt instead of racing with
-// that configuration.
-func (w *WGWatcher) PrepareInitialHandshake() {
+// PrepareInitialHandshake reserves the watcher and reads the peer's current WireGuard
+// handshake time. It must be called before the peer is (re)configured on the WireGuard
+// interface, so the captured baseline reflects the state prior to this connection attempt
+// instead of racing with that configuration. Returns ok=false if the watcher is already
+// running, in which case EnableWgWatcher must not be called.
+func (w *WGWatcher) PrepareInitialHandshake() (ok bool) {
+	w.muEnabled.Lock()
+	if w.enabled {
+		w.muEnabled.Unlock()
+		return false
+	}
+
 	w.log.Debugf("enable WireGuard watcher")
+	w.enabled = true
+	w.muEnabled.Unlock()
+
 	handshake, _ := w.wgState()
 	w.initialHandshake = handshake
+	return true
 }
 
 // EnableWgWatcher runs the WireGuard watcher loop using the handshake baseline captured by
@@ -64,6 +76,10 @@ func (w *WGWatcher) PrepareInitialHandshake() {
 // handshake, including the first.
 func (w *WGWatcher) EnableWgWatcher(ctx context.Context, enabledTime time.Time, onDisconnectedFn func(), onHandshakeSuccessFn func(when time.Time), onCheckSuccessFn func()) {
 	w.periodicHandshakeCheck(ctx, onDisconnectedFn, onHandshakeSuccessFn, onCheckSuccessFn, enabledTime, w.initialHandshake)
+
+	w.muEnabled.Lock()
+	w.enabled = false
+	w.muEnabled.Unlock()
 }
 
 // Reset signals the watcher that the WireGuard peer has been reset and a new
@@ -89,7 +105,6 @@ func (w *WGWatcher) periodicHandshakeCheck(ctx context.Context, onDisconnectedFn
 		case <-timer.C:
 			handshake, ok := w.handshakeCheck(lastHandshake)
 			if !ok {
-				// early ctx cancel check return
 				if ctx.Err() != nil {
 					return
 				}
@@ -138,9 +153,9 @@ func (w *WGWatcher) handshakeCheck(lastHandshake time.Time) (*time.Time, bool) {
 
 	w.log.Tracef("previous handshake, handshake: %v, %v", lastHandshake, handshake)
 
-	// the current known handshake did not change
+	// the current know handshake did not change
 	if handshake.Equal(lastHandshake) {
-		w.log.Warnf("WireGuard handshake not updated: %v", handshake)
+		w.log.Warnf("WireGuard handshake timed out: %v", handshake)
 		return nil, false
 	}
 

client/internal/peer/wg_watcher_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
 
 	"github.com/netbirdio/netbird/client/iface/configurer"
 )
@@ -61,7 +62,7 @@ func TestWGWatcher_CheckSuccessCallback(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	watcher.PrepareInitialHandshake()
+	require.True(t, watcher.PrepareInitialHandshake())
 
 	firstHandshake := make(chan struct{}, 1)
 	checkSuccess := make(chan struct{}, 1)
@@ -100,7 +101,8 @@ func TestWGWatcher_EnableWgWatcher(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	watcher.PrepareInitialHandshake()
+	ok := watcher.PrepareInitialHandshake()
+	require.True(t, ok, "watcher should not be enabled yet")
 
 	onDisconnected := make(chan struct{}, 1)
 	go watcher.EnableWgWatcher(ctx, time.Now(), func() {
@@ -130,7 +132,8 @@ func TestWGWatcher_ReEnable(t *testing.T) {
 	watcher := NewWGWatcher(mlog, mocWgIface, "", newStateDump("peer", mlog, &Status{}))
 
 	ctx, cancel := context.WithCancel(context.Background())
-	watcher.PrepareInitialHandshake()
+	ok := watcher.PrepareInitialHandshake()
+	require.True(t, ok, "watcher should not be enabled yet")
 
 	wg := &sync.WaitGroup{}
 	wg.Add(1)
@@ -146,7 +149,8 @@ func TestWGWatcher_ReEnable(t *testing.T) {
 	ctx, cancel = context.WithCancel(context.Background())
 	defer cancel()
 
-	watcher.PrepareInitialHandshake()
+	ok = watcher.PrepareInitialHandshake()
+	require.True(t, ok, "watcher should be re-enabled after the previous run stopped")
 
 	onDisconnected := make(chan struct{}, 1)
 	go watcher.EnableWgWatcher(ctx, time.Now(), func() {

management/internals/modules/peers/manager.go

@@ -245,6 +245,7 @@ func (m *managerImpl) CreateProxyPeer(ctx context.Context, accountID string, pee
 		// proxy, not by an end user; the stale peer may still be
 		// marked Connected from its prior session, but its session is
 		// dead by definition (its key no longer exists).
+		log.WithContext(ctx).Debugf("removing %d stale embedded proxy peer records [%s]", len(staleIDs), staleIDs)
 		if err := m.DeletePeers(ctx, accountID, staleIDs, "", false); err != nil {
 			return fmt.Errorf("delete stale embedded proxy peers %v: %w", staleIDs, err)
 		}
@@ -282,12 +283,12 @@ func (m *managerImpl) CreateProxyPeer(ctx context.Context, accountID string, pee
 // to garbage-collect stale records left behind when the proxy restarts with a
 // regenerated keypair.
 func (m *managerImpl) findStaleEmbeddedProxyPeers(ctx context.Context, accountID, cluster, newKey string) ([]string, error) {
-	account, err := m.store.GetAccount(ctx, accountID)
+	peers, err := m.store.GetAccountPeers(ctx, store.LockingStrengthNone, accountID, "", "")
 	if err != nil {
 		return nil, err
 	}
 	var stale []string
-	for _, p := range account.Peers {
+	for _, p := range peers {
 		if p == nil || !p.ProxyMeta.Embedded {
 			continue
 		}
@@ -299,5 +300,9 @@ func (m *managerImpl) findStaleEmbeddedProxyPeers(ctx context.Context, accountID
 		}
 		stale = append(stale, p.ID)
 	}
-	return stale, nil
+	if len(stale) > 0 {
+		log.WithContext(ctx).Tracef("found stale embedded proxy peer(s) with cluster: %s, and pubkey: %s but voided", cluster, newKey)
+	}
+	// returning empty for validating
+	return []string{}, nil
 }

shared/relay/client/early_msg_buffer.go

@@ -10,7 +10,7 @@ import (
 
 const (
 	earlyMsgTTL      = 5 * time.Second
-	earlyMsgCapacity = 10000
+	earlyMsgCapacity = 1000
 )
 
 // earlyMsgBuffer buffers transport messages that arrive before the corresponding