Refactor sync fast path tests and fix CI flakiness

- Introduce `skipOnWindows` helper to properly skip tests relying on Unix specific paths.
- Replace fixed sleep with `require.Eventually` in `waitForPeerDisconnect` to address flakiness in CI.
- Split `commitFastPath` logic out of `runFastPathSync` to close race conditions and improve clarity.
- Update tests to leverage new helpers and more precise assertions (e.g., `waitForPeerDisconnect`).
- Add `flakyStore` test helper to exercise fail-closed behavior in flag handling.
- Enhance `RunFastPathFlagRoutine` to disable the flag on store read errors.

management/server/sync_fast_path_test.go

@@ -12,10 +12,21 @@ import (
 
 	"github.com/netbirdio/netbird/encryption"
 	"github.com/netbirdio/netbird/management/internals/server/config"
+	"github.com/netbirdio/netbird/management/server/store"
 	mgmtProto "github.com/netbirdio/netbird/shared/management/proto"
 	"github.com/netbirdio/netbird/util"
 )
 
+// skipOnWindows skips the calling test on Windows. The in-process gRPC
+// harness uses Unix socket / path conventions that do not cleanly map to
+// Windows.
+func skipOnWindows(t *testing.T) {
+	t.Helper()
+	if runtime.GOOS == "windows" {
+		t.Skip("skipping on windows; harness uses unix path conventions")
+	}
+}
+
 func fastPathTestConfig(t *testing.T) *config.Config {
 	t.Helper()
 	return &config.Config{
@@ -72,12 +83,21 @@ func openSync(t *testing.T, client mgmtProto.ManagementServiceClient, serverKey,
 	return resp, cancel
 }
 
-// waitForPeerDisconnect gives the server's handleUpdates goroutine a moment to
-// notice the cancelled stream and run cancelPeerRoutines before the next open.
-// Without this the new stream can race with the old one's channel close and
-// trigger a spurious disconnect.
-func waitForPeerDisconnect() {
-	time.Sleep(50 * time.Millisecond)
+// waitForPeerDisconnect polls until the account manager reports the peer as
+// disconnected (Status.Connected == false), which happens once the server's
+// handleUpdates goroutine has run cancelPeerRoutines for the cancelled
+// stream. The deadline is bounded so a stuck server fails the test rather
+// than hanging. Replaces the former fixed 50ms sleep which was CI-flaky
+// under load or with the race detector on.
+func waitForPeerDisconnect(t *testing.T, am *DefaultAccountManager, peerPubKey string) {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		peer, err := am.Store.GetPeerByPeerPubKey(context.Background(), store.LockingStrengthNone, peerPubKey)
+		if err != nil {
+			return false
+		}
+		return !peer.Status.Connected
+	}, 2*time.Second, 10*time.Millisecond, "peer %s should be marked disconnected after stream cancel", peerPubKey)
 }
 
 func baseLinuxMeta() *mgmtProto.PeerSystemMeta {
@@ -103,9 +123,7 @@ func androidMeta() *mgmtProto.PeerSystemMeta {
 }
 
 func TestSyncFastPath_FirstSync_SendsFullMap(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("skipping on windows; harness uses unix path conventions")
-	}
+	skipOnWindows(t)
 	mgmtServer, _, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
 	require.NoError(t, err)
 	defer cleanup()
@@ -128,7 +146,8 @@ func TestSyncFastPath_FirstSync_SendsFullMap(t *testing.T) {
 }
 
 func TestSyncFastPath_SecondSync_MatchingSerial_SkipsMap(t *testing.T) {
-	mgmtServer, _, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
+	skipOnWindows(t)
+	mgmtServer, am, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
 	require.NoError(t, err)
 	defer cleanup()
 	defer mgmtServer.GracefulStop()
@@ -145,7 +164,7 @@ func TestSyncFastPath_SecondSync_MatchingSerial_SkipsMap(t *testing.T) {
 	first, cancel1 := openSync(t, client, *serverKey, *keys[0], baseLinuxMeta())
 	require.NotNil(t, first.NetworkMap, "first sync primes cache with a full map")
 	cancel1()
-	waitForPeerDisconnect()
+	waitForPeerDisconnect(t, am, keys[0].PublicKey().String())
 
 	second, cancel2 := openSync(t, client, *serverKey, *keys[0], baseLinuxMeta())
 	defer cancel2()
@@ -158,7 +177,8 @@ func TestSyncFastPath_SecondSync_MatchingSerial_SkipsMap(t *testing.T) {
 }
 
 func TestSyncFastPath_AndroidNeverSkips(t *testing.T) {
-	mgmtServer, _, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
+	skipOnWindows(t)
+	mgmtServer, am, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
 	require.NoError(t, err)
 	defer cleanup()
 	defer mgmtServer.GracefulStop()
@@ -175,7 +195,7 @@ func TestSyncFastPath_AndroidNeverSkips(t *testing.T) {
 	first, cancel1 := openSync(t, client, *serverKey, *keys[0], androidMeta())
 	require.NotNil(t, first.NetworkMap, "android first sync must deliver a full map")
 	cancel1()
-	waitForPeerDisconnect()
+	waitForPeerDisconnect(t, am, keys[0].PublicKey().String())
 
 	second, cancel2 := openSync(t, client, *serverKey, *keys[0], androidMeta())
 	defer cancel2()
@@ -184,7 +204,8 @@ func TestSyncFastPath_AndroidNeverSkips(t *testing.T) {
 }
 
 func TestSyncFastPath_MetaChanged_SendsFullMap(t *testing.T) {
-	mgmtServer, _, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
+	skipOnWindows(t)
+	mgmtServer, am, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
 	require.NoError(t, err)
 	defer cleanup()
 	defer mgmtServer.GracefulStop()
@@ -201,7 +222,7 @@ func TestSyncFastPath_MetaChanged_SendsFullMap(t *testing.T) {
 	first, cancel1 := openSync(t, client, *serverKey, *keys[0], baseLinuxMeta())
 	require.NotNil(t, first.NetworkMap, "first sync primes cache")
 	cancel1()
-	waitForPeerDisconnect()
+	waitForPeerDisconnect(t, am, keys[0].PublicKey().String())
 
 	changed := baseLinuxMeta()
 	changed.Hostname = "linux-host-renamed"
@@ -212,7 +233,8 @@ func TestSyncFastPath_MetaChanged_SendsFullMap(t *testing.T) {
 }
 
 func TestSyncFastPath_LoginInvalidatesCache(t *testing.T) {
-	mgmtServer, _, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
+	skipOnWindows(t)
+	mgmtServer, am, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
 	require.NoError(t, err)
 	defer cleanup()
 	defer mgmtServer.GracefulStop()
@@ -233,7 +255,7 @@ func TestSyncFastPath_LoginInvalidatesCache(t *testing.T) {
 	first, cancel1 := openSync(t, client, *serverKey, key, baseLinuxMeta())
 	require.NotNil(t, first.NetworkMap, "first sync primes cache")
 	cancel1()
-	waitForPeerDisconnect()
+	waitForPeerDisconnect(t, am, key.PublicKey().String())
 
 	// A subsequent login (e.g. SSH key rotation, re-auth) must clear the cache.
 	_, err = loginPeerWithValidSetupKey(key, client)
@@ -245,7 +267,8 @@ func TestSyncFastPath_LoginInvalidatesCache(t *testing.T) {
 }
 
 func TestSyncFastPath_OtherPeerRegistered_ForcesFullMap(t *testing.T) {
-	mgmtServer, _, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
+	skipOnWindows(t)
+	mgmtServer, am, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
 	require.NoError(t, err)
 	defer cleanup()
 	defer mgmtServer.GracefulStop()
@@ -262,7 +285,7 @@ func TestSyncFastPath_OtherPeerRegistered_ForcesFullMap(t *testing.T) {
 	first, cancel1 := openSync(t, client, *serverKey, *keys[0], baseLinuxMeta())
 	require.NotNil(t, first.NetworkMap, "first sync primes cache at serial N")
 	cancel1()
-	waitForPeerDisconnect()
+	waitForPeerDisconnect(t, am, keys[0].PublicKey().String())
 
 	// Registering another peer bumps the account serial via IncrementNetworkSerial.
 	_, err = registerPeers(1, client)

management/server/sync_legacy_wire_test.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/golang/protobuf/proto" //nolint:staticcheck // matches the generator
 	"github.com/stretchr/testify/require"
@@ -46,6 +45,7 @@ func sendWireFixture(t *testing.T, client mgmtProto.ManagementServiceClient, ser
 }
 
 func TestSync_WireFixture_LegacyClients_AlwaysReceiveFullMap(t *testing.T) {
+	skipOnWindows(t)
 	cases := []struct {
 		name    string
 		fixture string
@@ -87,12 +87,13 @@ func TestSync_WireFixture_LegacyClient_ReconnectStillGetsFullMap(t *testing.T) {
 	// readInitialSettings — they error on nil NetworkMap. Without extra opt-in
 	// signalling there is no way for the server to know this is a GetNetworkMap
 	// call rather than a main Sync, so the server's fast path would break them
-	// on reconnect. This test documents the currently accepted tradeoff: a
-	// legacy client always gets a full map on the first Sync, but a warm cache
-	// entry for the same peer key (set by a previous modern-client flow) does
-	// lead to the fast path. When a future proto opt-in lands, this test must
-	// be tightened to assert full map even on a cache hit for legacy meta.
-	mgmtServer, _, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
+	// on reconnect. This test pins the currently accepted tradeoff: a legacy
+	// v0.40 client gets a full map on the first Sync but a reconnect with an
+	// unchanged metaHash hits the primed cache and goes through the fast path.
+	// When a future proto opt-in lets the server distinguish these clients,
+	// this assertion must be tightened to require.NotNil(second.NetworkMap).
+	skipOnWindows(t)
+	mgmtServer, am, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
 	require.NoError(t, err)
 	defer cleanup()
 	defer mgmtServer.GracefulStop()
@@ -110,16 +111,19 @@ func TestSync_WireFixture_LegacyClient_ReconnectStillGetsFullMap(t *testing.T) {
 	require.NoError(t, err)
 
 	first, cancel1 := sendWireFixture(t, client, *serverKey, *keys[0], abs)
-	cancel1()
 	require.NotNil(t, first.NetworkMap, "first legacy sync receives full map and primes cache")
+	cancel1()
+	waitForPeerDisconnect(t, am, keys[0].PublicKey().String())
 
-	// Give server-side handleUpdates time to tear down the first stream before
-	// we reopen for the same peer.
-	time.Sleep(50 * time.Millisecond)
+	second, cancel2 := sendWireFixture(t, client, *serverKey, *keys[0], abs)
+	defer cancel2()
+	require.Nil(t, second.NetworkMap, "documented legacy-reconnect tradeoff: warm cache entry causes fast path; update when proto opt-in lands")
+	require.NotNil(t, second.NetbirdConfig, "fast path still delivers NetbirdConfig")
 }
 
 func TestSync_WireFixture_AndroidReconnect_NeverSkips(t *testing.T) {
-	mgmtServer, _, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
+	skipOnWindows(t)
+	mgmtServer, am, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
 	require.NoError(t, err)
 	defer cleanup()
 	defer mgmtServer.GracefulStop()
@@ -139,7 +143,7 @@ func TestSync_WireFixture_AndroidReconnect_NeverSkips(t *testing.T) {
 	first, cancel1 := sendWireFixture(t, client, *serverKey, *keys[0], abs)
 	require.NotNil(t, first.NetworkMap, "android first sync must deliver a full map")
 	cancel1()
-	waitForPeerDisconnect()
+	waitForPeerDisconnect(t, am, keys[0].PublicKey().String())
 
 	second, cancel2 := sendWireFixture(t, client, *serverKey, *keys[0], abs)
 	defer cancel2()
@@ -147,7 +151,8 @@ func TestSync_WireFixture_AndroidReconnect_NeverSkips(t *testing.T) {
 }
 
 func TestSync_WireFixture_ModernClientReconnect_TakesFastPath(t *testing.T) {
-	mgmtServer, _, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
+	skipOnWindows(t)
+	mgmtServer, am, addr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sql", fastPathTestConfig(t))
 	require.NoError(t, err)
 	defer cleanup()
 	defer mgmtServer.GracefulStop()
@@ -167,7 +172,7 @@ func TestSync_WireFixture_ModernClientReconnect_TakesFastPath(t *testing.T) {
 	first, cancel1 := sendWireFixture(t, client, *serverKey, *keys[0], abs)
 	require.NotNil(t, first.NetworkMap, "modern first sync primes cache")
 	cancel1()
-	waitForPeerDisconnect()
+	waitForPeerDisconnect(t, am, keys[0].PublicKey().String())
 
 	second, cancel2 := sendWireFixture(t, client, *serverKey, *keys[0], abs)
 	defer cancel2()