From fe3c9150003db21b90fd29126ecf8460031b31a4 Mon Sep 17 00:00:00 2001 From: pascal Date: Tue, 27 Oct 2026 13:51:33 +0100 Subject: [PATCH] non blocking heartbeat --- .../management/network_map_db/base_data.sql | 12 ++++++------ .../network_map_db/network_map_data.sql | 12 ++++++------ .../shared/grpc/peer_update_handler.go | 19 ++++++++++++++----- .../shared/grpc/peer_update_handler_test.go | 11 ++++++----- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/integration_tests/management/network_map_db/base_data.sql b/integration_tests/management/network_map_db/base_data.sql index 136df00ac..0e41bd54e 100644 --- a/integration_tests/management/network_map_db/base_data.sql +++ b/integration_tests/management/network_map_db/base_data.sql @@ -21,32 +21,32 @@ insert into group_peers (account_id, peer_id, group_id) VALUES('account-1','peer insert into group_peers (account_id, peer_id, group_id) VALUES('account-1','peer-id-2','group-two-resources-id'); insert into group_peers (account_id, peer_id, group_id) VALUES('account-1','peer-id-3','group-two-resources-id'); insert into peers (id, account_id, "key", ssh_key, dns_label, extra_dns_labels, user_id, ssh_enabled, login_expiration_enabled, last_login, ip, ipv6, - peer_status_requires_approval, peer_status_connected, proxy_meta_embedded, proxy_meta_cluster, + peer_status_requires_approval, peer_status_connected, peer_status_last_seen, proxy_meta_embedded, proxy_meta_cluster, meta_wt_version, meta_go_os, meta_os_version, meta_kernel_version, meta_network_addresses, meta_files, meta_capabilities, meta_flags, meta_sync_message_version, location_country_code, location_city_name, location_connection_ip) values('peer-id-1','account-1','key-1','ssh-key-1','peer-1','["extra-peer-1"]','user-id-1',true,true,'2026-08-06 13:25:59.12999','"10.10.10.1"','"fdf4:ba80:6aa5:89f1:44d7:8701:8699:4940"', - false,true,true,'cluster-1.netbird.services', + false,true,CURRENT_TIMESTAMP,true,'cluster-1.netbird.services', '0.76.0','linux','26.4.1','6.8.0-134-generic','[{"NetIP":"fe80::8b4c:973f:a76b:3771/64","Mac":"00:15:5d:24:0c:ac"},{"NetIP":"192.168.16.1/20","Mac":"00:15:5d:24:0c:ac"}]','[{"Path":"/usr/bin/netbird","Exist":false,"ProcessIsRunning":false}]', '[1,2]','{"RosenpassEnabled":false,"RosenpassPermissive":false,"ServerSSHAllowed":true,"DisableClientRoutes":false,"DisableServerRoutes":false,"DisableDNS":false,"DisableFirewall":false,"BlockLANAccess":false,"BlockInbound":false,"DisableIPv6":false,"LazyConnectionEnabled":false}',1, 'DE','Berlin','"46.201.148.187"'); insert into peers (id,account_id,"key", ssh_key, dns_label, extra_dns_labels, user_id, ssh_enabled, login_expiration_enabled, last_login, ip, ipv6, - peer_status_requires_approval, peer_status_connected, proxy_meta_embedded, proxy_meta_cluster, + peer_status_requires_approval, peer_status_connected, peer_status_last_seen, proxy_meta_embedded, proxy_meta_cluster, meta_wt_version, meta_go_os, meta_os_version, meta_kernel_version, meta_network_addresses, meta_files, meta_capabilities, meta_flags, meta_sync_message_version, location_country_code, location_city_name, location_connection_ip) values('peer-id-2','account-1','key-2','ssh-key-2','peer-2','["extra-peer-2"]','user-id-2',true,true,'2026-08-06 14:25:59.12999','"10.10.100.1"','"fdf5:ba80:6aa5:89f1:44d7:8701:8699:4940"', - false,true,true,'cluster-2.netbird.services', + false,true,CURRENT_TIMESTAMP,true,'cluster-2.netbird.services', '0.76.1','linux','26.4.2','6.8.0-135-generic','[{"NetIP":"fe81::8b4c:973f:a76b:3771/64","Mac":"00:15:5d:24:0c:ad"},{"NetIP":"192.168.17.1/20","Mac":"00:15:5d:24:0c:ad"}]','[{"Path":"/usr/bin/netbird","Exist":false,"ProcessIsRunning":false}]', '[1,2]','{"RosenpassEnabled":false,"RosenpassPermissive":false,"ServerSSHAllowed":true,"DisableClientRoutes":false,"DisableServerRoutes":false,"DisableDNS":false,"DisableFirewall":false,"BlockLANAccess":false,"BlockInbound":false,"DisableIPv6":false,"LazyConnectionEnabled":false}',0, 'DE','Berlin','"46.201.149.187"'); insert into peers (id,account_id,"key", ssh_key, dns_label, extra_dns_labels, user_id, ssh_enabled, login_expiration_enabled, last_login, ip, ipv6, - peer_status_requires_approval, peer_status_connected, proxy_meta_embedded, proxy_meta_cluster, + peer_status_requires_approval, peer_status_connected, peer_status_last_seen, proxy_meta_embedded, proxy_meta_cluster, meta_wt_version, meta_go_os, meta_os_version, meta_kernel_version, meta_network_addresses, meta_files, meta_capabilities, meta_flags, meta_sync_message_version, location_country_code, location_city_name, location_connection_ip) values('peer-id-3','account-1','key-3','ssh-key-3','peer-3','["extra-peer-3"]','user-id-3',true,true,'2026-08-06 12:25:59.12999','"10.10.200.1"','"fdf6:ba80:6aa5:89f1:44d7:8701:8699:4940"', - false,true,true,'cluster-3.netbird.services', + false,true,CURRENT_TIMESTAMP,true,'cluster-3.netbird.services', '0.76.2','linux','26.4.3','6.8.0-136-generic','[{"NetIP":"fe82::8b4c:973f:a76b:3771/64","Mac":"00:15:5d:24:0c:ae"},{"NetIP":"192.168.18.1/20","Mac":"00:15:5d:24:0c:ae"}]','[{"Path":"/usr/bin/netbird","Exist":false,"ProcessIsRunning":false}]', '[1,2]','{"RosenpassEnabled":false,"RosenpassPermissive":false,"ServerSSHAllowed":true,"DisableClientRoutes":false,"DisableServerRoutes":false,"DisableDNS":false,"DisableFirewall":false,"BlockLANAccess":false,"BlockInbound":false,"DisableIPv6":false,"LazyConnectionEnabled":false}',1, 'DE','Berlin','"46.201.150.187"'); diff --git a/integration_tests/management/network_map_db/network_map_data.sql b/integration_tests/management/network_map_db/network_map_data.sql index d94e2f4aa..5a8a903bd 100644 --- a/integration_tests/management/network_map_db/network_map_data.sql +++ b/integration_tests/management/network_map_db/network_map_data.sql @@ -13,32 +13,32 @@ insert into group_peers (account_id, peer_id, group_id) VALUES('account-33','pee insert into group_peers (account_id, peer_id, group_id) VALUES('account-33','peer-id-332','33-group-two-resources-id'); insert into group_peers (account_id, peer_id, group_id) VALUES('account-33','peer-id-333','33-group-two-resources-id'); insert into peers (id, account_id, "key", ssh_key, dns_label, extra_dns_labels, user_id, ssh_enabled, login_expiration_enabled, last_login, ip, ipv6, - peer_status_requires_approval, peer_status_connected, proxy_meta_embedded, proxy_meta_cluster, + peer_status_requires_approval, peer_status_connected, peer_status_last_seen, proxy_meta_embedded, proxy_meta_cluster, meta_wt_version, meta_go_os, meta_os_version, meta_kernel_version, meta_network_addresses, meta_files, meta_capabilities, meta_flags, meta_sync_message_version, location_country_code, location_city_name, location_connection_ip) values('peer-id-331','account-33','key-331','ssh-key-1','peer-1','["extra-peer-1"]','user-id-1',true,true,'2026-08-06 13:25:59.12999','"10.10.10.1"','"fdf4:ba80:6aa5:89f1:44d7:8701:8699:4940"', - false,true,true,'cluster-1.netbird.services', + false,true,CURRENT_TIMESTAMP,true,'cluster-1.netbird.services', '0.76.0','linux','26.4.1','6.8.0-134-generic','[{"NetIP":"fe80::8b4c:973f:a76b:3771/64","Mac":"00:15:5d:24:0c:ac"},{"NetIP":"192.168.16.1/20","Mac":"00:15:5d:24:0c:ac"}]','[{"Path":"/usr/bin/netbird","Exist":false,"ProcessIsRunning":false}]', '[1,2]','{"RosenpassEnabled":false,"RosenpassPermissive":false,"ServerSSHAllowed":true,"DisableClientRoutes":false,"DisableServerRoutes":false,"DisableDNS":false,"DisableFirewall":false,"BlockLANAccess":false,"BlockInbound":false,"DisableIPv6":false,"LazyConnectionEnabled":false}',1, 'DE','Berlin','"46.201.148.187"'); insert into peers (id,account_id,"key", ssh_key, dns_label, extra_dns_labels, user_id, ssh_enabled, login_expiration_enabled, last_login, ip, ipv6, - peer_status_requires_approval, peer_status_connected, proxy_meta_embedded, proxy_meta_cluster, + peer_status_requires_approval, peer_status_connected, peer_status_last_seen, proxy_meta_embedded, proxy_meta_cluster, meta_wt_version, meta_go_os, meta_os_version, meta_kernel_version, meta_network_addresses, meta_files, meta_capabilities, meta_flags, meta_sync_message_version, location_country_code, location_city_name, location_connection_ip) values('peer-id-332','account-33','key-332','ssh-key-2','peer-2','["extra-peer-2"]','user-id-2',true,true,'2026-08-06 14:25:59.12999','"10.10.100.1"','"fdf5:ba80:6aa5:89f1:44d7:8701:8699:4940"', - false,true,true,'cluster-2.netbird.services', + false,true,CURRENT_TIMESTAMP,true,'cluster-2.netbird.services', '0.76.1','linux','26.4.2','6.8.0-135-generic','[{"NetIP":"fe81::8b4c:973f:a76b:3771/64","Mac":"00:15:5d:24:0c:ad"},{"NetIP":"192.168.17.1/20","Mac":"00:15:5d:24:0c:ad"}]','[{"Path":"/usr/bin/netbird","Exist":false,"ProcessIsRunning":false}]', '[1,2]','{"RosenpassEnabled":false,"RosenpassPermissive":false,"ServerSSHAllowed":true,"DisableClientRoutes":false,"DisableServerRoutes":false,"DisableDNS":false,"DisableFirewall":false,"BlockLANAccess":false,"BlockInbound":false,"DisableIPv6":false,"LazyConnectionEnabled":false}',0, 'DE','Berlin','"46.201.149.187"'); insert into peers (id,account_id,"key", ssh_key, dns_label, extra_dns_labels, user_id, ssh_enabled, login_expiration_enabled, last_login, ip, ipv6, - peer_status_requires_approval, peer_status_connected, proxy_meta_embedded, proxy_meta_cluster, + peer_status_requires_approval, peer_status_connected, peer_status_last_seen, proxy_meta_embedded, proxy_meta_cluster, meta_wt_version, meta_go_os, meta_os_version, meta_kernel_version, meta_network_addresses, meta_files, meta_capabilities, meta_flags, meta_sync_message_version, location_country_code, location_city_name, location_connection_ip) values('peer-id-333','account-33','key-333','ssh-key-3','peer-3','["extra-peer-3"]','user-id-3',true,true,'2026-08-06 12:25:59.12999','"10.10.200.1"','"fdf6:ba80:6aa5:89f1:44d7:8701:8699:4940"', - false,true,true,'cluster-3.netbird.services', + false,true,CURRENT_TIMESTAMP,true,'cluster-3.netbird.services', '0.76.2','linux','26.4.3','6.8.0-136-generic','[{"NetIP":"fe82::8b4c:973f:a76b:3771/64","Mac":"00:15:5d:24:0c:ae"},{"NetIP":"192.168.18.1/20","Mac":"00:15:5d:24:0c:ae"}]','[{"Path":"/usr/bin/netbird","Exist":false,"ProcessIsRunning":false}]', '[1,2]','{"RosenpassEnabled":false,"RosenpassPermissive":false,"ServerSSHAllowed":true,"DisableClientRoutes":false,"DisableServerRoutes":false,"DisableDNS":false,"DisableFirewall":false,"BlockLANAccess":false,"BlockInbound":false,"DisableIPv6":false,"LazyConnectionEnabled":false}',1, 'DE','Berlin','"46.201.150.187"'); diff --git a/management/internals/shared/grpc/peer_update_handler.go b/management/internals/shared/grpc/peer_update_handler.go index 53734993f..1297066cd 100644 --- a/management/internals/shared/grpc/peer_update_handler.go +++ b/management/internals/shared/grpc/peer_update_handler.go @@ -2,18 +2,20 @@ package grpc import ( "context" + "sync/atomic" "time" + log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/management/internals/controllers/network_map" nbpeer "github.com/netbirdio/netbird/management/server/peer" "github.com/netbirdio/netbird/management/server/telemetry" "github.com/netbirdio/netbird/shared/management/certposture" "github.com/netbirdio/netbird/shared/management/proto" - log "github.com/sirupsen/logrus" - "golang.zx2c4.com/wireguard/wgctrl/wgtypes" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) func PeerUpdateHandlerFactory( @@ -77,6 +79,7 @@ func (pu *PeerUpdateHandler) HandleUpdates(ctx context.Context) error { defer pu.debouncer.Stop() var heartbeat <-chan time.Time + var heartbeatRunning atomic.Bool if pu.heartbeat != nil { ticker := time.NewTicker(nbpeer.HeartbeatInterval) defer ticker.Stop() @@ -122,7 +125,13 @@ func (pu *PeerUpdateHandler) HandleUpdates(ctx context.Context) error { } case <-heartbeat: - pu.heartbeat() + // A heartbeat that finishes after cleanup writes nothing: it is fenced on this stream's session. + if heartbeatRunning.CompareAndSwap(false, true) { + go func() { + defer heartbeatRunning.Store(false) + pu.heartbeat() + }() + } // condition when client <-> server connection has been terminated case <-pu.srv.Context().Done(): diff --git a/management/internals/shared/grpc/peer_update_handler_test.go b/management/internals/shared/grpc/peer_update_handler_test.go index 548fc94d6..7871471d3 100644 --- a/management/internals/shared/grpc/peer_update_handler_test.go +++ b/management/internals/shared/grpc/peer_update_handler_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "sync/atomic" "testing" "testing/synctest" "time" @@ -128,7 +129,7 @@ func TestSendPeerUpdates_HeartbeatOnlyOnInterval(t *testing.T) { updateDebouncer := NewMockDebouncer(ctrl) syncSender := NewMocksyncSender(ctrl) - var heartbeats int + var heartbeats atomic.Int32 pu := PeerUpdateHandler{ peerKey: mustGenerateKey(t), updates: make(chan *network_map.UpdateMessage), @@ -137,7 +138,7 @@ func TestSendPeerUpdates_HeartbeatOnlyOnInterval(t *testing.T) { debouncer: updateDebouncer, srv: syncSender, cleanupFunc: func() {}, - heartbeat: func() { heartbeats++ }, + heartbeat: func() { heartbeats.Add(1) }, } msg := network_map.UpdateMessage{ @@ -161,15 +162,15 @@ func TestSendPeerUpdates_HeartbeatOnlyOnInterval(t *testing.T) { pu.updates <- &msg synctest.Wait() - assert.Zero(t, heartbeats, "an update must not write a heartbeat") + assert.Zero(t, heartbeats.Load(), "an update must not write a heartbeat") time.Sleep(nbpeer.HeartbeatInterval) synctest.Wait() - assert.Equal(t, 1, heartbeats, "one heartbeat per interval") + assert.Equal(t, int32(1), heartbeats.Load(), "one heartbeat per interval") time.Sleep(nbpeer.HeartbeatInterval) synctest.Wait() - assert.Equal(t, 2, heartbeats, "one heartbeat per interval") + assert.Equal(t, int32(2), heartbeats.Load(), "one heartbeat per interval") cancel() <-done