[management] use peer.Status.LastSeen field to detect stale peer.Status.Connected values (#7268)

management/internals/shared/grpc/peer_update_handler.go

@@ -2,17 +2,20 @@ package grpc
 
 import (
 	"context"
+	"sync/atomic"
 	"time"
 
-	"github.com/netbirdio/netbird/encryption"
-	"github.com/netbirdio/netbird/management/internals/controllers/network_map"
-	"github.com/netbirdio/netbird/management/server/telemetry"
-	"github.com/netbirdio/netbird/shared/management/certposture"
-	"github.com/netbirdio/netbird/shared/management/proto"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/encryption"
+	"github.com/netbirdio/netbird/management/internals/controllers/network_map"
+	nbpeer "github.com/netbirdio/netbird/management/server/peer"
+	"github.com/netbirdio/netbird/management/server/telemetry"
+	"github.com/netbirdio/netbird/shared/management/certposture"
+	"github.com/netbirdio/netbird/shared/management/proto"
 )
 
 func PeerUpdateHandlerFactory(
@@ -21,6 +24,7 @@ func PeerUpdateHandlerFactory(
 	secretsManager SecretsManager,
 	challenger *certposture.Challenger,
 	onChallengeStamped func(),
+	heartbeat func(),
 	srv proto.ManagementService_SyncServer,
 	cleanupfunc func()) *PeerUpdateHandler {
 	return &PeerUpdateHandler{
@@ -29,6 +33,7 @@ func PeerUpdateHandlerFactory(
 		secretsManager:     secretsManager,
 		challenger:         challenger,
 		onChallengeStamped: onChallengeStamped,
+		heartbeat:          heartbeat,
 		srv:                srv,
 		encrypter:          encryption.DefaultEncrypter{},
 		debouncer:          NewUpdateDebouncer(1000 * time.Millisecond),
@@ -54,6 +59,7 @@ type PeerUpdateHandler struct {
 	encrypter          encryption.Encrypter
 	debouncer          Debouncer
 	cleanupFunc        func()
+	heartbeat          func()
 }
 
 func (pu *PeerUpdateHandler) WithMetrics(appMetrics telemetry.AppMetrics) *PeerUpdateHandler {
@@ -72,6 +78,14 @@ func (pu *PeerUpdateHandler) HandleUpdates(ctx context.Context) error {
 
 	defer pu.debouncer.Stop()
 
+	var heartbeat <-chan time.Time
+	var heartbeatRunning atomic.Bool
+	if pu.heartbeat != nil {
+		ticker := time.NewTicker(nbpeer.HeartbeatInterval)
+		defer ticker.Stop()
+		heartbeat = ticker.C
+	}
+
 	for {
 		select {
 		// condition when there are some updates
@@ -110,6 +124,15 @@ func (pu *PeerUpdateHandler) HandleUpdates(ctx context.Context) error {
 				}
 			}
 
+		case <-heartbeat:
+			// A heartbeat that finishes after cleanup writes nothing: it is fenced on this stream's session.
+			if heartbeatRunning.CompareAndSwap(false, true) {
+				go func() {
+					defer heartbeatRunning.Store(false)
+					pu.heartbeat()
+				}()
+			}
+
 		// condition when client <-> server connection has been terminated
 		case <-pu.srv.Context().Done():
 			// happens when connection drops, e.g. client disconnects

management/internals/shared/grpc/peer_update_handler_test.go

@@ -4,15 +4,19 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"testing"
+	"testing/synctest"
 	"time"
 
 	pb "github.com/golang/protobuf/proto" //nolint
-	"github.com/netbirdio/netbird/management/internals/controllers/network_map"
-	"github.com/netbirdio/netbird/shared/management/proto"
 	"github.com/stretchr/testify/assert"
 	"go.uber.org/mock/gomock"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	"github.com/netbirdio/netbird/management/internals/controllers/network_map"
+	nbpeer "github.com/netbirdio/netbird/management/server/peer"
+	"github.com/netbirdio/netbird/shared/management/proto"
 )
 
 func TestSendPeerUpdates_FirstUpdate(t *testing.T) {
@@ -118,6 +122,61 @@ func TestSendPeerUpdates_ServerContextDone(t *testing.T) {
 	wg.Wait()
 }
 
+func TestSendPeerUpdates_HeartbeatOnlyOnInterval(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		secretsManager := NewMockSecretsManager(ctrl)
+		updateDebouncer := NewMockDebouncer(ctrl)
+		syncSender := NewMocksyncSender(ctrl)
+
+		var heartbeats atomic.Int32
+		pu := PeerUpdateHandler{
+			peerKey:        mustGenerateKey(t),
+			updates:        make(chan *network_map.UpdateMessage),
+			secretsManager: secretsManager,
+			encrypter:      testEncrypter{},
+			debouncer:      updateDebouncer,
+			srv:            syncSender,
+			cleanupFunc:    func() {},
+			heartbeat:      func() { heartbeats.Add(1) },
+		}
+
+		msg := network_map.UpdateMessage{
+			Update: &proto.SyncResponse{Version: 1},
+		}
+
+		srvCtx, cancel := context.WithCancel(context.Background())
+		srvKey := mustGenerateKey(t)
+		updateDebouncer.EXPECT().ProcessUpdate(gomock.Eq(&msg)).Return(true)
+		updateDebouncer.EXPECT().TimerChannel().AnyTimes().Return(make(chan time.Time))
+		syncSender.EXPECT().Context().AnyTimes().Return(srvCtx)
+		secretsManager.EXPECT().GetWGKey().Return(srvKey, nil)
+		syncSender.EXPECT().Send(gomock.Any())
+		updateDebouncer.EXPECT().Stop()
+
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			pu.HandleUpdates(context.Background()) //nolint:errcheck
+		}()
+
+		pu.updates <- &msg
+		synctest.Wait()
+		assert.Zero(t, heartbeats.Load(), "an update must not write a heartbeat")
+
+		time.Sleep(nbpeer.HeartbeatInterval)
+		synctest.Wait()
+		assert.Equal(t, int32(1), heartbeats.Load(), "one heartbeat per interval")
+
+		time.Sleep(nbpeer.HeartbeatInterval)
+		synctest.Wait()
+		assert.Equal(t, int32(2), heartbeats.Load(), "one heartbeat per interval")
+
+		cancel()
+		<-done
+	})
+}
+
 func mustGenerateKey(t *testing.T) wgtypes.Key {
 	t.Helper()
 	k, err := wgtypes.GenerateKey()

management/internals/shared/grpc/server.go

@@ -57,6 +57,8 @@ const (
 	envConcurrentSyncs = "NB_MAX_CONCURRENT_SYNCS"
 
 	defaultSyncLim = 1000
+
+	peerHeartbeatTimeout = 5 * time.Second
 )
 
 // Server an instance of a Management gRPC API server
@@ -368,7 +370,7 @@ func (s *Server) Sync(req *proto.EncryptedMessage, srv proto.ManagementService_S
 	s.syncSem.Add(-1)
 
 	return PeerUpdateHandlerFactory(peerKey, updates, s.secretsManager, s.challenger,
-		trackChallenge,
+		trackChallenge, func() { s.heartbeatPeerSession(ctx, accountID, peer.ID, syncStart) },
 		srv, func() { s.cancelPeerRoutines(ctx, accountID, peer, syncStart, updates) }).
 		WithMetrics(s.appMetrics).HandleUpdates(ctx)
 }
@@ -462,6 +464,17 @@ func (s *Server) sendJob(ctx context.Context, peerKey wgtypes.Key, job *job.Even
 	return nil
 }
 
+// heartbeatPeerSession records that the peer of a live sync stream is still connected.
+// A failed heartbeat keeps the stream open: the peer only reads as disconnected after
+// nbpeer.StaleAfter without one, and reads as connected again once a heartbeat lands.
+func (s *Server) heartbeatPeerSession(ctx context.Context, accountID, peerID string, streamStartTime time.Time) {
+	ctx, cancel := context.WithTimeout(ctx, peerHeartbeatTimeout)
+	defer cancel()
+	if _, err := s.accountManager.GetStore().HeartbeatPeerSession(ctx, accountID, peerID, streamStartTime.UnixNano()); err != nil && !errors.Is(ctx.Err(), context.Canceled) {
+		log.WithContext(ctx).Warnf("heartbeat of peer %s: %v", peerID, err)
+	}
+}
+
 // cancelPeerRoutines tears down the stream under the peer lock. While draining, the
 // disconnect is queued before waiting for the lock, so the shutdown flush covers the
 // stream even when the lock is held past the gRPC stop timeout.