[client] Keep HA routing peer selection latency-aware and hold switches that reset connections (#7689)

client/internal/peer/status.go

@@ -28,6 +28,11 @@ import (
 
 const eventQueueSize = 10
 
+// routerLatencyNotifyThreshold is the latency or noise change that notifies the route
+// watchers subscribed to a peer. Latency is resampled continuously, so without
+// a threshold every sample would wake every watcher of every routing peer.
+const routerLatencyNotifyThreshold = 5 * time.Millisecond
+
 type ResolvedDomainInfo struct {
 	Prefixes     []netip.Prefix
 	ParentDomain domain.Domain
@@ -41,11 +46,21 @@ type EventListener interface {
 	OnEvent(event *proto.SystemEvent)
 }
 
+// LatencySample is a peer's smoothed round trip time together with the
+// uncertainty of that estimate, which is what the jitter of the path leaves
+// behind after smoothing. Comparing two peers is only meaningful when their
+// difference is larger than this.
+type LatencySample struct {
+	Latency time.Duration
+	Noise   time.Duration
+}
+
 // RouterState status for router peers. This contains relevant fields for route manager
 type RouterState struct {
-	Status  ConnStatus
-	Relayed bool
-	Latency time.Duration
+	Status       ConnStatus
+	Relayed      bool
+	Latency      time.Duration
+	LatencyNoise time.Duration
 }
 
 // State contains the latest state of a peer
@@ -67,9 +82,11 @@ type State struct {
 	BytesTx                    int64
 	BytesRx                    int64
 	Latency                    time.Duration
-	RosenpassEnabled           bool
-	SSHHostKey                 []byte
-	routes                     map[string]struct{}
+	// LatencyNoise is the uncertainty of Latency on a jittery path
+	LatencyNoise     time.Duration
+	RosenpassEnabled bool
+	SSHHostKey       []byte
+	routes           map[string]struct{}
 }
 
 // AddRoute add a single route to routes map
@@ -103,6 +120,13 @@ func (s *State) GetRoutes() map[string]struct{} {
 	return maps.Clone(s.routes)
 }
 
+// clearLatency drops the latency estimate, which is only measured over a direct
+// connection and says nothing about the path once that connection is gone.
+func (s *State) clearLatency() {
+	s.Latency = 0
+	s.LatencyNoise = 0
+}
+
 // LocalPeerState contains the latest state of the local peer
 type LocalPeerState struct {
 	IP              string
@@ -163,11 +187,25 @@ type FullStatus struct {
 	Events                []*proto.SystemEvent
 }
 
+// routerSnapshot is a router-state map stamped with a sequence number taken
+// under the status lock. Snapshots are built under the lock but delivered
+// outside it, so two dispatchers can race on the way to a subscriber; the
+// stamp lets delivery drop the older snapshot instead of letting it overwrite
+// a newer one.
+type routerSnapshot struct {
+	seq    uint64
+	states map[string]RouterState
+}
+
 type StatusChangeSubscription struct {
 	peerID     string
 	id         string
 	eventsChan chan map[string]RouterState
 	ctx        context.Context
+
+	// sendMu serializes deliveries and guards lastSeq
+	sendMu  sync.Mutex
+	lastSeq uint64
 }
 
 func newStatusChangeSubscription(ctx context.Context, peerID string) *StatusChangeSubscription {
@@ -184,17 +222,54 @@ func (s *StatusChangeSubscription) Events() chan map[string]RouterState {
 	return s.eventsChan
 }
 
+// deliver forwards a snapshot to the subscriber, dropping it when a newer one
+// has already been delivered. A blocking deliver waits for buffer space, since
+// peer state transitions must not be lost. A non-blocking one gives up when the
+// subscriber is backlogged or another delivery is in flight; it is meant for
+// latency refreshes, which are superseded by the next sample anyway and are
+// also re-read from the recorder on the route watchers' periodic re-evaluation.
+// Only a snapshot that reached the subscriber supersedes older ones.
+func (s *StatusChangeSubscription) deliver(snapshot *routerSnapshot, block bool) {
+	if block {
+		s.sendMu.Lock()
+	} else if !s.sendMu.TryLock() {
+		return
+	}
+	defer s.sendMu.Unlock()
+
+	if snapshot.seq <= s.lastSeq {
+		return
+	}
+
+	if block {
+		select {
+		case s.eventsChan <- snapshot.states:
+			s.lastSeq = snapshot.seq
+		case <-s.ctx.Done():
+		}
+		return
+	}
+
+	select {
+	case s.eventsChan <- snapshot.states:
+		s.lastSeq = snapshot.seq
+	default:
+	}
+}
+
 // Status holds a state of peers, signal, management connections and relays.
 // mux is an RWMutex so hot read paths (notably PeerStateByIP, called for
 // every private-service request) don't contend against each other.
 // Pure read methods take RLock; anything that mutates state takes Lock.
 type Status struct {
-	mux                 sync.RWMutex
-	muxRelays           sync.RWMutex
-	peers               map[string]State
-	ipToKey             map[string]string
-	activeRoutePeers    map[route.HAUniqueID]string
-	changeNotify        map[string]map[string]*StatusChangeSubscription // map[peerID]map[subscriptionID]*StatusChangeSubscription
+	mux              sync.RWMutex
+	muxRelays        sync.RWMutex
+	peers            map[string]State
+	ipToKey          map[string]string
+	activeRoutePeers map[route.HAUniqueID]string
+	changeNotify     map[string]map[string]*StatusChangeSubscription // map[peerID]map[subscriptionID]*StatusChangeSubscription
+	// routerSeq stamps router-state snapshots, see routerSnapshot
+	routerSeq           uint64
 	signalState         bool
 	signalError         error
 	managementState     bool
@@ -386,6 +461,7 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 		peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
 		peerState.RelayServerAddress = receivedState.RelayServerAddress
 		peerState.RosenpassEnabled = receivedState.RosenpassEnabled
+		peerState.clearLatency()
 	}
 
 	d.peers[receivedState.PubKey] = peerState
@@ -402,7 +478,7 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 		d.notifier.peerListChanged(numPeers)
 	}
 	if notifyRouter {
-		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
+		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot, true)
 	}
 	d.notifyStateChange()
 	return nil
@@ -522,7 +598,7 @@ func (d *Status) UpdatePeerICEState(receivedState State) error {
 		d.notifier.peerListChanged(numPeers)
 	}
 	if notifyRouter {
-		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
+		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot, true)
 	}
 	d.notifyStateChange()
 	return nil
@@ -559,7 +635,7 @@ func (d *Status) UpdatePeerRelayedState(receivedState State) error {
 		d.notifier.peerListChanged(numPeers)
 	}
 	if notifyRouter {
-		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
+		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot, true)
 	}
 	d.notifyStateChange()
 	return nil
@@ -595,7 +671,7 @@ func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error
 		d.notifier.peerListChanged(numPeers)
 	}
 	if notifyRouter {
-		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
+		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot, true)
 	}
 	d.notifyStateChange()
 	return nil
@@ -620,6 +696,7 @@ func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error {
 	peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
 	peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
 	peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
+	peerState.clearLatency()
 
 	d.peers[receivedState.PubKey] = peerState
 
@@ -634,7 +711,7 @@ func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error {
 		d.notifier.peerListChanged(numPeers)
 	}
 	if notifyRouter {
-		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
+		d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot, true)
 	}
 	d.notifyStateChange()
 	return nil
@@ -714,7 +791,7 @@ func (d *Status) FinishPeerListModifications() {
 	// snapshot per-peer router state to deliver after the lock is released
 	type routerDispatch struct {
 		peerID   string
-		snapshot map[string]RouterState
+		snapshot *routerSnapshot
 	}
 	dispatches := make([]routerDispatch, 0, len(d.peers))
 	for key := range d.peers {
@@ -728,7 +805,7 @@ func (d *Status) FinishPeerListModifications() {
 
 	d.notifier.peerListChanged(numPeers)
 	for _, rd := range dispatches {
-		d.dispatchRouterPeers(rd.peerID, rd.snapshot)
+		d.dispatchRouterPeers(rd.peerID, rd.snapshot, true)
 	}
 	d.notifyStateChange()
 }
@@ -1057,22 +1134,42 @@ func (d *Status) GetManagementState() ManagementState {
 	}
 }
 
-func (d *Status) UpdateLatency(pubKey string, latency time.Duration) error {
-	if latency <= 0 {
+func (d *Status) UpdateLatency(pubKey string, sample LatencySample) error {
+	if sample.Latency <= 0 {
 		return nil
 	}
 
 	d.mux.Lock()
-	defer d.mux.Unlock()
 	peerState, ok := d.peers[pubKey]
 	if !ok {
+		d.mux.Unlock()
 		return errors.New("peer doesn't exist")
 	}
-	peerState.Latency = latency
+
+	previous, previousNoise := peerState.Latency, peerState.LatencyNoise
+	peerState.Latency = sample.Latency
+	peerState.LatencyNoise = sample.Noise
 	d.peers[pubKey] = peerState
+
+	// the noise sets the switch margin, so it counts on its own
+	notifyRouter := absDuration(sample.Latency-previous) >= routerLatencyNotifyThreshold ||
+		absDuration(sample.Noise-previousNoise) >= routerLatencyNotifyThreshold
+	routerSnapshot := d.snapshotRouterPeersLocked(pubKey, notifyRouter)
+
+	d.mux.Unlock()
+
+	d.dispatchRouterPeers(pubKey, routerSnapshot, false)
+
 	return nil
 }
 
+func absDuration(d time.Duration) time.Duration {
+	if d < 0 {
+		return -d
+	}
+	return d
+}
+
 // IsLoginRequired determines if a peer's login has expired.
 func (d *Status) IsLoginRequired() bool {
 	d.mux.RLock()
@@ -1229,7 +1326,7 @@ func (d *Status) RemoveConnectionListener() {
 // Caller MUST hold d.mux. Returns nil when there are no subscribers for peerID
 // or when notify is false. The snapshot is consumed later by dispatchRouterPeers
 // outside the lock so the channel send cannot stall any d.mux holder.
-func (d *Status) snapshotRouterPeersLocked(peerID string, notify bool) map[string]RouterState {
+func (d *Status) snapshotRouterPeersLocked(peerID string, notify bool) *routerSnapshot {
 	if !notify {
 		return nil
 	}
@@ -1244,22 +1341,29 @@ func (d *Status) snapshotRouterPeersLocked(peerID string, notify bool) map[strin
 			continue
 		}
 		routerPeers[pid] = RouterState{
-			Status:  s.ConnStatus,
-			Relayed: s.Relayed,
-			Latency: s.Latency,
+			Status:       s.ConnStatus,
+			Relayed:      s.Relayed,
+			Latency:      s.Latency,
+			LatencyNoise: s.LatencyNoise,
 		}
 	}
-	return routerPeers
+
+	d.routerSeq++
+
+	return &routerSnapshot{seq: d.routerSeq, states: routerPeers}
 }
 
 // dispatchRouterPeers delivers a previously snapshotted router-state map to
 // the peer's subscribers. Caller MUST NOT hold d.mux. The method takes a
 // fresh, short read of d.changeNotify under the lock to grab subscriber
 // channels, then sends outside the lock so a slow consumer cannot block other
-// d.mux holders. The send itself stays blocking (only short-circuited by the
-// subscriber's context) so peer state transitions are not silently dropped.
-func (d *Status) dispatchRouterPeers(peerID string, routerPeers map[string]RouterState) {
-	if routerPeers == nil {
+// d.mux holders. With block set, the send waits for buffer space (only
+// short-circuited by the subscriber's context) so peer state transitions are
+// not silently dropped; without it, a backlogged subscriber loses the snapshot
+// instead of stalling the caller. Out-of-order deliveries are dropped either
+// way, see StatusChangeSubscription.deliver.
+func (d *Status) dispatchRouterPeers(peerID string, snapshot *routerSnapshot, block bool) {
+	if snapshot == nil {
 		return
 	}
 
@@ -1274,10 +1378,7 @@ func (d *Status) dispatchRouterPeers(peerID string, routerPeers map[string]Route
 	d.mux.Unlock()
 
 	for _, sub := range subs {
-		select {
-		case sub.eventsChan <- routerPeers:
-		case <-sub.ctx.Done():
-		}
+		sub.deliver(snapshot, block)
 	}
 }
 

client/internal/peer/status_test.go

@@ -44,6 +44,184 @@ func TestGetPeer(t *testing.T) {
 	assert.Error(t, err, "should return error when peer doesn't exist")
 }
 
+func TestUpdateLatencyNotifiesRouteWatchers(t *testing.T) {
+	key := "abc"
+	status := NewRecorder("https://mgm")
+	require.NoError(t, status.AddPeer(key, "abc.netbird", "100.108.254.1", ""))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	sub := status.SubscribeToPeerStateChanges(ctx, key)
+
+	require.NoError(t, status.UpdateLatency(key, LatencySample{Latency: 40 * time.Millisecond}))
+	select {
+	case states := <-sub.Events():
+		assert.Equal(t, 40*time.Millisecond, states[key].Latency, "the notified state should carry the new latency")
+	default:
+		t.Fatal("expected a router state notification for the first latency sample")
+	}
+
+	// below routerLatencyNotifyThreshold: recorded, but not worth waking the watchers
+	require.NoError(t, status.UpdateLatency(key, LatencySample{Latency: 42 * time.Millisecond}))
+	select {
+	case <-sub.Events():
+		t.Fatal("a latency change below the threshold should not notify")
+	default:
+	}
+
+	peerState, err := status.GetPeer(key)
+	require.NoError(t, err)
+	assert.Equal(t, 42*time.Millisecond, peerState.Latency, "the latency should be recorded even without a notification")
+
+	require.NoError(t, status.UpdateLatency(key, LatencySample{Latency: 100 * time.Millisecond}))
+	select {
+	case states := <-sub.Events():
+		assert.Equal(t, 100*time.Millisecond, states[key].Latency, "a significant latency change should notify")
+	default:
+		t.Fatal("expected a router state notification for a significant latency change")
+	}
+
+	// the noise feeds the switch margin, so a large change in it matters on its
+	// own even when the latency stays put
+	require.NoError(t, status.UpdateLatency(key, LatencySample{Latency: 100 * time.Millisecond, Noise: 20 * time.Millisecond}))
+	select {
+	case states := <-sub.Events():
+		assert.Equal(t, 20*time.Millisecond, states[key].LatencyNoise, "a significant noise change should notify")
+	default:
+		t.Fatal("expected a router state notification for a significant noise change")
+	}
+
+	require.NoError(t, status.UpdateLatency(key, LatencySample{Latency: 100 * time.Millisecond, Noise: 21 * time.Millisecond}))
+	select {
+	case <-sub.Events():
+		t.Fatal("a noise change below the threshold should not notify")
+	default:
+	}
+}
+
+// TestSubscriptionDeliverOrdering covers the sequence stamping: snapshots are
+// built under the status lock but delivered outside it, so a dispatcher that
+// is descheduled between the two can deliver an outdated snapshot after a
+// newer one. The older snapshot must be dropped, not delivered.
+func TestSubscriptionDeliverOrdering(t *testing.T) {
+	sub := newStatusChangeSubscription(context.Background(), "peer")
+
+	newer := &routerSnapshot{seq: 2, states: map[string]RouterState{"peer": {Status: StatusConnecting}}}
+	older := &routerSnapshot{seq: 1, states: map[string]RouterState{"peer": {Status: StatusConnected}}}
+
+	sub.deliver(newer, true)
+	sub.deliver(older, true)
+
+	select {
+	case states := <-sub.Events():
+		assert.Equal(t, StatusConnecting, states["peer"].Status, "the newer snapshot should be delivered")
+	default:
+		t.Fatal("expected the newer snapshot to be delivered")
+	}
+
+	select {
+	case <-sub.Events():
+		t.Fatal("the outdated snapshot must be dropped, not delivered after the newer one")
+	default:
+	}
+}
+
+// TestSubscriptionDeliverNonBlocking covers the latency dispatch path: a
+// backlogged subscriber must not block the sender, and the dropped snapshot is
+// recovered from the recorder by the route watchers' periodic re-evaluation.
+func TestSubscriptionDeliverNonBlocking(t *testing.T) {
+	sub := newStatusChangeSubscription(context.Background(), "peer")
+
+	// fill the subscription buffer with no consumer draining it
+	seq := uint64(1)
+	for {
+		before := len(sub.eventsChan)
+		sub.deliver(&routerSnapshot{seq: seq, states: map[string]RouterState{}}, false)
+		seq++
+		if len(sub.eventsChan) == before {
+			break
+		}
+	}
+
+	done := make(chan struct{})
+	go func() {
+		sub.deliver(&routerSnapshot{seq: seq, states: map[string]RouterState{}}, false)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("a non-blocking deliver must return with the buffer full")
+	}
+}
+
+// A snapshot the subscriber never received must not count as delivered: a
+// state transition dispatched just before it, but delivered just after, would
+// otherwise be dropped as outdated and never reach the subscriber.
+func TestSubscriptionDroppedDeliveryDoesNotSupersede(t *testing.T) {
+	sub := newStatusChangeSubscription(context.Background(), "peer")
+
+	for seq := uint64(1); seq <= uint64(cap(sub.eventsChan)); seq++ {
+		sub.deliver(&routerSnapshot{seq: seq, states: map[string]RouterState{}}, false)
+	}
+	require.Len(t, sub.eventsChan, cap(sub.eventsChan), "the buffer must be full")
+
+	// a latency refresh stamped after the transition wins the race to the
+	// subscriber and is dropped on the full buffer
+	transition := &routerSnapshot{seq: 100, states: map[string]RouterState{"peer": {Status: StatusConnecting}}}
+	latency := &routerSnapshot{seq: 101, states: map[string]RouterState{"peer": {Status: StatusConnected}}}
+	sub.deliver(latency, false)
+
+	for range cap(sub.eventsChan) {
+		<-sub.eventsChan
+	}
+	sub.deliver(transition, true)
+
+	select {
+	case states := <-sub.Events():
+		assert.Equal(t, StatusConnecting, states["peer"].Status, "the transition must be delivered")
+	default:
+		t.Fatal("a transition must not be dropped because a later snapshot was dropped")
+	}
+}
+
+// Latency is only measured over a direct connection. Once it is gone the last
+// sample describes a path the traffic no longer takes, and must not be compared
+// against other routing peers as if it were current.
+func TestLatencyClearedWithDirectConnection(t *testing.T) {
+	const key = "abc"
+
+	newStatus := func(t *testing.T) *Status {
+		t.Helper()
+		status := NewRecorder("https://mgm")
+		require.NoError(t, status.AddPeer(key, "peer-a.netbird.local", "10.10.10.10", ""))
+		require.NoError(t, status.UpdatePeerICEState(State{PubKey: key, ConnStatus: StatusConnected}))
+		require.NoError(t, status.UpdateLatency(key, LatencySample{Latency: 20 * time.Millisecond, Noise: time.Millisecond}))
+		return status
+	}
+
+	assertCleared := func(t *testing.T, status *Status) {
+		t.Helper()
+		state, err := status.GetPeer(key)
+		require.NoError(t, err)
+		assert.Zero(t, state.Latency, "the latency of the lost direct path must be cleared")
+		assert.Zero(t, state.LatencyNoise, "the noise of the lost direct path must be cleared")
+	}
+
+	t.Run("ice disconnected, relayed", func(t *testing.T) {
+		status := newStatus(t)
+		require.NoError(t, status.UpdatePeerICEStateToDisconnected(State{PubKey: key, ConnStatus: StatusConnected, Relayed: true}))
+		assertCleared(t, status)
+	})
+
+	t.Run("connection closed", func(t *testing.T) {
+		status := newStatus(t)
+		require.NoError(t, status.UpdatePeerState(State{PubKey: key, ConnStatus: StatusIdle}))
+		assertCleared(t, status)
+	})
+}
+
 func TestUpdatePeerState(t *testing.T) {
 	key := "abc"
 	ip := "10.10.10.10"

client/internal/peer/worker_ice.go

@@ -3,6 +3,7 @@ package peer
 import (
 	"context"
 	"fmt"
+	"math"
 	"net"
 	"strconv"
 	"sync"
@@ -20,6 +21,26 @@ import (
 	"github.com/netbirdio/netbird/route"
 )
 
+const (
+	// latencySampleInterval is how often the round trip time of the selected
+	// candidate pair is read. The ICE agent refreshes it on every consent
+	// binding request, so sampling is cheap and needs no extra traffic.
+	latencySampleInterval = 4 * time.Second
+
+	// latencyEWMAAlpha is the weight of a new sample in the exponentially
+	// weighted moving average. At latencySampleInterval this yields a time
+	// constant of roughly 27 seconds: a sustained change is reflected within
+	// tens of seconds, while the residual movement on a jittery path stays
+	// below the switch margin that route selection applies, so noise alone
+	// cannot move a route. Raising it widens that residual movement.
+	latencyEWMAAlpha = 0.15
+)
+
+// latencySmoothedNoiseFactor converts the spread of the raw samples into the
+// spread of the smoothed value: for an exponentially weighted moving average
+// with weight a, the variance of the output is a/(2-a) of the input variance.
+var latencySmoothedNoiseFactor = math.Sqrt(latencyEWMAAlpha / (2 - latencyEWMAAlpha))
+
 type ICEConnInfo struct {
 	RemoteConn                 net.Conn
 	RosenpassPubKey            []byte
@@ -137,7 +158,7 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 		w.log.Debugf("recreate ICE agent: %s / %s", w.sessionID, *remoteOfferAnswer.SessionID)
 	}
 	dialerCtx, dialerCancel := context.WithCancel(w.ctx)
-	agent, err := w.reCreateAgent(dialerCancel, preferredCandidateTypes)
+	agent, sampleNow, err := w.reCreateAgent(dialerCancel, preferredCandidateTypes)
 	if err != nil {
 		w.log.Errorf("failed to recreate ICE Agent: %s", err)
 		return
@@ -153,7 +174,7 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 
 	// Capture the cancel func at spawn time: connect reads it from the argument
 	// instead of the field, which a newer OnNewOffer may already have replaced.
-	go w.connect(dialerCtx, dialerCancel, agent, remoteOfferAnswer)
+	go w.connect(dialerCtx, dialerCancel, agent, sampleNow, remoteOfferAnswer)
 }
 
 // OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer.
@@ -217,29 +238,33 @@ func (w *WorkerICE) Close() {
 	w.abandonNegotiation()
 }
 
-func (w *WorkerICE) reCreateAgent(dialerCancel context.CancelFunc, candidates []ice.CandidateType) (*icemaker.ThreadSafeAgent, error) {
+// reCreateAgent creates a new ICE agent with its callbacks registered. The
+// returned channel wakes that agent's latency sampler when its selected
+// candidate pair changes.
+func (w *WorkerICE) reCreateAgent(dialerCancel context.CancelFunc, candidates []ice.CandidateType) (*icemaker.ThreadSafeAgent, chan struct{}, error) {
 	w.portForwardAttempted = false
 
 	agent, err := icemaker.NewAgent(w.ctx, w.iFaceDiscover, w.config.ICEConfig, candidates, w.localUfrag, w.localPwd)
 	if err != nil {
-		return nil, fmt.Errorf("create agent: %w", err)
+		return nil, nil, fmt.Errorf("create agent: %w", err)
 	}
 
 	if err := agent.OnCandidate(w.onICECandidate); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if err := agent.OnConnectionStateChange(w.onConnectionStateChange(agent, dialerCancel)); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	if err := agent.OnSelectedCandidatePairChange(func(c1, c2 ice.Candidate) {
-		w.onICESelectedCandidatePair(agent, c1, c2)
-	}); err != nil {
-		return nil, err
+	// one channel per agent, so the sampler of a replaced agent cannot take a
+	// wakeup meant for the new one
+	sampleNow := make(chan struct{}, 1)
+	if err := agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair(sampleNow)); err != nil {
+		return nil, nil, err
 	}
 
-	return agent, nil
+	return agent, sampleNow, nil
 }
 
 func (w *WorkerICE) SessionID() ICESessionID {
@@ -252,7 +277,7 @@ func (w *WorkerICE) SessionID() ICESessionID {
 // will block until connection succeeded
 // but it won't release if ICE Agent went into Disconnected or Failed state,
 // so we have to cancel it with the provided context once agent detected a broken connection
-func (w *WorkerICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) {
+func (w *WorkerICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agent *icemaker.ThreadSafeAgent, sampleNow <-chan struct{}, remoteOfferAnswer *OfferAnswer) {
 	w.log.Debugf("gather candidates")
 	if err := agent.GatherCandidates(); err != nil {
 		w.log.Warnf("failed to gather candidates: %s", err)
@@ -343,6 +368,9 @@ func (w *WorkerICE) connect(ctx context.Context, dialerCancel context.CancelFunc
 	w.lastSuccess = time.Now()
 	w.muxAgent.Unlock()
 
+	// ctx is the agent's dialer context, so sampling stops when the agent is closed
+	go w.sampleLatency(ctx, agent, sampleNow)
+
 	// todo: the potential problem is a race between the onConnectionStateChange
 	// and the delivery below: after this unlock, a newer offer can replace
 	// w.agent before onICEConnectionIsReady runs, delivering this (now stale)
@@ -532,21 +560,63 @@ func (w *WorkerICE) createForwardedCandidate(srflxCandidate ice.Candidate, mappi
 	return candidate, nil
 }
 
-func (w *WorkerICE) onICESelectedCandidatePair(agent *icemaker.ThreadSafeAgent, c1, c2 ice.Candidate) {
-	w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(),
-		w.config.Key)
+func (w *WorkerICE) onICESelectedCandidatePair(sampleNow chan<- struct{}) func(c1, c2 ice.Candidate) {
+	return func(c1, c2 ice.Candidate) {
+		w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(),
+			w.config.Key)
 
+		// refresh the recorded latency right away: it still describes the previous
+		// path, and route selection may run before the next sampling tick
+		select {
+		case sampleNow <- struct{}{}:
+		default:
+		}
+	}
+}
+
+// sampleLatency records the smoothed round trip time of the agent's selected
+// candidate pair until the agent is closed. Route selection compares the
+// recorded latencies of high availability peers, so it has to keep following
+// the path instead of freezing at the value observed when the pair was chosen.
+func (w *WorkerICE) sampleLatency(ctx context.Context, agent *icemaker.ThreadSafeAgent, sampleNow <-chan struct{}) {
+	var sampler latencySampler
+
+	ticker := time.NewTicker(latencySampleInterval)
+	defer ticker.Stop()
+
+	for {
+		w.updateLatency(ctx, agent, &sampler)
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		case <-sampleNow:
+		}
+	}
+}
+
+func (w *WorkerICE) updateLatency(ctx context.Context, agent *icemaker.ThreadSafeAgent, sampler *latencySampler) {
 	pairStat, ok := agent.GetSelectedCandidatePairStats()
 	if !ok {
-		w.log.Warnf("failed to get selected candidate pair stats")
+		w.log.Debugf("failed to get selected candidate pair stats")
 		return
 	}
 
-	duration := time.Duration(pairStat.CurrentRoundTripTime * float64(time.Second))
-	if err := w.statusRecorder.UpdateLatency(w.config.Key, duration); err != nil {
-		w.log.Debugf("failed to update latency for peer: %s", err)
+	sample := sampler.observe(pairStat)
+	if sample.Latency <= 0 {
 		return
 	}
+
+	// the agent may have been closed while sampling, and its connection's
+	// latency cleared: a late sample must not bring the stale value back
+	if ctx.Err() != nil {
+		return
+	}
+
+	if err := w.statusRecorder.UpdateLatency(w.config.Key, sample); err != nil {
+		w.log.Debugf("failed to update latency for peer: %s", err)
+	}
 }
 
 func (w *WorkerICE) logSuccessfulPaths(agent *icemaker.ThreadSafeAgent) {
@@ -612,6 +682,59 @@ func (w *WorkerICE) agentDial(ctx context.Context, agent *icemaker.ThreadSafeAge
 	}
 }
 
+// latencySampler smooths the round trip time samples of a single ICE agent and
+// tracks how much they scatter, so consumers can tell a real difference between
+// two paths from the noise of a jittery one.
+type latencySampler struct {
+	smoothed  time.Duration
+	deviation time.Duration
+	localID   string
+	remoteID  string
+}
+
+// observe folds a candidate pair stat into the moving average and returns the
+// smoothed latency with the uncertainty of that estimate. A different candidate
+// pair starts over: the path changed, so the previous samples describe a route
+// that no longer carries the traffic.
+func (s *latencySampler) observe(stat ice.CandidatePairStats) LatencySample {
+	rtt := time.Duration(stat.CurrentRoundTripTime * float64(time.Second))
+	if rtt <= 0 {
+		return LatencySample{}
+	}
+
+	if s.smoothed <= 0 || s.localID != stat.LocalCandidateID || s.remoteID != stat.RemoteCandidateID {
+		s.localID = stat.LocalCandidateID
+		s.remoteID = stat.RemoteCandidateID
+		s.smoothed = rtt
+		// A fresh path is described by a single raw sample, which is exactly
+		// when the estimate deserves the least trust: a lucky outlier reported
+		// with no uncertainty could move a route on its own. Seed the deviation
+		// at half the observed value so the reported noise stays conservative
+		// until real samples accumulate, and keep a larger deviation carried
+		// over from the previous path, since the link's jitter usually outlives
+		// a candidate pair change.
+		if seed := rtt / 2; s.deviation < seed {
+			s.deviation = seed
+		}
+		return s.sample()
+	}
+
+	// mean absolute deviation of the samples around the average, the same
+	// estimator TCP keeps next to its smoothed round trip time
+	delta := absDuration(rtt - s.smoothed)
+	s.deviation = time.Duration(latencyEWMAAlpha*float64(delta) + (1-latencyEWMAAlpha)*float64(s.deviation))
+	s.smoothed = time.Duration(latencyEWMAAlpha*float64(rtt) + (1-latencyEWMAAlpha)*float64(s.smoothed))
+
+	return s.sample()
+}
+
+func (s *latencySampler) sample() LatencySample {
+	return LatencySample{
+		Latency: s.smoothed,
+		Noise:   time.Duration(latencySmoothedNoiseFactor * float64(s.deviation)),
+	}
+}
+
 func shouldAddExtraCandidate(candidate ice.Candidate) bool {
 	if candidate.Type() != ice.CandidateTypeServerReflexive {
 		return false

client/internal/peer/worker_ice_test.go

@@ -0,0 +1,168 @@
+package peer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/pion/ice/v4"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func pairStat(local, remote string, rtt time.Duration) ice.CandidatePairStats {
+	return ice.CandidatePairStats{
+		LocalCandidateID:     local,
+		RemoteCandidateID:    remote,
+		CurrentRoundTripTime: rtt.Seconds(),
+	}
+}
+
+func TestLatencySamplerObserve(t *testing.T) {
+	t.Run("first sample is taken as is", func(t *testing.T) {
+		var s latencySampler
+
+		got := s.observe(pairStat("local", "remote", 40*time.Millisecond))
+		assert.Equal(t, 40*time.Millisecond, got.Latency, "the first sample has nothing to average with")
+	})
+
+	t.Run("zero round trip time is not recorded", func(t *testing.T) {
+		var s latencySampler
+
+		require.Zero(t, s.observe(pairStat("local", "remote", 0)).Latency, "a pair without an RTT sample yields no latency")
+
+		// the first real sample still seeds the average
+		assert.Equal(t, 20*time.Millisecond, s.observe(pairStat("local", "remote", 20*time.Millisecond)).Latency)
+	})
+
+	t.Run("single spike is damped", func(t *testing.T) {
+		var s latencySampler
+
+		s.observe(pairStat("local", "remote", 20*time.Millisecond))
+		spiked := s.observe(pairStat("local", "remote", 220*time.Millisecond)).Latency
+
+		// a single sample only contributes its weight of the 200ms delta
+		assert.Equal(t, 50*time.Millisecond, spiked, "a single spike must not be reported in full")
+
+		// and it decays again once the path is back to normal
+		for range 10 {
+			s.observe(pairStat("local", "remote", 20*time.Millisecond))
+		}
+		assert.Less(t, s.observe(pairStat("local", "remote", 20*time.Millisecond)).Latency, 30*time.Millisecond,
+			"the average should return towards the steady state")
+	})
+
+	t.Run("sustained change is followed", func(t *testing.T) {
+		var s latencySampler
+
+		s.observe(pairStat("local", "remote", 20*time.Millisecond))
+
+		// the route selection switch margin, see routemanager/client
+		const switchMargin = 20 * time.Millisecond
+
+		// at a 4s sample interval, 3 samples are about 12 seconds
+		var latency time.Duration
+		for range 3 {
+			latency = s.observe(pairStat("local", "remote", 80*time.Millisecond)).Latency
+		}
+		assert.Greater(t, latency-20*time.Millisecond, switchMargin,
+			"a sustained 60ms degradation must clear the switch margin within seconds")
+
+		// and 10 more samples, about 40 seconds in total, get most of the way there
+		for range 10 {
+			latency = s.observe(pairStat("local", "remote", 80*time.Millisecond)).Latency
+		}
+		assert.Greater(t, latency, 70*time.Millisecond, "a sustained degradation must be tracked within tens of seconds")
+	})
+
+	// Smoothing narrows the scatter of a jittery path but leaves a residue, and
+	// that residue is what route selection has to compare its margins against.
+	// The reported noise has to describe it, so it has to grow with the jitter
+	// and cover the movement of the smoothed value.
+	t.Run("reported noise reflects the jitter of the path", func(t *testing.T) {
+		// 100ms +/- 40ms, the same shape the netem jitter test uses
+		jittery := []int{
+			104, 138, 61, 96, 141, 72, 118, 65, 133, 88,
+			59, 127, 92, 136, 68, 111, 143, 74, 99, 121,
+		}
+		steady := []int{
+			100, 101, 99, 100, 102, 99, 100, 101, 100, 99, 101, 100, 99, 100, 101, 100, 99, 101, 100, 100,
+			101, 99, 100, 102, 99, 100, 101, 100, 99, 101, 100, 99, 100, 101, 100, 99, 101, 100, 100, 101,
+		}
+
+		spreadAndNoise := func(raw []int) (spread, noise time.Duration) {
+			var s latencySampler
+			var minSmoothed, maxSmoothed time.Duration
+
+			for i, ms := range raw {
+				got := s.observe(pairStat("local", "remote", time.Duration(ms)*time.Millisecond))
+				noise = got.Noise
+
+				// skip the warmup, the first sample is taken as is
+				if i < 5 {
+					continue
+				}
+				if minSmoothed == 0 || got.Latency < minSmoothed {
+					minSmoothed = got.Latency
+				}
+				if got.Latency > maxSmoothed {
+					maxSmoothed = got.Latency
+				}
+			}
+
+			return maxSmoothed - minSmoothed, noise
+		}
+
+		jitterySpread, jitteryNoise := spreadAndNoise(jittery)
+		steadySpread, steadyNoise := spreadAndNoise(steady)
+
+		assert.Greater(t, jitteryNoise, steadyNoise, "a jittery path must report more noise than a steady one")
+		assert.Less(t, steadyNoise, time.Millisecond, "a steady path must report almost no noise, so the fixed margins decide")
+
+		// the noise is a deviation, so a few of them have to cover the movement
+		// of the smoothed value, which is what the switch margin relies on
+		assert.Greater(t, 4*jitteryNoise, jitterySpread,
+			"reported noise %v must account for the smoothed spread %v", jitteryNoise, jitterySpread)
+		assert.Greater(t, 4*steadyNoise+time.Millisecond, steadySpread,
+			"reported noise %v must account for the smoothed spread %v", steadyNoise, steadySpread)
+	})
+
+	t.Run("new candidate pair discards the previous average", func(t *testing.T) {
+		var s latencySampler
+
+		s.observe(pairStat("local", "remote", 200*time.Millisecond))
+		got := s.observe(pairStat("local2", "remote2", 20*time.Millisecond)).Latency
+
+		assert.Equal(t, 20*time.Millisecond, got, "samples of the previous path must not be carried over")
+	})
+
+	// A single raw sample is when the estimate deserves the least trust: with
+	// zero reported noise, a lucky outlier could clear the switch margins and
+	// move a route on its own.
+	t.Run("a fresh path reports conservative noise", func(t *testing.T) {
+		var s latencySampler
+
+		first := s.observe(pairStat("local", "remote", 100*time.Millisecond))
+		assert.Greater(t, first.Noise, 10*time.Millisecond, "a single-sample estimate must not be reported as noise-free")
+
+		// steady samples shrink the uncertainty again
+		var got LatencySample
+		for range 40 {
+			got = s.observe(pairStat("local", "remote", 100*time.Millisecond))
+		}
+		assert.Less(t, got.Noise, 2*time.Millisecond, "steady samples should earn the estimate its trust back")
+	})
+
+	t.Run("a pair change keeps the larger deviation of a jittery link", func(t *testing.T) {
+		var s latencySampler
+
+		// build up a deviation well above the reseed floor of the new path
+		for _, ms := range []int{100, 180, 60, 170, 50, 190, 70, 160} {
+			s.observe(pairStat("local", "remote", time.Duration(ms)*time.Millisecond))
+		}
+		before := s.deviation
+		require.Greater(t, before, 25*time.Millisecond, "the jittery path should have accumulated deviation")
+
+		s.observe(pairStat("local2", "remote2", 50*time.Millisecond))
+		assert.GreaterOrEqual(t, s.deviation, before, "a pair change must not shrink the known jitter of the link")
+	})
+}