[client] Keep HA routing peer selection latency-aware and hold switches that reset connections (#7689)

client/internal/peer/worker_ice.go

@@ -3,6 +3,7 @@ package peer
 import (
 	"context"
 	"fmt"
+	"math"
 	"net"
 	"strconv"
 	"sync"
@@ -20,6 +21,26 @@ import (
 	"github.com/netbirdio/netbird/route"
 )
 
+const (
+	// latencySampleInterval is how often the round trip time of the selected
+	// candidate pair is read. The ICE agent refreshes it on every consent
+	// binding request, so sampling is cheap and needs no extra traffic.
+	latencySampleInterval = 4 * time.Second
+
+	// latencyEWMAAlpha is the weight of a new sample in the exponentially
+	// weighted moving average. At latencySampleInterval this yields a time
+	// constant of roughly 27 seconds: a sustained change is reflected within
+	// tens of seconds, while the residual movement on a jittery path stays
+	// below the switch margin that route selection applies, so noise alone
+	// cannot move a route. Raising it widens that residual movement.
+	latencyEWMAAlpha = 0.15
+)
+
+// latencySmoothedNoiseFactor converts the spread of the raw samples into the
+// spread of the smoothed value: for an exponentially weighted moving average
+// with weight a, the variance of the output is a/(2-a) of the input variance.
+var latencySmoothedNoiseFactor = math.Sqrt(latencyEWMAAlpha / (2 - latencyEWMAAlpha))
+
 type ICEConnInfo struct {
 	RemoteConn                 net.Conn
 	RosenpassPubKey            []byte
@@ -137,7 +158,7 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 		w.log.Debugf("recreate ICE agent: %s / %s", w.sessionID, *remoteOfferAnswer.SessionID)
 	}
 	dialerCtx, dialerCancel := context.WithCancel(w.ctx)
-	agent, err := w.reCreateAgent(dialerCancel, preferredCandidateTypes)
+	agent, sampleNow, err := w.reCreateAgent(dialerCancel, preferredCandidateTypes)
 	if err != nil {
 		w.log.Errorf("failed to recreate ICE Agent: %s", err)
 		return
@@ -153,7 +174,7 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 
 	// Capture the cancel func at spawn time: connect reads it from the argument
 	// instead of the field, which a newer OnNewOffer may already have replaced.
-	go w.connect(dialerCtx, dialerCancel, agent, remoteOfferAnswer)
+	go w.connect(dialerCtx, dialerCancel, agent, sampleNow, remoteOfferAnswer)
 }
 
 // OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer.
@@ -217,29 +238,33 @@ func (w *WorkerICE) Close() {
 	w.abandonNegotiation()
 }
 
-func (w *WorkerICE) reCreateAgent(dialerCancel context.CancelFunc, candidates []ice.CandidateType) (*icemaker.ThreadSafeAgent, error) {
+// reCreateAgent creates a new ICE agent with its callbacks registered. The
+// returned channel wakes that agent's latency sampler when its selected
+// candidate pair changes.
+func (w *WorkerICE) reCreateAgent(dialerCancel context.CancelFunc, candidates []ice.CandidateType) (*icemaker.ThreadSafeAgent, chan struct{}, error) {
 	w.portForwardAttempted = false
 
 	agent, err := icemaker.NewAgent(w.ctx, w.iFaceDiscover, w.config.ICEConfig, candidates, w.localUfrag, w.localPwd)
 	if err != nil {
-		return nil, fmt.Errorf("create agent: %w", err)
+		return nil, nil, fmt.Errorf("create agent: %w", err)
 	}
 
 	if err := agent.OnCandidate(w.onICECandidate); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if err := agent.OnConnectionStateChange(w.onConnectionStateChange(agent, dialerCancel)); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	if err := agent.OnSelectedCandidatePairChange(func(c1, c2 ice.Candidate) {
-		w.onICESelectedCandidatePair(agent, c1, c2)
-	}); err != nil {
-		return nil, err
+	// one channel per agent, so the sampler of a replaced agent cannot take a
+	// wakeup meant for the new one
+	sampleNow := make(chan struct{}, 1)
+	if err := agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair(sampleNow)); err != nil {
+		return nil, nil, err
 	}
 
-	return agent, nil
+	return agent, sampleNow, nil
 }
 
 func (w *WorkerICE) SessionID() ICESessionID {
@@ -252,7 +277,7 @@ func (w *WorkerICE) SessionID() ICESessionID {
 // will block until connection succeeded
 // but it won't release if ICE Agent went into Disconnected or Failed state,
 // so we have to cancel it with the provided context once agent detected a broken connection
-func (w *WorkerICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) {
+func (w *WorkerICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agent *icemaker.ThreadSafeAgent, sampleNow <-chan struct{}, remoteOfferAnswer *OfferAnswer) {
 	w.log.Debugf("gather candidates")
 	if err := agent.GatherCandidates(); err != nil {
 		w.log.Warnf("failed to gather candidates: %s", err)
@@ -343,6 +368,9 @@ func (w *WorkerICE) connect(ctx context.Context, dialerCancel context.CancelFunc
 	w.lastSuccess = time.Now()
 	w.muxAgent.Unlock()
 
+	// ctx is the agent's dialer context, so sampling stops when the agent is closed
+	go w.sampleLatency(ctx, agent, sampleNow)
+
 	// todo: the potential problem is a race between the onConnectionStateChange
 	// and the delivery below: after this unlock, a newer offer can replace
 	// w.agent before onICEConnectionIsReady runs, delivering this (now stale)
@@ -532,21 +560,63 @@ func (w *WorkerICE) createForwardedCandidate(srflxCandidate ice.Candidate, mappi
 	return candidate, nil
 }
 
-func (w *WorkerICE) onICESelectedCandidatePair(agent *icemaker.ThreadSafeAgent, c1, c2 ice.Candidate) {
-	w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(),
-		w.config.Key)
+func (w *WorkerICE) onICESelectedCandidatePair(sampleNow chan<- struct{}) func(c1, c2 ice.Candidate) {
+	return func(c1, c2 ice.Candidate) {
+		w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(),
+			w.config.Key)
 
+		// refresh the recorded latency right away: it still describes the previous
+		// path, and route selection may run before the next sampling tick
+		select {
+		case sampleNow <- struct{}{}:
+		default:
+		}
+	}
+}
+
+// sampleLatency records the smoothed round trip time of the agent's selected
+// candidate pair until the agent is closed. Route selection compares the
+// recorded latencies of high availability peers, so it has to keep following
+// the path instead of freezing at the value observed when the pair was chosen.
+func (w *WorkerICE) sampleLatency(ctx context.Context, agent *icemaker.ThreadSafeAgent, sampleNow <-chan struct{}) {
+	var sampler latencySampler
+
+	ticker := time.NewTicker(latencySampleInterval)
+	defer ticker.Stop()
+
+	for {
+		w.updateLatency(ctx, agent, &sampler)
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		case <-sampleNow:
+		}
+	}
+}
+
+func (w *WorkerICE) updateLatency(ctx context.Context, agent *icemaker.ThreadSafeAgent, sampler *latencySampler) {
 	pairStat, ok := agent.GetSelectedCandidatePairStats()
 	if !ok {
-		w.log.Warnf("failed to get selected candidate pair stats")
+		w.log.Debugf("failed to get selected candidate pair stats")
 		return
 	}
 
-	duration := time.Duration(pairStat.CurrentRoundTripTime * float64(time.Second))
-	if err := w.statusRecorder.UpdateLatency(w.config.Key, duration); err != nil {
-		w.log.Debugf("failed to update latency for peer: %s", err)
+	sample := sampler.observe(pairStat)
+	if sample.Latency <= 0 {
 		return
 	}
+
+	// the agent may have been closed while sampling, and its connection's
+	// latency cleared: a late sample must not bring the stale value back
+	if ctx.Err() != nil {
+		return
+	}
+
+	if err := w.statusRecorder.UpdateLatency(w.config.Key, sample); err != nil {
+		w.log.Debugf("failed to update latency for peer: %s", err)
+	}
 }
 
 func (w *WorkerICE) logSuccessfulPaths(agent *icemaker.ThreadSafeAgent) {
@@ -612,6 +682,59 @@ func (w *WorkerICE) agentDial(ctx context.Context, agent *icemaker.ThreadSafeAge
 	}
 }
 
+// latencySampler smooths the round trip time samples of a single ICE agent and
+// tracks how much they scatter, so consumers can tell a real difference between
+// two paths from the noise of a jittery one.
+type latencySampler struct {
+	smoothed  time.Duration
+	deviation time.Duration
+	localID   string
+	remoteID  string
+}
+
+// observe folds a candidate pair stat into the moving average and returns the
+// smoothed latency with the uncertainty of that estimate. A different candidate
+// pair starts over: the path changed, so the previous samples describe a route
+// that no longer carries the traffic.
+func (s *latencySampler) observe(stat ice.CandidatePairStats) LatencySample {
+	rtt := time.Duration(stat.CurrentRoundTripTime * float64(time.Second))
+	if rtt <= 0 {
+		return LatencySample{}
+	}
+
+	if s.smoothed <= 0 || s.localID != stat.LocalCandidateID || s.remoteID != stat.RemoteCandidateID {
+		s.localID = stat.LocalCandidateID
+		s.remoteID = stat.RemoteCandidateID
+		s.smoothed = rtt
+		// A fresh path is described by a single raw sample, which is exactly
+		// when the estimate deserves the least trust: a lucky outlier reported
+		// with no uncertainty could move a route on its own. Seed the deviation
+		// at half the observed value so the reported noise stays conservative
+		// until real samples accumulate, and keep a larger deviation carried
+		// over from the previous path, since the link's jitter usually outlives
+		// a candidate pair change.
+		if seed := rtt / 2; s.deviation < seed {
+			s.deviation = seed
+		}
+		return s.sample()
+	}
+
+	// mean absolute deviation of the samples around the average, the same
+	// estimator TCP keeps next to its smoothed round trip time
+	delta := absDuration(rtt - s.smoothed)
+	s.deviation = time.Duration(latencyEWMAAlpha*float64(delta) + (1-latencyEWMAAlpha)*float64(s.deviation))
+	s.smoothed = time.Duration(latencyEWMAAlpha*float64(rtt) + (1-latencyEWMAAlpha)*float64(s.smoothed))
+
+	return s.sample()
+}
+
+func (s *latencySampler) sample() LatencySample {
+	return LatencySample{
+		Latency: s.smoothed,
+		Noise:   time.Duration(latencySmoothedNoiseFactor * float64(s.deviation)),
+	}
+}
+
 func shouldAddExtraCandidate(candidate ice.Candidate) bool {
 	if candidate.Type() != ice.CandidateTypeServerReflexive {
 		return false