[client] Keep HA routing peer selection latency-aware and hold switches that reset connections (#7689)

client/internal/routemanager/client/client.go

@@ -3,8 +3,10 @@ package client
 import (
 	"context"
 	"fmt"
+	"math"
 	"net/netip"
 	"reflect"
+	"sync"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -26,6 +28,48 @@ const (
 	handlerTypeStatic
 )
 
+const (
+	// latencySwitchMinGain is the absolute latency improvement an alternative
+	// routing peer must show before an otherwise equivalent current peer is
+	// replaced. An absolute margin alone would be meaningless on slow paths,
+	// hence latencySwitchGainPercent on top.
+	latencySwitchMinGain = 20 * time.Millisecond
+
+	// latencySwitchGainPercent is the relative latency improvement required in
+	// addition to latencySwitchMinGain. A relative margin alone would churn on
+	// fast paths, where a few milliseconds are a large fraction of the latency.
+	latencySwitchGainPercent = 20
+
+	// latencyNoiseSigmas is how many standard deviations of the measurement
+	// noise a latency gain has to exceed. Smoothing narrows the scatter of a
+	// jittery path but does not remove it, so on such a path two equivalent
+	// peers keep producing differences of tens of milliseconds. Requiring the
+	// gain to stand out from that scatter is what separates a peer that is
+	// really closer from one that only looks closer right now.
+	latencyNoiseSigmas = 4
+
+	// latencySwitchMinDwell is how long a routing peer is kept before latency
+	// can move the choice again. Changes in connection status, metric or relay
+	// state ignore it so failover is never delayed.
+	latencySwitchMinDwell = 30 * time.Second
+
+	// routeReevalInterval is how often routes are re-evaluated without an
+	// incoming peer state change. A switch deferred by latencySwitchMinDwell,
+	// or a latency that drifts in steps too small to notify, would otherwise
+	// wait for an unrelated event to be reconsidered.
+	routeReevalInterval = latencySwitchMinDwell
+
+	// unknownLatency ranks peers without a latency sample below any peer with a
+	// real measurement, without excluding them from selection. It doubles as the
+	// upper bound for samples, so a pathological value cannot dominate, and sits
+	// well above any usable path so that slow paths keep their order.
+	unknownLatency = 10 * time.Second
+
+	// holdReportInterval is how long a routing peer held back by a switch rule
+	// is reported at debug level only, after being reported at info level once.
+	holdReportInterval = 10 * time.Minute
+)
+
 type reason int
 
 const (
@@ -36,10 +80,133 @@ const (
 	reasonHA
 )
 
+// switchReason describes why a routing peer replaced the current one.
+type switchReason string
+
+const (
+	switchReasonNone         switchReason = ""
+	switchReasonUnavailable  switchReason = "current routing peer unavailable"
+	switchReasonConnected    switchReason = "connected peer preferred over idle"
+	switchReasonMetric       switchReason = "lower route metric"
+	switchReasonDirect       switchReason = "direct connection preferred over relayed"
+	switchReasonLatency      switchReason = "lower latency"
+	switchReasonRouteChanged switchReason = "route assigned to a different routing peer"
+)
+
+// holdKind is the rule that keeps the current routing peer over a better one.
+type holdKind int
+
+const (
+	holdLatencyMargin holdKind = iota
+	holdLatencyDwell
+	holdRelayedDelay
+)
+
 type routerPeerStatus struct {
 	status  peer.ConnStatus
 	relayed bool
 	latency time.Duration
+	// noise is the uncertainty of latency, see peer.LatencySample
+	noise time.Duration
+}
+
+func (s routerPeerStatus) isDirect() bool {
+	return s.status == peer.StatusConnected && !s.relayed
+}
+
+// routeCandidate is a route considered for selection, together with the
+// selection criteria derived from the state of its routing peer.
+type routeCandidate struct {
+	id        route.ID
+	peer      string
+	metric    int
+	connected bool
+	relayed   bool
+	latency   time.Duration
+	noise     time.Duration
+}
+
+func newRouteCandidate(r *route.Route, status routerPeerStatus) routeCandidate {
+	latency := status.latency
+	if latency <= 0 || latency > unknownLatency {
+		latency = unknownLatency
+	}
+
+	return routeCandidate{
+		id:        r.ID,
+		peer:      r.Peer,
+		metric:    r.Metric,
+		connected: status.status == peer.StatusConnected,
+		relayed:   status.relayed,
+		latency:   latency,
+		noise:     status.noise,
+	}
+}
+
+// tierCompare orders c against o on the categorical criteria: connection status,
+// then route metric, then direct versus relayed. It returns a positive number
+// when c outranks o, a negative one when o outranks c, and zero when the two are
+// equivalent. Latency is excluded on purpose: it is the tie-break applied by
+// betterThan and the only criterion the switch margins in shouldSwitchRoute
+// apply to.
+func (c routeCandidate) tierCompare(o routeCandidate) int {
+	if c.connected != o.connected {
+		if c.connected {
+			return 1
+		}
+		return -1
+	}
+	if c.metric != o.metric {
+		return o.metric - c.metric
+	}
+	if c.relayed != o.relayed {
+		if c.relayed {
+			return -1
+		}
+		return 1
+	}
+
+	return 0
+}
+
+func (c routeCandidate) isDirect() bool {
+	return c.connected && !c.relayed
+}
+
+func (c routeCandidate) tierBetterThan(o routeCandidate) bool {
+	return c.tierCompare(o) > 0
+}
+
+// winsOnlyOnRelay reports whether c outranks o solely by being direct while o
+// is relayed, with connection status and metric equal.
+func (c routeCandidate) winsOnlyOnRelay(o routeCandidate) bool {
+	return c.connected == o.connected && c.metric == o.metric && o.relayed && !c.relayed
+}
+
+// switchReasonOver names the criterion on which c replaces the current o.
+func (c routeCandidate) switchReasonOver(o routeCandidate) switchReason {
+	switch {
+	case c.connected != o.connected:
+		return switchReasonConnected
+	case c.metric != o.metric:
+		return switchReasonMetric
+	case c.relayed != o.relayed:
+		return switchReasonDirect
+	default:
+		return switchReasonLatency
+	}
+}
+
+func (c routeCandidate) betterThan(o routeCandidate) bool {
+	if tier := c.tierCompare(o); tier != 0 {
+		return tier > 0
+	}
+
+	return c.latency < o.latency
+}
+
+func (c routeCandidate) String() string {
+	return fmt.Sprintf("connected: %t, metric: %d, relayed: %t, latency: %v +/- %v", c.connected, c.metric, c.relayed, c.latency, c.noise)
 }
 
 type RoutesUpdate struct {
@@ -69,18 +236,72 @@ type WatcherConfig struct {
 // Once stopped, it cannot be reused.
 // The methods are not thread-safe and should be synchronized externally.
 type Watcher struct {
-	ctx                 context.Context
-	cancel              context.CancelFunc
-	statusRecorder      *peer.Status
-	wgInterface         iface.WGIface
-	routes              map[route.ID]*route.Route
-	routeUpdate         chan RoutesUpdate
-	peerStateUpdate     chan map[string]peer.RouterState
+	ctx            context.Context
+	cancel         context.CancelFunc
+	statusRecorder *peer.Status
+	wgInterface    iface.WGIface
+	routes         map[route.ID]*route.Route
+	routeUpdate    chan RoutesUpdate
+	// peerStateUpdate holds at most one pending routing peer state notification,
+	// see watchPeerStatusChanges
+	peerStateUpdate     chan struct{}
 	routePeersNotifiers map[string]chan struct{} // map of peer key to channel for peer state changes
 	currentChosen       *route.Route
 	currentChosenStatus *routerPeerStatus
-	handler             RouteHandler
-	updateSerial        uint64
+	// lastSwitch is when currentChosen was last replaced by a latency-driven
+	// switch, used to enforce latencySwitchMinDwell. Categorical switches
+	// (failover, metric, relay state) leave it untouched: the dwell exists to
+	// stop latency ping-pong, and arming it on a failover would delay the
+	// return to a recovered better peer.
+	lastSwitch time.Time
+	// latencySwitch marks the evaluation's switch decision as latency-driven;
+	// set by shouldSwitchRoute, consumed when the switch is applied
+	latencySwitch bool
+	// switchReason describes why the evaluation replaced the current routing
+	// peer; set by getBestRouteFromStatuses, consumed when the switch is applied
+	switchReason switchReason
+	// relayedSwitch is how long a current routing peer that dropped from direct
+	// to relayed is kept over an otherwise equivalent direct one. The zero value
+	// switches at once.
+	relayedSwitch relayedSwitchPolicy
+	// trackedCurrent, trackedPeer and trackedDirect are the current route, its
+	// routing peer and whether that peer was connected directly as of the
+	// previous evaluation, used to tell a peer that dropped to relay while
+	// carrying the route from one that was never direct
+	trackedCurrent route.ID
+	trackedPeer    string
+	trackedDirect  bool
+	// degradedSince is when the current routing peer dropped from direct to
+	// relayed, zero if it is direct or was already relayed when chosen
+	degradedSince time.Time
+	// relayedHold fires when a relayed switch being held back becomes due, so it
+	// is applied on time rather than on the next periodic re-evaluation
+	relayedHold *time.Timer
+	// heldCandidate and heldKind identify the alternative last held back by the
+	// switch rules and the rule holding it, and heldReported is when that was
+	// reported at info level. Routes are re-evaluated on every latency change,
+	// so a hold is reported once per episode instead of on every evaluation.
+	heldCandidate route.ID
+	heldKind      holdKind
+	heldReported  time.Time
+	// reportedUnassigned tracks that the current no-routing-peer episode has
+	// been reported, so the periodic re-evaluation does not repeat it
+	reportedUnassigned bool
+	// done is closed when the Start loop exits, so Stop can wait for it before
+	// removing the routes the loop might still be updating
+	done chan struct{}
+	// lifecycleMu orders Start against Stop: Start runs in its own goroutine
+	// and may begin after Stop, or never, so Stop only waits for done when the
+	// loop has actually started
+	lifecycleMu  sync.Mutex
+	started      bool
+	stopped      bool
+	handler      RouteHandler
+	updateSerial uint64
+}
+
+func newPeerStateUpdate() chan struct{} {
+	return make(chan struct{}, 1)
 }
 
 func NewWatcher(config WatcherConfig) *Watcher {
@@ -94,9 +315,11 @@ func NewWatcher(config WatcherConfig) *Watcher {
 		routes:              make(map[route.ID]*route.Route),
 		routePeersNotifiers: make(map[string]chan struct{}),
 		routeUpdate:         make(chan RoutesUpdate),
-		peerStateUpdate:     make(chan map[string]peer.RouterState),
+		peerStateUpdate:     newPeerStateUpdate(),
+		done:                make(chan struct{}),
 		handler:             config.Handler,
 		currentChosenStatus: nil,
+		relayedSwitch:       relayedSwitch(),
 	}
 	return client
 }
@@ -113,23 +336,7 @@ func (w *Watcher) getRouterPeerStatuses() map[route.ID]routerPeerStatus {
 			status:  peerStatus.ConnStatus,
 			relayed: peerStatus.Relayed,
 			latency: peerStatus.Latency,
-		}
-	}
-	return routePeerStatuses
-}
-
-func (w *Watcher) convertRouterPeerStatuses(states map[string]peer.RouterState) map[route.ID]routerPeerStatus {
-	routePeerStatuses := make(map[route.ID]routerPeerStatus)
-	for _, r := range w.routes {
-		peerStatus, ok := states[r.Peer]
-		if !ok {
-			log.Warnf("couldn't fetch peer state: %v", r.Peer)
-			continue
-		}
-		routePeerStatuses[r.ID] = routerPeerStatus{
-			status:  peerStatus.Status,
-			relayed: peerStatus.Relayed,
-			latency: peerStatus.Latency,
+			noise:   peerStatus.LatencyNoise,
 		}
 	}
 	return routePeerStatuses
@@ -139,122 +346,277 @@ func (w *Watcher) convertRouterPeerStatuses(states map[string]peer.RouterState)
 // within a Watcher, taking into account peer connection status, route metrics, and
 // preference for non-relayed and direct connections.
 //
-// It follows these prioritization rules:
+// Candidates are ranked by the categorical criteria first, in this order:
 // * Connection status: Both connected and idle peers are considered, but connected peers always take precedence.
-// * Idle peer penalty: Idle peers receive a significant score penalty to ensure any connected peer is preferred.
 // * Metric: Routes with lower metrics (better) are prioritized.
 // * Non-relayed: Routes without relays are preferred.
-// * Latency: Routes with lower latency are prioritized.
-// * Allowed IPs: Idle peers can still receive allowed IPs to enable lazy connection triggering.
-// * we compare the current score + 10ms to the chosen score to avoid flapping between routes
-// * Stability: In case of equal scores, the currently active route (if any) is maintained.
+// Peers that are equal on all of them are then ranked by latency, lowest first.
+//
+// Idle peers can still receive allowed IPs to enable lazy connection triggering.
+// Connecting peers are skipped: they have no endpoint to assign allowed IPs to.
+//
+// Replacing the current route is subject to shouldSwitchRoute, so latency alone
+// only moves the route when the gain is worth the interruption. In case of a
+// tie, the currently active route (if any) is maintained.
 //
 // It returns the ID of the selected optimal route.
 func (w *Watcher) getBestRouteFromStatuses(routePeerStatuses map[route.ID]routerPeerStatus) (route.ID, routerPeerStatus) {
-	var chosen route.ID
-	chosenScore := float64(0)
-	currScore := float64(0)
-
 	var currID route.ID
 	if w.currentChosen != nil {
 		currID = w.currentChosen.ID
 	}
 
-	var chosenStatus routerPeerStatus
+	var best, current routeCandidate
+	var haveBest, haveCurrent bool
 
 	for _, r := range w.routes {
-		tempScore := float64(0)
 		peerStatus, found := routePeerStatuses[r.ID]
-		// connecting status equals disconnected: no wireguard endpoint to assign allowed IPs to
+		// connecting status equals disconnected: no endpoint to assign allowed IPs to
 		if !found || peerStatus.status == peer.StatusConnecting {
 			continue
 		}
 
-		if r.Metric < route.MaxMetric {
-			metricDiff := route.MaxMetric - r.Metric
-			tempScore = float64(metricDiff) * 10
-		}
-
-		// in some temporal cases, latency can be 0, so we set it to 999ms to not block but try to avoid this route
-		latency := 999 * time.Millisecond
-		if peerStatus.latency != 0 {
-			latency = peerStatus.latency
-		} else if !peerStatus.relayed && peerStatus.status != peer.StatusIdle {
-			log.Tracef("peer %s has 0 latency: [%v]", r.Peer, w.handler)
-		}
-
-		// avoid negative tempScore on the higher latency calculation
-		if latency > 1*time.Second {
-			latency = 999 * time.Millisecond
-		}
-
-		// higher latency is worse score
-		tempScore += 1 - latency.Seconds()
-
-		// apply significant penalty for idle peers to ensure connected peers always take precedence
-		if peerStatus.status == peer.StatusConnected {
-			tempScore += 100_000
-		}
-
-		if !peerStatus.relayed {
-			tempScore++
-		}
-
-		if tempScore > chosenScore || (tempScore == chosenScore && chosen == "") {
-			chosen = r.ID
-			chosenStatus = peerStatus
-			chosenScore = tempScore
-		}
-
-		if chosen == "" && currID == "" {
-			chosen = r.ID
-			chosenStatus = peerStatus
-			chosenScore = tempScore
-		}
-
+		candidate := newRouteCandidate(r, peerStatus)
 		if r.ID == currID {
-			currScore = tempScore
+			current, haveCurrent = candidate, true
+		}
+		if !haveBest || candidate.betterThan(best) {
+			best, haveBest = candidate, true
 		}
 	}
 
-	chosenID := chosen
-	if chosen == "" {
-		chosenID = "<none>"
-	}
-	currentID := currID
-	if currID == "" {
-		currentID = "<none>"
+	// Candidates that tie must not displace the current peer: the routes are
+	// held in a map, so the winner of a tie would otherwise depend on iteration
+	// order and change from one evaluation to the next.
+	if haveCurrent && !best.betterThan(current) {
+		best = current
 	}
 
-	log.Debugf("chosen route: %s, chosen score: %f, current route: %s, current score: %f", chosenID, chosenScore, currentID, currScore)
+	w.trackRelayed(current, haveCurrent)
+
+	if !haveBest {
+		w.reportUnassigned()
+		w.heldCandidate = ""
+
+		return "", routerPeerStatus{}
+	}
+	w.reportedUnassigned = false
+
+	if haveCurrent && current.id != best.id {
+		if log.IsLevelEnabled(log.DebugLevel) {
+			log.Debugf("best routing peer for [%v] is %s (%s), current is %s (%s)",
+				w.handler, best.peer, best, current.peer, current)
+		}
+
+		if !w.shouldSwitchRoute(current, best) {
+			return current.id, routePeerStatuses[current.id]
+		}
+	}
 
 	switch {
-	case chosen == "":
-		var peers []string
-		for _, r := range w.routes {
-			peers = append(peers, r.Peer)
-		}
-
-		log.Infof("network [%v] has not been assigned a routing peer as no peers from the list %s are currently available", w.handler, peers)
-	case chosen != currID:
-		// we compare the current score + 10ms to the chosen score to avoid flapping between routes
-		if currScore != 0 && currScore+0.01 > chosenScore {
-			log.Debugf("keeping current routing peer %s for [%v]: the score difference with latency is less than 0.01(10ms): current: %f, new: %f",
-				w.currentChosen.Peer, w.handler, currScore, chosenScore)
-			return currID, chosenStatus
-		}
-		var p string
-		if rt := w.routes[chosen]; rt != nil {
-			p = rt.Peer
-		}
-		log.Infof("New chosen route is %s with peer %s with score %f for network [%v]", chosen, p, chosenScore, w.handler)
+	case haveCurrent && current.id == best.id:
+	case haveCurrent:
+		w.switchReason = best.switchReasonOver(current)
+		log.Warnf("switching routing peer for network [%v] from %s to %s: %s (%s)",
+			w.handler, current.peer, best.peer, w.switchReason, best)
+	case w.currentChosen != nil:
+		w.switchReason = switchReasonUnavailable
+		log.Warnf("switching routing peer for network [%v] from %s to %s: %s (%s)",
+			w.handler, w.currentChosen.Peer, best.peer, w.switchReason, best)
+	default:
+		log.Infof("new chosen route is %s with peer %s for network [%v]: %s",
+			best.id, best.peer, w.handler, best)
 	}
 
-	return chosen, chosenStatus
+	return best.id, routePeerStatuses[best.id]
 }
 
-func (w *Watcher) watchPeerStatusChanges(ctx context.Context, peerKey string, peerStateUpdate chan map[string]peer.RouterState, closer chan struct{}) {
-	subscription := w.statusRecorder.SubscribeToPeerStateChanges(ctx, peerKey)
+// reportUnassigned reports that no routing peer is available for the network,
+// once per episode: the periodic re-evaluation would otherwise repeat it for as
+// long as the routing peers stay unavailable.
+func (w *Watcher) reportUnassigned() {
+	if w.reportedUnassigned {
+		return
+	}
+	w.reportedUnassigned = true
+
+	var peers []string
+	for _, r := range w.routes {
+		peers = append(peers, r.Peer)
+	}
+	log.Infof("network [%v] has not been assigned a routing peer as no peers from the list %s are currently available", w.handler, peers)
+}
+
+// trackRelayed records when the current routing peer dropped from a direct to a
+// relayed connection while carrying the route, which starts the relayedSwitch
+// delay. A peer that was never direct, because it was relayed when chosen or
+// came up over relay from idle, is not degraded: holding on to it would only
+// keep the route on the slower path.
+func (w *Watcher) trackRelayed(current routeCandidate, haveCurrent bool) {
+	if !haveCurrent {
+		w.trackedCurrent, w.trackedPeer, w.trackedDirect, w.degradedSince = "", "", false, time.Time{}
+		return
+	}
+
+	switch {
+	case current.id != w.trackedCurrent, current.peer != w.trackedPeer, !current.relayed:
+		w.degradedSince = time.Time{}
+	case w.trackedDirect:
+		w.degradedSince = time.Now()
+		// a new degradation, reported as such even if an earlier one held
+		// back the same candidate
+		w.heldCandidate = ""
+	}
+	w.trackedCurrent, w.trackedPeer, w.trackedDirect = current.id, current.peer, current.isDirect()
+}
+
+// shouldSwitchRoute reports whether the current routing peer should be replaced
+// by the candidate, which outranks it.
+//
+// A candidate that wins on a categorical criterion (it is connected while the
+// current peer is not, has a better metric, or is direct while the current peer
+// is relayed) takes over immediately. The exception is a current peer that only
+// just dropped from direct to relayed, see relayedSwitchDue.
+//
+// A candidate that only has lower latency has to beat the current peer by the
+// switch margin, and the current peer has to have been in place for
+// latencySwitchMinDwell. Every switch interrupts the traffic using the route, so
+// a gain that is marginal, or that the measurement noise of the two paths can
+// account for on its own, is not worth acting on.
+//
+// An approved latency-driven switch sets w.latencySwitch, which arms the dwell
+// time once the switch is applied.
+func (w *Watcher) shouldSwitchRoute(current, candidate routeCandidate) bool {
+	if candidate.winsOnlyOnRelay(current) {
+		return w.relayedSwitchDue(current, candidate)
+	}
+	if candidate.tierBetterThan(current) {
+		return true
+	}
+
+	gain := current.latency - candidate.latency
+	if margin := switchMargin(current, candidate); gain < margin {
+		if level := w.holdLevel(candidate, holdLatencyMargin); log.IsLevelEnabled(level) {
+			log.StandardLogger().Logf(level,
+				"keeping routing peer %s for [%v] over %s: latency gain %v on %v is below the %v switch margin (%s)",
+				current.peer, w.handler, candidate.peer, gain, current.latency, margin, candidate)
+		}
+		return false
+	}
+
+	if dwelled := time.Since(w.lastSwitch); dwelled < latencySwitchMinDwell {
+		if level := w.holdLevel(candidate, holdLatencyDwell); log.IsLevelEnabled(level) {
+			log.StandardLogger().Logf(level,
+				"keeping routing peer %s for [%v] over %s: chosen %v ago, below the %v minimum dwell time",
+				current.peer, w.handler, candidate.peer, dwelled.Round(time.Second), latencySwitchMinDwell)
+		}
+		return false
+	}
+
+	w.latencySwitch = true
+
+	return true
+}
+
+// relayedSwitchDue reports whether a relayed current routing peer should give
+// way to a candidate that is better only by being direct.
+//
+// A peer that dropped from direct to relayed while carrying the route is kept
+// for the relayedSwitch delay: the relay still carries the traffic, staying
+// keeps every flow through this routing peer alive, and a direct connection that
+// dropped often recovers within seconds. A peer that was already relayed when
+// chosen gives way at once, so the route is not stuck on the slower path when a
+// direct one becomes available. A switch that is not due yet arms relayedHold so
+// it is applied as soon as it becomes due.
+func (w *Watcher) relayedSwitchDue(current, candidate routeCandidate) bool {
+	if w.degradedSince.IsZero() {
+		return true
+	}
+
+	policy := w.relayedSwitch
+	if !policy.never {
+		remaining := policy.delay - time.Since(w.degradedSince)
+		if remaining <= 0 {
+			return true
+		}
+		w.armRelayedHold(remaining)
+	}
+
+	if level := w.holdLevel(candidate, holdRelayedDelay); log.IsLevelEnabled(level) {
+		log.StandardLogger().Logf(level,
+			"keeping relayed routing peer %s for [%v] over direct %s: dropped to relay %v ago, switch delay %s",
+			current.peer, w.handler, candidate.peer, time.Since(w.degradedSince).Round(time.Second), policy)
+	}
+
+	return false
+}
+
+func (w *Watcher) armRelayedHold(d time.Duration) {
+	if w.relayedHold == nil {
+		w.relayedHold = time.NewTimer(d)
+		return
+	}
+	w.relayedHold.Reset(d)
+}
+
+// relayedHoldC is the channel of relayedHold, nil while it was never armed so
+// that selecting on it blocks.
+func (w *Watcher) relayedHoldC() <-chan time.Time {
+	if w.relayedHold == nil {
+		return nil
+	}
+	return w.relayedHold.C
+}
+
+// holdLevel is the level to report a decision to keep the current routing peer
+// at. The first evaluation that holds a given candidate back under a given rule
+// is reported at info level, because that is what someone wondering why a route
+// did not move needs to see. The repeats stay at debug level for
+// holdReportInterval, including when two near-equal peers keep trading places.
+func (w *Watcher) holdLevel(candidate routeCandidate, kind holdKind) log.Level {
+	if w.heldCandidate == candidate.id && w.heldKind == kind && time.Since(w.heldReported) < holdReportInterval {
+		return log.DebugLevel
+	}
+
+	w.heldCandidate, w.heldKind, w.heldReported = candidate.id, kind, time.Now()
+
+	return log.InfoLevel
+}
+
+// switchMargin is how much lower the candidate's latency has to be before the
+// route moves: at least latencySwitchMinGain, at least latencySwitchGainPercent
+// of the current latency so the margin scales with slow paths, and more than the
+// combined measurement noise of the two paths. The noise term is what keeps a
+// jittery path from moving the route on its own; it vanishes on a stable path,
+// where the fixed margins decide.
+func switchMargin(current, candidate routeCandidate) time.Duration {
+	margin := latencySwitchMinGain
+
+	if relative := current.latency * latencySwitchGainPercent / 100; relative > margin {
+		margin = relative
+	}
+
+	// the two paths vary independently, so their noise adds in quadrature
+	noise := time.Duration(latencyNoiseSigmas * math.Hypot(float64(current.noise), float64(candidate.noise)))
+	if noise > margin {
+		margin = noise
+	}
+
+	return margin
+}
+
+// watchPeerStatusChanges forwards the state changes of a routing peer to the
+// watcher as a notification to re-evaluate. The watcher reads the current peer
+// states when it gets to it, so a notification still pending covers any number
+// of later changes, and the forwarder never waits for a busy watcher: that
+// would stop it draining the subscription and in turn block the peer's updates.
+func (w *Watcher) watchPeerStatusChanges(ctx context.Context, peerKey string, peerStateUpdate chan<- struct{}, closer chan struct{}) {
+	// the subscription gets its own context so a delivery blocked on it is
+	// released as soon as this forwarder exits, not only when the watcher stops
+	subCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	subscription := w.statusRecorder.SubscribeToPeerStateChanges(subCtx, peerKey)
 	defer w.statusRecorder.UnsubscribePeerStateChanges(subscription)
 
 	for {
@@ -263,14 +625,11 @@ func (w *Watcher) watchPeerStatusChanges(ctx context.Context, peerKey string, pe
 			return
 		case <-closer:
 			return
-		case routerStates := <-subscription.Events():
+		case <-subscription.Events():
 			select {
-			case peerStateUpdate <- routerStates:
+			case peerStateUpdate <- struct{}{}:
 				log.Debugf("triggered route state update for Peer: %s", peerKey)
-			case <-ctx.Done():
-				return
-			case <-closer:
-				return
+			default:
 			}
 		}
 	}
@@ -337,6 +696,8 @@ func (w *Watcher) shouldSkipRecalculation(newChosenID route.ID, newStatus router
 }
 
 func (w *Watcher) recalculateRoutes(rsn reason, routerPeerStatuses map[route.ID]routerPeerStatus) error {
+	w.latencySwitch = false
+	w.switchReason = switchReasonNone
 	newChosenID, newStatus := w.getBestRouteFromStatuses(routerPeerStatuses)
 
 	// If no route is chosen, remove the route from the peer
@@ -348,6 +709,9 @@ func (w *Watcher) recalculateRoutes(rsn reason, routerPeerStatuses map[route.ID]
 		if err := w.removeAllowedIPs(w.currentChosen, rsn); err != nil {
 			return fmt.Errorf("remove obsolete: %w", err)
 		}
+		if rsn == reasonPeerUpdate {
+			w.routingPeerLostEvent(w.currentChosen)
+		}
 
 		w.currentChosen = nil
 		w.currentChosenStatus = nil
@@ -360,6 +724,17 @@ func (w *Watcher) recalculateRoutes(rsn reason, routerPeerStatuses map[route.ID]
 		return nil
 	}
 
+	previous := w.currentChosen
+	newChosenRoute := w.routes[newChosenID]
+	// a route update can hand the same route to a different routing peer
+	peerReplaced := previous != nil && previous.ID == newChosenID && previous.Peer != newChosenRoute.Peer
+	switched := previous == nil || previous.ID != newChosenID || peerReplaced
+	if peerReplaced {
+		w.switchReason = switchReasonRouteChanged
+		log.Warnf("switching routing peer for network [%v] from %s to %s: %s",
+			w.handler, previous.Peer, newChosenRoute.Peer, w.switchReason)
+	}
+
 	// If the chosen route was assigned to a different peer, remove the allowed IPs first
 	if isNew := w.currentChosen == nil; !isNew {
 		if err := w.removeAllowedIPs(w.currentChosen, reasonHA); err != nil {
@@ -367,7 +742,6 @@ func (w *Watcher) recalculateRoutes(rsn reason, routerPeerStatuses map[route.ID]
 		}
 	}
 
-	newChosenRoute := w.routes[newChosenID]
 	if err := w.addAllowedIPs(newChosenRoute); err != nil {
 		return fmt.Errorf("add new: %w", err)
 	}
@@ -377,20 +751,86 @@ func (w *Watcher) recalculateRoutes(rsn reason, routerPeerStatuses map[route.ID]
 
 	w.currentChosen = newChosenRoute
 	w.currentChosenStatus = &newStatus
+	if !switched {
+		return nil
+	}
+
+	// start tracking the new routing peer from the state it was chosen in, so a
+	// drop to relay seen by the next evaluation counts as a degradation
+	w.trackedCurrent, w.trackedPeer = newChosenID, newChosenRoute.Peer
+	w.trackedDirect, w.degradedSince = newStatus.isDirect(), time.Time{}
+	if w.latencySwitch {
+		w.lastSwitch = time.Now()
+	}
+	if previous != nil {
+		w.routingPeerSwitchEvent(previous, newChosenRoute)
+	}
 
 	return nil
 }
 
-func (w *Watcher) connectEvent(route *route.Route) {
-	var defaultRoute bool
-	for _, r := range w.routes {
-		if r.Network.Bits() == 0 {
-			defaultRoute = true
-			break
-		}
+// routingPeerSwitchEvent publishes a replaced routing peer as a system event,
+// so it shows in the status output and debug bundles whatever the log level.
+// Moving a masqueraded route resets the connections that used it. For default
+// routes this adds the reason to the events of disconnectEvent and
+// connectEvent, and carries no user message so it is not shown as a
+// notification a second time.
+func (w *Watcher) routingPeerSwitchEvent(previous, next *route.Route) {
+	if w.statusRecorder == nil {
+		return
 	}
 
-	if !defaultRoute {
+	reason := w.switchReason
+	if reason == switchReasonNone {
+		reason = switchReasonUnavailable
+	}
+
+	w.statusRecorder.PublishEvent(
+		proto.SystemEvent_WARNING,
+		proto.SystemEvent_NETWORK,
+		"Routing peer changed",
+		"",
+		map[string]string{
+			"network":       w.handler.String(),
+			"id":            string(next.NetID),
+			"previous_peer": previous.Peer,
+			"peer":          next.Peer,
+			"reason":        string(reason),
+		},
+	)
+}
+
+// routingPeerLostEvent publishes the loss of the last available routing peer of
+// a network as a system event. Default routes are left to disconnectEvent.
+func (w *Watcher) routingPeerLostEvent(previous *route.Route) {
+	if w.statusRecorder == nil || w.hasDefaultRoute() {
+		return
+	}
+
+	w.statusRecorder.PublishEvent(
+		proto.SystemEvent_WARNING,
+		proto.SystemEvent_NETWORK,
+		"No routing peer available",
+		"",
+		map[string]string{
+			"network":       w.handler.String(),
+			"id":            string(previous.NetID),
+			"previous_peer": previous.Peer,
+		},
+	)
+}
+
+func (w *Watcher) hasDefaultRoute() bool {
+	for _, r := range w.routes {
+		if r.Network.Bits() == 0 {
+			return true
+		}
+	}
+	return false
+}
+
+func (w *Watcher) connectEvent(route *route.Route) {
+	if !w.hasDefaultRoute() {
 		return
 	}
 
@@ -411,15 +851,7 @@ func (w *Watcher) connectEvent(route *route.Route) {
 }
 
 func (w *Watcher) disconnectEvent(route *route.Route, rsn reason) {
-	var defaultRoute bool
-	for _, r := range w.routes {
-		if r.Network.Bits() == 0 {
-			defaultRoute = true
-			break
-		}
-	}
-
-	if !defaultRoute {
+	if !w.hasDefaultRoute() {
 		return
 	}
 
@@ -505,13 +937,41 @@ func (w *Watcher) classifyUpdate(update RoutesUpdate) bool {
 // Start is the main point of reacting on client network routing events.
 // All the processing related to the client network should be done here. Thread-safe.
 func (w *Watcher) Start() {
+	w.lifecycleMu.Lock()
+	if w.stopped || w.started {
+		w.lifecycleMu.Unlock()
+		return
+	}
+	w.started = true
+	w.lifecycleMu.Unlock()
+
+	defer close(w.done)
+
+	reeval := time.NewTicker(routeReevalInterval)
+	defer reeval.Stop()
+	defer func() {
+		if w.relayedHold != nil {
+			w.relayedHold.Stop()
+		}
+	}()
+
 	for {
 		select {
 		case <-w.ctx.Done():
 			return
-		case routersStates := <-w.peerStateUpdate:
-			routerPeerStatuses := w.convertRouterPeerStatuses(routersStates)
-			if err := w.recalculateRoutes(reasonPeerUpdate, routerPeerStatuses); err != nil {
+		case <-reeval.C:
+			if err := w.recalculateRoutes(reasonPeerUpdate, w.getRouterPeerStatuses()); err != nil {
+				log.Errorf("Failed to re-evaluate routes for network [%v]: %v", w.handler, err)
+			}
+		case <-w.relayedHoldC():
+			if err := w.recalculateRoutes(reasonPeerUpdate, w.getRouterPeerStatuses()); err != nil {
+				log.Errorf("Failed to re-evaluate routes for network [%v]: %v", w.handler, err)
+			}
+		case <-w.peerStateUpdate:
+			// The notification only says that a routing peer changed. The states it
+			// carries can be older than ones already applied, since every routing
+			// peer has its own subscription, so the current states are read instead.
+			if err := w.recalculateRoutes(reasonPeerUpdate, w.getRouterPeerStatuses()); err != nil {
 				log.Errorf("Failed to recalculate routes for network [%v]: %v", w.handler, err)
 			}
 		case update := <-w.routeUpdate:
@@ -550,7 +1010,22 @@ func (w *Watcher) handleRouteUpdate(update RoutesUpdate) {
 func (w *Watcher) Stop() {
 	log.Debugf("Stopping watcher for network [%v]", w.handler)
 
+	w.lifecycleMu.Lock()
+	if w.stopped {
+		w.lifecycleMu.Unlock()
+		return
+	}
+	w.stopped = true
+	started := w.started
+	w.lifecycleMu.Unlock()
+
 	w.cancel()
+	// wait for the Start loop to exit: it may be inside a recalculation, and
+	// cleaning up concurrently would race on currentChosen and could leave the
+	// allowed IPs of a route chosen after this point behind
+	if started {
+		<-w.done
+	}
 
 	if w.currentChosen == nil {
 		return

client/internal/routemanager/client/client_bench_test.go

@@ -23,6 +23,12 @@ var benchmarkTiers = []benchmarkTier{
 	{"Small", 100, 50, 4},
 	{"Medium", 1000, 200, 16},
 	{"Large", 5000, 500, 32},
+	// the tiers below are deliberately past anything realistic: a watcher holds
+	// the routes of a single network, while these give one watcher every route
+	// in the account at once
+	{"ManyPeers", 20000, 500, 32},
+	{"ManyRoutes", 1000, 5000, 16},
+	{"ManyBoth", 20000, 5000, 32},
 }
 
 type mockRouteHandler struct {
@@ -136,7 +142,7 @@ func BenchmarkRecalculateRoutes(b *testing.B) {
 				routes:              routes,
 				routePeersNotifiers: make(map[string]chan struct{}),
 				routeUpdate:         make(chan RoutesUpdate),
-				peerStateUpdate:     make(chan map[string]peer.RouterState),
+				peerStateUpdate:     newPeerStateUpdate(),
 				handler:             &mockRouteHandler{network: "benchmark"},
 				currentChosenStatus: nil,
 			}

client/internal/routemanager/client/client_test.go

@@ -1,11 +1,17 @@
 package client
 
 import (
+	"context"
 	"fmt"
 	"net/netip"
+	"sync"
 	"testing"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/internal/routemanager/common"
 	"github.com/netbirdio/netbird/client/internal/routemanager/static"
@@ -316,6 +322,145 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 			currentRoute:    "route1",
 			expectedRouteID: "route2",
 		},
+		{
+			// 40ms gain clears latencySwitchMinGain but not the 20% of 300ms
+			name: "latency gain below the relative margin should not change the route",
+			statuses: map[route.ID]routerPeerStatus{
+				"route1": {
+					status:  peer.StatusConnected,
+					latency: 300 * time.Millisecond,
+				},
+				"route2": {
+					status:  peer.StatusConnected,
+					latency: 260 * time.Millisecond,
+				},
+			},
+			existingRoutes: map[route.ID]*route.Route{
+				"route1": {
+					ID:     "route1",
+					Metric: route.MaxMetric,
+					Peer:   "peer1",
+				},
+				"route2": {
+					ID:     "route2",
+					Metric: route.MaxMetric,
+					Peer:   "peer2",
+				},
+			},
+			currentRoute:    "route1",
+			expectedRouteID: "route1",
+		},
+		{
+			name: "latency gain above both margins should change the route",
+			statuses: map[route.ID]routerPeerStatus{
+				"route1": {
+					status:  peer.StatusConnected,
+					latency: 300 * time.Millisecond,
+				},
+				"route2": {
+					status:  peer.StatusConnected,
+					latency: 200 * time.Millisecond,
+				},
+			},
+			existingRoutes: map[route.ID]*route.Route{
+				"route1": {
+					ID:     "route1",
+					Metric: route.MaxMetric,
+					Peer:   "peer1",
+				},
+				"route2": {
+					ID:     "route2",
+					Metric: route.MaxMetric,
+					Peer:   "peer2",
+				},
+			},
+			currentRoute:    "route1",
+			expectedRouteID: "route2",
+		},
+		{
+			// 7ms gain is 58% of the current latency but below latencySwitchMinGain
+			name: "latency gain below the absolute margin should not change the route",
+			statuses: map[route.ID]routerPeerStatus{
+				"route1": {
+					status:  peer.StatusConnected,
+					latency: 12 * time.Millisecond,
+				},
+				"route2": {
+					status:  peer.StatusConnected,
+					latency: 5 * time.Millisecond,
+				},
+			},
+			existingRoutes: map[route.ID]*route.Route{
+				"route1": {
+					ID:     "route1",
+					Metric: route.MaxMetric,
+					Peer:   "peer1",
+				},
+				"route2": {
+					ID:     "route2",
+					Metric: route.MaxMetric,
+					Peer:   "peer2",
+				},
+			},
+			currentRoute:    "route1",
+			expectedRouteID: "route1",
+		},
+		{
+			name: "current relayed peer should be replaced by a direct peer with worse latency",
+			statuses: map[route.ID]routerPeerStatus{
+				"route1": {
+					status:  peer.StatusConnected,
+					relayed: true,
+					latency: 10 * time.Millisecond,
+				},
+				"route2": {
+					status:  peer.StatusConnected,
+					relayed: false,
+					latency: 200 * time.Millisecond,
+				},
+			},
+			existingRoutes: map[route.ID]*route.Route{
+				"route1": {
+					ID:     "route1",
+					Metric: route.MaxMetric,
+					Peer:   "peer1",
+				},
+				"route2": {
+					ID:     "route2",
+					Metric: route.MaxMetric,
+					Peer:   "peer2",
+				},
+			},
+			currentRoute:    "route1",
+			expectedRouteID: "route2",
+		},
+		{
+			name: "better metric should replace the current peer with better latency",
+			statuses: map[route.ID]routerPeerStatus{
+				"route1": {
+					status:  peer.StatusConnected,
+					latency: 10 * time.Millisecond,
+				},
+				"route2": {
+					status:  peer.StatusConnected,
+					latency: 300 * time.Millisecond,
+				},
+			},
+			existingRoutes: map[route.ID]*route.Route{
+				"route1": {
+					ID:     "route1",
+					Metric: route.MaxMetric,
+					Peer:   "peer1",
+				},
+				"route2": {
+					ID:     "route2",
+					Metric: 5000,
+					Peer:   "peer2",
+				},
+			},
+			currentRoute:    "route1",
+			expectedRouteID: "route2",
+		},
 		{
 			name: "current chosen route doesn't exist anymore",
 			statuses: map[route.ID]routerPeerStatus{
@@ -828,3 +973,510 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 		})
 	}
 }
+
+// newTestWatcher builds a watcher over two routes of equal metric, with route1
+// as the currently chosen one.
+func newTestWatcher(t *testing.T, lastSwitch time.Time) *Watcher {
+	t.Helper()
+
+	routes := map[route.ID]*route.Route{
+		"route1": {ID: "route1", Metric: route.MaxMetric, Peer: "peer1"},
+		"route2": {ID: "route2", Metric: route.MaxMetric, Peer: "peer2"},
+	}
+
+	params := common.HandlerParams{
+		Route: &route.Route{Network: netip.MustParsePrefix("192.168.0.0/24")},
+	}
+
+	return &Watcher{
+		handler:       static.NewRoute(params),
+		routes:        routes,
+		currentChosen: routes["route1"],
+		lastSwitch:    lastSwitch,
+	}
+}
+
+func TestLatencySwitchDwellTime(t *testing.T) {
+	// a gain large enough to clear both switch margins
+	statuses := map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 300 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 100 * time.Millisecond},
+	}
+
+	t.Run("latency switch is held during the dwell time", func(t *testing.T) {
+		w := newTestWatcher(t, time.Now())
+
+		chosen, _ := w.getBestRouteFromStatuses(statuses)
+		assert.Equal(t, route.ID("route1"), chosen, "route should be kept while the dwell time has not elapsed")
+	})
+
+	t.Run("latency switch is allowed after the dwell time", func(t *testing.T) {
+		w := newTestWatcher(t, time.Now().Add(-latencySwitchMinDwell-time.Second))
+
+		chosen, _ := w.getBestRouteFromStatuses(statuses)
+		assert.Equal(t, route.ID("route2"), chosen, "route should switch once the dwell time has elapsed")
+	})
+
+	t.Run("categorical improvement ignores the dwell time", func(t *testing.T) {
+		w := newTestWatcher(t, time.Now())
+
+		// route1 is no longer connected, so the dwell time must not delay failover
+		chosen, _ := w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+			"route1": {status: peer.StatusIdle, latency: 10 * time.Millisecond},
+			"route2": {status: peer.StatusConnected, latency: 300 * time.Millisecond},
+		})
+		assert.Equal(t, route.ID("route2"), chosen, "failover must not be delayed by the dwell time")
+	})
+}
+
+// TestLatencySwitchNoiseMargin covers the noise aware margin: on a jittery path
+// the smoothed latencies of two equivalent peers keep drifting apart by tens of
+// milliseconds, and that difference must not move the route even though it
+// clears the fixed margins.
+// Measurements on slow paths must keep their order: a peer measured at 1.2s is
+// better than one at 2.5s, and a peer without a sample still ranks below both.
+func TestSlowPathLatenciesKeepTheirOrder(t *testing.T) {
+	w := newTestWatcher(t, time.Time{})
+
+	chosen, _ := w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 2500 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 1200 * time.Millisecond},
+	})
+	assert.Equal(t, route.ID("route2"), chosen, "a 1.3s gain on a 2.5s path must move the route")
+
+	unsampled := newRouteCandidate(w.routes["route1"], routerPeerStatus{status: peer.StatusConnected})
+	slow := newRouteCandidate(w.routes["route2"], routerPeerStatus{status: peer.StatusConnected, latency: 5 * time.Second})
+	assert.True(t, slow.betterThan(unsampled), "a measured peer must rank above one without a sample")
+}
+
+func TestLatencySwitchNoiseMargin(t *testing.T) {
+	t.Run("a gain within the measurement noise does not switch", func(t *testing.T) {
+		w := newTestWatcher(t, time.Time{})
+
+		chosen, _ := w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+			"route1": {status: peer.StatusConnected, latency: 120 * time.Millisecond, noise: 12 * time.Millisecond},
+			"route2": {status: peer.StatusConnected, latency: 80 * time.Millisecond, noise: 12 * time.Millisecond},
+		})
+		assert.Equal(t, route.ID("route1"), chosen, "a 40ms gain between two paths jittering by 12ms is not a real difference")
+	})
+
+	t.Run("a gain beyond the measurement noise switches", func(t *testing.T) {
+		w := newTestWatcher(t, time.Time{})
+
+		chosen, _ := w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+			"route1": {status: peer.StatusConnected, latency: 300 * time.Millisecond, noise: 12 * time.Millisecond},
+			"route2": {status: peer.StatusConnected, latency: 80 * time.Millisecond, noise: 12 * time.Millisecond},
+		})
+		assert.Equal(t, route.ID("route2"), chosen, "a 220ms gain stands out from 12ms of jitter")
+	})
+
+	// the held candidate is tracked so the decision is reported once per episode
+	// rather than on every re-evaluation
+	t.Run("holding a candidate back is tracked as one episode", func(t *testing.T) {
+		w := newTestWatcher(t, time.Time{})
+
+		within := map[route.ID]routerPeerStatus{
+			"route1": {status: peer.StatusConnected, latency: 120 * time.Millisecond, noise: 12 * time.Millisecond},
+			"route2": {status: peer.StatusConnected, latency: 80 * time.Millisecond, noise: 12 * time.Millisecond},
+		}
+
+		w.getBestRouteFromStatuses(within)
+		assert.Equal(t, route.ID("route2"), w.heldCandidate, "the held candidate should be recorded on the first hold")
+
+		w.getBestRouteFromStatuses(within)
+		assert.Equal(t, route.ID("route2"), w.heldCandidate, "a repeated hold of the same candidate stays the same episode")
+
+		// the smoothed latencies of two near-equal peers keep crossing: route2
+		// falling behind for a moment must not start a new episode
+		w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+			"route1": {status: peer.StatusConnected, latency: 80 * time.Millisecond, noise: 12 * time.Millisecond},
+			"route2": {status: peer.StatusConnected, latency: 120 * time.Millisecond, noise: 12 * time.Millisecond},
+		})
+		held := routeCandidate{id: "route2"}
+		assert.Equal(t, log.DebugLevel, w.holdLevel(held, holdLatencyMargin), "holding route2 back again after a crossing is a repeat")
+
+		// a hold reported long ago is worth reporting again
+		w.heldReported = w.heldReported.Add(-holdReportInterval - time.Second)
+		assert.Equal(t, log.InfoLevel, w.holdLevel(held, holdLatencyMargin), "a hold is reported again once the report interval passed")
+	})
+
+	t.Run("a stable path is decided by the fixed margins", func(t *testing.T) {
+		w := newTestWatcher(t, time.Time{})
+
+		// the same 40ms gain as the first case, on paths that barely vary
+		chosen, _ := w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+			"route1": {status: peer.StatusConnected, latency: 120 * time.Millisecond, noise: 300 * time.Microsecond},
+			"route2": {status: peer.StatusConnected, latency: 80 * time.Millisecond, noise: 300 * time.Microsecond},
+		})
+		assert.Equal(t, route.ID("route2"), chosen, "without jitter a 40ms gain on a 120ms path is a real improvement")
+	})
+}
+
+// recordingHandler records the allowed IPs calls a watcher makes.
+type recordingHandler struct {
+	added   []string
+	removed int
+}
+
+func (h *recordingHandler) String() string                 { return "recording" }
+func (h *recordingHandler) AddRoute(context.Context) error { return nil }
+func (h *recordingHandler) RemoveRoute() error             { return nil }
+func (h *recordingHandler) AddAllowedIPs(peerKey string) error {
+	h.added = append(h.added, peerKey)
+	return nil
+}
+
+func (h *recordingHandler) RemoveAllowedIPs() error {
+	h.removed++
+	return nil
+}
+
+func newRecalcWatcher(t *testing.T) (*Watcher, *recordingHandler) {
+	t.Helper()
+
+	handler := &recordingHandler{}
+	routes := map[route.ID]*route.Route{
+		"route1": {ID: "route1", Metric: route.MaxMetric, Peer: "peer1"},
+		"route2": {ID: "route2", Metric: route.MaxMetric, Peer: "peer2"},
+	}
+
+	return &Watcher{
+		statusRecorder: peer.NewRecorder("https://mgm"),
+		routes:         routes,
+		handler:        handler,
+	}, handler
+}
+
+// TestRecalculateRoutesAssignsIdlePeer covers the lazy connection path: an idle
+// routing peer has no established connection yet, but must still receive the
+// allowed IPs so that traffic for the route can trigger the connection.
+func TestRecalculateRoutesAssignsIdlePeer(t *testing.T) {
+	w, handler := newRecalcWatcher(t)
+
+	err := w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusIdle},
+	})
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{"peer1"}, handler.added, "an idle routing peer should still get the allowed IPs")
+	require.NotNil(t, w.currentChosen, "an idle peer should be chosen when it is the only candidate")
+	assert.Equal(t, route.ID("route1"), w.currentChosen.ID)
+}
+
+func TestRecalculateRoutesDwellTime(t *testing.T) {
+	w, handler := newRecalcWatcher(t)
+
+	// route2 has no status yet, so route1 is the only candidate
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 300 * time.Millisecond},
+	}))
+	require.Equal(t, []string{"peer1"}, handler.added)
+	require.True(t, w.lastSwitch.IsZero(), "the initial assignment is not latency-driven and must not arm the dwell time")
+
+	betterRoute2 := map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 300 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 100 * time.Millisecond},
+	}
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, betterRoute2))
+	assert.Equal(t, []string{"peer1", "peer2"}, handler.added, "a latency switch with no recent switch should be allowed")
+	assert.Equal(t, 1, handler.removed)
+	require.False(t, w.lastSwitch.IsZero(), "a latency-driven switch must arm the dwell time")
+
+	// the peers trade places: switching back is again latency-only, but now
+	// inside the dwell window
+	betterRoute1 := map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 100 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 300 * time.Millisecond},
+	}
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, betterRoute1))
+	assert.Equal(t, []string{"peer1", "peer2"}, handler.added, "the route should be held during the dwell time")
+
+	w.lastSwitch = w.lastSwitch.Add(-latencySwitchMinDwell - time.Second)
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, betterRoute1))
+	assert.Equal(t, []string{"peer1", "peer2", "peer1"}, handler.added, "the route should switch once the dwell time elapsed")
+	assert.Equal(t, 2, handler.removed)
+
+	// the switch re-armed the dwell time, which must not delay failover
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnecting, latency: 100 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 300 * time.Millisecond},
+	}))
+	assert.Equal(t, []string{"peer1", "peer2", "peer1", "peer2"}, handler.added, "losing the routing peer must fail over regardless of the dwell time")
+	assert.Equal(t, 3, handler.removed)
+}
+
+// TestFailoverDoesNotArmDwellTime covers the recovery path: a failover to a
+// worse peer must not start the dwell time, or traffic would be stuck on the
+// worse path for the full dwell after the better peer comes back.
+func TestFailoverDoesNotArmDwellTime(t *testing.T) {
+	w, handler := newRecalcWatcher(t)
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 20 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 200 * time.Millisecond},
+	}))
+	require.Equal(t, []string{"peer1"}, handler.added)
+
+	// peer1 blips into connecting: unguarded failover to the worse peer2
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnecting, latency: 20 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 200 * time.Millisecond},
+	}))
+	require.Equal(t, []string{"peer1", "peer2"}, handler.added, "failover to the remaining peer should happen")
+	require.True(t, w.lastSwitch.IsZero(), "a failover must not arm the dwell time")
+
+	// peer1 recovers seconds later: the return is latency-only but must not be
+	// held back by a dwell the failover would have armed
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 20 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 200 * time.Millisecond},
+	}))
+	assert.Equal(t, []string{"peer1", "peer2", "peer1"}, handler.added, "the recovered better peer should take the route back immediately")
+}
+
+// blockingHandler records the allowed IPs calls in order and blocks the first
+// AddAllowedIPs until released, to hold a recalculation in flight.
+type blockingHandler struct {
+	mu      sync.Mutex
+	calls   []string
+	entered chan struct{}
+	release chan struct{}
+	once    sync.Once
+}
+
+func (h *blockingHandler) String() string                 { return "blocking" }
+func (h *blockingHandler) AddRoute(context.Context) error { return nil }
+func (h *blockingHandler) RemoveRoute() error             { return nil }
+
+func (h *blockingHandler) AddAllowedIPs(peerKey string) error {
+	h.once.Do(func() {
+		close(h.entered)
+		<-h.release
+	})
+	h.record("add " + peerKey)
+	return nil
+}
+
+func (h *blockingHandler) RemoveAllowedIPs() error {
+	h.record("remove")
+	return nil
+}
+
+func (h *blockingHandler) record(call string) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.calls = append(h.calls, call)
+}
+
+// TestStopWaitsForStart covers the shutdown ordering: Stop must not clean up
+// while the Start loop is still inside a recalculation, or it would remove the
+// allowed IPs before the recalculation adds them and leave them behind.
+func TestStopWaitsForStart(t *testing.T) {
+	w, _ := newRecalcWatcher(t)
+	handler := &blockingHandler{entered: make(chan struct{}), release: make(chan struct{})}
+	w.handler = handler
+
+	require.NoError(t, w.statusRecorder.AddPeer("peer1", "peer1.netbird.cloud", "100.64.0.1", ""))
+	require.NoError(t, w.statusRecorder.UpdatePeerState(peer.State{PubKey: "peer1", ConnStatus: peer.StatusConnected}))
+
+	w.ctx, w.cancel = context.WithCancel(context.Background())
+	w.done = make(chan struct{})
+	w.peerStateUpdate = make(chan struct{})
+	go w.Start()
+
+	// the unbuffered send returns once the loop took the notification, so the
+	// loop is running and about to recalculate
+	w.peerStateUpdate <- struct{}{}
+	select {
+	case <-handler.entered:
+	case <-time.After(2 * time.Second):
+		t.Fatal("the notification should start a recalculation")
+	}
+
+	finished := make(chan struct{})
+	go func() {
+		w.Stop()
+		close(finished)
+	}()
+
+	select {
+	case <-finished:
+		t.Fatal("Stop must wait for the recalculation in flight")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	close(handler.release)
+	select {
+	case <-finished:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Stop should return once the Start loop has exited")
+	}
+
+	handler.mu.Lock()
+	defer handler.mu.Unlock()
+	assert.Equal(t, []string{"add peer1", "remove"}, handler.calls, "Stop must clean up after the recalculation, not before it")
+}
+
+// A notification carries no peer states: every routing peer has its own
+// subscription and forwarder feeding the same channel, so a snapshot could
+// arrive after a newer one. The watcher acts on the states in the recorder as
+// of when it handles the notification.
+func TestPeerStateNotificationUsesRecordedStates(t *testing.T) {
+	w, handler := newRecalcWatcher(t)
+
+	recorder := w.statusRecorder
+	for _, key := range []string{"peer1", "peer2"} {
+		require.NoError(t, recorder.AddPeer(key, key+".netbird.cloud", "100.64.0.1", ""))
+	}
+	require.NoError(t, recorder.UpdatePeerICEState(peer.State{PubKey: "peer1", ConnStatus: peer.StatusConnected}))
+	require.NoError(t, recorder.UpdatePeerICEState(peer.State{PubKey: "peer2", ConnStatus: peer.StatusConnecting}))
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, w.getRouterPeerStatuses()))
+	require.Equal(t, []string{"peer1"}, handler.added)
+
+	w.ctx, w.cancel = context.WithCancel(context.Background())
+	w.done = make(chan struct{})
+	w.peerStateUpdate = make(chan struct{})
+	go w.Start()
+
+	require.NoError(t, recorder.UpdatePeerICEState(peer.State{PubKey: "peer1", ConnStatus: peer.StatusConnecting}))
+	require.NoError(t, recorder.UpdatePeerICEState(peer.State{PubKey: "peer2", ConnStatus: peer.StatusConnected}))
+	w.peerStateUpdate <- struct{}{}
+	w.Stop()
+
+	assert.Equal(t, []string{"peer1", "peer2"}, handler.added, "the notification must move the route according to the recorded states")
+}
+
+// While the watcher is busy, for instance inside a recalculation, the peer state
+// forwarder must keep draining its subscription. Otherwise the subscription
+// buffer fills up and the peer's next state update blocks its caller, the
+// peer's connection handling, until the watcher is free again.
+func TestPeerStateForwarderDoesNotBlockWhileWatcherIsBusy(t *testing.T) {
+	w, _ := newRecalcWatcher(t)
+	w.peerStateUpdate = newPeerStateUpdate()
+
+	recorder := w.statusRecorder
+	require.NoError(t, recorder.AddPeer("peer1", "peer1.netbird.cloud", "100.64.0.1", ""))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	closer := make(chan struct{})
+	go w.watchPeerStatusChanges(ctx, "peer1", w.peerStateUpdate, closer)
+
+	// wait for the forwarder's subscription to exist, so the updates below reach it
+	require.Eventually(t, func() bool {
+		require.NoError(t, recorder.UpdatePeerICEState(peer.State{PubKey: "peer1", ConnStatus: peer.StatusConnected}))
+		require.NoError(t, recorder.UpdatePeerICEState(peer.State{PubKey: "peer1", ConnStatus: peer.StatusConnecting}))
+		select {
+		case <-w.peerStateUpdate:
+			return true
+		case <-time.After(50 * time.Millisecond):
+			return false
+		}
+	}, 2*time.Second, 10*time.Millisecond, "the forwarder must pass on a notification")
+
+	// nobody reads peerStateUpdate: the watcher is busy
+	updated := make(chan struct{})
+	go func() {
+		defer close(updated)
+		for i := 0; i < 50; i++ {
+			status := peer.StatusConnected
+			if i%2 == 1 {
+				status = peer.StatusConnecting
+			}
+			if err := recorder.UpdatePeerICEState(peer.State{PubKey: "peer1", ConnStatus: status}); err != nil {
+				t.Errorf("update peer state: %v", err)
+				return
+			}
+		}
+	}()
+
+	select {
+	case <-updated:
+	case <-time.After(2 * time.Second):
+		t.Fatal("peer state updates must not block while the watcher is busy")
+	}
+}
+
+// TestStopWithoutStart covers a watcher whose Start goroutine never ran, or ran
+// only after Stop: Stop must return instead of waiting for a loop that does not
+// exist, and a late Start must not begin working on a stopped watcher.
+func TestStopWithoutStart(t *testing.T) {
+	w := NewWatcher(WatcherConfig{
+		Context:        context.Background(),
+		StatusRecorder: peer.NewRecorder("https://mgm"),
+		Handler:        &recordingHandler{},
+	})
+
+	finished := make(chan struct{})
+	go func() {
+		w.Stop()
+		close(finished)
+	}()
+
+	select {
+	case <-finished:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Stop must return when Start was never called")
+	}
+
+	started := make(chan struct{})
+	go func() {
+		w.Start()
+		close(started)
+	}()
+
+	select {
+	case <-started:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Start after Stop must return immediately")
+	}
+}
+
+// TestUnassignedNetworkReportedOncePerEpisode covers the no-routing-peer log:
+// the periodic re-evaluation hits the same branch every tick, so the episode
+// must be reported once, and again only after a route was assigned in between.
+func TestUnassignedNetworkReportedOncePerEpisode(t *testing.T) {
+	w, _ := newRecalcWatcher(t)
+
+	unassigned := map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnecting},
+	}
+	assigned := map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 10 * time.Millisecond},
+	}
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, unassigned))
+	assert.True(t, w.reportedUnassigned, "the first unassigned evaluation should be reported")
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, unassigned))
+	assert.True(t, w.reportedUnassigned, "repeats stay within the same episode")
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, assigned))
+	assert.False(t, w.reportedUnassigned, "an assigned route ends the episode")
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, unassigned))
+	assert.True(t, w.reportedUnassigned, "losing the route again starts a new episode")
+}
+
+// TestLatencyJitterDoesNotSwitchRoute walks two equivalent peers through jitter
+// around the same mean. Neither peer is genuinely better, so the route must not
+// move at any point.
+func TestLatencyJitterDoesNotSwitchRoute(t *testing.T) {
+	w := newTestWatcher(t, time.Time{})
+
+	jitter := []int{-6, 3, -4, 8, -2, 5, -7, 1, 6, -3}
+	for i, offsetMs := range jitter {
+		offset := time.Duration(offsetMs) * time.Millisecond
+		statuses := map[route.ID]routerPeerStatus{
+			"route1": {status: peer.StatusConnected, latency: 20*time.Millisecond + offset},
+			"route2": {status: peer.StatusConnected, latency: 20*time.Millisecond - offset},
+		}
+
+		chosen, _ := w.getBestRouteFromStatuses(statuses)
+		require.Equal(t, route.ID("route1"), chosen, "route changed on jitter at sample %d", i)
+	}
+}

client/internal/routemanager/client/env.go

@@ -0,0 +1,77 @@
+package client
+
+import (
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	// envRelayedSwitchDelay sets how long a routing peer that dropped from a
+	// direct to a relayed connection while carrying the route is kept before the
+	// route moves to an equivalent peer with a direct connection. Takes a
+	// duration ("30s", "0" to switch at once) or "never" to only leave such a
+	// peer when it disconnects. A routing peer that was already relayed when
+	// chosen always gives way to a direct one at once.
+	envRelayedSwitchDelay = "NB_HA_RELAYED_SWITCH_DELAY"
+
+	relayedSwitchDelayNever = "never"
+
+	// defaultRelayedSwitchDelay covers the common case of a direct connection
+	// that drops to relay and recovers within seconds. Moving the route
+	// resets every masqueraded flow using it, staying on the relay does not.
+	defaultRelayedSwitchDelay = 30 * time.Second
+)
+
+// relayedSwitchPolicy is the resolved envRelayedSwitchDelay setting.
+type relayedSwitchPolicy struct {
+	delay time.Duration
+	// never keeps a connected relayed routing peer regardless of delay
+	never bool
+}
+
+func (p relayedSwitchPolicy) String() string {
+	if p.never {
+		return relayedSwitchDelayNever
+	}
+	return p.delay.String()
+}
+
+var (
+	relayedSwitchOnce     sync.Once
+	resolvedRelayedSwitch relayedSwitchPolicy
+)
+
+// relayedSwitch returns the process-wide relayed switch policy, read from the
+// environment on first use.
+func relayedSwitch() relayedSwitchPolicy {
+	relayedSwitchOnce.Do(func() {
+		resolvedRelayedSwitch = parseRelayedSwitchPolicy(os.Getenv(envRelayedSwitchDelay))
+		log.Infof("routing peer relayed switch delay: %s", resolvedRelayedSwitch)
+	})
+	return resolvedRelayedSwitch
+}
+
+func parseRelayedSwitchPolicy(raw string) relayedSwitchPolicy {
+	def := relayedSwitchPolicy{delay: defaultRelayedSwitchDelay}
+
+	val := strings.TrimSpace(raw)
+	if val == "" {
+		return def
+	}
+	if strings.EqualFold(val, relayedSwitchDelayNever) {
+		return relayedSwitchPolicy{never: true}
+	}
+
+	delay, err := time.ParseDuration(val)
+	if err != nil || delay < 0 {
+		log.Warnf("invalid %s value %q, using the default of %s: want a duration or %q",
+			envRelayedSwitchDelay, raw, def, relayedSwitchDelayNever)
+		return def
+	}
+
+	return relayedSwitchPolicy{delay: delay}
+}

client/internal/routemanager/client/relayed_switch_test.go

@@ -0,0 +1,358 @@
+package client
+
+import (
+	"context"
+	"net/netip"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/proto"
+	"github.com/netbirdio/netbird/route"
+)
+
+func TestParseRelayedSwitchPolicy(t *testing.T) {
+	testCases := []struct {
+		raw  string
+		want relayedSwitchPolicy
+	}{
+		{raw: "", want: relayedSwitchPolicy{delay: defaultRelayedSwitchDelay}},
+		{raw: "  ", want: relayedSwitchPolicy{delay: defaultRelayedSwitchDelay}},
+		{raw: "0", want: relayedSwitchPolicy{}},
+		{raw: "10s", want: relayedSwitchPolicy{delay: 10 * time.Second}},
+		{raw: " 2m ", want: relayedSwitchPolicy{delay: 2 * time.Minute}},
+		{raw: "never", want: relayedSwitchPolicy{never: true}},
+		{raw: "NEVER", want: relayedSwitchPolicy{never: true}},
+		// unrecognised values keep the default rather than guessing
+		{raw: "-5s", want: relayedSwitchPolicy{delay: defaultRelayedSwitchDelay}},
+		{raw: "soon", want: relayedSwitchPolicy{delay: defaultRelayedSwitchDelay}},
+		{raw: "30", want: relayedSwitchPolicy{delay: defaultRelayedSwitchDelay}},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.raw, func(t *testing.T) {
+			assert.Equal(t, tc.want, parseRelayedSwitchPolicy(tc.raw), "policy parsed from %q", tc.raw)
+		})
+	}
+}
+
+// newRelayedTestWatcher is a watcher over two equal-metric routes, currently on
+// route1, for the given policy.
+func newRelayedTestWatcher(t *testing.T, policy relayedSwitchPolicy) *Watcher {
+	t.Helper()
+
+	w := newTestWatcher(t, time.Time{})
+	w.relayedSwitch = policy
+	t.Cleanup(func() {
+		if w.relayedHold != nil {
+			w.relayedHold.Stop()
+		}
+	})
+	return w
+}
+
+// degrade walks the current routing peer route1 from direct to relayed, the way
+// it is seen when a direct connection carrying the route drops to relay.
+func degrade(t *testing.T, w *Watcher) {
+	t.Helper()
+
+	chosen, _ := w.getBestRouteFromStatuses(currentDirect)
+	require.Equal(t, route.ID("route1"), chosen, "route1 must carry the route while direct")
+
+	chosen, _ = w.getBestRouteFromStatuses(currentRelayed)
+	require.Equal(t, route.ID("route1"), chosen, "a routing peer that just dropped to relay must be kept")
+	require.False(t, w.degradedSince.IsZero(), "the drop to relay must be recorded")
+}
+
+var (
+	// route1 is relayed, route2 is an equivalent peer with a direct connection
+	currentRelayed = map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, relayed: true, latency: 50 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 60 * time.Millisecond},
+	}
+	currentDirect = map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 50 * time.Millisecond},
+		"route2": {status: peer.StatusConnected, latency: 60 * time.Millisecond},
+	}
+)
+
+func TestDegradedRoutingPeerIsKeptDuringDelay(t *testing.T) {
+	w := newRelayedTestWatcher(t, relayedSwitchPolicy{delay: 30 * time.Second})
+
+	degrade(t, w)
+	require.NotNil(t, w.relayedHold, "the pending switch must arm a timer so it is applied when due")
+
+	// the direct connection comes back within the delay: nothing moved, and the
+	// degraded period ends
+	chosen, _ := w.getBestRouteFromStatuses(currentDirect)
+	assert.Equal(t, route.ID("route1"), chosen, "the recovered routing peer must stay chosen")
+	assert.True(t, w.degradedSince.IsZero(), "the degraded period must end once the peer is direct again")
+}
+
+func TestDegradedRoutingPeerIsReplacedAfterDelay(t *testing.T) {
+	w := newRelayedTestWatcher(t, relayedSwitchPolicy{delay: 30 * time.Second})
+
+	degrade(t, w)
+	w.degradedSince = w.degradedSince.Add(-31 * time.Second)
+
+	chosen, _ := w.getBestRouteFromStatuses(currentRelayed)
+	assert.Equal(t, route.ID("route2"), chosen, "a routing peer relayed for longer than the delay must give way to a direct one")
+	assert.Equal(t, switchReasonDirect, w.switchReason, "the switch reason must name the relayed connection")
+}
+
+// A routing peer that was relayed when it was chosen is not degraded: when an
+// equivalent direct peer shows up the route must upgrade at once instead of
+// staying on the slower path for the delay.
+func TestRelayedRoutingPeerUpgradesAtOnce(t *testing.T) {
+	w := newRelayedTestWatcher(t, relayedSwitchPolicy{never: true})
+
+	// route1 was chosen while relayed and route2 was still connecting
+	chosen, _ := w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, relayed: true},
+		"route2": {status: peer.StatusConnecting},
+	})
+	require.Equal(t, route.ID("route1"), chosen)
+	require.True(t, w.degradedSince.IsZero(), "a peer relayed from the start is not degraded")
+
+	chosen, _ = w.getBestRouteFromStatuses(currentRelayed)
+	assert.Equal(t, route.ID("route2"), chosen, "a direct peer must take over from one that was relayed all along")
+	assert.Equal(t, switchReasonDirect, w.switchReason)
+	assert.Nil(t, w.relayedHold, "an upgrade must not schedule anything")
+}
+
+// An idle routing peer woken by traffic whose relay connection comes up before
+// the direct one was never direct: it is not degraded, and must give way to an
+// equivalent direct peer at once.
+func TestIdleRoutingPeerWokenOverRelayUpgradesAtOnce(t *testing.T) {
+	w := newRelayedTestWatcher(t, relayedSwitchPolicy{never: true})
+
+	chosen, _ := w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusIdle},
+		"route2": {status: peer.StatusIdle},
+	})
+	require.Equal(t, route.ID("route1"), chosen)
+
+	chosen, _ = w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, relayed: true},
+		"route2": {status: peer.StatusConnecting},
+	})
+	require.Equal(t, route.ID("route1"), chosen)
+	require.True(t, w.degradedSince.IsZero(), "waking up over relay is not a drop from direct")
+
+	chosen, _ = w.getBestRouteFromStatuses(currentRelayed)
+	assert.Equal(t, route.ID("route2"), chosen, "a direct peer must take over from one that was never direct")
+	assert.Nil(t, w.relayedHold, "an upgrade must not schedule anything")
+}
+
+// A routing peer that drops to relay again, after recovering from an earlier
+// drop, is a new degradation and must be reported at info level again.
+func TestRepeatedDegradationIsReportedAgain(t *testing.T) {
+	w := newRelayedTestWatcher(t, relayedSwitchPolicy{delay: 30 * time.Second})
+
+	degrade(t, w)
+	require.Equal(t, route.ID("route2"), w.heldCandidate, "the first drop must be reported")
+
+	// the first report is well within the report interval
+	w.heldReported = w.heldReported.Add(-time.Minute)
+	firstReport := w.heldReported
+
+	degrade(t, w)
+	assert.True(t, w.heldReported.After(firstReport), "a new drop to relay must be reported, not treated as a repeat")
+}
+
+func TestDegradedRoutingPeerNeverPolicy(t *testing.T) {
+	w := newRelayedTestWatcher(t, relayedSwitchPolicy{never: true})
+
+	degrade(t, w)
+	w.degradedSince = w.degradedSince.Add(-time.Hour)
+
+	chosen, _ := w.getBestRouteFromStatuses(currentRelayed)
+	assert.Equal(t, route.ID("route1"), chosen, "with never, a degraded routing peer is kept while connected")
+	assert.Nil(t, w.relayedHold, "with never there is nothing to schedule")
+}
+
+func TestDegradedRoutingPeerZeroDelaySwitchesAtOnce(t *testing.T) {
+	w := newRelayedTestWatcher(t, relayedSwitchPolicy{})
+
+	chosen, _ := w.getBestRouteFromStatuses(currentDirect)
+	require.Equal(t, route.ID("route1"), chosen)
+
+	chosen, _ = w.getBestRouteFromStatuses(currentRelayed)
+	assert.Equal(t, route.ID("route2"), chosen, "a zero delay must keep the immediate switch to a direct peer")
+}
+
+// The delay only covers a candidate that is better solely by being direct. A
+// better metric or a lost routing peer must move the route at once.
+func TestRelayedDelayDoesNotHoldOtherSwitches(t *testing.T) {
+	t.Run("better metric", func(t *testing.T) {
+		w := newRelayedTestWatcher(t, relayedSwitchPolicy{never: true})
+		degrade(t, w)
+		w.routes["route2"].Metric = route.MaxMetric - 1
+
+		chosen, _ := w.getBestRouteFromStatuses(currentRelayed)
+		assert.Equal(t, route.ID("route2"), chosen, "a better metric must win over a degraded peer immediately")
+		assert.Equal(t, switchReasonMetric, w.switchReason)
+	})
+
+	t.Run("current peer disconnected", func(t *testing.T) {
+		w := newRelayedTestWatcher(t, relayedSwitchPolicy{never: true})
+		degrade(t, w)
+
+		chosen, _ := w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+			"route1": {status: peer.StatusConnecting, relayed: true},
+			"route2": {status: peer.StatusConnected, latency: 60 * time.Millisecond},
+		})
+		assert.Equal(t, route.ID("route2"), chosen, "a routing peer that is gone must be replaced regardless of the delay")
+		assert.Equal(t, switchReasonUnavailable, w.switchReason)
+	})
+
+	t.Run("both relayed", func(t *testing.T) {
+		w := newRelayedTestWatcher(t, relayedSwitchPolicy{delay: 30 * time.Second})
+
+		chosen, _ := w.getBestRouteFromStatuses(map[route.ID]routerPeerStatus{
+			"route1": {status: peer.StatusConnected, relayed: true},
+			"route2": {status: peer.StatusConnected, relayed: true},
+		})
+		assert.Equal(t, route.ID("route1"), chosen, "equally relayed peers are a tie and the current one stays")
+		assert.Nil(t, w.relayedHold, "a tie must not schedule a switch")
+	})
+}
+
+// TestRelayedHoldTimerAppliesSwitch drives the Start loop: once the delay has
+// passed, the held switch must be applied without waiting for a peer update or
+// the periodic re-evaluation.
+func TestRelayedHoldTimerAppliesSwitch(t *testing.T) {
+	w, handler := newRecalcWatcher(t)
+	w.relayedSwitch = relayedSwitchPolicy{delay: 50 * time.Millisecond}
+
+	recorder := w.statusRecorder
+	for _, key := range []string{"peer1", "peer2"} {
+		require.NoError(t, recorder.AddPeer(key, key+".netbird.cloud", "100.64.0.1", ""))
+	}
+	require.NoError(t, recorder.UpdatePeerState(peer.State{PubKey: "peer1", ConnStatus: peer.StatusConnected, Relayed: true}))
+	require.NoError(t, recorder.UpdatePeerState(peer.State{PubKey: "peer2", ConnStatus: peer.StatusConnected}))
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected},
+	}))
+	require.Equal(t, []string{"peer1"}, handler.added)
+
+	// route1 drops to relay while route2 is direct: held for the delay
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, relayed: true},
+		"route2": {status: peer.StatusConnected},
+	}))
+	require.Equal(t, []string{"peer1"}, handler.added, "the switch must be held while the delay runs")
+
+	w.ctx, w.cancel = context.WithCancel(context.Background())
+	w.done = make(chan struct{})
+	go w.Start()
+	t.Cleanup(w.Stop)
+
+	// the loop owns the watcher state now, so observe the switch through the
+	// event it publishes rather than reading its fields
+	require.Eventually(t, func() bool {
+		for _, e := range recorder.GetEventHistory() {
+			if e.Message == "Routing peer changed" && e.Metadata["peer"] == "peer2" {
+				return true
+			}
+		}
+		return false
+	}, 2*time.Second, 10*time.Millisecond, "the held switch must be applied once the delay passed")
+}
+
+// A route update can keep a route's ID and hand it to a different routing peer.
+// That is a switch like any other: it is reported, and the new peer is tracked
+// from its own state rather than inheriting the previous peer's.
+func TestRouteReassignedToDifferentPeerIsASwitch(t *testing.T) {
+	w, handler := newRecalcWatcher(t)
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected},
+	}))
+	require.Equal(t, []string{"peer1"}, handler.added)
+
+	reassigned := *w.routes["route1"]
+	reassigned.Peer = "peer3"
+	w.routes["route1"] = &reassigned
+
+	require.NoError(t, w.recalculateRoutes(reasonRouteUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, relayed: true},
+	}))
+	assert.Equal(t, []string{"peer1", "peer3"}, handler.added, "the allowed IPs must move to the new routing peer")
+	assert.Equal(t, "peer3", w.trackedPeer, "the new routing peer must be tracked")
+	assert.False(t, w.trackedDirect, "the new routing peer is tracked from its own state")
+
+	events := w.statusRecorder.GetEventHistory()
+	require.Len(t, events, 1, "the reassignment must publish a switch event")
+	assert.Equal(t, "Routing peer changed", events[0].Message)
+	assert.Equal(t, "peer1", events[0].Metadata["previous_peer"])
+	assert.Equal(t, "peer3", events[0].Metadata["peer"])
+	assert.Equal(t, string(switchReasonRouteChanged), events[0].Metadata["reason"])
+}
+
+// Exit nodes report their own connect and disconnect events, but those carry
+// no reason: the switch event is published for default routes too.
+func TestDefaultRouteSwitchPublishesEvent(t *testing.T) {
+	w, _ := newRecalcWatcher(t)
+	for _, r := range w.routes {
+		r.Network = netip.MustParsePrefix("0.0.0.0/0")
+	}
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected},
+	}))
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnecting},
+		"route2": {status: peer.StatusConnected},
+	}))
+
+	var switchEvents []*proto.SystemEvent
+	for _, e := range w.statusRecorder.GetEventHistory() {
+		if e.Message == "Routing peer changed" {
+			switchEvents = append(switchEvents, e)
+		}
+	}
+	require.Len(t, switchEvents, 1, "an exit node switch must publish a switch event")
+	assert.Equal(t, string(switchReasonUnavailable), switchEvents[0].Metadata["reason"])
+	assert.Empty(t, switchEvents[0].UserMessage, "the switch event must not notify the user a second time")
+}
+
+func TestRoutingPeerSwitchPublishesEvent(t *testing.T) {
+	w, _ := newRecalcWatcher(t)
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnected, latency: 20 * time.Millisecond},
+	}))
+	assert.Empty(t, w.statusRecorder.GetEventHistory(), "the initial assignment is not a switch")
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnecting},
+		"route2": {status: peer.StatusConnected, latency: 20 * time.Millisecond},
+	}))
+
+	events := w.statusRecorder.GetEventHistory()
+	require.Len(t, events, 1, "a switch must publish exactly one event")
+	assert.Equal(t, proto.SystemEvent_WARNING, events[0].Severity)
+	assert.Equal(t, "Routing peer changed", events[0].Message)
+	assert.Equal(t, map[string]string{
+		"network":       "recording",
+		"id":            "",
+		"previous_peer": "peer1",
+		"peer":          "peer2",
+		"reason":        string(switchReasonUnavailable),
+	}, events[0].Metadata, "the event must name both peers and the reason")
+
+	require.NoError(t, w.recalculateRoutes(reasonPeerUpdate, map[route.ID]routerPeerStatus{
+		"route1": {status: peer.StatusConnecting},
+		"route2": {status: peer.StatusConnecting},
+	}))
+
+	events = w.statusRecorder.GetEventHistory()
+	require.Len(t, events, 2, "losing the last routing peer must publish an event")
+	assert.Equal(t, "No routing peer available", events[1].Message)
+	assert.Equal(t, "peer2", events[1].Metadata["previous_peer"])
+}