Merge branch 'main' into refactor/relay-foreign-cache

# Conflicts:
#	shared/relay/client/manager.go
This commit is contained in:
Zoltán Papp
2026-07-21 11:27:09 +02:00
789 changed files with 101091 additions and 9250 deletions
+124 -15
View File
@@ -6,6 +6,7 @@ import (
"net"
"net/netip"
"runtime"
"slices"
"sync"
"time"
@@ -29,6 +30,11 @@ import (
relayClient "github.com/netbirdio/netbird/shared/relay/client"
)
// wgTimeoutEscalationThreshold is the number of consecutive WireGuard
// handshake timeouts after which the rosenpass state for the peer is
// considered desynced and gets reset.
const wgTimeoutEscalationThreshold = 3
// MetricsRecorder is an interface for recording peer connection metrics
type MetricsRecorder interface {
RecordConnectionStages(
@@ -117,6 +123,9 @@ type Conn struct {
wgWatcher *WGWatcher
wgWatcherWg sync.WaitGroup
wgWatcherCancel context.CancelFunc
// wgTimeouts counts consecutive WireGuard handshake timeouts without a
// successful handshake in between. Guarded by mu.
wgTimeouts int
// used to store the remote Rosenpass key for Relayed connection in case of connection update from ice
rosenpassRemoteKey []byte
@@ -136,6 +145,39 @@ type Conn struct {
// Connection stage timestamps for metrics
metricsRecorder MetricsRecorder
metricsStages *MetricsStages
// pendingFirstPacket is the lazyconn-captured handshake init, replayed once the real
// transport is up.
pendingFirstPacket []byte
}
// injectPendingFirstPacket replays the captured handshake through the proxy if present, else
// directly through the ICE conn. The packet is cleared only after a successful write, so a failed
// or transport-less attempt leaves it available for a later reinjection. Caller must hold conn.mu.
func (conn *Conn) injectPendingFirstPacket(proxy wgproxy.Proxy, directConn net.Conn) {
pkt := conn.pendingFirstPacket
if len(pkt) == 0 {
return
}
switch {
case proxy != nil:
if err := proxy.InjectPacket(pkt); err != nil {
conn.Log.Debugf("failed to reinject captured first packet via proxy: %v", err)
return
}
case directConn != nil:
if _, err := directConn.Write(pkt); err != nil {
conn.Log.Debugf("failed to reinject captured first packet via direct conn: %v", err)
return
}
default:
conn.Log.Debugf("no transport available to reinject captured first packet")
return
}
conn.pendingFirstPacket = nil
conn.Log.Debugf("reinjected captured first packet (%d bytes)", len(pkt))
}
// NewConn creates a new not opened Conn to the remote peer.
@@ -161,7 +203,6 @@ func NewConn(config ConnConfig, services ServiceDependencies) (*Conn, error) {
statusICE: worker.NewAtomicStatus(),
dumpState: dumpState,
endpointUpdater: NewEndpointUpdater(connLog, config.WgConfig, isController(config)),
wgWatcher: NewWGWatcher(connLog, config.WgConfig.WgInterface, config.Key, dumpState),
metricsRecorder: services.MetricsRecorder,
}
@@ -172,6 +213,16 @@ func NewConn(config ConnConfig, services ServiceDependencies) (*Conn, error) {
// It will try to establish a connection using ICE and in parallel with relay. The higher priority connection type will
// be used.
func (conn *Conn) Open(engineCtx context.Context) error {
return conn.open(engineCtx, nil)
}
// OpenWithFirstPacket opens the connection like Open and stashes firstPacket to be replayed once
// the real transport is established. The packet is retained only on a successful open.
func (conn *Conn) OpenWithFirstPacket(engineCtx context.Context, firstPacket []byte) error {
return conn.open(engineCtx, firstPacket)
}
func (conn *Conn) open(engineCtx context.Context, firstPacket []byte) error {
conn.mu.Lock()
defer conn.mu.Unlock()
@@ -227,6 +278,9 @@ func (conn *Conn) Open(engineCtx context.Context) error {
defer conn.wg.Done()
conn.guard.Start(conn.ctx, conn.onGuardEvent)
}()
if len(firstPacket) > 0 {
conn.pendingFirstPacket = slices.Clone(firstPacket)
}
conn.opened = true
return nil
}
@@ -423,6 +477,8 @@ func (conn *Conn) onICEConnectionIsReady(priority conntype.ConnPriority, iceConn
conn.wgProxyRelay.RedirectAs(ep)
}
conn.injectPendingFirstPacket(wgProxy, iceConnInfo.RemoteConn)
conn.currentConnPriority = priority
conn.statusICE.SetConnected()
conn.updateIceState(iceConnInfo, updateTime)
@@ -546,6 +602,8 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) {
wgConfigWorkaround()
conn.injectPendingFirstPacket(wgProxy, nil)
conn.rosenpassRemoteKey = rci.rosenpassPubKey
conn.currentConnPriority = conntype.Relay
conn.statusRelay.SetConnected()
@@ -612,11 +670,12 @@ func (conn *Conn) onGuardEvent() {
}
}
func (conn *Conn) onWGDisconnected() {
func (conn *Conn) onWGDisconnected(watcherCtx context.Context) {
conn.mu.Lock()
defer conn.mu.Unlock()
if conn.ctx.Err() != nil {
// watcherCtx guards against a stale watcher tearing down a connection that already superseded it.
if conn.ctx.Err() != nil || watcherCtx.Err() != nil {
return
}
@@ -632,6 +691,29 @@ func (conn *Conn) onWGDisconnected() {
default:
conn.Log.Debugf("No active connection to close on WG timeout")
}
conn.escalateWGTimeoutLocked()
}
// escalateWGTimeoutLocked resets the peer's rosenpass state after repeated
// handshake timeouts. With rosenpass enabled, persistent timeouts mean the
// preshared keys have desynced; the renewal exchange runs over the dead
// tunnel and cannot resync them. Reporting the peer disconnected drops its
// rosenpass state, so the next connection configuration programs the
// rendezvous key and the tunnel can bootstrap again. Callers must hold mu.
func (conn *Conn) escalateWGTimeoutLocked() {
if conn.config.RosenpassConfig.PubKey == nil {
return
}
conn.wgTimeouts++
if conn.wgTimeouts < wgTimeoutEscalationThreshold || conn.onDisconnected == nil {
return
}
conn.wgTimeouts = 0
conn.Log.Warnf("%d consecutive WireGuard handshake timeouts, resetting rosenpass state for peer", wgTimeoutEscalationThreshold)
conn.onDisconnected(conn.config.WgConfig.RemoteKey)
}
func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []byte, updateTime time.Time) {
@@ -751,23 +833,39 @@ func (conn *Conn) isConnectedOnAllWay() (status guard.ConnStatus) {
})
}
// enableWgWatcherIfNeeded starts a fresh watcher instance per connection attempt, so its
// lifecycle stays bound to conn.mu and enable/disable can't race an old goroutine's shutdown.
// Caller must hold conn.mu.
func (conn *Conn) enableWgWatcherIfNeeded(enabledTime time.Time) {
if !conn.wgWatcher.IsEnabled() {
wgWatcherCtx, wgWatcherCancel := context.WithCancel(conn.ctx)
conn.wgWatcherCancel = wgWatcherCancel
conn.wgWatcherWg.Add(1)
go func() {
defer conn.wgWatcherWg.Done()
conn.wgWatcher.EnableWgWatcher(wgWatcherCtx, enabledTime, conn.onWGDisconnected, conn.onWGHandshakeSuccess)
}()
if conn.wgWatcher != nil {
return
}
watcher := NewWGWatcher(conn.Log, conn.config.WgConfig.WgInterface, conn.config.Key, conn.dumpState)
watcher.PrepareInitialHandshake()
wgWatcherCtx, wgWatcherCancel := context.WithCancel(conn.ctx)
conn.wgWatcher = watcher
conn.wgWatcherCancel = wgWatcherCancel
conn.wgWatcherWg.Add(1)
go func() {
defer conn.wgWatcherWg.Done()
onDisconnected := func() { conn.onWGDisconnected(wgWatcherCtx) }
watcher.EnableWgWatcher(wgWatcherCtx, enabledTime, onDisconnected, conn.onWGHandshakeSuccess, conn.onWGCheckSuccess)
}()
}
// disableWgWatcherIfNeeded cancels and drops the watcher once no transport is active. It never
// waits for the goroutine: the timeout path reentrantly calls back here under conn.mu, so
// blocking would deadlock. Caller must hold conn.mu.
func (conn *Conn) disableWgWatcherIfNeeded() {
if conn.currentConnPriority == conntype.None && conn.wgWatcherCancel != nil {
conn.wgWatcherCancel()
conn.wgWatcherCancel = nil
if conn.currentConnPriority != conntype.None || conn.wgWatcher == nil {
return
}
conn.wgWatcherCancel()
conn.wgWatcher = nil
conn.wgWatcherCancel = nil
}
func (conn *Conn) newProxy(remoteConn net.Conn) (wgproxy.Proxy, error) {
@@ -790,7 +888,9 @@ func (conn *Conn) resetEndpoint() {
return
}
conn.Log.Infof("reset wg endpoint")
conn.wgWatcher.Reset()
if conn.wgWatcher != nil {
conn.wgWatcher.Reset()
}
if err := conn.endpointUpdater.RemoveEndpointAddress(); err != nil {
conn.Log.Warnf("failed to remove endpoint address before update: %v", err)
}
@@ -839,6 +939,15 @@ func (conn *Conn) onWGHandshakeSuccess(when time.Time) {
conn.recordConnectionMetrics()
}
// onWGCheckSuccess is called for every watcher check that observed a fresh
// handshake, including handshakes of connections that were already up when
// the watcher started.
func (conn *Conn) onWGCheckSuccess() {
conn.mu.Lock()
conn.wgTimeouts = 0
conn.mu.Unlock()
}
// recordConnectionMetrics records connection stage timestamps as metrics
func (conn *Conn) recordConnectionMetrics() {
if conn.metricsRecorder == nil {
+82
View File
@@ -7,6 +7,7 @@ import (
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/netbirdio/netbird/client/iface"
@@ -304,3 +305,84 @@ func TestConn_presharedKey_RosenpassManaged(t *testing.T) {
t.Fatalf("expected non-nil presharedKey before Rosenpass manages PSK")
}
}
func newWGTimeoutTestConn(rosenpassEnabled bool, disconnected *[]string) *Conn {
cfg := ConnConfig{
Key: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
LocalKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
WgConfig: WgConfig{RemoteKey: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU="},
}
if rosenpassEnabled {
cfg.RosenpassConfig = RosenpassConfig{PubKey: []byte("dummykey")}
}
conn := &Conn{
ctx: context.Background(),
config: cfg,
Log: log.WithField("peer", cfg.Key),
metricsStages: &MetricsStages{},
}
conn.SetOnDisconnected(func(remotePeer string) {
*disconnected = append(*disconnected, remotePeer)
})
return conn
}
// TestConn_onWGDisconnected_EscalatesToRosenpassReset: repeated handshake
// timeouts with rosenpass enabled mean the preshared keys have desynced. The
// renewal exchange runs over the dead tunnel and cannot resync them, so after
// wgTimeoutEscalationThreshold consecutive timeouts the conn must report the
// peer disconnected, dropping its rosenpass state so the next configuration
// programs the rendezvous key.
func TestConn_onWGDisconnected_EscalatesToRosenpassReset(t *testing.T) {
var disconnected []string
conn := newWGTimeoutTestConn(true, &disconnected)
for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
conn.onWGDisconnected(conn.ctx)
}
assert.Empty(t, disconnected, "escalation must not fire below the threshold")
conn.onWGDisconnected(conn.ctx)
assert.Equal(t, []string{conn.config.WgConfig.RemoteKey}, disconnected,
"reaching the threshold must report the peer disconnected once")
for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
conn.onWGDisconnected(conn.ctx)
}
assert.Len(t, disconnected, 1, "escalation must restart counting after firing")
conn.onWGDisconnected(conn.ctx)
assert.Len(t, disconnected, 2, "continued timeouts must escalate again")
}
// TestConn_onWGDisconnected_CheckSuccessResetsEscalation: a successful
// handshake between timeouts means the tunnel recovered; the counter must
// start over.
func TestConn_onWGDisconnected_CheckSuccessResetsEscalation(t *testing.T) {
var disconnected []string
conn := newWGTimeoutTestConn(true, &disconnected)
for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
conn.onWGDisconnected(conn.ctx)
}
conn.onWGCheckSuccess()
for i := 0; i < wgTimeoutEscalationThreshold-1; i++ {
conn.onWGDisconnected(conn.ctx)
}
assert.Empty(t, disconnected, "handshake success must reset the timeout count")
}
// TestConn_onWGDisconnected_NoEscalationWithoutRosenpass: without rosenpass
// there is no per-peer key state to reset; repeated timeouts must not report
// disconnects.
func TestConn_onWGDisconnected_NoEscalationWithoutRosenpass(t *testing.T) {
var disconnected []string
conn := newWGTimeoutTestConn(false, &disconnected)
for i := 0; i < wgTimeoutEscalationThreshold*3; i++ {
conn.onWGDisconnected(conn.ctx)
}
assert.Empty(t, disconnected, "escalation must be limited to rosenpass connections")
}
+5 -1
View File
@@ -85,7 +85,11 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) {
defer g.srWatcher.RemoveListener(srReconnectedChan)
ticker := g.initialTicker(ctx)
defer ticker.Stop()
defer func() {
// If backoff.Ticker.send is blocked, context.Done will not close the Ticker goroutine.
// We have to explicitly call Stop, even if we use backoff.WithContext.
ticker.Stop()
}()
tickerChannel := ticker.C
@@ -0,0 +1,92 @@
package guard
import (
"context"
"runtime"
"strings"
"sync"
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/peer/ice"
)
func newTestGuard(status connStatusFunc) *Guard {
srw := NewSRWatcher(nil, nil, nil, ice.Config{})
return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw)
}
// countBackoffTickerGoroutines returns how many goroutines are currently sitting
// in backoff/v4.(*Ticker).run (a ticker goroutine that has not exited).
func countBackoffTickerGoroutines() int {
buf := make([]byte, 1<<25) // 32MB
n := runtime.Stack(buf, true)
return strings.Count(string(buf[:n]), "backoff/v4.(*Ticker).run")
}
// TestGuard_ReconnectTicker_NoGoroutineLeakOnShutdown reproduces a observed
// leak: after a shutdown burst, ticker run/send goroutines stay parked
// forever even though every reconnect loop has exited.
func TestGuard_ReconnectTicker_NoGoroutineLeakOnShutdown(t *testing.T) {
before := countBackoffTickerGoroutines()
const peers = 6000
cancels := make([]context.CancelFunc, 0, peers)
var wg sync.WaitGroup
// A status check slower than the tick cadence. This models the real
// isConnectedOnAllWay/callback doing work: while the loop is busy in the
// handler, the ticker fires the next tick and parks in send(), because
// send() never selects on ctx.
slowStatus := func() ConnStatus {
time.Sleep(70 * time.Millisecond)
return ConnStatusConnected
}
for range peers {
g := newTestGuard(slowStatus)
ctx, cancel := context.WithCancel(context.Background())
cancels = append(cancels, cancel)
wg.Add(1)
go func() {
defer wg.Done()
g.Start(ctx, func() {})
}()
// Force the live ticker to be a newReconnectTicker.
g.SetRelayedConnDisconnected()
}
// Let the replacement tickers get past their 800ms initial interval, so
// many are parked in send() waiting on the (slow) consumer when we tear
// everything down.
time.Sleep(1500 * time.Millisecond)
// Shutdown burst: cancel every peer at once, like engine teardown.
for _, c := range cancels {
c()
}
// Every reconnect loop must return
waitCh := make(chan struct{})
go func() { wg.Wait(); close(waitCh) }()
select {
case <-waitCh:
case <-time.After(30 * time.Second):
t.Fatal("not all reconnect loops returned after ctx cancel")
}
// Give any correctly-stopped ticker goroutines time to unwind.
for range 50 {
runtime.Gosched()
time.Sleep(10 * time.Millisecond)
}
leaked := countBackoffTickerGoroutines() - before
t.Logf("backoff Ticker.run goroutines still parked after teardown of %d peers: %d", peers, leaked)
if leaked > 0 {
t.Errorf("LEAK: %d backoff ticker goroutines parked after all reconnect loops exited "+
"(defer ticker.Stop() stops the initial ticker, not the live replacement)", leaked)
}
}
+184 -17
View File
@@ -7,6 +7,7 @@ import (
"net/netip"
"slices"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
@@ -191,23 +192,30 @@ func (s *StatusChangeSubscription) Events() chan map[string]RouterState {
// every private-service request) don't contend against each other.
// Pure read methods take RLock; anything that mutates state takes Lock.
type Status struct {
mux sync.RWMutex
muxRelays sync.RWMutex
peers map[string]State
ipToKey map[string]string
changeNotify map[string]map[string]*StatusChangeSubscription // map[peerID]map[subscriptionID]*StatusChangeSubscription
signalState bool
signalError error
managementState bool
managementError error
relayStates []relay.ProbeResult
localPeer LocalPeerState
offlinePeers []State
mgmAddress string
signalAddress string
notifier *notifier
rosenpassEnabled bool
rosenpassPermissive bool
mux sync.RWMutex
muxRelays sync.RWMutex
peers map[string]State
ipToKey map[string]string
changeNotify map[string]map[string]*StatusChangeSubscription // map[peerID]map[subscriptionID]*StatusChangeSubscription
signalState bool
signalError error
managementState bool
managementError error
relayStates []relay.ProbeResult
localPeer LocalPeerState
offlinePeers []State
mgmAddress string
signalAddress string
notifier *notifier
rosenpassEnabled bool
rosenpassPermissive bool
// sessionExpiresAt is the absolute UTC instant at which the peer's SSO
// session expires. Zero when the peer is not SSO-tracked or login
// expiration is disabled. Populated from management LoginResponse /
// SyncResponse and exposed via the daemon's Status / SubscribeStatus RPC
// so the UI can show remaining time without itself talking to mgm.
sessionExpiresAt time.Time
nsGroupStates []NSGroupState
resolvedDomainsStates map[domain.Domain]ResolvedDomainInfo
lazyConnectionEnabled bool
@@ -223,6 +231,21 @@ type Status struct {
eventStreams map[string]chan *proto.SystemEvent
eventQueue *EventQueue
// stateChangeStreams fan-out connection-state changes (connected /
// disconnected / connecting / address change / peers list change) to
// every active SubscribeStatus gRPC stream. Each subscriber gets a
// buffered chan; the notifier non-blockingly pings them so a slow
// consumer can never stall the daemon.
stateChangeMux sync.Mutex
stateChangeStreams map[string]chan struct{}
// networksRevision bumps whenever the routed-networks set or their
// selected state changes (driven by the route manager). Surfaced in the
// status snapshot so the UI can fingerprint on it and re-fetch
// ListNetworks only on a real change. Atomic so the snapshot builder can
// read it without taking mux.
networksRevision atomic.Uint64
ingressGwMgr *ingressgw.Manager
routeIDLookup routeIDLookup
@@ -237,6 +260,7 @@ func NewRecorder(mgmAddress string) *Status {
changeNotify: make(map[string]map[string]*StatusChangeSubscription),
eventStreams: make(map[string]chan *proto.SystemEvent),
eventQueue: NewEventQueue(eventQueueSize),
stateChangeStreams: make(map[string]chan struct{}),
offlinePeers: make([]State, 0),
notifier: newNotifier(),
mgmAddress: mgmAddress,
@@ -401,6 +425,7 @@ func (d *Status) UpdatePeerState(receivedState State) error {
if notifyRouter {
d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
}
d.notifyStateChange()
return nil
}
@@ -426,6 +451,7 @@ func (d *Status) AddPeerStateRoute(peer string, route string, resourceId route.R
// todo: consider to make sense of this notification or not
d.notifier.peerListChanged(numPeers)
d.notifyStateChange()
return nil
}
@@ -451,6 +477,7 @@ func (d *Status) RemovePeerStateRoute(peer string, route string) error {
// todo: consider to make sense of this notification or not
d.notifier.peerListChanged(numPeers)
d.notifyStateChange()
return nil
}
@@ -500,6 +527,7 @@ func (d *Status) UpdatePeerICEState(receivedState State) error {
if notifyRouter {
d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
}
d.notifyStateChange()
return nil
}
@@ -536,6 +564,7 @@ func (d *Status) UpdatePeerRelayedState(receivedState State) error {
if notifyRouter {
d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
}
d.notifyStateChange()
return nil
}
@@ -571,6 +600,7 @@ func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error
if notifyRouter {
d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
}
d.notifyStateChange()
return nil
}
@@ -609,6 +639,7 @@ func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error {
if notifyRouter {
d.dispatchRouterPeers(receivedState.PubKey, routerSnapshot)
}
d.notifyStateChange()
return nil
}
@@ -702,6 +733,7 @@ func (d *Status) FinishPeerListModifications() {
for _, rd := range dispatches {
d.dispatchRouterPeers(rd.peerID, rd.snapshot)
}
d.notifyStateChange()
}
func (d *Status) SubscribeToPeerStateChanges(ctx context.Context, peerID string) *StatusChangeSubscription {
@@ -760,6 +792,41 @@ func (d *Status) UpdateLocalPeerState(localPeerState LocalPeerState) {
d.mux.Unlock()
d.notifier.localAddressChanged(fqdn, ip)
d.notifyStateChange()
}
// SetSessionExpiresAt records the absolute UTC instant at which the peer's
// SSO session is set to expire. Pass the zero value to clear (e.g. when the
// management server stops publishing a deadline because login expiration was
// disabled or the peer is not SSO-tracked). Same-value updates are no-ops;
// real changes fan out via notifyStateChange so SubscribeStatus consumers
// pick up the new deadline on their next read.
func (d *Status) SetSessionExpiresAt(deadline time.Time) {
d.mux.Lock()
if d.sessionExpiresAt.Equal(deadline) {
d.mux.Unlock()
return
}
d.sessionExpiresAt = deadline
d.mux.Unlock()
d.notifyStateChange()
}
// GetSessionExpiresAt returns the most recently recorded SSO session deadline,
// or the zero value when no deadline is tracked. A deadline that has already
// slipped into the past reports as "none": once the session has expired it is
// no longer a meaningful countdown, and the sessionwatch.Watcher does not
// arm a timer at the deadline itself to clear it (only the two pre-expiry
// warnings). Without this guard the UI would keep painting a stale
// "expires in …" against a moment that has passed until the next login,
// extend, or teardown rewrote the value.
func (d *Status) GetSessionExpiresAt() time.Time {
d.mux.Lock()
defer d.mux.Unlock()
if !d.sessionExpiresAt.IsZero() && d.sessionExpiresAt.Before(time.Now()) {
return time.Time{}
}
return d.sessionExpiresAt
}
// AddLocalPeerStateRoute adds a route to the local peer state
@@ -828,11 +895,19 @@ func (d *Status) CleanLocalPeerState() {
d.mux.Unlock()
d.notifier.localAddressChanged(fqdn, ip)
d.notifyStateChange()
}
// MarkManagementDisconnected sets ManagementState to disconnected
func (d *Status) MarkManagementDisconnected(err error) {
d.mux.Lock()
// Health checks re-mark the same state on every probe; skip the fan-out
// when nothing actually changed so we don't flood SubscribeStatus
// consumers with identical snapshots.
if !d.managementState && errors.Is(d.managementError, err) {
d.mux.Unlock()
return
}
d.managementState = false
d.managementError = err
mgm := d.managementState
@@ -840,11 +915,16 @@ func (d *Status) MarkManagementDisconnected(err error) {
d.mux.Unlock()
d.notifier.updateServerStates(mgm, sig)
d.notifyStateChange()
}
// MarkManagementConnected sets ManagementState to connected
func (d *Status) MarkManagementConnected() {
d.mux.Lock()
if d.managementState && d.managementError == nil {
d.mux.Unlock()
return
}
d.managementState = true
d.managementError = nil
mgm := d.managementState
@@ -852,6 +932,7 @@ func (d *Status) MarkManagementConnected() {
d.mux.Unlock()
d.notifier.updateServerStates(mgm, sig)
d.notifyStateChange()
}
// UpdateSignalAddress update the address of the signal server
@@ -885,6 +966,10 @@ func (d *Status) UpdateLazyConnection(enabled bool) {
// MarkSignalDisconnected sets SignalState to disconnected
func (d *Status) MarkSignalDisconnected(err error) {
d.mux.Lock()
if !d.signalState && errors.Is(d.signalError, err) {
d.mux.Unlock()
return
}
d.signalState = false
d.signalError = err
mgm := d.managementState
@@ -892,11 +977,16 @@ func (d *Status) MarkSignalDisconnected(err error) {
d.mux.Unlock()
d.notifier.updateServerStates(mgm, sig)
d.notifyStateChange()
}
// MarkSignalConnected sets SignalState to connected
func (d *Status) MarkSignalConnected() {
d.mux.Lock()
if d.signalState && d.signalError == nil {
d.mux.Unlock()
return
}
d.signalState = true
d.signalError = nil
mgm := d.managementState
@@ -904,6 +994,7 @@ func (d *Status) MarkSignalConnected() {
d.mux.Unlock()
d.notifier.updateServerStates(mgm, sig)
d.notifyStateChange()
}
func (d *Status) UpdateRelayStates(relayResults []relay.ProbeResult) {
@@ -1110,16 +1201,19 @@ func (d *Status) GetFullStatus() FullStatus {
// ClientStart will notify all listeners about the new service state
func (d *Status) ClientStart() {
d.notifier.clientStart()
d.notifyStateChange()
}
// ClientStop will notify all listeners about the new service state
func (d *Status) ClientStop() {
d.notifier.clientStop()
d.notifyStateChange()
}
// ClientTeardown will notify all listeners about the service is under teardown
func (d *Status) ClientTeardown() {
d.notifier.clientTearDown()
d.notifyStateChange()
}
// SetConnectionListener set a listener to the notifier
@@ -1261,6 +1355,79 @@ func (d *Status) GetEventHistory() []*proto.SystemEvent {
return d.eventQueue.GetAll()
}
// SubscribeToStateChanges hands back a channel that receives a tick on
// every connection-state change (connected / disconnected / connecting /
// address change / peers-list change). The channel is buffered to one
// pending tick so a coalesced burst still wakes the consumer exactly
// once. Pass the returned id to UnsubscribeFromStateChanges to detach.
func (d *Status) SubscribeToStateChanges() (string, <-chan struct{}) {
d.stateChangeMux.Lock()
defer d.stateChangeMux.Unlock()
id := uuid.New().String()
ch := make(chan struct{}, 1)
d.stateChangeStreams[id] = ch
return id, ch
}
// UnsubscribeFromStateChanges releases a SubscribeToStateChanges channel
// and closes it so any consumer goroutine selecting on the channel
// unblocks cleanly.
func (d *Status) UnsubscribeFromStateChanges(id string) {
d.stateChangeMux.Lock()
defer d.stateChangeMux.Unlock()
if ch, ok := d.stateChangeStreams[id]; ok {
close(ch)
delete(d.stateChangeStreams, id)
}
}
// notifyStateChange wakes every SubscribeToStateChanges subscriber. Drops
// the tick if a subscriber's buffer is full — by definition the consumer
// is already going to fetch the latest snapshot, so multiple pending ticks
// would be redundant.
func (d *Status) notifyStateChange() {
d.stateChangeMux.Lock()
defer d.stateChangeMux.Unlock()
for _, ch := range d.stateChangeStreams {
select {
case ch <- struct{}{}:
default:
}
}
}
// NotifyStateChange is the public wake-the-subscribers entry point used by
// callers that mutate state outside the peer recorder — most importantly
// the connect-state machine, which writes StatusNeedsLogin into the
// shared contextState (client/internal/state.go) without touching any
// recorder field. Without this push the SubscribeStatus stream stays on
// the previous snapshot until an unrelated peer/management/signal
// change happens to fire notifyStateChange, leaving the UI's status
// out of sync with the daemon.
func (d *Status) NotifyStateChange() {
d.notifyStateChange()
}
// BumpNetworksRevision increments the routed-networks revision and wakes every
// SubscribeStatus subscriber. The route manager calls it when a network map
// changes the available routes or when a selection is applied — the peer
// status itself only records actively-routed (chosen) networks, so without
// this bump a candidate route appearing/disappearing would never reach the UI.
func (d *Status) BumpNetworksRevision() {
d.networksRevision.Add(1)
d.notifyStateChange()
}
// GetNetworksRevision returns the current routed-networks revision, surfaced in
// the status snapshot so the UI can detect route/selection changes (see
// BumpNetworksRevision).
func (d *Status) GetNetworksRevision() uint64 {
return d.networksRevision.Load()
}
func (d *Status) SetWgIface(wgInterface WGIfaceStatus) {
d.mux.Lock()
defer d.mux.Unlock()
+36
View File
@@ -314,3 +314,39 @@ func TestGetFullStatus(t *testing.T) {
assert.Equal(t, signalState, fullStatus.SignalState, "signal status should be equal")
assert.ElementsMatch(t, []State{peerState1, peerState2}, fullStatus.Peers, "peers states should match")
}
// notified reports whether a state-change tick is pending on ch, draining it.
func notified(ch <-chan struct{}) bool {
select {
case <-ch:
return true
default:
return false
}
}
func TestMarkServerStateDoesNotNotifyWhenUnchanged(t *testing.T) {
status := NewRecorder("https://mgm")
_, ch := status.SubscribeToStateChanges()
// First transition is a real change and must notify.
status.MarkManagementConnected()
require.True(t, notified(ch), "first connect should notify")
// Re-marking the same state must not notify again.
status.MarkManagementConnected()
assert.False(t, notified(ch), "redundant connect should not notify")
// Same for signal.
status.MarkSignalConnected()
require.True(t, notified(ch), "first signal connect should notify")
status.MarkSignalConnected()
assert.False(t, notified(ch), "redundant signal connect should not notify")
// A genuine change (disconnect with an error) notifies again.
err := errors.New("boom")
status.MarkManagementDisconnected(err)
require.True(t, notified(ch), "disconnect should notify")
status.MarkManagementDisconnected(err)
assert.False(t, notified(ch), "redundant disconnect should not notify")
}
+30 -34
View File
@@ -3,7 +3,6 @@ package peer
import (
"context"
"fmt"
"sync"
"time"
log "github.com/sirupsen/logrus"
@@ -24,14 +23,16 @@ type WGInterfaceStater interface {
GetStats() (map[string]configurer.WGStats, error)
}
// WGWatcher is single-shot: one instance per connection attempt, run once, then discarded.
// Lifecycle is owned by Conn under conn.mu, so it keeps no "enabled" state to go stale.
type WGWatcher struct {
log *log.Entry
wgIfaceStater WGInterfaceStater
peerKey string
stateDump *stateDump
enabled bool
muEnabled sync.RWMutex
// initialHandshake is not thread-safe; never call PrepareInitialHandshake and EnableWgWatcher concurrently.
initialHandshake time.Time
resetCh chan struct{}
}
@@ -46,36 +47,23 @@ func NewWGWatcher(log *log.Entry, wgIfaceStater WGInterfaceStater, peerKey strin
}
}
// EnableWgWatcher starts the WireGuard watcher. If it is already enabled, it will return immediately and do nothing.
// The watcher runs until ctx is cancelled. Caller is responsible for context lifecycle management.
func (w *WGWatcher) EnableWgWatcher(ctx context.Context, enabledTime time.Time, onDisconnectedFn func(), onHandshakeSuccessFn func(when time.Time)) {
w.muEnabled.Lock()
if w.enabled {
w.muEnabled.Unlock()
return
}
// PrepareInitialHandshake reads the peer's current WireGuard handshake time. It must be
// called before the peer is (re)configured on the WireGuard interface, so the captured
// baseline reflects the state prior to this connection attempt instead of racing with
// that configuration.
func (w *WGWatcher) PrepareInitialHandshake() {
w.log.Debugf("enable WireGuard watcher")
w.enabled = true
w.muEnabled.Unlock()
initialHandshake, err := w.wgState()
if err != nil {
w.log.Warnf("failed to read initial wg stats: %v", err)
}
w.periodicHandshakeCheck(ctx, onDisconnectedFn, onHandshakeSuccessFn, enabledTime, initialHandshake)
w.muEnabled.Lock()
w.enabled = false
w.muEnabled.Unlock()
handshake, _ := w.wgState()
w.initialHandshake = handshake
}
// IsEnabled returns true if the WireGuard watcher is currently enabled
func (w *WGWatcher) IsEnabled() bool {
w.muEnabled.RLock()
defer w.muEnabled.RUnlock()
return w.enabled
// EnableWgWatcher runs the WireGuard watcher loop using the handshake baseline captured by
// PrepareInitialHandshake. The watcher runs until ctx is cancelled. Caller is responsible
// for context lifecycle management. onHandshakeSuccessFn is called only for the first
// handshake observed by this run, onCheckSuccessFn for every check that observed a fresh
// handshake, including the first.
func (w *WGWatcher) EnableWgWatcher(ctx context.Context, enabledTime time.Time, onDisconnectedFn func(), onHandshakeSuccessFn func(when time.Time), onCheckSuccessFn func()) {
w.periodicHandshakeCheck(ctx, onDisconnectedFn, onHandshakeSuccessFn, onCheckSuccessFn, enabledTime, w.initialHandshake)
}
// Reset signals the watcher that the WireGuard peer has been reset and a new
@@ -88,7 +76,7 @@ func (w *WGWatcher) Reset() {
}
// wgStateCheck help to check the state of the WireGuard handshake and relay connection
func (w *WGWatcher) periodicHandshakeCheck(ctx context.Context, onDisconnectedFn func(), onHandshakeSuccessFn func(when time.Time), enabledTime time.Time, initialHandshake time.Time) {
func (w *WGWatcher) periodicHandshakeCheck(ctx context.Context, onDisconnectedFn func(), onHandshakeSuccessFn func(when time.Time), onCheckSuccessFn func(), enabledTime time.Time, initialHandshake time.Time) {
w.log.Infof("WireGuard watcher started")
timer := time.NewTimer(wgHandshakeOvertime)
@@ -101,17 +89,25 @@ func (w *WGWatcher) periodicHandshakeCheck(ctx context.Context, onDisconnectedFn
case <-timer.C:
handshake, ok := w.handshakeCheck(lastHandshake)
if !ok {
// early ctx cancel check return
if ctx.Err() != nil {
return
}
onDisconnectedFn()
return
}
if lastHandshake.IsZero() {
elapsed := calcElapsed(enabledTime, *handshake)
w.log.Infof("first wg handshake detected within: %.2fsec, (%s)", elapsed, handshake)
if onHandshakeSuccessFn != nil {
if onHandshakeSuccessFn != nil && ctx.Err() == nil {
onHandshakeSuccessFn(*handshake)
}
}
if onCheckSuccessFn != nil && ctx.Err() == nil {
onCheckSuccessFn()
}
lastHandshake = *handshake
resetTime := time.Until(handshake.Add(checkPeriod))
@@ -142,9 +138,9 @@ func (w *WGWatcher) handshakeCheck(lastHandshake time.Time) (*time.Time, bool) {
w.log.Tracef("previous handshake, handshake: %v, %v", lastHandshake, handshake)
// the current know handshake did not change
// the current known handshake did not change
if handshake.Equal(lastHandshake) {
w.log.Warnf("WireGuard handshake timed out: %v", handshake)
w.log.Warnf("WireGuard handshake not updated: %v", handshake)
return nil, false
}
+75 -3
View File
@@ -23,6 +23,72 @@ func (m *MocWgIface) disconnect() {
m.stop = true
}
type mockHandshakeStats struct {
mu sync.Mutex
handshake time.Time
}
func (m *mockHandshakeStats) GetStats() (map[string]configurer.WGStats, error) {
m.mu.Lock()
defer m.mu.Unlock()
return map[string]configurer.WGStats{"": {LastHandshake: m.handshake}}, nil
}
func (m *mockHandshakeStats) advance() {
m.mu.Lock()
defer m.mu.Unlock()
m.handshake = time.Now()
}
// TestWGWatcher_CheckSuccessCallback: onCheckSuccessFn must fire for a fresh
// handshake even when the watcher started with an existing handshake baseline,
// the case where onHandshakeSuccessFn stays silent.
func TestWGWatcher_CheckSuccessCallback(t *testing.T) {
// checkPeriod bounds how stale a handshake may be before the watcher treats it
// as a suspended-machine timeout. The first check fires after wgHandshakeOvertime,
// so keep checkPeriod well above any scheduling jitter to avoid a false timeout
// converting the expected success into a disconnect on a loaded runner.
checkPeriod = 1 * time.Minute
wgHandshakeOvertime = 1 * time.Second
mlog := log.WithField("peer", "tet")
// Use an old baseline so advance() yields a strictly newer handshake even on
// platforms with coarse clock resolution (Windows), where two time.Now() calls
// microseconds apart can return the same instant and read as a timed-out handshake.
stats := &mockHandshakeStats{handshake: time.Now().Add(-time.Hour)}
watcher := NewWGWatcher(mlog, stats, "", newStateDump("peer", mlog, &Status{}))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
watcher.PrepareInitialHandshake()
firstHandshake := make(chan struct{}, 1)
checkSuccess := make(chan struct{}, 1)
go watcher.EnableWgWatcher(ctx, time.Now(), func() {}, func(when time.Time) {
firstHandshake <- struct{}{}
}, func() {
select {
case checkSuccess <- struct{}{}:
default:
}
})
stats.advance()
select {
case <-checkSuccess:
case <-time.After(10 * time.Second):
t.Errorf("timeout waiting for check success callback")
}
select {
case <-firstHandshake:
t.Errorf("first-handshake callback must not fire for a non-zero baseline")
default:
}
}
func TestWGWatcher_EnableWgWatcher(t *testing.T) {
checkPeriod = 5 * time.Second
wgHandshakeOvertime = 1 * time.Second
@@ -34,13 +100,15 @@ func TestWGWatcher_EnableWgWatcher(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
watcher.PrepareInitialHandshake()
onDisconnected := make(chan struct{}, 1)
go watcher.EnableWgWatcher(ctx, time.Now(), func() {
mlog.Infof("onDisconnectedFn")
onDisconnected <- struct{}{}
}, func(when time.Time) {
mlog.Infof("onHandshakeSuccess: %v", when)
})
}, nil)
// wait for initial reading
time.Sleep(2 * time.Second)
@@ -62,11 +130,13 @@ func TestWGWatcher_ReEnable(t *testing.T) {
watcher := NewWGWatcher(mlog, mocWgIface, "", newStateDump("peer", mlog, &Status{}))
ctx, cancel := context.WithCancel(context.Background())
watcher.PrepareInitialHandshake()
wg := &sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
watcher.EnableWgWatcher(ctx, time.Now(), func() {}, func(when time.Time) {})
watcher.EnableWgWatcher(ctx, time.Now(), func() {}, func(when time.Time) {}, nil)
}()
cancel()
@@ -76,10 +146,12 @@ func TestWGWatcher_ReEnable(t *testing.T) {
ctx, cancel = context.WithCancel(context.Background())
defer cancel()
watcher.PrepareInitialHandshake()
onDisconnected := make(chan struct{}, 1)
go watcher.EnableWgWatcher(ctx, time.Now(), func() {
onDisconnected <- struct{}{}
}, func(when time.Time) {})
}, func(when time.Time) {}, nil)
time.Sleep(2 * time.Second)
mocWgIface.disconnect()