[client] Merge main into peer event bus refactor

This commit is contained in:
Zoltán Papp
2026-09-10 01:48:41 +02:00
1102 changed files with 94422 additions and 19596 deletions
+26 -11
View File
@@ -30,6 +30,7 @@ import (
"github.com/netbirdio/netbird/client/internal/portforward"
"github.com/netbirdio/netbird/client/internal/rosenpass"
"github.com/netbirdio/netbird/client/internal/stdnet"
"github.com/netbirdio/netbird/client/netevents"
"github.com/netbirdio/netbird/route"
relayClient "github.com/netbirdio/netbird/shared/relay/client"
)
@@ -104,6 +105,10 @@ type ConnConfig struct {
// ICEConfig ICE protocol configuration
ICEConfig icemaker.Config
// NetMgr gates the reconnection guard on OS-reported network
// availability; nil disables gating.
NetMgr *netevents.Manager
}
func (c ConnConfig) IsController() bool {
@@ -265,7 +270,7 @@ func (conn *Conn) open(engineCtx context.Context, firstPacket []byte) error {
RosenpassAddr: conn.config.RosenpassConfig.Addr,
}, conn.signaler, iceWorker, conn.relayManager)
conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher)
conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher, conn.config.NetMgr)
conn.relayDialInFlight = false
conn.pendingRelayOffer = nil
@@ -481,6 +486,7 @@ func (conn *Conn) teardown(mb *mailbox, leftover []event, signalToRemote bool, d
if conn.wgWatcherCancel != nil {
conn.wgWatcherCancel()
conn.wgWatcher = nil
conn.wgWatcherCancel = nil
}
conn.workerRelay.CloseConn()
@@ -650,7 +656,7 @@ func (conn *Conn) handleICEReady(priority worker.ConnPriority, iceConnInfo worke
conn.dumpState.NewLocalProxy()
wgProxy, err = conn.newProxy(iceConnInfo.RemoteConn)
if err != nil {
conn.Log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
conn.Log.Errorf("failed to add relayed net.Conn to local proxy: %v", err)
return
}
ep = wgProxy.EndpointAddr()
@@ -1142,9 +1148,8 @@ func (conn *Conn) newProxy(remoteConn net.Conn) (wgproxy.Proxy, error) {
}
wgProxy := conn.config.WgConfig.WgInterface.GetProxy()
if err := wgProxy.AddTurnConn(conn.ctx, udpAddr, remoteConn); err != nil {
conn.Log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
return nil, err
if err := wgProxy.AddRelayedConn(conn.ctx, udpAddr, remoteConn); err != nil {
return nil, fmt.Errorf("add relayed conn to proxy: %w", err)
}
return wgProxy, nil
}
@@ -1205,12 +1210,9 @@ func (conn *Conn) recordConnectionMetrics() {
return
}
var connType metrics.ConnectionType
switch conn.currentConnPriority {
case worker.Relay:
connType = metrics.ConnectionTypeRelay
default:
connType = metrics.ConnectionTypeICE
connType := metricsConnType(conn.currentConnPriority)
if connType == metrics.ConnectionTypeUnknown {
return
}
// Record metrics with timestamps - duration calculation happens in metrics package
@@ -1298,3 +1300,16 @@ func boolToConnStatus(connected bool) guard.ConnStatus {
}
return guard.ConnStatusDisconnected
}
func metricsConnType(priority worker.ConnPriority) metrics.ConnectionType {
switch priority {
case worker.Relay:
return metrics.ConnectionTypeRelay
case worker.ICETurn:
return metrics.ConnectionTypeICETurn
case worker.ICEP2P:
return metrics.ConnectionTypeICEP2P
default:
return metrics.ConnectionTypeUnknown
}
}
@@ -0,0 +1,52 @@
package peer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/client/internal/peer/metricsstages"
"github.com/netbirdio/netbird/client/internal/peer/signaling"
)
func TestConn_AnswerBeforeEventLoop(t *testing.T) {
for _, tc := range []struct {
name string
ports []int
}{
{name: "holds early answer", ports: []int{51820}},
{name: "keeps latest answer", ports: []int{1111, 2222}},
} {
t.Run(tc.name, func(t *testing.T) {
conn, err := NewConn(connConf, ServiceDependencies{})
require.NoError(t, err)
conn.metricsStages = &metricsstages.MetricsStages{}
conn.handshaker = signaling.NewHandshaker(conn.Log, signaling.Config{}, nil, nil, nil)
// A relay dial in progress retains the dispatched answer as its next offer.
conn.relayDialInFlight = true
mb := newMailbox()
conn.mailbox.Store(mb)
// Incoming answers can arrive after Open publishes the mailbox but
// before the event loop gets scheduled to consume it.
for _, port := range tc.ports {
conn.OnRemoteAnswer(signaling.OfferAnswer{WgListenPort: port})
}
select {
case <-mb.wake:
default:
t.Fatal("an early answer must wake the event loop")
}
events := mb.drain()
require.Len(t, events, 1, "only the latest answer should reach the event loop")
for _, ev := range events {
conn.handleEvent(ev)
}
require.NotNil(t, conn.pendingRelayOffer, "the answer must reach relay dispatch")
assert.Equal(t, tc.ports[len(tc.ports)-1], conn.pendingRelayOffer.WgListenPort,
"relay dispatch must receive the latest queued answer")
})
}
}
+32
View File
@@ -13,11 +13,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/client/iface"
"github.com/netbirdio/netbird/client/internal/metrics"
"github.com/netbirdio/netbird/client/internal/peer/guard"
"github.com/netbirdio/netbird/client/internal/peer/ice"
"github.com/netbirdio/netbird/client/internal/peer/metricsstages"
"github.com/netbirdio/netbird/client/internal/peer/signaling"
"github.com/netbirdio/netbird/client/internal/peer/status"
"github.com/netbirdio/netbird/client/internal/peer/worker"
"github.com/netbirdio/netbird/client/internal/stdnet"
"github.com/netbirdio/netbird/util"
)
@@ -354,3 +356,33 @@ func TestConn_onWGDisconnected_NoEscalationWithoutRosenpass(t *testing.T) {
}
assert.Empty(t, disconnected, "escalation must be limited to rosenpass connections")
}
func TestMetricsConnType(t *testing.T) {
tests := []struct {
name string
priority worker.ConnPriority
expected metrics.ConnectionType
}{
{"relay", worker.Relay, metrics.ConnectionTypeRelay},
{"ice over turn is relayed, not p2p", worker.ICETurn, metrics.ConnectionTypeICETurn},
{"direct p2p", worker.ICEP2P, metrics.ConnectionTypeICEP2P},
{"unset priority is unknown, not p2p", worker.None, metrics.ConnectionTypeUnknown},
{"unrecognised priority is unknown", worker.ConnPriority(99), metrics.ConnectionTypeUnknown},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.expected, metricsConnType(tc.priority))
})
}
}
func TestMetricsConnType_RelayedMatchesIsRelayed(t *testing.T) {
for _, priority := range []worker.ConnPriority{worker.None, worker.Relay, worker.ICETurn, worker.ICEP2P} {
conn := &Conn{currentConnPriority: priority}
tag := metricsConnType(priority)
relayedTag := tag == metrics.ConnectionTypeRelay || tag == metrics.ConnectionTypeICETurn
assert.Equal(t, conn.isRelayed(), relayedTag,
"priority %s: isRelayed and the %q metric tag must agree", priority, tag)
}
}
+44 -5
View File
@@ -22,6 +22,12 @@ const (
type connStatusFunc func() ConnStatus
// NetworkWatcher is the availability view the guard gates reconnects on.
type NetworkWatcher interface {
IsOnline() bool
Changed() <-chan struct{}
}
// Guard is responsible for the reconnection logic.
// It will trigger to send an offer to the peer then has connection issues.
// Watch these events:
@@ -31,20 +37,26 @@ type connStatusFunc func() ConnStatus
// - Relayed connection disconnected
// - ICE candidate changes
type Guard struct {
log *log.Entry
isConnectedOnAllWay connStatusFunc
timeout time.Duration
srWatcher *SRWatcher
log *log.Entry
isConnectedOnAllWay connStatusFunc
timeout time.Duration
srWatcher *SRWatcher
// netWatcher gates reconnect attempts on OS-reported network availability;
// nil disables gating.
netWatcher NetworkWatcher
relayedConnDisconnected chan struct{}
iCEConnDisconnected chan struct{}
}
func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher) *Guard {
// NewGuard creates a reconnection guard for a peer connection. A nil netWatcher
// disables network availability gating.
func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher, netWatcher NetworkWatcher) *Guard {
return &Guard{
log: log,
isConnectedOnAllWay: isConnectedFn,
timeout: timeout,
srWatcher: srWatcher,
netWatcher: netWatcher,
relayedConnDisconnected: make(chan struct{}, 1),
iCEConnDisconnected: make(chan struct{}, 1),
}
@@ -96,9 +108,19 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) {
iceState := &iceRetryState{log: g.log}
defer iceState.reset()
var netChanged <-chan struct{}
if g.netWatcher != nil {
netChanged = g.netWatcher.Changed()
}
for {
select {
case <-tickerChannel:
// skip attempts while the OS reports no usable network; the
// netChanged case below resumes the loop once it returns
if g.netWatcher != nil && !g.netWatcher.IsOnline() {
continue
}
switch g.isConnectedOnAllWay() {
case ConnStatusConnected:
// all good, nothing to do
@@ -135,6 +157,23 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) {
tickerChannel = ticker.C
iceState.reset()
case <-netChanged:
// Re-arm for the next transition before acting on this one.
netChanged = g.netWatcher.Changed()
if !g.netWatcher.IsOnline() {
continue
}
// Ticks skipped while offline drove the backoff towards its
// maximum without ever attempting, and left the ICE budget
// frozen — possibly in hourly mode. Recover on our own so the
// peer does not depend on a signal or relay event that never
// comes when both stayed up across the outage.
g.log.Debugf("network is back, reset reconnection ticker")
ticker.Stop()
ticker = g.newReconnectTicker(ctx)
tickerChannel = ticker.C
iceState.reset()
case <-ctx.Done():
g.log.Debugf("context is done, stop reconnect loop")
return
@@ -15,7 +15,7 @@ import (
func newTestGuard(status connStatusFunc) *Guard {
srw := NewSRWatcher(nil, nil, nil, ice.Config{})
return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw)
return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw, nil)
}
// countBackoffTickerGoroutines returns how many goroutines are currently sitting
@@ -0,0 +1,107 @@
package guard
import (
"context"
"sync/atomic"
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/peer/ice"
"github.com/netbirdio/netbird/client/netevents/netstate"
)
// newTestGuardWithNetState builds a guard with a realistic MaxInterval: the
// backoff must be able to grow well past the outage, as it does in production
// where the timeout is seconds to minutes.
func newTestGuardWithNetState(status connStatusFunc, netState *netstate.State) *Guard {
srw := NewSRWatcher(nil, nil, nil, ice.Config{})
return NewGuard(log.WithField("test", "guard"), status, 30*time.Second, srw, netState)
}
// TestGuard_RecoversAfterOfflineToOnline covers a peer that stays disconnected
// across a network outage while neither signal nor relay reports an event —
// both stayed up, as on a short airplane mode toggle over Wi-Fi.
//
// Every tick taken while offline is skipped, but it still advances the
// exponential backoff, so by the time the network returns the next tick can be
// tens of seconds away. Without an explicit reaction to the transition the
// peer waits out that interval for a recovery that could start immediately.
func TestGuard_RecoversAfterOfflineToOnline(t *testing.T) {
netState := netstate.New()
var attempts atomic.Int32
g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Start from the reconnect ticker (800ms initial interval), the state a
// peer is in after it loses its connection.
go g.Start(ctx, func() { attempts.Add(1) })
g.SetRelayedConnDisconnected()
// Let the backoff climb: 0.8s, 1.6s, 3.2s, 6.4s ... every tick is skipped
// while offline, but each one doubles the wait for the next.
netState.Set(false)
time.Sleep(8 * time.Second)
offlineAttempts := attempts.Load()
if offlineAttempts != 0 {
t.Fatalf("callback ran %d times while offline, want 0", offlineAttempts)
}
netState.Set(true)
// The next organic tick is now several seconds out, so anything within
// this window can only come from reacting to the transition itself.
pollCtx, stopPolling := context.WithTimeout(ctx, 2*time.Second)
defer stopPolling()
select {
case <-pollCtx.Done():
t.Fatal("peer was not retried within 2s of the network coming back, " +
"with neither a signal nor a relay event to fall back on")
case <-pollUntil(pollCtx, func() bool { return attempts.Load() > 0 }):
}
}
// TestGuard_OfflineTransitionDoesNotRetry checks the other direction: going
// offline must not itself trigger an attempt.
func TestGuard_OfflineTransitionDoesNotRetry(t *testing.T) {
netState := netstate.New()
var attempts atomic.Int32
g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go g.Start(ctx, func() { attempts.Add(1) })
netState.Set(false)
time.Sleep(5 * time.Second)
if got := attempts.Load(); got != 0 {
t.Fatalf("callback ran %d times after going offline, want 0", got)
}
}
// pollUntil closes the returned channel once cond holds. It gives up when ctx
// is done, so the polling goroutine never outlives the test that started it.
func pollUntil(ctx context.Context, cond func() bool) <-chan struct{} {
done := make(chan struct{})
go func() {
for {
if cond() {
close(done)
return
}
select {
case <-ctx.Done():
return
case <-time.After(10 * time.Millisecond):
}
}
}()
return done
}
+40
View File
@@ -0,0 +1,40 @@
package status
// ClientState identifies the client connection state delivered via
// Listener.OnStateChanged.
type ClientState int
// Client states. The numeric values cross the gomobile boundary (the mobile
// bindings re-export them as integer constants), so they are a wire format:
// append new states at the end, never reorder or insert.
const (
ClientStateDisconnected ClientState = iota
ClientStateConnected
ClientStateConnecting
ClientStateDisconnecting
// ClientStateNoNetwork is an overlay state: it is never stored as the
// last notification, only derived from ClientStateConnecting while the
// OS reports no usable network (see notifier.effectiveState).
ClientStateNoNetwork
)
// Listener is a callback type about the NetBird network connection state
type Listener interface {
// OnStateChanged reports every client state transition. New states are
// delivered only through this callback; the per-state callbacks below
// are kept for compatibility and will be removed once all consumers
// have migrated.
OnStateChanged(state ClientState)
// Deprecated: consume OnStateChanged instead.
OnConnected()
// Deprecated: consume OnStateChanged instead.
OnDisconnected()
// Deprecated: consume OnStateChanged instead.
OnConnecting()
// Deprecated: consume OnStateChanged instead.
OnDisconnecting()
OnAddressChanged(string, string)
OnPeersListChanged(int)
}
+81 -40
View File
@@ -4,41 +4,64 @@ import (
"sync"
)
const (
stateDisconnected = iota
stateConnected
stateConnecting
stateDisconnecting
)
// Listener is a callback type about the NetBird network connection state
type Listener interface {
OnConnected()
OnDisconnected()
OnConnecting()
OnDisconnecting()
OnAddressChanged(string, string)
OnPeersListChanged(int)
}
type notifier struct {
// publishLock orders state publication: it is held across computing the
// effective state and handing it to the listener, so a transition cannot
// overtake a newer one and leave the listener on a stale state.
publishLock sync.Mutex
serverStateLock sync.Mutex
listenersLock sync.Mutex
listener Listener
currentClientState bool
lastNotification int
lastNotification ClientState
lastNumberOfPeers int
lastFqdnAddress string
lastIPAddress string
networkAvailable bool
}
func newNotifier() *notifier {
return &notifier{}
return &notifier{
networkAvailable: true,
}
}
// effectiveState maps the computed state to what listeners should see:
// while the OS reports no usable network, "Connecting" would be a lie —
// connection attempts are suspended — so it is reported as NoNetwork.
// Caller must hold serverStateLock.
func (n *notifier) effectiveState(state ClientState) ClientState {
if !n.networkAvailable && state == ClientStateConnecting {
return ClientStateNoNetwork
}
return state
}
// setNetworkAvailable records the OS network availability and re-notifies
// the listener when the flag flips the effective state (Connecting <->
// NoNetwork).
func (n *notifier) setNetworkAvailable(available bool) {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
if n.networkAvailable == available {
n.serverStateLock.Unlock()
return
}
previous := n.effectiveState(n.lastNotification)
n.networkAvailable = available
current := n.effectiveState(n.lastNotification)
n.serverStateLock.Unlock()
if previous != current {
n.notify(current)
}
}
func (n *notifier) setListener(listener Listener) {
n.serverStateLock.Lock()
lastNotification := n.lastNotification
lastNotification := n.effectiveState(n.lastNotification)
numOfPeers := n.lastNumberOfPeers
fqdnAddress := n.lastFqdnAddress
address := n.lastIPAddress
@@ -62,6 +85,9 @@ func (n *notifier) removeListener() {
}
func (n *notifier) updateServerStates(mgmState bool, signalState bool) {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
calculatedState := n.calculateState(mgmState, signalState)
@@ -71,43 +97,54 @@ func (n *notifier) updateServerStates(mgmState bool, signalState bool) {
}
n.lastNotification = calculatedState
effective := n.effectiveState(calculatedState)
n.serverStateLock.Unlock()
n.notify(calculatedState)
n.notify(effective)
}
func (n *notifier) clientStart() {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
n.currentClientState = true
n.lastNotification = stateConnecting
n.lastNotification = ClientStateConnecting
effective := n.effectiveState(ClientStateConnecting)
n.serverStateLock.Unlock()
n.notify(stateConnecting)
n.notify(effective)
}
func (n *notifier) clientStop() {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
n.currentClientState = false
n.lastNotification = stateDisconnected
n.lastNotification = ClientStateDisconnected
n.serverStateLock.Unlock()
n.notify(stateDisconnected)
n.notify(ClientStateDisconnected)
}
func (n *notifier) clientTearDown() {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
n.currentClientState = false
n.lastNotification = stateDisconnecting
n.lastNotification = ClientStateDisconnecting
n.serverStateLock.Unlock()
n.notify(stateDisconnecting)
n.notify(ClientStateDisconnecting)
}
func (n *notifier) isServerStateChanged(newState int) bool {
func (n *notifier) isServerStateChanged(newState ClientState) bool {
return n.lastNotification != newState
}
func (n *notifier) notify(state int) {
func (n *notifier) notify(state ClientState) {
n.listenersLock.Lock()
listener := n.listener
n.listenersLock.Unlock()
@@ -119,20 +156,20 @@ func (n *notifier) notify(state int) {
notifyListener(listener, state)
}
func (n *notifier) calculateState(managementConn, signalConn bool) int {
func (n *notifier) calculateState(managementConn, signalConn bool) ClientState {
if managementConn && signalConn {
return stateConnected
return ClientStateConnected
}
if !managementConn && !signalConn && !n.currentClientState {
return stateDisconnected
return ClientStateDisconnected
}
if n.lastNotification == stateDisconnecting {
return stateDisconnecting
if n.lastNotification == ClientStateDisconnecting {
return ClientStateDisconnecting
}
return stateConnecting
return ClientStateConnecting
}
func (n *notifier) peerListChanged(numOfPeers int) {
@@ -169,15 +206,19 @@ func (n *notifier) localAddressChanged(fqdn, address string) {
listener.OnAddressChanged(fqdn, address)
}
func notifyListener(l Listener, state int) {
func notifyListener(l Listener, state ClientState) {
// legacy per-state callbacks; NoNetwork is delivered only via
// OnStateChanged below
switch state {
case stateDisconnected:
case ClientStateDisconnected:
l.OnDisconnected()
case stateConnected:
case ClientStateConnected:
l.OnConnected()
case stateConnecting:
case ClientStateConnecting:
l.OnConnecting()
case stateDisconnecting:
case ClientStateDisconnecting:
l.OnDisconnecting()
}
l.OnStateChanged(state)
}
@@ -0,0 +1,108 @@
package status
import (
"sync"
"testing"
"time"
)
type recordingListener struct {
mu sync.Mutex
states []ClientState
onState func(ClientState)
}
func (l *recordingListener) OnStateChanged(state ClientState) {
l.mu.Lock()
l.states = append(l.states, state)
hook := l.onState
l.mu.Unlock()
if hook != nil {
hook(state)
}
}
func (l *recordingListener) last() (ClientState, bool) {
l.mu.Lock()
defer l.mu.Unlock()
if len(l.states) == 0 {
return 0, false
}
return l.states[len(l.states)-1], true
}
func (l *recordingListener) snapshot() []ClientState {
l.mu.Lock()
defer l.mu.Unlock()
return append([]ClientState(nil), l.states...)
}
func (l *recordingListener) OnConnected() {}
func (l *recordingListener) OnDisconnected() {}
func (l *recordingListener) OnConnecting() {}
func (l *recordingListener) OnDisconnecting() {}
func (l *recordingListener) OnAddressChanged(string, string) {}
func (l *recordingListener) OnPeersListChanged(int) {}
// TestNotifier_ConcurrentAvailabilityFlipOrdersPublication holds the first
// transition inside the listener callback and flips availability again from
// another goroutine while it is parked. The second flip must not publish
// ahead of the one in flight, otherwise the listener ends up on a state the
// notifier already superseded.
func TestNotifier_ConcurrentAvailabilityFlipOrdersPublication(t *testing.T) {
n := newNotifier()
n.currentClientState = true
n.lastNotification = ClientStateConnecting
entered := make(chan struct{})
release := make(chan struct{})
l := &recordingListener{}
l.onState = func(state ClientState) {
if state != ClientStateNoNetwork {
return
}
l.mu.Lock()
l.onState = nil
l.mu.Unlock()
close(entered)
<-release
}
n.listener = l
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
n.setNetworkAvailable(false)
}()
<-entered
flipped := make(chan struct{})
go func() {
defer close(flipped)
n.setNetworkAvailable(true)
}()
select {
case <-flipped:
t.Fatal("the online transition published while the offline one was " +
"still in flight; publication is not serialized")
case <-time.After(200 * time.Millisecond):
}
close(release)
<-flipped
wg.Wait()
got, ok := l.last()
if !ok {
t.Fatal("listener never observed a state")
}
if got != ClientStateConnecting {
t.Fatalf("listener holds %v after the network came back, want Connecting; sequence: %v",
got, l.snapshot())
}
}
+15 -12
View File
@@ -6,29 +6,32 @@ import (
)
type mocListener struct {
lastState int
lastState ClientState
wg sync.WaitGroup
peersWg sync.WaitGroup
peers int
}
func (l *mocListener) OnConnected() {
l.lastState = stateConnected
l.lastState = ClientStateConnected
l.wg.Done()
}
func (l *mocListener) OnDisconnected() {
l.lastState = stateDisconnected
l.lastState = ClientStateDisconnected
l.wg.Done()
}
func (l *mocListener) OnConnecting() {
l.lastState = stateConnecting
l.lastState = ClientStateConnecting
l.wg.Done()
}
func (l *mocListener) OnDisconnecting() {
l.lastState = stateDisconnecting
l.lastState = ClientStateDisconnecting
l.wg.Done()
}
func (l *mocListener) OnStateChanged(state ClientState) {
}
func (l *mocListener) OnAddressChanged(host, addr string) {
}
@@ -57,15 +60,15 @@ func Test_notifier_serverState(t *testing.T) {
type scenario struct {
name string
expected int
expected ClientState
mgmState bool
signalState bool
}
scenarios := []scenario{
{"connected", stateConnected, true, true},
{"mgm down", stateConnecting, false, true},
{"signal down", stateConnecting, true, false},
{"disconnected", stateDisconnected, false, false},
{"connected", ClientStateConnected, true, true},
{"mgm down", ClientStateConnecting, false, true},
{"signal down", ClientStateConnecting, true, false},
{"disconnected", ClientStateDisconnected, false, false},
}
for _, tt := range scenarios {
@@ -85,7 +88,7 @@ func Test_notifier_SetListener(t *testing.T) {
listener.setPeersWaiter()
n := newNotifier()
n.lastNotification = stateConnecting
n.lastNotification = ClientStateConnecting
n.setListener(listener)
listener.wait()
listener.waitPeers()
@@ -99,7 +102,7 @@ func Test_notifier_RemoveListener(t *testing.T) {
listener.setWaiter()
listener.setPeersWaiter()
n := newNotifier()
n.lastNotification = stateConnecting
n.lastNotification = ClientStateConnecting
n.setListener(listener)
// setListener replays cached state on a goroutine; wait for both the state
// and peers callbacks to finish so we don't race on listener.peers.
+18
View File
@@ -991,6 +991,18 @@ func (d *Recorder) GetResolvedDomainsStates() map[domain.Domain]ResolvedDomainIn
return maps.Clone(d.resolvedDomainsStates)
}
// GetPeerStates returns a snapshot of all known peer states, including offline peers.
func (d *Recorder) GetPeerStates() []State {
d.mux.RLock()
defer d.mux.RUnlock()
states := make([]State, 0, d.numOfPeers())
for _, state := range d.peers {
states = append(states, state)
}
return append(states, d.offlinePeers...)
}
// GetFullStatus gets full status
func (d *Recorder) GetFullStatus() FullStatus {
fullStatus := FullStatus{
@@ -1035,6 +1047,12 @@ func (d *Recorder) ClientTeardown() {
d.notifyStateChange()
}
// SetNetworkAvailable records the OS-reported network availability; while
// unavailable, listeners see NoNetwork instead of Connecting.
func (d *Recorder) SetNetworkAvailable(available bool) {
d.notifier.setNetworkAvailable(available)
}
// SetConnectionListener set a listener to the notifier
func (d *Recorder) SetConnectionListener(listener Listener) {
d.notifier.setListener(listener)
@@ -129,6 +129,28 @@ func TestStatus_PeerStateByIP_RemovedPeer(t *testing.T) {
req.False(ok, "removed peer must not resolve by IPv6 tunnel address")
}
// TestStatus_GetPeerStates_IncludesOfflinePeers keeps the snapshot in line with
// GetFullStatus: offline peers are known peers, so a consumer counting peers
// must see the same total the status command reports.
func TestStatus_GetPeerStates_IncludesOfflinePeers(t *testing.T) {
status := NewRecorder("https://mgm")
req := require.New(t)
req.NoError(status.AddPeer("pk-online", "online.netbird", "100.64.0.10", "fd00::1"))
status.ReplaceOfflinePeers([]State{
{PubKey: "pk-offline", FQDN: "offline.netbird", IP: "100.64.0.20", ConnStatus: StatusIdle},
})
states := status.GetPeerStates()
req.Len(states, 2, "snapshot must carry both the online and the offline peer")
keys := make([]string, 0, len(states))
for _, s := range states {
keys = append(keys, s.PubKey)
}
req.ElementsMatch([]string{"pk-online", "pk-offline"}, keys, "snapshot must carry both peers")
}
func TestStatus_UpdatePeerFQDN(t *testing.T) {
key := "abc"
fqdn := "peer-a.netbird.local"
+7
View File
@@ -6,6 +6,7 @@ import "github.com/netbirdio/netbird/client/internal/peer/status"
// package. Callers are being migrated to reference the status package
// directly; these aliases will be removed once the migration completes.
type (
ClientState = status.ClientState
Status = status.Recorder
State = status.State
ConnStatus = status.ConnStatus
@@ -26,6 +27,12 @@ type (
)
const (
ClientStateDisconnected = status.ClientStateDisconnected
ClientStateConnected = status.ClientStateConnected
ClientStateConnecting = status.ClientStateConnecting
ClientStateDisconnecting = status.ClientStateDisconnecting
ClientStateNoNetwork = status.ClientStateNoNetwork
StatusIdle = status.StatusIdle
StatusConnecting = status.StatusConnecting
StatusConnected = status.StatusConnected
+67 -20
View File
@@ -74,6 +74,9 @@ type ICE struct {
// portForwardAttempted tracks if we've already tried port forwarding this session
portForwardAttempted bool
// dialFunc, when non-nil, replaces agentDial in connect(). Only for tests.
dialFunc func(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *signaling.OfferAnswer) (net.Conn, error)
}
func NewICE(log *log.Entry, key string, iceConfig icemaker.Config, isController bool, onConnReady func(ConnPriority, ICEConnInfo), onStatusDisconnect func(bool), services ICEDependencies, hasRelayOnLocally bool) (*ICE, error) {
@@ -135,7 +138,7 @@ func (w *ICE) OnNewOffer(ctx context.Context, remoteOfferAnswer *signaling.Offer
w.log.Errorf("failed to create new session ID: %s", err)
}
w.sessionID = sessionID
w.agent = nil
w.abandonNegotiation()
}
var preferredCandidateTypes []ice.CandidateType
@@ -163,6 +166,8 @@ func (w *ICE) OnNewOffer(ctx context.Context, remoteOfferAnswer *signaling.Offer
w.remoteSessionID = ""
}
// Capture the cancel func at spawn time: connect reads it from the argument
// instead of the field, which a newer OnNewOffer may already have replaced.
go w.connect(dialerCtx, dialerCancel, agent, remoteOfferAnswer)
}
@@ -218,16 +223,16 @@ func (w *ICE) Close() {
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
if w.agent == nil {
return
if w.agent != nil {
w.agentDialerCancel()
if err := w.agent.Close(); err != nil {
w.log.Warnf("failed to close ICE agent: %s", err)
}
}
w.agentDialerCancel()
if err := w.agent.Close(); err != nil {
w.log.Warnf("failed to close ICE agent: %s", err)
}
w.agent = nil
// Unconditional: a dial goroutine racing this Close skips its own cleanup
// (closeAgent finds a nil agent), so the flags must be dropped here too or
// the reconnection guard reads the stale state as Connected forever.
w.abandonNegotiation()
}
func (w *ICE) reCreateAgent(ctx context.Context, dialerCancel context.CancelFunc, candidates []ice.CandidateType) (*icemaker.ThreadSafeAgent, error) {
@@ -273,8 +278,14 @@ func (w *ICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agen
return
}
w.log.Debugf("turn agent dial")
remoteConn, err := w.turnAgentDial(ctx, agent, remoteOfferAnswer)
w.log.Debugf("agent dial")
dial := func(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *signaling.OfferAnswer) (net.Conn, error) {
return w.agentDial(ctx, agent, remoteOfferAnswer)
}
if w.dialFunc != nil {
dial = w.dialFunc
}
remoteConn, err := dial(ctx, agent, remoteOfferAnswer)
if err != nil {
w.log.Debugf("failed to dial the remote peer: %s", err)
w.closeAgent(agent, dialerCancel)
@@ -282,6 +293,19 @@ func (w *ICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agen
}
w.log.Debugf("agent dial succeeded")
// A newer negotiation may have replaced this agent during the dial.
// Discard its connection before querying candidates or punching ports.
w.muxAgent.Lock()
stale := w.agent != agent
w.muxAgent.Unlock()
if stale {
if err := remoteConn.Close(); err != nil {
w.log.Warnf("failed to close stale ICE connection: %s", err)
}
w.log.Warnf("discarding connection from a stale ICE negotiation")
return
}
pair, err := agent.GetSelectedCandidatePair()
if err != nil {
w.closeAgent(agent, dialerCancel)
@@ -318,9 +342,14 @@ func (w *ICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agen
w.log.Debugf("on ICE conn is ready to use")
w.muxAgent.Lock()
// Keep the ownership check atomic with the state update so a stale dial
// cannot overwrite a newer negotiation.
if w.agent != agent {
w.muxAgent.Unlock()
w.log.Debugf("agent has been replaced during connect, dropping obsolete connection")
if err := remoteConn.Close(); err != nil {
w.log.Warnf("failed to close stale ICE connection: %s", err)
}
w.log.Warnf("discarding connection from a stale ICE negotiation")
return
}
w.agentConnecting = false
@@ -344,20 +373,27 @@ func (w *ICE) closeAgent(agent *icemaker.ThreadSafeAgent, cancel context.CancelF
sessionChanged := w.remoteSessionChanged
w.remoteSessionChanged = false
// Only the owner of the current session may reset its state: a stale dial
// goroutine waking after a newer attempt must not clobber it.
if w.agent == agent {
// consider to remove from here and move to the OnNewOffer
sessionID, err := icemaker.NewSessionID()
if err != nil {
w.log.Errorf("failed to create new session ID: %s", err)
}
w.sessionID = sessionID
w.agent = nil
w.agentConnecting = false
w.remoteSessionID = ""
w.abandonNegotiation()
}
return sessionChanged
}
// Clearing the agent and connecting flag together keeps retries from stalling.
// Callers must dispose of the agent first and hold muxAgent.
func (w *ICE) abandonNegotiation() {
w.agent = nil
w.agentConnecting = false
w.remoteSessionID = ""
}
func (w *ICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) {
// wait local endpoint configuration
time.Sleep(time.Second)
@@ -412,6 +448,17 @@ func (w *ICE) injectPortForwardedCandidate(srflxCandidate ice.Candidate) {
return
}
// A forwarded candidate only makes sense for an IPv4 mapping, which
// translates a port on the gateway's address. An IPv6 pinhole translates
// nothing: it unblocks the address ICE already gathers as a host candidate,
// so there is no second address to advertise. Injecting one here would also
// paste an IPv6 address onto whichever server-reflexive candidate arrived
// first, which is usually IPv4.
if mapping.ExternalIP != nil && mapping.ExternalIP.To4() == nil {
w.log.Debugf("skipping port-forwarded candidate: %s mapping is IPv6-only", mapping.NATType)
return
}
w.muxAgent.Lock()
if w.portForwardAttempted {
w.muxAgent.Unlock()
@@ -541,8 +588,8 @@ func (w *ICE) onConnectionStateChange(agent *icemaker.ThreadSafeAgent, dialerCan
connected = true
w.logSuccessfulPaths(agent)
case ice.ConnectionStateFailed, ice.ConnectionStateDisconnected, ice.ConnectionStateClosed:
// ice.ConnectionStateClosed happens when we recreate the agent. For the P2P to TURN switch important to
// notify the conn.onICEStateDisconnected changes to update the current used priority
// ice.ConnectionStateClosed happens when we recreate the agent. The P2P to relay switch requires
// notifying conn.onICEStateDisconnected so it can update the currently used priority.
sessionChanged := w.closeAgent(agent, dialerCancel)
@@ -567,7 +614,7 @@ func (w *ICE) onConnectionStateChange(agent *icemaker.ThreadSafeAgent, dialerCan
}
}
func (w *ICE) turnAgentDial(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *signaling.OfferAnswer) (*ice.Conn, error) {
func (w *ICE) agentDial(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *signaling.OfferAnswer) (*ice.Conn, error) {
if w.isController {
return agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
} else {
@@ -0,0 +1,258 @@
package worker
import (
"context"
"net"
"sync/atomic"
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
"github.com/netbirdio/netbird/client/internal/peer/signaling"
signal "github.com/netbirdio/netbird/shared/signal/client"
sProto "github.com/netbirdio/netbird/shared/signal/proto"
)
// stubSignalClient satisfies signal.Client as a no-op so the candidate
// goroutine spawned by a real GatherCandidates never dereferences a nil
// signaler in tests.
type stubSignalClient struct{}
func (stubSignalClient) Close() error { return nil }
func (stubSignalClient) StreamConnected() bool { return false }
func (stubSignalClient) GetStatus() signal.Status { return signal.StreamDisconnected }
func (stubSignalClient) Receive(context.Context, func(*sProto.Message) error) error { return nil }
func (stubSignalClient) Ready() bool { return false }
func (stubSignalClient) IsHealthy() bool { return false }
func (stubSignalClient) WaitStreamConnected(context.Context) {}
func (stubSignalClient) SendToStream(*sProto.EncryptedMessage) error { return nil }
func (stubSignalClient) Send(*sProto.Message) error { return nil }
func (stubSignalClient) SetOnReconnectedListener(func()) {}
// newTestWorkerICE builds a worker with real pion plumbing and no-op signaling.
func newTestWorkerICE(t *testing.T) *ICE {
t.Helper()
config := icemaker.Config{}
stunTurn := &icemaker.StunTurn{}
stunTurn.Store(nil)
config.StunTurn = stunTurn
w, err := NewICE(log.WithField("test", t.Name()), "test-peer", config, true, nil, nil,
ICEDependencies{Signaler: signaling.NewSignaler(stubSignalClient{}, wgtypes.Key{})}, false)
require.NoError(t, err, "worker setup must succeed")
return w
}
// TestWorkerICE_CloseDuringDial_ClearsConnectingFlag drives the teardown race
// through the real dial goroutine instead of simulating its cleanup.
//
// The real-world sequence this models:
// 1. OnNewOffer starts a negotiation: agent set, agentConnecting = true,
// go connect()
// 2. The network dies and connect() stays blocked inside GatherCandidates/Dial
// 3. A WG handshake timeout calls Close(): the agent is released and the dial
// context cancelled, but agentConnecting is not reset
// 4. The real goroutine wakes with an error and runs its own cleanup
// (closeAgent), where `w.agent == agent` is now false, so the flag reset
// is skipped
//
// There is no remote responder, so Dial can never succeed: whatever point the
// goroutine is at, closing first forces it down the error path. Before the fix
// the flag stays true forever and the deadline below expires.
func TestWorkerICE_CloseDuringDial_ClearsConnectingFlag(t *testing.T) {
w := newTestWorkerICE(t)
sid := icemaker.SessionID("test-session-id")
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
IceCredentials: signaling.IceCredentials{
UFrag: "testufrag",
Pwd: "testpwdtestpwdtestpwd12",
},
SessionID: &sid,
})
require.True(t, w.InProgress(), "OnNewOffer must mark the negotiation as in progress")
// Teardown wins the race while connect() is still running.
w.Close()
// Close drops the flags synchronously, so the assertion below does not
// converge on the goroutine: the deadline only absorbs the dial goroutine
// waking up in the background, proving nothing re-wedges it afterwards.
require.Eventually(t, func() bool {
return !w.InProgress()
}, 10*time.Second, 50*time.Millisecond,
"Close must leave the negotiation idle even while the dial goroutine is still winding down")
// abandonNegotiation owns these three fields together; the worker is idle
// only when all of them are dropped.
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
assert.Nil(t, w.agent, "no agent may survive the teardown")
assert.False(t, w.agentConnecting, "the connecting flag must match the nil agent")
assert.Empty(t, w.remoteSessionID, "a dead session's remote ID must not linger")
}
// TestWorkerICE_CloseClearsResidualConnectingState covers Close on a worker whose
// agent is already gone but whose flag is stuck on true, e.g. after an aborted
// recreate in OnNewOffer or after a first Close raced a dial goroutine.
func TestWorkerICE_CloseClearsResidualConnectingState(t *testing.T) {
w := newTestWorkerICE(t)
w.muxAgent.Lock()
w.agentConnecting = true
w.muxAgent.Unlock()
w.Close()
assert.False(t, w.InProgress(), "Close must drop residual connecting state even without a live agent")
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
assert.Nil(t, w.agent)
assert.False(t, w.agentConnecting)
assert.Empty(t, w.remoteSessionID)
}
// TestWorkerICE_StaleCloseAgentKeepsCurrentSession pins the ownership guard in
// closeAgent: a late-waking dial goroutine from an older session must not reset
// the state of a newer negotiation that reused the worker. The newer session
// must survive wholesale - agent, flag and remote session identity alike.
func TestWorkerICE_StaleCloseAgentKeepsCurrentSession(t *testing.T) {
w := newTestWorkerICE(t)
t.Cleanup(w.Close)
sidA := icemaker.SessionID("session-a")
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
IceCredentials: signaling.IceCredentials{UFrag: "ufragaaaa", Pwd: "pwdpwdpwdpwdpwdpwdpwdp1"},
SessionID: &sidA,
})
w.muxAgent.Lock()
oldAgent := w.agent
oldCancel := w.agentDialerCancel
w.muxAgent.Unlock()
require.NotNil(t, oldAgent, "OnNewOffer must have created an ICE agent")
w.Close()
sidB := icemaker.SessionID("session-b")
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
IceCredentials: signaling.IceCredentials{UFrag: "ufragbbbb", Pwd: "pwdpwdpwdpwdpwdpwdpwdp2"},
SessionID: &sidB,
})
require.True(t, w.InProgress(), "the second negotiation must be in flight")
w.muxAgent.Lock()
newAgent := w.agent
w.muxAgent.Unlock()
// The old dial goroutine finally wakes and cleans up its captured agent.
w.closeAgent(oldAgent, oldCancel)
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
assert.Same(t, newAgent, w.agent, "the current agent must be untouched by the stale cleanup")
assert.True(t, w.agentConnecting, "the current negotiation must stay in flight")
// Read live under the lock: a snapshot captured before the stale cleanup
// would pass even if the cleanup wiped current state.
assert.Equal(t, sidB, w.remoteSessionID, "the remote session identity must be preserved")
}
// closeTrackConn records Close calls so a test can assert that a discarded
// connection was actually released.
type closeTrackConn struct {
net.Conn
closed atomic.Bool
}
func (c *closeTrackConn) Close() error {
c.closed.Store(true)
return c.Conn.Close()
}
// TestWorkerICE_StaleDialSuccessKeepsNewerNegotiation pins the ownership guard
// in connect()'s success path: a dial that came back after a newer negotiation
// replaced the agent must discard its connection and leave the newer session's
// state - agent, agentConnecting, remoteSessionID, lastSuccess - intact.
//
// The dial hook holds session A's goroutine open until session B is installed,
// then returns a live connection, mimicking the vendored pion dial which hands
// out a live *ice.Conn when a pair is selected without checking afterwards
// whether the agent was replaced meanwhile. Releasing A's dial therefore
// exercises the stale-success commit path deterministically instead of racing
// real ICE.
func TestWorkerICE_StaleDialSuccessKeepsNewerNegotiation(t *testing.T) {
w := newTestWorkerICE(t)
t.Cleanup(w.Close)
dialStarted := make(chan struct{})
releaseDial := make(chan struct{})
staleConn := &closeTrackConn{}
var calls atomic.Int32
w.dialFunc = func(ctx context.Context, _ *icemaker.ThreadSafeAgent, _ *signaling.OfferAnswer) (net.Conn, error) {
if calls.Add(1) == 1 {
// Session A: hold the goroutine open until session B is installed,
// then return a live connection, mimicking the vendored pion dial
// which hands out a live *ice.Conn once a pair is selected without
// re-checking whether the agent was replaced meanwhile. Releasing
// the dial therefore exercises the stale-success commit path
// deterministically instead of racing real ICE.
close(dialStarted)
<-releaseDial
client, _ := net.Pipe()
staleConn.Conn = client
return staleConn, nil
}
// A newer negotiation parks on its dialer context, cancelled by the
// t.Cleanup Close at test end.
<-ctx.Done()
return nil, ctx.Err()
}
sidA := icemaker.SessionID("session-a")
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
IceCredentials: signaling.IceCredentials{UFrag: "ufragaaaa", Pwd: "pwdpwdpwdpwdpwdpwdpwdp1"},
SessionID: &sidA,
})
require.True(t, w.InProgress(), "session A must be in flight")
// Session A's goroutine is now parked in the dial hook.
<-dialStarted
sidB := icemaker.SessionID("session-b")
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
IceCredentials: signaling.IceCredentials{UFrag: "ufragbbbb", Pwd: "pwdpwdpwdpwdpwdpwdpwdp2"},
SessionID: &sidB,
})
w.muxAgent.Lock()
agentB := w.agent
w.lastSuccess = time.Time{}
w.muxAgent.Unlock()
require.NotNil(t, agentB, "session B must have created an ICE agent")
require.True(t, w.InProgress(), "session B must be in flight")
// Release session A's dial: it must be recognized as stale and discarded.
close(releaseDial)
require.Eventually(t, func() bool {
return staleConn.closed.Load()
}, 10*time.Second, 10*time.Millisecond,
"the stale connection must be closed by the ownership guard")
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
assert.Same(t, agentB, w.agent, "session A must not uninstall session B's agent")
assert.True(t, w.agentConnecting, "session A must not clear session B's connecting flag")
assert.Equal(t, sidB, w.remoteSessionID, "session A must not clear session B's remote session identity")
assert.True(t, w.lastSuccess.IsZero(), "session A must not record a success for session B")
// The commit block guards agentConnecting, lastSuccess and
// onICEConnectionIsReady together, so the state assertions above imply the
// callback never ran for session A; the nil conn would have panicked the
// stale goroutine on any invocation.
}