mirror of
https://github.com/netbirdio/netbird.git
synced 2026-09-18 12:49:07 +02:00
[client] Merge main into peer event bus refactor
This commit is contained in:
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal/portforward"
|
||||
"github.com/netbirdio/netbird/client/internal/rosenpass"
|
||||
"github.com/netbirdio/netbird/client/internal/stdnet"
|
||||
"github.com/netbirdio/netbird/client/netevents"
|
||||
"github.com/netbirdio/netbird/route"
|
||||
relayClient "github.com/netbirdio/netbird/shared/relay/client"
|
||||
)
|
||||
@@ -104,6 +105,10 @@ type ConnConfig struct {
|
||||
|
||||
// ICEConfig ICE protocol configuration
|
||||
ICEConfig icemaker.Config
|
||||
|
||||
// NetMgr gates the reconnection guard on OS-reported network
|
||||
// availability; nil disables gating.
|
||||
NetMgr *netevents.Manager
|
||||
}
|
||||
|
||||
func (c ConnConfig) IsController() bool {
|
||||
@@ -265,7 +270,7 @@ func (conn *Conn) open(engineCtx context.Context, firstPacket []byte) error {
|
||||
RosenpassAddr: conn.config.RosenpassConfig.Addr,
|
||||
}, conn.signaler, iceWorker, conn.relayManager)
|
||||
|
||||
conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher)
|
||||
conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher, conn.config.NetMgr)
|
||||
|
||||
conn.relayDialInFlight = false
|
||||
conn.pendingRelayOffer = nil
|
||||
@@ -481,6 +486,7 @@ func (conn *Conn) teardown(mb *mailbox, leftover []event, signalToRemote bool, d
|
||||
|
||||
if conn.wgWatcherCancel != nil {
|
||||
conn.wgWatcherCancel()
|
||||
conn.wgWatcher = nil
|
||||
conn.wgWatcherCancel = nil
|
||||
}
|
||||
conn.workerRelay.CloseConn()
|
||||
@@ -650,7 +656,7 @@ func (conn *Conn) handleICEReady(priority worker.ConnPriority, iceConnInfo worke
|
||||
conn.dumpState.NewLocalProxy()
|
||||
wgProxy, err = conn.newProxy(iceConnInfo.RemoteConn)
|
||||
if err != nil {
|
||||
conn.Log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
|
||||
conn.Log.Errorf("failed to add relayed net.Conn to local proxy: %v", err)
|
||||
return
|
||||
}
|
||||
ep = wgProxy.EndpointAddr()
|
||||
@@ -1142,9 +1148,8 @@ func (conn *Conn) newProxy(remoteConn net.Conn) (wgproxy.Proxy, error) {
|
||||
}
|
||||
|
||||
wgProxy := conn.config.WgConfig.WgInterface.GetProxy()
|
||||
if err := wgProxy.AddTurnConn(conn.ctx, udpAddr, remoteConn); err != nil {
|
||||
conn.Log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
|
||||
return nil, err
|
||||
if err := wgProxy.AddRelayedConn(conn.ctx, udpAddr, remoteConn); err != nil {
|
||||
return nil, fmt.Errorf("add relayed conn to proxy: %w", err)
|
||||
}
|
||||
return wgProxy, nil
|
||||
}
|
||||
@@ -1205,12 +1210,9 @@ func (conn *Conn) recordConnectionMetrics() {
|
||||
return
|
||||
}
|
||||
|
||||
var connType metrics.ConnectionType
|
||||
switch conn.currentConnPriority {
|
||||
case worker.Relay:
|
||||
connType = metrics.ConnectionTypeRelay
|
||||
default:
|
||||
connType = metrics.ConnectionTypeICE
|
||||
connType := metricsConnType(conn.currentConnPriority)
|
||||
if connType == metrics.ConnectionTypeUnknown {
|
||||
return
|
||||
}
|
||||
|
||||
// Record metrics with timestamps - duration calculation happens in metrics package
|
||||
@@ -1298,3 +1300,16 @@ func boolToConnStatus(connected bool) guard.ConnStatus {
|
||||
}
|
||||
return guard.ConnStatusDisconnected
|
||||
}
|
||||
|
||||
func metricsConnType(priority worker.ConnPriority) metrics.ConnectionType {
|
||||
switch priority {
|
||||
case worker.Relay:
|
||||
return metrics.ConnectionTypeRelay
|
||||
case worker.ICETurn:
|
||||
return metrics.ConnectionTypeICETurn
|
||||
case worker.ICEP2P:
|
||||
return metrics.ConnectionTypeICEP2P
|
||||
default:
|
||||
return metrics.ConnectionTypeUnknown
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package peer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/client/internal/peer/metricsstages"
|
||||
"github.com/netbirdio/netbird/client/internal/peer/signaling"
|
||||
)
|
||||
|
||||
func TestConn_AnswerBeforeEventLoop(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ports []int
|
||||
}{
|
||||
{name: "holds early answer", ports: []int{51820}},
|
||||
{name: "keeps latest answer", ports: []int{1111, 2222}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
conn, err := NewConn(connConf, ServiceDependencies{})
|
||||
require.NoError(t, err)
|
||||
conn.metricsStages = &metricsstages.MetricsStages{}
|
||||
conn.handshaker = signaling.NewHandshaker(conn.Log, signaling.Config{}, nil, nil, nil)
|
||||
// A relay dial in progress retains the dispatched answer as its next offer.
|
||||
conn.relayDialInFlight = true
|
||||
mb := newMailbox()
|
||||
conn.mailbox.Store(mb)
|
||||
|
||||
// Incoming answers can arrive after Open publishes the mailbox but
|
||||
// before the event loop gets scheduled to consume it.
|
||||
for _, port := range tc.ports {
|
||||
conn.OnRemoteAnswer(signaling.OfferAnswer{WgListenPort: port})
|
||||
}
|
||||
|
||||
select {
|
||||
case <-mb.wake:
|
||||
default:
|
||||
t.Fatal("an early answer must wake the event loop")
|
||||
}
|
||||
events := mb.drain()
|
||||
require.Len(t, events, 1, "only the latest answer should reach the event loop")
|
||||
for _, ev := range events {
|
||||
conn.handleEvent(ev)
|
||||
}
|
||||
require.NotNil(t, conn.pendingRelayOffer, "the answer must reach relay dispatch")
|
||||
assert.Equal(t, tc.ports[len(tc.ports)-1], conn.pendingRelayOffer.WgListenPort,
|
||||
"relay dispatch must receive the latest queued answer")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -13,11 +13,13 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/client/iface"
|
||||
"github.com/netbirdio/netbird/client/internal/metrics"
|
||||
"github.com/netbirdio/netbird/client/internal/peer/guard"
|
||||
"github.com/netbirdio/netbird/client/internal/peer/ice"
|
||||
"github.com/netbirdio/netbird/client/internal/peer/metricsstages"
|
||||
"github.com/netbirdio/netbird/client/internal/peer/signaling"
|
||||
"github.com/netbirdio/netbird/client/internal/peer/status"
|
||||
"github.com/netbirdio/netbird/client/internal/peer/worker"
|
||||
"github.com/netbirdio/netbird/client/internal/stdnet"
|
||||
"github.com/netbirdio/netbird/util"
|
||||
)
|
||||
@@ -354,3 +356,33 @@ func TestConn_onWGDisconnected_NoEscalationWithoutRosenpass(t *testing.T) {
|
||||
}
|
||||
assert.Empty(t, disconnected, "escalation must be limited to rosenpass connections")
|
||||
}
|
||||
|
||||
func TestMetricsConnType(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
priority worker.ConnPriority
|
||||
expected metrics.ConnectionType
|
||||
}{
|
||||
{"relay", worker.Relay, metrics.ConnectionTypeRelay},
|
||||
{"ice over turn is relayed, not p2p", worker.ICETurn, metrics.ConnectionTypeICETurn},
|
||||
{"direct p2p", worker.ICEP2P, metrics.ConnectionTypeICEP2P},
|
||||
{"unset priority is unknown, not p2p", worker.None, metrics.ConnectionTypeUnknown},
|
||||
{"unrecognised priority is unknown", worker.ConnPriority(99), metrics.ConnectionTypeUnknown},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.expected, metricsConnType(tc.priority))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsConnType_RelayedMatchesIsRelayed(t *testing.T) {
|
||||
for _, priority := range []worker.ConnPriority{worker.None, worker.Relay, worker.ICETurn, worker.ICEP2P} {
|
||||
conn := &Conn{currentConnPriority: priority}
|
||||
tag := metricsConnType(priority)
|
||||
relayedTag := tag == metrics.ConnectionTypeRelay || tag == metrics.ConnectionTypeICETurn
|
||||
assert.Equal(t, conn.isRelayed(), relayedTag,
|
||||
"priority %s: isRelayed and the %q metric tag must agree", priority, tag)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,12 @@ const (
|
||||
|
||||
type connStatusFunc func() ConnStatus
|
||||
|
||||
// NetworkWatcher is the availability view the guard gates reconnects on.
|
||||
type NetworkWatcher interface {
|
||||
IsOnline() bool
|
||||
Changed() <-chan struct{}
|
||||
}
|
||||
|
||||
// Guard is responsible for the reconnection logic.
|
||||
// It will trigger to send an offer to the peer then has connection issues.
|
||||
// Watch these events:
|
||||
@@ -31,20 +37,26 @@ type connStatusFunc func() ConnStatus
|
||||
// - Relayed connection disconnected
|
||||
// - ICE candidate changes
|
||||
type Guard struct {
|
||||
log *log.Entry
|
||||
isConnectedOnAllWay connStatusFunc
|
||||
timeout time.Duration
|
||||
srWatcher *SRWatcher
|
||||
log *log.Entry
|
||||
isConnectedOnAllWay connStatusFunc
|
||||
timeout time.Duration
|
||||
srWatcher *SRWatcher
|
||||
// netWatcher gates reconnect attempts on OS-reported network availability;
|
||||
// nil disables gating.
|
||||
netWatcher NetworkWatcher
|
||||
relayedConnDisconnected chan struct{}
|
||||
iCEConnDisconnected chan struct{}
|
||||
}
|
||||
|
||||
func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher) *Guard {
|
||||
// NewGuard creates a reconnection guard for a peer connection. A nil netWatcher
|
||||
// disables network availability gating.
|
||||
func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher, netWatcher NetworkWatcher) *Guard {
|
||||
return &Guard{
|
||||
log: log,
|
||||
isConnectedOnAllWay: isConnectedFn,
|
||||
timeout: timeout,
|
||||
srWatcher: srWatcher,
|
||||
netWatcher: netWatcher,
|
||||
relayedConnDisconnected: make(chan struct{}, 1),
|
||||
iCEConnDisconnected: make(chan struct{}, 1),
|
||||
}
|
||||
@@ -96,9 +108,19 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) {
|
||||
iceState := &iceRetryState{log: g.log}
|
||||
defer iceState.reset()
|
||||
|
||||
var netChanged <-chan struct{}
|
||||
if g.netWatcher != nil {
|
||||
netChanged = g.netWatcher.Changed()
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-tickerChannel:
|
||||
// skip attempts while the OS reports no usable network; the
|
||||
// netChanged case below resumes the loop once it returns
|
||||
if g.netWatcher != nil && !g.netWatcher.IsOnline() {
|
||||
continue
|
||||
}
|
||||
switch g.isConnectedOnAllWay() {
|
||||
case ConnStatusConnected:
|
||||
// all good, nothing to do
|
||||
@@ -135,6 +157,23 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) {
|
||||
tickerChannel = ticker.C
|
||||
iceState.reset()
|
||||
|
||||
case <-netChanged:
|
||||
// Re-arm for the next transition before acting on this one.
|
||||
netChanged = g.netWatcher.Changed()
|
||||
if !g.netWatcher.IsOnline() {
|
||||
continue
|
||||
}
|
||||
// Ticks skipped while offline drove the backoff towards its
|
||||
// maximum without ever attempting, and left the ICE budget
|
||||
// frozen — possibly in hourly mode. Recover on our own so the
|
||||
// peer does not depend on a signal or relay event that never
|
||||
// comes when both stayed up across the outage.
|
||||
g.log.Debugf("network is back, reset reconnection ticker")
|
||||
ticker.Stop()
|
||||
ticker = g.newReconnectTicker(ctx)
|
||||
tickerChannel = ticker.C
|
||||
iceState.reset()
|
||||
|
||||
case <-ctx.Done():
|
||||
g.log.Debugf("context is done, stop reconnect loop")
|
||||
return
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
func newTestGuard(status connStatusFunc) *Guard {
|
||||
srw := NewSRWatcher(nil, nil, nil, ice.Config{})
|
||||
return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw)
|
||||
return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw, nil)
|
||||
}
|
||||
|
||||
// countBackoffTickerGoroutines returns how many goroutines are currently sitting
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package guard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/client/internal/peer/ice"
|
||||
"github.com/netbirdio/netbird/client/netevents/netstate"
|
||||
)
|
||||
|
||||
// newTestGuardWithNetState builds a guard with a realistic MaxInterval: the
|
||||
// backoff must be able to grow well past the outage, as it does in production
|
||||
// where the timeout is seconds to minutes.
|
||||
func newTestGuardWithNetState(status connStatusFunc, netState *netstate.State) *Guard {
|
||||
srw := NewSRWatcher(nil, nil, nil, ice.Config{})
|
||||
return NewGuard(log.WithField("test", "guard"), status, 30*time.Second, srw, netState)
|
||||
}
|
||||
|
||||
// TestGuard_RecoversAfterOfflineToOnline covers a peer that stays disconnected
|
||||
// across a network outage while neither signal nor relay reports an event —
|
||||
// both stayed up, as on a short airplane mode toggle over Wi-Fi.
|
||||
//
|
||||
// Every tick taken while offline is skipped, but it still advances the
|
||||
// exponential backoff, so by the time the network returns the next tick can be
|
||||
// tens of seconds away. Without an explicit reaction to the transition the
|
||||
// peer waits out that interval for a recovery that could start immediately.
|
||||
func TestGuard_RecoversAfterOfflineToOnline(t *testing.T) {
|
||||
netState := netstate.New()
|
||||
|
||||
var attempts atomic.Int32
|
||||
g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
// Start from the reconnect ticker (800ms initial interval), the state a
|
||||
// peer is in after it loses its connection.
|
||||
go g.Start(ctx, func() { attempts.Add(1) })
|
||||
g.SetRelayedConnDisconnected()
|
||||
|
||||
// Let the backoff climb: 0.8s, 1.6s, 3.2s, 6.4s ... every tick is skipped
|
||||
// while offline, but each one doubles the wait for the next.
|
||||
netState.Set(false)
|
||||
time.Sleep(8 * time.Second)
|
||||
|
||||
offlineAttempts := attempts.Load()
|
||||
if offlineAttempts != 0 {
|
||||
t.Fatalf("callback ran %d times while offline, want 0", offlineAttempts)
|
||||
}
|
||||
|
||||
netState.Set(true)
|
||||
|
||||
// The next organic tick is now several seconds out, so anything within
|
||||
// this window can only come from reacting to the transition itself.
|
||||
pollCtx, stopPolling := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer stopPolling()
|
||||
|
||||
select {
|
||||
case <-pollCtx.Done():
|
||||
t.Fatal("peer was not retried within 2s of the network coming back, " +
|
||||
"with neither a signal nor a relay event to fall back on")
|
||||
case <-pollUntil(pollCtx, func() bool { return attempts.Load() > 0 }):
|
||||
}
|
||||
}
|
||||
|
||||
// TestGuard_OfflineTransitionDoesNotRetry checks the other direction: going
|
||||
// offline must not itself trigger an attempt.
|
||||
func TestGuard_OfflineTransitionDoesNotRetry(t *testing.T) {
|
||||
netState := netstate.New()
|
||||
|
||||
var attempts atomic.Int32
|
||||
g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go g.Start(ctx, func() { attempts.Add(1) })
|
||||
|
||||
netState.Set(false)
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
if got := attempts.Load(); got != 0 {
|
||||
t.Fatalf("callback ran %d times after going offline, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// pollUntil closes the returned channel once cond holds. It gives up when ctx
|
||||
// is done, so the polling goroutine never outlives the test that started it.
|
||||
func pollUntil(ctx context.Context, cond func() bool) <-chan struct{} {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
if cond() {
|
||||
close(done)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}()
|
||||
return done
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package status
|
||||
|
||||
// ClientState identifies the client connection state delivered via
|
||||
// Listener.OnStateChanged.
|
||||
type ClientState int
|
||||
|
||||
// Client states. The numeric values cross the gomobile boundary (the mobile
|
||||
// bindings re-export them as integer constants), so they are a wire format:
|
||||
// append new states at the end, never reorder or insert.
|
||||
const (
|
||||
ClientStateDisconnected ClientState = iota
|
||||
ClientStateConnected
|
||||
ClientStateConnecting
|
||||
ClientStateDisconnecting
|
||||
// ClientStateNoNetwork is an overlay state: it is never stored as the
|
||||
// last notification, only derived from ClientStateConnecting while the
|
||||
// OS reports no usable network (see notifier.effectiveState).
|
||||
ClientStateNoNetwork
|
||||
)
|
||||
|
||||
// Listener is a callback type about the NetBird network connection state
|
||||
type Listener interface {
|
||||
// OnStateChanged reports every client state transition. New states are
|
||||
// delivered only through this callback; the per-state callbacks below
|
||||
// are kept for compatibility and will be removed once all consumers
|
||||
// have migrated.
|
||||
OnStateChanged(state ClientState)
|
||||
|
||||
// Deprecated: consume OnStateChanged instead.
|
||||
OnConnected()
|
||||
// Deprecated: consume OnStateChanged instead.
|
||||
OnDisconnected()
|
||||
// Deprecated: consume OnStateChanged instead.
|
||||
OnConnecting()
|
||||
// Deprecated: consume OnStateChanged instead.
|
||||
OnDisconnecting()
|
||||
|
||||
OnAddressChanged(string, string)
|
||||
OnPeersListChanged(int)
|
||||
}
|
||||
@@ -4,41 +4,64 @@ import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
stateDisconnected = iota
|
||||
stateConnected
|
||||
stateConnecting
|
||||
stateDisconnecting
|
||||
)
|
||||
|
||||
// Listener is a callback type about the NetBird network connection state
|
||||
type Listener interface {
|
||||
OnConnected()
|
||||
OnDisconnected()
|
||||
OnConnecting()
|
||||
OnDisconnecting()
|
||||
OnAddressChanged(string, string)
|
||||
OnPeersListChanged(int)
|
||||
}
|
||||
|
||||
type notifier struct {
|
||||
// publishLock orders state publication: it is held across computing the
|
||||
// effective state and handing it to the listener, so a transition cannot
|
||||
// overtake a newer one and leave the listener on a stale state.
|
||||
publishLock sync.Mutex
|
||||
serverStateLock sync.Mutex
|
||||
listenersLock sync.Mutex
|
||||
listener Listener
|
||||
currentClientState bool
|
||||
lastNotification int
|
||||
lastNotification ClientState
|
||||
lastNumberOfPeers int
|
||||
lastFqdnAddress string
|
||||
lastIPAddress string
|
||||
networkAvailable bool
|
||||
}
|
||||
|
||||
func newNotifier() *notifier {
|
||||
return ¬ifier{}
|
||||
return ¬ifier{
|
||||
networkAvailable: true,
|
||||
}
|
||||
}
|
||||
|
||||
// effectiveState maps the computed state to what listeners should see:
|
||||
// while the OS reports no usable network, "Connecting" would be a lie —
|
||||
// connection attempts are suspended — so it is reported as NoNetwork.
|
||||
// Caller must hold serverStateLock.
|
||||
func (n *notifier) effectiveState(state ClientState) ClientState {
|
||||
if !n.networkAvailable && state == ClientStateConnecting {
|
||||
return ClientStateNoNetwork
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// setNetworkAvailable records the OS network availability and re-notifies
|
||||
// the listener when the flag flips the effective state (Connecting <->
|
||||
// NoNetwork).
|
||||
func (n *notifier) setNetworkAvailable(available bool) {
|
||||
n.publishLock.Lock()
|
||||
defer n.publishLock.Unlock()
|
||||
|
||||
n.serverStateLock.Lock()
|
||||
if n.networkAvailable == available {
|
||||
n.serverStateLock.Unlock()
|
||||
return
|
||||
}
|
||||
previous := n.effectiveState(n.lastNotification)
|
||||
n.networkAvailable = available
|
||||
current := n.effectiveState(n.lastNotification)
|
||||
n.serverStateLock.Unlock()
|
||||
|
||||
if previous != current {
|
||||
n.notify(current)
|
||||
}
|
||||
}
|
||||
|
||||
func (n *notifier) setListener(listener Listener) {
|
||||
n.serverStateLock.Lock()
|
||||
lastNotification := n.lastNotification
|
||||
lastNotification := n.effectiveState(n.lastNotification)
|
||||
numOfPeers := n.lastNumberOfPeers
|
||||
fqdnAddress := n.lastFqdnAddress
|
||||
address := n.lastIPAddress
|
||||
@@ -62,6 +85,9 @@ func (n *notifier) removeListener() {
|
||||
}
|
||||
|
||||
func (n *notifier) updateServerStates(mgmState bool, signalState bool) {
|
||||
n.publishLock.Lock()
|
||||
defer n.publishLock.Unlock()
|
||||
|
||||
n.serverStateLock.Lock()
|
||||
calculatedState := n.calculateState(mgmState, signalState)
|
||||
|
||||
@@ -71,43 +97,54 @@ func (n *notifier) updateServerStates(mgmState bool, signalState bool) {
|
||||
}
|
||||
|
||||
n.lastNotification = calculatedState
|
||||
effective := n.effectiveState(calculatedState)
|
||||
n.serverStateLock.Unlock()
|
||||
|
||||
n.notify(calculatedState)
|
||||
n.notify(effective)
|
||||
}
|
||||
|
||||
func (n *notifier) clientStart() {
|
||||
n.publishLock.Lock()
|
||||
defer n.publishLock.Unlock()
|
||||
|
||||
n.serverStateLock.Lock()
|
||||
n.currentClientState = true
|
||||
n.lastNotification = stateConnecting
|
||||
n.lastNotification = ClientStateConnecting
|
||||
effective := n.effectiveState(ClientStateConnecting)
|
||||
n.serverStateLock.Unlock()
|
||||
|
||||
n.notify(stateConnecting)
|
||||
n.notify(effective)
|
||||
}
|
||||
|
||||
func (n *notifier) clientStop() {
|
||||
n.publishLock.Lock()
|
||||
defer n.publishLock.Unlock()
|
||||
|
||||
n.serverStateLock.Lock()
|
||||
n.currentClientState = false
|
||||
n.lastNotification = stateDisconnected
|
||||
n.lastNotification = ClientStateDisconnected
|
||||
n.serverStateLock.Unlock()
|
||||
|
||||
n.notify(stateDisconnected)
|
||||
n.notify(ClientStateDisconnected)
|
||||
}
|
||||
|
||||
func (n *notifier) clientTearDown() {
|
||||
n.publishLock.Lock()
|
||||
defer n.publishLock.Unlock()
|
||||
|
||||
n.serverStateLock.Lock()
|
||||
n.currentClientState = false
|
||||
n.lastNotification = stateDisconnecting
|
||||
n.lastNotification = ClientStateDisconnecting
|
||||
n.serverStateLock.Unlock()
|
||||
|
||||
n.notify(stateDisconnecting)
|
||||
n.notify(ClientStateDisconnecting)
|
||||
}
|
||||
|
||||
func (n *notifier) isServerStateChanged(newState int) bool {
|
||||
func (n *notifier) isServerStateChanged(newState ClientState) bool {
|
||||
return n.lastNotification != newState
|
||||
}
|
||||
|
||||
func (n *notifier) notify(state int) {
|
||||
func (n *notifier) notify(state ClientState) {
|
||||
n.listenersLock.Lock()
|
||||
listener := n.listener
|
||||
n.listenersLock.Unlock()
|
||||
@@ -119,20 +156,20 @@ func (n *notifier) notify(state int) {
|
||||
notifyListener(listener, state)
|
||||
}
|
||||
|
||||
func (n *notifier) calculateState(managementConn, signalConn bool) int {
|
||||
func (n *notifier) calculateState(managementConn, signalConn bool) ClientState {
|
||||
if managementConn && signalConn {
|
||||
return stateConnected
|
||||
return ClientStateConnected
|
||||
}
|
||||
|
||||
if !managementConn && !signalConn && !n.currentClientState {
|
||||
return stateDisconnected
|
||||
return ClientStateDisconnected
|
||||
}
|
||||
|
||||
if n.lastNotification == stateDisconnecting {
|
||||
return stateDisconnecting
|
||||
if n.lastNotification == ClientStateDisconnecting {
|
||||
return ClientStateDisconnecting
|
||||
}
|
||||
|
||||
return stateConnecting
|
||||
return ClientStateConnecting
|
||||
}
|
||||
|
||||
func (n *notifier) peerListChanged(numOfPeers int) {
|
||||
@@ -169,15 +206,19 @@ func (n *notifier) localAddressChanged(fqdn, address string) {
|
||||
listener.OnAddressChanged(fqdn, address)
|
||||
}
|
||||
|
||||
func notifyListener(l Listener, state int) {
|
||||
func notifyListener(l Listener, state ClientState) {
|
||||
// legacy per-state callbacks; NoNetwork is delivered only via
|
||||
// OnStateChanged below
|
||||
switch state {
|
||||
case stateDisconnected:
|
||||
case ClientStateDisconnected:
|
||||
l.OnDisconnected()
|
||||
case stateConnected:
|
||||
case ClientStateConnected:
|
||||
l.OnConnected()
|
||||
case stateConnecting:
|
||||
case ClientStateConnecting:
|
||||
l.OnConnecting()
|
||||
case stateDisconnecting:
|
||||
case ClientStateDisconnecting:
|
||||
l.OnDisconnecting()
|
||||
}
|
||||
|
||||
l.OnStateChanged(state)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package status
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type recordingListener struct {
|
||||
mu sync.Mutex
|
||||
states []ClientState
|
||||
onState func(ClientState)
|
||||
}
|
||||
|
||||
func (l *recordingListener) OnStateChanged(state ClientState) {
|
||||
l.mu.Lock()
|
||||
l.states = append(l.states, state)
|
||||
hook := l.onState
|
||||
l.mu.Unlock()
|
||||
|
||||
if hook != nil {
|
||||
hook(state)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *recordingListener) last() (ClientState, bool) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if len(l.states) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return l.states[len(l.states)-1], true
|
||||
}
|
||||
|
||||
func (l *recordingListener) snapshot() []ClientState {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return append([]ClientState(nil), l.states...)
|
||||
}
|
||||
|
||||
func (l *recordingListener) OnConnected() {}
|
||||
func (l *recordingListener) OnDisconnected() {}
|
||||
func (l *recordingListener) OnConnecting() {}
|
||||
func (l *recordingListener) OnDisconnecting() {}
|
||||
func (l *recordingListener) OnAddressChanged(string, string) {}
|
||||
func (l *recordingListener) OnPeersListChanged(int) {}
|
||||
|
||||
// TestNotifier_ConcurrentAvailabilityFlipOrdersPublication holds the first
|
||||
// transition inside the listener callback and flips availability again from
|
||||
// another goroutine while it is parked. The second flip must not publish
|
||||
// ahead of the one in flight, otherwise the listener ends up on a state the
|
||||
// notifier already superseded.
|
||||
func TestNotifier_ConcurrentAvailabilityFlipOrdersPublication(t *testing.T) {
|
||||
n := newNotifier()
|
||||
n.currentClientState = true
|
||||
n.lastNotification = ClientStateConnecting
|
||||
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
l := &recordingListener{}
|
||||
l.onState = func(state ClientState) {
|
||||
if state != ClientStateNoNetwork {
|
||||
return
|
||||
}
|
||||
l.mu.Lock()
|
||||
l.onState = nil
|
||||
l.mu.Unlock()
|
||||
close(entered)
|
||||
<-release
|
||||
}
|
||||
n.listener = l
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
n.setNetworkAvailable(false)
|
||||
}()
|
||||
|
||||
<-entered
|
||||
|
||||
flipped := make(chan struct{})
|
||||
go func() {
|
||||
defer close(flipped)
|
||||
n.setNetworkAvailable(true)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-flipped:
|
||||
t.Fatal("the online transition published while the offline one was " +
|
||||
"still in flight; publication is not serialized")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(release)
|
||||
<-flipped
|
||||
wg.Wait()
|
||||
|
||||
got, ok := l.last()
|
||||
if !ok {
|
||||
t.Fatal("listener never observed a state")
|
||||
}
|
||||
if got != ClientStateConnecting {
|
||||
t.Fatalf("listener holds %v after the network came back, want Connecting; sequence: %v",
|
||||
got, l.snapshot())
|
||||
}
|
||||
}
|
||||
@@ -6,29 +6,32 @@ import (
|
||||
)
|
||||
|
||||
type mocListener struct {
|
||||
lastState int
|
||||
lastState ClientState
|
||||
wg sync.WaitGroup
|
||||
peersWg sync.WaitGroup
|
||||
peers int
|
||||
}
|
||||
|
||||
func (l *mocListener) OnConnected() {
|
||||
l.lastState = stateConnected
|
||||
l.lastState = ClientStateConnected
|
||||
l.wg.Done()
|
||||
}
|
||||
func (l *mocListener) OnDisconnected() {
|
||||
l.lastState = stateDisconnected
|
||||
l.lastState = ClientStateDisconnected
|
||||
l.wg.Done()
|
||||
}
|
||||
func (l *mocListener) OnConnecting() {
|
||||
l.lastState = stateConnecting
|
||||
l.lastState = ClientStateConnecting
|
||||
l.wg.Done()
|
||||
}
|
||||
func (l *mocListener) OnDisconnecting() {
|
||||
l.lastState = stateDisconnecting
|
||||
l.lastState = ClientStateDisconnecting
|
||||
l.wg.Done()
|
||||
}
|
||||
|
||||
func (l *mocListener) OnStateChanged(state ClientState) {
|
||||
|
||||
}
|
||||
func (l *mocListener) OnAddressChanged(host, addr string) {
|
||||
|
||||
}
|
||||
@@ -57,15 +60,15 @@ func Test_notifier_serverState(t *testing.T) {
|
||||
|
||||
type scenario struct {
|
||||
name string
|
||||
expected int
|
||||
expected ClientState
|
||||
mgmState bool
|
||||
signalState bool
|
||||
}
|
||||
scenarios := []scenario{
|
||||
{"connected", stateConnected, true, true},
|
||||
{"mgm down", stateConnecting, false, true},
|
||||
{"signal down", stateConnecting, true, false},
|
||||
{"disconnected", stateDisconnected, false, false},
|
||||
{"connected", ClientStateConnected, true, true},
|
||||
{"mgm down", ClientStateConnecting, false, true},
|
||||
{"signal down", ClientStateConnecting, true, false},
|
||||
{"disconnected", ClientStateDisconnected, false, false},
|
||||
}
|
||||
|
||||
for _, tt := range scenarios {
|
||||
@@ -85,7 +88,7 @@ func Test_notifier_SetListener(t *testing.T) {
|
||||
listener.setPeersWaiter()
|
||||
|
||||
n := newNotifier()
|
||||
n.lastNotification = stateConnecting
|
||||
n.lastNotification = ClientStateConnecting
|
||||
n.setListener(listener)
|
||||
listener.wait()
|
||||
listener.waitPeers()
|
||||
@@ -99,7 +102,7 @@ func Test_notifier_RemoveListener(t *testing.T) {
|
||||
listener.setWaiter()
|
||||
listener.setPeersWaiter()
|
||||
n := newNotifier()
|
||||
n.lastNotification = stateConnecting
|
||||
n.lastNotification = ClientStateConnecting
|
||||
n.setListener(listener)
|
||||
// setListener replays cached state on a goroutine; wait for both the state
|
||||
// and peers callbacks to finish so we don't race on listener.peers.
|
||||
|
||||
@@ -991,6 +991,18 @@ func (d *Recorder) GetResolvedDomainsStates() map[domain.Domain]ResolvedDomainIn
|
||||
return maps.Clone(d.resolvedDomainsStates)
|
||||
}
|
||||
|
||||
// GetPeerStates returns a snapshot of all known peer states, including offline peers.
|
||||
func (d *Recorder) GetPeerStates() []State {
|
||||
d.mux.RLock()
|
||||
defer d.mux.RUnlock()
|
||||
|
||||
states := make([]State, 0, d.numOfPeers())
|
||||
for _, state := range d.peers {
|
||||
states = append(states, state)
|
||||
}
|
||||
return append(states, d.offlinePeers...)
|
||||
}
|
||||
|
||||
// GetFullStatus gets full status
|
||||
func (d *Recorder) GetFullStatus() FullStatus {
|
||||
fullStatus := FullStatus{
|
||||
@@ -1035,6 +1047,12 @@ func (d *Recorder) ClientTeardown() {
|
||||
d.notifyStateChange()
|
||||
}
|
||||
|
||||
// SetNetworkAvailable records the OS-reported network availability; while
|
||||
// unavailable, listeners see NoNetwork instead of Connecting.
|
||||
func (d *Recorder) SetNetworkAvailable(available bool) {
|
||||
d.notifier.setNetworkAvailable(available)
|
||||
}
|
||||
|
||||
// SetConnectionListener set a listener to the notifier
|
||||
func (d *Recorder) SetConnectionListener(listener Listener) {
|
||||
d.notifier.setListener(listener)
|
||||
|
||||
@@ -129,6 +129,28 @@ func TestStatus_PeerStateByIP_RemovedPeer(t *testing.T) {
|
||||
req.False(ok, "removed peer must not resolve by IPv6 tunnel address")
|
||||
}
|
||||
|
||||
// TestStatus_GetPeerStates_IncludesOfflinePeers keeps the snapshot in line with
|
||||
// GetFullStatus: offline peers are known peers, so a consumer counting peers
|
||||
// must see the same total the status command reports.
|
||||
func TestStatus_GetPeerStates_IncludesOfflinePeers(t *testing.T) {
|
||||
status := NewRecorder("https://mgm")
|
||||
req := require.New(t)
|
||||
|
||||
req.NoError(status.AddPeer("pk-online", "online.netbird", "100.64.0.10", "fd00::1"))
|
||||
status.ReplaceOfflinePeers([]State{
|
||||
{PubKey: "pk-offline", FQDN: "offline.netbird", IP: "100.64.0.20", ConnStatus: StatusIdle},
|
||||
})
|
||||
|
||||
states := status.GetPeerStates()
|
||||
req.Len(states, 2, "snapshot must carry both the online and the offline peer")
|
||||
|
||||
keys := make([]string, 0, len(states))
|
||||
for _, s := range states {
|
||||
keys = append(keys, s.PubKey)
|
||||
}
|
||||
req.ElementsMatch([]string{"pk-online", "pk-offline"}, keys, "snapshot must carry both peers")
|
||||
}
|
||||
|
||||
func TestStatus_UpdatePeerFQDN(t *testing.T) {
|
||||
key := "abc"
|
||||
fqdn := "peer-a.netbird.local"
|
||||
|
||||
@@ -6,6 +6,7 @@ import "github.com/netbirdio/netbird/client/internal/peer/status"
|
||||
// package. Callers are being migrated to reference the status package
|
||||
// directly; these aliases will be removed once the migration completes.
|
||||
type (
|
||||
ClientState = status.ClientState
|
||||
Status = status.Recorder
|
||||
State = status.State
|
||||
ConnStatus = status.ConnStatus
|
||||
@@ -26,6 +27,12 @@ type (
|
||||
)
|
||||
|
||||
const (
|
||||
ClientStateDisconnected = status.ClientStateDisconnected
|
||||
ClientStateConnected = status.ClientStateConnected
|
||||
ClientStateConnecting = status.ClientStateConnecting
|
||||
ClientStateDisconnecting = status.ClientStateDisconnecting
|
||||
ClientStateNoNetwork = status.ClientStateNoNetwork
|
||||
|
||||
StatusIdle = status.StatusIdle
|
||||
StatusConnecting = status.StatusConnecting
|
||||
StatusConnected = status.StatusConnected
|
||||
|
||||
@@ -74,6 +74,9 @@ type ICE struct {
|
||||
|
||||
// portForwardAttempted tracks if we've already tried port forwarding this session
|
||||
portForwardAttempted bool
|
||||
|
||||
// dialFunc, when non-nil, replaces agentDial in connect(). Only for tests.
|
||||
dialFunc func(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *signaling.OfferAnswer) (net.Conn, error)
|
||||
}
|
||||
|
||||
func NewICE(log *log.Entry, key string, iceConfig icemaker.Config, isController bool, onConnReady func(ConnPriority, ICEConnInfo), onStatusDisconnect func(bool), services ICEDependencies, hasRelayOnLocally bool) (*ICE, error) {
|
||||
@@ -135,7 +138,7 @@ func (w *ICE) OnNewOffer(ctx context.Context, remoteOfferAnswer *signaling.Offer
|
||||
w.log.Errorf("failed to create new session ID: %s", err)
|
||||
}
|
||||
w.sessionID = sessionID
|
||||
w.agent = nil
|
||||
w.abandonNegotiation()
|
||||
}
|
||||
|
||||
var preferredCandidateTypes []ice.CandidateType
|
||||
@@ -163,6 +166,8 @@ func (w *ICE) OnNewOffer(ctx context.Context, remoteOfferAnswer *signaling.Offer
|
||||
w.remoteSessionID = ""
|
||||
}
|
||||
|
||||
// Capture the cancel func at spawn time: connect reads it from the argument
|
||||
// instead of the field, which a newer OnNewOffer may already have replaced.
|
||||
go w.connect(dialerCtx, dialerCancel, agent, remoteOfferAnswer)
|
||||
}
|
||||
|
||||
@@ -218,16 +223,16 @@ func (w *ICE) Close() {
|
||||
w.muxAgent.Lock()
|
||||
defer w.muxAgent.Unlock()
|
||||
|
||||
if w.agent == nil {
|
||||
return
|
||||
if w.agent != nil {
|
||||
w.agentDialerCancel()
|
||||
if err := w.agent.Close(); err != nil {
|
||||
w.log.Warnf("failed to close ICE agent: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
w.agentDialerCancel()
|
||||
if err := w.agent.Close(); err != nil {
|
||||
w.log.Warnf("failed to close ICE agent: %s", err)
|
||||
}
|
||||
|
||||
w.agent = nil
|
||||
// Unconditional: a dial goroutine racing this Close skips its own cleanup
|
||||
// (closeAgent finds a nil agent), so the flags must be dropped here too or
|
||||
// the reconnection guard reads the stale state as Connected forever.
|
||||
w.abandonNegotiation()
|
||||
}
|
||||
|
||||
func (w *ICE) reCreateAgent(ctx context.Context, dialerCancel context.CancelFunc, candidates []ice.CandidateType) (*icemaker.ThreadSafeAgent, error) {
|
||||
@@ -273,8 +278,14 @@ func (w *ICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agen
|
||||
return
|
||||
}
|
||||
|
||||
w.log.Debugf("turn agent dial")
|
||||
remoteConn, err := w.turnAgentDial(ctx, agent, remoteOfferAnswer)
|
||||
w.log.Debugf("agent dial")
|
||||
dial := func(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *signaling.OfferAnswer) (net.Conn, error) {
|
||||
return w.agentDial(ctx, agent, remoteOfferAnswer)
|
||||
}
|
||||
if w.dialFunc != nil {
|
||||
dial = w.dialFunc
|
||||
}
|
||||
remoteConn, err := dial(ctx, agent, remoteOfferAnswer)
|
||||
if err != nil {
|
||||
w.log.Debugf("failed to dial the remote peer: %s", err)
|
||||
w.closeAgent(agent, dialerCancel)
|
||||
@@ -282,6 +293,19 @@ func (w *ICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agen
|
||||
}
|
||||
w.log.Debugf("agent dial succeeded")
|
||||
|
||||
// A newer negotiation may have replaced this agent during the dial.
|
||||
// Discard its connection before querying candidates or punching ports.
|
||||
w.muxAgent.Lock()
|
||||
stale := w.agent != agent
|
||||
w.muxAgent.Unlock()
|
||||
if stale {
|
||||
if err := remoteConn.Close(); err != nil {
|
||||
w.log.Warnf("failed to close stale ICE connection: %s", err)
|
||||
}
|
||||
w.log.Warnf("discarding connection from a stale ICE negotiation")
|
||||
return
|
||||
}
|
||||
|
||||
pair, err := agent.GetSelectedCandidatePair()
|
||||
if err != nil {
|
||||
w.closeAgent(agent, dialerCancel)
|
||||
@@ -318,9 +342,14 @@ func (w *ICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agen
|
||||
w.log.Debugf("on ICE conn is ready to use")
|
||||
|
||||
w.muxAgent.Lock()
|
||||
// Keep the ownership check atomic with the state update so a stale dial
|
||||
// cannot overwrite a newer negotiation.
|
||||
if w.agent != agent {
|
||||
w.muxAgent.Unlock()
|
||||
w.log.Debugf("agent has been replaced during connect, dropping obsolete connection")
|
||||
if err := remoteConn.Close(); err != nil {
|
||||
w.log.Warnf("failed to close stale ICE connection: %s", err)
|
||||
}
|
||||
w.log.Warnf("discarding connection from a stale ICE negotiation")
|
||||
return
|
||||
}
|
||||
w.agentConnecting = false
|
||||
@@ -344,20 +373,27 @@ func (w *ICE) closeAgent(agent *icemaker.ThreadSafeAgent, cancel context.CancelF
|
||||
sessionChanged := w.remoteSessionChanged
|
||||
w.remoteSessionChanged = false
|
||||
|
||||
// Only the owner of the current session may reset its state: a stale dial
|
||||
// goroutine waking after a newer attempt must not clobber it.
|
||||
if w.agent == agent {
|
||||
// consider to remove from here and move to the OnNewOffer
|
||||
sessionID, err := icemaker.NewSessionID()
|
||||
if err != nil {
|
||||
w.log.Errorf("failed to create new session ID: %s", err)
|
||||
}
|
||||
w.sessionID = sessionID
|
||||
w.agent = nil
|
||||
w.agentConnecting = false
|
||||
w.remoteSessionID = ""
|
||||
w.abandonNegotiation()
|
||||
}
|
||||
return sessionChanged
|
||||
}
|
||||
|
||||
// Clearing the agent and connecting flag together keeps retries from stalling.
|
||||
// Callers must dispose of the agent first and hold muxAgent.
|
||||
func (w *ICE) abandonNegotiation() {
|
||||
w.agent = nil
|
||||
w.agentConnecting = false
|
||||
w.remoteSessionID = ""
|
||||
}
|
||||
|
||||
func (w *ICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) {
|
||||
// wait local endpoint configuration
|
||||
time.Sleep(time.Second)
|
||||
@@ -412,6 +448,17 @@ func (w *ICE) injectPortForwardedCandidate(srflxCandidate ice.Candidate) {
|
||||
return
|
||||
}
|
||||
|
||||
// A forwarded candidate only makes sense for an IPv4 mapping, which
|
||||
// translates a port on the gateway's address. An IPv6 pinhole translates
|
||||
// nothing: it unblocks the address ICE already gathers as a host candidate,
|
||||
// so there is no second address to advertise. Injecting one here would also
|
||||
// paste an IPv6 address onto whichever server-reflexive candidate arrived
|
||||
// first, which is usually IPv4.
|
||||
if mapping.ExternalIP != nil && mapping.ExternalIP.To4() == nil {
|
||||
w.log.Debugf("skipping port-forwarded candidate: %s mapping is IPv6-only", mapping.NATType)
|
||||
return
|
||||
}
|
||||
|
||||
w.muxAgent.Lock()
|
||||
if w.portForwardAttempted {
|
||||
w.muxAgent.Unlock()
|
||||
@@ -541,8 +588,8 @@ func (w *ICE) onConnectionStateChange(agent *icemaker.ThreadSafeAgent, dialerCan
|
||||
connected = true
|
||||
w.logSuccessfulPaths(agent)
|
||||
case ice.ConnectionStateFailed, ice.ConnectionStateDisconnected, ice.ConnectionStateClosed:
|
||||
// ice.ConnectionStateClosed happens when we recreate the agent. For the P2P to TURN switch important to
|
||||
// notify the conn.onICEStateDisconnected changes to update the current used priority
|
||||
// ice.ConnectionStateClosed happens when we recreate the agent. The P2P to relay switch requires
|
||||
// notifying conn.onICEStateDisconnected so it can update the currently used priority.
|
||||
|
||||
sessionChanged := w.closeAgent(agent, dialerCancel)
|
||||
|
||||
@@ -567,7 +614,7 @@ func (w *ICE) onConnectionStateChange(agent *icemaker.ThreadSafeAgent, dialerCan
|
||||
}
|
||||
}
|
||||
|
||||
func (w *ICE) turnAgentDial(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *signaling.OfferAnswer) (*ice.Conn, error) {
|
||||
func (w *ICE) agentDial(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *signaling.OfferAnswer) (*ice.Conn, error) {
|
||||
if w.isController {
|
||||
return agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
|
||||
|
||||
icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
|
||||
"github.com/netbirdio/netbird/client/internal/peer/signaling"
|
||||
signal "github.com/netbirdio/netbird/shared/signal/client"
|
||||
sProto "github.com/netbirdio/netbird/shared/signal/proto"
|
||||
)
|
||||
|
||||
// stubSignalClient satisfies signal.Client as a no-op so the candidate
|
||||
// goroutine spawned by a real GatherCandidates never dereferences a nil
|
||||
// signaler in tests.
|
||||
type stubSignalClient struct{}
|
||||
|
||||
func (stubSignalClient) Close() error { return nil }
|
||||
func (stubSignalClient) StreamConnected() bool { return false }
|
||||
func (stubSignalClient) GetStatus() signal.Status { return signal.StreamDisconnected }
|
||||
func (stubSignalClient) Receive(context.Context, func(*sProto.Message) error) error { return nil }
|
||||
func (stubSignalClient) Ready() bool { return false }
|
||||
func (stubSignalClient) IsHealthy() bool { return false }
|
||||
func (stubSignalClient) WaitStreamConnected(context.Context) {}
|
||||
func (stubSignalClient) SendToStream(*sProto.EncryptedMessage) error { return nil }
|
||||
func (stubSignalClient) Send(*sProto.Message) error { return nil }
|
||||
func (stubSignalClient) SetOnReconnectedListener(func()) {}
|
||||
|
||||
// newTestWorkerICE builds a worker with real pion plumbing and no-op signaling.
|
||||
func newTestWorkerICE(t *testing.T) *ICE {
|
||||
t.Helper()
|
||||
|
||||
config := icemaker.Config{}
|
||||
stunTurn := &icemaker.StunTurn{}
|
||||
stunTurn.Store(nil)
|
||||
config.StunTurn = stunTurn
|
||||
|
||||
w, err := NewICE(log.WithField("test", t.Name()), "test-peer", config, true, nil, nil,
|
||||
ICEDependencies{Signaler: signaling.NewSignaler(stubSignalClient{}, wgtypes.Key{})}, false)
|
||||
require.NoError(t, err, "worker setup must succeed")
|
||||
return w
|
||||
}
|
||||
|
||||
// TestWorkerICE_CloseDuringDial_ClearsConnectingFlag drives the teardown race
|
||||
// through the real dial goroutine instead of simulating its cleanup.
|
||||
//
|
||||
// The real-world sequence this models:
|
||||
// 1. OnNewOffer starts a negotiation: agent set, agentConnecting = true,
|
||||
// go connect()
|
||||
// 2. The network dies and connect() stays blocked inside GatherCandidates/Dial
|
||||
// 3. A WG handshake timeout calls Close(): the agent is released and the dial
|
||||
// context cancelled, but agentConnecting is not reset
|
||||
// 4. The real goroutine wakes with an error and runs its own cleanup
|
||||
// (closeAgent), where `w.agent == agent` is now false, so the flag reset
|
||||
// is skipped
|
||||
//
|
||||
// There is no remote responder, so Dial can never succeed: whatever point the
|
||||
// goroutine is at, closing first forces it down the error path. Before the fix
|
||||
// the flag stays true forever and the deadline below expires.
|
||||
func TestWorkerICE_CloseDuringDial_ClearsConnectingFlag(t *testing.T) {
|
||||
w := newTestWorkerICE(t)
|
||||
|
||||
sid := icemaker.SessionID("test-session-id")
|
||||
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
|
||||
IceCredentials: signaling.IceCredentials{
|
||||
UFrag: "testufrag",
|
||||
Pwd: "testpwdtestpwdtestpwd12",
|
||||
},
|
||||
SessionID: &sid,
|
||||
})
|
||||
require.True(t, w.InProgress(), "OnNewOffer must mark the negotiation as in progress")
|
||||
|
||||
// Teardown wins the race while connect() is still running.
|
||||
w.Close()
|
||||
|
||||
// Close drops the flags synchronously, so the assertion below does not
|
||||
// converge on the goroutine: the deadline only absorbs the dial goroutine
|
||||
// waking up in the background, proving nothing re-wedges it afterwards.
|
||||
require.Eventually(t, func() bool {
|
||||
return !w.InProgress()
|
||||
}, 10*time.Second, 50*time.Millisecond,
|
||||
"Close must leave the negotiation idle even while the dial goroutine is still winding down")
|
||||
|
||||
// abandonNegotiation owns these three fields together; the worker is idle
|
||||
// only when all of them are dropped.
|
||||
w.muxAgent.Lock()
|
||||
defer w.muxAgent.Unlock()
|
||||
assert.Nil(t, w.agent, "no agent may survive the teardown")
|
||||
assert.False(t, w.agentConnecting, "the connecting flag must match the nil agent")
|
||||
assert.Empty(t, w.remoteSessionID, "a dead session's remote ID must not linger")
|
||||
}
|
||||
|
||||
// TestWorkerICE_CloseClearsResidualConnectingState covers Close on a worker whose
|
||||
// agent is already gone but whose flag is stuck on true, e.g. after an aborted
|
||||
// recreate in OnNewOffer or after a first Close raced a dial goroutine.
|
||||
func TestWorkerICE_CloseClearsResidualConnectingState(t *testing.T) {
|
||||
w := newTestWorkerICE(t)
|
||||
|
||||
w.muxAgent.Lock()
|
||||
w.agentConnecting = true
|
||||
w.muxAgent.Unlock()
|
||||
|
||||
w.Close()
|
||||
|
||||
assert.False(t, w.InProgress(), "Close must drop residual connecting state even without a live agent")
|
||||
|
||||
w.muxAgent.Lock()
|
||||
defer w.muxAgent.Unlock()
|
||||
assert.Nil(t, w.agent)
|
||||
assert.False(t, w.agentConnecting)
|
||||
assert.Empty(t, w.remoteSessionID)
|
||||
}
|
||||
|
||||
// TestWorkerICE_StaleCloseAgentKeepsCurrentSession pins the ownership guard in
|
||||
// closeAgent: a late-waking dial goroutine from an older session must not reset
|
||||
// the state of a newer negotiation that reused the worker. The newer session
|
||||
// must survive wholesale - agent, flag and remote session identity alike.
|
||||
func TestWorkerICE_StaleCloseAgentKeepsCurrentSession(t *testing.T) {
|
||||
w := newTestWorkerICE(t)
|
||||
t.Cleanup(w.Close)
|
||||
|
||||
sidA := icemaker.SessionID("session-a")
|
||||
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
|
||||
IceCredentials: signaling.IceCredentials{UFrag: "ufragaaaa", Pwd: "pwdpwdpwdpwdpwdpwdpwdp1"},
|
||||
SessionID: &sidA,
|
||||
})
|
||||
w.muxAgent.Lock()
|
||||
oldAgent := w.agent
|
||||
oldCancel := w.agentDialerCancel
|
||||
w.muxAgent.Unlock()
|
||||
require.NotNil(t, oldAgent, "OnNewOffer must have created an ICE agent")
|
||||
|
||||
w.Close()
|
||||
|
||||
sidB := icemaker.SessionID("session-b")
|
||||
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
|
||||
IceCredentials: signaling.IceCredentials{UFrag: "ufragbbbb", Pwd: "pwdpwdpwdpwdpwdpwdpwdp2"},
|
||||
SessionID: &sidB,
|
||||
})
|
||||
require.True(t, w.InProgress(), "the second negotiation must be in flight")
|
||||
|
||||
w.muxAgent.Lock()
|
||||
newAgent := w.agent
|
||||
w.muxAgent.Unlock()
|
||||
|
||||
// The old dial goroutine finally wakes and cleans up its captured agent.
|
||||
w.closeAgent(oldAgent, oldCancel)
|
||||
|
||||
w.muxAgent.Lock()
|
||||
defer w.muxAgent.Unlock()
|
||||
assert.Same(t, newAgent, w.agent, "the current agent must be untouched by the stale cleanup")
|
||||
assert.True(t, w.agentConnecting, "the current negotiation must stay in flight")
|
||||
// Read live under the lock: a snapshot captured before the stale cleanup
|
||||
// would pass even if the cleanup wiped current state.
|
||||
assert.Equal(t, sidB, w.remoteSessionID, "the remote session identity must be preserved")
|
||||
}
|
||||
|
||||
// closeTrackConn records Close calls so a test can assert that a discarded
|
||||
// connection was actually released.
|
||||
type closeTrackConn struct {
|
||||
net.Conn
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func (c *closeTrackConn) Close() error {
|
||||
c.closed.Store(true)
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
// TestWorkerICE_StaleDialSuccessKeepsNewerNegotiation pins the ownership guard
|
||||
// in connect()'s success path: a dial that came back after a newer negotiation
|
||||
// replaced the agent must discard its connection and leave the newer session's
|
||||
// state - agent, agentConnecting, remoteSessionID, lastSuccess - intact.
|
||||
//
|
||||
// The dial hook holds session A's goroutine open until session B is installed,
|
||||
// then returns a live connection, mimicking the vendored pion dial which hands
|
||||
// out a live *ice.Conn when a pair is selected without checking afterwards
|
||||
// whether the agent was replaced meanwhile. Releasing A's dial therefore
|
||||
// exercises the stale-success commit path deterministically instead of racing
|
||||
// real ICE.
|
||||
func TestWorkerICE_StaleDialSuccessKeepsNewerNegotiation(t *testing.T) {
|
||||
w := newTestWorkerICE(t)
|
||||
t.Cleanup(w.Close)
|
||||
|
||||
dialStarted := make(chan struct{})
|
||||
releaseDial := make(chan struct{})
|
||||
staleConn := &closeTrackConn{}
|
||||
|
||||
var calls atomic.Int32
|
||||
w.dialFunc = func(ctx context.Context, _ *icemaker.ThreadSafeAgent, _ *signaling.OfferAnswer) (net.Conn, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
// Session A: hold the goroutine open until session B is installed,
|
||||
// then return a live connection, mimicking the vendored pion dial
|
||||
// which hands out a live *ice.Conn once a pair is selected without
|
||||
// re-checking whether the agent was replaced meanwhile. Releasing
|
||||
// the dial therefore exercises the stale-success commit path
|
||||
// deterministically instead of racing real ICE.
|
||||
close(dialStarted)
|
||||
<-releaseDial
|
||||
client, _ := net.Pipe()
|
||||
staleConn.Conn = client
|
||||
return staleConn, nil
|
||||
}
|
||||
// A newer negotiation parks on its dialer context, cancelled by the
|
||||
// t.Cleanup Close at test end.
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
sidA := icemaker.SessionID("session-a")
|
||||
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
|
||||
IceCredentials: signaling.IceCredentials{UFrag: "ufragaaaa", Pwd: "pwdpwdpwdpwdpwdpwdpwdp1"},
|
||||
SessionID: &sidA,
|
||||
})
|
||||
require.True(t, w.InProgress(), "session A must be in flight")
|
||||
|
||||
// Session A's goroutine is now parked in the dial hook.
|
||||
<-dialStarted
|
||||
|
||||
sidB := icemaker.SessionID("session-b")
|
||||
w.OnNewOffer(t.Context(), &signaling.OfferAnswer{
|
||||
IceCredentials: signaling.IceCredentials{UFrag: "ufragbbbb", Pwd: "pwdpwdpwdpwdpwdpwdpwdp2"},
|
||||
SessionID: &sidB,
|
||||
})
|
||||
|
||||
w.muxAgent.Lock()
|
||||
agentB := w.agent
|
||||
w.lastSuccess = time.Time{}
|
||||
w.muxAgent.Unlock()
|
||||
require.NotNil(t, agentB, "session B must have created an ICE agent")
|
||||
require.True(t, w.InProgress(), "session B must be in flight")
|
||||
|
||||
// Release session A's dial: it must be recognized as stale and discarded.
|
||||
close(releaseDial)
|
||||
require.Eventually(t, func() bool {
|
||||
return staleConn.closed.Load()
|
||||
}, 10*time.Second, 10*time.Millisecond,
|
||||
"the stale connection must be closed by the ownership guard")
|
||||
|
||||
w.muxAgent.Lock()
|
||||
defer w.muxAgent.Unlock()
|
||||
assert.Same(t, agentB, w.agent, "session A must not uninstall session B's agent")
|
||||
assert.True(t, w.agentConnecting, "session A must not clear session B's connecting flag")
|
||||
assert.Equal(t, sidB, w.remoteSessionID, "session A must not clear session B's remote session identity")
|
||||
assert.True(t, w.lastSuccess.IsZero(), "session A must not record a success for session B")
|
||||
// The commit block guards agentConnecting, lastSuccess and
|
||||
// onICEConnectionIsReady together, so the state assertions above imply the
|
||||
// callback never ran for session A; the nil conn would have panicked the
|
||||
// stale goroutine on any invocation.
|
||||
}
|
||||
Reference in New Issue
Block a user