Merge branch 'main' into fix/pkg-loss

client/internal/peer/conn.go

@@ -5,61 +5,61 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
+	"net/netip"
 	"os"
 	"runtime"
-	"strings"
 	"sync"
 	"time"
 
-	"github.com/pion/ice/v3"
+	"github.com/pion/ice/v4"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
 	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/client/iface/wgproxy"
+	"github.com/netbirdio/netbird/client/internal/peer/conntype"
+	"github.com/netbirdio/netbird/client/internal/peer/dispatcher"
 	"github.com/netbirdio/netbird/client/internal/peer/guard"
 	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
+	"github.com/netbirdio/netbird/client/internal/peer/id"
+	"github.com/netbirdio/netbird/client/internal/peer/worker"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
-	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
-	nbnet "github.com/netbirdio/netbird/util/net"
+	relayClient "github.com/netbirdio/netbird/shared/relay/client"
 	semaphoregroup "github.com/netbirdio/netbird/util/semaphore-group"
 )
 
-type ConnPriority int
-
-func (cp ConnPriority) String() string {
-	switch cp {
-	case connPriorityNone:
-		return "None"
-	case connPriorityRelay:
-		return "PriorityRelay"
-	case connPriorityICETurn:
-		return "PriorityICETurn"
-	case connPriorityICEP2P:
-		return "PriorityICEP2P"
-	default:
-		return fmt.Sprintf("ConnPriority(%d)", cp)
-	}
-}
-
 const (
 	defaultWgKeepAlive = 25 * time.Second
-
-	connPriorityNone    ConnPriority = 0
-	connPriorityRelay   ConnPriority = 1
-	connPriorityICETurn ConnPriority = 2
-	connPriorityICEP2P  ConnPriority = 3
 )
 
+type ServiceDependencies struct {
+	StatusRecorder     *Status
+	Signaler           *Signaler
+	IFaceDiscover      stdnet.ExternalIFaceDiscover
+	RelayManager       *relayClient.Manager
+	SrWatcher          *guard.SRWatcher
+	Semaphore          *semaphoregroup.SemaphoreGroup
+	PeerConnDispatcher *dispatcher.ConnectionDispatcher
+}
+
 type WgConfig struct {
 	WgListenPort int
 	RemoteKey    string
 	WgInterface  WGIface
-	AllowedIps   string
+	AllowedIps   []netip.Prefix
 	PreSharedKey *wgtypes.Key
 }
 
+type RosenpassConfig struct {
+	// RosenpassPubKey is this peer's Rosenpass public key
+	PubKey []byte
+	// RosenpassPubKey is this peer's RosenpassAddr server address (IP:port)
+	Addr string
+
+	PermissiveMode bool
+}
+
 // ConnConfig is a peer Connection configuration
 type ConnConfig struct {
 	// Key is a public key of a remote peer
@@ -67,157 +67,169 @@ type ConnConfig struct {
 	// LocalKey is a public key of a local peer
 	LocalKey string
 
+	AgentVersion string
+
 	Timeout time.Duration
 
 	WgConfig WgConfig
 
 	LocalWgPort int
 
-	// RosenpassPubKey is this peer's Rosenpass public key
-	RosenpassPubKey []byte
-	// RosenpassPubKey is this peer's RosenpassAddr server address (IP:port)
-	RosenpassAddr string
+	RosenpassConfig RosenpassConfig
 
 	// ICEConfig ICE protocol configuration
 	ICEConfig icemaker.Config
 }
 
 type Conn struct {
-	log            *log.Entry
+	Log            *log.Entry
 	mu             sync.Mutex
 	ctx            context.Context
 	ctxCancel      context.CancelFunc
 	config         ConnConfig
 	statusRecorder *Status
 	signaler       *Signaler
+	iFaceDiscover  stdnet.ExternalIFaceDiscover
 	relayManager   *relayClient.Manager
-	allowedIP      net.IP
-	handshaker     *Handshaker
+	srWatcher      *guard.SRWatcher
 
 	onConnected    func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)
-	onDisconnected func(remotePeer string, wgIP string)
+	onDisconnected func(remotePeer string)
 
-	statusRelay         *AtomicConnStatus
-	statusICE           *AtomicConnStatus
-	currentConnPriority ConnPriority
+	statusRelay         *worker.AtomicWorkerStatus
+	statusICE           *worker.AtomicWorkerStatus
+	currentConnPriority conntype.ConnPriority
 	opened              bool // this flag is used to prevent close in case of not opened connection
 
 	workerICE   *WorkerICE
 	workerRelay *WorkerRelay
+	wgWatcherWg sync.WaitGroup
 
-	connIDRelay          nbnet.ConnectionID
-	connIDICE            nbnet.ConnectionID
-	beforeAddPeerHooks   []nbnet.AddHookFunc
-	afterRemovePeerHooks []nbnet.RemoveHookFunc
+	// used to store the remote Rosenpass key for Relayed connection in case of connection update from ice
+	rosenpassRemoteKey []byte
 
 	wgProxyICE   wgproxy.Proxy
 	wgProxyRelay wgproxy.Proxy
+	handshaker   *Handshaker
 
 	guard     *guard.Guard
 	semaphore *semaphoregroup.SemaphoreGroup
+	wg        sync.WaitGroup
+
+	// debug purpose
+	dumpState *stateDump
 }
 
 // NewConn creates a new not opened Conn to the remote peer.
 // To establish a connection run Conn.Open
-func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager, srWatcher *guard.SRWatcher, semaphore *semaphoregroup.SemaphoreGroup) (*Conn, error) {
-	allowedIP, _, err := net.ParseCIDR(config.WgConfig.AllowedIps)
-	if err != nil {
-		log.Errorf("failed to parse allowedIPS: %v", err)
-		return nil, err
+func NewConn(config ConnConfig, services ServiceDependencies) (*Conn, error) {
+	if len(config.WgConfig.AllowedIps) == 0 {
+		return nil, fmt.Errorf("allowed IPs is empty")
 	}
 
-	ctx, ctxCancel := context.WithCancel(engineCtx)
 	connLog := log.WithField("peer", config.Key)
 
 	var conn = &Conn{
-		log:            connLog,
-		ctx:            ctx,
-		ctxCancel:      ctxCancel,
+		Log:            connLog,
 		config:         config,
-		statusRecorder: statusRecorder,
-		signaler:       signaler,
-		relayManager:   relayManager,
-		allowedIP:      allowedIP,
-		statusRelay:    NewAtomicConnStatus(),
-		statusICE:      NewAtomicConnStatus(),
-		semaphore:      semaphore,
+		statusRecorder: services.StatusRecorder,
+		signaler:       services.Signaler,
+		iFaceDiscover:  services.IFaceDiscover,
+		relayManager:   services.RelayManager,
+		srWatcher:      services.SrWatcher,
+		semaphore:      services.Semaphore,
+		statusRelay:    worker.NewAtomicStatus(),
+		statusICE:      worker.NewAtomicStatus(),
+		dumpState:      newStateDump(config.Key, connLog, services.StatusRecorder),
 	}
 
-	ctrl := isController(config)
-	conn.workerRelay = NewWorkerRelay(connLog, ctrl, config, conn, relayManager)
-
-	relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally()
-	conn.workerICE, err = NewWorkerICE(ctx, connLog, config, conn, signaler, iFaceDiscover, statusRecorder, relayIsSupportedLocally)
-	if err != nil {
-		return nil, err
-	}
-
-	conn.handshaker = NewHandshaker(ctx, connLog, config, signaler, conn.workerICE, conn.workerRelay)
-
-	conn.handshaker.AddOnNewOfferListener(conn.workerRelay.OnNewOffer)
-	if os.Getenv("NB_FORCE_RELAY") != "true" {
-		conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer)
-	}
-
-	conn.guard = guard.NewGuard(connLog, ctrl, conn.isConnectedOnAllWay, config.Timeout, srWatcher)
-
-	go conn.handshaker.Listen()
-
 	return conn, nil
 }
 
 // Open opens connection to the remote peer
 // It will try to establish a connection using ICE and in parallel with relay. The higher priority connection type will
 // be used.
-func (conn *Conn) Open() {
-	conn.semaphore.Add(conn.ctx)
-	conn.log.Debugf("open connection to peer")
+func (conn *Conn) Open(engineCtx context.Context) error {
+	conn.semaphore.Add(engineCtx)
 
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
-	conn.opened = true
+
+	if conn.opened {
+		conn.semaphore.Done(engineCtx)
+		return nil
+	}
+
+	conn.ctx, conn.ctxCancel = context.WithCancel(engineCtx)
+
+	conn.workerRelay = NewWorkerRelay(conn.ctx, conn.Log, isController(conn.config), conn.config, conn, conn.relayManager, conn.dumpState)
+
+	relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally()
+	workerICE, err := NewWorkerICE(conn.ctx, conn.Log, conn.config, conn, conn.signaler, conn.iFaceDiscover, conn.statusRecorder, relayIsSupportedLocally)
+	if err != nil {
+		return err
+	}
+	conn.workerICE = workerICE
+
+	conn.handshaker = NewHandshaker(conn.Log, conn.config, conn.signaler, conn.workerICE, conn.workerRelay)
+
+	conn.handshaker.AddOnNewOfferListener(conn.workerRelay.OnNewOffer)
+	if os.Getenv("NB_FORCE_RELAY") != "true" {
+		conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer)
+	}
+
+	conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher)
+
+	conn.wg.Add(1)
+	go func() {
+		defer conn.wg.Done()
+		conn.handshaker.Listen(conn.ctx)
+	}()
+	go conn.dumpState.Start(conn.ctx)
 
 	peerState := State{
 		PubKey:           conn.config.Key,
-		IP:               strings.Split(conn.config.WgConfig.AllowedIps, "/")[0],
 		ConnStatusUpdate: time.Now(),
-		ConnStatus:       StatusDisconnected,
+		ConnStatus:       StatusConnecting,
 		Mux:              new(sync.RWMutex),
 	}
-	err := conn.statusRecorder.UpdatePeerState(peerState)
-	if err != nil {
-		conn.log.Warnf("error while updating the state err: %v", err)
+	if err := conn.statusRecorder.UpdatePeerState(peerState); err != nil {
+		conn.Log.Warnf("error while updating the state err: %v", err)
 	}
 
-	go conn.startHandshakeAndReconnect(conn.ctx)
-}
+	conn.wg.Add(1)
+	go func() {
+		defer conn.wg.Done()
 
-func (conn *Conn) startHandshakeAndReconnect(ctx context.Context) {
-	defer conn.semaphore.Done(conn.ctx)
-	conn.waitInitialRandomSleepTime(ctx)
+		conn.waitInitialRandomSleepTime(conn.ctx)
+		conn.semaphore.Done(conn.ctx)
 
-	err := conn.handshaker.sendOffer()
-	if err != nil {
-		conn.log.Errorf("failed to send initial offer: %v", err)
-	}
-
-	go conn.guard.Start(ctx)
-	go conn.listenGuardEvent(ctx)
+		conn.guard.Start(conn.ctx, conn.onGuardEvent)
+	}()
+	conn.opened = true
+	return nil
 }
 
 // Close closes this peer Conn issuing a close event to the Conn closeCh
-func (conn *Conn) Close() {
+func (conn *Conn) Close(signalToRemote bool) {
 	conn.mu.Lock()
+	defer conn.wgWatcherWg.Wait()
 	defer conn.mu.Unlock()
 
-	conn.log.Infof("close peer connection")
-	conn.ctxCancel()
-
 	if !conn.opened {
-		conn.log.Debugf("ignore close connection to peer")
+		conn.Log.Debugf("ignore close connection to peer")
 		return
 	}
 
+	if signalToRemote {
+		if err := conn.signaler.SignalIdle(conn.config.Key); err != nil {
+			conn.Log.Errorf("failed to signal idle state to peer: %v", err)
+		}
+	}
+
+	conn.Log.Infof("close peer connection")
+	conn.ctxCancel()
+
 	conn.workerRelay.DisableWgWatcher()
 	conn.workerRelay.CloseConn()
 	conn.workerICE.Close()
@@ -225,7 +237,7 @@ func (conn *Conn) Close() {
 	if conn.wgProxyRelay != nil {
 		err := conn.wgProxyRelay.CloseConn()
 		if err != nil {
-			conn.log.Errorf("failed to close wg proxy for relay: %v", err)
+			conn.Log.Errorf("failed to close wg proxy for relay: %v", err)
 		}
 		conn.wgProxyRelay = nil
 	}
@@ -233,56 +245,53 @@ func (conn *Conn) Close() {
 	if conn.wgProxyICE != nil {
 		err := conn.wgProxyICE.CloseConn()
 		if err != nil {
-			conn.log.Errorf("failed to close wg proxy for ice: %v", err)
+			conn.Log.Errorf("failed to close wg proxy for ice: %v", err)
 		}
 		conn.wgProxyICE = nil
 	}
 
 	if err := conn.removeWgPeer(); err != nil {
-		conn.log.Errorf("failed to remove wg endpoint: %v", err)
+		conn.Log.Errorf("failed to remove wg endpoint: %v", err)
 	}
 
-	conn.freeUpConnID()
-
 	if conn.evalStatus() == StatusConnected && conn.onDisconnected != nil {
-		conn.onDisconnected(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps)
+		conn.onDisconnected(conn.config.WgConfig.RemoteKey)
 	}
 
 	conn.setStatusToDisconnected()
+	conn.opened = false
+	conn.wg.Wait()
+	conn.Log.Infof("peer connection closed")
 }
 
 // OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
 // doesn't block, discards the message if connection wasn't ready
-func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool {
-	conn.log.Debugf("OnRemoteAnswer, status ICE: %s, status relay: %s", conn.statusICE, conn.statusRelay)
-	return conn.handshaker.OnRemoteAnswer(answer)
+func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) {
+	conn.dumpState.RemoteAnswer()
+	conn.Log.Infof("OnRemoteAnswer, priority: %s, status ICE: %s, status relay: %s", conn.currentConnPriority, conn.statusICE, conn.statusRelay)
+	conn.handshaker.OnRemoteAnswer(answer)
 }
 
 // OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer.
 func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) {
+	conn.dumpState.RemoteCandidate()
 	conn.workerICE.OnRemoteCandidate(candidate, haRoutes)
 }
 
-func (conn *Conn) AddBeforeAddPeerHook(hook nbnet.AddHookFunc) {
-	conn.beforeAddPeerHooks = append(conn.beforeAddPeerHooks, hook)
-}
-func (conn *Conn) AddAfterRemovePeerHook(hook nbnet.RemoveHookFunc) {
-	conn.afterRemovePeerHooks = append(conn.afterRemovePeerHooks, hook)
-}
-
 // SetOnConnected sets a handler function to be triggered by Conn when a new connection to a remote peer established
 func (conn *Conn) SetOnConnected(handler func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)) {
 	conn.onConnected = handler
 }
 
 // SetOnDisconnected sets a handler function to be triggered by Conn when a connection to a remote disconnected
-func (conn *Conn) SetOnDisconnected(handler func(remotePeer string, wgIP string)) {
+func (conn *Conn) SetOnDisconnected(handler func(remotePeer string)) {
 	conn.onDisconnected = handler
 }
 
-func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool {
-	conn.log.Debugf("OnRemoteOffer, on status ICE: %s, status Relay: %s", conn.statusICE, conn.statusRelay)
-	return conn.handshaker.OnRemoteOffer(offer)
+func (conn *Conn) OnRemoteOffer(offer OfferAnswer) {
+	conn.dumpState.RemoteOffer()
+	conn.Log.Infof("OnRemoteOffer, on status ICE: %s, status Relay: %s", conn.statusICE, conn.statusRelay)
+	conn.handshaker.OnRemoteOffer(offer)
 }
 
 // WgConfig returns the WireGuard config
@@ -290,19 +299,24 @@ func (conn *Conn) WgConfig() WgConfig {
 	return conn.config.WgConfig
 }
 
-// Status returns current status of the Conn
-func (conn *Conn) Status() ConnStatus {
+// IsConnected returns true if the peer is connected
+func (conn *Conn) IsConnected() bool {
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
-	return conn.evalStatus()
+
+	return conn.evalStatus() == StatusConnected
 }
 
 func (conn *Conn) GetKey() string {
 	return conn.config.Key
 }
 
+func (conn *Conn) ConnID() id.ConnID {
+	return id.ConnID(conn)
+}
+
 // configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected
-func (conn *Conn) onICEConnectionIsReady(priority ConnPriority, iceConnInfo ICEConnInfo) {
+func (conn *Conn) onICEConnectionIsReady(priority conntype.ConnPriority, iceConnInfo ICEConnInfo) {
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
 
@@ -310,21 +324,22 @@ func (conn *Conn) onICEConnectionIsReady(priority ConnPriority, iceConnInfo ICEC
 		return
 	}
 
-	if remoteConnNil(conn.log, iceConnInfo.RemoteConn) {
-		conn.log.Errorf("remote ICE connection is nil")
+	if remoteConnNil(conn.Log, iceConnInfo.RemoteConn) {
+		conn.Log.Errorf("remote ICE connection is nil")
 		return
 	}
 
 	// this never should happen, because Relay is the lower priority and ICE always close the deprecated connection before upgrade
 	// todo consider to remove this check
 	if conn.currentConnPriority > priority {
-		conn.log.Infof("current connection priority (%s) is higher than the new one (%s), do not upgrade connection", conn.currentConnPriority, priority)
-		conn.statusICE.Set(StatusConnected)
+		conn.Log.Infof("current connection priority (%s) is higher than the new one (%s), do not upgrade connection", conn.currentConnPriority, priority)
+		conn.statusICE.SetConnected()
 		conn.updateIceState(iceConnInfo)
 		return
 	}
 
-	conn.log.Infof("set ICE to active connection")
+	conn.Log.Infof("set ICE to active connection")
+	conn.dumpState.P2PConnected()
 
 	var (
 		ep      *net.UDPAddr
@@ -332,9 +347,10 @@ func (conn *Conn) onICEConnectionIsReady(priority ConnPriority, iceConnInfo ICEC
 		err     error
 	)
 	if iceConnInfo.RelayedOnLocal {
+		conn.dumpState.NewLocalProxy()
 		wgProxy, err = conn.newProxy(iceConnInfo.RemoteConn)
 		if err != nil {
-			conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
+			conn.Log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
 			return
 		}
 		ep = wgProxy.EndpointAddr()
@@ -349,36 +365,32 @@ func (conn *Conn) onICEConnectionIsReady(priority ConnPriority, iceConnInfo ICEC
 		ep = directEp
 	}
 
-	if err := conn.runBeforeAddPeerHooks(ep.IP); err != nil {
-		conn.log.Errorf("Before add peer hook failed: %v", err)
-	}
-
 	conn.workerRelay.DisableWgWatcher()
+	// todo consider to run conn.wgWatcherWg.Wait() here
 
 	if conn.wgProxyRelay != nil {
-		conn.log.Debugf("pause Relayed proxy")
 		conn.wgProxyRelay.Pause()
 	}
 
 	if wgProxy != nil {
-		conn.log.Debugf("run ICE proxy")
 		wgProxy.Work()
 	}
 
-	conn.log.Infof("configure WireGuard endpoint to: %s", ep.String())
-	if err = conn.configureWGEndpoint(ep); err != nil {
+	conn.Log.Infof("configure WireGuard endpoint to: %s", ep.String())
+	if err = conn.configureWGEndpoint(ep, iceConnInfo.RosenpassPubKey); err != nil {
 		conn.handleConfigurationFailure(err, wgProxy)
 		return
 	}
 	wgConfigWorkaround()
 
+
 	if conn.wgProxyRelay != nil {
-		conn.log.Debugf("redirect packages from relayed conn to WireGuard")
+		conn.Log.Debugf("redirect packages from relayed conn to WireGuard")
 		conn.wgProxyRelay.RedirectAs(ep)
 	}
 
 	conn.currentConnPriority = priority
-	conn.statusICE.Set(StatusConnected)
+	conn.statusICE.SetConnected()
 	conn.updateIceState(iceConnInfo)
 	conn.doOnConnected(iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr)
 }
@@ -391,35 +403,41 @@ func (conn *Conn) onICEStateDisconnected() {
 		return
 	}
 
-	conn.log.Tracef("ICE connection state changed to disconnected")
+	conn.Log.Tracef("ICE connection state changed to disconnected")
 
 	if conn.wgProxyICE != nil {
 		if err := conn.wgProxyICE.CloseConn(); err != nil {
-			conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err)
+			conn.Log.Warnf("failed to close deprecated wg proxy conn: %v", err)
 		}
 	}
 
 	// switch back to relay connection
 	if conn.isReadyToUpgrade() {
-		conn.log.Infof("ICE disconnected, set Relay to active connection")
+		conn.Log.Infof("ICE disconnected, set Relay to active connection")
+		conn.dumpState.SwitchToRelay()
+		conn.wgProxyRelay.Work()
 
-		if err := conn.configureWGEndpoint(conn.wgProxyRelay.EndpointAddr()); err != nil {
-			conn.log.Errorf("failed to switch to relay conn: %v", err)
+		if err := conn.configureWGEndpoint(conn.wgProxyRelay.EndpointAddr(), conn.rosenpassRemoteKey); err != nil {
+			conn.Log.Errorf("failed to switch to relay conn: %v", err)
 		}
 
+		conn.wgWatcherWg.Add(1)
+		go func() {
+			defer conn.wgWatcherWg.Done()
+			conn.workerRelay.EnableWgWatcher(conn.ctx)
+		}()
 		conn.wgProxyRelay.Work()
-		conn.workerRelay.EnableWgWatcher(conn.ctx)
-		conn.currentConnPriority = connPriorityRelay
+		conn.currentConnPriority = conntype.Relay
 	} else {
-		conn.log.Infof("ICE disconnected, do not switch to Relay. Reset priority to: %s", connPriorityNone.String())
-		conn.currentConnPriority = connPriorityNone
+		conn.Log.Infof("ICE disconnected, do not switch to Relay. Reset priority to: %s", conntype.None.String())
+		conn.currentConnPriority = conntype.None
 	}
 
-	changed := conn.statusICE.Get() != StatusDisconnected
+	changed := conn.statusICE.Get() != worker.StatusDisconnected
 	if changed {
 		conn.guard.SetICEConnDisconnected()
 	}
-	conn.statusICE.Set(StatusDisconnected)
+	conn.statusICE.SetDisconnected()
 
 	peerState := State{
 		PubKey:           conn.config.Key,
@@ -430,7 +448,7 @@ func (conn *Conn) onICEStateDisconnected() {
 
 	err := conn.statusRecorder.UpdatePeerICEStateToDisconnected(peerState)
 	if err != nil {
-		conn.log.Warnf("unable to set peer's state to disconnected ice, got error: %v", err)
+		conn.Log.Warnf("unable to set peer's state to disconnected ice, got error: %v", err)
 	}
 }
 
@@ -440,49 +458,55 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) {
 
 	if conn.ctx.Err() != nil {
 		if err := rci.relayedConn.Close(); err != nil {
-			conn.log.Warnf("failed to close unnecessary relayed connection: %v", err)
+			conn.Log.Warnf("failed to close unnecessary relayed connection: %v", err)
 		}
 		return
 	}
 
-	conn.log.Debugf("Relay connection has been established, setup the WireGuard")
+	conn.dumpState.RelayConnected()
+	conn.Log.Debugf("Relay connection has been established, setup the WireGuard")
 
 	wgProxy, err := conn.newProxy(rci.relayedConn)
 	if err != nil {
-		conn.log.Errorf("failed to add relayed net.Conn to local proxy: %v", err)
+		conn.Log.Errorf("failed to add relayed net.Conn to local proxy: %v", err)
 		return
 	}
+	wgProxy.SetDisconnectListener(conn.onRelayDisconnected)
 
-	conn.log.Infof("created new wgProxy for relay connection: %s", wgProxy.EndpointAddr().String())
+	conn.dumpState.NewLocalProxy()
 
-	if conn.iceP2PIsActive() {
-		conn.log.Debugf("do not switch to relay because current priority is: %s", conn.currentConnPriority.String())
+	conn.Log.Infof("created new wgProxy for relay connection: %s", wgProxy.EndpointAddr().String())
+
+	if conn.isICEActive() {
+		conn.Log.Debugf("do not switch to relay because current priority is: %s", conn.currentConnPriority.String())
 		conn.setRelayedProxy(wgProxy)
-		conn.statusRelay.Set(StatusConnected)
+		conn.statusRelay.SetConnected()
 		conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey)
 		return
 	}
 
-	if err := conn.runBeforeAddPeerHooks(wgProxy.EndpointAddr().IP); err != nil {
-		conn.log.Errorf("Before add peer hook failed: %v", err)
-	}
-
 	wgProxy.Work()
-	if err := conn.configureWGEndpoint(wgProxy.EndpointAddr()); err != nil {
+	if err := conn.configureWGEndpoint(wgProxy.EndpointAddr(), rci.rosenpassPubKey); err != nil {
 		if err := wgProxy.CloseConn(); err != nil {
-			conn.log.Warnf("Failed to close relay connection: %v", err)
+			conn.Log.Warnf("Failed to close relay connection: %v", err)
 		}
-		conn.log.Errorf("Failed to update WireGuard peer configuration: %v", err)
+		conn.Log.Errorf("Failed to update WireGuard peer configuration: %v", err)
 		return
 	}
-	conn.workerRelay.EnableWgWatcher(conn.ctx)
+
+	conn.wgWatcherWg.Add(1)
+	go func() {
+		defer conn.wgWatcherWg.Done()
+		conn.workerRelay.EnableWgWatcher(conn.ctx)
+	}()
 
 	wgConfigWorkaround()
-	conn.currentConnPriority = connPriorityRelay
-	conn.statusRelay.Set(StatusConnected)
+	conn.rosenpassRemoteKey = rci.rosenpassPubKey
+	conn.currentConnPriority = conntype.Relay
+	conn.statusRelay.SetConnected()
 	conn.setRelayedProxy(wgProxy)
 	conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey)
-	conn.log.Infof("start to communicate with peer via relay")
+	conn.Log.Infof("start to communicate with peer via relay")
 	conn.doOnConnected(rci.rosenpassPubKey, rci.rosenpassAddr)
 }
 
@@ -494,13 +518,11 @@ func (conn *Conn) onRelayDisconnected() {
 		return
 	}
 
-	conn.log.Debugf("relay connection is disconnected")
+	conn.Log.Debugf("relay connection is disconnected")
 
-	if conn.currentConnPriority == connPriorityRelay {
-		conn.log.Debugf("clean up WireGuard config")
-		if err := conn.removeWgPeer(); err != nil {
-			conn.log.Errorf("failed to remove wg endpoint: %v", err)
-		}
+	if conn.currentConnPriority == conntype.Relay {
+		conn.Log.Debugf("clean up WireGuard config")
+		conn.currentConnPriority = conntype.None
 	}
 
 	if conn.wgProxyRelay != nil {
@@ -508,11 +530,11 @@ func (conn *Conn) onRelayDisconnected() {
 		conn.wgProxyRelay = nil
 	}
 
-	changed := conn.statusRelay.Get() != StatusDisconnected
+	changed := conn.statusRelay.Get() != worker.StatusDisconnected
 	if changed {
 		conn.guard.SetRelayedConnDisconnected()
 	}
-	conn.statusRelay.Set(StatusDisconnected)
+	conn.statusRelay.SetDisconnected()
 
 	peerState := State{
 		PubKey:           conn.config.Key,
@@ -521,31 +543,25 @@ func (conn *Conn) onRelayDisconnected() {
 		ConnStatusUpdate: time.Now(),
 	}
 	if err := conn.statusRecorder.UpdatePeerRelayedStateToDisconnected(peerState); err != nil {
-		conn.log.Warnf("unable to save peer's state to Relay disconnected, got error: %v", err)
+		conn.Log.Warnf("unable to save peer's state to Relay disconnected, got error: %v", err)
 	}
 }
 
-func (conn *Conn) listenGuardEvent(ctx context.Context) {
-	for {
-		select {
-		case <-conn.guard.Reconnect:
-			conn.log.Debugf("send offer to peer")
-			if err := conn.handshaker.SendOffer(); err != nil {
-				conn.log.Errorf("failed to send offer: %v", err)
-			}
-		case <-ctx.Done():
-			return
-		}
+func (conn *Conn) onGuardEvent() {
+	conn.dumpState.SendOffer()
+	if err := conn.handshaker.SendOffer(); err != nil {
+		conn.Log.Errorf("failed to send offer: %v", err)
 	}
 }
 
-func (conn *Conn) configureWGEndpoint(addr *net.UDPAddr) error {
+func (conn *Conn) configureWGEndpoint(addr *net.UDPAddr, remoteRPKey []byte) error {
+	presharedKey := conn.presharedKey(remoteRPKey)
 	return conn.config.WgConfig.WgInterface.UpdatePeer(
 		conn.config.WgConfig.RemoteKey,
 		conn.config.WgConfig.AllowedIps,
 		defaultWgKeepAlive,
 		addr,
-		conn.config.WgConfig.PreSharedKey,
+		presharedKey,
 	)
 }
 
@@ -561,7 +577,7 @@ func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []by
 
 	err := conn.statusRecorder.UpdatePeerRelayedState(peerState)
 	if err != nil {
-		conn.log.Warnf("unable to save peer's Relay state, got error: %v", err)
+		conn.Log.Warnf("unable to save peer's Relay state, got error: %v", err)
 	}
 }
 
@@ -580,17 +596,18 @@ func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo) {
 
 	err := conn.statusRecorder.UpdatePeerICEState(peerState)
 	if err != nil {
-		conn.log.Warnf("unable to save peer's ICE state, got error: %v", err)
+		conn.Log.Warnf("unable to save peer's ICE state, got error: %v", err)
 	}
 }
 
 func (conn *Conn) setStatusToDisconnected() {
-	conn.statusRelay.Set(StatusDisconnected)
-	conn.statusICE.Set(StatusDisconnected)
+	conn.statusRelay.SetDisconnected()
+	conn.statusICE.SetDisconnected()
+	conn.currentConnPriority = conntype.None
 
 	peerState := State{
 		PubKey:           conn.config.Key,
-		ConnStatus:       StatusDisconnected,
+		ConnStatus:       StatusIdle,
 		ConnStatusUpdate: time.Now(),
 		Mux:              new(sync.RWMutex),
 	}
@@ -598,10 +615,10 @@ func (conn *Conn) setStatusToDisconnected() {
 	if err != nil {
 		// pretty common error because by that time Engine can already remove the peer and status won't be available.
 		// todo rethink status updates
-		conn.log.Debugf("error while updating peer's state, err: %v", err)
+		conn.Log.Debugf("error while updating peer's state, err: %v", err)
 	}
 	if err := conn.statusRecorder.UpdateWireGuardPeerState(conn.config.Key, configurer.WGStats{}); err != nil {
-		conn.log.Debugf("failed to reset wireguard stats for peer: %s", err)
+		conn.Log.Debugf("failed to reset wireguard stats for peer: %s", err)
 	}
 }
 
@@ -611,7 +628,7 @@ func (conn *Conn) doOnConnected(remoteRosenpassPubKey []byte, remoteRosenpassAdd
 	}
 
 	if conn.onConnected != nil {
-		conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.allowedIP.String(), remoteRosenpassAddr)
+		conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.config.WgConfig.AllowedIps[0].Addr().String(), remoteRosenpassAddr)
 	}
 }
 
@@ -629,32 +646,24 @@ func (conn *Conn) waitInitialRandomSleepTime(ctx context.Context) {
 }
 
 func (conn *Conn) isRelayed() bool {
-	if conn.statusRelay.Get() == StatusDisconnected && (conn.statusICE.Get() == StatusDisconnected || conn.statusICE.Get() == StatusConnecting) {
+	switch conn.currentConnPriority {
+	case conntype.Relay, conntype.ICETurn:
+		return true
+	default:
 		return false
 	}
-
-	if conn.currentConnPriority == connPriorityICEP2P {
-		return false
-	}
-
-	return true
 }
 
 func (conn *Conn) evalStatus() ConnStatus {
-	if conn.statusRelay.Get() == StatusConnected || conn.statusICE.Get() == StatusConnected {
+	if conn.statusRelay.Get() == worker.StatusConnected || conn.statusICE.Get() == worker.StatusConnected {
 		return StatusConnected
 	}
 
-	if conn.statusRelay.Get() == StatusConnecting || conn.statusICE.Get() == StatusConnecting {
-		return StatusConnecting
-	}
-
-	return StatusDisconnected
+	return StatusConnecting
 }
 
 func (conn *Conn) isConnectedOnAllWay() (connected bool) {
-	conn.mu.Lock()
-	defer conn.mu.Unlock()
+	// would be better to protect this with a mutex, but it could cause deadlock with Close function
 
 	defer func() {
 		if !connected {
@@ -662,12 +671,12 @@ func (conn *Conn) isConnectedOnAllWay() (connected bool) {
 		}
 	}()
 
-	if conn.statusICE.Get() == StatusDisconnected {
+	if conn.statusICE.Get() == worker.StatusDisconnected && !conn.workerICE.InProgress() {
 		return false
 	}
 
 	if conn.workerRelay.IsRelayConnectionSupportedWithPeer() {
-		if conn.statusRelay.Get() != StatusConnected {
+		if conn.statusRelay.Get() == worker.StatusDisconnected {
 			return false
 		}
 	}
@@ -675,57 +684,27 @@ func (conn *Conn) isConnectedOnAllWay() (connected bool) {
 	return true
 }
 
-func (conn *Conn) runBeforeAddPeerHooks(ip net.IP) error {
-	conn.connIDICE = nbnet.GenerateConnID()
-	for _, hook := range conn.beforeAddPeerHooks {
-		if err := hook(conn.connIDICE, ip); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (conn *Conn) freeUpConnID() {
-	if conn.connIDRelay != "" {
-		for _, hook := range conn.afterRemovePeerHooks {
-			if err := hook(conn.connIDRelay); err != nil {
-				conn.log.Errorf("After remove peer hook failed: %v", err)
-			}
-		}
-		conn.connIDRelay = ""
-	}
-
-	if conn.connIDICE != "" {
-		for _, hook := range conn.afterRemovePeerHooks {
-			if err := hook(conn.connIDICE); err != nil {
-				conn.log.Errorf("After remove peer hook failed: %v", err)
-			}
-		}
-		conn.connIDICE = ""
-	}
-}
-
 func (conn *Conn) newProxy(remoteConn net.Conn) (wgproxy.Proxy, error) {
-	conn.log.Debugf("setup proxied WireGuard connection")
+	conn.Log.Debugf("setup proxied WireGuard connection")
 	udpAddr := &net.UDPAddr{
-		IP:   conn.allowedIP,
+		IP:   conn.config.WgConfig.AllowedIps[0].Addr().AsSlice(),
 		Port: conn.config.WgConfig.WgListenPort,
 	}
 
 	wgProxy := conn.config.WgConfig.WgInterface.GetProxy()
 	if err := wgProxy.AddTurnConn(conn.ctx, udpAddr, remoteConn); err != nil {
-		conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
+		conn.Log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
 		return nil, err
 	}
 	return wgProxy, nil
 }
 
 func (conn *Conn) isReadyToUpgrade() bool {
-	return conn.wgProxyRelay != nil && conn.currentConnPriority != connPriorityRelay
+	return conn.wgProxyRelay != nil && conn.currentConnPriority != conntype.Relay
 }
 
-func (conn *Conn) iceP2PIsActive() bool {
-	return conn.currentConnPriority == connPriorityICEP2P && conn.statusICE.Get() == StatusConnected
+func (conn *Conn) isICEActive() bool {
+	return (conn.currentConnPriority == conntype.ICEP2P || conn.currentConnPriority == conntype.ICETurn) && conn.statusICE.Get() == worker.StatusConnected
 }
 
 func (conn *Conn) removeWgPeer() error {
@@ -733,10 +712,10 @@ func (conn *Conn) removeWgPeer() error {
 }
 
 func (conn *Conn) handleConfigurationFailure(err error, wgProxy wgproxy.Proxy) {
-	conn.log.Warnf("Failed to update wg peer configuration: %v", err)
+	conn.Log.Warnf("Failed to update wg peer configuration: %v", err)
 	if wgProxy != nil {
 		if ierr := wgProxy.CloseConn(); ierr != nil {
-			conn.log.Warnf("Failed to close wg proxy: %v", ierr)
+			conn.Log.Warnf("Failed to close wg proxy: %v", ierr)
 		}
 	}
 	if conn.wgProxyRelay != nil {
@@ -746,24 +725,66 @@ func (conn *Conn) handleConfigurationFailure(err error, wgProxy wgproxy.Proxy) {
 
 func (conn *Conn) logTraceConnState() {
 	if conn.workerRelay.IsRelayConnectionSupportedWithPeer() {
-		conn.log.Tracef("connectivity guard check, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE)
+		conn.Log.Tracef("connectivity guard check, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE)
 	} else {
-		conn.log.Tracef("connectivity guard check, ice state: %s", conn.statusICE)
+		conn.Log.Tracef("connectivity guard check, ice state: %s", conn.statusICE)
 	}
 }
 
 func (conn *Conn) setRelayedProxy(proxy wgproxy.Proxy) {
 	if conn.wgProxyRelay != nil {
 		if err := conn.wgProxyRelay.CloseConn(); err != nil {
-			conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err)
+			conn.Log.Warnf("failed to close deprecated wg proxy conn: %v", err)
 		}
 	}
 	conn.wgProxyRelay = proxy
 }
 
 // AllowedIP returns the allowed IP of the remote peer
-func (conn *Conn) AllowedIP() net.IP {
-	return conn.allowedIP
+func (conn *Conn) AllowedIP() netip.Addr {
+	return conn.config.WgConfig.AllowedIps[0].Addr()
+}
+
+func (conn *Conn) AgentVersionString() string {
+	return conn.config.AgentVersion
+}
+
+func (conn *Conn) presharedKey(remoteRosenpassKey []byte) *wgtypes.Key {
+	if conn.config.RosenpassConfig.PubKey == nil {
+		return conn.config.WgConfig.PreSharedKey
+	}
+
+	if remoteRosenpassKey == nil && conn.config.RosenpassConfig.PermissiveMode {
+		return conn.config.WgConfig.PreSharedKey
+	}
+
+	determKey, err := conn.rosenpassDetermKey()
+	if err != nil {
+		conn.Log.Errorf("failed to generate Rosenpass initial key: %v", err)
+		return conn.config.WgConfig.PreSharedKey
+	}
+
+	return determKey
+}
+
+// todo: move this logic into Rosenpass package
+func (conn *Conn) rosenpassDetermKey() (*wgtypes.Key, error) {
+	lk := []byte(conn.config.LocalKey)
+	rk := []byte(conn.config.Key) // remote key
+	var keyInput []byte
+	if string(lk) > string(rk) {
+		//nolint:gocritic
+		keyInput = append(lk[:16], rk[:16]...)
+	} else {
+		//nolint:gocritic
+		keyInput = append(rk[:16], lk[:16]...)
+	}
+
+	key, err := wgtypes.NewKey(keyInput)
+	if err != nil {
+		return nil, err
+	}
+	return &key, nil
 }
 
 func isController(config ConnConfig) bool {

client/internal/peer/conn_status.go

@@ -1,58 +1,29 @@
 package peer
 
 import (
-	"sync/atomic"
-
 	log "github.com/sirupsen/logrus"
 )
 
 const (
-	// StatusConnected indicate the peer is in connected state
-	StatusConnected ConnStatus = iota
+	// StatusIdle indicate the peer is in disconnected state
+	StatusIdle ConnStatus = iota
 	// StatusConnecting indicate the peer is in connecting state
 	StatusConnecting
-	// StatusDisconnected indicate the peer is in disconnected state
-	StatusDisconnected
+	// StatusConnected indicate the peer is in connected state
+	StatusConnected
 )
 
 // ConnStatus describe the status of a peer's connection
 type ConnStatus int32
 
-// AtomicConnStatus is a thread-safe wrapper for ConnStatus
-type AtomicConnStatus struct {
-	status atomic.Int32
-}
-
-// NewAtomicConnStatus creates a new AtomicConnStatus with the given initial status
-func NewAtomicConnStatus() *AtomicConnStatus {
-	acs := &AtomicConnStatus{}
-	acs.Set(StatusDisconnected)
-	return acs
-}
-
-// Get returns the current connection status
-func (acs *AtomicConnStatus) Get() ConnStatus {
-	return ConnStatus(acs.status.Load())
-}
-
-// Set updates the connection status
-func (acs *AtomicConnStatus) Set(status ConnStatus) {
-	acs.status.Store(int32(status))
-}
-
-// String returns the string representation of the current status
-func (acs *AtomicConnStatus) String() string {
-	return acs.Get().String()
-}
-
 func (s ConnStatus) String() string {
 	switch s {
 	case StatusConnecting:
 		return "Connecting"
 	case StatusConnected:
 		return "Connected"
-	case StatusDisconnected:
-		return "Disconnected"
+	case StatusIdle:
+		return "Idle"
 	default:
 		log.Errorf("unknown status: %d", s)
 		return "INVALID_PEER_CONNECTION_STATUS"

client/internal/peer/conn_status_test.go

@@ -14,7 +14,7 @@ func TestConnStatus_String(t *testing.T) {
 		want   string
 	}{
 		{"StatusConnected", StatusConnected, "Connected"},
-		{"StatusDisconnected", StatusDisconnected, "Disconnected"},
+		{"StatusIdle", StatusIdle, "Idle"},
 		{"StatusConnecting", StatusConnecting, "Connecting"},
 	}
 
@@ -24,5 +24,4 @@ func TestConnStatus_String(t *testing.T) {
 			assert.Equal(t, got, table.want, "they should be equal")
 		})
 	}
-
 }

client/internal/peer/conn_test.go

@@ -2,14 +2,15 @@ package peer
 
 import (
 	"context"
+	"fmt"
 	"os"
-	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 
 	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/internal/peer/dispatcher"
 	"github.com/netbirdio/netbird/client/internal/peer/guard"
 	"github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
@@ -17,6 +18,8 @@ import (
 	semaphoregroup "github.com/netbirdio/netbird/util/semaphore-group"
 )
 
+var testDispatcher = dispatcher.NewConnectionDispatcher()
+
 var connConf = ConnConfig{
 	Key:         "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
 	LocalKey:    "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
@@ -28,7 +31,7 @@ var connConf = ConnConfig{
 }
 
 func TestMain(m *testing.M) {
-	_ = util.InitLog("trace", "console")
+	_ = util.InitLog("trace", util.LogConsole)
 	code := m.Run()
 	os.Exit(code)
 }
@@ -47,7 +50,13 @@ func TestNewConn_interfaceFilter(t *testing.T) {
 
 func TestConn_GetKey(t *testing.T) {
 	swWatcher := guard.NewSRWatcher(nil, nil, nil, connConf.ICEConfig)
-	conn, err := NewConn(context.Background(), connConf, nil, nil, nil, nil, swWatcher, semaphoregroup.NewSemaphoreGroup(1))
+
+	sd := ServiceDependencies{
+		SrWatcher:          swWatcher,
+		Semaphore:          semaphoregroup.NewSemaphoreGroup(1),
+		PeerConnDispatcher: testDispatcher,
+	}
+	conn, err := NewConn(connConf, sd)
 	if err != nil {
 		return
 	}
@@ -59,105 +68,219 @@ func TestConn_GetKey(t *testing.T) {
 
 func TestConn_OnRemoteOffer(t *testing.T) {
 	swWatcher := guard.NewSRWatcher(nil, nil, nil, connConf.ICEConfig)
-	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), nil, nil, nil, swWatcher, semaphoregroup.NewSemaphoreGroup(1))
+	sd := ServiceDependencies{
+		StatusRecorder:     NewRecorder("https://mgm"),
+		SrWatcher:          swWatcher,
+		Semaphore:          semaphoregroup.NewSemaphoreGroup(1),
+		PeerConnDispatcher: testDispatcher,
+	}
+	conn, err := NewConn(connConf, sd)
 	if err != nil {
 		return
 	}
 
-	wg := sync.WaitGroup{}
-	wg.Add(2)
-	go func() {
-		<-conn.handshaker.remoteOffersCh
-		wg.Done()
-	}()
+	onNewOffeChan := make(chan struct{})
 
-	go func() {
-		for {
-			accepted := conn.OnRemoteOffer(OfferAnswer{
-				IceCredentials: IceCredentials{
-					UFrag: "test",
-					Pwd:   "test",
-				},
-				WgListenPort: 0,
-				Version:      "",
-			})
-			if accepted {
-				wg.Done()
-				return
-			}
-		}
-	}()
+	conn.handshaker.AddOnNewOfferListener(func(remoteOfferAnswer *OfferAnswer) {
+		onNewOffeChan <- struct{}{}
+	})
 
-	wg.Wait()
+	conn.OnRemoteOffer(OfferAnswer{
+		IceCredentials: IceCredentials{
+			UFrag: "test",
+			Pwd:   "test",
+		},
+		WgListenPort: 0,
+		Version:      "",
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	select {
+	case <-onNewOffeChan:
+		// success
+	case <-ctx.Done():
+		t.Error("expected to receive a new offer notification, but timed out")
+	}
 }
 
 func TestConn_OnRemoteAnswer(t *testing.T) {
 	swWatcher := guard.NewSRWatcher(nil, nil, nil, connConf.ICEConfig)
-	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), nil, nil, nil, swWatcher, semaphoregroup.NewSemaphoreGroup(1))
+	sd := ServiceDependencies{
+		StatusRecorder:     NewRecorder("https://mgm"),
+		SrWatcher:          swWatcher,
+		Semaphore:          semaphoregroup.NewSemaphoreGroup(1),
+		PeerConnDispatcher: testDispatcher,
+	}
+	conn, err := NewConn(connConf, sd)
 	if err != nil {
 		return
 	}
 
-	wg := sync.WaitGroup{}
-	wg.Add(2)
-	go func() {
-		<-conn.handshaker.remoteAnswerCh
-		wg.Done()
-	}()
+	onNewOffeChan := make(chan struct{})
 
-	go func() {
-		for {
-			accepted := conn.OnRemoteAnswer(OfferAnswer{
-				IceCredentials: IceCredentials{
-					UFrag: "test",
-					Pwd:   "test",
-				},
-				WgListenPort: 0,
-				Version:      "",
-			})
-			if accepted {
-				wg.Done()
-				return
-			}
-		}
-	}()
+	conn.handshaker.AddOnNewOfferListener(func(remoteOfferAnswer *OfferAnswer) {
+		onNewOffeChan <- struct{}{}
+	})
 
-	wg.Wait()
+	conn.OnRemoteAnswer(OfferAnswer{
+		IceCredentials: IceCredentials{
+			UFrag: "test",
+			Pwd:   "test",
+		},
+		WgListenPort: 0,
+		Version:      "",
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	select {
+	case <-onNewOffeChan:
+		// success
+	case <-ctx.Done():
+		t.Error("expected to receive a new offer notification, but timed out")
+	}
 }
-func TestConn_Status(t *testing.T) {
-	swWatcher := guard.NewSRWatcher(nil, nil, nil, connConf.ICEConfig)
-	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), nil, nil, nil, swWatcher, semaphoregroup.NewSemaphoreGroup(1))
-	if err != nil {
-		return
+
+func TestConn_presharedKey(t *testing.T) {
+	conn1 := Conn{
+		config: ConnConfig{
+			Key:             "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
+			LocalKey:        "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
+			RosenpassConfig: RosenpassConfig{},
+		},
+	}
+	conn2 := Conn{
+		config: ConnConfig{
+			Key:             "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
+			LocalKey:        "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
+			RosenpassConfig: RosenpassConfig{},
+		},
 	}
 
-	tables := []struct {
-		name        string
-		statusIce   ConnStatus
-		statusRelay ConnStatus
-		want        ConnStatus
+	tests := []struct {
+		conn1Permissive         bool
+		conn1RosenpassEnabled   bool
+		conn2Permissive         bool
+		conn2RosenpassEnabled   bool
+		conn1ExpectedInitialKey bool
+		conn2ExpectedInitialKey bool
 	}{
-		{"StatusConnected", StatusConnected, StatusConnected, StatusConnected},
-		{"StatusDisconnected", StatusDisconnected, StatusDisconnected, StatusDisconnected},
-		{"StatusConnecting", StatusConnecting, StatusConnecting, StatusConnecting},
-		{"StatusConnectingIce", StatusConnecting, StatusDisconnected, StatusConnecting},
-		{"StatusConnectingIceAlternative", StatusConnecting, StatusConnected, StatusConnected},
-		{"StatusConnectingRelay", StatusDisconnected, StatusConnecting, StatusConnecting},
-		{"StatusConnectingRelayAlternative", StatusConnected, StatusConnecting, StatusConnected},
+		{
+			conn1Permissive:         false,
+			conn1RosenpassEnabled:   false,
+			conn2Permissive:         false,
+			conn2RosenpassEnabled:   false,
+			conn1ExpectedInitialKey: false,
+			conn2ExpectedInitialKey: false,
+		},
+		{
+			conn1Permissive:         false,
+			conn1RosenpassEnabled:   true,
+			conn2Permissive:         false,
+			conn2RosenpassEnabled:   true,
+			conn1ExpectedInitialKey: true,
+			conn2ExpectedInitialKey: true,
+		},
+		{
+			conn1Permissive:         false,
+			conn1RosenpassEnabled:   true,
+			conn2Permissive:         false,
+			conn2RosenpassEnabled:   false,
+			conn1ExpectedInitialKey: true,
+			conn2ExpectedInitialKey: false,
+		},
+		{
+			conn1Permissive:         false,
+			conn1RosenpassEnabled:   false,
+			conn2Permissive:         false,
+			conn2RosenpassEnabled:   true,
+			conn1ExpectedInitialKey: false,
+			conn2ExpectedInitialKey: true,
+		},
+		{
+			conn1Permissive:         true,
+			conn1RosenpassEnabled:   true,
+			conn2Permissive:         false,
+			conn2RosenpassEnabled:   false,
+			conn1ExpectedInitialKey: false,
+			conn2ExpectedInitialKey: false,
+		},
+		{
+			conn1Permissive:         false,
+			conn1RosenpassEnabled:   false,
+			conn2Permissive:         true,
+			conn2RosenpassEnabled:   true,
+			conn1ExpectedInitialKey: false,
+			conn2ExpectedInitialKey: false,
+		},
+		{
+			conn1Permissive:         true,
+			conn1RosenpassEnabled:   true,
+			conn2Permissive:         true,
+			conn2RosenpassEnabled:   true,
+			conn1ExpectedInitialKey: true,
+			conn2ExpectedInitialKey: true,
+		},
+		{
+			conn1Permissive:         false,
+			conn1RosenpassEnabled:   false,
+			conn2Permissive:         false,
+			conn2RosenpassEnabled:   true,
+			conn1ExpectedInitialKey: false,
+			conn2ExpectedInitialKey: true,
+		},
+		{
+			conn1Permissive:         false,
+			conn1RosenpassEnabled:   true,
+			conn2Permissive:         true,
+			conn2RosenpassEnabled:   true,
+			conn1ExpectedInitialKey: true,
+			conn2ExpectedInitialKey: true,
+		},
 	}
 
-	for _, table := range tables {
-		t.Run(table.name, func(t *testing.T) {
-			si := NewAtomicConnStatus()
-			si.Set(table.statusIce)
-			conn.statusICE = si
+	conn1.config.RosenpassConfig.PermissiveMode = true
+	for i, test := range tests {
+		tcase := i + 1
+		t.Run(fmt.Sprintf("Rosenpass test case %d", tcase), func(t *testing.T) {
+			conn1.config.RosenpassConfig = RosenpassConfig{}
+			conn2.config.RosenpassConfig = RosenpassConfig{}
 
-			sr := NewAtomicConnStatus()
-			sr.Set(table.statusRelay)
-			conn.statusRelay = sr
+			if test.conn1RosenpassEnabled {
+				conn1.config.RosenpassConfig.PubKey = []byte("dummykey")
+			}
+			conn1.config.RosenpassConfig.PermissiveMode = test.conn1Permissive
 
-			got := conn.Status()
-			assert.Equal(t, got, table.want, "they should be equal")
+			if test.conn2RosenpassEnabled {
+				conn2.config.RosenpassConfig.PubKey = []byte("dummykey")
+			}
+			conn2.config.RosenpassConfig.PermissiveMode = test.conn2Permissive
+
+			conn1PresharedKey := conn1.presharedKey(conn2.config.RosenpassConfig.PubKey)
+			conn2PresharedKey := conn2.presharedKey(conn1.config.RosenpassConfig.PubKey)
+
+			if test.conn1ExpectedInitialKey {
+				if conn1PresharedKey == nil {
+					t.Errorf("Case %d: Expected conn1 to have a non-nil key, but got nil", tcase)
+				}
+			} else {
+				if conn1PresharedKey != nil {
+					t.Errorf("Case %d: Expected conn1 to have a nil key, but got %v", tcase, conn1PresharedKey)
+				}
+			}
+
+			// Assert conn2's key expectation
+			if test.conn2ExpectedInitialKey {
+				if conn2PresharedKey == nil {
+					t.Errorf("Case %d: Expected conn2 to have a non-nil key, but got nil", tcase)
+				}
+			} else {
+				if conn2PresharedKey != nil {
+					t.Errorf("Case %d: Expected conn2 to have a nil key, but got %v", tcase, conn2PresharedKey)
+				}
+			}
 		})
 	}
 }

client/internal/peer/conntype/priority.go

@@ -0,0 +1,29 @@
+package conntype
+
+import (
+	"fmt"
+)
+
+const (
+	None    ConnPriority = 0
+	Relay   ConnPriority = 1
+	ICETurn ConnPriority = 2
+	ICEP2P  ConnPriority = 3
+)
+
+type ConnPriority int
+
+func (cp ConnPriority) String() string {
+	switch cp {
+	case None:
+		return "None"
+	case Relay:
+		return "PriorityRelay"
+	case ICETurn:
+		return "PriorityICETurn"
+	case ICEP2P:
+		return "PriorityICEP2P"
+	default:
+		return fmt.Sprintf("ConnPriority(%d)", cp)
+	}
+}

client/internal/peer/dispatcher/dispatcher.go

@@ -0,0 +1,52 @@
+package dispatcher
+
+import (
+	"sync"
+
+	"github.com/netbirdio/netbird/client/internal/peer/id"
+)
+
+type ConnectionListener struct {
+	OnConnected    func(peerID id.ConnID)
+	OnDisconnected func(peerID id.ConnID)
+}
+
+type ConnectionDispatcher struct {
+	listeners map[*ConnectionListener]struct{}
+	mu        sync.Mutex
+}
+
+func NewConnectionDispatcher() *ConnectionDispatcher {
+	return &ConnectionDispatcher{
+		listeners: make(map[*ConnectionListener]struct{}),
+	}
+}
+
+func (e *ConnectionDispatcher) AddListener(listener *ConnectionListener) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.listeners[listener] = struct{}{}
+}
+
+func (e *ConnectionDispatcher) RemoveListener(listener *ConnectionListener) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	delete(e.listeners, listener)
+}
+
+func (e *ConnectionDispatcher) NotifyConnected(peerConnID id.ConnID) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	for listener := range e.listeners {
+		listener.OnConnected(peerConnID)
+	}
+}
+
+func (e *ConnectionDispatcher) NotifyDisconnected(peerConnID id.ConnID) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	for listener := range e.listeners {
+		listener.OnDisconnected(peerConnID)
+	}
+}

client/internal/peer/guard/guard.go

@@ -8,10 +8,6 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-const (
-	reconnectMaxElapsedTime = 30 * time.Minute
-)
-
 type isConnectedFunc func() bool
 
 // Guard is responsible for the reconnection logic.
@@ -23,9 +19,7 @@ type isConnectedFunc func() bool
 // - Relayed connection disconnected
 // - ICE candidate changes
 type Guard struct {
-	Reconnect               chan struct{}
 	log                     *log.Entry
-	isController            bool
 	isConnectedOnAllWay     isConnectedFunc
 	timeout                 time.Duration
 	srWatcher               *SRWatcher
@@ -33,11 +27,9 @@ type Guard struct {
 	iCEConnDisconnected     chan struct{}
 }
 
-func NewGuard(log *log.Entry, isController bool, isConnectedFn isConnectedFunc, timeout time.Duration, srWatcher *SRWatcher) *Guard {
+func NewGuard(log *log.Entry, isConnectedFn isConnectedFunc, timeout time.Duration, srWatcher *SRWatcher) *Guard {
 	return &Guard{
-		Reconnect:               make(chan struct{}, 1),
 		log:                     log,
-		isController:            isController,
 		isConnectedOnAllWay:     isConnectedFn,
 		timeout:                 timeout,
 		srWatcher:               srWatcher,
@@ -46,12 +38,9 @@ func NewGuard(log *log.Entry, isController bool, isConnectedFn isConnectedFunc,
 	}
 }
 
-func (g *Guard) Start(ctx context.Context) {
-	if g.isController {
-		g.reconnectLoopWithRetry(ctx)
-	} else {
-		g.listenForDisconnectEvents(ctx)
-	}
+func (g *Guard) Start(ctx context.Context, eventCallback func()) {
+	g.log.Infof("starting guard for reconnection with MaxInterval: %s", g.timeout)
+	g.reconnectLoopWithRetry(ctx, eventCallback)
 }
 
 func (g *Guard) SetRelayedConnDisconnected() {
@@ -68,20 +57,17 @@ func (g *Guard) SetICEConnDisconnected() {
 	}
 }
 
-// reconnectLoopWithRetry periodically check (max 30 min) the connection status.
+// reconnectLoopWithRetry periodically check the connection status.
 // Try to send offer while the P2P is not established or while the Relay is not connected if is it supported
-func (g *Guard) reconnectLoopWithRetry(ctx context.Context) {
-	waitForInitialConnectionTry(ctx)
-
+func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) {
 	srReconnectedChan := g.srWatcher.NewListener()
 	defer g.srWatcher.RemoveListener(srReconnectedChan)
 
-	ticker := g.prepareExponentTicker(ctx)
+	ticker := g.initialTicker(ctx)
 	defer ticker.Stop()
 
 	tickerChannel := ticker.C
 
-	g.log.Infof("start reconnect loop...")
 	for {
 		select {
 		case t := <-tickerChannel:
@@ -93,9 +79,8 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context) {
 			}
 
 			if !g.isConnectedOnAllWay() {
-				g.triggerOfferSending()
+				callback()
 			}
-
 		case <-g.relayedConnDisconnected:
 			g.log.Debugf("Relay connection changed, reset reconnection ticker")
 			ticker.Stop()
@@ -121,30 +106,18 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context) {
 	}
 }
 
-// listenForDisconnectEvents is used when the peer is not a controller and it should reconnect to the peer
-// when the connection is lost. It will try to establish a connection only once time if before the connection was established
-// It track separately the ice and relay connection status. Just because a lower priority connection reestablished it does not
-// mean that to switch to it. We always force to use the higher priority connection.
-func (g *Guard) listenForDisconnectEvents(ctx context.Context) {
-	srReconnectedChan := g.srWatcher.NewListener()
-	defer g.srWatcher.RemoveListener(srReconnectedChan)
+// initialTicker give chance to the peer to establish the initial connection.
+func (g *Guard) initialTicker(ctx context.Context) *backoff.Ticker {
+	bo := backoff.WithContext(&backoff.ExponentialBackOff{
+		InitialInterval:     3 * time.Second,
+		RandomizationFactor: 0.1,
+		Multiplier:          2,
+		MaxInterval:         g.timeout,
+		Stop:                backoff.Stop,
+		Clock:               backoff.SystemClock,
+	}, ctx)
 
-	g.log.Infof("start listen for reconnect events...")
-	for {
-		select {
-		case <-g.relayedConnDisconnected:
-			g.log.Debugf("Relay connection changed, triggering reconnect")
-			g.triggerOfferSending()
-		case <-g.iCEConnDisconnected:
-			g.log.Debugf("ICE state changed, try to send new offer")
-			g.triggerOfferSending()
-		case <-srReconnectedChan:
-			g.triggerOfferSending()
-		case <-ctx.Done():
-			g.log.Debugf("context is done, stop reconnect loop")
-			return
-		}
-	}
+	return backoff.NewTicker(bo)
 }
 
 func (g *Guard) prepareExponentTicker(ctx context.Context) *backoff.Ticker {
@@ -153,7 +126,6 @@ func (g *Guard) prepareExponentTicker(ctx context.Context) *backoff.Ticker {
 		RandomizationFactor: 0.1,
 		Multiplier:          2,
 		MaxInterval:         g.timeout,
-		MaxElapsedTime:      reconnectMaxElapsedTime,
 		Stop:                backoff.Stop,
 		Clock:               backoff.SystemClock,
 	}, ctx)
@@ -163,20 +135,3 @@ func (g *Guard) prepareExponentTicker(ctx context.Context) *backoff.Ticker {
 
 	return ticker
 }
-
-func (g *Guard) triggerOfferSending() {
-	select {
-	case g.Reconnect <- struct{}{}:
-	default:
-	}
-}
-
-// Give chance to the peer to establish the initial connection.
-// With it, we can decrease to send necessary offer
-func waitForInitialConnectionTry(ctx context.Context) {
-	select {
-	case <-ctx.Done():
-		return
-	case <-time.After(3 * time.Second):
-	}
-}

client/internal/peer/guard/ice_monitor.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/pion/ice/v3"
+	"github.com/pion/ice/v4"
 	log "github.com/sirupsen/logrus"
 
 	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"

client/internal/peer/handshaker.go

@@ -39,11 +39,19 @@ type OfferAnswer struct {
 
 	// relay server address
 	RelaySrvAddress string
+	// SessionID is the unique identifier of the session, used to discard old messages
+	SessionID *ICESessionID
+}
+
+func (oa *OfferAnswer) SessionIDString() string {
+	if oa.SessionID == nil {
+		return "unknown"
+	}
+	return oa.SessionID.String()
 }
 
 type Handshaker struct {
 	mu                  sync.Mutex
-	ctx                 context.Context
 	log                 *log.Entry
 	config              ConnConfig
 	signaler            *Signaler
@@ -57,9 +65,8 @@ type Handshaker struct {
 	remoteAnswerCh chan OfferAnswer
 }
 
-func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ice *WorkerICE, relay *WorkerRelay) *Handshaker {
+func NewHandshaker(log *log.Entry, config ConnConfig, signaler *Signaler, ice *WorkerICE, relay *WorkerRelay) *Handshaker {
 	return &Handshaker{
-		ctx:            ctx,
 		log:            log,
 		config:         config,
 		signaler:       signaler,
@@ -74,23 +81,27 @@ func (h *Handshaker) AddOnNewOfferListener(offer func(remoteOfferAnswer *OfferAn
 	h.onNewOfferListeners = append(h.onNewOfferListeners, offer)
 }
 
-func (h *Handshaker) Listen() {
+func (h *Handshaker) Listen(ctx context.Context) {
 	for {
-		h.log.Debugf("wait for remote offer confirmation")
-		remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation()
-		if err != nil {
-			var connectionClosedError *ConnectionClosedError
-			if errors.As(err, &connectionClosedError) {
-				h.log.Tracef("stop handshaker")
-				return
+		select {
+		case remoteOfferAnswer := <-h.remoteOffersCh:
+			// received confirmation from the remote peer -> ready to proceed
+			if err := h.sendAnswer(); err != nil {
+				h.log.Errorf("failed to send remote offer confirmation: %s", err)
+				continue
 			}
-			h.log.Errorf("failed to received remote offer confirmation: %s", err)
-			continue
-		}
-
-		h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort)
-		for _, listener := range h.onNewOfferListeners {
-			go listener(remoteOfferAnswer)
+			for _, listener := range h.onNewOfferListeners {
+				listener(&remoteOfferAnswer)
+			}
+			h.log.Infof("received offer, running version %s, remote WireGuard listen port %d, session id: %s", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString())
+		case remoteOfferAnswer := <-h.remoteAnswerCh:
+			h.log.Infof("received answer, running version %s, remote WireGuard listen port %d, session id: %s", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString())
+			for _, listener := range h.onNewOfferListeners {
+				listener(&remoteOfferAnswer)
+			}
+		case <-ctx.Done():
+			h.log.Infof("stop listening for remote offers and answers")
+			return
 		}
 	}
 }
@@ -103,44 +114,27 @@ func (h *Handshaker) SendOffer() error {
 
 // OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
 // doesn't block, discards the message if connection wasn't ready
-func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool {
+func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) {
 	select {
 	case h.remoteOffersCh <- offer:
-		return true
+		return
 	default:
-		h.log.Debugf("OnRemoteOffer skipping message because is not ready")
+		h.log.Warnf("skipping remote offer message because receiver not ready")
 		// connection might not be ready yet to receive so we ignore the message
-		return false
+		return
 	}
 }
 
 // OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
 // doesn't block, discards the message if connection wasn't ready
-func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool {
+func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) {
 	select {
 	case h.remoteAnswerCh <- answer:
-		return true
+		return
 	default:
 		// connection might not be ready yet to receive so we ignore the message
-		h.log.Debugf("OnRemoteAnswer skipping message because is not ready")
-		return false
-	}
-}
-
-func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) {
-	select {
-	case remoteOfferAnswer := <-h.remoteOffersCh:
-		// received confirmation from the remote peer -> ready to proceed
-		err := h.sendAnswer()
-		if err != nil {
-			return nil, err
-		}
-		return &remoteOfferAnswer, nil
-	case remoteOfferAnswer := <-h.remoteAnswerCh:
-		return &remoteOfferAnswer, nil
-	case <-h.ctx.Done():
-		// closed externally
-		return nil, NewConnectionClosedError(h.config.Key)
+		h.log.Warnf("skipping remote answer message because receiver not ready")
+		return
 	}
 }
 
@@ -150,43 +144,34 @@ func (h *Handshaker) sendOffer() error {
 		return ErrSignalIsNotReady
 	}
 
-	iceUFrag, icePwd := h.ice.GetLocalUserCredentials()
-	offer := OfferAnswer{
-		IceCredentials:  IceCredentials{iceUFrag, icePwd},
-		WgListenPort:    h.config.LocalWgPort,
-		Version:         version.NetbirdVersion(),
-		RosenpassPubKey: h.config.RosenpassPubKey,
-		RosenpassAddr:   h.config.RosenpassAddr,
-	}
-
-	addr, err := h.relay.RelayInstanceAddress()
-	if err == nil {
-		offer.RelaySrvAddress = addr
-	}
+	offer := h.buildOfferAnswer()
+	h.log.Infof("sending offer with serial: %s", offer.SessionIDString())
 
 	return h.signaler.SignalOffer(offer, h.config.Key)
 }
 
 func (h *Handshaker) sendAnswer() error {
-	h.log.Debugf("sending answer")
-	uFrag, pwd := h.ice.GetLocalUserCredentials()
+	answer := h.buildOfferAnswer()
+	h.log.Infof("sending answer with serial: %s", answer.SessionIDString())
 
+	return h.signaler.SignalAnswer(answer, h.config.Key)
+}
+
+func (h *Handshaker) buildOfferAnswer() OfferAnswer {
+	uFrag, pwd := h.ice.GetLocalUserCredentials()
+	sid := h.ice.SessionID()
 	answer := OfferAnswer{
 		IceCredentials:  IceCredentials{uFrag, pwd},
 		WgListenPort:    h.config.LocalWgPort,
 		Version:         version.NetbirdVersion(),
-		RosenpassPubKey: h.config.RosenpassPubKey,
-		RosenpassAddr:   h.config.RosenpassAddr,
+		RosenpassPubKey: h.config.RosenpassConfig.PubKey,
+		RosenpassAddr:   h.config.RosenpassConfig.Addr,
+		SessionID:       &sid,
 	}
-	addr, err := h.relay.RelayInstanceAddress()
-	if err == nil {
+
+	if addr, err := h.relay.RelayInstanceAddress(); err == nil {
 		answer.RelaySrvAddress = addr
 	}
 
-	err = h.signaler.SignalAnswer(answer, h.config.Key)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return answer
 }

client/internal/peer/ice/StunTurn.go

@@ -3,7 +3,7 @@ package ice
 import (
 	"sync/atomic"
 
-	"github.com/pion/stun/v2"
+	"github.com/pion/stun/v3"
 )
 
 type StunTurn atomic.Value

client/internal/peer/ice/agent.go

@@ -1,9 +1,11 @@
 package ice
 
 import (
+	"sync"
 	"time"
 
-	"github.com/pion/ice/v3"
+	"github.com/pion/ice/v4"
+	"github.com/pion/logging"
 	"github.com/pion/randutil"
 	log "github.com/sirupsen/logrus"
 
@@ -17,17 +19,28 @@ const (
 
 	iceKeepAliveDefault           = 4 * time.Second
 	iceDisconnectedTimeoutDefault = 6 * time.Second
+	iceFailedTimeoutDefault       = 6 * time.Second
 	// iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package
 	iceRelayAcceptanceMinWaitDefault = 2 * time.Second
 )
 
-var (
-	failedTimeout = 6 * time.Second
-)
+type ThreadSafeAgent struct {
+	*ice.Agent
+	once sync.Once
+}
 
-func NewAgent(iFaceDiscover stdnet.ExternalIFaceDiscover, config Config, candidateTypes []ice.CandidateType, ufrag string, pwd string) (*ice.Agent, error) {
+func (a *ThreadSafeAgent) Close() error {
+	var err error
+	a.once.Do(func() {
+		err = a.Agent.Close()
+	})
+	return err
+}
+
+func NewAgent(iFaceDiscover stdnet.ExternalIFaceDiscover, config Config, candidateTypes []ice.CandidateType, ufrag string, pwd string) (*ThreadSafeAgent, error) {
 	iceKeepAlive := iceKeepAlive()
 	iceDisconnectedTimeout := iceDisconnectedTimeout()
+	iceFailedTimeout := iceFailedTimeout()
 	iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait()
 
 	transportNet, err := newStdNet(iFaceDiscover, config.InterfaceBlackList)
@@ -35,6 +48,10 @@ func NewAgent(iFaceDiscover stdnet.ExternalIFaceDiscover, config Config, candida
 		log.Errorf("failed to create pion's stdnet: %s", err)
 	}
 
+	fac := logging.NewDefaultLoggerFactory()
+
+	//fac.Writer = log.StandardLogger().Writer()
+
 	agentConfig := &ice.AgentConfig{
 		MulticastDNSMode:       ice.MulticastDNSModeDisabled,
 		NetworkTypes:           []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6},
@@ -45,19 +62,25 @@ func NewAgent(iFaceDiscover stdnet.ExternalIFaceDiscover, config Config, candida
 		UDPMuxSrflx:            config.UDPMuxSrflx,
 		NAT1To1IPs:             config.NATExternalIPs,
 		Net:                    transportNet,
-		FailedTimeout:          &failedTimeout,
+		FailedTimeout:          &iceFailedTimeout,
 		DisconnectedTimeout:    &iceDisconnectedTimeout,
 		KeepaliveInterval:      &iceKeepAlive,
 		RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait,
 		LocalUfrag:             ufrag,
 		LocalPwd:               pwd,
+		LoggerFactory:          fac,
 	}
 
 	if config.DisableIPv6Discovery {
 		agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4}
 	}
 
-	return ice.NewAgent(agentConfig)
+	agent, err := ice.NewAgent(agentConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ThreadSafeAgent{Agent: agent}, nil
 }
 
 func GenerateICECredentials() (string, string, error) {

client/internal/peer/ice/config.go

@@ -1,7 +1,7 @@
 package ice
 
 import (
-	"github.com/pion/ice/v3"
+	"github.com/pion/ice/v4"
 )
 
 type Config struct {

client/internal/peer/ice/env.go

@@ -13,6 +13,7 @@ const (
 	envICEForceRelayConn            = "NB_ICE_FORCE_RELAY_CONN"
 	envICEKeepAliveIntervalSec      = "NB_ICE_KEEP_ALIVE_INTERVAL_SEC"
 	envICEDisconnectedTimeoutSec    = "NB_ICE_DISCONNECTED_TIMEOUT_SEC"
+	envICEFailedTimeoutSec          = "NB_ICE_FAILED_TIMEOUT_SEC"
 	envICERelayAcceptanceMinWaitSec = "NB_ICE_RELAY_ACCEPTANCE_MIN_WAIT_SEC"
 
 	msgWarnInvalidValue = "invalid value %s set for %s, using default %v"
@@ -55,6 +56,22 @@ func iceDisconnectedTimeout() time.Duration {
 	return time.Duration(disconnectedTimeoutSec) * time.Second
 }
 
+func iceFailedTimeout() time.Duration {
+	failedTimeoutEnv := os.Getenv(envICEFailedTimeoutSec)
+	if failedTimeoutEnv == "" {
+		return iceFailedTimeoutDefault
+	}
+
+	log.Infof("setting ICE failed timeout to %s seconds", failedTimeoutEnv)
+	failedTimeoutSec, err := strconv.Atoi(failedTimeoutEnv)
+	if err != nil {
+		log.Warnf(msgWarnInvalidValue, failedTimeoutEnv, envICEFailedTimeoutSec, iceFailedTimeoutDefault)
+		return iceFailedTimeoutDefault
+	}
+
+	return time.Duration(failedTimeoutSec) * time.Second
+}
+
 func iceRelayAcceptanceMinWait() time.Duration {
 	iceRelayAcceptanceMinWaitEnv := os.Getenv(envICERelayAcceptanceMinWaitSec)
 	if iceRelayAcceptanceMinWaitEnv == "" {

client/internal/peer/id/connid.go

@@ -0,0 +1,5 @@
+package id
+
+import "unsafe"
+
+type ConnID unsafe.Pointer

client/internal/peer/iface.go

@@ -2,16 +2,20 @@ package peer
 
 import (
 	"net"
+	"net/netip"
 	"time"
 
-	"github.com/netbirdio/netbird/client/iface/configurer"
-	"github.com/netbirdio/netbird/client/iface/wgproxy"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	"github.com/netbirdio/netbird/client/iface/configurer"
+	"github.com/netbirdio/netbird/client/iface/wgaddr"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
 )
 
 type WGIface interface {
-	UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
+	UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
 	RemovePeer(peerKey string) error
-	GetStats(peerKey string) (configurer.WGStats, error)
+	GetStats() (map[string]configurer.WGStats, error)
 	GetProxy() wgproxy.Proxy
+	Address() wgaddr.Address
 }

client/internal/peer/notifier.go

@@ -18,6 +18,8 @@ type notifier struct {
 	currentClientState bool
 	lastNotification   int
 	lastNumberOfPeers  int
+	lastFqdnAddress    string
+	lastIPAddress      string
 }
 
 func newNotifier() *notifier {
@@ -25,15 +27,22 @@ func newNotifier() *notifier {
 }
 
 func (n *notifier) setListener(listener Listener) {
+	n.serverStateLock.Lock()
+	lastNotification := n.lastNotification
+	numOfPeers := n.lastNumberOfPeers
+	fqdnAddress := n.lastFqdnAddress
+	address := n.lastIPAddress
+	n.serverStateLock.Unlock()
+
 	n.listenersLock.Lock()
 	defer n.listenersLock.Unlock()
 
-	n.serverStateLock.Lock()
-	n.notifyListener(listener, n.lastNotification)
-	listener.OnPeersListChanged(n.lastNumberOfPeers)
-	n.serverStateLock.Unlock()
-
 	n.listener = listener
+
+	listener.OnAddressChanged(fqdnAddress, address)
+	notifyListener(listener, lastNotification)
+	// run on go routine to avoid on Java layer to call go functions on same thread
+	go listener.OnPeersListChanged(numOfPeers)
 }
 
 func (n *notifier) removeListener() {
@@ -44,41 +53,44 @@ func (n *notifier) removeListener() {
 
 func (n *notifier) updateServerStates(mgmState bool, signalState bool) {
 	n.serverStateLock.Lock()
-	defer n.serverStateLock.Unlock()
-
 	calculatedState := n.calculateState(mgmState, signalState)
 
 	if !n.isServerStateChanged(calculatedState) {
+		n.serverStateLock.Unlock()
 		return
 	}
 
 	n.lastNotification = calculatedState
+	n.serverStateLock.Unlock()
 
-	n.notify(n.lastNotification)
+	n.notify(calculatedState)
 }
 
 func (n *notifier) clientStart() {
 	n.serverStateLock.Lock()
-	defer n.serverStateLock.Unlock()
 	n.currentClientState = true
 	n.lastNotification = stateConnecting
-	n.notify(n.lastNotification)
+	n.serverStateLock.Unlock()
+
+	n.notify(stateConnecting)
 }
 
 func (n *notifier) clientStop() {
 	n.serverStateLock.Lock()
-	defer n.serverStateLock.Unlock()
 	n.currentClientState = false
 	n.lastNotification = stateDisconnected
-	n.notify(n.lastNotification)
+	n.serverStateLock.Unlock()
+
+	n.notify(stateDisconnected)
 }
 
 func (n *notifier) clientTearDown() {
 	n.serverStateLock.Lock()
-	defer n.serverStateLock.Unlock()
 	n.currentClientState = false
 	n.lastNotification = stateDisconnecting
-	n.notify(n.lastNotification)
+	n.serverStateLock.Unlock()
+
+	n.notify(stateDisconnecting)
 }
 
 func (n *notifier) isServerStateChanged(newState int) bool {
@@ -87,26 +99,14 @@ func (n *notifier) isServerStateChanged(newState int) bool {
 
 func (n *notifier) notify(state int) {
 	n.listenersLock.Lock()
-	defer n.listenersLock.Unlock()
-	if n.listener == nil {
+	listener := n.listener
+	n.listenersLock.Unlock()
+
+	if listener == nil {
 		return
 	}
-	n.notifyListener(n.listener, state)
-}
 
-func (n *notifier) notifyListener(l Listener, state int) {
-	go func() {
-		switch state {
-		case stateDisconnected:
-			l.OnDisconnected()
-		case stateConnected:
-			l.OnConnected()
-		case stateConnecting:
-			l.OnConnecting()
-		case stateDisconnecting:
-			l.OnDisconnecting()
-		}
-	}()
+	notifyListener(listener, state)
 }
 
 func (n *notifier) calculateState(managementConn, signalConn bool) int {
@@ -126,20 +126,48 @@ func (n *notifier) calculateState(managementConn, signalConn bool) int {
 }
 
 func (n *notifier) peerListChanged(numOfPeers int) {
+	n.serverStateLock.Lock()
 	n.lastNumberOfPeers = numOfPeers
+	n.serverStateLock.Unlock()
+
 	n.listenersLock.Lock()
-	defer n.listenersLock.Unlock()
-	if n.listener == nil {
+	listener := n.listener
+	n.listenersLock.Unlock()
+
+	if listener == nil {
 		return
 	}
-	n.listener.OnPeersListChanged(numOfPeers)
+
+	// run on go routine to avoid on Java layer to call go functions on same thread
+	go listener.OnPeersListChanged(numOfPeers)
 }
 
 func (n *notifier) localAddressChanged(fqdn, address string) {
+	n.serverStateLock.Lock()
+	n.lastFqdnAddress = fqdn
+	n.lastIPAddress = address
+	n.serverStateLock.Unlock()
+
 	n.listenersLock.Lock()
-	defer n.listenersLock.Unlock()
-	if n.listener == nil {
+	listener := n.listener
+	n.listenersLock.Unlock()
+
+	if listener == nil {
 		return
 	}
-	n.listener.OnAddressChanged(fqdn, address)
+
+	listener.OnAddressChanged(fqdn, address)
+}
+
+func notifyListener(l Listener, state int) {
+	switch state {
+	case stateDisconnected:
+		l.OnDisconnected()
+	case stateConnected:
+		l.OnConnected()
+	case stateConnecting:
+		l.OnConnecting()
+	case stateDisconnecting:
+		l.OnDisconnecting()
+	}
 }

client/internal/peer/route.go

@@ -0,0 +1,133 @@
+package peer
+
+import (
+	"net/netip"
+	"sort"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/route"
+)
+
+// routeEntry holds the route prefix and the corresponding resource ID.
+type routeEntry struct {
+	prefix     netip.Prefix
+	resourceID route.ResID
+}
+
+type routeIDLookup struct {
+	localRoutes []routeEntry
+	localLock   sync.RWMutex
+
+	remoteRoutes []routeEntry
+	remoteLock   sync.RWMutex
+
+	resolvedIPs sync.Map
+}
+
+func (r *routeIDLookup) AddLocalRouteID(resourceID route.ResID, route netip.Prefix) {
+	r.localLock.Lock()
+	defer r.localLock.Unlock()
+
+	// update the resource id if the route already exists.
+	for i, entry := range r.localRoutes {
+		if entry.prefix == route {
+			r.localRoutes[i].resourceID = resourceID
+			log.Tracef("resourceID for route %v updated to %s in local routes", route, resourceID)
+			return
+		}
+	}
+
+	// append and sort descending by prefix bits (more specific first)
+	r.localRoutes = append(r.localRoutes, routeEntry{prefix: route, resourceID: resourceID})
+	sort.Slice(r.localRoutes, func(i, j int) bool {
+		return r.localRoutes[i].prefix.Bits() > r.localRoutes[j].prefix.Bits()
+	})
+}
+
+func (r *routeIDLookup) RemoveLocalRouteID(route netip.Prefix) {
+	r.localLock.Lock()
+	defer r.localLock.Unlock()
+
+	for i, entry := range r.localRoutes {
+		if entry.prefix == route {
+			r.localRoutes = append(r.localRoutes[:i], r.localRoutes[i+1:]...)
+			return
+		}
+	}
+}
+
+func (r *routeIDLookup) AddRemoteRouteID(resourceID route.ResID, route netip.Prefix) {
+	r.remoteLock.Lock()
+	defer r.remoteLock.Unlock()
+
+	for i, entry := range r.remoteRoutes {
+		if entry.prefix == route {
+			r.remoteRoutes[i].resourceID = resourceID
+			log.Tracef("resourceID for route %v updated to %s in remote routes", route, resourceID)
+			return
+		}
+	}
+
+	// append and sort descending by prefix bits.
+	r.remoteRoutes = append(r.remoteRoutes, routeEntry{prefix: route, resourceID: resourceID})
+	sort.Slice(r.remoteRoutes, func(i, j int) bool {
+		return r.remoteRoutes[i].prefix.Bits() > r.remoteRoutes[j].prefix.Bits()
+	})
+}
+
+func (r *routeIDLookup) RemoveRemoteRouteID(route netip.Prefix) {
+	r.remoteLock.Lock()
+	defer r.remoteLock.Unlock()
+
+	for i, entry := range r.remoteRoutes {
+		if entry.prefix == route {
+			r.remoteRoutes = append(r.remoteRoutes[:i], r.remoteRoutes[i+1:]...)
+			return
+		}
+	}
+}
+
+func (r *routeIDLookup) AddResolvedIP(resourceID route.ResID, route netip.Prefix) {
+	r.resolvedIPs.Store(route.Addr(), resourceID)
+}
+
+func (r *routeIDLookup) RemoveResolvedIP(route netip.Prefix) {
+	r.resolvedIPs.Delete(route.Addr())
+}
+
+// Lookup returns the resource ID for the given IP address
+// and a bool indicating if the IP is an exit node.
+func (r *routeIDLookup) Lookup(ip netip.Addr) (route.ResID, bool) {
+	if res, ok := r.resolvedIPs.Load(ip); ok {
+		return res.(route.ResID), false
+	}
+
+	var resourceID route.ResID
+	var isExitNode bool
+
+	r.localLock.RLock()
+	for _, entry := range r.localRoutes {
+		if entry.prefix.Contains(ip) {
+			resourceID = entry.resourceID
+			isExitNode = entry.prefix.Bits() == 0
+			break
+		}
+	}
+	r.localLock.RUnlock()
+
+	if resourceID == "" {
+		r.remoteLock.RLock()
+		for _, entry := range r.remoteRoutes {
+			if entry.prefix.Contains(ip) {
+				resourceID = entry.resourceID
+				isExitNode = entry.prefix.Bits() == 0
+				break
+			}
+		}
+		r.remoteLock.RUnlock()
+	}
+
+	return resourceID, isExitNode
+}

client/internal/peer/session_id.go

@@ -0,0 +1,47 @@
+package peer
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+)
+
+const sessionIDSize = 5
+
+type ICESessionID string
+
+// NewICESessionID generates a new session ID for distinguishing sessions
+func NewICESessionID() (ICESessionID, error) {
+	b := make([]byte, sessionIDSize)
+	if _, err := io.ReadFull(rand.Reader, b); err != nil {
+		return "", fmt.Errorf("failed to generate session ID: %w", err)
+	}
+	return ICESessionID(hex.EncodeToString(b)), nil
+}
+
+func ICESessionIDFromBytes(b []byte) (ICESessionID, error) {
+	if len(b) != sessionIDSize {
+		return "", fmt.Errorf("invalid session ID length: %d", len(b))
+	}
+	return ICESessionID(hex.EncodeToString(b)), nil
+}
+
+// Bytes returns the raw bytes of the session ID for protobuf serialization
+func (id ICESessionID) Bytes() ([]byte, error) {
+	if len(id) == 0 {
+		return nil, fmt.Errorf("ICE session ID is empty")
+	}
+	b, err := hex.DecodeString(string(id))
+	if err != nil {
+		return nil, fmt.Errorf("invalid ICE session ID encoding: %w", err)
+	}
+	if len(b) != sessionIDSize {
+		return nil, fmt.Errorf("invalid ICE session ID length: expected %d bytes, got %d", sessionIDSize, len(b))
+	}
+	return b, nil
+}
+
+func (id ICESessionID) String() string {
+	return string(id)
+}

client/internal/peer/signaler.go

@@ -1,11 +1,12 @@
 package peer
 
 import (
-	"github.com/pion/ice/v3"
+	"github.com/pion/ice/v4"
+	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
-	signal "github.com/netbirdio/netbird/signal/client"
-	sProto "github.com/netbirdio/netbird/signal/proto"
+	signal "github.com/netbirdio/netbird/shared/signal/client"
+	sProto "github.com/netbirdio/netbird/shared/signal/proto"
 )
 
 type Signaler struct {
@@ -45,6 +46,10 @@ func (s *Signaler) Ready() bool {
 
 // SignalOfferAnswer signals either an offer or an answer to remote peer
 func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, bodyType sProto.Body_Type) error {
+	sessionIDBytes, err := offerAnswer.SessionID.Bytes()
+	if err != nil {
+		log.Warnf("failed to get session ID bytes: %v", err)
+	}
 	msg, err := signal.MarshalCredential(
 		s.wgPrivateKey,
 		offerAnswer.WgListenPort,
@@ -56,15 +61,25 @@ func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string,
 		bodyType,
 		offerAnswer.RosenpassPubKey,
 		offerAnswer.RosenpassAddr,
-		offerAnswer.RelaySrvAddress)
+		offerAnswer.RelaySrvAddress,
+		sessionIDBytes)
 	if err != nil {
 		return err
 	}
 
-	err = s.signal.Send(msg)
-	if err != nil {
+	if err = s.signal.Send(msg); err != nil {
 		return err
 	}
 
 	return nil
 }
+
+func (s *Signaler) SignalIdle(remoteKey string) error {
+	return s.signal.Send(&sProto.Message{
+		Key:       s.wgPrivateKey.PublicKey().String(),
+		RemoteKey: remoteKey,
+		Body: &sProto.Body{
+			Type: sProto.Body_GO_IDLE,
+		},
+	})
+}

client/internal/peer/state_dump.go

@@ -0,0 +1,122 @@
+package peer
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type stateDump struct {
+	log    *log.Entry
+	status *Status
+	key    string
+
+	sentOffer       int
+	remoteOffer     int
+	remoteAnswer    int
+	remoteCandidate int
+	p2pConnected    int
+	switchToRelay   int
+	wgCheckSuccess  int
+	relayConnected  int
+	localProxies    int
+
+	mu sync.Mutex
+}
+
+func newStateDump(key string, log *log.Entry, statusRecorder *Status) *stateDump {
+	return &stateDump{
+		log:    log,
+		status: statusRecorder,
+		key:    key,
+	}
+}
+
+func (s *stateDump) Start(ctx context.Context) {
+	ticker := time.NewTicker(10 * time.Minute)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			s.dumpState()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (s *stateDump) RemoteOffer() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.remoteOffer++
+}
+
+func (s *stateDump) RemoteCandidate() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.remoteCandidate++
+}
+
+func (s *stateDump) SendOffer() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.sentOffer++
+}
+
+func (s *stateDump) dumpState() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	status := "unknown"
+	state, e := s.status.GetPeer(s.key)
+	if e == nil {
+		status = state.ConnStatus.String()
+	}
+
+	s.log.Infof("Dump stat: Status: %s, SentOffer: %d, RemoteOffer: %d, RemoteAnswer: %d, RemoteCandidate: %d, P2PConnected: %d, SwitchToRelay: %d, WGCheckSuccess: %d, RelayConnected: %d, LocalProxies: %d",
+		status, s.sentOffer, s.remoteOffer, s.remoteAnswer, s.remoteCandidate, s.p2pConnected, s.switchToRelay, s.wgCheckSuccess, s.relayConnected, s.localProxies)
+}
+
+func (s *stateDump) RemoteAnswer() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.remoteAnswer++
+}
+
+func (s *stateDump) P2PConnected() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.p2pConnected++
+}
+
+func (s *stateDump) SwitchToRelay() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.switchToRelay++
+}
+
+func (s *stateDump) WGcheckSuccess() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.wgCheckSuccess++
+}
+
+func (s *stateDump) RelayConnected() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.relayConnected++
+}
+
+func (s *stateDump) NewLocalProxy() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.localProxies++
+}

client/internal/peer/status.go

@@ -1,7 +1,9 @@
 package peer
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"net/netip"
 	"slices"
 	"sync"
@@ -14,11 +16,14 @@ import (
 	gstatus "google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/iface/configurer"
+	"github.com/netbirdio/netbird/client/internal/ingressgw"
 	"github.com/netbirdio/netbird/client/internal/relay"
 	"github.com/netbirdio/netbird/client/proto"
-	"github.com/netbirdio/netbird/management/domain"
-	relayClient "github.com/netbirdio/netbird/relay/client"
+	"github.com/netbirdio/netbird/shared/management/domain"
+	relayClient "github.com/netbirdio/netbird/shared/relay/client"
+	"github.com/netbirdio/netbird/route"
 )
 
 const eventQueueSize = 10
@@ -28,10 +33,21 @@ type ResolvedDomainInfo struct {
 	ParentDomain domain.Domain
 }
 
+type WGIfaceStatus interface {
+	FullStats() (*configurer.Stats, error)
+}
+
 type EventListener interface {
 	OnEvent(event *proto.SystemEvent)
 }
 
+// RouterState status for router peers. This contains relevant fields for route manager
+type RouterState struct {
+	Status  ConnStatus
+	Relayed bool
+	Latency time.Duration
+}
+
 // State contains the latest state of a peer
 type State struct {
 	Mux                        *sync.RWMutex
@@ -124,7 +140,7 @@ type RosenpassState struct {
 // whether it's enabled, and the last error message encountered during probing.
 type NSGroupState struct {
 	ID      string
-	Servers []string
+	Servers []netip.AddrPort
 	Domains []string
 	Enabled bool
 	Error   error
@@ -132,20 +148,43 @@ type NSGroupState struct {
 
 // FullStatus contains the full state held by the Status instance
 type FullStatus struct {
-	Peers           []State
-	ManagementState ManagementState
-	SignalState     SignalState
-	LocalPeerState  LocalPeerState
-	RosenpassState  RosenpassState
-	Relays          []relay.ProbeResult
-	NSGroupStates   []NSGroupState
+	Peers                 []State
+	ManagementState       ManagementState
+	SignalState           SignalState
+	LocalPeerState        LocalPeerState
+	RosenpassState        RosenpassState
+	Relays                []relay.ProbeResult
+	NSGroupStates         []NSGroupState
+	NumOfForwardingRules  int
+	LazyConnectionEnabled bool
+}
+
+type StatusChangeSubscription struct {
+	peerID     string
+	id         string
+	eventsChan chan map[string]RouterState
+	ctx        context.Context
+}
+
+func newStatusChangeSubscription(ctx context.Context, peerID string) *StatusChangeSubscription {
+	return &StatusChangeSubscription{
+		ctx:    ctx,
+		peerID: peerID,
+		id:     uuid.New().String(),
+		// it is a buffer for notifications to block less the status recorded
+		eventsChan: make(chan map[string]RouterState, 8),
+	}
+}
+
+func (s *StatusChangeSubscription) Events() chan map[string]RouterState {
+	return s.eventsChan
 }
 
 // Status holds a state of peers, signal, management connections and relays
 type Status struct {
 	mux                   sync.Mutex
 	peers                 map[string]State
-	changeNotify          map[string]chan struct{}
+	changeNotify          map[string]map[string]*StatusChangeSubscription // map[peerID]map[subscriptionID]*StatusChangeSubscription
 	signalState           bool
 	signalError           error
 	managementState       bool
@@ -160,6 +199,7 @@ type Status struct {
 	rosenpassPermissive   bool
 	nsGroupStates         []NSGroupState
 	resolvedDomainsStates map[domain.Domain]ResolvedDomainInfo
+	lazyConnectionEnabled bool
 
 	// To reduce the number of notification invocation this bool will be true when need to call the notification
 	// Some Peer actions mostly used by in a batch when the network map has been synchronized. In these type of events
@@ -171,13 +211,18 @@ type Status struct {
 	eventMux     sync.RWMutex
 	eventStreams map[string]chan *proto.SystemEvent
 	eventQueue   *EventQueue
+
+	ingressGwMgr *ingressgw.Manager
+
+	routeIDLookup routeIDLookup
+	wgIface       WGIfaceStatus
 }
 
 // NewRecorder returns a new Status instance
 func NewRecorder(mgmAddress string) *Status {
 	return &Status{
 		peers:                 make(map[string]State),
-		changeNotify:          make(map[string]chan struct{}),
+		changeNotify:          make(map[string]map[string]*StatusChangeSubscription),
 		eventStreams:          make(map[string]chan *proto.SystemEvent),
 		eventQueue:            NewEventQueue(eventQueueSize),
 		offlinePeers:          make([]State, 0),
@@ -193,6 +238,12 @@ func (d *Status) SetRelayMgr(manager *relayClient.Manager) {
 	d.relayMgr = manager
 }
 
+func (d *Status) SetIngressGwMgr(ingressGwMgr *ingressgw.Manager) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	d.ingressGwMgr = ingressGwMgr
+}
+
 // ReplaceOfflinePeers replaces
 func (d *Status) ReplaceOfflinePeers(replacement []State) {
 	d.mux.Lock()
@@ -205,7 +256,7 @@ func (d *Status) ReplaceOfflinePeers(replacement []State) {
 }
 
 // AddPeer adds peer to Daemon status map
-func (d *Status) AddPeer(peerPubKey string, fqdn string) error {
+func (d *Status) AddPeer(peerPubKey string, fqdn string, ip string) error {
 	d.mux.Lock()
 	defer d.mux.Unlock()
 
@@ -215,7 +266,8 @@ func (d *Status) AddPeer(peerPubKey string, fqdn string) error {
 	}
 	d.peers[peerPubKey] = State{
 		PubKey:     peerPubKey,
-		ConnStatus: StatusDisconnected,
+		IP:         ip,
+		ConnStatus: StatusIdle,
 		FQDN:       fqdn,
 		Mux:        new(sync.RWMutex),
 	}
@@ -235,6 +287,18 @@ func (d *Status) GetPeer(peerPubKey string) (State, error) {
 	return state, nil
 }
 
+func (d *Status) PeerByIP(ip string) (string, bool) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	for _, state := range d.peers {
+		if state.IP == ip {
+			return state.FQDN, true
+		}
+	}
+	return "", false
+}
+
 // RemovePeer removes peer from Daemon status map
 func (d *Status) RemovePeer(peerPubKey string) error {
 	d.mux.Lock()
@@ -260,11 +324,7 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 		return errors.New("peer doesn't exist")
 	}
 
-	if receivedState.IP != "" {
-		peerState.IP = receivedState.IP
-	}
-
-	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+	oldState := peerState.ConnStatus
 
 	if receivedState.ConnStatus != peerState.ConnStatus {
 		peerState.ConnStatus = receivedState.ConnStatus
@@ -280,15 +340,18 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 
 	d.peers[receivedState.PubKey] = peerState
 
-	if skipNotification {
-		return nil
+	if hasConnStatusChanged(oldState, receivedState.ConnStatus) {
+		d.notifyPeerListChanged()
 	}
 
-	d.notifyPeerListChanged()
+	// when we close the connection we will not notify the router manager
+	if receivedState.ConnStatus == StatusIdle {
+		d.notifyPeerStateChangeListeners(receivedState.PubKey)
+	}
 	return nil
 }
 
-func (d *Status) AddPeerStateRoute(peer string, route string) error {
+func (d *Status) AddPeerStateRoute(peer string, route string, resourceId route.ResID) error {
 	d.mux.Lock()
 	defer d.mux.Unlock()
 
@@ -300,6 +363,11 @@ func (d *Status) AddPeerStateRoute(peer string, route string) error {
 	peerState.AddRoute(route)
 	d.peers[peer] = peerState
 
+	pref, err := netip.ParsePrefix(route)
+	if err == nil {
+		d.routeIDLookup.AddRemoteRouteID(resourceId, pref)
+	}
+
 	// todo: consider to make sense of this notification or not
 	d.notifyPeerListChanged()
 	return nil
@@ -317,11 +385,26 @@ func (d *Status) RemovePeerStateRoute(peer string, route string) error {
 	peerState.DeleteRoute(route)
 	d.peers[peer] = peerState
 
+	pref, err := netip.ParsePrefix(route)
+	if err == nil {
+		d.routeIDLookup.RemoveRemoteRouteID(pref)
+	}
+
 	// todo: consider to make sense of this notification or not
 	d.notifyPeerListChanged()
 	return nil
 }
 
+// CheckRoutes checks if the source and destination addresses are within the same route
+// and returns the resource ID of the route that contains the addresses
+func (d *Status) CheckRoutes(ip netip.Addr) ([]byte, bool) {
+	if d == nil {
+		return nil, false
+	}
+	resId, isExitNode := d.routeIDLookup.Lookup(ip)
+	return []byte(resId), isExitNode
+}
+
 func (d *Status) UpdatePeerICEState(receivedState State) error {
 	d.mux.Lock()
 	defer d.mux.Unlock()
@@ -331,11 +414,8 @@ func (d *Status) UpdatePeerICEState(receivedState State) error {
 		return errors.New("peer doesn't exist")
 	}
 
-	if receivedState.IP != "" {
-		peerState.IP = receivedState.IP
-	}
-
-	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+	oldState := peerState.ConnStatus
+	oldIsRelayed := peerState.Relayed
 
 	peerState.ConnStatus = receivedState.ConnStatus
 	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
@@ -348,12 +428,13 @@ func (d *Status) UpdatePeerICEState(receivedState State) error {
 
 	d.peers[receivedState.PubKey] = peerState
 
-	if skipNotification {
-		return nil
+	if hasConnStatusChanged(oldState, receivedState.ConnStatus) {
+		d.notifyPeerListChanged()
 	}
 
-	d.notifyPeerStateChangeListeners(receivedState.PubKey)
-	d.notifyPeerListChanged()
+	if hasStatusOrRelayedChange(oldState, receivedState.ConnStatus, oldIsRelayed, receivedState.Relayed) {
+		d.notifyPeerStateChangeListeners(receivedState.PubKey)
+	}
 	return nil
 }
 
@@ -366,7 +447,8 @@ func (d *Status) UpdatePeerRelayedState(receivedState State) error {
 		return errors.New("peer doesn't exist")
 	}
 
-	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+	oldState := peerState.ConnStatus
+	oldIsRelayed := peerState.Relayed
 
 	peerState.ConnStatus = receivedState.ConnStatus
 	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
@@ -376,12 +458,13 @@ func (d *Status) UpdatePeerRelayedState(receivedState State) error {
 
 	d.peers[receivedState.PubKey] = peerState
 
-	if skipNotification {
-		return nil
+	if hasConnStatusChanged(oldState, receivedState.ConnStatus) {
+		d.notifyPeerListChanged()
 	}
 
-	d.notifyPeerStateChangeListeners(receivedState.PubKey)
-	d.notifyPeerListChanged()
+	if hasStatusOrRelayedChange(oldState, receivedState.ConnStatus, oldIsRelayed, receivedState.Relayed) {
+		d.notifyPeerStateChangeListeners(receivedState.PubKey)
+	}
 	return nil
 }
 
@@ -394,7 +477,8 @@ func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error
 		return errors.New("peer doesn't exist")
 	}
 
-	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+	oldState := peerState.ConnStatus
+	oldIsRelayed := peerState.Relayed
 
 	peerState.ConnStatus = receivedState.ConnStatus
 	peerState.Relayed = receivedState.Relayed
@@ -403,12 +487,13 @@ func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error
 
 	d.peers[receivedState.PubKey] = peerState
 
-	if skipNotification {
-		return nil
+	if hasConnStatusChanged(oldState, receivedState.ConnStatus) {
+		d.notifyPeerListChanged()
 	}
 
-	d.notifyPeerStateChangeListeners(receivedState.PubKey)
-	d.notifyPeerListChanged()
+	if hasStatusOrRelayedChange(oldState, receivedState.ConnStatus, oldIsRelayed, receivedState.Relayed) {
+		d.notifyPeerStateChangeListeners(receivedState.PubKey)
+	}
 	return nil
 }
 
@@ -421,7 +506,8 @@ func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error {
 		return errors.New("peer doesn't exist")
 	}
 
-	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+	oldState := peerState.ConnStatus
+	oldIsRelayed := peerState.Relayed
 
 	peerState.ConnStatus = receivedState.ConnStatus
 	peerState.Relayed = receivedState.Relayed
@@ -433,12 +519,13 @@ func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error {
 
 	d.peers[receivedState.PubKey] = peerState
 
-	if skipNotification {
-		return nil
+	if hasConnStatusChanged(oldState, receivedState.ConnStatus) {
+		d.notifyPeerListChanged()
 	}
 
-	d.notifyPeerStateChangeListeners(receivedState.PubKey)
-	d.notifyPeerListChanged()
+	if hasStatusOrRelayedChange(oldState, receivedState.ConnStatus, oldIsRelayed, receivedState.Relayed) {
+		d.notifyPeerStateChangeListeners(receivedState.PubKey)
+	}
 	return nil
 }
 
@@ -461,17 +548,12 @@ func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats configurer.WGSt
 	return nil
 }
 
-func shouldSkipNotify(receivedConnStatus ConnStatus, curr State) bool {
-	switch {
-	case receivedConnStatus == StatusConnecting:
-		return true
-	case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting:
-		return true
-	case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected:
-		return curr.IP != ""
-	default:
-		return false
-	}
+func hasStatusOrRelayedChange(oldConnStatus, newConnStatus ConnStatus, oldRelayed, newRelayed bool) bool {
+	return oldRelayed != newRelayed || hasConnStatusChanged(newConnStatus, oldConnStatus)
+}
+
+func hasConnStatusChanged(oldStatus, newStatus ConnStatus) bool {
+	return newStatus != oldStatus
 }
 
 // UpdatePeerFQDN update peer's state fqdn only
@@ -493,30 +575,55 @@ func (d *Status) UpdatePeerFQDN(peerPubKey, fqdn string) error {
 // FinishPeerListModifications this event invoke the notification
 func (d *Status) FinishPeerListModifications() {
 	d.mux.Lock()
+	defer d.mux.Unlock()
 
 	if !d.peerListChangedForNotification {
-		d.mux.Unlock()
 		return
 	}
 	d.peerListChangedForNotification = false
-	d.mux.Unlock()
 
 	d.notifyPeerListChanged()
+
+	for key := range d.peers {
+		d.notifyPeerStateChangeListeners(key)
+	}
 }
 
-// GetPeerStateChangeNotifier returns a change notifier channel for a peer
-func (d *Status) GetPeerStateChangeNotifier(peer string) <-chan struct{} {
+func (d *Status) SubscribeToPeerStateChanges(ctx context.Context, peerID string) *StatusChangeSubscription {
 	d.mux.Lock()
 	defer d.mux.Unlock()
 
-	ch, found := d.changeNotify[peer]
-	if found {
-		return ch
+	sub := newStatusChangeSubscription(ctx, peerID)
+	if _, ok := d.changeNotify[peerID]; !ok {
+		d.changeNotify[peerID] = make(map[string]*StatusChangeSubscription)
+	}
+	d.changeNotify[peerID][sub.id] = sub
+
+	return sub
+}
+
+func (d *Status) UnsubscribePeerStateChanges(subscription *StatusChangeSubscription) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	if subscription == nil {
+		return
 	}
 
-	ch = make(chan struct{})
-	d.changeNotify[peer] = ch
-	return ch
+	channels, ok := d.changeNotify[subscription.peerID]
+	if !ok {
+		return
+	}
+
+	sub, exists := channels[subscription.id]
+	if !exists {
+		return
+	}
+
+	delete(channels, subscription.id)
+	if len(channels) == 0 {
+		delete(d.changeNotify, sub.peerID)
+	}
 }
 
 // GetLocalPeerState returns the local peer state
@@ -535,6 +642,63 @@ func (d *Status) UpdateLocalPeerState(localPeerState LocalPeerState) {
 	d.notifyAddressChanged()
 }
 
+// AddLocalPeerStateRoute adds a route to the local peer state
+func (d *Status) AddLocalPeerStateRoute(route string, resourceId route.ResID) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	pref, err := netip.ParsePrefix(route)
+	if err == nil {
+		d.routeIDLookup.AddLocalRouteID(resourceId, pref)
+	}
+
+	if d.localPeer.Routes == nil {
+		d.localPeer.Routes = map[string]struct{}{}
+	}
+
+	d.localPeer.Routes[route] = struct{}{}
+}
+
+// RemoveLocalPeerStateRoute removes a route from the local peer state
+func (d *Status) RemoveLocalPeerStateRoute(route string) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	pref, err := netip.ParsePrefix(route)
+	if err == nil {
+		d.routeIDLookup.RemoveLocalRouteID(pref)
+	}
+
+	delete(d.localPeer.Routes, route)
+}
+
+// AddResolvedIPLookupEntry adds a resolved IP lookup entry
+func (d *Status) AddResolvedIPLookupEntry(prefix netip.Prefix, resourceId route.ResID) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	d.routeIDLookup.AddResolvedIP(resourceId, prefix)
+}
+
+// RemoveResolvedIPLookupEntry removes a resolved IP lookup entry
+func (d *Status) RemoveResolvedIPLookupEntry(route string) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	pref, err := netip.ParsePrefix(route)
+	if err == nil {
+		d.routeIDLookup.RemoveResolvedIP(pref)
+	}
+}
+
+// CleanLocalPeerStateRoutes cleans all routes from the local peer state
+func (d *Status) CleanLocalPeerStateRoutes() {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	d.localPeer.Routes = map[string]struct{}{}
+}
+
 // CleanLocalPeerState cleans local peer status
 func (d *Status) CleanLocalPeerState() {
 	d.mux.Lock()
@@ -586,6 +750,12 @@ func (d *Status) UpdateRosenpass(rosenpassEnabled, rosenpassPermissive bool) {
 	d.rosenpassEnabled = rosenpassEnabled
 }
 
+func (d *Status) UpdateLazyConnection(enabled bool) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	d.lazyConnectionEnabled = enabled
+}
+
 // MarkSignalDisconnected sets SignalState to disconnected
 func (d *Status) MarkSignalDisconnected(err error) {
 	d.mux.Lock()
@@ -618,7 +788,7 @@ func (d *Status) UpdateDNSStates(dnsStates []NSGroupState) {
 	d.nsGroupStates = dnsStates
 }
 
-func (d *Status) UpdateResolvedDomainsStates(originalDomain domain.Domain, resolvedDomain domain.Domain, prefixes []netip.Prefix) {
+func (d *Status) UpdateResolvedDomainsStates(originalDomain domain.Domain, resolvedDomain domain.Domain, prefixes []netip.Prefix, resourceId route.ResID) {
 	d.mux.Lock()
 	defer d.mux.Unlock()
 
@@ -627,6 +797,10 @@ func (d *Status) UpdateResolvedDomainsStates(originalDomain domain.Domain, resol
 		Prefixes:     prefixes,
 		ParentDomain: originalDomain,
 	}
+
+	for _, prefix := range prefixes {
+		d.routeIDLookup.AddResolvedIP(resourceId, prefix)
+	}
 }
 
 func (d *Status) DeleteResolvedDomainsStates(domain domain.Domain) {
@@ -637,6 +811,10 @@ func (d *Status) DeleteResolvedDomainsStates(domain domain.Domain) {
 	for k, v := range d.resolvedDomainsStates {
 		if v.ParentDomain == domain {
 			delete(d.resolvedDomainsStates, k)
+
+			for _, prefix := range v.Prefixes {
+				d.routeIDLookup.RemoveResolvedIP(prefix)
+			}
 		}
 	}
 }
@@ -650,6 +828,12 @@ func (d *Status) GetRosenpassState() RosenpassState {
 	}
 }
 
+func (d *Status) GetLazyConnection() bool {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	return d.lazyConnectionEnabled
+}
+
 func (d *Status) GetManagementState() ManagementState {
 	d.mux.Lock()
 	defer d.mux.Unlock()
@@ -734,6 +918,16 @@ func (d *Status) GetRelayStates() []relay.ProbeResult {
 	return append(relayStates, relayState)
 }
 
+func (d *Status) ForwardingRules() []firewall.ForwardRule {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	if d.ingressGwMgr == nil {
+		return nil
+	}
+
+	return d.ingressGwMgr.Rules()
+}
+
 func (d *Status) GetDNSStates() []NSGroupState {
 	d.mux.Lock()
 	defer d.mux.Unlock()
@@ -751,11 +945,13 @@ func (d *Status) GetResolvedDomainsStates() map[domain.Domain]ResolvedDomainInfo
 // GetFullStatus gets full status
 func (d *Status) GetFullStatus() FullStatus {
 	fullStatus := FullStatus{
-		ManagementState: d.GetManagementState(),
-		SignalState:     d.GetSignalState(),
-		Relays:          d.GetRelayStates(),
-		RosenpassState:  d.GetRosenpassState(),
-		NSGroupStates:   d.GetDNSStates(),
+		ManagementState:       d.GetManagementState(),
+		SignalState:           d.GetSignalState(),
+		Relays:                d.GetRelayStates(),
+		RosenpassState:        d.GetRosenpassState(),
+		NSGroupStates:         d.GetDNSStates(),
+		NumOfForwardingRules:  len(d.ForwardingRules()),
+		LazyConnectionEnabled: d.GetLazyConnection(),
 	}
 
 	d.mux.Lock()
@@ -802,13 +998,33 @@ func (d *Status) onConnectionChanged() {
 
 // notifyPeerStateChangeListeners notifies route manager about the change in peer state
 func (d *Status) notifyPeerStateChangeListeners(peerID string) {
-	ch, found := d.changeNotify[peerID]
-	if !found {
+	subs, ok := d.changeNotify[peerID]
+	if !ok {
 		return
 	}
 
-	close(ch)
-	delete(d.changeNotify, peerID)
+	// collect the relevant data for router peers
+	routerPeers := make(map[string]RouterState, len(d.changeNotify))
+	for pid := range d.changeNotify {
+		s, ok := d.peers[pid]
+		if !ok {
+			log.Warnf("router peer not found in peers list: %s", pid)
+			continue
+		}
+
+		routerPeers[pid] = RouterState{
+			Status:  s.ConnStatus,
+			Relayed: s.Relayed,
+			Latency: s.Latency,
+		}
+	}
+
+	for _, sub := range subs {
+		select {
+		case sub.eventsChan <- routerPeers:
+		case <-sub.ctx.Done():
+		}
+	}
 }
 
 func (d *Status) notifyPeerListChanged() {
@@ -892,6 +1108,23 @@ func (d *Status) GetEventHistory() []*proto.SystemEvent {
 	return d.eventQueue.GetAll()
 }
 
+func (d *Status) SetWgIface(wgInterface WGIfaceStatus) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	d.wgIface = wgInterface
+}
+
+func (d *Status) PeersStatus() (*configurer.Stats, error) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	if d.wgIface == nil {
+		return nil, fmt.Errorf("wgInterface is nil, cannot retrieve peers status")
+	}
+
+	return d.wgIface.FullStats()
+}
+
 type EventQueue struct {
 	maxSize int
 	events  []*proto.SystemEvent

client/internal/peer/status_test.go

@@ -1,31 +1,35 @@
 package peer
 
 import (
+	"context"
 	"errors"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestAddPeer(t *testing.T) {
 	key := "abc"
+	ip := "100.108.254.1"
 	status := NewRecorder("https://mgm")
-	err := status.AddPeer(key, "abc.netbird")
+	err := status.AddPeer(key, "abc.netbird", ip)
 	assert.NoError(t, err, "shouldn't return error")
 
 	_, exists := status.peers[key]
 	assert.True(t, exists, "value was found")
 
-	err = status.AddPeer(key, "abc.netbird")
+	err = status.AddPeer(key, "abc.netbird", ip)
 
 	assert.Error(t, err, "should return error on duplicate")
 }
 
 func TestGetPeer(t *testing.T) {
 	key := "abc"
+	ip := "100.108.254.1"
 	status := NewRecorder("https://mgm")
-	err := status.AddPeer(key, "abc.netbird")
+	err := status.AddPeer(key, "abc.netbird", ip)
 	assert.NoError(t, err, "shouldn't return error")
 
 	peerStatus, err := status.GetPeer(key)
@@ -40,16 +44,16 @@ func TestGetPeer(t *testing.T) {
 func TestUpdatePeerState(t *testing.T) {
 	key := "abc"
 	ip := "10.10.10.10"
+	fqdn := "peer-a.netbird.local"
 	status := NewRecorder("https://mgm")
+	_ = status.AddPeer(key, fqdn, ip)
+
 	peerState := State{
-		PubKey: key,
-		Mux:    new(sync.RWMutex),
+		PubKey:           key,
+		ConnStatusUpdate: time.Now(),
+		ConnStatus:       StatusConnecting,
 	}
 
-	status.peers[key] = peerState
-
-	peerState.IP = ip
-
 	err := status.UpdatePeerState(peerState)
 	assert.NoError(t, err, "shouldn't return error")
 
@@ -81,25 +85,27 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) {
 	key := "abc"
 	ip := "10.10.10.10"
 	status := NewRecorder("https://mgm")
+	_ = status.AddPeer(key, "abc.netbird", ip)
+
+	sub := status.SubscribeToPeerStateChanges(context.Background(), key)
+	assert.NotNil(t, sub, "channel shouldn't be nil")
+
 	peerState := State{
-		PubKey: key,
-		Mux:    new(sync.RWMutex),
+		PubKey:           key,
+		ConnStatus:       StatusConnecting,
+		Relayed:          false,
+		ConnStatusUpdate: time.Now(),
 	}
 
-	status.peers[key] = peerState
-
-	ch := status.GetPeerStateChangeNotifier(key)
-	assert.NotNil(t, ch, "channel shouldn't be nil")
-
-	peerState.IP = ip
-
 	err := status.UpdatePeerRelayedStateToDisconnected(peerState)
 	assert.NoError(t, err, "shouldn't return error")
 
+	timeoutCtx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
 	select {
-	case <-ch:
-	default:
-		t.Errorf("channel wasn't closed after update")
+	case <-sub.eventsChan:
+	case <-timeoutCtx.Done():
+		t.Errorf("timed out waiting for event")
 	}
 }
 

client/internal/peer/wg_watcher.go

@@ -2,6 +2,7 @@ package peer
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
@@ -20,25 +21,26 @@ var (
 )
 
 type WGInterfaceStater interface {
-	GetStats(key string) (configurer.WGStats, error)
+	GetStats() (map[string]configurer.WGStats, error)
 }
 
 type WGWatcher struct {
 	log           *log.Entry
 	wgIfaceStater WGInterfaceStater
 	peerKey       string
+	stateDump     *stateDump
 
 	ctx       context.Context
 	ctxCancel context.CancelFunc
 	ctxLock   sync.Mutex
-	waitGroup sync.WaitGroup
 }
 
-func NewWGWatcher(log *log.Entry, wgIfaceStater WGInterfaceStater, peerKey string) *WGWatcher {
+func NewWGWatcher(log *log.Entry, wgIfaceStater WGInterfaceStater, peerKey string, stateDump *stateDump) *WGWatcher {
 	return &WGWatcher{
 		log:           log,
 		wgIfaceStater: wgIfaceStater,
 		peerKey:       peerKey,
+		stateDump:     stateDump,
 	}
 }
 
@@ -46,24 +48,24 @@ func NewWGWatcher(log *log.Entry, wgIfaceStater WGInterfaceStater, peerKey strin
 func (w *WGWatcher) EnableWgWatcher(parentCtx context.Context, onDisconnectedFn func()) {
 	w.log.Debugf("enable WireGuard watcher")
 	w.ctxLock.Lock()
-	defer w.ctxLock.Unlock()
 
 	if w.ctx != nil && w.ctx.Err() == nil {
 		w.log.Errorf("WireGuard watcher already enabled")
+		w.ctxLock.Unlock()
 		return
 	}
 
 	ctx, ctxCancel := context.WithCancel(parentCtx)
 	w.ctx = ctx
 	w.ctxCancel = ctxCancel
+	w.ctxLock.Unlock()
 
 	initialHandshake, err := w.wgState()
 	if err != nil {
 		w.log.Warnf("failed to read initial wg stats: %v", err)
 	}
 
-	w.waitGroup.Add(1)
-	go w.periodicHandshakeCheck(ctx, ctxCancel, onDisconnectedFn, initialHandshake)
+	w.periodicHandshakeCheck(ctx, ctxCancel, onDisconnectedFn, initialHandshake)
 }
 
 // DisableWgWatcher stops the WireGuard watcher and wait for the watcher to exit
@@ -79,13 +81,11 @@ func (w *WGWatcher) DisableWgWatcher() {
 
 	w.ctxCancel()
 	w.ctxCancel = nil
-	w.waitGroup.Wait()
 }
 
 // wgStateCheck help to check the state of the WireGuard handshake and relay connection
 func (w *WGWatcher) periodicHandshakeCheck(ctx context.Context, ctxCancel context.CancelFunc, onDisconnectedFn func(), initialHandshake time.Time) {
 	w.log.Infof("WireGuard watcher started")
-	defer w.waitGroup.Done()
 
 	timer := time.NewTimer(wgHandshakeOvertime)
 	defer timer.Stop()
@@ -105,6 +105,7 @@ func (w *WGWatcher) periodicHandshakeCheck(ctx context.Context, ctxCancel contex
 
 			resetTime := time.Until(handshake.Add(checkPeriod))
 			timer.Reset(resetTime)
+			w.stateDump.WGcheckSuccess()
 
 			w.log.Debugf("WireGuard watcher reset timer: %v", resetTime)
 		case <-ctx.Done():
@@ -146,9 +147,13 @@ func (w *WGWatcher) handshakeCheck(lastHandshake time.Time) (*time.Time, bool) {
 }
 
 func (w *WGWatcher) wgState() (time.Time, error) {
-	wgState, err := w.wgIfaceStater.GetStats(w.peerKey)
+	wgStates, err := w.wgIfaceStater.GetStats()
 	if err != nil {
 		return time.Time{}, err
 	}
+	wgState, ok := wgStates[w.peerKey]
+	if !ok {
+		return time.Time{}, fmt.Errorf("peer %s not found in WireGuard endpoints", w.peerKey)
+	}
 	return wgState.LastHandshake, nil
 }

client/internal/peer/wg_watcher_test.go

@@ -11,26 +11,11 @@ import (
 )
 
 type MocWgIface struct {
-	initial       bool
-	lastHandshake time.Time
-	stop          bool
+	stop bool
 }
 
-func (m *MocWgIface) GetStats(key string) (configurer.WGStats, error) {
-	if !m.initial {
-		m.initial = true
-		return configurer.WGStats{}, nil
-	}
-
-	if !m.stop {
-		m.lastHandshake = time.Now()
-	}
-
-	stats := configurer.WGStats{
-		LastHandshake: m.lastHandshake,
-	}
-
-	return stats, nil
+func (m *MocWgIface) GetStats() (map[string]configurer.WGStats, error) {
+	return map[string]configurer.WGStats{}, nil
 }
 
 func (m *MocWgIface) disconnect() {
@@ -43,13 +28,13 @@ func TestWGWatcher_EnableWgWatcher(t *testing.T) {
 
 	mlog := log.WithField("peer", "tet")
 	mocWgIface := &MocWgIface{}
-	watcher := NewWGWatcher(mlog, mocWgIface, "")
+	watcher := NewWGWatcher(mlog, mocWgIface, "", newStateDump("peer", mlog, &Status{}))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	onDisconnected := make(chan struct{}, 1)
-	watcher.EnableWgWatcher(ctx, func() {
+	go watcher.EnableWgWatcher(ctx, func() {
 		mlog.Infof("onDisconnectedFn")
 		onDisconnected <- struct{}{}
 	})
@@ -72,17 +57,18 @@ func TestWGWatcher_ReEnable(t *testing.T) {
 
 	mlog := log.WithField("peer", "tet")
 	mocWgIface := &MocWgIface{}
-	watcher := NewWGWatcher(mlog, mocWgIface, "")
+	watcher := NewWGWatcher(mlog, mocWgIface, "", newStateDump("peer", mlog, &Status{}))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	onDisconnected := make(chan struct{}, 1)
 
-	watcher.EnableWgWatcher(ctx, func() {})
+	go watcher.EnableWgWatcher(ctx, func() {})
+	time.Sleep(1 * time.Second)
 	watcher.DisableWgWatcher()
 
-	watcher.EnableWgWatcher(ctx, func() {
+	go watcher.EnableWgWatcher(ctx, func() {
 		onDisconnected <- struct{}{}
 	})
 

client/internal/peer/worker/state.go

@@ -0,0 +1,55 @@
+package worker
+
+import (
+	"sync/atomic"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	StatusDisconnected Status = iota
+	StatusConnected
+)
+
+type Status int32
+
+func (s Status) String() string {
+	switch s {
+	case StatusDisconnected:
+		return "Disconnected"
+	case StatusConnected:
+		return "Connected"
+	default:
+		log.Errorf("unknown status: %d", s)
+		return "unknown"
+	}
+}
+
+// AtomicWorkerStatus is a thread-safe wrapper for worker status
+type AtomicWorkerStatus struct {
+	status atomic.Int32
+}
+
+func NewAtomicStatus() *AtomicWorkerStatus {
+	acs := &AtomicWorkerStatus{}
+	acs.SetDisconnected()
+	return acs
+}
+
+// Get returns the current connection status
+func (acs *AtomicWorkerStatus) Get() Status {
+	return Status(acs.status.Load())
+}
+
+func (acs *AtomicWorkerStatus) SetConnected() {
+	acs.status.Store(int32(StatusConnected))
+}
+
+func (acs *AtomicWorkerStatus) SetDisconnected() {
+	acs.status.Store(int32(StatusDisconnected))
+}
+
+// String returns the string representation of the current status
+func (acs *AtomicWorkerStatus) String() string {
+	return acs.Get().String()
+}

client/internal/peer/worker_ice.go

@@ -8,12 +8,13 @@ import (
 	"sync"
 	"time"
 
-	"github.com/pion/ice/v3"
+	"github.com/pion/ice/v4"
 	"github.com/pion/stun/v2"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/internal/peer/conntype"
 	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	"github.com/netbirdio/netbird/route"
@@ -41,8 +42,18 @@ type WorkerICE struct {
 	statusRecorder    *Status
 	hasRelayOnLocally bool
 
-	agent    *ice.Agent
-	muxAgent sync.Mutex
+	agent             *icemaker.ThreadSafeAgent
+	agentDialerCancel context.CancelFunc
+	agentConnecting   bool      // while it is true, drop all incoming offers
+	lastSuccess       time.Time // with this avoid the too frequent ICE agent recreation
+	// remoteSessionID represents the peer's session identifier from the latest remote offer.
+	remoteSessionID ICESessionID
+	// sessionID is used to track the current session ID of the ICE agent
+	// increase by one when disconnecting the agent
+	// with it the remote peer can discard the already deprecated offer/answer
+	// Without it the remote peer may recreate a workable ICE connection
+	sessionID ICESessionID
+	muxAgent  sync.Mutex
 
 	StunTurn []*stun.URI
 
@@ -56,6 +67,11 @@ type WorkerICE struct {
 }
 
 func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, conn *Conn, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool) (*WorkerICE, error) {
+	sessionID, err := NewICESessionID()
+	if err != nil {
+		return nil, err
+	}
+
 	w := &WorkerICE{
 		ctx:               ctx,
 		log:               log,
@@ -65,6 +81,8 @@ func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, conn *
 		iFaceDiscover:     ifaceDiscover,
 		statusRecorder:    statusRecorder,
 		hasRelayOnLocally: hasRelayOnLocally,
+		lastKnownState:    ice.ConnectionStateDisconnected,
+		sessionID:         sessionID,
 	}
 
 	localUfrag, localPwd, err := icemaker.GenerateICECredentials()
@@ -77,15 +95,36 @@ func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, conn *
 }
 
 func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
-	w.log.Debugf("OnNewOffer for ICE")
+	w.log.Debugf("OnNewOffer for ICE, serial: %s", remoteOfferAnswer.SessionIDString())
 	w.muxAgent.Lock()
 
-	if w.agent != nil {
-		w.log.Debugf("agent already exists, skipping the offer")
+	if w.agentConnecting {
+		w.log.Debugf("agent connection is in progress, skipping the offer")
 		w.muxAgent.Unlock()
 		return
 	}
 
+	if w.agent != nil {
+		// backward compatibility with old clients that do not send session ID
+		if remoteOfferAnswer.SessionID == nil {
+			w.log.Debugf("agent already exists, skipping the offer")
+			w.muxAgent.Unlock()
+			return
+		}
+		if w.remoteSessionID == *remoteOfferAnswer.SessionID {
+			w.log.Debugf("agent already exists and session ID matches, skipping the offer: %s", remoteOfferAnswer.SessionIDString())
+			w.muxAgent.Unlock()
+			return
+		}
+		w.log.Debugf("agent already exists, recreate the connection")
+		w.agentDialerCancel()
+		if err := w.agent.Close(); err != nil {
+			w.log.Warnf("failed to close ICE agent: %s", err)
+		}
+		w.agent = nil
+		// todo consider to switch to Relay connection while establishing a new ICE connection
+	}
+
 	var preferredCandidateTypes []ice.CandidateType
 	if w.hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" {
 		preferredCandidateTypes = icemaker.CandidateTypesP2P()
@@ -94,36 +133,125 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 	}
 
 	w.log.Debugf("recreate ICE agent")
-	agentCtx, agentCancel := context.WithCancel(w.ctx)
-	agent, err := w.reCreateAgent(agentCancel, preferredCandidateTypes)
+	dialerCtx, dialerCancel := context.WithCancel(w.ctx)
+	agent, err := w.reCreateAgent(dialerCancel, preferredCandidateTypes)
 	if err != nil {
 		w.log.Errorf("failed to recreate ICE Agent: %s", err)
 		w.muxAgent.Unlock()
 		return
 	}
+	w.sentExtraSrflx = false
 	w.agent = agent
+	w.agentDialerCancel = dialerCancel
+	w.agentConnecting = true
 	w.muxAgent.Unlock()
 
+	go w.connect(dialerCtx, agent, remoteOfferAnswer)
+}
+
+// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer.
+func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+	w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String())
+	if w.agent == nil {
+		w.log.Warnf("ICE Agent is not initialized yet")
+		return
+	}
+
+	if candidateViaRoutes(candidate, haRoutes) {
+		return
+	}
+
+	if err := w.agent.AddRemoteCandidate(candidate); err != nil {
+		w.log.Errorf("error while handling remote candidate")
+		return
+	}
+}
+
+func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string) {
+	return w.localUfrag, w.localPwd
+}
+
+func (w *WorkerICE) InProgress() bool {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+
+	return w.agentConnecting
+}
+
+func (w *WorkerICE) Close() {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+
+	if w.agent == nil {
+		return
+	}
+
+	w.agentDialerCancel()
+	if err := w.agent.Close(); err != nil {
+		w.log.Warnf("failed to close ICE agent: %s", err)
+	}
+
+	w.agent = nil
+}
+
+func (w *WorkerICE) reCreateAgent(dialerCancel context.CancelFunc, candidates []ice.CandidateType) (*icemaker.ThreadSafeAgent, error) {
+	agent, err := icemaker.NewAgent(w.iFaceDiscover, w.config.ICEConfig, candidates, w.localUfrag, w.localPwd)
+	if err != nil {
+		return nil, fmt.Errorf("create agent: %w", err)
+	}
+
+	if err := agent.OnCandidate(w.onICECandidate); err != nil {
+		return nil, err
+	}
+
+	if err := agent.OnConnectionStateChange(w.onConnectionStateChange(agent, dialerCancel)); err != nil {
+		return nil, err
+	}
+
+	if err := agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair); err != nil {
+		return nil, err
+	}
+
+	return agent, nil
+}
+
+func (w *WorkerICE) SessionID() ICESessionID {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+
+	return w.sessionID
+}
+
+// will block until connection succeeded
+// but it won't release if ICE Agent went into Disconnected or Failed state,
+// so we have to cancel it with the provided context once agent detected a broken connection
+func (w *WorkerICE) connect(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) {
 	w.log.Debugf("gather candidates")
-	err = w.agent.GatherCandidates()
-	if err != nil {
-		w.log.Debugf("failed to gather candidates: %s", err)
+	if err := agent.GatherCandidates(); err != nil {
+		w.log.Warnf("failed to gather candidates: %s", err)
+		w.closeAgent(agent, w.agentDialerCancel)
 		return
 	}
 
-	// will block until connection succeeded
-	// but it won't release if ICE Agent went into Disconnected or Failed state,
-	// so we have to cancel it with the provided context once agent detected a broken connection
 	w.log.Debugf("turn agent dial")
-	remoteConn, err := w.turnAgentDial(agentCtx, remoteOfferAnswer)
+	remoteConn, err := w.turnAgentDial(ctx, agent, remoteOfferAnswer)
 	if err != nil {
 		w.log.Debugf("failed to dial the remote peer: %s", err)
+		w.closeAgent(agent, w.agentDialerCancel)
 		return
 	}
 	w.log.Debugf("agent dial succeeded")
 
-	pair, err := w.agent.GetSelectedCandidatePair()
+	pair, err := agent.GetSelectedCandidatePair()
 	if err != nil {
+		w.closeAgent(agent, w.agentDialerCancel)
+		return
+	}
+	if pair == nil {
+		w.log.Warnf("selected candidate pair is nil, cannot proceed")
+		w.closeAgent(agent, w.agentDialerCancel)
 		return
 	}
 
@@ -150,114 +278,39 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 		RelayedOnLocal:             isRelayCandidate(pair.Local),
 	}
 	w.log.Debugf("on ICE conn is ready to use")
-	go w.conn.onICEConnectionIsReady(selectedPriority(pair), ci)
-}
 
-// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer.
-func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) {
+	w.log.Infof("connection succeeded with offer session: %s", remoteOfferAnswer.SessionIDString())
 	w.muxAgent.Lock()
-	defer w.muxAgent.Unlock()
-	w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String())
-	if w.agent == nil {
-		w.log.Warnf("ICE Agent is not initialized yet")
-		return
+	w.agentConnecting = false
+	w.lastSuccess = time.Now()
+	if remoteOfferAnswer.SessionID != nil {
+		w.remoteSessionID = *remoteOfferAnswer.SessionID
 	}
+	w.muxAgent.Unlock()
 
-	if candidateViaRoutes(candidate, haRoutes) {
-		return
-	}
-
-	err := w.agent.AddRemoteCandidate(candidate)
-	if err != nil {
-		w.log.Errorf("error while handling remote candidate")
-		return
-	}
+	// todo: the potential problem is a race between the onConnectionStateChange
+	w.conn.onICEConnectionIsReady(selectedPriority(pair), ci)
 }
 
-func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string) {
-	w.muxAgent.Lock()
-	defer w.muxAgent.Unlock()
-	return w.localUfrag, w.localPwd
-}
-
-func (w *WorkerICE) Close() {
-	w.muxAgent.Lock()
-	defer w.muxAgent.Unlock()
-
-	if w.agent == nil {
-		return
-	}
-
-	if err := w.agent.Close(); err != nil {
-		w.log.Warnf("failed to close ICE agent: %s", err)
-	}
-}
-
-func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, candidates []ice.CandidateType) (*ice.Agent, error) {
-	w.sentExtraSrflx = false
-
-	agent, err := icemaker.NewAgent(w.iFaceDiscover, w.config.ICEConfig, candidates, w.localUfrag, w.localPwd)
-	if err != nil {
-		return nil, fmt.Errorf("create agent: %w", err)
-	}
-
-	err = agent.OnCandidate(w.onICECandidate)
-	if err != nil {
-		return nil, err
-	}
-
-	err = agent.OnConnectionStateChange(func(state ice.ConnectionState) {
-		w.log.Debugf("ICE ConnectionState has changed to %s", state.String())
-		switch state {
-		case ice.ConnectionStateConnected:
-			w.lastKnownState = ice.ConnectionStateConnected
-			return
-		case ice.ConnectionStateFailed, ice.ConnectionStateDisconnected:
-			if w.lastKnownState != ice.ConnectionStateDisconnected {
-				w.lastKnownState = ice.ConnectionStateDisconnected
-				w.conn.onICEStateDisconnected()
-			}
-			w.closeAgent(agentCancel)
-		default:
-			return
-		}
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	err = agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair)
-	if err != nil {
-		return nil, err
-	}
-
-	err = agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) {
-		err := w.statusRecorder.UpdateLatency(w.config.Key, p.Latency())
-		if err != nil {
-			w.log.Debugf("failed to update latency for peer: %s", err)
-			return
-		}
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed setting binding response callback: %w", err)
-	}
-
-	return agent, nil
-}
-
-func (w *WorkerICE) closeAgent(cancel context.CancelFunc) {
-	w.muxAgent.Lock()
-	defer w.muxAgent.Unlock()
-
+func (w *WorkerICE) closeAgent(agent *icemaker.ThreadSafeAgent, cancel context.CancelFunc) {
 	cancel()
-	if w.agent == nil {
-		return
-	}
-
-	if err := w.agent.Close(); err != nil {
+	if err := agent.Close(); err != nil {
 		w.log.Warnf("failed to close ICE agent: %s", err)
 	}
-	w.agent = nil
+
+	w.muxAgent.Lock()
+	// todo review does it make sense to generate new session ID all the time when w.agent==agent
+	sessionID, err := NewICESessionID()
+	if err != nil {
+		w.log.Errorf("failed to create new session ID: %s", err)
+	}
+	w.sessionID = sessionID
+
+	if w.agent == agent {
+		w.agent = nil
+		w.agentConnecting = false
+	}
+	w.muxAgent.Unlock()
 }
 
 func (w *WorkerICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) {
@@ -327,6 +380,46 @@ func (w *WorkerICE) onICECandidate(candidate ice.Candidate) {
 func (w *WorkerICE) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) {
 	w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(),
 		w.config.Key)
+
+	w.muxAgent.Lock()
+
+	pair, err := w.agent.GetSelectedCandidatePair()
+	if err != nil {
+		w.log.Warnf("failed to get selected candidate pair: %s", err)
+		w.muxAgent.Unlock()
+		return
+	}
+	if pair == nil {
+		w.log.Warnf("selected candidate pair is nil, cannot proceed")
+		w.muxAgent.Unlock()
+		return
+	}
+	w.muxAgent.Unlock()
+
+	duration := time.Duration(pair.CurrentRoundTripTime() * float64(time.Second))
+	if err := w.statusRecorder.UpdateLatency(w.config.Key, duration); err != nil {
+		w.log.Debugf("failed to update latency for peer: %s", err)
+		return
+	}
+}
+
+func (w *WorkerICE) onConnectionStateChange(agent *icemaker.ThreadSafeAgent, dialerCancel context.CancelFunc) func(ice.ConnectionState) {
+	return func(state ice.ConnectionState) {
+		w.log.Debugf("ICE ConnectionState has changed to %s", state.String())
+		switch state {
+		case ice.ConnectionStateConnected:
+			w.lastKnownState = ice.ConnectionStateConnected
+			return
+		case ice.ConnectionStateFailed, ice.ConnectionStateDisconnected:
+			if w.lastKnownState == ice.ConnectionStateConnected {
+				w.lastKnownState = ice.ConnectionStateDisconnected
+				w.conn.onICEStateDisconnected()
+			}
+			w.closeAgent(agent, dialerCancel)
+		default:
+			return
+		}
+	}
 }
 
 func (w *WorkerICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool {
@@ -336,18 +429,18 @@ func (w *WorkerICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool
 	return false
 }
 
-func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) {
+func (w *WorkerICE) turnAgentDial(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) {
 	isControlling := w.config.LocalKey > w.config.Key
 	if isControlling {
-		return w.agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
+		return agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
 	} else {
-		return w.agent.Accept(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
+		return agent.Accept(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
 	}
 }
 
 func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) {
 	relatedAdd := candidate.RelatedAddress()
-	return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{
+	ec, err := ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{
 		Network:   candidate.NetworkType().String(),
 		Address:   candidate.Address(),
 		Port:      relatedAdd.Port,
@@ -355,9 +448,26 @@ func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive
 		RelAddr:   relatedAdd.Address,
 		RelPort:   relatedAdd.Port,
 	})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, e := range candidate.Extensions() {
+		if err := ec.AddExtension(e); err != nil {
+			return nil, err
+		}
+	}
+
+	return ec, nil
 }
 
 func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool {
+	addr, err := netip.ParseAddr(candidate.Address())
+	if err != nil {
+		log.Errorf("Failed to parse IP address %s: %v", candidate.Address(), err)
+		return false
+	}
+
 	var routePrefixes []netip.Prefix
 	for _, routes := range clientRoutes {
 		if len(routes) > 0 && routes[0] != nil {
@@ -365,14 +475,8 @@ func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool
 		}
 	}
 
-	addr, err := netip.ParseAddr(candidate.Address())
-	if err != nil {
-		log.Errorf("Failed to parse IP address %s: %v", candidate.Address(), err)
-		return false
-	}
-
 	for _, prefix := range routePrefixes {
-		// default route is
+		// default route is handled by route exclusion / ip rules
 		if prefix.Bits() == 0 {
 			continue
 		}
@@ -396,10 +500,10 @@ func isRelayed(pair *ice.CandidatePair) bool {
 	return false
 }
 
-func selectedPriority(pair *ice.CandidatePair) ConnPriority {
+func selectedPriority(pair *ice.CandidatePair) conntype.ConnPriority {
 	if isRelayed(pair) {
-		return connPriorityICETurn
+		return conntype.ICETurn
 	} else {
-		return connPriorityICEP2P
+		return conntype.ICEP2P
 	}
 }

client/internal/peer/worker_relay.go

@@ -9,7 +9,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	relayClient "github.com/netbirdio/netbird/relay/client"
+	relayClient "github.com/netbirdio/netbird/shared/relay/client"
 )
 
 type RelayConnInfo struct {
@@ -19,11 +19,12 @@ type RelayConnInfo struct {
 }
 
 type WorkerRelay struct {
+	peerCtx      context.Context
 	log          *log.Entry
 	isController bool
 	config       ConnConfig
 	conn         *Conn
-	relayManager relayClient.ManagerService
+	relayManager *relayClient.Manager
 
 	relayedConn net.Conn
 	relayLock   sync.Mutex
@@ -33,14 +34,15 @@ type WorkerRelay struct {
 	wgWatcher *WGWatcher
 }
 
-func NewWorkerRelay(log *log.Entry, ctrl bool, config ConnConfig, conn *Conn, relayManager relayClient.ManagerService) *WorkerRelay {
+func NewWorkerRelay(ctx context.Context, log *log.Entry, ctrl bool, config ConnConfig, conn *Conn, relayManager *relayClient.Manager, stateDump *stateDump) *WorkerRelay {
 	r := &WorkerRelay{
+		peerCtx:      ctx,
 		log:          log,
 		isController: ctrl,
 		config:       config,
 		conn:         conn,
 		relayManager: relayManager,
-		wgWatcher:    NewWGWatcher(log, config.WgConfig.WgInterface, config.Key),
+		wgWatcher:    NewWGWatcher(log, config.WgConfig.WgInterface, config.Key, stateDump),
 	}
 	return r
 }
@@ -62,7 +64,7 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 
 	srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress)
 
-	relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key)
+	relayedConn, err := w.relayManager.OpenConn(w.peerCtx, srv, w.config.Key)
 	if err != nil {
 		if errors.Is(err, relayClient.ErrConnAlreadyExists) {
 			w.log.Debugf("handled offer by reusing existing relay connection")