[client] Fix/pkg loss (#3338)

The Relayed connection setup is optimistic. It does not have any confirmation of an established end-to-end connection. Peers start sending WireGuard handshake packets immediately after the successful offer-answer handshake.
Meanwhile, for successful P2P connection negotiation, we change the WireGuard endpoint address, but this change does not trigger new handshake initiation. Because the peer switched from Relayed connection to P2P, the packets from the Relay server are dropped and must wait for the next WireGuard handshake via P2P.

To avoid this scenario, the relayed WireGuard proxy no longer drops the packets. Instead, it rewrites the source address to the new P2P endpoint and continues forwarding the packets.

We still have one corner case: if the Relayed server negotiation chooses a server that has not been used before. In this case, one side of the peer connection will be slower to reach the Relay server, and the Relay server will drop the handshake packet.

If everything goes well we should see exactly 5 seconds improvements between the WireGuard configuration time and the handshake time.

client/iface/wgproxy/bind/proxy.go

@@ -16,28 +16,37 @@ import (
 	"github.com/netbirdio/netbird/client/iface/wgproxy/listener"
 )
 
+type IceBind interface {
+	SetEndpoint(fakeIP netip.Addr, conn net.Conn)
+	RemoveEndpoint(fakeIP netip.Addr)
+	Recv(ctx context.Context, msg bind.RecvMessage)
+	MTU() uint16
+}
+
 type ProxyBind struct {
-	Bind *bind.ICEBind
+	bind IceBind
 
-	fakeNetIP      *netip.AddrPort
-	wgBindEndpoint *bind.Endpoint
-	remoteConn     net.Conn
-	ctx            context.Context
-	cancel         context.CancelFunc
-	closeMu        sync.Mutex
-	closed         bool
+	// wgRelayedEndpoint is a fake address that generated by the Bind.SetEndpoint based on the remote NetBird peer address
+	wgRelayedEndpoint *bind.Endpoint
+	wgCurrentUsed     *bind.Endpoint
+	remoteConn        net.Conn
+	ctx               context.Context
+	cancel            context.CancelFunc
+	closeMu           sync.Mutex
+	closed            bool
 
-	pausedMu  sync.Mutex
-	paused    bool
-	isStarted bool
+	paused     bool
+	pausedCond *sync.Cond
+	isStarted  bool
 
 	closeListener *listener.CloseListener
 }
 
-func NewProxyBind(bind *bind.ICEBind) *ProxyBind {
+func NewProxyBind(bind IceBind) *ProxyBind {
 	p := &ProxyBind{
-		Bind:          bind,
+		bind:          bind,
 		closeListener: listener.NewCloseListener(),
+		pausedCond:    sync.NewCond(&sync.Mutex{}),
 	}
 
 	return p
@@ -46,25 +55,25 @@ func NewProxyBind(bind *bind.ICEBind) *ProxyBind {
 // AddTurnConn adds a new connection to the bind.
 // endpoint is the NetBird address of the remote peer. The SetEndpoint return with the address what will be used in the
 // WireGuard configuration.
+//
+// Parameters:
+//   - ctx: Context is used for proxyToLocal to avoid unnecessary error messages
+//   - nbAddr: The NetBird UDP address of the remote peer, it required to generate fake address
+//   - remoteConn: The established TURN connection to the remote peer
 func (p *ProxyBind) AddTurnConn(ctx context.Context, nbAddr *net.UDPAddr, remoteConn net.Conn) error {
 	fakeNetIP, err := fakeAddress(nbAddr)
 	if err != nil {
 		return err
 	}
-
-	p.fakeNetIP = fakeNetIP
-	p.wgBindEndpoint = &bind.Endpoint{AddrPort: *fakeNetIP}
+	p.wgRelayedEndpoint = &bind.Endpoint{AddrPort: *fakeNetIP}
 	p.remoteConn = remoteConn
 	p.ctx, p.cancel = context.WithCancel(ctx)
 	return nil
 
 }
+
 func (p *ProxyBind) EndpointAddr() *net.UDPAddr {
-	return &net.UDPAddr{
-		IP:   p.fakeNetIP.Addr().AsSlice(),
-		Port: int(p.fakeNetIP.Port()),
-		Zone: p.fakeNetIP.Addr().Zone(),
-	}
+	return bind.EndpointToUDPAddr(*p.wgRelayedEndpoint)
 }
 
 func (p *ProxyBind) SetDisconnectListener(disconnected func()) {
@@ -76,17 +85,21 @@ func (p *ProxyBind) Work() {
 		return
 	}
 
-	p.Bind.SetEndpoint(p.fakeNetIP.Addr(), p.remoteConn)
+	p.bind.SetEndpoint(p.wgRelayedEndpoint.Addr(), p.remoteConn)
 
-	p.pausedMu.Lock()
+	p.pausedCond.L.Lock()
 	p.paused = false
-	p.pausedMu.Unlock()
+
+	p.wgCurrentUsed = p.wgRelayedEndpoint
 
 	// Start the proxy only once
 	if !p.isStarted {
 		p.isStarted = true
 		go p.proxyToLocal(p.ctx)
 	}
+
+	p.pausedCond.Signal()
+	p.pausedCond.L.Unlock()
 }
 
 func (p *ProxyBind) Pause() {
@@ -94,9 +107,25 @@ func (p *ProxyBind) Pause() {
 		return
 	}
 
-	p.pausedMu.Lock()
+	p.pausedCond.L.Lock()
 	p.paused = true
-	p.pausedMu.Unlock()
+	p.pausedCond.L.Unlock()
+}
+
+func (p *ProxyBind) RedirectAs(endpoint *net.UDPAddr) {
+	p.pausedCond.L.Lock()
+	p.paused = false
+
+	p.wgCurrentUsed = addrToEndpoint(endpoint)
+
+	p.pausedCond.Signal()
+	p.pausedCond.L.Unlock()
+}
+
+func addrToEndpoint(addr *net.UDPAddr) *bind.Endpoint {
+	ip, _ := netip.AddrFromSlice(addr.IP.To4())
+	addrPort := netip.AddrPortFrom(ip, uint16(addr.Port))
+	return &bind.Endpoint{AddrPort: addrPort}
 }
 
 func (p *ProxyBind) CloseConn() error {
@@ -107,6 +136,10 @@ func (p *ProxyBind) CloseConn() error {
 }
 
 func (p *ProxyBind) close() error {
+	if p.remoteConn == nil {
+		return nil
+	}
+
 	p.closeMu.Lock()
 	defer p.closeMu.Unlock()
 
@@ -120,7 +153,12 @@ func (p *ProxyBind) close() error {
 
 	p.cancel()
 
-	p.Bind.RemoveEndpoint(p.fakeNetIP.Addr())
+	p.pausedCond.L.Lock()
+	p.paused = false
+	p.pausedCond.Signal()
+	p.pausedCond.L.Unlock()
+
+	p.bind.RemoveEndpoint(p.wgRelayedEndpoint.Addr())
 
 	if rErr := p.remoteConn.Close(); rErr != nil && !errors.Is(rErr, net.ErrClosed) {
 		return rErr
@@ -136,7 +174,7 @@ func (p *ProxyBind) proxyToLocal(ctx context.Context) {
 	}()
 
 	for {
-		buf := make([]byte, p.Bind.MTU()+bufsize.WGBufferOverhead)
+		buf := make([]byte, p.bind.MTU()+bufsize.WGBufferOverhead)
 		n, err := p.remoteConn.Read(buf)
 		if err != nil {
 			if ctx.Err() != nil {
@@ -147,18 +185,17 @@ func (p *ProxyBind) proxyToLocal(ctx context.Context) {
 			return
 		}
 
-		p.pausedMu.Lock()
-		if p.paused {
-			p.pausedMu.Unlock()
-			continue
+		p.pausedCond.L.Lock()
+		for p.paused {
+			p.pausedCond.Wait()
 		}
 
 		msg := bind.RecvMessage{
-			Endpoint: p.wgBindEndpoint,
+			Endpoint: p.wgCurrentUsed,
 			Buffer:   buf[:n],
 		}
-		p.Bind.RecvChan <- msg
-		p.pausedMu.Unlock()
+		p.bind.Recv(ctx, msg)
+		p.pausedCond.L.Unlock()
 	}
 }