[client] Use native windows sock opts to avoid routing loops (#4314)

- Move `util/grpc` and `util/net` to `client` so `internal` packages can be accessed
 - Add methods to return the next best interface after the NetBird interface.
- Use `IP_UNICAST_IF` sock opt to force the outgoing interface for the NetBird `net.Dialer` and `net.ListenerConfig` to avoid routing loops. The interface is picked by the new route lookup method.
- Some refactoring to avoid import cycles
- Old behavior is available through `NB_USE_LEGACY_ROUTING=true` env var

client/net/conn.go

@@ -0,0 +1,49 @@
+//go:build !ios
+
+package net
+
+import (
+	"io"
+	"net"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/net/hooks"
+)
+
+// Conn wraps a net.Conn to override the Close method
+type Conn struct {
+	net.Conn
+	ID hooks.ConnectionID
+}
+
+// Close overrides the net.Conn Close method to execute all registered hooks after closing the connection
+// Close overrides the net.Conn Close method to execute all registered hooks before closing the connection.
+func (c *Conn) Close() error {
+	return closeConn(c.ID, c.Conn)
+}
+
+// TCPConn wraps net.TCPConn to override its Close method to include hook functionality.
+type TCPConn struct {
+	*net.TCPConn
+	ID hooks.ConnectionID
+}
+
+// Close overrides the net.TCPConn Close method to execute all registered hooks before closing the connection.
+func (c *TCPConn) Close() error {
+	return closeConn(c.ID, c.TCPConn)
+}
+
+// closeConn is a helper function to close connections and execute close hooks.
+func closeConn(id hooks.ConnectionID, conn io.Closer) error {
+	err := conn.Close()
+
+	closeHooks := hooks.GetCloseHooks()
+	for _, hook := range closeHooks {
+		if err := hook(id); err != nil {
+			log.Errorf("Error executing close hook: %v", err)
+		}
+	}
+
+	return err
+}

client/net/dial.go

@@ -0,0 +1,82 @@
+//go:build !ios
+
+package net
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/pion/transport/v3"
+	log "github.com/sirupsen/logrus"
+)
+
+func DialUDP(network string, laddr, raddr *net.UDPAddr) (transport.UDPConn, error) {
+	if CustomRoutingDisabled() {
+		return net.DialUDP(network, laddr, raddr)
+	}
+
+	dialer := NewDialer()
+	dialer.LocalAddr = laddr
+
+	conn, err := dialer.Dial(network, raddr.String())
+	if err != nil {
+		return nil, fmt.Errorf("dialing UDP %s: %w", raddr.String(), err)
+	}
+
+	switch c := conn.(type) {
+	case *net.UDPConn:
+		// Advanced routing: plain connection
+		return c, nil
+	case *Conn:
+		// Legacy routing: wrapped connection preserves close hooks
+		udpConn, ok := c.Conn.(*net.UDPConn)
+		if !ok {
+			if err := conn.Close(); err != nil {
+				log.Errorf("Failed to close connection: %v", err)
+			}
+			return nil, fmt.Errorf("expected UDP connection, got %T", c.Conn)
+		}
+		return &UDPConn{UDPConn: udpConn, ID: c.ID, seenAddrs: &sync.Map{}}, nil
+	}
+
+	if err := conn.Close(); err != nil {
+		log.Errorf("failed to close connection: %v", err)
+	}
+	return nil, fmt.Errorf("unexpected connection type: %T", conn)
+}
+
+func DialTCP(network string, laddr, raddr *net.TCPAddr) (transport.TCPConn, error) {
+	if CustomRoutingDisabled() {
+		return net.DialTCP(network, laddr, raddr)
+	}
+
+	dialer := NewDialer()
+	dialer.LocalAddr = laddr
+
+	conn, err := dialer.Dial(network, raddr.String())
+	if err != nil {
+		return nil, fmt.Errorf("dialing TCP %s: %w", raddr.String(), err)
+	}
+
+	switch c := conn.(type) {
+	case *net.TCPConn:
+		// Advanced routing: plain connection
+		return c, nil
+	case *Conn:
+		// Legacy routing: wrapped connection preserves close hooks
+		tcpConn, ok := c.Conn.(*net.TCPConn)
+		if !ok {
+			if err := conn.Close(); err != nil {
+				log.Errorf("Failed to close connection: %v", err)
+			}
+			return nil, fmt.Errorf("expected TCP connection, got %T", c.Conn)
+		}
+		return &TCPConn{TCPConn: tcpConn, ID: c.ID}, nil
+	}
+
+	if err := conn.Close(); err != nil {
+		log.Errorf("failed to close connection: %v", err)
+	}
+	return nil, fmt.Errorf("unexpected connection type: %T", conn)
+}

client/net/dial_ios.go

@@ -0,0 +1,13 @@
+package net
+
+import (
+	"net"
+)
+
+func DialUDP(network string, laddr, raddr *net.UDPAddr) (*net.UDPConn, error) {
+	return net.DialUDP(network, laddr, raddr)
+}
+
+func DialTCP(network string, laddr, raddr *net.TCPAddr) (*net.TCPConn, error) {
+	return net.DialTCP(network, laddr, raddr)
+}

client/net/dialer.go

@@ -0,0 +1,20 @@
+package net
+
+import (
+	"net"
+)
+
+// Dialer extends the standard net.Dialer with the ability to execute hooks before
+// and after connections. This can be used to bypass the VPN for connections using this dialer.
+type Dialer struct {
+	*net.Dialer
+}
+
+// NewDialer returns a customized net.Dialer with overridden Control method
+func NewDialer() *Dialer {
+	dialer := &Dialer{
+		Dialer: &net.Dialer{},
+	}
+	dialer.init()
+	return dialer
+}

client/net/dialer_dial.go

@@ -0,0 +1,87 @@
+//go:build !ios
+
+package net
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+
+	nberrors "github.com/netbirdio/netbird/client/errors"
+	"github.com/netbirdio/netbird/client/internal/routemanager/util"
+	"github.com/netbirdio/netbird/client/net/hooks"
+)
+
+// DialContext wraps the net.Dialer's DialContext method to use the custom connection
+func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	log.Debugf("Dialing %s %s", network, address)
+
+	if CustomRoutingDisabled() || AdvancedRouting() {
+		return d.Dialer.DialContext(ctx, network, address)
+	}
+
+	connID := hooks.GenerateConnID()
+	if err := callDialerHooks(ctx, connID, address, d.Resolver); err != nil {
+		log.Errorf("Failed to call dialer hooks: %v", err)
+	}
+
+	conn, err := d.Dialer.DialContext(ctx, network, address)
+	if err != nil {
+		return nil, fmt.Errorf("d.Dialer.DialContext: %w", err)
+	}
+
+	// Wrap the connection in Conn to handle Close with hooks
+	return &Conn{Conn: conn, ID: connID}, nil
+}
+
+// Dial wraps the net.Dialer's Dial method to use the custom connection
+func (d *Dialer) Dial(network, address string) (net.Conn, error) {
+	return d.DialContext(context.Background(), network, address)
+}
+
+func callDialerHooks(ctx context.Context, connID hooks.ConnectionID, address string, customResolver *net.Resolver) error {
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
+	writeHooks := hooks.GetWriteHooks()
+	if len(writeHooks) == 0 {
+		return nil
+	}
+
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		return fmt.Errorf("split host and port: %w", err)
+	}
+
+	resolver := customResolver
+	if resolver == nil {
+		resolver = net.DefaultResolver
+	}
+
+	ips, err := resolver.LookupIPAddr(ctx, host)
+	if err != nil {
+		return fmt.Errorf("failed to resolve address %s: %w", address, err)
+	}
+
+	log.Debugf("Dialer resolved IPs for %s: %v", address, ips)
+
+	var merr *multierror.Error
+	for _, ip := range ips {
+		prefix, err := util.GetPrefixFromIP(ip.IP)
+		if err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("convert IP %s to prefix: %w", ip.IP, err))
+			continue
+		}
+		for _, hook := range writeHooks {
+			if err := hook(connID, prefix); err != nil {
+				merr = multierror.Append(merr, fmt.Errorf("executing dial hook for IP %s: %w", ip.IP, err))
+			}
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}

client/net/dialer_init_android.go

@@ -0,0 +1,5 @@
+package net
+
+func (d *Dialer) init() {
+	d.Dialer.Control = ControlProtectSocket
+}

client/net/dialer_init_generic.go

@@ -0,0 +1,7 @@
+//go:build !linux && !windows
+
+package net
+
+func (d *Dialer) init() {
+	// implemented on Linux, Android, and Windows only
+}

client/net/dialer_init_linux.go

@@ -0,0 +1,12 @@
+//go:build !android
+
+package net
+
+import "syscall"
+
+// init configures the net.Dialer Control function to set the fwmark on the socket
+func (d *Dialer) init() {
+	d.Dialer.Control = func(_, _ string, c syscall.RawConn) error {
+		return setRawSocketMark(c)
+	}
+}

client/net/dialer_init_windows.go

@@ -0,0 +1,5 @@
+package net
+
+func (d *Dialer) init() {
+	d.Dialer.Control = applyUnicastIFToSocket
+}

client/net/env.go

@@ -0,0 +1,35 @@
+package net
+
+import (
+	"os"
+	"strconv"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/iface/netstack"
+)
+
+const (
+	envDisableCustomRouting = "NB_DISABLE_CUSTOM_ROUTING"
+	envUseLegacyRouting     = "NB_USE_LEGACY_ROUTING"
+)
+
+// CustomRoutingDisabled returns true if custom routing is disabled.
+// This will fall back to the operation mode before the exit node functionality was implemented.
+// In particular exclusion routes won't be set up and all dialers and listeners will use net.Dial and net.Listen, respectively.
+func CustomRoutingDisabled() bool {
+	if netstack.IsEnabled() {
+		return true
+	}
+
+	var customRoutingDisabled bool
+	if val := os.Getenv(envDisableCustomRouting); val != "" {
+		var err error
+		customRoutingDisabled, err = strconv.ParseBool(val)
+		if err != nil {
+			log.Warnf("failed to parse %s: %v", envDisableCustomRouting, err)
+		}
+	}
+
+	return customRoutingDisabled
+}

client/net/env_android.go

@@ -0,0 +1,24 @@
+//go:build android
+
+package net
+
+// Init initializes the network environment for Android
+func Init() {
+	// No initialization needed on Android
+}
+
+// AdvancedRouting reports whether routing loops can be avoided without using exclusion routes.
+// Always returns true on Android since we cannot handle routes dynamically.
+func AdvancedRouting() bool {
+	return true
+}
+
+// SetVPNInterfaceName is a no-op on Android
+func SetVPNInterfaceName(name string) {
+	// No-op on Android - not needed for Android VPN service
+}
+
+// GetVPNInterfaceName returns empty string on Android
+func GetVPNInterfaceName() string {
+	return ""
+}

client/net/env_generic.go

@@ -0,0 +1,23 @@
+//go:build !linux && !windows && !android
+
+package net
+
+// Init initializes the network environment (no-op on non-Linux/Windows platforms)
+func Init() {
+	// No-op on non-Linux/Windows platforms
+}
+
+// AdvancedRouting returns false on non-Linux/Windows platforms
+func AdvancedRouting() bool {
+	return false
+}
+
+// SetVPNInterfaceName is a no-op on non-Windows platforms
+func SetVPNInterfaceName(name string) {
+	// No-op on non-Windows platforms
+}
+
+// GetVPNInterfaceName returns empty string on non-Windows platforms
+func GetVPNInterfaceName() string {
+	return ""
+}

client/net/env_linux.go

@@ -0,0 +1,141 @@
+//go:build linux && !android
+
+package net
+
+import (
+	"errors"
+	"os"
+	"strconv"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vishvananda/netlink"
+
+	"github.com/netbirdio/netbird/client/iface/netstack"
+)
+
+const (
+	// these have the same effect, skip socket env supported for backward compatibility
+	envSkipSocketMark = "NB_SKIP_SOCKET_MARK"
+)
+
+var advancedRoutingSupported bool
+
+func Init() {
+	advancedRoutingSupported = checkAdvancedRoutingSupport()
+}
+
+// AdvancedRouting reports whether routing loops can be avoided without using exclusion routes
+func AdvancedRouting() bool {
+	return advancedRoutingSupported
+}
+
+func checkAdvancedRoutingSupport() bool {
+	var err error
+
+	var legacyRouting bool
+	if val := os.Getenv(envUseLegacyRouting); val != "" {
+		legacyRouting, err = strconv.ParseBool(val)
+		if err != nil {
+			log.Warnf("failed to parse %s: %v", envUseLegacyRouting, err)
+		}
+	}
+
+	var skipSocketMark bool
+	if val := os.Getenv(envSkipSocketMark); val != "" {
+		skipSocketMark, err = strconv.ParseBool(val)
+		if err != nil {
+			log.Warnf("failed to parse %s: %v", envSkipSocketMark, err)
+		}
+	}
+
+	// requested to disable advanced routing
+	if legacyRouting || skipSocketMark ||
+		// envCustomRoutingDisabled disables the custom dialers.
+		// There is no point in using advanced routing without those, as they set up fwmarks on the sockets.
+		CustomRoutingDisabled() ||
+		// netstack mode doesn't need routing at all
+		netstack.IsEnabled() {
+
+		log.Info("advanced routing has been requested to be disabled")
+		return false
+	}
+
+	if !CheckFwmarkSupport() || !CheckRuleOperationsSupport() {
+		log.Warn("system doesn't support required routing features, falling back to legacy routing")
+		return false
+	}
+
+	log.Info("system supports advanced routing")
+
+	return true
+}
+
+func CheckFwmarkSupport() bool {
+	// temporarily enable advanced routing to check if fwmarks are supported
+	old := advancedRoutingSupported
+	advancedRoutingSupported = true
+	defer func() {
+		advancedRoutingSupported = old
+	}()
+
+	dialer := NewDialer()
+	dialer.Timeout = 100 * time.Millisecond
+
+	conn, err := dialer.Dial("udp", "127.0.0.1:9")
+	if err != nil {
+		log.Warnf("failed to dial with fwmark: %v", err)
+		return false
+	}
+
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Warnf("failed to close connection: %v", err)
+		}
+	}()
+
+	if err := conn.SetWriteDeadline(time.Now().Add(time.Millisecond * 100)); err != nil {
+		log.Warnf("failed to set write deadline: %v", err)
+		return false
+	}
+
+	if _, err := conn.Write([]byte("")); err != nil {
+		log.Warnf("failed to write to fwmark connection: %v", err)
+		return false
+	}
+
+	return true
+}
+
+func CheckRuleOperationsSupport() bool {
+	rule := netlink.NewRule()
+	// low precedence, semi-random
+	rule.Priority = 32321
+	rule.Table = syscall.RT_TABLE_MAIN
+	rule.Family = netlink.FAMILY_V4
+
+	if err := netlink.RuleAdd(rule); err != nil {
+		if errors.Is(err, syscall.EOPNOTSUPP) {
+			log.Warn("IP rule operations are not supported")
+			return false
+		}
+		log.Warnf("failed to test rule support: %v", err)
+		return false
+	}
+
+	if err := netlink.RuleDel(rule); err != nil {
+		log.Warnf("failed to delete test rule: %v", err)
+	}
+	return true
+}
+
+// SetVPNInterfaceName is a no-op on Linux
+func SetVPNInterfaceName(name string) {
+	// No-op on Linux - not needed for fwmark-based routing
+}
+
+// GetVPNInterfaceName returns empty string on Linux
+func GetVPNInterfaceName() string {
+	return ""
+}

client/net/env_windows.go

@@ -0,0 +1,67 @@
+//go:build windows
+
+package net
+
+import (
+	"os"
+	"strconv"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/iface/netstack"
+)
+
+var (
+	vpnInterfaceName string
+	vpnInitMutex     sync.RWMutex
+
+	advancedRoutingSupported bool
+)
+
+func Init() {
+	advancedRoutingSupported = checkAdvancedRoutingSupport()
+}
+
+func checkAdvancedRoutingSupport() bool {
+	var err error
+	var legacyRouting bool
+	if val := os.Getenv(envUseLegacyRouting); val != "" {
+		legacyRouting, err = strconv.ParseBool(val)
+		if err != nil {
+			log.Warnf("failed to parse %s: %v", envUseLegacyRouting, err)
+		}
+	}
+
+	if legacyRouting || netstack.IsEnabled() {
+		log.Info("advanced routing has been requested to be disabled")
+		return false
+	}
+
+	log.Info("system supports advanced routing")
+
+	return true
+}
+
+// AdvancedRouting reports whether routing loops can be avoided without using exclusion routes
+func AdvancedRouting() bool {
+	return advancedRoutingSupported
+}
+
+// GetVPNInterfaceName returns the stored VPN interface name
+func GetVPNInterfaceName() string {
+	vpnInitMutex.RLock()
+	defer vpnInitMutex.RUnlock()
+	return vpnInterfaceName
+}
+
+// SetVPNInterfaceName sets the VPN interface name for lazy initialization
+func SetVPNInterfaceName(name string) {
+	vpnInitMutex.Lock()
+	defer vpnInitMutex.Unlock()
+	vpnInterfaceName = name
+
+	if name != "" {
+		log.Infof("VPN interface name set to %s for route exclusion", name)
+	}
+}

client/net/hooks/hooks.go

@@ -0,0 +1,93 @@
+package hooks
+
+import (
+	"net/netip"
+	"slices"
+	"sync"
+
+	"github.com/google/uuid"
+)
+
+// ConnectionID provides a globally unique identifier for network connections.
+// It's used to track connections throughout their lifecycle so the close hook can correlate with the dial hook.
+type ConnectionID string
+
+// GenerateConnID generates a unique identifier for each connection.
+func GenerateConnID() ConnectionID {
+	return ConnectionID(uuid.NewString())
+}
+
+type WriteHookFunc func(connID ConnectionID, prefix netip.Prefix) error
+type CloseHookFunc func(connID ConnectionID) error
+type AddressRemoveHookFunc func(connID ConnectionID, prefix netip.Prefix) error
+
+var (
+	hooksMutex sync.RWMutex
+
+	writeHooks         []WriteHookFunc
+	closeHooks         []CloseHookFunc
+	addressRemoveHooks []AddressRemoveHookFunc
+)
+
+// AddWriteHook allows adding a new hook to be executed before writing/dialing.
+func AddWriteHook(hook WriteHookFunc) {
+	hooksMutex.Lock()
+	defer hooksMutex.Unlock()
+	writeHooks = append(writeHooks, hook)
+}
+
+// AddCloseHook allows adding a new hook to be executed on connection close.
+func AddCloseHook(hook CloseHookFunc) {
+	hooksMutex.Lock()
+	defer hooksMutex.Unlock()
+	closeHooks = append(closeHooks, hook)
+}
+
+// RemoveWriteHooks removes all write hooks.
+func RemoveWriteHooks() {
+	hooksMutex.Lock()
+	defer hooksMutex.Unlock()
+	writeHooks = nil
+}
+
+// RemoveCloseHooks removes all close hooks.
+func RemoveCloseHooks() {
+	hooksMutex.Lock()
+	defer hooksMutex.Unlock()
+	closeHooks = nil
+}
+
+// AddAddressRemoveHook allows adding a new hook to be executed when an address is removed.
+func AddAddressRemoveHook(hook AddressRemoveHookFunc) {
+	hooksMutex.Lock()
+	defer hooksMutex.Unlock()
+	addressRemoveHooks = append(addressRemoveHooks, hook)
+}
+
+// RemoveAddressRemoveHooks removes all listener address hooks.
+func RemoveAddressRemoveHooks() {
+	hooksMutex.Lock()
+	defer hooksMutex.Unlock()
+	addressRemoveHooks = nil
+}
+
+// GetWriteHooks returns a copy of the current write hooks.
+func GetWriteHooks() []WriteHookFunc {
+	hooksMutex.RLock()
+	defer hooksMutex.RUnlock()
+	return slices.Clone(writeHooks)
+}
+
+// GetCloseHooks returns a copy of the current close hooks.
+func GetCloseHooks() []CloseHookFunc {
+	hooksMutex.RLock()
+	defer hooksMutex.RUnlock()
+	return slices.Clone(closeHooks)
+}
+
+// GetAddressRemoveHooks returns a copy of the current listener address remove hooks.
+func GetAddressRemoveHooks() []AddressRemoveHookFunc {
+	hooksMutex.RLock()
+	defer hooksMutex.RUnlock()
+	return slices.Clone(addressRemoveHooks)
+}

client/net/listen.go

@@ -0,0 +1,47 @@
+//go:build !ios
+
+package net
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/pion/transport/v3"
+	log "github.com/sirupsen/logrus"
+)
+
+// ListenUDP listens on the network address and returns a transport.UDPConn
+// which includes support for write and close hooks.
+func ListenUDP(network string, laddr *net.UDPAddr) (transport.UDPConn, error) {
+	if CustomRoutingDisabled() {
+		return net.ListenUDP(network, laddr)
+	}
+
+	conn, err := NewListener().ListenPacket(context.Background(), network, laddr.String())
+	if err != nil {
+		return nil, fmt.Errorf("listen UDP: %w", err)
+	}
+
+	switch c := conn.(type) {
+	case *net.UDPConn:
+		// Advanced routing: plain connection
+		return c, nil
+	case *PacketConn:
+		// Legacy routing: wrapped connection for hooks
+		udpConn, ok := c.PacketConn.(*net.UDPConn)
+		if !ok {
+			if err := c.Close(); err != nil {
+				log.Errorf("Failed to close connection: %v", err)
+			}
+			return nil, fmt.Errorf("expected UDPConn, got %T", c.PacketConn)
+		}
+		return &UDPConn{UDPConn: udpConn, ID: c.ID, seenAddrs: &sync.Map{}}, nil
+	}
+
+	if err := conn.Close(); err != nil {
+		log.Errorf("failed to close connection: %v", err)
+	}
+	return nil, fmt.Errorf("unexpected connection type: %T", conn)
+}

client/net/listen_ios.go

@@ -0,0 +1,11 @@
+//go:build ios
+
+package net
+
+import (
+	"net"
+)
+
+func ListenUDP(network string, laddr *net.UDPAddr) (*net.UDPConn, error) {
+	return net.ListenUDP(network, laddr)
+}

client/net/listener.go

@@ -0,0 +1,19 @@
+package net
+
+import (
+	"net"
+)
+
+// ListenerConfig extends the standard net.ListenConfig with the ability to execute hooks before
+// responding via the socket and after closing. This can be used to bypass the VPN for listeners.
+type ListenerConfig struct {
+	net.ListenConfig
+}
+
+// NewListener creates a new ListenerConfig instance.
+func NewListener() *ListenerConfig {
+	listener := &ListenerConfig{}
+	listener.init()
+
+	return listener
+}

client/net/listener_init_android.go

@@ -0,0 +1,6 @@
+package net
+
+// init configures the net.ListenerConfig Control function to set the fwmark on the socket
+func (l *ListenerConfig) init() {
+	l.ListenConfig.Control = ControlProtectSocket
+}

client/net/listener_init_generic.go

@@ -0,0 +1,7 @@
+//go:build !linux && !windows
+
+package net
+
+func (l *ListenerConfig) init() {
+	// implemented on Linux, Android, and Windows only
+}

client/net/listener_init_linux.go

@@ -0,0 +1,14 @@
+//go:build !android
+
+package net
+
+import (
+	"syscall"
+)
+
+// init configures the net.ListenerConfig Control function to set the fwmark on the socket
+func (l *ListenerConfig) init() {
+	l.ListenConfig.Control = func(_, _ string, c syscall.RawConn) error {
+		return setRawSocketMark(c)
+	}
+}

client/net/listener_init_windows.go

@@ -0,0 +1,8 @@
+package net
+
+func (l *ListenerConfig) init() {
+	// TODO: this will select a single source interface, but for UDP we can have various source interfaces and IP addresses.
+	// For now we stick to the one that matches the request IP address, which can be the unspecified IP. In this case
+	// the interface will be selected that serves the default route.
+	l.ListenConfig.Control = applyUnicastIFToSocket
+}

client/net/listener_listen.go

@@ -0,0 +1,153 @@
+//go:build !ios
+
+package net
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+
+	nberrors "github.com/netbirdio/netbird/client/errors"
+	"github.com/netbirdio/netbird/client/internal/routemanager/util"
+	"github.com/netbirdio/netbird/client/net/hooks"
+)
+
+// ListenPacket listens on the network address and returns a PacketConn
+// which includes support for write hooks.
+func (l *ListenerConfig) ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error) {
+	if CustomRoutingDisabled() || AdvancedRouting() {
+		return l.ListenConfig.ListenPacket(ctx, network, address)
+	}
+
+	pc, err := l.ListenConfig.ListenPacket(ctx, network, address)
+	if err != nil {
+		return nil, fmt.Errorf("listen packet: %w", err)
+	}
+	connID := hooks.GenerateConnID()
+
+	return &PacketConn{PacketConn: pc, ID: connID, seenAddrs: &sync.Map{}}, nil
+}
+
+// PacketConn wraps net.PacketConn to override its WriteTo and Close methods to include hook functionality.
+type PacketConn struct {
+	net.PacketConn
+	ID        hooks.ConnectionID
+	seenAddrs *sync.Map
+}
+
+// WriteTo writes a packet with payload b to addr, executing registered write hooks beforehand.
+func (c *PacketConn) WriteTo(b []byte, addr net.Addr) (n int, err error) {
+	if err := callWriteHooks(c.ID, c.seenAddrs, addr); err != nil {
+		log.Errorf("Failed to call write hooks: %v", err)
+	}
+	return c.PacketConn.WriteTo(b, addr)
+}
+
+// Close overrides the net.PacketConn Close method to execute all registered hooks before closing the connection.
+func (c *PacketConn) Close() error {
+	defer c.seenAddrs.Clear()
+	return closeConn(c.ID, c.PacketConn)
+}
+
+// UDPConn wraps net.UDPConn to override its WriteTo and Close methods to include hook functionality.
+type UDPConn struct {
+	*net.UDPConn
+	ID        hooks.ConnectionID
+	seenAddrs *sync.Map
+}
+
+// WriteTo writes a packet with payload b to addr, executing registered write hooks beforehand.
+func (c *UDPConn) WriteTo(b []byte, addr net.Addr) (n int, err error) {
+	if err := callWriteHooks(c.ID, c.seenAddrs, addr); err != nil {
+		log.Errorf("Failed to call write hooks: %v", err)
+	}
+	return c.UDPConn.WriteTo(b, addr)
+}
+
+// Close overrides the net.UDPConn Close method to execute all registered hooks before closing the connection.
+func (c *UDPConn) Close() error {
+	defer c.seenAddrs.Clear()
+	return closeConn(c.ID, c.UDPConn)
+}
+
+// RemoveAddress removes an address from the seen cache and triggers removal hooks.
+func (c *PacketConn) RemoveAddress(addr string) {
+	if _, exists := c.seenAddrs.LoadAndDelete(addr); !exists {
+		return
+	}
+
+	ipStr, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		log.Errorf("Error splitting IP address and port: %v", err)
+		return
+	}
+
+	ipAddr, err := netip.ParseAddr(ipStr)
+	if err != nil {
+		log.Errorf("Error parsing IP address %s: %v", ipStr, err)
+		return
+	}
+
+	prefix := netip.PrefixFrom(ipAddr.Unmap(), ipAddr.BitLen())
+
+	addressRemoveHooks := hooks.GetAddressRemoveHooks()
+	if len(addressRemoveHooks) == 0 {
+		return
+	}
+
+	for _, hook := range addressRemoveHooks {
+		if err := hook(c.ID, prefix); err != nil {
+			log.Errorf("Error executing listener address remove hook: %v", err)
+		}
+	}
+}
+
+// WrapPacketConn wraps an existing net.PacketConn with nbnet hook functionality
+func WrapPacketConn(conn net.PacketConn) net.PacketConn {
+	if AdvancedRouting() {
+		// hooks not required for advanced routing
+		return conn
+	}
+	return &PacketConn{
+		PacketConn: conn,
+		ID:         hooks.GenerateConnID(),
+		seenAddrs:  &sync.Map{},
+	}
+}
+
+func callWriteHooks(id hooks.ConnectionID, seenAddrs *sync.Map, addr net.Addr) error {
+	if _, loaded := seenAddrs.LoadOrStore(addr.String(), true); loaded {
+		return nil
+	}
+
+	writeHooks := hooks.GetWriteHooks()
+	if len(writeHooks) == 0 {
+		return nil
+	}
+
+	udpAddr, ok := addr.(*net.UDPAddr)
+	if !ok {
+		return fmt.Errorf("expected *net.UDPAddr for packet connection, got %T", addr)
+	}
+
+	prefix, err := util.GetPrefixFromIP(udpAddr.IP)
+	if err != nil {
+		return fmt.Errorf("convert UDP IP %s to prefix: %w", udpAddr.IP, err)
+	}
+
+	log.Debugf("Listener resolved IP for %s: %s", addr, prefix)
+
+	var merr *multierror.Error
+	for _, hook := range writeHooks {
+		if err := hook(id, prefix); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("execute write hook: %w", err))
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}

client/net/listener_listen_ios.go

@@ -0,0 +1,10 @@
+package net
+
+import (
+	"net"
+)
+
+// WrapPacketConn on iOS just returns the original connection since iOS handles its own networking
+func WrapPacketConn(conn *net.UDPConn) *net.UDPConn {
+	return conn
+}

client/net/net.go

@@ -0,0 +1,69 @@
+package net
+
+import (
+	"fmt"
+	"math/big"
+	"net"
+	"net/netip"
+)
+
+const (
+	// ControlPlaneMark is the fwmark value used to mark packets that should not be routed through the NetBird interface to
+	// avoid routing loops.
+	// This includes all control plane traffic (mgmt, signal, flows), relay, ICE/stun/turn and everything that is emitted by the wireguard socket.
+	// It doesn't collide with the other marks, as the others are used for data plane traffic only.
+	ControlPlaneMark = 0x1BD00
+
+	// Data plane marks (0x1BD10 - 0x1BDFF)
+
+	// DataPlaneMarkLower is the lowest value for the data plane range
+	DataPlaneMarkLower = 0x1BD10
+	// DataPlaneMarkUpper is the highest value for the data plane range
+	DataPlaneMarkUpper = 0x1BDFF
+
+	// DataPlaneMarkIn is the mark for inbound data plane traffic.
+	DataPlaneMarkIn = 0x1BD10
+
+	// DataPlaneMarkOut is the mark for outbound data plane traffic.
+	DataPlaneMarkOut = 0x1BD11
+
+	// PreroutingFwmarkRedirected is applied to packets that are were redirected (input -> forward, e.g. by Docker or Podman) for special handling.
+	PreroutingFwmarkRedirected = 0x1BD20
+
+	// PreroutingFwmarkMasquerade is applied to packets that arrive from the NetBird interface and should be masqueraded.
+	PreroutingFwmarkMasquerade = 0x1BD21
+
+	// PreroutingFwmarkMasqueradeReturn is applied to packets that will leave through the NetBird interface and should be masqueraded.
+	PreroutingFwmarkMasqueradeReturn = 0x1BD22
+)
+
+// IsDataPlaneMark determines if a fwmark is in the data plane range (0x1BD10-0x1BDFF)
+func IsDataPlaneMark(fwmark uint32) bool {
+	return fwmark >= DataPlaneMarkLower && fwmark <= DataPlaneMarkUpper
+}
+
+func GetLastIPFromNetwork(network netip.Prefix, fromEnd int) (netip.Addr, error) {
+	var endIP net.IP
+	addr := network.Addr().AsSlice()
+	mask := net.CIDRMask(network.Bits(), len(addr)*8)
+
+	for i := 0; i < len(addr); i++ {
+		endIP = append(endIP, addr[i]|^mask[i])
+	}
+
+	// convert to big.Int
+	endInt := big.NewInt(0)
+	endInt.SetBytes(endIP)
+
+	// subtract fromEnd from the last ip
+	fromEndBig := big.NewInt(int64(fromEnd))
+	resultInt := big.NewInt(0)
+	resultInt.Sub(endInt, fromEndBig)
+
+	ip, ok := netip.AddrFromSlice(resultInt.Bytes())
+	if !ok {
+		return netip.Addr{}, fmt.Errorf("invalid IP address from network %s", network)
+	}
+
+	return ip.Unmap(), nil
+}

client/net/net_linux.go

@@ -0,0 +1,55 @@
+//go:build !android
+
+package net
+
+import (
+	"fmt"
+	"syscall"
+)
+
+// SetSocketMark sets the SO_MARK option on the given socket connection
+func SetSocketMark(conn syscall.Conn) error {
+	if !AdvancedRouting() {
+		return nil
+	}
+
+	sysconn, err := conn.SyscallConn()
+	if err != nil {
+		return fmt.Errorf("get raw conn: %w", err)
+	}
+
+	return setRawSocketMark(sysconn)
+}
+
+// SetSocketOpt sets the SO_MARK option on the given file descriptor
+func SetSocketOpt(fd int) error {
+	if !AdvancedRouting() {
+		return nil
+	}
+
+	return setSocketOptInt(fd)
+}
+
+func setRawSocketMark(conn syscall.RawConn) error {
+	var setErr error
+
+	err := conn.Control(func(fd uintptr) {
+		if !AdvancedRouting() {
+			return
+		}
+		setErr = setSocketOptInt(int(fd))
+	})
+	if err != nil {
+		return fmt.Errorf("control: %w", err)
+	}
+
+	if setErr != nil {
+		return fmt.Errorf("set SO_MARK: %w", setErr)
+	}
+
+	return nil
+}
+
+func setSocketOptInt(fd int) error {
+	return syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, ControlPlaneMark)
+}

client/net/net_test.go

@@ -0,0 +1,94 @@
+package net
+
+import (
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetLastIPFromNetwork(t *testing.T) {
+	tests := []struct {
+		name      string
+		network   string
+		fromEnd   int
+		expected  string
+		expectErr bool
+	}{
+		{
+			name:     "IPv4 /24 network - last IP (fromEnd=0)",
+			network:  "192.168.1.0/24",
+			fromEnd:  0,
+			expected: "192.168.1.255",
+		},
+		{
+			name:     "IPv4 /24 network - fromEnd=1",
+			network:  "192.168.1.0/24",
+			fromEnd:  1,
+			expected: "192.168.1.254",
+		},
+		{
+			name:     "IPv4 /24 network - fromEnd=5",
+			network:  "192.168.1.0/24",
+			fromEnd:  5,
+			expected: "192.168.1.250",
+		},
+		{
+			name:     "IPv4 /16 network - last IP",
+			network:  "10.0.0.0/16",
+			fromEnd:  0,
+			expected: "10.0.255.255",
+		},
+		{
+			name:     "IPv4 /16 network - fromEnd=256",
+			network:  "10.0.0.0/16",
+			fromEnd:  256,
+			expected: "10.0.254.255",
+		},
+		{
+			name:     "IPv4 /32 network - single host",
+			network:  "192.168.1.100/32",
+			fromEnd:  0,
+			expected: "192.168.1.100",
+		},
+		{
+			name:     "IPv6 /64 network - last IP",
+			network:  "2001:db8::/64",
+			fromEnd:  0,
+			expected: "2001:db8::ffff:ffff:ffff:ffff",
+		},
+		{
+			name:     "IPv6 /64 network - fromEnd=1",
+			network:  "2001:db8::/64",
+			fromEnd:  1,
+			expected: "2001:db8::ffff:ffff:ffff:fffe",
+		},
+		{
+			name:     "IPv6 /128 network - single host",
+			network:  "2001:db8::1/128",
+			fromEnd:  0,
+			expected: "2001:db8::1",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			network, err := netip.ParsePrefix(tt.network)
+			require.NoError(t, err, "Failed to parse network prefix")
+
+			result, err := GetLastIPFromNetwork(network, tt.fromEnd)
+
+			if tt.expectErr {
+				assert.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			expectedIP, err := netip.ParseAddr(tt.expected)
+			require.NoError(t, err, "Failed to parse expected IP")
+
+			assert.Equal(t, expectedIP, result, "IP mismatch for network %s with fromEnd=%d", tt.network, tt.fromEnd)
+		})
+	}
+}

client/net/net_windows.go

@@ -0,0 +1,284 @@
+package net
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+	"unsafe"
+
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sys/windows"
+)
+
+const (
+	// https://learn.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options
+	IpUnicastIf   = 31
+	Ipv6UnicastIf = 31
+
+	// https://learn.microsoft.com/en-us/windows/win32/winsock/ipproto-ipv6-socket-options
+	Ipv6V6only = 27
+)
+
+// GetBestInterfaceFunc is set at runtime to avoid import cycle
+var GetBestInterfaceFunc func(dest netip.Addr, vpnIntf string) (*net.Interface, error)
+
+// nativeToBigEndian converts a uint32 from native byte order to big-endian
+func nativeToBigEndian(v uint32) uint32 {
+	return (v&0xff)<<24 | (v&0xff00)<<8 | (v&0xff0000)>>8 | (v&0xff000000)>>24
+}
+
+// parseDestinationAddress parses the destination address from various formats
+func parseDestinationAddress(network, address string) (netip.Addr, error) {
+	if address == "" {
+		if strings.HasSuffix(network, "6") {
+			return netip.IPv6Unspecified(), nil
+		}
+		return netip.IPv4Unspecified(), nil
+	}
+
+	if addrPort, err := netip.ParseAddrPort(address); err == nil {
+		return addrPort.Addr(), nil
+	}
+
+	if dest, err := netip.ParseAddr(address); err == nil {
+		return dest, nil
+	}
+
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		// No port, treat whole string as host
+		host = address
+	}
+
+	if host == "" {
+		if strings.HasSuffix(network, "6") {
+			return netip.IPv6Unspecified(), nil
+		}
+		return netip.IPv4Unspecified(), nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
+	if err != nil || len(ips) == 0 {
+		return netip.Addr{}, fmt.Errorf("resolve destination %s: %w", host, err)
+	}
+
+	dest, ok := netip.AddrFromSlice(ips[0].IP)
+	if !ok {
+		return netip.Addr{}, fmt.Errorf("convert IP %v to netip.Addr", ips[0].IP)
+	}
+
+	if ips[0].Zone != "" {
+		dest = dest.WithZone(ips[0].Zone)
+	}
+
+	return dest, nil
+}
+
+func getInterfaceFromZone(zone string) *net.Interface {
+	if zone == "" {
+		return nil
+	}
+
+	idx, err := strconv.Atoi(zone)
+	if err != nil {
+		log.Debugf("invalid zone format for Windows (expected numeric): %s", zone)
+		return nil
+	}
+
+	iface, err := net.InterfaceByIndex(idx)
+	if err != nil {
+		log.Debugf("failed to get interface by index %d from zone: %v", idx, err)
+		return nil
+	}
+
+	return iface
+}
+
+type interfaceSelection struct {
+	iface4 *net.Interface
+	iface6 *net.Interface
+}
+
+func selectInterfaceForZone(dest netip.Addr, zone string) *interfaceSelection {
+	iface := getInterfaceFromZone(zone)
+	if iface == nil {
+		return nil
+	}
+
+	if dest.Is6() {
+		return &interfaceSelection{iface6: iface}
+	}
+	return &interfaceSelection{iface4: iface}
+}
+
+func selectInterfaceForUnspecified() (*interfaceSelection, error) {
+	if GetBestInterfaceFunc == nil {
+		return nil, errors.New("GetBestInterfaceFunc not initialized")
+	}
+
+	var result interfaceSelection
+	vpnIfaceName := GetVPNInterfaceName()
+
+	if iface4, err := GetBestInterfaceFunc(netip.IPv4Unspecified(), vpnIfaceName); err == nil {
+		result.iface4 = iface4
+	} else {
+		log.Debugf("No IPv4 default route found: %v", err)
+	}
+
+	if iface6, err := GetBestInterfaceFunc(netip.IPv6Unspecified(), vpnIfaceName); err == nil {
+		result.iface6 = iface6
+	} else {
+		log.Debugf("No IPv6 default route found: %v", err)
+	}
+
+	if result.iface4 == nil && result.iface6 == nil {
+		return nil, errors.New("no default routes found")
+	}
+
+	return &result, nil
+}
+
+func selectInterface(dest netip.Addr) (*interfaceSelection, error) {
+	if zone := dest.Zone(); zone != "" {
+		if selection := selectInterfaceForZone(dest, zone); selection != nil {
+			return selection, nil
+		}
+	}
+
+	if dest.IsUnspecified() {
+		return selectInterfaceForUnspecified()
+	}
+
+	if GetBestInterfaceFunc == nil {
+		return nil, errors.New("GetBestInterfaceFunc not initialized")
+	}
+
+	iface, err := GetBestInterfaceFunc(dest, GetVPNInterfaceName())
+	if err != nil {
+		return nil, fmt.Errorf("find route for %s: %w", dest, err)
+	}
+
+	if dest.Is6() {
+		return &interfaceSelection{iface6: iface}, nil
+	}
+	return &interfaceSelection{iface4: iface}, nil
+}
+
+func setIPv4UnicastIF(fd uintptr, iface *net.Interface) error {
+	ifaceIndexBE := nativeToBigEndian(uint32(iface.Index))
+	if err := windows.SetsockoptInt(windows.Handle(fd), windows.IPPROTO_IP, IpUnicastIf, int(ifaceIndexBE)); err != nil {
+		return fmt.Errorf("set IP_UNICAST_IF: %w (interface: %s, index: %d)", err, iface.Name, iface.Index)
+	}
+	return nil
+}
+
+func setIPv6UnicastIF(fd uintptr, iface *net.Interface) error {
+	if err := windows.SetsockoptInt(windows.Handle(fd), windows.IPPROTO_IPV6, Ipv6UnicastIf, iface.Index); err != nil {
+		return fmt.Errorf("set IPV6_UNICAST_IF: %w (interface: %s, index: %d)", err, iface.Name, iface.Index)
+	}
+	return nil
+}
+
+func setUnicastIf(fd uintptr, network string, selection *interfaceSelection, address string) error {
+	// The Go runtime always passes specific network types to Control (udp4, udp6, tcp4, tcp6, etc.)
+	// Never generic ones (udp, tcp, ip)
+
+	switch {
+	case strings.HasSuffix(network, "4"):
+		// IPv4-only socket (udp4, tcp4, ip4)
+		return setUnicastIfIPv4(fd, network, selection, address)
+
+	case strings.HasSuffix(network, "6"):
+		// IPv6 socket (udp6, tcp6, ip6) - could be dual-stack or IPv6-only
+		return setUnicastIfIPv6(fd, network, selection, address)
+	}
+
+	// Shouldn't reach here based on Go's documented behavior
+	return fmt.Errorf("unexpected network type: %s", network)
+}
+
+func setUnicastIfIPv4(fd uintptr, network string, selection *interfaceSelection, address string) error {
+	if selection.iface4 == nil {
+		return nil
+	}
+
+	if err := setIPv4UnicastIF(fd, selection.iface4); err != nil {
+		return err
+	}
+
+	log.Debugf("Set IP_UNICAST_IF=%d on %s for %s to %s", selection.iface4.Index, selection.iface4.Name, network, address)
+	return nil
+}
+
+func setUnicastIfIPv6(fd uintptr, network string, selection *interfaceSelection, address string) error {
+	isDualStack := checkDualStack(fd)
+
+	// For dual-stack sockets, also set the IPv4 option
+	if isDualStack && selection.iface4 != nil {
+		if err := setIPv4UnicastIF(fd, selection.iface4); err != nil {
+			return err
+		}
+		log.Debugf("Set IP_UNICAST_IF=%d on %s for %s to %s (dual-stack)", selection.iface4.Index, selection.iface4.Name, network, address)
+	}
+
+	if selection.iface6 == nil {
+		return nil
+	}
+
+	if err := setIPv6UnicastIF(fd, selection.iface6); err != nil {
+		return err
+	}
+
+	log.Debugf("Set IPV6_UNICAST_IF=%d on %s for %s to %s", selection.iface6.Index, selection.iface6.Name, network, address)
+	return nil
+}
+
+func checkDualStack(fd uintptr) bool {
+	var v6Only int
+	v6OnlyLen := int32(unsafe.Sizeof(v6Only))
+	err := windows.Getsockopt(windows.Handle(fd), windows.IPPROTO_IPV6, Ipv6V6only, (*byte)(unsafe.Pointer(&v6Only)), &v6OnlyLen)
+	return err == nil && v6Only == 0
+}
+
+// applyUnicastIFToSocket applies IpUnicastIf to a socket based on the destination address
+func applyUnicastIFToSocket(network string, address string, c syscall.RawConn) error {
+	if !AdvancedRouting() {
+		return nil
+	}
+
+	dest, err := parseDestinationAddress(network, address)
+	if err != nil {
+		return err
+	}
+
+	dest = dest.Unmap()
+
+	if !dest.IsValid() {
+		return fmt.Errorf("invalid destination address for %s", address)
+	}
+
+	selection, err := selectInterface(dest)
+	if err != nil {
+		return err
+	}
+
+	var controlErr error
+	err = c.Control(func(fd uintptr) {
+		controlErr = setUnicastIf(fd, network, selection, address)
+	})
+
+	if err != nil {
+		return fmt.Errorf("control: %w", err)
+	}
+
+	return controlErr
+}

client/net/protectsocket_android.go

@@ -0,0 +1,47 @@
+package net
+
+import (
+	"fmt"
+	"sync"
+	"syscall"
+
+	"github.com/netbirdio/netbird/client/iface/netstack"
+)
+
+var (
+	androidProtectSocketLock sync.Mutex
+	androidProtectSocket     func(fd int32) bool
+)
+
+func SetAndroidProtectSocketFn(fn func(fd int32) bool) {
+	androidProtectSocketLock.Lock()
+	androidProtectSocket = fn
+	androidProtectSocketLock.Unlock()
+}
+
+// ControlProtectSocket is a Control function that sets the fwmark on the socket
+func ControlProtectSocket(_, _ string, c syscall.RawConn) error {
+	if netstack.IsEnabled() {
+		return nil
+	}
+	var aErr error
+	err := c.Control(func(fd uintptr) {
+		androidProtectSocketLock.Lock()
+		defer androidProtectSocketLock.Unlock()
+
+		if androidProtectSocket == nil {
+			aErr = fmt.Errorf("socket protection function not set")
+			return
+		}
+
+		if !androidProtectSocket(int32(fd)) {
+			aErr = fmt.Errorf("failed to protect socket via Android")
+		}
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return aErr
+}