[client] Replace exclusion routes with scoped default + IP_BOUND_IF on macOS (#5918)

client/internal/routemanager/systemops/systemops_bsd_other.go

@@ -0,0 +1,10 @@
+//go:build (dragonfly || freebsd || netbsd || openbsd) && !darwin
+
+package systemops
+
+// Non-darwin BSDs don't support the IP_BOUND_IF + scoped default model. They
+// always fall through to the ref-counter exclusion-route path; these stubs
+// exist only so systemops_unix.go compiles.
+func (r *SysOps) setupAdvancedRouting() error   { return nil }
+func (r *SysOps) cleanupAdvancedRouting() error { return nil }
+func (r *SysOps) flushPlatformExtras() error    { return nil }

client/internal/routemanager/systemops/systemops_darwin.go

@@ -0,0 +1,241 @@
+//go:build darwin && !ios
+
+package systemops
+
+import (
+	"errors"
+	"fmt"
+	"net/netip"
+	"os"
+	"time"
+
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/route"
+	"golang.org/x/sys/unix"
+
+	nberrors "github.com/netbirdio/netbird/client/errors"
+	"github.com/netbirdio/netbird/client/internal/routemanager/vars"
+	nbnet "github.com/netbirdio/netbird/client/net"
+)
+
+// scopedRouteBudget bounds retries for the scoped default route. Installing or
+// deleting it matters enough that we're willing to spend longer waiting for the
+// kernel reply than for per-prefix exclusion routes.
+const scopedRouteBudget = 5 * time.Second
+
+// setupAdvancedRouting installs an RTF_IFSCOPE default route per address family
+// pinned to the current physical egress, so IP_BOUND_IF scoped lookups can
+// resolve gateway'd destinations while the VPN's split default owns the
+// unscoped table.
+//
+// Timing note: this runs during routeManager.Init, which happens before the
+// VPN interface is created and before any peer routes propagate. The initial
+// mgmt / signal / relay TCP dials always fire before this runs, so those
+// sockets miss the IP_BOUND_IF binding and rely on the kernel's normal route
+// lookup, which at that point correctly picks the physical default. Those
+// already-established TCP flows keep their originally-selected interface for
+// their lifetime on Darwin because the kernel caches the egress route
+// per-socket at connect time; adding the VPN's 0/1 + 128/1 split default
+// afterwards does not migrate them since the original en0 default stays in
+// the table. Any subsequent reconnect via nbnet.NewDialer picks up the
+// populated bound-iface cache and gets IP_BOUND_IF set cleanly.
+func (r *SysOps) setupAdvancedRouting() error {
+	// Drop any previously-cached egress interface before reinstalling. On a
+	// refresh, a family that no longer resolves would otherwise keep the stale
+	// binding, causing new sockets to scope to an interface without a matching
+	// scoped default.
+	nbnet.ClearBoundInterfaces()
+
+	if err := r.flushScopedDefaults(); err != nil {
+		log.Warnf("flush residual scoped defaults: %v", err)
+	}
+
+	var merr *multierror.Error
+	installed := 0
+
+	for _, unspec := range []netip.Addr{netip.IPv4Unspecified(), netip.IPv6Unspecified()} {
+		ok, err := r.installScopedDefaultFor(unspec)
+		if err != nil {
+			merr = multierror.Append(merr, err)
+			continue
+		}
+		if ok {
+			installed++
+		}
+	}
+
+	if installed == 0 && merr != nil {
+		return nberrors.FormatErrorOrNil(merr)
+	}
+	if merr != nil {
+		log.Warnf("advanced routing setup partially succeeded: %v", nberrors.FormatErrorOrNil(merr))
+	}
+	return nil
+}
+
+// installScopedDefaultFor resolves the physical default nexthop for the given
+// address family, installs a scoped default via it, and caches the iface for
+// subsequent IP_BOUND_IF / IPV6_BOUND_IF socket binds.
+func (r *SysOps) installScopedDefaultFor(unspec netip.Addr) (bool, error) {
+	nexthop, err := GetNextHop(unspec)
+	if err != nil {
+		if errors.Is(err, vars.ErrRouteNotFound) {
+			return false, nil
+		}
+		return false, fmt.Errorf("get default nexthop for %s: %w", unspec, err)
+	}
+	if nexthop.Intf == nil {
+		return false, fmt.Errorf("unusable default nexthop for %s (no interface)", unspec)
+	}
+
+	if err := r.addScopedDefault(unspec, nexthop); err != nil {
+		return false, fmt.Errorf("add scoped default on %s: %w", nexthop.Intf.Name, err)
+	}
+
+	af := unix.AF_INET
+	if unspec.Is6() {
+		af = unix.AF_INET6
+	}
+	nbnet.SetBoundInterface(af, nexthop.Intf)
+	via := "point-to-point"
+	if nexthop.IP.IsValid() {
+		via = nexthop.IP.String()
+	}
+	log.Infof("installed scoped default route via %s on %s for %s", via, nexthop.Intf.Name, afOf(unspec))
+	return true, nil
+}
+
+func (r *SysOps) cleanupAdvancedRouting() error {
+	nbnet.ClearBoundInterfaces()
+	return r.flushScopedDefaults()
+}
+
+// flushPlatformExtras runs darwin-specific residual cleanup hooked into the
+// generic FlushMarkedRoutes path, so a crashed daemon's scoped defaults get
+// removed on the next boot regardless of whether a profile is brought up.
+func (r *SysOps) flushPlatformExtras() error {
+	return r.flushScopedDefaults()
+}
+
+// flushScopedDefaults removes any scoped default routes tagged with routeProtoFlag.
+// Safe to call at startup to clear residual entries from a prior session.
+func (r *SysOps) flushScopedDefaults() error {
+	rib, err := retryFetchRIB()
+	if err != nil {
+		return fmt.Errorf("fetch routing table: %w", err)
+	}
+
+	msgs, err := route.ParseRIB(route.RIBTypeRoute, rib)
+	if err != nil {
+		return fmt.Errorf("parse routing table: %w", err)
+	}
+
+	var merr *multierror.Error
+	removed := 0
+
+	for _, msg := range msgs {
+		rtMsg, ok := msg.(*route.RouteMessage)
+		if !ok {
+			continue
+		}
+		if rtMsg.Flags&routeProtoFlag == 0 {
+			continue
+		}
+		if rtMsg.Flags&unix.RTF_IFSCOPE == 0 {
+			continue
+		}
+
+		info, err := MsgToRoute(rtMsg)
+		if err != nil {
+			log.Debugf("skip scoped flush: %v", err)
+			continue
+		}
+		if !info.Dst.IsValid() || info.Dst.Bits() != 0 {
+			continue
+		}
+
+		if err := r.deleteScopedRoute(rtMsg); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("delete scoped default %s on index %d: %w",
+				info.Dst, rtMsg.Index, err))
+			continue
+		}
+		removed++
+		log.Debugf("flushed residual scoped default %s on index %d", info.Dst, rtMsg.Index)
+	}
+
+	if removed > 0 {
+		log.Infof("flushed %d residual scoped default route(s)", removed)
+	}
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+func (r *SysOps) addScopedDefault(unspec netip.Addr, nexthop Nexthop) error {
+	return r.scopedRouteSocket(unix.RTM_ADD, unspec, nexthop)
+}
+
+func (r *SysOps) deleteScopedRoute(rtMsg *route.RouteMessage) error {
+	// Preserve identifying flags from the stored route (including RTF_GATEWAY
+	// only if present); kernel-set bits like RTF_DONE don't belong on RTM_DELETE.
+	keep := unix.RTF_UP | unix.RTF_STATIC | unix.RTF_GATEWAY | unix.RTF_IFSCOPE | routeProtoFlag
+	del := &route.RouteMessage{
+		Type:    unix.RTM_DELETE,
+		Flags:   rtMsg.Flags & keep,
+		Version: unix.RTM_VERSION,
+		Seq:     r.getSeq(),
+		Index:   rtMsg.Index,
+		Addrs:   rtMsg.Addrs,
+	}
+	return r.writeRouteMessage(del, scopedRouteBudget)
+}
+
+func (r *SysOps) scopedRouteSocket(action int, unspec netip.Addr, nexthop Nexthop) error {
+	flags := unix.RTF_UP | unix.RTF_STATIC | unix.RTF_IFSCOPE | routeProtoFlag
+
+	msg := &route.RouteMessage{
+		Type:    action,
+		Flags:   flags,
+		Version: unix.RTM_VERSION,
+		ID:      uintptr(os.Getpid()),
+		Seq:     r.getSeq(),
+		Index:   nexthop.Intf.Index,
+	}
+
+	const numAddrs = unix.RTAX_NETMASK + 1
+	addrs := make([]route.Addr, numAddrs)
+
+	dst, err := addrToRouteAddr(unspec)
+	if err != nil {
+		return fmt.Errorf("build destination: %w", err)
+	}
+	mask, err := prefixToRouteNetmask(netip.PrefixFrom(unspec, 0))
+	if err != nil {
+		return fmt.Errorf("build netmask: %w", err)
+	}
+	addrs[unix.RTAX_DST] = dst
+	addrs[unix.RTAX_NETMASK] = mask
+
+	if nexthop.IP.IsValid() {
+		msg.Flags |= unix.RTF_GATEWAY
+		gw, err := addrToRouteAddr(nexthop.IP.Unmap())
+		if err != nil {
+			return fmt.Errorf("build gateway: %w", err)
+		}
+		addrs[unix.RTAX_GATEWAY] = gw
+	} else {
+		addrs[unix.RTAX_GATEWAY] = &route.LinkAddr{
+			Index: nexthop.Intf.Index,
+			Name:  nexthop.Intf.Name,
+		}
+	}
+	msg.Addrs = addrs
+
+	return r.writeRouteMessage(msg, scopedRouteBudget)
+}
+
+func afOf(a netip.Addr) string {
+	if a.Is4() {
+		return "IPv4"
+	}
+	return "IPv6"
+}

client/internal/routemanager/systemops/systemops_generic.go

@@ -21,6 +21,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/routemanager/util"
 	"github.com/netbirdio/netbird/client/internal/routemanager/vars"
 	"github.com/netbirdio/netbird/client/internal/statemanager"
+	nbnet "github.com/netbirdio/netbird/client/net"
 	"github.com/netbirdio/netbird/client/net/hooks"
 )
 
@@ -31,8 +32,6 @@ var splitDefaultv4_2 = netip.PrefixFrom(netip.AddrFrom4([4]byte{128}), 1)
 var splitDefaultv6_1 = netip.PrefixFrom(netip.IPv6Unspecified(), 1)
 var splitDefaultv6_2 = netip.PrefixFrom(netip.AddrFrom16([16]byte{0x80}), 1)
 
-var ErrRoutingIsSeparate = errors.New("routing is separate")
-
 func (r *SysOps) setupRefCounter(initAddresses []net.IP, stateManager *statemanager.Manager) error {
 	stateManager.RegisterState(&ShutdownState{})
 
@@ -397,12 +396,16 @@ func ipToAddr(ip net.IP, intf *net.Interface) (netip.Addr, error) {
 }
 
 // IsAddrRouted checks if the candidate address would route to the vpn, in which case it returns true and the matched prefix.
+// When advanced routing is active the WG socket is bound to the physical interface (fwmark on linux,
+// IP_UNICAST_IF on windows, IP_BOUND_IF on darwin) and bypasses the main routing table, so the check is skipped.
 func IsAddrRouted(addr netip.Addr, vpnRoutes []netip.Prefix) (bool, netip.Prefix) {
-	localRoutes, err := hasSeparateRouting()
+	if nbnet.AdvancedRouting() {
+		return false, netip.Prefix{}
+	}
+
+	localRoutes, err := GetRoutesFromTable()
 	if err != nil {
-		if !errors.Is(err, ErrRoutingIsSeparate) {
-			log.Errorf("Failed to get routes: %v", err)
-		}
+		log.Errorf("Failed to get routes: %v", err)
 		return false, netip.Prefix{}
 	}
 

client/internal/routemanager/systemops/systemops_js.go

@@ -22,10 +22,6 @@ func GetRoutesFromTable() ([]netip.Prefix, error) {
 	return []netip.Prefix{}, nil
 }
 
-func hasSeparateRouting() ([]netip.Prefix, error) {
-	return []netip.Prefix{}, nil
-}
-
 // GetDetailedRoutesFromTable returns empty routes for WASM.
 func GetDetailedRoutesFromTable() ([]DetailedRoute, error) {
 	return []DetailedRoute{}, nil

client/internal/routemanager/systemops/systemops_linux.go

@@ -894,13 +894,6 @@ func getAddressFamily(prefix netip.Prefix) int {
 	return netlink.FAMILY_V6
 }
 
-func hasSeparateRouting() ([]netip.Prefix, error) {
-	if !nbnet.AdvancedRouting() {
-		return GetRoutesFromTable()
-	}
-	return nil, ErrRoutingIsSeparate
-}
-
 func isOpErr(err error) bool {
 	// EAFTNOSUPPORT when ipv6 is disabled via sysctl, EOPNOTSUPP when disabled in boot options or otherwise not supported
 	if errors.Is(err, syscall.EAFNOSUPPORT) || errors.Is(err, syscall.EOPNOTSUPP) {

client/internal/routemanager/systemops/systemops_nonlinux.go

@@ -48,10 +48,6 @@ func EnableIPForwarding() error {
 	return nil
 }
 
-func hasSeparateRouting() ([]netip.Prefix, error) {
-	return GetRoutesFromTable()
-}
-
 // GetIPRules returns IP rules for debugging (not supported on non-Linux platforms)
 func GetIPRules() ([]IPRule, error) {
 	log.Infof("IP rules collection is not supported on %s", runtime.GOOS)

client/internal/routemanager/systemops/systemops_unix.go

@@ -25,6 +25,9 @@ import (
 
 const (
 	envRouteProtoFlag = "NB_ROUTE_PROTO_FLAG"
+
+	// routeBudget bounds retries for per-prefix exclusion route programming.
+	routeBudget = 1 * time.Second
 )
 
 var routeProtoFlag int
@@ -41,26 +44,42 @@ func init() {
 }
 
 func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager.Manager, advancedRouting bool) error {
+	if advancedRouting {
+		return r.setupAdvancedRouting()
+	}
+
+	log.Infof("Using legacy routing setup with ref counters")
 	return r.setupRefCounter(initAddresses, stateManager)
 }
 
 func (r *SysOps) CleanupRouting(stateManager *statemanager.Manager, advancedRouting bool) error {
+	if advancedRouting {
+		return r.cleanupAdvancedRouting()
+	}
+
 	return r.cleanupRefCounter(stateManager)
 }
 
 // FlushMarkedRoutes removes single IP exclusion routes marked with the configured RTF_PROTO flag.
+// On darwin it also flushes residual RTF_IFSCOPE scoped default routes so a
+// crashed prior session can't leave crud in the table.
 func (r *SysOps) FlushMarkedRoutes() error {
+	var merr *multierror.Error
+
+	if err := r.flushPlatformExtras(); err != nil {
+		merr = multierror.Append(merr, fmt.Errorf("flush platform extras: %w", err))
+	}
+
 	rib, err := retryFetchRIB()
 	if err != nil {
-		return fmt.Errorf("fetch routing table: %w", err)
+		return nberrors.FormatErrorOrNil(multierror.Append(merr, fmt.Errorf("fetch routing table: %w", err)))
 	}
 
 	msgs, err := route.ParseRIB(route.RIBTypeRoute, rib)
 	if err != nil {
-		return fmt.Errorf("parse routing table: %w", err)
+		return nberrors.FormatErrorOrNil(multierror.Append(merr, fmt.Errorf("parse routing table: %w", err)))
 	}
 
-	var merr *multierror.Error
 	flushedCount := 0
 
 	for _, msg := range msgs {
@@ -117,12 +136,12 @@ func (r *SysOps) routeSocket(action int, prefix netip.Prefix, nexthop Nexthop) e
 		return fmt.Errorf("invalid prefix: %s", prefix)
 	}
 
-	expBackOff := backoff.NewExponentialBackOff()
-	expBackOff.InitialInterval = 50 * time.Millisecond
-	expBackOff.MaxInterval = 500 * time.Millisecond
-	expBackOff.MaxElapsedTime = 1 * time.Second
+	msg, err := r.buildRouteMessage(action, prefix, nexthop)
+	if err != nil {
+		return fmt.Errorf("build route message: %w", err)
+	}
 
-	if err := backoff.Retry(r.routeOp(action, prefix, nexthop), expBackOff); err != nil {
+	if err := r.writeRouteMessage(msg, routeBudget); err != nil {
 		a := "add"
 		if action == unix.RTM_DELETE {
 			a = "remove"
@@ -132,50 +151,91 @@ func (r *SysOps) routeSocket(action int, prefix netip.Prefix, nexthop Nexthop) e
 	return nil
 }
 
-func (r *SysOps) routeOp(action int, prefix netip.Prefix, nexthop Nexthop) func() error {
-	operation := func() error {
-		fd, err := unix.Socket(syscall.AF_ROUTE, syscall.SOCK_RAW, syscall.AF_UNSPEC)
-		if err != nil {
-			return fmt.Errorf("open routing socket: %w", err)
+// writeRouteMessage sends a route message over AF_ROUTE and waits for the
+// kernel's matching reply, retrying transient failures until budget elapses.
+// Callers do not need to manage sockets or seq numbers themselves.
+func (r *SysOps) writeRouteMessage(msg *route.RouteMessage, budget time.Duration) error {
+	expBackOff := backoff.NewExponentialBackOff()
+	expBackOff.InitialInterval = 50 * time.Millisecond
+	expBackOff.MaxInterval = 500 * time.Millisecond
+	expBackOff.MaxElapsedTime = budget
+
+	return backoff.Retry(func() error { return routeMessageRoundtrip(msg) }, expBackOff)
+}
+
+func routeMessageRoundtrip(msg *route.RouteMessage) error {
+	fd, err := unix.Socket(syscall.AF_ROUTE, syscall.SOCK_RAW, syscall.AF_UNSPEC)
+	if err != nil {
+		return fmt.Errorf("open routing socket: %w", err)
+	}
+	defer func() {
+		if err := unix.Close(fd); err != nil && !errors.Is(err, unix.EBADF) {
+			log.Warnf("close routing socket: %v", err)
 		}
-		defer func() {
-			if err := unix.Close(fd); err != nil && !errors.Is(err, unix.EBADF) {
-				log.Warnf("failed to close routing socket: %v", err)
+	}()
+
+	tv := unix.Timeval{Sec: 1}
+	if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv); err != nil {
+		return backoff.Permanent(fmt.Errorf("set recv timeout: %w", err))
+	}
+
+	// AF_ROUTE is a broadcast channel: every route socket on the host sees
+	// every RTM_* event. With concurrent route programming the default
+	// per-socket queue overflows and our own reply gets dropped.
+	if err := unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, 1<<20); err != nil {
+		log.Debugf("set SO_RCVBUF on route socket: %v", err)
+	}
+
+	bytes, err := msg.Marshal()
+	if err != nil {
+		return backoff.Permanent(fmt.Errorf("marshal: %w", err))
+	}
+
+	if _, err = unix.Write(fd, bytes); err != nil {
+		if errors.Is(err, unix.ENOBUFS) || errors.Is(err, unix.EAGAIN) {
+			return fmt.Errorf("write: %w", err)
+		}
+		return backoff.Permanent(fmt.Errorf("write: %w", err))
+	}
+	return readRouteResponse(fd, msg.Type, msg.Seq)
+}
+
+// readRouteResponse reads from the AF_ROUTE socket until it sees a reply
+// matching our write (same type, seq, and pid). AF_ROUTE SOCK_RAW is a
+// broadcast channel: interface up/down, third-party route changes and neighbor
+// discovery events can all land between our write and read, so we must filter.
+func readRouteResponse(fd, wantType, wantSeq int) error {
+	pid := int32(os.Getpid())
+	resp := make([]byte, 2048)
+	deadline := time.Now().Add(time.Second)
+	for {
+		if time.Now().After(deadline) {
+			// Transient: under concurrent pressure the kernel can drop our reply
+			// from the socket buffer. Let backoff.Retry re-send with a fresh seq.
+			return fmt.Errorf("read: timeout waiting for route reply type=%d seq=%d", wantType, wantSeq)
+		}
+		n, err := unix.Read(fd, resp)
+		if err != nil {
+			if errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EWOULDBLOCK) {
+				// SO_RCVTIMEO fired while waiting; loop to re-check the absolute deadline.
+				continue
 			}
-		}()
-
-		msg, err := r.buildRouteMessage(action, prefix, nexthop)
-		if err != nil {
-			return backoff.Permanent(fmt.Errorf("build route message: %w", err))
+			return backoff.Permanent(fmt.Errorf("read: %w", err))
 		}
-
-		msgBytes, err := msg.Marshal()
-		if err != nil {
-			return backoff.Permanent(fmt.Errorf("marshal route message: %w", err))
+		if n < int(unsafe.Sizeof(unix.RtMsghdr{})) {
+			continue
 		}
-
-		if _, err = unix.Write(fd, msgBytes); err != nil {
-			if errors.Is(err, unix.ENOBUFS) || errors.Is(err, unix.EAGAIN) {
-				return fmt.Errorf("write: %w", err)
-			}
-			return backoff.Permanent(fmt.Errorf("write: %w", err))
+		hdr := (*unix.RtMsghdr)(unsafe.Pointer(&resp[0]))
+		// Darwin reflects the sender's pid on replies; matching (Type, Seq, Pid)
+		// uniquely identifies our own reply among broadcast traffic.
+		if int(hdr.Type) != wantType || int(hdr.Seq) != wantSeq || hdr.Pid != pid {
+			continue
 		}
-
-		respBuf := make([]byte, 2048)
-		n, err := unix.Read(fd, respBuf)
-		if err != nil {
-			return backoff.Permanent(fmt.Errorf("read route response: %w", err))
+		if hdr.Errno != 0 {
+			return backoff.Permanent(fmt.Errorf("kernel: %w", syscall.Errno(hdr.Errno)))
 		}
-
-		if n > 0 {
-			if err := r.parseRouteResponse(respBuf[:n]); err != nil {
-				return backoff.Permanent(err)
-			}
-		}
-
 		return nil
 	}
-	return operation
 }
 
 func (r *SysOps) buildRouteMessage(action int, prefix netip.Prefix, nexthop Nexthop) (msg *route.RouteMessage, err error) {
@@ -183,6 +243,7 @@ func (r *SysOps) buildRouteMessage(action int, prefix netip.Prefix, nexthop Next
 		Type:    action,
 		Flags:   unix.RTF_UP | routeProtoFlag,
 		Version: unix.RTM_VERSION,
+		ID:      uintptr(os.Getpid()),
 		Seq:     r.getSeq(),
 	}
 
@@ -221,19 +282,6 @@ func (r *SysOps) buildRouteMessage(action int, prefix netip.Prefix, nexthop Next
 	return msg, nil
 }
 
-func (r *SysOps) parseRouteResponse(buf []byte) error {
-	if len(buf) < int(unsafe.Sizeof(unix.RtMsghdr{})) {
-		return nil
-	}
-
-	rtMsg := (*unix.RtMsghdr)(unsafe.Pointer(&buf[0]))
-	if rtMsg.Errno != 0 {
-		return fmt.Errorf("parse: %d", rtMsg.Errno)
-	}
-
-	return nil
-}
-
 // addrToRouteAddr converts a netip.Addr to the appropriate route.Addr (*route.Inet4Addr or *route.Inet6Addr).
 func addrToRouteAddr(addr netip.Addr) (route.Addr, error) {
 	if addr.Is4() {