[client] Check for fwmark support and use fallback routing if not supported (#3220)

util/grpc/dialer.go

@@ -40,7 +40,6 @@ func WithCustomDialer() grpc.DialOption {
 			}
 		}
 
-		log.Debug("Using nbnet.NewDialer()")
 		conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr)
 		if err != nil {
 			log.Errorf("Failed to dial: %s", err)

util/net/env.go

@@ -2,6 +2,7 @@ package net
 
 import (
 	"os"
+	"strconv"
 
 	log "github.com/sirupsen/logrus"
 
@@ -10,20 +11,24 @@ import (
 
 const (
 	envDisableCustomRouting = "NB_DISABLE_CUSTOM_ROUTING"
-	envSkipSocketMark       = "NB_SKIP_SOCKET_MARK"
 )
 
+// CustomRoutingDisabled returns true if custom routing is disabled.
+// This will fall back to the operation mode before the exit node functionality was implemented.
+// In particular exclusion routes won't be set up and all dialers and listeners will use net.Dial and net.Listen, respectively.
 func CustomRoutingDisabled() bool {
 	if netstack.IsEnabled() {
 		return true
 	}
-	return os.Getenv(envDisableCustomRouting) == "true"
-}
 
-func SkipSocketMark() bool {
-	if skipSocketMark := os.Getenv(envSkipSocketMark); skipSocketMark == "true" {
-		log.Infof("%s is set to true, skipping SO_MARK", envSkipSocketMark)
-		return true
+	var customRoutingDisabled bool
+	if val := os.Getenv(envDisableCustomRouting); val != "" {
+		var err error
+		customRoutingDisabled, err = strconv.ParseBool(val)
+		if err != nil {
+			log.Warnf("failed to parse %s: %v", envDisableCustomRouting, err)
+		}
 	}
-	return false
+
+	return customRoutingDisabled
 }

util/net/env_generic.go

@@ -0,0 +1,12 @@
+//go:build !linux || android
+
+package net
+
+func Init() {
+	// nothing to do on non-linux
+}
+
+func AdvancedRouting() bool {
+	// non-linux currently doesn't support advanced routing
+	return false
+}

util/net/env_linux.go

@@ -0,0 +1,119 @@
+//go:build linux && !android
+
+package net
+
+import (
+	"errors"
+	"os"
+	"strconv"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vishvananda/netlink"
+
+	"github.com/netbirdio/netbird/client/iface/netstack"
+)
+
+const (
+	// these have the same effect, skip socket env supported for backward compatibility
+	envSkipSocketMark   = "NB_SKIP_SOCKET_MARK"
+	envUseLegacyRouting = "NB_USE_LEGACY_ROUTING"
+)
+
+var advancedRoutingSupported bool
+
+func Init() {
+	advancedRoutingSupported = checkAdvancedRoutingSupport()
+}
+
+func AdvancedRouting() bool {
+	return advancedRoutingSupported
+}
+
+func checkAdvancedRoutingSupport() bool {
+	var err error
+
+	var legacyRouting bool
+	if val := os.Getenv(envUseLegacyRouting); val != "" {
+		legacyRouting, err = strconv.ParseBool(val)
+		if err != nil {
+			log.Warnf("failed to parse %s: %v", envUseLegacyRouting, err)
+		}
+	}
+
+	var skipSocketMark bool
+	if val := os.Getenv(envSkipSocketMark); val != "" {
+		skipSocketMark, err = strconv.ParseBool(val)
+		if err != nil {
+			log.Warnf("failed to parse %s: %v", envSkipSocketMark, err)
+		}
+	}
+
+	// requested to disable advanced routing
+	if legacyRouting || skipSocketMark ||
+		// envCustomRoutingDisabled disables the custom dialers.
+		// There is no point in using advanced routing without those, as they set up fwmarks on the sockets.
+		CustomRoutingDisabled() ||
+		// netstack mode doesn't need routing at all
+		netstack.IsEnabled() {
+
+		log.Info("advanced routing has been requested to be disabled")
+		return false
+	}
+
+	if !CheckFwmarkSupport() || !CheckRuleOperationsSupport() {
+		log.Warn("system doesn't support required routing features, falling back to legacy routing")
+		return false
+	}
+
+	log.Info("system supports advanced routing")
+
+	return true
+}
+
+func CheckFwmarkSupport() bool {
+	// temporarily enable advanced routing to check fwmarks are supported
+	old := advancedRoutingSupported
+	advancedRoutingSupported = true
+	defer func() {
+		advancedRoutingSupported = old
+	}()
+
+	dialer := NewDialer()
+	dialer.Timeout = 100 * time.Millisecond
+
+	conn, err := dialer.Dial("udp", "127.0.0.1:9")
+	if err != nil {
+		log.Warnf("failed to dial with fwmark: %v", err)
+		return false
+	}
+	if err := conn.Close(); err != nil {
+		log.Warnf("failed to close connection: %v", err)
+
+	}
+
+	return true
+}
+
+func CheckRuleOperationsSupport() bool {
+	rule := netlink.NewRule()
+	// low precedence, semi-random
+	rule.Priority = 32321
+	rule.Table = syscall.RT_TABLE_MAIN
+	rule.Family = netlink.FAMILY_V4
+
+	if err := netlink.RuleAdd(rule); err != nil {
+		if errors.Is(err, syscall.EOPNOTSUPP) {
+			log.Warn("IP rule operations are not supported")
+			return false
+		}
+		log.Warnf("failed to test rule support: %v", err)
+		return false
+	}
+
+	if err := netlink.RuleDel(rule); err != nil {
+		log.Warnf("failed to delete test rule: %v", err)
+	}
+	return true
+}

util/net/net_linux.go

@@ -5,13 +5,11 @@ package net
 import (
 	"fmt"
 	"syscall"
-
-	log "github.com/sirupsen/logrus"
 )
 
 // SetSocketMark sets the SO_MARK option on the given socket connection
 func SetSocketMark(conn syscall.Conn) error {
-	if isSocketMarkDisabled() {
+	if !AdvancedRouting() {
 		return nil
 	}
 
@@ -25,7 +23,7 @@ func SetSocketMark(conn syscall.Conn) error {
 
 // SetSocketOpt sets the SO_MARK option on the given file descriptor
 func SetSocketOpt(fd int) error {
-	if isSocketMarkDisabled() {
+	if !AdvancedRouting() {
 		return nil
 	}
 
@@ -36,7 +34,7 @@ func setRawSocketMark(conn syscall.RawConn) error {
 	var setErr error
 
 	err := conn.Control(func(fd uintptr) {
-		if isSocketMarkDisabled() {
+		if !AdvancedRouting() {
 			return
 		}
 		setErr = setSocketOptInt(int(fd))
@@ -55,15 +53,3 @@ func setRawSocketMark(conn syscall.RawConn) error {
 func setSocketOptInt(fd int) error {
 	return syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, NetbirdFwmark)
 }
-
-func isSocketMarkDisabled() bool {
-	if CustomRoutingDisabled() {
-		log.Infof("Custom routing is disabled, skipping SO_MARK")
-		return true
-	}
-
-	if SkipSocketMark() {
-		return true
-	}
-	return false
-}