fix tests

Co-authored-by: Viktor Liu <viktor@netbird.io>

client/internal/engine.go

@@ -794,6 +794,7 @@ func (e *Engine) updateOfflinePeers(offlinePeers []*mgmProto.RemotePeerConfig) {
 			FQDN:             offlinePeer.GetFqdn(),
 			ConnStatus:       peer.StatusDisconnected,
 			ConnStatusUpdate: time.Now(),
+			Mux:              new(sync.RWMutex),
 		}
 	}
 	e.statusRecorder.ReplaceOfflinePeers(replacement)

client/internal/peer/conn.go

@@ -229,7 +229,6 @@ func (conn *Conn) reCreateAgent() error {
 	}
 
 	conn.agent, err = ice.NewAgent(agentConfig)
-
 	if err != nil {
 		return err
 	}
@@ -285,6 +284,7 @@ func (conn *Conn) Open() error {
 		IP:               strings.Split(conn.config.WgConfig.AllowedIps, "/")[0],
 		ConnStatusUpdate: time.Now(),
 		ConnStatus:       conn.status,
+		Mux:              new(sync.RWMutex),
 	}
 	err := conn.statusRecorder.UpdatePeerState(peerState)
 	if err != nil {
@@ -344,6 +344,7 @@ func (conn *Conn) Open() error {
 		PubKey:           conn.config.Key,
 		ConnStatus:       conn.status,
 		ConnStatusUpdate: time.Now(),
+		Mux:              new(sync.RWMutex),
 	}
 	err = conn.statusRecorder.UpdatePeerState(peerState)
 	if err != nil {
@@ -468,6 +469,7 @@ func (conn *Conn) configureConnection(remoteConn net.Conn, remoteWgPort int, rem
 		RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Local.Port()),
 		Direct:                     !isRelayCandidate(pair.Local),
 		RosenpassEnabled:           rosenpassEnabled,
+		Mux:                        new(sync.RWMutex),
 	}
 	if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay {
 		peerState.Relayed = true
@@ -558,6 +560,7 @@ func (conn *Conn) cleanup() error {
 		PubKey:           conn.config.Key,
 		ConnStatus:       conn.status,
 		ConnStatusUpdate: time.Now(),
+		Mux:              new(sync.RWMutex),
 	}
 	err := conn.statusRecorder.UpdatePeerState(peerState)
 	if err != nil {

client/internal/peer/status.go

@@ -14,6 +14,7 @@ import (
 
 // State contains the latest state of a peer
 type State struct {
+	Mux                        *sync.RWMutex
 	IP                         string
 	PubKey                     string
 	FQDN                       string
@@ -30,7 +31,38 @@ type State struct {
 	BytesRx                    int64
 	Latency                    time.Duration
 	RosenpassEnabled           bool
-	Routes                     map[string]struct{}
+	routes                     map[string]struct{}
+}
+
+// AddRoute add a single route to routes map
+func (s *State) AddRoute(network string) {
+	s.Mux.Lock()
+	if s.routes == nil {
+		s.routes = make(map[string]struct{})
+	}
+	s.routes[network] = struct{}{}
+	s.Mux.Unlock()
+}
+
+// SetRoutes set state routes
+func (s *State) SetRoutes(routes map[string]struct{}) {
+	s.Mux.Lock()
+	s.routes = routes
+	s.Mux.Unlock()
+}
+
+// DeleteRoute removes a route from the network amp
+func (s *State) DeleteRoute(network string) {
+	s.Mux.Lock()
+	delete(s.routes, network)
+	s.Mux.Unlock()
+}
+
+// GetRoutes return routes map
+func (s *State) GetRoutes() map[string]struct{} {
+	s.Mux.RLock()
+	defer s.Mux.RUnlock()
+	return s.routes
 }
 
 // LocalPeerState contains the latest state of the local peer
@@ -143,6 +175,7 @@ func (d *Status) AddPeer(peerPubKey string, fqdn string) error {
 		PubKey:     peerPubKey,
 		ConnStatus: StatusDisconnected,
 		FQDN:       fqdn,
+		Mux:        new(sync.RWMutex),
 	}
 	d.peerListChangedForNotification = true
 	return nil
@@ -189,8 +222,8 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 		peerState.IP = receivedState.IP
 	}
 
-	if receivedState.Routes != nil {
-		peerState.Routes = receivedState.Routes
+	if receivedState.GetRoutes() != nil {
+		peerState.SetRoutes(receivedState.GetRoutes())
 	}
 
 	skipNotification := shouldSkipNotify(receivedState, peerState)
@@ -440,7 +473,6 @@ func (d *Status) IsLoginRequired() bool {
 	s, ok := gstatus.FromError(d.managementError)
 	if ok && (s.Code() == codes.InvalidArgument || s.Code() == codes.PermissionDenied) {
 		return true
-
 	}
 	return false
 }

client/internal/peer/status_test.go

@@ -3,6 +3,7 @@ package peer
 import (
 	"errors"
 	"testing"
+	"sync"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -42,6 +43,7 @@ func TestUpdatePeerState(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
+		Mux:	new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState
@@ -62,6 +64,7 @@ func TestStatus_UpdatePeerFQDN(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
+		Mux:	new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState
@@ -80,6 +83,7 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
+		Mux:	new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState
@@ -104,6 +108,7 @@ func TestRemovePeer(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
+		Mux:	new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState

client/internal/routemanager/client.go

@@ -39,6 +39,7 @@ type clientNetwork struct {
 	chosenRoute         *route.Route
 	network             netip.Prefix
 	updateSerial        uint64
+	systemUpdated       bool
 }
 
 func newClientNetworkWatcher(ctx context.Context, wgInterface *iface.WGIface, statusRecorder *peer.Status, network netip.Prefix) *clientNetwork {
@@ -196,7 +197,7 @@ func (c *clientNetwork) removeRouteFromWireguardPeer(peerKey string) error {
 		return fmt.Errorf("get peer state: %v", err)
 	}
 
-	delete(state.Routes, c.network.String())
+	state.DeleteRoute(c.network.String())
 	if err := c.statusRecorder.UpdatePeerState(state); err != nil {
 		log.Warnf("Failed to update peer state: %v", err)
 	}
@@ -215,8 +216,13 @@ func (c *clientNetwork) removeRouteFromWireguardPeer(peerKey string) error {
 
 func (c *clientNetwork) removeRouteFromPeerAndSystem() error {
 	if c.chosenRoute != nil {
-		if err := removeVPNRoute(c.network, c.wgInterface.Name()); err != nil {
-			return fmt.Errorf("remove route %s from system, err: %v", c.network, err)
+		if c.systemUpdated {
+			if err := removeVPNRoute(c.network, c.wgInterface.Name()); err != nil {
+				return fmt.Errorf("remove route %s from system, err: %v", c.network, err)
+			}
+			c.systemUpdated = false
+		} else {
+			log.Debugf("system is not updated for network %s, skiping removal from system route table", c.network)
 		}
 
 		if err := c.removeRouteFromWireguardPeer(c.chosenRoute.Peer); err != nil {
@@ -260,6 +266,8 @@ func (c *clientNetwork) recalculateRouteAndUpdatePeerAndSystem() error {
 			return fmt.Errorf("route %s couldn't be added for peer %s, err: %v",
 				c.network.String(), c.wgInterface.Address().IP.String(), err)
 		}
+
+		c.systemUpdated = true
 	}
 
 	c.chosenRoute = c.routes[chosen]
@@ -268,10 +276,7 @@ func (c *clientNetwork) recalculateRouteAndUpdatePeerAndSystem() error {
 	if err != nil {
 		log.Errorf("Failed to get peer state: %v", err)
 	} else {
-		if state.Routes == nil {
-			state.Routes = map[string]struct{}{}
-		}
-		state.Routes[c.network.String()] = struct{}{}
+		state.AddRoute(c.network.String())
 		if err := c.statusRecorder.UpdatePeerState(state); err != nil {
 			log.Warnf("Failed to update peer state: %v", err)
 		}

client/internal/routemanager/manager.go

@@ -16,6 +16,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/iface"
 	"github.com/netbirdio/netbird/route"
+	nbnet "github.com/netbirdio/netbird/util/net"
 	"github.com/netbirdio/netbird/version"
 )
 
@@ -68,6 +69,10 @@ func NewManager(ctx context.Context, pubKey string, wgInterface *iface.WGIface,
 
 // Init sets up the routing
 func (m *DefaultManager) Init() (peer.BeforeAddPeerHookFunc, peer.AfterRemovePeerHookFunc, error) {
+	if nbnet.CustomRoutingDisabled() {
+		return nil, nil, nil
+	}
+
 	if err := cleanupRouting(); err != nil {
 		log.Warnf("Failed cleaning up routing: %v", err)
 	}
@@ -99,11 +104,15 @@ func (m *DefaultManager) Stop() {
 	if m.serverRouter != nil {
 		m.serverRouter.cleanUp()
 	}
-	if err := cleanupRouting(); err != nil {
-		log.Errorf("Error cleaning up routing: %v", err)
-	} else {
-		log.Info("Routing cleanup complete")
+
+	if !nbnet.CustomRoutingDisabled() {
+		if err := cleanupRouting(); err != nil {
+			log.Errorf("Error cleaning up routing: %v", err)
+		} else {
+			log.Info("Routing cleanup complete")
+		}
 	}
+
 	m.ctx = nil
 }
 
@@ -210,9 +219,11 @@ func (m *DefaultManager) clientRoutes(initialRoutes []*route.Route) []*route.Rou
 }
 
 func isPrefixSupported(prefix netip.Prefix) bool {
-	switch runtime.GOOS {
-	case "linux", "windows", "darwin":
-		return true
+	if !nbnet.CustomRoutingDisabled() {
+		switch runtime.GOOS {
+		case "linux", "windows", "darwin":
+			return true
+		}
 	}
 
 	// If prefix is too small, lets assume it is a possible default prefix which is not yet supported

client/internal/routemanager/systemops.go

@@ -256,8 +256,7 @@ func addNonExistingRoute(prefix netip.Prefix, intf string) error {
 		return fmt.Errorf("exists in route table: %w", err)
 	}
 	if ok {
-		log.Warnf("Skipping adding a new route for network %s because it already exists", prefix)
-		return nil
+		return fmt.Errorf("skipping adding a new route for network %s because it already exists", prefix)
 	}
 
 	ok, err = isSubRange(prefix)

client/internal/routemanager/systemops_linux.go

@@ -4,14 +4,14 @@ package routemanager
 
 import (
 	"bufio"
-	"context"
 	"errors"
 	"fmt"
 	"net"
 	"net/netip"
 	"os"
+	"strconv"
+	"strings"
 	"syscall"
-	"time"
 
 	"github.com/hashicorp/go-multierror"
 	log "github.com/sirupsen/logrus"
@@ -32,19 +32,31 @@ const (
 	rtTablesPath = "/etc/iproute2/rt_tables"
 
 	// ipv4ForwardingPath is the path to the file containing the IP forwarding setting.
-	ipv4ForwardingPath = "/proc/sys/net/ipv4/ip_forward"
+	ipv4ForwardingPath = "net.ipv4.ip_forward"
+
+	rpFilterPath          = "net.ipv4.conf.all.rp_filter"
+	rpFilterInterfacePath = "net.ipv4.conf.%s.rp_filter"
+	srcValidMarkPath      = "net.ipv4.conf.all.src_valid_mark"
 )
 
 var ErrTableIDExists = errors.New("ID exists with different name")
 
 var routeManager = &RouteManager{}
-var isLegacy = os.Getenv("NB_USE_LEGACY_ROUTING") == "true"
+
+// originalSysctl stores the original sysctl values before they are modified
+var originalSysctl map[string]int
+
+// determines whether to use the legacy routing setup
+var isLegacy = os.Getenv("NB_USE_LEGACY_ROUTING") == "true" || nbnet.CustomRoutingDisabled()
+
+// sysctlFailed is used as an indicator to emit a warning when default routes are configured
+var sysctlFailed bool
 
 type ruleParams struct {
+	priority       int
 	fwmark         int
 	tableID        int
 	family         int
-	priority       int
 	invert         bool
 	suppressPrefix int
 	description    string
@@ -52,10 +64,10 @@ type ruleParams struct {
 
 func getSetupRules() []ruleParams {
 	return []ruleParams{
-		{nbnet.NetbirdFwmark, NetbirdVPNTableID, netlink.FAMILY_V4, -1, true, -1, "rule v4 netbird"},
-		{nbnet.NetbirdFwmark, NetbirdVPNTableID, netlink.FAMILY_V6, -1, true, -1, "rule v6 netbird"},
-		{-1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V4, -1, false, 0, "rule with suppress prefixlen v4"},
-		{-1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V6, -1, false, 0, "rule with suppress prefixlen v6"},
+		{100, -1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V4, false, 0, "rule with suppress prefixlen v4"},
+		{100, -1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V6, false, 0, "rule with suppress prefixlen v6"},
+		{110, nbnet.NetbirdFwmark, NetbirdVPNTableID, netlink.FAMILY_V4, true, -1, "rule v4 netbird"},
+		{110, nbnet.NetbirdFwmark, NetbirdVPNTableID, netlink.FAMILY_V6, true, -1, "rule v6 netbird"},
 	}
 }
 
@@ -69,8 +81,6 @@ func getSetupRules() []ruleParams {
 // Rule 2 (VPN Traffic Routing): Directs all remaining traffic to the 'NetbirdVPNTableID' custom routing table.
 // This table is where a default route or other specific routes received from the management server are configured,
 // enabling VPN connectivity.
-//
-// The rules are inserted in reverse order, as rules are added from the bottom up in the rule list.
 func setupRouting(initAddresses []net.IP, wgIface *iface.WGIface) (_ peer.BeforeAddPeerHookFunc, _ peer.AfterRemovePeerHookFunc, err error) {
 	if isLegacy {
 		log.Infof("Using legacy routing setup")
@@ -81,6 +91,13 @@ func setupRouting(initAddresses []net.IP, wgIface *iface.WGIface) (_ peer.Before
 		log.Errorf("Error adding routing table name: %v", err)
 	}
 
+	originalValues, err := setupSysctl(wgIface)
+	if err != nil {
+		log.Errorf("Error setting up sysctl: %v", err)
+		sysctlFailed = true
+	}
+	originalSysctl = originalValues
+
 	defer func() {
 		if err != nil {
 			if cleanErr := cleanupRouting(); cleanErr != nil {
@@ -123,11 +140,17 @@ func cleanupRouting() error {
 
 	rules := getSetupRules()
 	for _, rule := range rules {
-		if err := removeAllRules(rule); err != nil && !errors.Is(err, syscall.EOPNOTSUPP) {
+		if err := removeRule(rule); err != nil {
 			result = multierror.Append(result, fmt.Errorf("%s: %w", rule.description, err))
 		}
 	}
 
+	if err := cleanupSysctl(originalSysctl); err != nil {
+		result = multierror.Append(result, fmt.Errorf("cleanup sysctl: %w", err))
+	}
+	originalSysctl = nil
+	sysctlFailed = false
+
 	return result.ErrorOrNil()
 }
 
@@ -144,6 +167,10 @@ func addVPNRoute(prefix netip.Prefix, intf string) error {
 		return genericAddVPNRoute(prefix, intf)
 	}
 
+	if sysctlFailed && (prefix == defaultv4 || prefix == defaultv6) {
+		log.Warnf("Default route is configured but sysctl operations failed, VPN traffic may not be routed correctly, consider using NB_USE_LEGACY_ROUTING=true or setting net.ipv4.conf.*.rp_filter to 2 (loose) or 0 (off)")
+	}
+
 	// No need to check if routes exist as main table takes precedence over the VPN table via Rule 1
 
 	// TODO remove this once we have ipv6 support
@@ -336,22 +363,8 @@ func flushRoutes(tableID, family int) error {
 }
 
 func enableIPForwarding() error {
-	bytes, err := os.ReadFile(ipv4ForwardingPath)
-	if err != nil {
-		return fmt.Errorf("read file %s: %w", ipv4ForwardingPath, err)
-	}
-
-	// check if it is already enabled
-	// see more: https://github.com/netbirdio/netbird/issues/872
-	if len(bytes) > 0 && bytes[0] == 49 {
-		return nil
-	}
-
-	//nolint:gosec
-	if err := os.WriteFile(ipv4ForwardingPath, []byte("1"), 0644); err != nil {
-		return fmt.Errorf("write file %s: %w", ipv4ForwardingPath, err)
-	}
-	return nil
+	_, err := setSysctl(ipv4ForwardingPath, 1, false)
+	return err
 }
 
 // entryExists checks if the specified ID or name already exists in the rt_tables file
@@ -429,7 +442,7 @@ func addRule(params ruleParams) error {
 	rule.Invert = params.invert
 	rule.SuppressPrefixlen = params.suppressPrefix
 
-	if err := netlink.RuleAdd(rule); err != nil && !errors.Is(err, syscall.EAFNOSUPPORT) {
+	if err := netlink.RuleAdd(rule); err != nil && !errors.Is(err, syscall.EEXIST) && !errors.Is(err, syscall.EAFNOSUPPORT) {
 		return fmt.Errorf("add routing rule: %w", err)
 	}
 
@@ -446,43 +459,13 @@ func removeRule(params ruleParams) error {
 	rule.Priority = params.priority
 	rule.SuppressPrefixlen = params.suppressPrefix
 
-	if err := netlink.RuleDel(rule); err != nil {
+	if err := netlink.RuleDel(rule); err != nil && !errors.Is(err, syscall.ENOENT) && !errors.Is(err, syscall.EAFNOSUPPORT) {
 		return fmt.Errorf("remove routing rule: %w", err)
 	}
 
 	return nil
 }
 
-func removeAllRules(params ruleParams) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-	defer cancel()
-
-	done := make(chan error, 1)
-	go func() {
-		for {
-			if ctx.Err() != nil {
-				done <- ctx.Err()
-				return
-			}
-			if err := removeRule(params); err != nil {
-				if errors.Is(err, syscall.ENOENT) || errors.Is(err, syscall.EAFNOSUPPORT) {
-					done <- nil
-					return
-				}
-				done <- err
-				return
-			}
-		}
-	}()
-
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case err := <-done:
-		return err
-	}
-}
-
 // addNextHop adds the gateway and device to the route.
 func addNextHop(addr netip.Addr, intf string, route *netlink.Route) error {
 	if addr.IsValid() {
@@ -509,3 +492,83 @@ func getAddressFamily(prefix netip.Prefix) int {
 	}
 	return netlink.FAMILY_V6
 }
+
+// setupSysctl configures sysctl settings for RP filtering and source validation.
+func setupSysctl(wgIface *iface.WGIface) (map[string]int, error) {
+	keys := map[string]int{}
+	var result *multierror.Error
+
+	oldVal, err := setSysctl(srcValidMarkPath, 1, false)
+	if err != nil {
+		result = multierror.Append(result, err)
+	} else {
+		keys[srcValidMarkPath] = oldVal
+	}
+
+	oldVal, err = setSysctl(rpFilterPath, 2, true)
+	if err != nil {
+		result = multierror.Append(result, err)
+	} else {
+		keys[rpFilterPath] = oldVal
+	}
+
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		result = multierror.Append(result, fmt.Errorf("list interfaces: %w", err))
+	}
+
+	for _, intf := range interfaces {
+		if intf.Name == "lo" || wgIface != nil && intf.Name == wgIface.Name() {
+			continue
+		}
+
+		i := fmt.Sprintf(rpFilterInterfacePath, intf.Name)
+		oldVal, err := setSysctl(i, 2, true)
+		if err != nil {
+			result = multierror.Append(result, err)
+		} else {
+			keys[i] = oldVal
+		}
+	}
+
+	return keys, result.ErrorOrNil()
+}
+
+// setSysctl sets a sysctl configuration, if onlyIfOne is true it will only set the new value if it's set to 1
+func setSysctl(key string, desiredValue int, onlyIfOne bool) (int, error) {
+	path := fmt.Sprintf("/proc/sys/%s", strings.ReplaceAll(key, ".", "/"))
+	currentValue, err := os.ReadFile(path)
+	if err != nil {
+		return -1, fmt.Errorf("read sysctl %s: %w", key, err)
+	}
+
+	currentV, err := strconv.Atoi(strings.TrimSpace(string(currentValue)))
+	if err != nil && len(currentValue) > 0 {
+		return -1, fmt.Errorf("convert current desiredValue to int: %w", err)
+	}
+
+	if currentV == desiredValue || onlyIfOne && currentV != 1 {
+		return currentV, nil
+	}
+
+	//nolint:gosec
+	if err := os.WriteFile(path, []byte(strconv.Itoa(desiredValue)), 0644); err != nil {
+		return currentV, fmt.Errorf("write sysctl %s: %w", key, err)
+	}
+	log.Debugf("Set sysctl %s from %d to %d", key, currentV, desiredValue)
+
+	return currentV, nil
+}
+
+func cleanupSysctl(originalSettings map[string]int) error {
+	var result *multierror.Error
+
+	for key, value := range originalSettings {
+		_, err := setSysctl(key, value, false)
+		if err != nil {
+			result = multierror.Append(result, err)
+		}
+	}
+
+	return result.ErrorOrNil()
+}

client/internal/routemanager/systemops_test.go

@@ -3,18 +3,14 @@
 package routemanager
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net"
 	"net/netip"
-	"os"
 	"runtime"
-	"strings"
 	"testing"
 
 	"github.com/pion/transport/v3/stdnet"
-	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
@@ -41,8 +37,8 @@ func TestAddRemoveRoutes(t *testing.T) {
 			shouldBeRemoved:        true,
 		},
 		{
-			name:                   "Should Not Add Or Remove Route 127.0.0.1/32",
-			prefix:                 netip.MustParsePrefix("127.0.0.1/32"),
+			name:                   "Should Not Add Or Remove Route 127.0.1.1/32",
+			prefix:                 netip.MustParsePrefix("127.0.1.1/32"),
 			shouldRouteToWireguard: false,
 			shouldBeRemoved:        false,
 		},
@@ -61,7 +57,7 @@ func TestAddRemoveRoutes(t *testing.T) {
 
 			err = wgInterface.Create()
 			require.NoError(t, err, "should create testing wireguard interface")
-			_, _, err = setupRouting(nil, nil)
+			_, _, err = setupRouting(nil, wgInterface)
 			require.NoError(t, err)
 			t.Cleanup(func() {
 				assert.NoError(t, cleanupRouting())
@@ -140,10 +136,10 @@ func TestGetNextHop(t *testing.T) {
 }
 
 func TestAddExistAndRemoveRoute(t *testing.T) {
-	defaultGateway, _, err := getNextHop(netip.MustParseAddr("0.0.0.0"))
+	defaultGateway, _, errGW := getNextHop(netip.MustParseAddr("0.0.0.0"))
 	t.Log("defaultGateway: ", defaultGateway)
-	if err != nil {
-		t.Fatal("shouldn't return error when fetching the gateway: ", err)
+	if errGW != nil {
+		t.Fatal("shouldn't return error when fetching the gateway: ", errGW)
 	}
 	testCases := []struct {
 		name              string
@@ -159,7 +155,7 @@ func TestAddExistAndRemoveRoute(t *testing.T) {
 		{
 			name:           "Should Not Add Route if overlaps with default gateway",
 			prefix:         netip.MustParsePrefix(defaultGateway.String() + "/31"),
-			shouldAddRoute: false,
+			shouldAddRoute: true,
 		},
 		{
 			name:              "Should Add Route if bigger network exists",
@@ -182,11 +178,6 @@ func TestAddExistAndRemoveRoute(t *testing.T) {
 	}
 
 	for n, testCase := range testCases {
-		var buf bytes.Buffer
-		log.SetOutput(&buf)
-		defer func() {
-			log.SetOutput(os.Stderr)
-		}()
 		t.Run(testCase.name, func(t *testing.T) {
 			peerPrivateKey, _ := wgtypes.GeneratePrivateKey()
 			newNet, err := stdnet.NewNet()
@@ -202,34 +193,33 @@ func TestAddExistAndRemoveRoute(t *testing.T) {
 
 			// Prepare the environment
 			if testCase.preExistingPrefix.IsValid() {
-				err := genericAddVPNRoute(testCase.preExistingPrefix, wgInterface.Name())
+				err = genericAddVPNRoute(testCase.preExistingPrefix, wgInterface.Name())
 				require.NoError(t, err, "should not return err when adding pre-existing route")
+				ok, err := existsInRouteTable(testCase.preExistingPrefix)
+				require.NoError(t, err, "should not return err")
+				require.True(t, ok, "route should exist")
 			}
 
 			// Add the route
 			err = genericAddVPNRoute(testCase.prefix, wgInterface.Name())
-			require.NoError(t, err, "should not return err when adding route")
-
-			if testCase.shouldAddRoute {
-				// test if route exists after adding
-				ok, err := existsInRouteTable(testCase.prefix)
-				require.NoError(t, err, "should not return err")
-				require.True(t, ok, "route should exist")
-
-				// remove route again if added
-				err = genericRemoveVPNRoute(testCase.prefix, wgInterface.Name())
-				require.NoError(t, err, "should not return err")
+			if !testCase.shouldAddRoute {
+				require.Error(t, err, "should return err when adding existing route")
+				return
 			}
-
-			// route should either not have been added or should have been removed
-			// In case of already existing route, it should not have been added (but still exist)
+			require.NoError(t, err, "should not return err when adding route")
+			// test if route exists after adding
 			ok, err := existsInRouteTable(testCase.prefix)
-			t.Log("Buffer string: ", buf.String())
+			require.NoError(t, err, "should not return err")
+			require.True(t, ok, "route should exist")
+
+			// remove route again if added
+			err = genericRemoveVPNRoute(testCase.prefix, wgInterface.Name())
 			require.NoError(t, err, "should not return err")
 
-			if !strings.Contains(buf.String(), "because it already exists") {
-				require.False(t, ok, "route should not exist")
-			}
+			// test if route was removed
+			ok, err = existsInRouteTable(testCase.prefix)
+			require.NoError(t, err, "should not return err")
+			require.False(t, ok, "route should not exist")
 		})
 	}
 }
@@ -355,7 +345,7 @@ func setupTestEnv(t *testing.T) {
 
 	// 10.0.0.0/8 route exists in main table and vpn table
 	err = addVPNRoute(netip.MustParsePrefix("10.0.0.0/8"), wgIface.Name())
-	require.NoError(t, err, "addVPNRoute should not return err")
+	require.Error(t, err, "addVPNRoute should return err")
 	t.Cleanup(func() {
 		err = removeVPNRoute(netip.MustParsePrefix("10.0.0.0/8"), wgIface.Name())
 		assert.NoError(t, err, "removeVPNRoute should not return err")

client/internal/routemanager/systemops_windows.go

@@ -73,7 +73,7 @@ func addRoutePowershell(prefix netip.Prefix, nexthop netip.Addr, intf, intfIdx s
 	}
 
 	script := fmt.Sprintf(
-		`%s -AddressFamily "%s" -DestinationPrefix "%s" -Confirm:$False -ErrorAction Stop`,
+		`%s -AddressFamily "%s" -DestinationPrefix "%s" -Confirm:$False -ErrorAction Stop -PolicyStore ActiveStore`,
 		psCmd, addressFamily, destinationPrefix,
 	)
 

client/internal/wgproxy/proxy_ebpf.go

@@ -230,7 +230,7 @@ func (p *WGEBPFProxy) prepareSenderRawSocket() (net.PacketConn, error) {
 	}
 
 	// Set the fwmark on the socket.
-	err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, nbnet.NetbirdFwmark)
+	err = nbnet.SetSocketOpt(fd)
 	if err != nil {
 		return nil, fmt.Errorf("setting fwmark failed: %w", err)
 	}

client/server/server.go

@@ -718,7 +718,7 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus {
 			BytesRx:                    peerState.BytesRx,
 			BytesTx:                    peerState.BytesTx,
 			RosenpassEnabled:           peerState.RosenpassEnabled,
-			Routes:                     maps.Keys(peerState.Routes),
+			Routes:                     maps.Keys(peerState.GetRoutes()),
 			Latency:                    durationpb.New(peerState.Latency),
 		}
 		pbFullStatus.Peers = append(pbFullStatus.Peers, pbPeerState)

iface/wg_configurer_kernel.go

@@ -10,8 +10,6 @@ import (
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
-
-	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 type wgKernelConfigurer struct {
@@ -31,7 +29,7 @@ func (c *wgKernelConfigurer) configureInterface(privateKey string, port int) err
 	if err != nil {
 		return err
 	}
-	fwmark := nbnet.NetbirdFwmark
+	fwmark := getFwmark()
 	config := wgtypes.Config{
 		PrivateKey:   &key,
 		ReplacePeers: true,

iface/wg_configurer_usp.go

@@ -349,7 +349,7 @@ func toWgUserspaceString(wgCfg wgtypes.Config) string {
 }
 
 func getFwmark() int {
-	if runtime.GOOS == "linux" {
+	if runtime.GOOS == "linux" && !nbnet.CustomRoutingDisabled() {
 		return nbnet.NetbirdFwmark
 	}
 	return 0

management/server/account.go

@@ -242,19 +242,19 @@ type UserPermissions struct {
 }
 
 type UserInfo struct {
-	ID                   string               `json:"id"`
-	Email                string               `json:"email"`
-	Name                 string               `json:"name"`
-	Role                 string               `json:"role"`
-	AutoGroups           []string             `json:"auto_groups"`
-	Status               string               `json:"-"`
-	IsServiceUser        bool                 `json:"is_service_user"`
-	IsBlocked            bool                 `json:"is_blocked"`
-	NonDeletable         bool                 `json:"non_deletable"`
-	LastLogin            time.Time            `json:"last_login"`
-	Issued               string               `json:"issued"`
+	ID                   string                                     `json:"id"`
+	Email                string                                     `json:"email"`
+	Name                 string                                     `json:"name"`
+	Role                 string                                     `json:"role"`
+	AutoGroups           []string                                   `json:"auto_groups"`
+	Status               string                                     `json:"-"`
+	IsServiceUser        bool                                       `json:"is_service_user"`
+	IsBlocked            bool                                       `json:"is_blocked"`
+	NonDeletable         bool                                       `json:"non_deletable"`
+	LastLogin            time.Time                                  `json:"last_login"`
+	Issued               string                                     `json:"issued"`
 	IntegrationReference integration_reference.IntegrationReference `json:"-"`
-	Permissions          UserPermissions      `json:"permissions"`
+	Permissions          UserPermissions                            `json:"permissions"`
 }
 
 // getRoutesToSync returns the enabled routes for the peer ID and the routes
@@ -1120,7 +1120,7 @@ func (am *DefaultAccountManager) DeleteAccount(accountID, userID string) error {
 		return status.Errorf(status.PermissionDenied, "user is not allowed to delete account")
 	}
 
-	if user.Id != account.CreatedBy {
+	if user.Role != UserRoleOwner {
 		return status.Errorf(status.PermissionDenied, "user is not allowed to delete account. Only account owner can delete account")
 	}
 	for _, otherUser := range account.Users {

management/server/peer.go

@@ -551,8 +551,8 @@ func (am *DefaultAccountManager) SyncPeer(sync PeerSync) (*nbpeer.Peer, *Network
 		return nil, nil, status.Errorf(status.PermissionDenied, "peer login has expired, please log in once more")
 	}
 
-	requiresApproval, isStatusChanged := am.integratedPeerValidator.IsNotValidPeer(account.Id, peer, account.GetPeerGroupsList(peer.ID), account.Settings.Extra)
-	if requiresApproval {
+	peerNotValid, isStatusChanged := am.integratedPeerValidator.IsNotValidPeer(account.Id, peer, account.GetPeerGroupsList(peer.ID), account.Settings.Extra)
+	if peerNotValid {
 		emptyMap := &NetworkMap{
 			Network: account.Network.Copy(),
 		}
@@ -563,11 +563,11 @@ func (am *DefaultAccountManager) SyncPeer(sync PeerSync) (*nbpeer.Peer, *Network
 		am.updateAccountPeers(account)
 	}
 
-	approvedPeersMap, err := am.GetValidatedPeers(account)
+	validPeersMap, err := am.GetValidatedPeers(account)
 	if err != nil {
 		return nil, nil, err
 	}
-	return peer, account.GetPeerNetworkMap(peer.ID, am.dnsDomain, approvedPeersMap), nil
+	return peer, account.GetPeerNetworkMap(peer.ID, am.dnsDomain, validPeersMap), nil
 }
 
 // LoginPeer logs in or registers a peer.

util/grpc/dialer.go

@@ -3,6 +3,8 @@ package grpc
 import (
 	"context"
 	"net"
+	"os/user"
+	"runtime"
 
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
@@ -12,6 +14,20 @@ import (
 
 func WithCustomDialer() grpc.DialOption {
 	return grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
+		if runtime.GOOS == "linux" {
+			currentUser, err := user.Current()
+			if err != nil {
+				log.Fatalf("failed to get current user: %v", err)
+			}
+
+			// the custom dialer requires root permissions which are not required for use cases run as non-root
+			if currentUser.Uid != "0" {
+				dialer := &net.Dialer{}
+				return dialer.DialContext(ctx, "tcp", addr)
+			}
+		}
+
+
 		conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr)
 		if err != nil {
 			log.Errorf("Failed to dial: %s", err)

util/net/dialer_generic.go

@@ -49,6 +49,10 @@ func RemoveDialerHooks() {
 
 // DialContext wraps the net.Dialer's DialContext method to use the custom connection
 func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	if CustomRoutingDisabled() {
+		return d.Dialer.DialContext(ctx, network, address)
+	}
+
 	var resolver *net.Resolver
 	if d.Resolver != nil {
 		resolver = d.Resolver
@@ -123,6 +127,10 @@ func callDialerHooks(ctx context.Context, connID ConnectionID, address string, r
 }
 
 func DialUDP(network string, laddr, raddr *net.UDPAddr) (*net.UDPConn, error) {
+	if CustomRoutingDisabled() {
+		return net.DialUDP(network, laddr, raddr)
+	}
+
 	dialer := NewDialer()
 	dialer.LocalAddr = laddr
 
@@ -143,6 +151,10 @@ func DialUDP(network string, laddr, raddr *net.UDPAddr) (*net.UDPConn, error) {
 }
 
 func DialTCP(network string, laddr, raddr *net.TCPAddr) (*net.TCPConn, error) {
+	if CustomRoutingDisabled() {
+		return net.DialTCP(network, laddr, raddr)
+	}
+
 	dialer := NewDialer()
 	dialer.LocalAddr = laddr
 

util/net/listener_generic.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"sync"
 
+	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -52,6 +53,10 @@ func RemoveListenerHooks() {
 // ListenPacket listens on the network address and returns a PacketConn
 // which includes support for write hooks.
 func (l *ListenerConfig) ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error) {
+	if CustomRoutingDisabled() {
+		return l.ListenConfig.ListenPacket(ctx, network, address)
+	}
+
 	pc, err := l.ListenConfig.ListenPacket(ctx, network, address)
 	if err != nil {
 		return nil, fmt.Errorf("listen packet: %w", err)
@@ -144,7 +149,11 @@ func closeConn(id ConnectionID, conn net.PacketConn) error {
 
 // ListenUDP listens on the network address and returns a transport.UDPConn
 // which includes support for write and close hooks.
-func ListenUDP(network string, laddr *net.UDPAddr) (*UDPConn, error) {
+func ListenUDP(network string, laddr *net.UDPAddr) (transport.UDPConn, error) {
+	if CustomRoutingDisabled() {
+		return net.ListenUDP(network, laddr)
+	}
+
 	conn, err := NewListener().ListenPacket(context.Background(), network, laddr.String())
 	if err != nil {
 		return nil, fmt.Errorf("listen UDP: %w", err)

util/net/net.go

@@ -1,10 +1,16 @@
 package net
 
-import "github.com/google/uuid"
+import (
+	"os"
+
+	"github.com/google/uuid"
+)
 
 const (
 	// NetbirdFwmark is the fwmark value used by Netbird via wireguard
 	NetbirdFwmark = 0x1BD00
+
+	envDisableCustomRouting = "NB_DISABLE_CUSTOM_ROUTING"
 )
 
 // ConnectionID provides a globally unique identifier for network connections.
@@ -15,3 +21,7 @@ type ConnectionID string
 func GenerateConnID() ConnectionID {
 	return ConnectionID(uuid.NewString())
 }
+
+func CustomRoutingDisabled() bool {
+	return os.Getenv(envDisableCustomRouting) == "true"
+}

util/net/net_linux.go

@@ -21,7 +21,7 @@ func SetRawSocketMark(conn syscall.RawConn) error {
 	var setErr error
 
 	err := conn.Control(func(fd uintptr) {
-		setErr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, NetbirdFwmark)
+		setErr = SetSocketOpt(int(fd))
 	})
 	if err != nil {
 		return fmt.Errorf("control: %w", err)
@@ -33,3 +33,11 @@ func SetRawSocketMark(conn syscall.RawConn) error {
 
 	return nil
 }
+
+func SetSocketOpt(fd int) error {
+	if CustomRoutingDisabled() {
+		return nil
+	}
+
+	return syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, NetbirdFwmark)
+}