Fix RemoveConnByUfrag

Handle routes updates from management

Manage routing firewall rules

Manage peer RIB table

Add get peer and get notification channel from the status recorder

Update interface peers allowed IPs

.github/workflows/golangci-lint.yml

@@ -6,12 +6,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.x
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libgtk-3-dev libappindicator3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
-          args: --timeout=6m
+          #  SA1019: "io/ioutil" has been deprecated since Go 1.16
+          args: --timeout=6m -e SA1019
 
 

.github/workflows/test-docker-compose-linux.yml

@@ -5,6 +5,12 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
+      - name: Install jq
+        run: sudo apt-get install -y jq
+
+      - name: Install curl
+        run: sudo apt-get install -y curl
+
       - name: Install Go
         uses: actions/setup-go@v2
         with:
@@ -28,20 +34,29 @@ jobs:
         working-directory: infrastructure_files
         run: bash -x configure.sh
         env:
-          CI_NETBIRD_AUTH0_DOMAIN: ${{ secrets.CI_NETBIRD_AUTH0_DOMAIN }}
+          CI_NETBIRD_AUTH_CLIENT_ID: ${{ secrets.CI_NETBIRD_AUTH_CLIENT_ID }}
-          CI_NETBIRD_AUTH0_CLIENT_ID: ${{ secrets.CI_NETBIRD_AUTH0_CLIENT_ID }}
+          CI_NETBIRD_AUTH_AUDIENCE: testing.ci
-          CI_NETBIRD_AUTH0_AUDIENCE: testing.ci
+          CI_NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT: https://example.eu.auth0.com/.well-known/openid-configuration
+          CI_NETBIRD_USE_AUTH0: true
 
       - name: check values
         working-directory: infrastructure_files
         env:
-          CI_NETBIRD_AUTH0_DOMAIN: ${{ secrets.CI_NETBIRD_AUTH0_DOMAIN }}
+          CI_NETBIRD_AUTH_CLIENT_ID: ${{ secrets.CI_NETBIRD_AUTH_CLIENT_ID }}
-          CI_NETBIRD_AUTH0_CLIENT_ID: ${{ secrets.CI_NETBIRD_AUTH0_CLIENT_ID }}
+          CI_NETBIRD_AUTH_AUDIENCE: testing.ci
-          CI_NETBIRD_AUTH0_AUDIENCE: testing.ci
+          CI_NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT: https://example.eu.auth0.com/.well-known/openid-configuration
+          CI_NETBIRD_USE_AUTH0: true
+          CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified"
+          CI_NETBIRD_AUTH_AUTHORITY: https://example.eu.auth0.com/
+          CI_NETBIRD_AUTH_JWT_CERTS: https://example.eu.auth0.com/.well-known/jwks.json
+          CI_NETBIRD_AUTH_TOKEN_ENDPOINT: https://example.eu.auth0.com/oauth/token
+          CI_NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT: https://example.eu.auth0.com/oauth/device/code
         run: |
-          grep AUTH0_DOMAIN docker-compose.yml | grep $CI_NETBIRD_AUTH0_DOMAIN
+          grep AUTH_CLIENT_ID docker-compose.yml | grep $CI_NETBIRD_AUTH_CLIENT_ID
-          grep AUTH0_CLIENT_ID docker-compose.yml | grep $CI_NETBIRD_AUTH0_CLIENT_ID
+          grep AUTH_AUTHORITY docker-compose.yml | grep $CI_NETBIRD_AUTH_AUTHORITY
-          grep AUTH0_AUDIENCE docker-compose.yml | grep $CI_NETBIRD_AUTH0_AUDIENCE
+          grep AUTH_AUDIENCE docker-compose.yml | grep $CI_NETBIRD_AUTH_AUDIENCE
+          grep AUTH_SUPPORTED_SCOPES docker-compose.yml | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES"
+          grep USE_AUTH0 docker-compose.yml | grep $CI_NETBIRD_USE_AUTH0
           grep NETBIRD_MGMT_API_ENDPOINT docker-compose.yml | grep "http://localhost:33073"  
 
       - name: run docker compose up

README.md

@@ -56,7 +56,7 @@ NetBird creates an overlay peer-to-peer network connecting machines automaticall
 -  \[x] Remote SSH access without managing SSH keys.
   
 **Coming soon:**
--  \[ ] Router nodes
+-  \[ ] Network Routes.
 -  \[ ] Private DNS.
 -  \[ ] Mobile clients.
 -  \[ ] Network Activity Monitoring.

client/cmd/login.go

@@ -169,7 +169,8 @@ func foregroundGetTokenInfo(ctx context.Context, cmd *cobra.Command, config *int
 	hostedClient := internal.NewHostedDeviceFlow(
 		providerConfig.ProviderConfig.Audience,
 		providerConfig.ProviderConfig.ClientID,
-		providerConfig.ProviderConfig.Domain,
+		providerConfig.ProviderConfig.TokenEndpoint,
+		providerConfig.ProviderConfig.DeviceAuthEndpoint,
 	)
 
 	flowInfo, err := hostedClient.RequestDeviceCode(context.TODO())

client/hhhh.go

@@ -0,0 +1,98 @@
+package main
+
+/*
+import (
+	"flag"
+	"github.com/netbirdio/netbird/iface"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+	"net"
+	"net/http"
+	_ "net/http/pprof"
+	"time"
+)
+
+var name = flag.String("name", "wg0", "WireGuard interface name")
+var addr = flag.String("addr", "100.64.0.1/24", "interface WireGuard IP addr")
+var key = flag.String("key", "100.64.0.1/24", "WireGuard private key")
+var port = flag.Int("port", 51820, "WireGuard port")
+
+var remoteKey = flag.String("remote-key", "", "remote WireGuard public key")
+var remoteAddr = flag.String("remote-addr", "100.64.0.2/32", "remote WireGuard IP addr")
+var remoteEndpoint = flag.String("remote-endpoint", "127.0.0.1:51820", "remote WireGuard endpoint")
+
+func fff() {
+
+	flag.Parse()
+
+	go func() {
+		log.Println(http.ListenAndServe("localhost:6060", nil))
+	}()
+
+	myKey, err := wgtypes.ParseKey(*key)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	log.Infof("public key and addr [%s] [%s] ", myKey.PublicKey().String(), *addr)
+
+	wgIFace, err := iface.NewWGIFace(*name, *addr, 1280)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	defer wgIFace.Close()
+
+	// todo wrap into UDPMux
+	sharedSock, _, err := listenNet("udp4", *port)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	defer sharedSock.Close()
+
+	//	err = wgIFace.Create()
+	err = wgIFace.CreateNew(sharedSock)
+	if err != nil {
+		log.Errorf("failed to create interface %s %v", *name, err)
+		return
+	}
+
+	err = wgIFace.Configure(*key, *port)
+	if err != nil {
+		log.Errorf("failed to configure interface %s %v", *name, err)
+		return
+	}
+
+	ip, err := net.ResolveUDPAddr("udp4", *remoteEndpoint)
+	if err != nil {
+		// handle error
+	}
+
+	err = wgIFace.UpdatePeer(*remoteKey, *remoteAddr, 20*time.Second, ip, nil)
+	if err != nil {
+		log.Errorf("failed to configure remote peer %s %v", *remoteKey, err)
+		return
+	}
+
+	select {}
+
+}
+
+func listenNet(network string, port int) (*net.UDPConn, int, error) {
+	conn, err := net.ListenUDP(network, &net.UDPAddr{Port: port})
+	if err != nil {
+		return nil, 0, err
+	}
+
+	// Retrieve port.
+	laddr := conn.LocalAddr()
+	uaddr, err := net.ResolveUDPAddr(
+		laddr.Network(),
+		laddr.String(),
+	)
+	if err != nil {
+		return nil, 0, err
+	}
+	return conn, uaddr.Port, nil
+}*/

client/internal/config.go

@@ -37,6 +37,7 @@ type Config struct {
 	ManagementURL  *url.URL
 	AdminURL       *url.URL
 	WgIface        string
+	WgPort         int
 	IFaceBlackList []string
 	// SSHKey is a private SSH key in a PEM format
 	SSHKey string
@@ -49,7 +50,13 @@ func createNewConfig(managementURL, adminURL, configPath, preSharedKey string) (
 	if err != nil {
 		return nil, err
 	}
-	config := &Config{SSHKey: string(pem), PrivateKey: wgKey, WgIface: iface.WgInterfaceDefault, IFaceBlackList: []string{}}
+	config := &Config{
+		SSHKey:         string(pem),
+		PrivateKey:     wgKey,
+		WgIface:        iface.WgInterfaceDefault,
+		WgPort:         iface.DefaultWgPort,
+		IFaceBlackList: []string{},
+	}
 	if managementURL != "" {
 		URL, err := ParseURL("Management URL", managementURL)
 		if err != nil {
@@ -72,8 +79,8 @@ func createNewConfig(managementURL, adminURL, configPath, preSharedKey string) (
 		config.AdminURL = newURL
 	}
 
-	config.IFaceBlackList = []string{iface.WgInterfaceDefault, "tun0", "zt", "ZeroTier", "utun", "wg", "ts",
+	config.IFaceBlackList = []string{iface.WgInterfaceDefault, "wt", "utun", "tun0", "zt", "ZeroTier", "utun", "wg", "ts",
-		"Tailscale", "tailscale"}
+		"Tailscale", "tailscale", "docker", "vet"}
 
 	err = util.WriteJson(configPath, config)
 	if err != nil {
@@ -150,6 +157,11 @@ func ReadConfig(managementURL, adminURL, configPath string, preSharedKey *string
 		refresh = true
 	}
 
+	if config.WgPort == 0 {
+		config.WgPort = iface.DefaultWgPort
+		refresh = true
+	}
+
 	if refresh {
 		// since we have new management URL, we need to update config file
 		if err := util.WriteJson(configPath, config); err != nil {
@@ -197,9 +209,14 @@ type ProviderConfig struct {
 	// ClientSecret An IDP application client secret
 	ClientSecret string
 	// Domain An IDP API domain
+	// Deprecated. Use OIDCConfigEndpoint instead
 	Domain string
 	// Audience An Audience for to authorization validation
 	Audience string
+	// TokenEndpoint is the endpoint of an IDP manager where clients can obtain access token
+	TokenEndpoint string
+	// DeviceAuthEndpoint is the endpoint of an IDP manager where clients can obtain device authorization code
+	DeviceAuthEndpoint string
 }
 
 func GetDeviceAuthorizationFlowInfo(ctx context.Context, config *Config) (DeviceAuthorizationFlow, error) {
@@ -221,7 +238,13 @@ func GetDeviceAuthorizationFlowInfo(ctx context.Context, config *Config) (Device
 		log.Errorf("failed connecting to Management Service %s %v", config.ManagementURL.String(), err)
 		return DeviceAuthorizationFlow{}, err
 	}
-	log.Debugf("connected to management Service %s", config.ManagementURL.String())
+	log.Debugf("connected to the Management service %s", config.ManagementURL.String())
+	defer func() {
+		err = mgmClient.Close()
+		if err != nil {
+			log.Warnf("failed to close the Management service client %v", err)
+		}
+	}()
 
 	serverKey, err := mgmClient.GetServerPublicKey()
 	if err != nil {
@@ -240,20 +263,16 @@ func GetDeviceAuthorizationFlowInfo(ctx context.Context, config *Config) (Device
 		}
 	}
 
-	err = mgmClient.Close()
-	if err != nil {
-		log.Errorf("failed closing Management Service client: %v", err)
-		return DeviceAuthorizationFlow{}, err
-	}
-
 	return DeviceAuthorizationFlow{
 		Provider: protoDeviceAuthorizationFlow.Provider.String(),
 
 		ProviderConfig: ProviderConfig{
-			Audience:     protoDeviceAuthorizationFlow.ProviderConfig.Audience,
+			Audience:           protoDeviceAuthorizationFlow.GetProviderConfig().GetAudience(),
-			ClientID:     protoDeviceAuthorizationFlow.ProviderConfig.ClientID,
+			ClientID:           protoDeviceAuthorizationFlow.GetProviderConfig().GetClientID(),
-			ClientSecret: protoDeviceAuthorizationFlow.ProviderConfig.ClientSecret,
+			ClientSecret:       protoDeviceAuthorizationFlow.GetProviderConfig().GetClientSecret(),
-			Domain:       protoDeviceAuthorizationFlow.ProviderConfig.Domain,
+			Domain:             protoDeviceAuthorizationFlow.GetProviderConfig().Domain,
+			TokenEndpoint:      protoDeviceAuthorizationFlow.GetProviderConfig().GetTokenEndpoint(),
+			DeviceAuthEndpoint: protoDeviceAuthorizationFlow.GetProviderConfig().GetDeviceAuthEndpoint(),
 		},
 	}, nil
 }

client/internal/connect.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"github.com/netbirdio/netbird/client/ssh"
 	nbStatus "github.com/netbirdio/netbird/client/status"
-	mgmtcmd "github.com/netbirdio/netbird/management/cmd"
 	"strings"
 	"time"
 
@@ -80,9 +79,21 @@ func RunClient(ctx context.Context, config *Config, statusRecorder *nbStatus.Sta
 			cancel()
 		}()
 
+		log.Debugf("conecting to the Management service %s", config.ManagementURL.Host)
+		mgmClient, err := mgm.NewClient(engineCtx, config.ManagementURL.Host, myPrivateKey, mgmTlsEnabled)
+		if err != nil {
+			return wrapErr(gstatus.Errorf(codes.FailedPrecondition, "failed connecting to Management Service : %s", err))
+		}
+		log.Debugf("connected to the Management service %s", config.ManagementURL.Host)
+		defer func() {
+			err = mgmClient.Close()
+			if err != nil {
+				log.Warnf("failed to close the Management service client %v", err)
+			}
+		}()
+
 		// connect (just a connection, no stream yet) and login to Management Service to get an initial global Wiretrustee config
-		mgmClient, loginResp, err := connectToManagement(engineCtx, config.ManagementURL.Host, myPrivateKey, mgmTlsEnabled,
+		loginResp, err := loginToManagement(engineCtx, mgmClient, publicSSHKey)
-			publicSSHKey)
 		if err != nil {
 			log.Debug(err)
 			if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.PermissionDenied) {
@@ -115,6 +126,12 @@ func RunClient(ctx context.Context, config *Config, statusRecorder *nbStatus.Sta
 			log.Error(err)
 			return wrapErr(err)
 		}
+		defer func() {
+			err = signalClient.Close()
+			if err != nil {
+				log.Warnf("failed closing Signal service client %v", err)
+			}
+		}()
 
 		statusRecorder.MarkSignalConnected(signalURL)
 
@@ -140,18 +157,6 @@ func RunClient(ctx context.Context, config *Config, statusRecorder *nbStatus.Sta
 
 		backOff.Reset()
 
-		err = mgmClient.Close()
-		if err != nil {
-			log.Errorf("failed closing Management Service client %v", err)
-			return wrapErr(err)
-		}
-
-		err = signalClient.Close()
-		if err != nil {
-			log.Errorf("failed closing Signal Service client %v", err)
-			return wrapErr(err)
-		}
-
 		err = engine.Stop()
 		if err != nil {
 			log.Errorf("failed stopping engine %v", err)
@@ -183,7 +188,7 @@ func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.Pe
 		WgAddr:         peerConfig.Address,
 		IFaceBlackList: config.IFaceBlackList,
 		WgPrivateKey:   key,
-		WgPort:         iface.DefaultWgPort,
+		WgPort:         config.WgPort,
 		SSHKey:         []byte(config.SSHKey),
 	}
 
@@ -216,31 +221,28 @@ func connectToSignal(ctx context.Context, wtConfig *mgmProto.WiretrusteeConfig,
 	return signalClient, nil
 }
 
-// connectToManagement creates Management Services client, establishes a connection, logs-in and gets a global Wiretrustee config (signal, turn, stun hosts, etc)
+// loginToManagement creates Management Services client, establishes a connection, logs-in and gets a global Wiretrustee config (signal, turn, stun hosts, etc)
-func connectToManagement(ctx context.Context, managementAddr string, ourPrivateKey wgtypes.Key, tlsEnabled bool, pubSSHKey []byte) (*mgm.GrpcClient, *mgmProto.LoginResponse, error) {
+func loginToManagement(ctx context.Context, client mgm.Client, pubSSHKey []byte) (*mgmProto.LoginResponse, error) {
-	log.Debugf("connecting to Management Service %s", managementAddr)
-	client, err := mgm.NewClient(ctx, managementAddr, ourPrivateKey, tlsEnabled)
-	if err != nil {
-		return nil, nil, gstatus.Errorf(codes.FailedPrecondition, "failed connecting to Management Service : %s", err)
-	}
-	log.Debugf("connected to management server %s", managementAddr)
 
 	serverPublicKey, err := client.GetServerPublicKey()
 	if err != nil {
-		return nil, nil, gstatus.Errorf(codes.FailedPrecondition, "failed while getting Management Service public key: %s", err)
+		return nil, gstatus.Errorf(codes.FailedPrecondition, "failed while getting Management Service public key: %s", err)
 	}
 
 	sysInfo := system.GetInfo(ctx)
 	loginResp, err := client.Login(*serverPublicKey, sysInfo, pubSSHKey)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	log.Debugf("peer logged in to Management Service %s", managementAddr)
+	return loginResp, nil
-
-	return client, loginResp, nil
 }
 
+// ManagementLegacyPort is the port that was used before by the Management gRPC server.
+// It is used for backward compatibility now.
+// NB: hardcoded from github.com/netbirdio/netbird/management/cmd to avoid import
+const ManagementLegacyPort = 33073
+
 // UpdateOldManagementPort checks whether client can switch to the new Management port 443.
 // If it can switch, then it updates the config and returns a new one. Otherwise, it returns the provided config.
 // The check is performed only for the NetBird's managed version.
@@ -261,7 +263,7 @@ func UpdateOldManagementPort(ctx context.Context, config *Config, configPath str
 		return config, nil
 	}
 
-	if mgmTlsEnabled && config.ManagementURL.Port() == fmt.Sprintf("%d", mgmtcmd.ManagementLegacyPort) {
+	if mgmTlsEnabled && config.ManagementURL.Port() == fmt.Sprintf("%d", ManagementLegacyPort) {
 
 		newURL, err := ParseURL("Management URL", fmt.Sprintf("%s://%s:%d",
 			config.ManagementURL.Scheme, config.ManagementURL.Hostname(), 443))
@@ -282,7 +284,12 @@ func UpdateOldManagementPort(ctx context.Context, config *Config, configPath str
 			log.Infof("couldn't switch to the new Management %s", newURL.String())
 			return config, err
 		}
-		defer client.Close() //nolint
+		defer func() {
+			err = client.Close()
+			if err != nil {
+				log.Warnf("failed to close the Management service client %v", err)
+			}
+		}()
 
 		// gRPC check
 		_, err = client.GetServerPublicKey()

client/internal/engine.go

@@ -3,10 +3,11 @@ package internal
 import (
 	"context"
 	"fmt"
+	"github.com/netbirdio/netbird/client/internal/routemanager"
 	nbssh "github.com/netbirdio/netbird/client/ssh"
 	nbstatus "github.com/netbirdio/netbird/client/status"
+	"github.com/netbirdio/netbird/route"
 	"math/rand"
-	"net"
 	"reflect"
 	"runtime"
 	"strings"
@@ -87,10 +88,7 @@ type Engine struct {
 
 	wgInterface *iface.WGIface
 
-	udpMux          ice.UDPMux
+	iceMux ice.UniversalUDPMux
-	udpMuxSrflx     ice.UniversalUDPMux
-	udpMuxConn      *net.UDPConn
-	udpMuxConnSrflx *net.UDPConn
 
 	// networkSerial is the latest CurrentSerial (state ID) of the network sent by the Management service
 	networkSerial uint64
@@ -99,6 +97,8 @@ type Engine struct {
 	sshServer     nbssh.Server
 
 	statusRecorder *nbstatus.Status
+
+	routeManager routemanager.Manager
 }
 
 // Peer is an instance of the Connection Peer
@@ -151,30 +151,6 @@ func (e *Engine) Stop() error {
 		}
 	}
 
-	if e.udpMux != nil {
-		if err := e.udpMux.Close(); err != nil {
-			log.Debugf("close udp mux: %v", err)
-		}
-	}
-
-	if e.udpMuxSrflx != nil {
-		if err := e.udpMuxSrflx.Close(); err != nil {
-			log.Debugf("close server reflexive udp mux: %v", err)
-		}
-	}
-
-	if e.udpMuxConn != nil {
-		if err := e.udpMuxConn.Close(); err != nil {
-			log.Debugf("close udp mux connection: %v", err)
-		}
-	}
-
-	if e.udpMuxConnSrflx != nil {
-		if err := e.udpMuxConnSrflx.Close(); err != nil {
-			log.Debugf("close server reflexive udp mux connection: %v", err)
-		}
-	}
-
 	if !isNil(e.sshServer) {
 		err := e.sshServer.Stop()
 		if err != nil {
@@ -182,6 +158,10 @@ func (e *Engine) Stop() error {
 		}
 	}
 
+	if e.routeManager != nil {
+		e.routeManager.Stop()
+	}
+
 	log.Infof("stopped Netbird Engine")
 
 	return nil
@@ -205,33 +185,35 @@ func (e *Engine) Start() error {
 		return err
 	}
 
-	e.udpMuxConn, err = net.ListenUDP("udp4", &net.UDPAddr{Port: e.config.UDPMuxPort})
+	bind := &iface.ICEBind{}
-	if err != nil {
+	err = e.wgInterface.CreateNew(bind)
-		log.Errorf("failed listening on UDP port %d: [%s]", e.config.UDPMuxPort, err.Error())
-		return err
-	}
-
-	e.udpMuxConnSrflx, err = net.ListenUDP("udp4", &net.UDPAddr{Port: e.config.UDPMuxSrflxPort})
-	if err != nil {
-		log.Errorf("failed listening on UDP port %d: [%s]", e.config.UDPMuxSrflxPort, err.Error())
-		return err
-	}
-
-	e.udpMux = ice.NewUDPMuxDefault(ice.UDPMuxParams{UDPConn: e.udpMuxConn})
-	e.udpMuxSrflx = ice.NewUniversalUDPMuxDefault(ice.UniversalUDPMuxParams{UDPConn: e.udpMuxConnSrflx})
-
-	err = e.wgInterface.Create()
 	if err != nil {
 		log.Errorf("failed creating tunnel interface %s: [%s]", wgIfaceName, err.Error())
 		return err
 	}
 
-	err = e.wgInterface.Configure(myPrivateKey.String(), e.config.WgPort)
+	port, err := e.wgInterface.GetListenPort()
+	if err != nil {
+		return err
+	}
+
+	err = e.wgInterface.Configure(myPrivateKey.String(), *port)
 	if err != nil {
 		log.Errorf("failed configuring Wireguard interface [%s]: %s", wgIfaceName, err.Error())
 		return err
 	}
 
+	iceMux, err := bind.GetICEMux()
+	if err != nil {
+		return err
+	}
+	e.iceMux = iceMux
+
+	log.Infof("NetBird Engine started listening on WireGuard port %d", *port)
+
+	e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.wgInterface, e.statusRecorder)
+	e.config.WgPort = *port
+
 	e.receiveSignalEvents()
 	e.receiveManagementEvents()
 
@@ -382,15 +364,14 @@ func signalCandidate(candidate ice.Candidate, myKey wgtypes.Key, remoteKey wgtyp
 		},
 	})
 	if err != nil {
-		log.Errorf("failed signaling candidate to the remote peer %s %s", remoteKey.String(), err)
-		// todo ??
 		return err
 	}
 
 	return nil
 }
 
-func signalAuth(uFrag string, pwd string, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client, isAnswer bool) error {
+// SignalOfferAnswer signals either an offer or an answer to remote peer
+func SignalOfferAnswer(offerAnswer peer.OfferAnswer, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client, isAnswer bool) error {
 	var t sProto.Body_Type
 	if isAnswer {
 		t = sProto.Body_ANSWER
@@ -398,9 +379,9 @@ func signalAuth(uFrag string, pwd string, myKey wgtypes.Key, remoteKey wgtypes.K
 		t = sProto.Body_OFFER
 	}
 
-	msg, err := signal.MarshalCredential(myKey, remoteKey, &signal.Credential{
+	msg, err := signal.MarshalCredential(myKey, offerAnswer.WgListenPort, remoteKey, &signal.Credential{
-		UFrag: uFrag,
+		UFrag: offerAnswer.IceCredentials.UFrag,
-		Pwd:   pwd,
+		Pwd:   offerAnswer.IceCredentials.Pwd,
 	}, t)
 	if err != nil {
 		return err
@@ -620,11 +601,37 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error {
 			}
 		}
 	}
+	protoRoutes := networkMap.GetRoutes()
+	if protoRoutes == nil {
+		protoRoutes = []*mgmProto.Route{}
+	}
+	err := e.routeManager.UpdateRoutes(serial, toRoutes(protoRoutes))
+	if err != nil {
+		log.Errorf("failed to update routes, err: %v", err)
+	}
 
 	e.networkSerial = serial
 	return nil
 }
 
+func toRoutes(protoRoutes []*mgmProto.Route) []*route.Route {
+	routes := make([]*route.Route, 0)
+	for _, protoRoute := range protoRoutes {
+		_, prefix, _ := route.ParseNetwork(protoRoute.Network)
+		convertedRoute := &route.Route{
+			ID:          protoRoute.ID,
+			Network:     prefix,
+			NetID:       protoRoute.NetID,
+			NetworkType: route.NetworkType(protoRoute.NetworkType),
+			Peer:        protoRoute.Peer,
+			Metric:      int(protoRoute.Metric),
+			Masquerade:  protoRoute.Masquerade,
+		}
+		routes = append(routes, convertedRoute)
+	}
+	return routes
+}
+
 // addNewPeers adds peers that were not know before but arrived from the Management service with the update
 func (e *Engine) addNewPeers(peersUpdate []*mgmProto.RemotePeerConfig) error {
 	for _, p := range peersUpdate {
@@ -704,6 +711,7 @@ func (e Engine) peerExists(peerKey string) bool {
 }
 
 func (e Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, error) {
+	log.Debugf("creating peer connection %s", pubKey)
 	var stunTurn []*ice.URL
 	stunTurn = append(stunTurn, e.STUNs...)
 	stunTurn = append(stunTurn, e.TURNs...)
@@ -724,9 +732,10 @@ func (e Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, er
 		StunTurn:           stunTurn,
 		InterfaceBlackList: e.config.IFaceBlackList,
 		Timeout:            timeout,
-		UDPMux:             e.udpMux,
+		UDPMux:             e.iceMux,
-		UDPMuxSrflx:        e.udpMuxSrflx,
+		UDPMuxSrflx:        e.iceMux,
 		ProxyConfig:        proxyConfig,
+		LocalWgPort:        e.config.WgPort,
 	}
 
 	peerConn, err := peer.NewConn(config, e.statusRecorder)
@@ -739,16 +748,16 @@ func (e Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, er
 		return nil, err
 	}
 
-	signalOffer := func(uFrag string, pwd string) error {
+	signalOffer := func(offerAnswer peer.OfferAnswer) error {
-		return signalAuth(uFrag, pwd, e.config.WgPrivateKey, wgPubKey, e.signal, false)
+		return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, false)
 	}
 
 	signalCandidate := func(candidate ice.Candidate) error {
 		return signalCandidate(candidate, e.config.WgPrivateKey, wgPubKey, e.signal)
 	}
 
-	signalAnswer := func(uFrag string, pwd string) error {
+	signalAnswer := func(offerAnswer peer.OfferAnswer) error {
-		return signalAuth(uFrag, pwd, e.config.WgPrivateKey, wgPubKey, e.signal, true)
+		return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, true)
 	}
 
 	peerConn.SetSignalCandidate(signalCandidate)
@@ -777,18 +786,26 @@ func (e *Engine) receiveSignalEvents() {
 				if err != nil {
 					return err
 				}
-				conn.OnRemoteOffer(peer.IceCredentials{
+				conn.OnRemoteOffer(peer.OfferAnswer{
-					UFrag: remoteCred.UFrag,
+					IceCredentials: peer.IceCredentials{
-					Pwd:   remoteCred.Pwd,
+						UFrag: remoteCred.UFrag,
+						Pwd:   remoteCred.Pwd,
+					},
+					WgListenPort: int(msg.GetBody().GetWgListenPort()),
+					Version:      msg.GetBody().GetNetBirdVersion(),
 				})
 			case sProto.Body_ANSWER:
 				remoteCred, err := signal.UnMarshalCredential(msg)
 				if err != nil {
 					return err
 				}
-				conn.OnRemoteAnswer(peer.IceCredentials{
+				conn.OnRemoteAnswer(peer.OfferAnswer{
-					UFrag: remoteCred.UFrag,
+					IceCredentials: peer.IceCredentials{
-					Pwd:   remoteCred.Pwd,
+						UFrag: remoteCred.UFrag,
+						Pwd:   remoteCred.Pwd,
+					},
+					WgListenPort: int(msg.GetBody().GetWgListenPort()),
+					Version:      msg.GetBody().GetNetBirdVersion(),
 				})
 			case sProto.Body_CANDIDATE:
 				candidate, err := ice.UnmarshalCandidate(msg.GetBody().Payload)

client/internal/engine_test.go

@@ -3,11 +3,14 @@ package internal
 import (
 	"context"
 	"fmt"
+	"github.com/netbirdio/netbird/client/internal/routemanager"
 	"github.com/netbirdio/netbird/client/ssh"
 	nbstatus "github.com/netbirdio/netbird/client/status"
 	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/route"
 	"github.com/stretchr/testify/assert"
 	"net"
+	"net/netip"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -196,6 +199,7 @@ func TestEngine_UpdateNetworkMap(t *testing.T) {
 		WgPort:       33100,
 	}, nbstatus.NewRecorder())
 	engine.wgInterface, err = iface.NewWGIFace("utun102", "100.64.0.1/24", iface.DefaultMTU)
+	engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), engine.wgInterface, engine.statusRecorder)
 
 	type testCase struct {
 		name       string
@@ -426,6 +430,142 @@ func TestEngine_Sync(t *testing.T) {
 	}
 }
 
+func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) {
+	testCases := []struct {
+		name           string
+		inputErr       error
+		networkMap     *mgmtProto.NetworkMap
+		expectedLen    int
+		expectedRoutes []*route.Route
+		expectedSerial uint64
+	}{
+		{
+			name: "Routes Update Should Be Passed To Manager",
+			networkMap: &mgmtProto.NetworkMap{
+				Serial:             1,
+				PeerConfig:         nil,
+				RemotePeersIsEmpty: false,
+				Routes: []*mgmtProto.Route{
+					{
+						ID:          "a",
+						Network:     "192.168.0.0/24",
+						NetID:       "n1",
+						Peer:        "p1",
+						NetworkType: 1,
+						Masquerade:  false,
+					},
+					{
+						ID:          "b",
+						Network:     "192.168.1.0/24",
+						NetID:       "n2",
+						Peer:        "p1",
+						NetworkType: 1,
+						Masquerade:  false,
+					},
+				},
+			},
+			expectedLen: 2,
+			expectedRoutes: []*route.Route{
+				{
+					ID:          "a",
+					Network:     netip.MustParsePrefix("192.168.0.0/24"),
+					NetID:       "n1",
+					Peer:        "p1",
+					NetworkType: 1,
+					Masquerade:  false,
+				},
+				{
+					ID:          "b",
+					Network:     netip.MustParsePrefix("192.168.1.0/24"),
+					NetID:       "n2",
+					Peer:        "p1",
+					NetworkType: 1,
+					Masquerade:  false,
+				},
+			},
+			expectedSerial: 1,
+		},
+		{
+			name: "Empty Routes Update Should Be Passed",
+			networkMap: &mgmtProto.NetworkMap{
+				Serial:             1,
+				PeerConfig:         nil,
+				RemotePeersIsEmpty: false,
+				Routes:             nil,
+			},
+			expectedLen:    0,
+			expectedRoutes: []*route.Route{},
+			expectedSerial: 1,
+		},
+		{
+			name:     "Error Shouldn't Break Engine",
+			inputErr: fmt.Errorf("mocking error"),
+			networkMap: &mgmtProto.NetworkMap{
+				Serial:             1,
+				PeerConfig:         nil,
+				RemotePeersIsEmpty: false,
+				Routes:             nil,
+			},
+			expectedLen:    0,
+			expectedRoutes: []*route.Route{},
+			expectedSerial: 1,
+		},
+	}
+
+	for n, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			// test setup
+			key, err := wgtypes.GeneratePrivateKey()
+			if err != nil {
+				t.Fatal(err)
+				return
+			}
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			wgIfaceName := fmt.Sprintf("utun%d", 104+n)
+			wgAddr := fmt.Sprintf("100.66.%d.1/24", n)
+
+			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+				WgIfaceName:  wgIfaceName,
+				WgAddr:       wgAddr,
+				WgPrivateKey: key,
+				WgPort:       33100,
+			}, nbstatus.NewRecorder())
+			engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, iface.DefaultMTU)
+			assert.NoError(t, err, "shouldn't return error")
+			input := struct {
+				inputSerial uint64
+				inputRoutes []*route.Route
+			}{}
+
+			mockRouteManager := &routemanager.MockManager{
+				UpdateRoutesFunc: func(updateSerial uint64, newRoutes []*route.Route) error {
+					input.inputSerial = updateSerial
+					input.inputRoutes = newRoutes
+					return testCase.inputErr
+				},
+			}
+
+			engine.routeManager = mockRouteManager
+
+			defer func() {
+				exitErr := engine.Stop()
+				if exitErr != nil {
+					return
+				}
+			}()
+
+			err = engine.updateNetworkMap(testCase.networkMap)
+			assert.NoError(t, err, "shouldn't return error")
+			assert.Equal(t, testCase.expectedSerial, input.inputSerial, "serial should match")
+			assert.Len(t, input.inputRoutes, testCase.expectedLen, "routes len should match")
+			assert.Equal(t, testCase.expectedRoutes, input.inputRoutes, "routes should match")
+		})
+	}
+}
+
 func TestEngine_MultiplePeers(t *testing.T) {
 	// log.SetLevel(log.DebugLevel)
 

client/internal/login.go

@@ -26,13 +26,19 @@ func Login(ctx context.Context, config *Config, setupKey string, jwtToken string
 		mgmTlsEnabled = true
 	}
 
-	log.Debugf("connecting to Management Service %s", config.ManagementURL.String())
+	log.Debugf("connecting to the Management service %s", config.ManagementURL.String())
 	mgmClient, err := mgm.NewClient(ctx, config.ManagementURL.Host, myPrivateKey, mgmTlsEnabled)
 	if err != nil {
-		log.Errorf("failed connecting to Management Service %s %v", config.ManagementURL.String(), err)
+		log.Errorf("failed connecting to the Management service %s %v", config.ManagementURL.String(), err)
 		return err
 	}
-	log.Debugf("connected to management Service %s", config.ManagementURL.String())
+	log.Debugf("connected to the Management service %s", config.ManagementURL.String())
+	defer func() {
+		err = mgmClient.Close()
+		if err != nil {
+			log.Warnf("failed to close the Management service client %v", err)
+		}
+	}()
 
 	serverKey, err := mgmClient.GetServerPublicKey()
 	if err != nil {
@@ -53,7 +59,7 @@ func Login(ctx context.Context, config *Config, setupKey string, jwtToken string
 
 	err = mgmClient.Close()
 	if err != nil {
-		log.Errorf("failed closing Management Service client: %v", err)
+		log.Errorf("failed to close the Management service client: %v", err)
 		return err
 	}
 

client/internal/oauth.go

@@ -5,8 +5,10 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"net/url"
+	"reflect"
 	"strings"
 	"time"
 )
@@ -14,7 +16,6 @@ import (
 // OAuthClient is a OAuth client interface for various idp providers
 type OAuthClient interface {
 	RequestDeviceCode(ctx context.Context) (DeviceAuthInfo, error)
-	RotateAccessToken(ctx context.Context, refreshToken string) (TokenInfo, error)
 	WaitToken(ctx context.Context, info DeviceAuthInfo) (TokenInfo, error)
 	GetClientID(ctx context.Context) string
 }
@@ -55,8 +56,10 @@ type Hosted struct {
 	Audience string
 	// Hosted Native application client id
 	ClientID string
-	// Hosted domain
+	// TokenEndpoint to request access token
-	Domain string
+	TokenEndpoint string
+	// DeviceAuthEndpoint to request device authorization code
+	DeviceAuthEndpoint string
 
 	HTTPClient HTTPClient
 }
@@ -84,11 +87,11 @@ type TokenRequestResponse struct {
 
 // Claims used when validating the access token
 type Claims struct {
-	Audience string `json:"aud"`
+	Audience interface{} `json:"aud"`
 }
 
 // NewHostedDeviceFlow returns an Hosted OAuth client
-func NewHostedDeviceFlow(audience string, clientID string, domain string) *Hosted {
+func NewHostedDeviceFlow(audience string, clientID string, tokenEndpoint string, deviceAuthEndpoint string) *Hosted {
 	httpTransport := http.DefaultTransport.(*http.Transport).Clone()
 	httpTransport.MaxIdleConns = 5
 
@@ -98,10 +101,11 @@ func NewHostedDeviceFlow(audience string, clientID string, domain string) *Hoste
 	}
 
 	return &Hosted{
-		Audience:   audience,
+		Audience:           audience,
-		ClientID:   clientID,
+		ClientID:           clientID,
-		Domain:     domain,
+		TokenEndpoint:      tokenEndpoint,
-		HTTPClient: httpClient,
+		HTTPClient:         httpClient,
+		DeviceAuthEndpoint: deviceAuthEndpoint,
 	}
 }
 
@@ -112,22 +116,15 @@ func (h *Hosted) GetClientID(ctx context.Context) string {
 
 // RequestDeviceCode requests a device code login flow information from Hosted
 func (h *Hosted) RequestDeviceCode(ctx context.Context) (DeviceAuthInfo, error) {
-	url := "https://" + h.Domain + "/oauth/device/code"
+	form := url.Values{}
-	codePayload := RequestDeviceCodePayload{
+	form.Add("client_id", h.ClientID)
-		Audience: h.Audience,
+	form.Add("audience", h.Audience)
-		ClientID: h.ClientID,
+	req, err := http.NewRequest("POST", h.DeviceAuthEndpoint,
-	}
+		strings.NewReader(form.Encode()))
-	p, err := json.Marshal(codePayload)
-	if err != nil {
-		return DeviceAuthInfo{}, fmt.Errorf("parsing payload failed with error: %v", err)
-	}
-	payload := strings.NewReader(string(p))
-	req, err := http.NewRequest("POST", url, payload)
 	if err != nil {
 		return DeviceAuthInfo{}, fmt.Errorf("creating request failed with error: %v", err)
 	}
-
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-	req.Header.Add("content-type", "application/json")
 
 	res, err := h.HTTPClient.Do(req)
 	if err != nil {
@@ -135,7 +132,7 @@ func (h *Hosted) RequestDeviceCode(ctx context.Context) (DeviceAuthInfo, error)
 	}
 
 	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
+	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		return DeviceAuthInfo{}, fmt.Errorf("reading body failed with error: %v", err)
 	}
@@ -153,6 +150,48 @@ func (h *Hosted) RequestDeviceCode(ctx context.Context) (DeviceAuthInfo, error)
 	return deviceCode, err
 }
 
+func (h *Hosted) requestToken(info DeviceAuthInfo) (TokenRequestResponse, error) {
+	form := url.Values{}
+	form.Add("client_id", h.ClientID)
+	form.Add("grant_type", HostedGrantType)
+	form.Add("device_code", info.DeviceCode)
+	req, err := http.NewRequest("POST", h.TokenEndpoint, strings.NewReader(form.Encode()))
+	if err != nil {
+		return TokenRequestResponse{}, fmt.Errorf("failed to create request access token: %v", err)
+	}
+
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+	res, err := h.HTTPClient.Do(req)
+	if err != nil {
+		return TokenRequestResponse{}, fmt.Errorf("failed to request access token with error: %v", err)
+	}
+
+	defer func() {
+		err := res.Body.Close()
+		if err != nil {
+			return
+		}
+	}()
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return TokenRequestResponse{}, fmt.Errorf("failed reading access token response body with error: %v", err)
+	}
+
+	if res.StatusCode > 499 {
+		return TokenRequestResponse{}, fmt.Errorf("access token response returned code: %s", string(body))
+	}
+
+	tokenResponse := TokenRequestResponse{}
+	err = json.Unmarshal(body, &tokenResponse)
+	if err != nil {
+		return TokenRequestResponse{}, fmt.Errorf("parsing token response failed with error: %v", err)
+	}
+
+	return tokenResponse, nil
+}
+
 // WaitToken waits user's login and authorize the app. Once the user's authorize
 // it retrieves the access token from Hosted's endpoint and validates it before returning
 func (h *Hosted) WaitToken(ctx context.Context, info DeviceAuthInfo) (TokenInfo, error) {
@@ -163,24 +202,8 @@ func (h *Hosted) WaitToken(ctx context.Context, info DeviceAuthInfo) (TokenInfo,
 		case <-ctx.Done():
 			return TokenInfo{}, ctx.Err()
 		case <-ticker.C:
-			url := "https://" + h.Domain + "/oauth/token"
-			tokenReqPayload := TokenRequestPayload{
-				GrantType:  HostedGrantType,
-				DeviceCode: info.DeviceCode,
-				ClientID:   h.ClientID,
-			}
 
-			body, statusCode, err := requestToken(h.HTTPClient, url, tokenReqPayload)
+			tokenResponse, err := h.requestToken(info)
-			if err != nil {
-				return TokenInfo{}, fmt.Errorf("wait for token: %v", err)
-			}
-
-			if statusCode > 499 {
-				return TokenInfo{}, fmt.Errorf("wait token code returned error: %s", string(body))
-			}
-
-			tokenResponse := TokenRequestResponse{}
-			err = json.Unmarshal(body, &tokenResponse)
 			if err != nil {
 				return TokenInfo{}, fmt.Errorf("parsing token response failed with error: %v", err)
 			}
@@ -214,71 +237,6 @@ func (h *Hosted) WaitToken(ctx context.Context, info DeviceAuthInfo) (TokenInfo,
 	}
 }
 
-// RotateAccessToken requests a new token using an existing refresh token
-func (h *Hosted) RotateAccessToken(ctx context.Context, refreshToken string) (TokenInfo, error) {
-	url := "https://" + h.Domain + "/oauth/token"
-	tokenReqPayload := TokenRequestPayload{
-		GrantType:    HostedRefreshGrant,
-		ClientID:     h.ClientID,
-		RefreshToken: refreshToken,
-	}
-
-	body, statusCode, err := requestToken(h.HTTPClient, url, tokenReqPayload)
-	if err != nil {
-		return TokenInfo{}, fmt.Errorf("rotate access token: %v", err)
-	}
-
-	if statusCode != 200 {
-		return TokenInfo{}, fmt.Errorf("rotating token returned error: %s", string(body))
-	}
-
-	tokenResponse := TokenRequestResponse{}
-	err = json.Unmarshal(body, &tokenResponse)
-	if err != nil {
-		return TokenInfo{}, fmt.Errorf("parsing token response failed with error: %v", err)
-	}
-
-	err = isValidAccessToken(tokenResponse.AccessToken, h.Audience)
-	if err != nil {
-		return TokenInfo{}, fmt.Errorf("validate access token failed with error: %v", err)
-	}
-
-	tokenInfo := TokenInfo{
-		AccessToken:  tokenResponse.AccessToken,
-		TokenType:    tokenResponse.TokenType,
-		RefreshToken: tokenResponse.RefreshToken,
-		IDToken:      tokenResponse.IDToken,
-		ExpiresIn:    tokenResponse.ExpiresIn,
-	}
-	return tokenInfo, err
-}
-
-func requestToken(client HTTPClient, url string, tokenReqPayload TokenRequestPayload) ([]byte, int, error) {
-	p, err := json.Marshal(tokenReqPayload)
-	if err != nil {
-		return nil, 0, fmt.Errorf("parsing token payload failed with error: %v", err)
-	}
-	payload := strings.NewReader(string(p))
-	req, err := http.NewRequest("POST", url, payload)
-	if err != nil {
-		return nil, 0, fmt.Errorf("creating token request failed with error: %v", err)
-	}
-
-	req.Header.Add("content-type", "application/json")
-
-	res, err := client.Do(req)
-	if err != nil {
-		return nil, 0, fmt.Errorf("doing token request failed with error: %v", err)
-	}
-
-	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, 0, fmt.Errorf("reading token body failed with error: %v", err)
-	}
-	return body, res.StatusCode, nil
-}
-
 // isValidAccessToken is a simple validation of the access token
 func isValidAccessToken(token string, audience string) error {
 	if token == "" {
@@ -297,9 +255,24 @@ func isValidAccessToken(token string, audience string) error {
 		return err
 	}
 
-	if claims.Audience != audience {
+	if claims.Audience == nil {
-		return fmt.Errorf("invalid audience")
+		return fmt.Errorf("required token field audience is absent")
 	}
 
-	return nil
+	// Audience claim of JWT can be a string or an array of strings
+	typ := reflect.TypeOf(claims.Audience)
+	switch typ.Kind() {
+	case reflect.String:
+		if claims.Audience == audience {
+			return nil
+		}
+	case reflect.Slice:
+		for _, aud := range claims.Audience.([]interface{}) {
+			if audience == aud {
+				return nil
+			}
+		}
+	}
+
+	return fmt.Errorf("invalid JWT token audience field")
 }

client/internal/oauth_test.go

@@ -2,12 +2,12 @@ package internal
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"github.com/golang-jwt/jwt"
 	"github.com/stretchr/testify/require"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"net/url"
 	"strings"
 	"testing"
 	"time"
@@ -24,7 +24,7 @@ type mockHTTPClient struct {
 }
 
 func (c *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
-	body, err := ioutil.ReadAll(req.Body)
+	body, err := io.ReadAll(req.Body)
 	if err == nil {
 		c.reqBody = string(body)
 	}
@@ -33,13 +33,13 @@ func (c *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
 		c.count++
 		return &http.Response{
 			StatusCode: c.code,
-			Body:       ioutil.NopCloser(strings.NewReader(c.countResBody)),
+			Body:       io.NopCloser(strings.NewReader(c.countResBody)),
 		}, c.err
 	}
 
 	return &http.Response{
 		StatusCode: c.code,
-		Body:       ioutil.NopCloser(strings.NewReader(c.resBody)),
+		Body:       io.NopCloser(strings.NewReader(c.resBody)),
 	}, c.err
 }
 
@@ -54,15 +54,19 @@ func TestHosted_RequestDeviceCode(t *testing.T) {
 		testingFunc      require.ComparisonAssertionFunc
 		expectedOut      DeviceAuthInfo
 		expectedMSG      string
-		expectPayload    RequestDeviceCodePayload
+		expectPayload    string
 	}
 
+	expectedAudience := "ok"
+	expectedClientID := "bla"
+	form := url.Values{}
+	form.Add("audience", expectedAudience)
+	form.Add("client_id", expectedClientID)
+	expectPayload := form.Encode()
+
 	testCase1 := test{
-		name: "Payload Is Valid",
+		name:           "Payload Is Valid",
-		expectPayload: RequestDeviceCodePayload{
+		expectPayload:  expectPayload,
-			Audience: "ok",
-			ClientID: "bla",
-		},
 		inputReqCode:   200,
 		testingErrFunc: require.Error,
 		testingFunc:    require.EqualValues,
@@ -74,6 +78,7 @@ func TestHosted_RequestDeviceCode(t *testing.T) {
 		testingErrFunc:   require.Error,
 		expectedErrorMSG: "should return error",
 		testingFunc:      require.EqualValues,
+		expectPayload:    expectPayload,
 	}
 
 	testCase3 := test{
@@ -82,15 +87,13 @@ func TestHosted_RequestDeviceCode(t *testing.T) {
 		testingErrFunc:   require.Error,
 		expectedErrorMSG: "should return error",
 		testingFunc:      require.EqualValues,
+		expectPayload:    expectPayload,
 	}
 	testCase4Out := DeviceAuthInfo{ExpiresIn: 10}
 	testCase4 := test{
-		name:         "Got Device Code",
+		name:           "Got Device Code",
-		inputResBody: fmt.Sprintf("{\"expires_in\":%d}", testCase4Out.ExpiresIn),
+		inputResBody:   fmt.Sprintf("{\"expires_in\":%d}", testCase4Out.ExpiresIn),
-		expectPayload: RequestDeviceCodePayload{
+		expectPayload:  expectPayload,
-			Audience: "ok",
-			ClientID: "bla",
-		},
 		inputReqCode:   200,
 		testingErrFunc: require.NoError,
 		testingFunc:    require.EqualValues,
@@ -108,18 +111,17 @@ func TestHosted_RequestDeviceCode(t *testing.T) {
 			}
 
 			hosted := Hosted{
-				Audience:   testCase.expectPayload.Audience,
+				Audience:           expectedAudience,
-				ClientID:   testCase.expectPayload.ClientID,
+				ClientID:           expectedClientID,
-				Domain:     "test.hosted.com",
+				TokenEndpoint:      "test.hosted.com/token",
-				HTTPClient: &httpClient,
+				DeviceAuthEndpoint: "test.hosted.com/device/auth",
+				HTTPClient:         &httpClient,
 			}
 
 			authInfo, err := hosted.RequestDeviceCode(context.TODO())
 			testCase.testingErrFunc(t, err, testCase.expectedErrorMSG)
 
-			payload, _ := json.Marshal(testCase.expectPayload)
+			require.EqualValues(t, expectPayload, httpClient.reqBody, "payload should match")
-
-			require.EqualValues(t, string(payload), httpClient.reqBody, "payload should match")
 
 			testCase.testingFunc(t, testCase.expectedOut, authInfo, testCase.expectedMSG)
 
@@ -143,7 +145,7 @@ func TestHosted_WaitToken(t *testing.T) {
 		testingFunc       require.ComparisonAssertionFunc
 		expectedOut       TokenInfo
 		expectedMSG       string
-		expectPayload     TokenRequestPayload
+		expectPayload     string
 	}
 
 	defaultInfo := DeviceAuthInfo{
@@ -152,11 +154,13 @@ func TestHosted_WaitToken(t *testing.T) {
 		Interval:   1,
 	}
 
-	tokenReqPayload := TokenRequestPayload{
+	clientID := "test"
-		GrantType:  HostedGrantType,
+
-		DeviceCode: defaultInfo.DeviceCode,
+	form := url.Values{}
-		ClientID:   "test",
+	form.Add("grant_type", HostedGrantType)
-	}
+	form.Add("device_code", defaultInfo.DeviceCode)
+	form.Add("client_id", clientID)
+	tokenReqPayload := form.Encode()
 
 	testCase1 := test{
 		name:           "Payload Is Valid",
@@ -268,10 +272,11 @@ func TestHosted_WaitToken(t *testing.T) {
 			}
 
 			hosted := Hosted{
-				Audience:   testCase.inputAudience,
+				Audience:           testCase.inputAudience,
-				ClientID:   testCase.expectPayload.ClientID,
+				ClientID:           clientID,
-				Domain:     "test.hosted.com",
+				TokenEndpoint:      "test.hosted.com/token",
-				HTTPClient: &httpClient,
+				DeviceAuthEndpoint: "test.hosted.com/device/auth",
+				HTTPClient:         &httpClient,
 			}
 
 			ctx, cancel := context.WithTimeout(context.TODO(), testCase.inputTimeout)
@@ -279,12 +284,7 @@ func TestHosted_WaitToken(t *testing.T) {
 			tokenInfo, err := hosted.WaitToken(ctx, testCase.inputInfo)
 			testCase.testingErrFunc(t, err, testCase.expectedErrorMSG)
 
-			var payload []byte
+			require.EqualValues(t, testCase.expectPayload, httpClient.reqBody, "payload should match")
-			var emptyPayload TokenRequestPayload
-			if testCase.expectPayload != emptyPayload {
-				payload, _ = json.Marshal(testCase.expectPayload)
-			}
-			require.EqualValues(t, string(payload), httpClient.reqBody, "payload should match")
 
 			testCase.testingFunc(t, testCase.expectedOut, tokenInfo, testCase.expectedMSG)
 
@@ -293,123 +293,3 @@ func TestHosted_WaitToken(t *testing.T) {
 		})
 	}
 }
-
-func TestHosted_RotateAccessToken(t *testing.T) {
-	type test struct {
-		name             string
-		inputResBody     string
-		inputReqCode     int
-		inputReqError    error
-		inputMaxReqs     int
-		inputInfo        DeviceAuthInfo
-		inputAudience    string
-		testingErrFunc   require.ErrorAssertionFunc
-		expectedErrorMSG string
-		testingFunc      require.ComparisonAssertionFunc
-		expectedOut      TokenInfo
-		expectedMSG      string
-		expectPayload    TokenRequestPayload
-	}
-
-	defaultInfo := DeviceAuthInfo{
-		DeviceCode: "test",
-		ExpiresIn:  10,
-		Interval:   1,
-	}
-
-	tokenReqPayload := TokenRequestPayload{
-		GrantType:    HostedRefreshGrant,
-		ClientID:     "test",
-		RefreshToken: "refresh_test",
-	}
-
-	testCase1 := test{
-		name:           "Payload Is Valid",
-		inputInfo:      defaultInfo,
-		inputReqCode:   200,
-		testingErrFunc: require.Error,
-		testingFunc:    require.EqualValues,
-		expectPayload:  tokenReqPayload,
-	}
-
-	testCase2 := test{
-		name:             "Exit On Network Error",
-		inputInfo:        defaultInfo,
-		expectPayload:    tokenReqPayload,
-		inputReqError:    fmt.Errorf("error"),
-		testingErrFunc:   require.Error,
-		expectedErrorMSG: "should return error",
-		testingFunc:      require.EqualValues,
-	}
-
-	testCase3 := test{
-		name:             "Exit On Non 200 Status Code",
-		inputInfo:        defaultInfo,
-		inputReqCode:     401,
-		expectPayload:    tokenReqPayload,
-		testingErrFunc:   require.Error,
-		expectedErrorMSG: "should return error",
-		testingFunc:      require.EqualValues,
-	}
-
-	audience := "test"
-
-	token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"aud": audience})
-	var hmacSampleSecret []byte
-	tokenString, _ := token.SignedString(hmacSampleSecret)
-
-	testCase4 := test{
-		name:           "Exit On Invalid Audience",
-		inputInfo:      defaultInfo,
-		inputResBody:   fmt.Sprintf("{\"access_token\":\"%s\"}", tokenString),
-		inputReqCode:   200,
-		inputAudience:  "super test",
-		testingErrFunc: require.Error,
-		testingFunc:    require.EqualValues,
-		expectPayload:  tokenReqPayload,
-	}
-
-	testCase5 := test{
-		name:           "Received Token Info",
-		inputInfo:      defaultInfo,
-		inputResBody:   fmt.Sprintf("{\"access_token\":\"%s\"}", tokenString),
-		inputReqCode:   200,
-		inputAudience:  audience,
-		testingErrFunc: require.NoError,
-		testingFunc:    require.EqualValues,
-		expectPayload:  tokenReqPayload,
-		expectedOut:    TokenInfo{AccessToken: tokenString},
-	}
-
-	for _, testCase := range []test{testCase1, testCase2, testCase3, testCase4, testCase5} {
-		t.Run(testCase.name, func(t *testing.T) {
-
-			httpClient := mockHTTPClient{
-				resBody: testCase.inputResBody,
-				code:    testCase.inputReqCode,
-				err:     testCase.inputReqError,
-				MaxReqs: testCase.inputMaxReqs,
-			}
-
-			hosted := Hosted{
-				Audience:   testCase.inputAudience,
-				ClientID:   testCase.expectPayload.ClientID,
-				Domain:     "test.hosted.com",
-				HTTPClient: &httpClient,
-			}
-
-			tokenInfo, err := hosted.RotateAccessToken(context.TODO(), testCase.expectPayload.RefreshToken)
-			testCase.testingErrFunc(t, err, testCase.expectedErrorMSG)
-
-			var payload []byte
-			var emptyPayload TokenRequestPayload
-			if testCase.expectPayload != emptyPayload {
-				payload, _ = json.Marshal(testCase.expectPayload)
-			}
-			require.EqualValues(t, string(payload), httpClient.reqBody, "payload should match")
-
-			testCase.testingFunc(t, testCase.expectedOut, tokenInfo, testCase.expectedMSG)
-
-		})
-	}
-}

client/internal/peer/conn.go

@@ -3,6 +3,7 @@ package peer
 import (
 	"context"
 	nbStatus "github.com/netbirdio/netbird/client/status"
+	"github.com/netbirdio/netbird/client/system"
 	"github.com/netbirdio/netbird/iface"
 	"golang.zx2c4.com/wireguard/wgctrl"
 	"net"
@@ -36,6 +37,20 @@ type ConnConfig struct {
 
 	UDPMux      ice.UDPMux
 	UDPMuxSrflx ice.UniversalUDPMux
+
+	LocalWgPort int
+}
+
+// OfferAnswer represents a session establishment offer or answer
+type OfferAnswer struct {
+	IceCredentials IceCredentials
+	// WgListenPort is a remote WireGuard listen port.
+	// This field is used when establishing a direct WireGuard connection without any proxy.
+	// We can set the remote peer's endpoint with this port.
+	WgListenPort int
+
+	// Version of NetBird Agent
+	Version string
 }
 
 // IceCredentials ICE protocol credentials struct
@@ -51,13 +66,13 @@ type Conn struct {
 	// signalCandidate is a handler function to signal remote peer about local connection candidate
 	signalCandidate func(candidate ice.Candidate) error
 	// signalOffer is a handler function to signal remote peer our connection offer (credentials)
-	signalOffer  func(uFrag string, pwd string) error
+	signalOffer  func(OfferAnswer) error
-	signalAnswer func(uFrag string, pwd string) error
+	signalAnswer func(OfferAnswer) error
 
 	// remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection
-	remoteOffersCh chan IceCredentials
+	remoteOffersCh chan OfferAnswer
 	// remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection
-	remoteAnswerCh     chan IceCredentials
+	remoteAnswerCh     chan OfferAnswer
 	closeCh            chan struct{}
 	ctx                context.Context
 	notifyDisconnected context.CancelFunc
@@ -88,8 +103,8 @@ func NewConn(config ConnConfig, statusRecorder *nbStatus.Status) (*Conn, error)
 		mu:             sync.Mutex{},
 		status:         StatusDisconnected,
 		closeCh:        make(chan struct{}),
-		remoteOffersCh: make(chan IceCredentials),
+		remoteOffersCh: make(chan OfferAnswer),
-		remoteAnswerCh: make(chan IceCredentials),
+		remoteAnswerCh: make(chan OfferAnswer),
 		statusRecorder: statusRecorder,
 	}, nil
 }
@@ -132,7 +147,7 @@ func (conn *Conn) reCreateAgent() error {
 		MulticastDNSMode: ice.MulticastDNSModeDisabled,
 		NetworkTypes:     []ice.NetworkType{ice.NetworkTypeUDP4},
 		Urls:             conn.config.StunTurn,
-		CandidateTypes:   []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay},
+		CandidateTypes:   []ice.CandidateType{ice.CandidateTypeServerReflexive, ice.CandidateTypeHost, ice.CandidateTypeRelay},
 		FailedTimeout:    &failedTimeout,
 		InterfaceFilter:  interfaceFilter(conn.config.InterfaceBlackList),
 		UDPMux:           conn.config.UDPMux,
@@ -200,15 +215,15 @@ func (conn *Conn) Open() error {
 	// Only continue once we got a connection confirmation from the remote peer.
 	// The connection timeout could have happened before a confirmation received from the remote.
 	// The connection could have also been closed externally (e.g. when we received an update from the management that peer shouldn't be connected)
-	var remoteCredentials IceCredentials
+	var remoteOfferAnswer OfferAnswer
 	select {
-	case remoteCredentials = <-conn.remoteOffersCh:
+	case remoteOfferAnswer = <-conn.remoteOffersCh:
 		// received confirmation from the remote peer -> ready to proceed
 		err = conn.sendAnswer()
 		if err != nil {
 			return err
 		}
-	case remoteCredentials = <-conn.remoteAnswerCh:
+	case remoteOfferAnswer = <-conn.remoteAnswerCh:
 	case <-time.After(conn.config.Timeout):
 		return NewConnectionTimeoutError(conn.config.Key, conn.config.Timeout)
 	case <-conn.closeCh:
@@ -216,7 +231,8 @@ func (conn *Conn) Open() error {
 		return NewConnectionClosedError(conn.config.Key)
 	}
 
-	log.Debugf("received connection confirmation from peer %s", conn.config.Key)
+	log.Debugf("received connection confirmation from peer %s running version %s and with remote WireGuard listen port %d",
+		conn.config.Key, remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort)
 
 	// at this point we received offer/answer and we are ready to gather candidates
 	conn.mu.Lock()
@@ -245,28 +261,26 @@ func (conn *Conn) Open() error {
 	isControlling := conn.config.LocalKey > conn.config.Key
 	var remoteConn *ice.Conn
 	if isControlling {
-		remoteConn, err = conn.agent.Dial(conn.ctx, remoteCredentials.UFrag, remoteCredentials.Pwd)
+		remoteConn, err = conn.agent.Dial(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
 	} else {
-		remoteConn, err = conn.agent.Accept(conn.ctx, remoteCredentials.UFrag, remoteCredentials.Pwd)
+		remoteConn, err = conn.agent.Accept(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
 	}
 	if err != nil {
 		return err
 	}
 
+	// dynamically set remote WireGuard port is other side specified a different one from the default one
+	remoteWgPort := iface.DefaultWgPort
+	if remoteOfferAnswer.WgListenPort != 0 {
+		remoteWgPort = remoteOfferAnswer.WgListenPort
+	}
 	// the ice connection has been established successfully so we are ready to start the proxy
-	err = conn.startProxy(remoteConn)
+	err = conn.startProxy(remoteConn, remoteWgPort)
 	if err != nil {
 		return err
 	}
 
-	if conn.proxy.Type() == proxy.TypeNoProxy {
+	log.Infof("connected to peer %s [laddr <-> raddr] [%s <-> %s]", conn.config.Key, remoteConn.LocalAddr().String(), remoteConn.RemoteAddr().String())
-		host, _, _ := net.SplitHostPort(remoteConn.LocalAddr().String())
-		rhost, _, _ := net.SplitHostPort(remoteConn.RemoteAddr().String())
-		// direct Wireguard connection
-		log.Infof("directly connected to peer %s [laddr <-> raddr] [%s:%d <-> %s:%d]", conn.config.Key, host, iface.DefaultWgPort, rhost, iface.DefaultWgPort)
-	} else {
-		log.Infof("connected to peer %s [laddr <-> raddr] [%s <-> %s]", conn.config.Key, remoteConn.LocalAddr().String(), remoteConn.RemoteAddr().String())
-	}
 
 	// wait until connection disconnected or has been closed externally (upper layer, e.g. engine)
 	select {
@@ -319,7 +333,7 @@ func IsPublicIP(ip net.IP) bool {
 }
 
 // startProxy starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected
-func (conn *Conn) startProxy(remoteConn net.Conn) error {
+func (conn *Conn) startProxy(remoteConn net.Conn, remoteWgPort int) error {
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
 
@@ -330,15 +344,16 @@ func (conn *Conn) startProxy(remoteConn net.Conn) error {
 	}
 
 	peerState := nbStatus.PeerState{PubKey: conn.config.Key}
-	useProxy := shouldUseProxy(pair)
+
 	var p proxy.Proxy
-	if useProxy {
+	if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay {
 		p = proxy.NewWireguardProxy(conn.config.ProxyConfig)
 		peerState.Direct = false
 	} else {
-		p = proxy.NewNoProxy(conn.config.ProxyConfig)
+		p = proxy.NewNoProxy(conn.config.ProxyConfig, remoteWgPort)
 		peerState.Direct = true
 	}
+
 	conn.proxy = p
 	err = p.Start(remoteConn)
 	if err != nil {
@@ -409,12 +424,12 @@ func (conn *Conn) cleanup() error {
 }
 
 // SetSignalOffer sets a handler function to be triggered by Conn when a new connection offer has to be signalled to the remote peer
-func (conn *Conn) SetSignalOffer(handler func(uFrag string, pwd string) error) {
+func (conn *Conn) SetSignalOffer(handler func(offer OfferAnswer) error) {
 	conn.signalOffer = handler
 }
 
 // SetSignalAnswer sets a handler function to be triggered by Conn when a new connection answer has to be signalled to the remote peer
-func (conn *Conn) SetSignalAnswer(handler func(uFrag string, pwd string) error) {
+func (conn *Conn) SetSignalAnswer(handler func(answer OfferAnswer) error) {
 	conn.signalAnswer = handler
 }
 
@@ -459,8 +474,12 @@ func (conn *Conn) sendAnswer() error {
 		return err
 	}
 
-	log.Debugf("sending asnwer to %s", conn.config.Key)
+	log.Debugf("sending answer to %s", conn.config.Key)
-	err = conn.signalAnswer(localUFrag, localPwd)
+	err = conn.signalAnswer(OfferAnswer{
+		IceCredentials: IceCredentials{localUFrag, localPwd},
+		WgListenPort:   conn.config.LocalWgPort,
+		Version:        system.NetbirdVersion(),
+	})
 	if err != nil {
 		return err
 	}
@@ -477,7 +496,11 @@ func (conn *Conn) sendOffer() error {
 	if err != nil {
 		return err
 	}
-	err = conn.signalOffer(localUFrag, localPwd)
+	err = conn.signalOffer(OfferAnswer{
+		IceCredentials: IceCredentials{localUFrag, localPwd},
+		WgListenPort:   conn.config.LocalWgPort,
+		Version:        system.NetbirdVersion(),
+	})
 	if err != nil {
 		return err
 	}
@@ -518,11 +541,11 @@ func (conn *Conn) Status() ConnStatus {
 
 // OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
 // doesn't block, discards the message if connection wasn't ready
-func (conn *Conn) OnRemoteOffer(remoteAuth IceCredentials) bool {
+func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool {
 	log.Debugf("OnRemoteOffer from peer %s on status %s", conn.config.Key, conn.status.String())
 
 	select {
-	case conn.remoteOffersCh <- remoteAuth:
+	case conn.remoteOffersCh <- offer:
 		return true
 	default:
 		log.Debugf("OnRemoteOffer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String())
@@ -533,11 +556,11 @@ func (conn *Conn) OnRemoteOffer(remoteAuth IceCredentials) bool {
 
 // OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
 // doesn't block, discards the message if connection wasn't ready
-func (conn *Conn) OnRemoteAnswer(remoteAuth IceCredentials) bool {
+func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool {
 	log.Debugf("OnRemoteAnswer from peer %s on status %s", conn.config.Key, conn.status.String())
 
 	select {
-	case conn.remoteAnswerCh <- remoteAuth:
+	case conn.remoteAnswerCh <- answer:
 		return true
 	default:
 		// connection might not be ready yet to receive so we ignore the message

client/internal/peer/conn_test.go

@@ -18,6 +18,7 @@ var connConf = ConnConfig{
 	InterfaceBlackList: nil,
 	Timeout:            time.Second,
 	ProxyConfig:        proxy.Config{},
+	LocalWgPort:        51820,
 }
 
 func TestNewConn_interfaceFilter(t *testing.T) {
@@ -59,9 +60,13 @@ func TestConn_OnRemoteOffer(t *testing.T) {
 
 	go func() {
 		for {
-			accepted := conn.OnRemoteOffer(IceCredentials{
+			accepted := conn.OnRemoteOffer(OfferAnswer{
-				UFrag: "test",
+				IceCredentials: IceCredentials{
-				Pwd:   "test",
+					UFrag: "test",
+					Pwd:   "test",
+				},
+				WgListenPort: 0,
+				Version:      "",
 			})
 			if accepted {
 				wg.Done()
@@ -89,9 +94,13 @@ func TestConn_OnRemoteAnswer(t *testing.T) {
 
 	go func() {
 		for {
-			accepted := conn.OnRemoteAnswer(IceCredentials{
+			accepted := conn.OnRemoteAnswer(OfferAnswer{
-				UFrag: "test",
+				IceCredentials: IceCredentials{
-				Pwd:   "test",
+					UFrag: "test",
+					Pwd:   "test",
+				},
+				WgListenPort: 0,
+				Version:      "",
 			})
 			if accepted {
 				wg.Done()

client/internal/proxy/noproxy.go

@@ -1,7 +1,6 @@
 package proxy
 
 import (
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
 	"net"
 )
@@ -14,10 +13,14 @@ import (
 // In order NoProxy to work, Wireguard port has to be fixed for the time being.
 type NoProxy struct {
 	config Config
+	// RemoteWgListenPort is a WireGuard port of a remote peer.
+	// It is used instead of the hardcoded 51820 port.
+	RemoteWgListenPort int
 }
 
-func NewNoProxy(config Config) *NoProxy {
+// NewNoProxy creates a new NoProxy with a provided config and remote peer's WireGuard listen port
-	return &NoProxy{config: config}
+func NewNoProxy(config Config, remoteWgPort int) *NoProxy {
+	return &NoProxy{config: config, RemoteWgListenPort: remoteWgPort}
 }
 
 func (p *NoProxy) Close() error {
@@ -36,7 +39,6 @@ func (p *NoProxy) Start(remoteConn net.Conn) error {
 	if err != nil {
 		return err
 	}
-	addr.Port = iface.DefaultWgPort
 	err = p.config.WgInterface.UpdatePeer(p.config.RemoteKey, p.config.AllowedIps, DefaultWgKeepAlive,
 		addr, p.config.PreSharedKey)
 

client/internal/routemanager/client.go

@@ -0,0 +1,285 @@
+package routemanager
+
+import (
+	"context"
+	"fmt"
+	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/status"
+	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/route"
+	log "github.com/sirupsen/logrus"
+	"net/netip"
+)
+
+type routerPeerStatus struct {
+	connected bool
+	relayed   bool
+	direct    bool
+}
+
+type routesUpdate struct {
+	updateSerial uint64
+	routes       []*route.Route
+}
+
+type clientNetwork struct {
+	ctx                 context.Context
+	stop                context.CancelFunc
+	statusRecorder      *status.Status
+	wgInterface         *iface.WGIface
+	routes              map[string]*route.Route
+	routeUpdate         chan routesUpdate
+	peerStateUpdate     chan struct{}
+	routePeersNotifiers map[string]chan struct{}
+	chosenRoute         *route.Route
+	network             netip.Prefix
+	updateSerial        uint64
+}
+
+func newClientNetworkWatcher(ctx context.Context, wgInterface *iface.WGIface, statusRecorder *status.Status, network netip.Prefix) *clientNetwork {
+	ctx, cancel := context.WithCancel(ctx)
+	client := &clientNetwork{
+		ctx:                 ctx,
+		stop:                cancel,
+		statusRecorder:      statusRecorder,
+		wgInterface:         wgInterface,
+		routes:              make(map[string]*route.Route),
+		routePeersNotifiers: make(map[string]chan struct{}),
+		routeUpdate:         make(chan routesUpdate),
+		peerStateUpdate:     make(chan struct{}),
+		network:             network,
+	}
+	return client
+}
+
+func getClientNetworkID(input *route.Route) string {
+	return input.NetID + "-" + input.Network.String()
+}
+
+func (c *clientNetwork) getRouterPeerStatuses() map[string]routerPeerStatus {
+	routePeerStatuses := make(map[string]routerPeerStatus)
+	for _, r := range c.routes {
+		peerStatus, err := c.statusRecorder.GetPeer(r.Peer)
+		if err != nil {
+			log.Debugf("couldn't fetch peer state: %v", err)
+			continue
+		}
+		routePeerStatuses[r.ID] = routerPeerStatus{
+			connected: peerStatus.ConnStatus == peer.StatusConnected.String(),
+			relayed:   peerStatus.Relayed,
+			direct:    peerStatus.Direct,
+		}
+	}
+	return routePeerStatuses
+}
+
+func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[string]routerPeerStatus) string {
+	var chosen string
+	chosenScore := 0
+
+	currID := ""
+	if c.chosenRoute != nil {
+		currID = c.chosenRoute.ID
+	}
+
+	for _, r := range c.routes {
+		tempScore := 0
+		peerStatus, found := routePeerStatuses[r.ID]
+		if !found || !peerStatus.connected {
+			continue
+		}
+		if r.Metric < route.MaxMetric {
+			metricDiff := route.MaxMetric - r.Metric
+			tempScore = metricDiff * 10
+		}
+		if !peerStatus.relayed {
+			tempScore++
+		}
+		if !peerStatus.direct {
+			tempScore++
+		}
+		if tempScore > chosenScore || (tempScore == chosenScore && currID == r.ID) {
+			chosen = r.ID
+			chosenScore = tempScore
+		}
+	}
+
+	if chosen == "" {
+		var peers []string
+		for _, r := range c.routes {
+			peers = append(peers, r.Peer)
+		}
+		log.Warnf("no route was chosen for network %s because no peers from list %s were connected", c.network, peers)
+	} else if chosen != currID {
+		log.Infof("new chosen route is %s with peer %s with score %d", chosen, c.routes[chosen].Peer, chosenScore)
+	}
+
+	return chosen
+}
+
+func (c *clientNetwork) watchPeerStatusChanges(ctx context.Context, peerKey string, peerStateUpdate chan struct{}, closer chan struct{}) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-closer:
+			return
+		case <-c.statusRecorder.GetPeerStateChangeNotifier(peerKey):
+			state, err := c.statusRecorder.GetPeer(peerKey)
+			if err != nil || state.ConnStatus == peer.StatusConnecting.String() {
+				continue
+			}
+			peerStateUpdate <- struct{}{}
+			log.Debugf("triggered route state update for Peer %s, state: %s", peerKey, state.ConnStatus)
+		}
+	}
+}
+
+func (c *clientNetwork) startPeersStatusChangeWatcher() {
+	for _, r := range c.routes {
+		_, found := c.routePeersNotifiers[r.Peer]
+		if !found {
+			c.routePeersNotifiers[r.Peer] = make(chan struct{})
+			go c.watchPeerStatusChanges(c.ctx, r.Peer, c.peerStateUpdate, c.routePeersNotifiers[r.Peer])
+		}
+	}
+}
+
+func (c *clientNetwork) removeRouteFromWireguardPeer(peerKey string) error {
+	state, err := c.statusRecorder.GetPeer(peerKey)
+	if err != nil || state.ConnStatus != peer.StatusConnected.String() {
+		return nil
+	}
+
+	err = c.wgInterface.RemoveAllowedIP(peerKey, c.network.String())
+	if err != nil {
+		return fmt.Errorf("couldn't remove allowed IP %s removed for peer %s, err: %v",
+			c.network, c.chosenRoute.Peer, err)
+	}
+	return nil
+}
+
+func (c *clientNetwork) removeRouteFromPeerAndSystem() error {
+	if c.chosenRoute != nil {
+		err := c.removeRouteFromWireguardPeer(c.chosenRoute.Peer)
+		if err != nil {
+			return err
+		}
+		err = removeFromRouteTableIfNonSystem(c.network, c.wgInterface.GetAddress().IP.String())
+		if err != nil {
+			return fmt.Errorf("couldn't remove route %s from system, err: %v",
+				c.network, err)
+		}
+	}
+	return nil
+}
+
+func (c *clientNetwork) recalculateRouteAndUpdatePeerAndSystem() error {
+
+	var err error
+
+	routerPeerStatuses := c.getRouterPeerStatuses()
+
+	chosen := c.getBestRouteFromStatuses(routerPeerStatuses)
+	if chosen == "" {
+		err = c.removeRouteFromPeerAndSystem()
+		if err != nil {
+			return err
+		}
+
+		c.chosenRoute = nil
+
+		return nil
+	}
+
+	if c.chosenRoute != nil && c.chosenRoute.ID == chosen {
+		if c.chosenRoute.IsEqual(c.routes[chosen]) {
+			return nil
+		}
+	}
+
+	if c.chosenRoute != nil {
+		err = c.removeRouteFromWireguardPeer(c.chosenRoute.Peer)
+		if err != nil {
+			return err
+		}
+	} else {
+		err = addToRouteTableIfNoExists(c.network, c.wgInterface.GetAddress().IP.String())
+		if err != nil {
+			return fmt.Errorf("route %s couldn't be added for peer %s, err: %v",
+				c.chosenRoute.Network.String(), c.wgInterface.GetAddress().IP.String(), err)
+		}
+	}
+
+	c.chosenRoute = c.routes[chosen]
+	err = c.wgInterface.AddAllowedIP(c.chosenRoute.Peer, c.network.String())
+	if err != nil {
+		log.Errorf("couldn't add allowed IP %s added for peer %s, err: %v",
+			c.network, c.chosenRoute.Peer, err)
+	}
+
+	return nil
+}
+
+func (c *clientNetwork) sendUpdateToClientNetworkWatcher(update routesUpdate) {
+	go func() {
+		c.routeUpdate <- update
+	}()
+}
+
+func (c *clientNetwork) handleUpdate(update routesUpdate) {
+	updateMap := make(map[string]*route.Route)
+
+	for _, r := range update.routes {
+		updateMap[r.ID] = r
+	}
+
+	for id, r := range c.routes {
+		_, found := updateMap[id]
+		if !found {
+			close(c.routePeersNotifiers[r.Peer])
+			delete(c.routePeersNotifiers, r.Peer)
+		}
+	}
+
+	c.routes = updateMap
+}
+
+// peersStateAndUpdateWatcher is the main point of reacting on client network routing events.
+// All the processing related to the client network should be done here. Thread-safe.
+func (c *clientNetwork) peersStateAndUpdateWatcher() {
+	for {
+		select {
+		case <-c.ctx.Done():
+			log.Debugf("stopping watcher for network %s", c.network)
+			err := c.removeRouteFromPeerAndSystem()
+			if err != nil {
+				log.Error(err)
+			}
+			return
+		case <-c.peerStateUpdate:
+			err := c.recalculateRouteAndUpdatePeerAndSystem()
+			if err != nil {
+				log.Error(err)
+			}
+		case update := <-c.routeUpdate:
+			if update.updateSerial < c.updateSerial {
+				log.Warnf("received a routes update with smaller serial number, ignoring it")
+				continue
+			}
+
+			log.Debugf("received a new client network route update for %s", c.network)
+
+			c.handleUpdate(update)
+
+			c.updateSerial = update.updateSerial
+
+			err := c.recalculateRouteAndUpdatePeerAndSystem()
+			if err != nil {
+				log.Error(err)
+			}
+
+			c.startPeersStatusChangeWatcher()
+		}
+	}
+}

client/internal/routemanager/common_linux_test.go

@@ -0,0 +1,75 @@
+package routemanager
+
+var insertRuleTestCases = []struct {
+	name      string
+	inputPair routerPair
+	ipVersion string
+}{
+	{
+		name: "Insert Forwarding IPV4 Rule",
+		inputPair: routerPair{
+			ID:          "zxa",
+			source:      "100.100.100.1/32",
+			destination: "100.100.200.0/24",
+			masquerade:  false,
+		},
+		ipVersion: ipv4,
+	},
+	{
+		name: "Insert Forwarding And Nat IPV4 Rules",
+		inputPair: routerPair{
+			ID:          "zxa",
+			source:      "100.100.100.1/32",
+			destination: "100.100.200.0/24",
+			masquerade:  true,
+		},
+		ipVersion: ipv4,
+	},
+	{
+		name: "Insert Forwarding IPV6 Rule",
+		inputPair: routerPair{
+			ID:          "zxa",
+			source:      "fc00::1/128",
+			destination: "fc12::/64",
+			masquerade:  false,
+		},
+		ipVersion: ipv6,
+	},
+	{
+		name: "Insert Forwarding And Nat IPV6 Rules",
+		inputPair: routerPair{
+			ID:          "zxa",
+			source:      "fc00::1/128",
+			destination: "fc12::/64",
+			masquerade:  true,
+		},
+		ipVersion: ipv6,
+	},
+}
+
+var removeRuleTestCases = []struct {
+	name      string
+	inputPair routerPair
+	ipVersion string
+}{
+	{
+		name: "Remove Forwarding And Nat IPV4 Rules",
+		inputPair: routerPair{
+			ID:          "zxa",
+			source:      "100.100.100.1/32",
+			destination: "100.100.200.0/24",
+			masquerade:  true,
+		},
+		ipVersion: ipv4,
+	},
+	{
+		name: "Remove Forwarding And Nat IPV6 Rules",
+		inputPair: routerPair{
+			ID:          "zxa",
+			source:      "fc00::1/128",
+			destination: "fc12::/64",
+			masquerade:  true,
+		},
+		ipVersion: ipv6,
+	},
+}

client/internal/routemanager/firewall.go

@@ -0,0 +1,12 @@
+package routemanager
+
+type firewallManager interface {
+	// RestoreOrCreateContainers restores or creates a firewall container set of rules, tables and default rules
+	RestoreOrCreateContainers() error
+	// InsertRoutingRules inserts a routing firewall rule
+	InsertRoutingRules(pair routerPair) error
+	// RemoveRoutingRules removes a routing firewall rule
+	RemoveRoutingRules(pair routerPair) error
+	// CleanRoutingRules cleans a firewall set of containers
+	CleanRoutingRules()
+}

client/internal/routemanager/firewall_linux.go

@@ -0,0 +1,55 @@
+package routemanager
+
+import (
+	"context"
+	"fmt"
+	"github.com/coreos/go-iptables/iptables"
+	log "github.com/sirupsen/logrus"
+)
+import "github.com/google/nftables"
+
+const (
+	ipv6Forwarding   = "netbird-rt-ipv6-forwarding"
+	ipv4Forwarding   = "netbird-rt-ipv4-forwarding"
+	ipv6Nat          = "netbird-rt-ipv6-nat"
+	ipv4Nat          = "netbird-rt-ipv4-nat"
+	natFormat        = "netbird-nat-%s"
+	forwardingFormat = "netbird-fwd-%s"
+	ipv6             = "ipv6"
+	ipv4             = "ipv4"
+)
+
+func genKey(format string, input string) string {
+	return fmt.Sprintf(format, input)
+}
+
+// NewFirewall if supported, returns an iptables manager, otherwise returns a nftables manager
+func NewFirewall(parentCTX context.Context) firewallManager {
+	ctx, cancel := context.WithCancel(parentCTX)
+
+	if isIptablesSupported() {
+		log.Debugf("iptables is supported")
+		ipv4Client, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+		ipv6Client, _ := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+
+		return &iptablesManager{
+			ctx:        ctx,
+			stop:       cancel,
+			ipv4Client: ipv4Client,
+			ipv6Client: ipv6Client,
+			rules:      make(map[string]map[string][]string),
+		}
+	}
+
+	log.Debugf("iptables is not supported, using nftables")
+
+	manager := &nftablesManager{
+		ctx:    ctx,
+		stop:   cancel,
+		conn:   &nftables.Conn{},
+		chains: make(map[string]map[string]*nftables.Chain),
+		rules:  make(map[string]*nftables.Rule),
+	}
+
+	return manager
+}

client/internal/routemanager/firewall_nonlinux.go

@@ -0,0 +1,27 @@
+//go:build !linux
+// +build !linux
+
+package routemanager
+
+import "context"
+
+type unimplementedFirewall struct{}
+
+func (unimplementedFirewall) RestoreOrCreateContainers() error {
+	return nil
+}
+func (unimplementedFirewall) InsertRoutingRules(pair routerPair) error {
+	return nil
+}
+func (unimplementedFirewall) RemoveRoutingRules(pair routerPair) error {
+	return nil
+}
+
+func (unimplementedFirewall) CleanRoutingRules() {
+	return
+}
+
+// NewFirewall returns an unimplemented Firewall manager
+func NewFirewall(parentCtx context.Context) firewallManager {
+	return unimplementedFirewall{}
+}

client/internal/routemanager/iptables_linux.go

@@ -0,0 +1,403 @@
+package routemanager
+
+import (
+	"context"
+	"fmt"
+	"github.com/coreos/go-iptables/iptables"
+	log "github.com/sirupsen/logrus"
+	"net/netip"
+	"os/exec"
+	"strings"
+	"sync"
+)
+
+func isIptablesSupported() bool {
+	_, err4 := exec.LookPath("iptables")
+	_, err6 := exec.LookPath("ip6tables")
+	return err4 == nil && err6 == nil
+}
+
+// constants needed to manage and create iptable rules
+const (
+	iptablesFilterTable            = "filter"
+	iptablesNatTable               = "nat"
+	iptablesForwardChain           = "FORWARD"
+	iptablesPostRoutingChain       = "POSTROUTING"
+	iptablesRoutingNatChain        = "NETBIRD-RT-NAT"
+	iptablesRoutingForwardingChain = "NETBIRD-RT-FWD"
+	routingFinalForwardJump        = "ACCEPT"
+	routingFinalNatJump            = "MASQUERADE"
+)
+
+// some presets for building nftable rules
+var (
+	iptablesDefaultForwardingRule        = []string{"-j", iptablesRoutingForwardingChain, "-m", "comment", "--comment"}
+	iptablesDefaultNetbirdForwardingRule = []string{"-j", "RETURN"}
+	iptablesDefaultNatRule               = []string{"-j", iptablesRoutingNatChain, "-m", "comment", "--comment"}
+	iptablesDefaultNetbirdNatRule        = []string{"-j", "RETURN"}
+)
+
+type iptablesManager struct {
+	ctx        context.Context
+	stop       context.CancelFunc
+	ipv4Client *iptables.IPTables
+	ipv6Client *iptables.IPTables
+	rules      map[string]map[string][]string
+	mux        sync.Mutex
+}
+
+// CleanRoutingRules cleans existing iptables resources that we created by the agent
+func (i *iptablesManager) CleanRoutingRules() {
+	i.mux.Lock()
+	defer i.mux.Unlock()
+
+	err := i.cleanJumpRules()
+	if err != nil {
+		log.Error(err)
+	}
+
+	log.Debug("flushing tables")
+	errMSGFormat := "iptables: failed cleaning %s chain %s,error: %v"
+	err = i.ipv4Client.ClearAndDeleteChain(iptablesFilterTable, iptablesRoutingForwardingChain)
+	if err != nil {
+		log.Errorf(errMSGFormat, ipv4, iptablesRoutingForwardingChain, err)
+	}
+
+	err = i.ipv4Client.ClearAndDeleteChain(iptablesNatTable, iptablesRoutingNatChain)
+	if err != nil {
+		log.Errorf(errMSGFormat, ipv4, iptablesRoutingNatChain, err)
+	}
+
+	err = i.ipv6Client.ClearAndDeleteChain(iptablesFilterTable, iptablesRoutingForwardingChain)
+	if err != nil {
+		log.Errorf(errMSGFormat, ipv6, iptablesRoutingForwardingChain, err)
+	}
+
+	err = i.ipv6Client.ClearAndDeleteChain(iptablesNatTable, iptablesRoutingNatChain)
+	if err != nil {
+		log.Errorf(errMSGFormat, ipv6, iptablesRoutingNatChain, err)
+	}
+
+	log.Info("done cleaning up iptables rules")
+}
+
+// RestoreOrCreateContainers restores existing iptables containers (chains and rules)
+// if they don't exist, we create them
+func (i *iptablesManager) RestoreOrCreateContainers() error {
+	i.mux.Lock()
+	defer i.mux.Unlock()
+
+	if i.rules[ipv4][ipv4Forwarding] != nil && i.rules[ipv6][ipv6Forwarding] != nil {
+		return nil
+	}
+
+	errMSGFormat := "iptables: failed creating %s chain %s,error: %v"
+
+	err := createChain(i.ipv4Client, iptablesFilterTable, iptablesRoutingForwardingChain)
+	if err != nil {
+		return fmt.Errorf(errMSGFormat, ipv4, iptablesRoutingForwardingChain, err)
+	}
+
+	err = createChain(i.ipv4Client, iptablesNatTable, iptablesRoutingNatChain)
+	if err != nil {
+		return fmt.Errorf(errMSGFormat, ipv4, iptablesRoutingNatChain, err)
+	}
+
+	err = createChain(i.ipv6Client, iptablesFilterTable, iptablesRoutingForwardingChain)
+	if err != nil {
+		return fmt.Errorf(errMSGFormat, ipv6, iptablesRoutingForwardingChain, err)
+	}
+
+	err = createChain(i.ipv6Client, iptablesNatTable, iptablesRoutingNatChain)
+	if err != nil {
+		return fmt.Errorf(errMSGFormat, ipv6, iptablesRoutingNatChain, err)
+	}
+
+	err = i.restoreRules(i.ipv4Client)
+	if err != nil {
+		return fmt.Errorf("iptables: error while restoring ipv4 rules: %v", err)
+	}
+
+	err = i.restoreRules(i.ipv6Client)
+	if err != nil {
+		return fmt.Errorf("iptables: error while restoring ipv6 rules: %v", err)
+	}
+
+	err = i.addJumpRules()
+	if err != nil {
+		return fmt.Errorf("iptables: error while creating jump rules: %v", err)
+	}
+
+	return nil
+}
+
+// addJumpRules create jump rules to send packets to NetBird chains
+func (i *iptablesManager) addJumpRules() error {
+	err := i.cleanJumpRules()
+	if err != nil {
+		return err
+	}
+	rule := append(iptablesDefaultForwardingRule, ipv4Forwarding)
+	err = i.ipv4Client.Insert(iptablesFilterTable, iptablesForwardChain, 1, rule...)
+	if err != nil {
+		return err
+	}
+
+	i.rules[ipv4][ipv4Forwarding] = rule
+
+	rule = append(iptablesDefaultNatRule, ipv4Nat)
+	err = i.ipv4Client.Insert(iptablesNatTable, iptablesPostRoutingChain, 1, rule...)
+	if err != nil {
+		return err
+	}
+	i.rules[ipv4][ipv4Nat] = rule
+
+	rule = append(iptablesDefaultForwardingRule, ipv6Forwarding)
+	err = i.ipv6Client.Insert(iptablesFilterTable, iptablesForwardChain, 1, rule...)
+	if err != nil {
+		return err
+	}
+	i.rules[ipv6][ipv6Forwarding] = rule
+
+	rule = append(iptablesDefaultNatRule, ipv6Nat)
+	err = i.ipv6Client.Insert(iptablesNatTable, iptablesPostRoutingChain, 1, rule...)
+	if err != nil {
+		return err
+	}
+	i.rules[ipv6][ipv6Nat] = rule
+
+	return nil
+}
+
+// cleanJumpRules cleans jump rules that was sending packets to NetBird chains
+func (i *iptablesManager) cleanJumpRules() error {
+	var err error
+	errMSGFormat := "iptables: failed cleaning rule from %s chain %s,err: %v"
+	rule, found := i.rules[ipv4][ipv4Forwarding]
+	if found {
+		log.Debugf("iptables: removing %s rule: %s ", ipv4, ipv4Forwarding)
+		err = i.ipv4Client.DeleteIfExists(iptablesFilterTable, iptablesForwardChain, rule...)
+		if err != nil {
+			return fmt.Errorf(errMSGFormat, ipv4, iptablesForwardChain, err)
+		}
+	}
+	rule, found = i.rules[ipv4][ipv4Nat]
+	if found {
+		log.Debugf("iptables: removing %s rule: %s ", ipv4, ipv4Nat)
+		err = i.ipv4Client.DeleteIfExists(iptablesNatTable, iptablesPostRoutingChain, rule...)
+		if err != nil {
+			return fmt.Errorf(errMSGFormat, ipv4, iptablesPostRoutingChain, err)
+		}
+	}
+	rule, found = i.rules[ipv6][ipv6Forwarding]
+	if found {
+		log.Debugf("iptables: removing %s rule: %s ", ipv6, ipv6Forwarding)
+		err = i.ipv6Client.DeleteIfExists(iptablesFilterTable, iptablesForwardChain, rule...)
+		if err != nil {
+			return fmt.Errorf(errMSGFormat, ipv6, iptablesForwardChain, err)
+		}
+	}
+	rule, found = i.rules[ipv6][ipv6Nat]
+	if found {
+		log.Debugf("iptables: removing %s rule: %s ", ipv6, ipv6Nat)
+		err = i.ipv6Client.DeleteIfExists(iptablesNatTable, iptablesPostRoutingChain, rule...)
+		if err != nil {
+			return fmt.Errorf(errMSGFormat, ipv6, iptablesPostRoutingChain, err)
+		}
+	}
+	return nil
+}
+
+func iptablesProtoToString(proto iptables.Protocol) string {
+	if proto == iptables.ProtocolIPv6 {
+		return ipv6
+	}
+	return ipv4
+}
+
+// restoreRules restores existing NetBird rules
+func (i *iptablesManager) restoreRules(iptablesClient *iptables.IPTables) error {
+	ipVersion := iptablesProtoToString(iptablesClient.Proto())
+
+	if i.rules[ipVersion] == nil {
+		i.rules[ipVersion] = make(map[string][]string)
+	}
+	table := iptablesFilterTable
+	for _, chain := range []string{iptablesForwardChain, iptablesRoutingForwardingChain} {
+		rules, err := iptablesClient.List(table, chain)
+		if err != nil {
+			return err
+		}
+		for _, ruleString := range rules {
+			rule := strings.Fields(ruleString)
+			id := getRuleRouteID(rule)
+			if id != "" {
+				i.rules[ipVersion][id] = rule[2:]
+			}
+		}
+	}
+
+	table = iptablesNatTable
+	for _, chain := range []string{iptablesPostRoutingChain, iptablesRoutingNatChain} {
+		rules, err := iptablesClient.List(table, chain)
+		if err != nil {
+			return err
+		}
+		for _, ruleString := range rules {
+			rule := strings.Fields(ruleString)
+			id := getRuleRouteID(rule)
+			if id != "" {
+				i.rules[ipVersion][id] = rule[2:]
+			}
+		}
+	}
+
+	return nil
+}
+
+// createChain create NetBird chains
+func createChain(iptables *iptables.IPTables, table, newChain string) error {
+	chains, err := iptables.ListChains(table)
+	if err != nil {
+		return fmt.Errorf("couldn't get %s %s table chains, error: %v", iptablesProtoToString(iptables.Proto()), table, err)
+	}
+
+	shouldCreateChain := true
+	for _, chain := range chains {
+		if chain == newChain {
+			shouldCreateChain = false
+		}
+	}
+
+	if shouldCreateChain {
+		err = iptables.NewChain(table, newChain)
+		if err != nil {
+			return fmt.Errorf("couldn't create %s chain %s in %s table, error: %v", iptablesProtoToString(iptables.Proto()), newChain, table, err)
+		}
+
+		if table == iptablesNatTable {
+			err = iptables.Append(table, newChain, iptablesDefaultNetbirdNatRule...)
+		} else {
+			err = iptables.Append(table, newChain, iptablesDefaultNetbirdForwardingRule...)
+		}
+		if err != nil {
+			return fmt.Errorf("couldn't create %s chain %s default rule, error: %v", iptablesProtoToString(iptables.Proto()), newChain, err)
+		}
+
+	}
+	return nil
+}
+
+// genRuleSpec generates rule specification with comment identifier
+func genRuleSpec(jump, id, source, destination string) []string {
+	return []string{"-s", source, "-d", destination, "-j", jump, "-m", "comment", "--comment", id}
+}
+
+// getRuleRouteID returns the rule ID if matches our prefix
+func getRuleRouteID(rule []string) string {
+	for i, flag := range rule {
+		if flag == "--comment" {
+			id := rule[i+1]
+			if strings.HasPrefix(id, "netbird-") {
+				return id
+			}
+		}
+	}
+	return ""
+}
+
+// InsertRoutingRules inserts an iptables rule pair to the forwarding chain and if enabled, to the nat chain
+func (i *iptablesManager) InsertRoutingRules(pair routerPair) error {
+	i.mux.Lock()
+	defer i.mux.Unlock()
+
+	var err error
+	prefix := netip.MustParsePrefix(pair.source)
+	ipVersion := ipv4
+	iptablesClient := i.ipv4Client
+	if prefix.Addr().Unmap().Is6() {
+		iptablesClient = i.ipv6Client
+		ipVersion = ipv6
+	}
+
+	forwardRuleKey := genKey(forwardingFormat, pair.ID)
+	forwardRule := genRuleSpec(routingFinalForwardJump, forwardRuleKey, pair.source, pair.destination)
+	existingRule, found := i.rules[ipVersion][forwardRuleKey]
+	if found {
+		err = iptablesClient.DeleteIfExists(iptablesFilterTable, iptablesRoutingForwardingChain, existingRule...)
+		if err != nil {
+			return fmt.Errorf("iptables: error while removing existing forwarding rule for %s: %v", pair.destination, err)
+		}
+		delete(i.rules[ipVersion], forwardRuleKey)
+	}
+	err = iptablesClient.Insert(iptablesFilterTable, iptablesRoutingForwardingChain, 1, forwardRule...)
+	if err != nil {
+		return fmt.Errorf("iptables: error while adding new forwarding rule for %s: %v", pair.destination, err)
+	}
+
+	i.rules[ipVersion][forwardRuleKey] = forwardRule
+
+	if !pair.masquerade {
+		return nil
+	}
+
+	natRuleKey := genKey(natFormat, pair.ID)
+	natRule := genRuleSpec(routingFinalNatJump, natRuleKey, pair.source, pair.destination)
+	existingRule, found = i.rules[ipVersion][natRuleKey]
+	if found {
+		err = iptablesClient.DeleteIfExists(iptablesNatTable, iptablesRoutingNatChain, existingRule...)
+		if err != nil {
+			return fmt.Errorf("iptables: error while removing existing nat rulefor %s: %v", pair.destination, err)
+		}
+		delete(i.rules[ipVersion], natRuleKey)
+	}
+	err = iptablesClient.Insert(iptablesNatTable, iptablesRoutingNatChain, 1, natRule...)
+	if err != nil {
+		return fmt.Errorf("iptables: error while adding new nat rulefor %s: %v", pair.destination, err)
+	}
+
+	i.rules[ipVersion][natRuleKey] = natRule
+
+	return nil
+}
+
+// RemoveRoutingRules removes an iptables rule pair from forwarding and nat chains
+func (i *iptablesManager) RemoveRoutingRules(pair routerPair) error {
+	i.mux.Lock()
+	defer i.mux.Unlock()
+
+	var err error
+	prefix := netip.MustParsePrefix(pair.source)
+	ipVersion := ipv4
+	iptablesClient := i.ipv4Client
+	if prefix.Addr().Unmap().Is6() {
+		iptablesClient = i.ipv6Client
+		ipVersion = ipv6
+	}
+
+	forwardRuleKey := genKey(forwardingFormat, pair.ID)
+	existingRule, found := i.rules[ipVersion][forwardRuleKey]
+	if found {
+		err = iptablesClient.DeleteIfExists(iptablesFilterTable, iptablesRoutingForwardingChain, existingRule...)
+		if err != nil {
+			return fmt.Errorf("iptables: error while removing existing forwarding rule for %s: %v", pair.destination, err)
+		}
+	}
+	delete(i.rules[ipVersion], forwardRuleKey)
+
+	if !pair.masquerade {
+		return nil
+	}
+
+	natRuleKey := genKey(natFormat, pair.ID)
+	existingRule, found = i.rules[ipVersion][natRuleKey]
+	if found {
+		err = iptablesClient.DeleteIfExists(iptablesNatTable, iptablesRoutingNatChain, existingRule...)
+		if err != nil {
+			return fmt.Errorf("iptables: error while removing existing nat rule for %s: %v", pair.destination, err)
+		}
+	}
+	delete(i.rules[ipVersion], natRuleKey)
+
+	return nil
+}

client/internal/routemanager/iptables_linux_test.go

@@ -0,0 +1,247 @@
+package routemanager
+
+import (
+	"context"
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) {
+
+	if !isIptablesSupported() {
+		t.SkipNow()
+	}
+
+	ctx, cancel := context.WithCancel(context.TODO())
+	ipv4Client, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	ipv6Client, _ := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+
+	manager := &iptablesManager{
+		ctx:        ctx,
+		stop:       cancel,
+		ipv4Client: ipv4Client,
+		ipv6Client: ipv6Client,
+		rules:      make(map[string]map[string][]string),
+	}
+
+	defer manager.CleanRoutingRules()
+
+	err := manager.RestoreOrCreateContainers()
+	require.NoError(t, err, "shouldn't return error")
+
+	require.Len(t, manager.rules, 2, "should have created maps for ipv4 and ipv6")
+
+	require.Len(t, manager.rules[ipv4], 2, "should have created minimal rules for ipv4")
+
+	exists, err := ipv4Client.Exists(iptablesFilterTable, iptablesForwardChain, manager.rules[ipv4][ipv4Forwarding]...)
+	require.NoError(t, err, "should be able to query the iptables %s %s table and %s chain", ipv4, iptablesFilterTable, iptablesForwardChain)
+	require.True(t, exists, "forwarding rule should exist")
+
+	exists, err = ipv4Client.Exists(iptablesNatTable, iptablesPostRoutingChain, manager.rules[ipv4][ipv4Nat]...)
+	require.NoError(t, err, "should be able to query the iptables %s %s table and %s chain", ipv4, iptablesNatTable, iptablesPostRoutingChain)
+	require.True(t, exists, "postrouting rule should exist")
+
+	require.Len(t, manager.rules[ipv6], 2, "should have created minimal rules for ipv6")
+
+	exists, err = ipv6Client.Exists(iptablesFilterTable, iptablesForwardChain, manager.rules[ipv6][ipv6Forwarding]...)
+	require.NoError(t, err, "should be able to query the iptables %s %s table and %s chain", ipv6, iptablesFilterTable, iptablesForwardChain)
+	require.True(t, exists, "forwarding rule should exist")
+
+	exists, err = ipv6Client.Exists(iptablesNatTable, iptablesPostRoutingChain, manager.rules[ipv6][ipv6Nat]...)
+	require.NoError(t, err, "should be able to query the iptables %s %s table and %s chain", ipv6, iptablesNatTable, iptablesPostRoutingChain)
+	require.True(t, exists, "postrouting rule should exist")
+
+	pair := routerPair{
+		ID:          "abc",
+		source:      "100.100.100.1/32",
+		destination: "100.100.100.0/24",
+		masquerade:  true,
+	}
+	forward4RuleKey := genKey(forwardingFormat, pair.ID)
+	forward4Rule := genRuleSpec(routingFinalForwardJump, forward4RuleKey, pair.source, pair.destination)
+
+	err = ipv4Client.Insert(iptablesFilterTable, iptablesRoutingForwardingChain, 1, forward4Rule...)
+	require.NoError(t, err, "inserting rule should not return error")
+
+	nat4RuleKey := genKey(natFormat, pair.ID)
+	nat4Rule := genRuleSpec(routingFinalNatJump, nat4RuleKey, pair.source, pair.destination)
+
+	err = ipv4Client.Insert(iptablesNatTable, iptablesRoutingNatChain, 1, nat4Rule...)
+	require.NoError(t, err, "inserting rule should not return error")
+
+	pair = routerPair{
+		ID:          "abc",
+		source:      "fc00::1/128",
+		destination: "fc11::/64",
+		masquerade:  true,
+	}
+
+	forward6RuleKey := genKey(forwardingFormat, pair.ID)
+	forward6Rule := genRuleSpec(routingFinalForwardJump, forward6RuleKey, pair.source, pair.destination)
+
+	err = ipv6Client.Insert(iptablesFilterTable, iptablesRoutingForwardingChain, 1, forward6Rule...)
+	require.NoError(t, err, "inserting rule should not return error")
+
+	nat6RuleKey := genKey(natFormat, pair.ID)
+	nat6Rule := genRuleSpec(routingFinalNatJump, nat6RuleKey, pair.source, pair.destination)
+
+	err = ipv6Client.Insert(iptablesNatTable, iptablesRoutingNatChain, 1, nat6Rule...)
+	require.NoError(t, err, "inserting rule should not return error")
+
+	delete(manager.rules, ipv4)
+	delete(manager.rules, ipv6)
+
+	err = manager.RestoreOrCreateContainers()
+	require.NoError(t, err, "shouldn't return error")
+
+	require.Len(t, manager.rules[ipv4], 4, "should have restored all rules for ipv4")
+
+	foundRule, found := manager.rules[ipv4][forward4RuleKey]
+	require.True(t, found, "forwarding rule should exist in the map")
+	require.Equal(t, forward4Rule[:4], foundRule[:4], "stored forwarding rule should match")
+
+	foundRule, found = manager.rules[ipv4][nat4RuleKey]
+	require.True(t, found, "nat rule should exist in the map")
+	require.Equal(t, nat4Rule[:4], foundRule[:4], "stored nat rule should match")
+
+	require.Len(t, manager.rules[ipv6], 4, "should have restored all rules for ipv6")
+
+	foundRule, found = manager.rules[ipv6][forward6RuleKey]
+	require.True(t, found, "forwarding rule should exist in the map")
+	require.Equal(t, forward6Rule[:4], foundRule[:4], "stored forward rule should match")
+
+	foundRule, found = manager.rules[ipv6][nat6RuleKey]
+	require.True(t, found, "nat rule should exist in the map")
+	require.Equal(t, nat6Rule[:4], foundRule[:4], "stored nat rule should match")
+}
+
+func TestIptablesManager_InsertRoutingRules(t *testing.T) {
+
+	if !isIptablesSupported() {
+		t.SkipNow()
+	}
+
+	for _, testCase := range insertRuleTestCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.TODO())
+			ipv4Client, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+			ipv6Client, _ := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+			iptablesClient := ipv4Client
+			if testCase.ipVersion == ipv6 {
+				iptablesClient = ipv6Client
+			}
+
+			manager := &iptablesManager{
+				ctx:        ctx,
+				stop:       cancel,
+				ipv4Client: ipv4Client,
+				ipv6Client: ipv6Client,
+				rules:      make(map[string]map[string][]string),
+			}
+
+			defer manager.CleanRoutingRules()
+
+			err := manager.RestoreOrCreateContainers()
+			require.NoError(t, err, "shouldn't return error")
+
+			err = manager.InsertRoutingRules(testCase.inputPair)
+			require.NoError(t, err, "forwarding pair should be inserted")
+
+			forwardRuleKey := genKey(forwardingFormat, testCase.inputPair.ID)
+			forwardRule := genRuleSpec(routingFinalForwardJump, forwardRuleKey, testCase.inputPair.source, testCase.inputPair.destination)
+
+			exists, err := iptablesClient.Exists(iptablesFilterTable, iptablesRoutingForwardingChain, forwardRule...)
+			require.NoError(t, err, "should be able to query the iptables %s %s table and %s chain", testCase.ipVersion, iptablesFilterTable, iptablesRoutingForwardingChain)
+			require.True(t, exists, "forwarding rule should exist")
+
+			foundRule, found := manager.rules[testCase.ipVersion][forwardRuleKey]
+			require.True(t, found, "forwarding rule should exist in the manager map")
+			require.Equal(t, forwardRule[:4], foundRule[:4], "stored forwarding rule should match")
+
+			natRuleKey := genKey(natFormat, testCase.inputPair.ID)
+			natRule := genRuleSpec(routingFinalNatJump, natRuleKey, testCase.inputPair.source, testCase.inputPair.destination)
+
+			exists, err = iptablesClient.Exists(iptablesNatTable, iptablesRoutingNatChain, natRule...)
+			require.NoError(t, err, "should be able to query the iptables %s %s table and %s chain", testCase.ipVersion, iptablesNatTable, iptablesRoutingNatChain)
+			if testCase.inputPair.masquerade {
+				require.True(t, exists, "nat rule should be created")
+				foundNatRule, foundNat := manager.rules[testCase.ipVersion][natRuleKey]
+				require.True(t, foundNat, "nat rule should exist in the map")
+				require.Equal(t, natRule[:4], foundNatRule[:4], "stored nat rule should match")
+			} else {
+				require.False(t, exists, "nat rule should not be created")
+				_, foundNat := manager.rules[testCase.ipVersion][natRuleKey]
+				require.False(t, foundNat, "nat rule should exist in the map")
+			}
+		})
+	}
+}
+
+func TestIptablesManager_RemoveRoutingRules(t *testing.T) {
+
+	if !isIptablesSupported() {
+		t.SkipNow()
+	}
+
+	for _, testCase := range removeRuleTestCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.TODO())
+			ipv4Client, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+			ipv6Client, _ := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+			iptablesClient := ipv4Client
+			if testCase.ipVersion == ipv6 {
+				iptablesClient = ipv6Client
+			}
+
+			manager := &iptablesManager{
+				ctx:        ctx,
+				stop:       cancel,
+				ipv4Client: ipv4Client,
+				ipv6Client: ipv6Client,
+				rules:      make(map[string]map[string][]string),
+			}
+
+			defer manager.CleanRoutingRules()
+
+			err := manager.RestoreOrCreateContainers()
+			require.NoError(t, err, "shouldn't return error")
+
+			forwardRuleKey := genKey(forwardingFormat, testCase.inputPair.ID)
+			forwardRule := genRuleSpec(routingFinalForwardJump, forwardRuleKey, testCase.inputPair.source, testCase.inputPair.destination)
+
+			err = iptablesClient.Insert(iptablesFilterTable, iptablesRoutingForwardingChain, 1, forwardRule...)
+			require.NoError(t, err, "inserting rule should not return error")
+
+			natRuleKey := genKey(natFormat, testCase.inputPair.ID)
+			natRule := genRuleSpec(routingFinalNatJump, natRuleKey, testCase.inputPair.source, testCase.inputPair.destination)
+
+			err = iptablesClient.Insert(iptablesNatTable, iptablesRoutingNatChain, 1, natRule...)
+			require.NoError(t, err, "inserting rule should not return error")
+
+			delete(manager.rules, ipv4)
+			delete(manager.rules, ipv6)
+
+			err = manager.RestoreOrCreateContainers()
+			require.NoError(t, err, "shouldn't return error")
+
+			err = manager.RemoveRoutingRules(testCase.inputPair)
+			require.NoError(t, err, "shouldn't return error")
+
+			exists, err := iptablesClient.Exists(iptablesFilterTable, iptablesRoutingForwardingChain, forwardRule...)
+			require.NoError(t, err, "should be able to query the iptables %s %s table and %s chain", testCase.ipVersion, iptablesFilterTable, iptablesRoutingForwardingChain)
+			require.False(t, exists, "forwarding rule should not exist")
+
+			_, found := manager.rules[testCase.ipVersion][forwardRuleKey]
+			require.False(t, found, "forwarding rule should exist in the manager map")
+
+			exists, err = iptablesClient.Exists(iptablesNatTable, iptablesRoutingNatChain, natRule...)
+			require.NoError(t, err, "should be able to query the iptables %s %s table and %s chain", testCase.ipVersion, iptablesNatTable, iptablesRoutingNatChain)
+			require.False(t, exists, "nat rule should not exist")
+
+			_, found = manager.rules[testCase.ipVersion][natRuleKey]
+			require.False(t, found, "forwarding rule should exist in the manager map")
+
+		})
+	}
+}

client/internal/routemanager/manager.go

@@ -0,0 +1,181 @@
+package routemanager
+
+import (
+	"context"
+	"fmt"
+	"github.com/netbirdio/netbird/client/status"
+	"github.com/netbirdio/netbird/client/system"
+	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/route"
+	log "github.com/sirupsen/logrus"
+	"runtime"
+	"sync"
+)
+
+// Manager is a route manager interface
+type Manager interface {
+	UpdateRoutes(updateSerial uint64, newRoutes []*route.Route) error
+	Stop()
+}
+
+// DefaultManager is the default instance of a route manager
+type DefaultManager struct {
+	ctx            context.Context
+	stop           context.CancelFunc
+	mux            sync.Mutex
+	clientNetworks map[string]*clientNetwork
+	serverRoutes   map[string]*route.Route
+	serverRouter   *serverRouter
+	statusRecorder *status.Status
+	wgInterface    *iface.WGIface
+	pubKey         string
+}
+
+// NewManager returns a new route manager
+func NewManager(ctx context.Context, pubKey string, wgInterface *iface.WGIface, statusRecorder *status.Status) *DefaultManager {
+	mCTX, cancel := context.WithCancel(ctx)
+	return &DefaultManager{
+		ctx:            mCTX,
+		stop:           cancel,
+		clientNetworks: make(map[string]*clientNetwork),
+		serverRoutes:   make(map[string]*route.Route),
+		serverRouter: &serverRouter{
+			routes:                   make(map[string]*route.Route),
+			netForwardHistoryEnabled: isNetForwardHistoryEnabled(),
+			firewall:                 NewFirewall(ctx),
+		},
+		statusRecorder: statusRecorder,
+		wgInterface:    wgInterface,
+		pubKey:         pubKey,
+	}
+}
+
+// Stop stops the manager watchers and clean firewall rules
+func (m *DefaultManager) Stop() {
+	m.stop()
+	m.serverRouter.firewall.CleanRoutingRules()
+}
+
+func (m *DefaultManager) updateClientNetworks(updateSerial uint64, networks map[string][]*route.Route) {
+	// removing routes that do not exist as per the update from the Management service.
+	for id, client := range m.clientNetworks {
+		_, found := networks[id]
+		if !found {
+			log.Debugf("stopping client network watcher, %s", id)
+			client.stop()
+			delete(m.clientNetworks, id)
+		}
+	}
+
+	for id, routes := range networks {
+		clientNetworkWatcher, found := m.clientNetworks[id]
+		if !found {
+			clientNetworkWatcher = newClientNetworkWatcher(m.ctx, m.wgInterface, m.statusRecorder, routes[0].Network)
+			m.clientNetworks[id] = clientNetworkWatcher
+			go clientNetworkWatcher.peersStateAndUpdateWatcher()
+		}
+		update := routesUpdate{
+			updateSerial: updateSerial,
+			routes:       routes,
+		}
+
+		clientNetworkWatcher.sendUpdateToClientNetworkWatcher(update)
+	}
+}
+
+func (m *DefaultManager) updateServerRoutes(routesMap map[string]*route.Route) error {
+	serverRoutesToRemove := make([]string, 0)
+
+	if len(routesMap) > 0 {
+		err := m.serverRouter.firewall.RestoreOrCreateContainers()
+		if err != nil {
+			return fmt.Errorf("couldn't initialize firewall containers, got err: %v", err)
+		}
+	}
+
+	for routeID := range m.serverRoutes {
+		update, found := routesMap[routeID]
+		if !found || !update.IsEqual(m.serverRoutes[routeID]) {
+			serverRoutesToRemove = append(serverRoutesToRemove, routeID)
+			continue
+		}
+	}
+
+	for _, routeID := range serverRoutesToRemove {
+		oldRoute := m.serverRoutes[routeID]
+		err := m.removeFromServerNetwork(oldRoute)
+		if err != nil {
+			log.Errorf("unable to remove route id: %s, network %s, from server, got: %v",
+				oldRoute.ID, oldRoute.Network, err)
+		}
+		delete(m.serverRoutes, routeID)
+	}
+
+	for id, newRoute := range routesMap {
+		_, found := m.serverRoutes[id]
+		if found {
+			continue
+		}
+
+		err := m.addToServerNetwork(newRoute)
+		if err != nil {
+			log.Errorf("unable to add route %s from server, got: %v", newRoute.ID, err)
+			continue
+		}
+		m.serverRoutes[id] = newRoute
+	}
+
+	if len(m.serverRoutes) > 0 {
+		err := enableIPForwarding()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// UpdateRoutes compares received routes with existing routes and remove, update or add them to the client and server maps
+func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Route) error {
+	select {
+	case <-m.ctx.Done():
+		log.Infof("not updating routes as context is closed")
+		return m.ctx.Err()
+	default:
+		m.mux.Lock()
+		defer m.mux.Unlock()
+
+		newClientRoutesIDMap := make(map[string][]*route.Route)
+		newServerRoutesMap := make(map[string]*route.Route)
+
+		for _, newRoute := range newRoutes {
+			// only linux is supported for now
+			if newRoute.Peer == m.pubKey {
+				if runtime.GOOS != "linux" {
+					log.Warnf("received a route to manage, but agent doesn't support router mode on %s OS", runtime.GOOS)
+					continue
+				}
+				newServerRoutesMap[newRoute.ID] = newRoute
+			} else {
+				// if prefix is too small, lets assume is a possible default route which is not yet supported
+				// we skip this route management
+				if newRoute.Network.Bits() < 7 {
+					log.Errorf("this agent version: %s, doesn't support default routes, received %s, skiping this route",
+						system.NetbirdVersion(), newRoute.Network)
+					continue
+				}
+				clientNetworkID := getClientNetworkID(newRoute)
+				newClientRoutesIDMap[clientNetworkID] = append(newClientRoutesIDMap[clientNetworkID], newRoute)
+			}
+		}
+
+		m.updateClientNetworks(updateSerial, newClientRoutesIDMap)
+
+		err := m.updateServerRoutes(newServerRoutesMap)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}

client/internal/routemanager/manager_test.go

@@ -0,0 +1,370 @@
+package routemanager
+
+import (
+	"context"
+	"fmt"
+	"github.com/netbirdio/netbird/client/status"
+	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/route"
+	"github.com/stretchr/testify/require"
+	"net/netip"
+	"runtime"
+	"testing"
+)
+
+// send 5 routes, one for server and 4 for clients, one normal and 2 HA and one small
+// if linux host, should have one for server in map
+// we should have 2 client manager
+// 2 ranges in our routing table
+
+const localPeerKey = "local"
+const remotePeerKey1 = "remote1"
+const remotePeerKey2 = "remote1"
+
+func TestManagerUpdateRoutes(t *testing.T) {
+	testCases := []struct {
+		name                          string
+		inputInitRoutes               []*route.Route
+		inputRoutes                   []*route.Route
+		inputSerial                   uint64
+		shouldCheckServerRoutes       bool
+		serverRoutesExpected          int
+		clientNetworkWatchersExpected int
+	}{
+		{
+			name:            "Should create 2 client networks",
+			inputInitRoutes: []*route.Route{},
+			inputRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("100.64.251.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "b",
+					NetID:       "routeB",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("8.8.8.8/32"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputSerial:                   1,
+			clientNetworkWatchersExpected: 2,
+		},
+		{
+			name: "Should Create 2 Server Routes",
+			inputRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        localPeerKey,
+					Network:     netip.MustParsePrefix("100.64.252.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "b",
+					NetID:       "routeB",
+					Peer:        localPeerKey,
+					Network:     netip.MustParsePrefix("8.8.8.9/32"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputSerial:                   1,
+			shouldCheckServerRoutes:       runtime.GOOS == "linux",
+			serverRoutesExpected:          2,
+			clientNetworkWatchersExpected: 0,
+		},
+		{
+			name: "Should Create 1 Route For Client And Server",
+			inputRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        localPeerKey,
+					Network:     netip.MustParsePrefix("100.64.30.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "b",
+					NetID:       "routeB",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("8.8.9.9/32"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputSerial:                   1,
+			shouldCheckServerRoutes:       runtime.GOOS == "linux",
+			serverRoutesExpected:          1,
+			clientNetworkWatchersExpected: 1,
+		},
+		{
+			name: "Should Create 1 HA Route and 1 Standalone",
+			inputRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("8.8.20.0/24"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "b",
+					NetID:       "routeA",
+					Peer:        remotePeerKey2,
+					Network:     netip.MustParsePrefix("8.8.20.0/24"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "c",
+					NetID:       "routeB",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("8.8.9.9/32"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputSerial:                   1,
+			clientNetworkWatchersExpected: 2,
+		},
+		{
+			name: "No Small Client Route Should Be Added",
+			inputRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("0.0.0.0/0"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputSerial:                   1,
+			clientNetworkWatchersExpected: 0,
+		},
+		{
+			name: "No Server Routes Should Be Added To Non Linux",
+			inputRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        localPeerKey,
+					Network:     netip.MustParsePrefix("1.2.3.4/32"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputSerial:                   1,
+			shouldCheckServerRoutes:       runtime.GOOS != "linux",
+			serverRoutesExpected:          0,
+			clientNetworkWatchersExpected: 0,
+		},
+		{
+			name: "Remove 1 Client Route",
+			inputInitRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("100.64.251.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "b",
+					NetID:       "routeB",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("8.8.8.8/32"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("100.64.251.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputSerial:                   1,
+			clientNetworkWatchersExpected: 1,
+		},
+		{
+			name: "Update Route to HA",
+			inputInitRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("100.64.251.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "b",
+					NetID:       "routeB",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("8.8.8.8/32"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("100.64.251.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "b",
+					NetID:       "routeA",
+					Peer:        remotePeerKey2,
+					Network:     netip.MustParsePrefix("100.64.251.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputSerial:                   1,
+			clientNetworkWatchersExpected: 1,
+		},
+		{
+			name: "Remove Client Routes",
+			inputInitRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("100.64.251.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "b",
+					NetID:       "routeB",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("8.8.8.8/32"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputRoutes:                   []*route.Route{},
+			inputSerial:                   1,
+			clientNetworkWatchersExpected: 0,
+		},
+		{
+			name: "Remove All Routes",
+			inputInitRoutes: []*route.Route{
+				{
+					ID:          "a",
+					NetID:       "routeA",
+					Peer:        localPeerKey,
+					Network:     netip.MustParsePrefix("100.64.251.250/30"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+				{
+					ID:          "b",
+					NetID:       "routeB",
+					Peer:        remotePeerKey1,
+					Network:     netip.MustParsePrefix("8.8.8.8/32"),
+					NetworkType: route.IPv4Network,
+					Metric:      9999,
+					Masquerade:  false,
+					Enabled:     true,
+				},
+			},
+			inputRoutes:                   []*route.Route{},
+			inputSerial:                   1,
+			shouldCheckServerRoutes:       true,
+			serverRoutesExpected:          0,
+			clientNetworkWatchersExpected: 0,
+		},
+	}
+
+	for n, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun43%d", n), "100.65.65.2/24", iface.DefaultMTU)
+			require.NoError(t, err, "should create testing WGIface interface")
+			defer wgInterface.Close()
+
+			err = wgInterface.Create()
+			require.NoError(t, err, "should create testing wireguard interface")
+
+			statusRecorder := status.NewRecorder()
+			ctx := context.TODO()
+			routeManager := NewManager(ctx, localPeerKey, wgInterface, statusRecorder)
+			defer routeManager.Stop()
+
+			if len(testCase.inputInitRoutes) > 0 {
+				err = routeManager.UpdateRoutes(testCase.inputSerial, testCase.inputRoutes)
+				require.NoError(t, err, "should update routes with init routes")
+			}
+
+			err = routeManager.UpdateRoutes(testCase.inputSerial+uint64(len(testCase.inputInitRoutes)), testCase.inputRoutes)
+			require.NoError(t, err, "should update routes")
+
+			require.Len(t, routeManager.clientNetworks, testCase.clientNetworkWatchersExpected, "client networks size should match")
+
+			if testCase.shouldCheckServerRoutes {
+				require.Len(t, routeManager.serverRoutes, testCase.serverRoutesExpected, "server networks size should match")
+			}
+		})
+	}
+}

client/internal/routemanager/mock.go

@@ -0,0 +1,27 @@
+package routemanager
+
+import (
+	"fmt"
+	"github.com/netbirdio/netbird/route"
+)
+
+// MockManager is the mock instance of a route manager
+type MockManager struct {
+	UpdateRoutesFunc func(updateSerial uint64, newRoutes []*route.Route) error
+	StopFunc         func()
+}
+
+// UpdateRoutes mock implementation of UpdateRoutes from Manager interface
+func (m *MockManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Route) error {
+	if m.UpdateRoutesFunc != nil {
+		return m.UpdateRoutesFunc(updateSerial, newRoutes)
+	}
+	return fmt.Errorf("method UpdateRoutes is not implemented")
+}
+
+// Stop mock implementation of Stop from Manager interface
+func (m *MockManager) Stop() {
+	if m.StopFunc != nil {
+		m.StopFunc()
+	}
+}

client/internal/routemanager/nftables_linux.go

@@ -0,0 +1,384 @@
+package routemanager
+
+import (
+	"context"
+	"fmt"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	log "github.com/sirupsen/logrus"
+	"net"
+	"net/netip"
+	"sync"
+)
+import "github.com/google/nftables"
+
+//
+const (
+	nftablesTable                  = "netbird-rt"
+	nftablesRoutingForwardingChain = "netbird-rt-fwd"
+	nftablesRoutingNatChain        = "netbird-rt-nat"
+)
+
+// constants needed to create nftable rules
+const (
+	ipv4Len                  = 4
+	ipv4SrcOffset            = 12
+	ipv4DestOffset           = 16
+	ipv6Len                  = 16
+	ipv6SrcOffset            = 8
+	ipv6DestOffset           = 24
+	exprDirectionSource      = "source"
+	exprDirectionDestination = "destination"
+)
+
+// some presets for building nftable rules
+var (
+	zeroXor = binaryutil.NativeEndian.PutUint32(0)
+
+	zeroXor6 = append(binaryutil.NativeEndian.PutUint64(0), binaryutil.NativeEndian.PutUint64(0)...)
+
+	exprAllowRelatedEstablished = []expr.Any{
+		&expr.Ct{
+			Register:       1,
+			SourceRegister: false,
+			Key:            0,
+		},
+		&expr.Bitwise{
+			DestRegister:   1,
+			SourceRegister: 1,
+			Len:            4,
+			Mask:           []uint8{0x6, 0x0, 0x0, 0x0},
+			Xor:            zeroXor,
+		},
+		&expr.Cmp{
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+
+	exprCounterAccept = []expr.Any{
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+)
+
+type nftablesManager struct {
+	ctx       context.Context
+	stop      context.CancelFunc
+	conn      *nftables.Conn
+	tableIPv4 *nftables.Table
+	tableIPv6 *nftables.Table
+	chains    map[string]map[string]*nftables.Chain
+	rules     map[string]*nftables.Rule
+	mux       sync.Mutex
+}
+
+// CleanRoutingRules cleans existing nftables rules from the system
+func (n *nftablesManager) CleanRoutingRules() {
+	n.mux.Lock()
+	defer n.mux.Unlock()
+	log.Debug("flushing tables")
+	n.conn.FlushTable(n.tableIPv6)
+	n.conn.FlushTable(n.tableIPv4)
+	log.Debugf("flushing tables result in: %v error", n.conn.Flush())
+}
+
+// RestoreOrCreateContainers restores existing nftables containers (tables and chains)
+// if they don't exist, we create them
+func (n *nftablesManager) RestoreOrCreateContainers() error {
+	n.mux.Lock()
+	defer n.mux.Unlock()
+
+	if n.tableIPv6 != nil && n.tableIPv4 != nil {
+		log.Debugf("nftables: containers already restored, skipping")
+		return nil
+	}
+
+	tables, err := n.conn.ListTables()
+	if err != nil {
+		return fmt.Errorf("nftables: unable to list tables: %v", err)
+	}
+
+	for _, table := range tables {
+		if table.Name == nftablesTable {
+			if table.Family == nftables.TableFamilyIPv4 {
+				n.tableIPv4 = table
+				continue
+			}
+			n.tableIPv6 = table
+		}
+	}
+
+	if n.tableIPv4 == nil {
+		n.tableIPv4 = n.conn.AddTable(&nftables.Table{
+			Name:   nftablesTable,
+			Family: nftables.TableFamilyIPv4,
+		})
+	}
+
+	if n.tableIPv6 == nil {
+		n.tableIPv6 = n.conn.AddTable(&nftables.Table{
+			Name:   nftablesTable,
+			Family: nftables.TableFamilyIPv6,
+		})
+	}
+
+	chains, err := n.conn.ListChains()
+	if err != nil {
+		return fmt.Errorf("nftables: unable to list chains: %v", err)
+	}
+
+	n.chains[ipv4] = make(map[string]*nftables.Chain)
+	n.chains[ipv6] = make(map[string]*nftables.Chain)
+
+	for _, chain := range chains {
+		switch {
+		case chain.Table.Name == nftablesTable && chain.Table.Family == nftables.TableFamilyIPv4:
+			n.chains[ipv4][chain.Name] = chain
+		case chain.Table.Name == nftablesTable && chain.Table.Family == nftables.TableFamilyIPv6:
+			n.chains[ipv6][chain.Name] = chain
+		}
+	}
+
+	if _, found := n.chains[ipv4][nftablesRoutingForwardingChain]; !found {
+		n.chains[ipv4][nftablesRoutingForwardingChain] = n.conn.AddChain(&nftables.Chain{
+			Name:     nftablesRoutingForwardingChain,
+			Table:    n.tableIPv4,
+			Hooknum:  nftables.ChainHookForward,
+			Priority: nftables.ChainPriorityNATDest + 1,
+			Type:     nftables.ChainTypeFilter,
+		})
+	}
+
+	if _, found := n.chains[ipv4][nftablesRoutingNatChain]; !found {
+		n.chains[ipv4][nftablesRoutingNatChain] = n.conn.AddChain(&nftables.Chain{
+			Name:     nftablesRoutingNatChain,
+			Table:    n.tableIPv4,
+			Hooknum:  nftables.ChainHookPostrouting,
+			Priority: nftables.ChainPriorityNATSource - 1,
+			Type:     nftables.ChainTypeNAT,
+		})
+	}
+
+	if _, found := n.chains[ipv6][nftablesRoutingForwardingChain]; !found {
+		n.chains[ipv6][nftablesRoutingForwardingChain] = n.conn.AddChain(&nftables.Chain{
+			Name:     nftablesRoutingForwardingChain,
+			Table:    n.tableIPv6,
+			Hooknum:  nftables.ChainHookForward,
+			Priority: nftables.ChainPriorityNATDest + 1,
+			Type:     nftables.ChainTypeFilter,
+		})
+	}
+
+	if _, found := n.chains[ipv6][nftablesRoutingNatChain]; !found {
+		n.chains[ipv6][nftablesRoutingNatChain] = n.conn.AddChain(&nftables.Chain{
+			Name:     nftablesRoutingNatChain,
+			Table:    n.tableIPv6,
+			Hooknum:  nftables.ChainHookPostrouting,
+			Priority: nftables.ChainPriorityNATSource - 1,
+			Type:     nftables.ChainTypeNAT,
+		})
+	}
+
+	err = n.refreshRulesMap()
+	if err != nil {
+		return err
+	}
+
+	n.checkOrCreateDefaultForwardingRules()
+	err = n.conn.Flush()
+	if err != nil {
+		return fmt.Errorf("nftables: unable to initialize table: %v", err)
+	}
+	return nil
+}
+
+// refreshRulesMap refreshes the rule map with the latest rules. this is useful to avoid
+// duplicates and to get missing attributes that we don't have when adding new rules
+func (n *nftablesManager) refreshRulesMap() error {
+	for _, registeredChains := range n.chains {
+		for _, chain := range registeredChains {
+			rules, err := n.conn.GetRules(chain.Table, chain)
+			if err != nil {
+				return fmt.Errorf("nftables: unable to list rules: %v", err)
+			}
+			for _, rule := range rules {
+				if len(rule.UserData) > 0 {
+					n.rules[string(rule.UserData)] = rule
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// checkOrCreateDefaultForwardingRules checks if the default forwarding rules are enabled
+func (n *nftablesManager) checkOrCreateDefaultForwardingRules() {
+	_, foundIPv4 := n.rules[ipv4Forwarding]
+	if !foundIPv4 {
+		n.rules[ipv4Forwarding] = n.conn.AddRule(&nftables.Rule{
+			Table:    n.tableIPv4,
+			Chain:    n.chains[ipv4][nftablesRoutingForwardingChain],
+			Exprs:    exprAllowRelatedEstablished,
+			UserData: []byte(ipv4Forwarding),
+		})
+	}
+
+	_, foundIPv6 := n.rules[ipv6Forwarding]
+	if !foundIPv6 {
+		n.rules[ipv6Forwarding] = n.conn.AddRule(&nftables.Rule{
+			Table:    n.tableIPv6,
+			Chain:    n.chains[ipv6][nftablesRoutingForwardingChain],
+			Exprs:    exprAllowRelatedEstablished,
+			UserData: []byte(ipv6Forwarding),
+		})
+	}
+}
+
+// InsertRoutingRules inserts a nftable rule pair to the forwarding chain and if enabled, to the nat chain
+func (n *nftablesManager) InsertRoutingRules(pair routerPair) error {
+	n.mux.Lock()
+	defer n.mux.Unlock()
+
+	prefix := netip.MustParsePrefix(pair.source)
+
+	sourceExp := generateCIDRMatcherExpressions("source", pair.source)
+	destExp := generateCIDRMatcherExpressions("destination", pair.destination)
+
+	forwardExp := append(sourceExp, append(destExp, exprCounterAccept...)...)
+	fwdKey := genKey(forwardingFormat, pair.ID)
+	if prefix.Addr().Unmap().Is4() {
+		n.rules[fwdKey] = n.conn.InsertRule(&nftables.Rule{
+			Table:    n.tableIPv4,
+			Chain:    n.chains[ipv4][nftablesRoutingForwardingChain],
+			Exprs:    forwardExp,
+			UserData: []byte(fwdKey),
+		})
+	} else {
+		n.rules[fwdKey] = n.conn.InsertRule(&nftables.Rule{
+			Table:    n.tableIPv6,
+			Chain:    n.chains[ipv6][nftablesRoutingForwardingChain],
+			Exprs:    forwardExp,
+			UserData: []byte(fwdKey),
+		})
+	}
+
+	if pair.masquerade {
+		natExp := append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...)
+		natKey := genKey(natFormat, pair.ID)
+
+		if prefix.Addr().Unmap().Is4() {
+			n.rules[natKey] = n.conn.InsertRule(&nftables.Rule{
+				Table:    n.tableIPv4,
+				Chain:    n.chains[ipv4][nftablesRoutingNatChain],
+				Exprs:    natExp,
+				UserData: []byte(natKey),
+			})
+		} else {
+			n.rules[natKey] = n.conn.InsertRule(&nftables.Rule{
+				Table:    n.tableIPv6,
+				Chain:    n.chains[ipv6][nftablesRoutingNatChain],
+				Exprs:    natExp,
+				UserData: []byte(natKey),
+			})
+		}
+	}
+
+	err := n.conn.Flush()
+	if err != nil {
+		return fmt.Errorf("nftables: unable to insert rules for %s: %v", pair.destination, err)
+	}
+	return nil
+}
+
+// RemoveRoutingRules removes a nftable rule pair from forwarding and nat chains
+func (n *nftablesManager) RemoveRoutingRules(pair routerPair) error {
+	n.mux.Lock()
+	defer n.mux.Unlock()
+
+	err := n.refreshRulesMap()
+	if err != nil {
+		return err
+	}
+
+	fwdKey := genKey(forwardingFormat, pair.ID)
+	natKey := genKey(natFormat, pair.ID)
+	fwdRule, found := n.rules[fwdKey]
+	if found {
+		err = n.conn.DelRule(fwdRule)
+		if err != nil {
+			return fmt.Errorf("nftables: unable to remove forwarding rule for %s: %v", pair.destination, err)
+		}
+		log.Debugf("nftables: removing forwarding rule for %s", pair.destination)
+		delete(n.rules, fwdKey)
+	}
+	natRule, found := n.rules[natKey]
+	if found {
+		err = n.conn.DelRule(natRule)
+		if err != nil {
+			return fmt.Errorf("nftables: unable to remove nat rule for %s: %v", pair.destination, err)
+		}
+		log.Debugf("nftables: removing nat rule for %s", pair.destination)
+		delete(n.rules, natKey)
+	}
+	err = n.conn.Flush()
+	if err != nil {
+		return fmt.Errorf("nftables: received error while applying rule removal for %s: %v", pair.destination, err)
+	}
+	log.Debugf("nftables: removed rules for %s", pair.destination)
+	return nil
+}
+
+// getPayloadDirectives get expression directives based on ip version and direction
+func getPayloadDirectives(direction string, isIPv4 bool, isIPv6 bool) (uint32, uint32, []byte) {
+	switch {
+	case direction == exprDirectionSource && isIPv4:
+		return ipv4SrcOffset, ipv4Len, zeroXor
+	case direction == exprDirectionDestination && isIPv4:
+		return ipv4DestOffset, ipv4Len, zeroXor
+	case direction == exprDirectionSource && isIPv6:
+		return ipv6SrcOffset, ipv6Len, zeroXor6
+	case direction == exprDirectionDestination && isIPv6:
+		return ipv6DestOffset, ipv6Len, zeroXor6
+	default:
+		panic("no matched payload directive")
+	}
+}
+
+// generateCIDRMatcherExpressions generates nftables expressions that matches a CIDR
+func generateCIDRMatcherExpressions(direction string, cidr string) []expr.Any {
+	ip, network, _ := net.ParseCIDR(cidr)
+	ipToAdd, _ := netip.AddrFromSlice(ip)
+	add := ipToAdd.Unmap()
+
+	offSet, packetLen, zeroXor := getPayloadDirectives(direction, add.Is4(), add.Is6())
+
+	return []expr.Any{
+		// fetch src add
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       offSet,
+			Len:          packetLen,
+		},
+		// net mask
+		&expr.Bitwise{
+			DestRegister:   1,
+			SourceRegister: 1,
+			Len:            packetLen,
+			Mask:           network.Mask,
+			Xor:            zeroXor,
+		},
+		// net address
+		&expr.Cmp{
+			Register: 1,
+			Data:     add.AsSlice(),
+		},
+	}
+}

client/internal/routemanager/nftables_linux_test.go

@@ -0,0 +1,270 @@
+package routemanager
+
+import (
+	"context"
+	"github.com/google/nftables"
+	"github.com/google/nftables/expr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestNftablesManager_RestoreOrCreateContainers(t *testing.T) {
+
+	ctx, cancel := context.WithCancel(context.TODO())
+
+	manager := &nftablesManager{
+		ctx:    ctx,
+		stop:   cancel,
+		conn:   &nftables.Conn{},
+		chains: make(map[string]map[string]*nftables.Chain),
+		rules:  make(map[string]*nftables.Rule),
+	}
+
+	nftablesTestingClient := &nftables.Conn{}
+
+	defer manager.CleanRoutingRules()
+
+	err := manager.RestoreOrCreateContainers()
+	require.NoError(t, err, "shouldn't return error")
+
+	require.Len(t, manager.chains, 2, "should have created chains for ipv4 and ipv6")
+	require.Len(t, manager.chains[ipv4], 2, "should have created chains for ipv4")
+	require.Len(t, manager.chains[ipv4], 2, "should have created chains for ipv6")
+	require.Len(t, manager.rules, 2, "should have created rules for ipv4 and ipv6")
+
+	pair := routerPair{
+		ID:          "abc",
+		source:      "100.100.100.1/32",
+		destination: "100.100.100.0/24",
+		masquerade:  true,
+	}
+
+	sourceExp := generateCIDRMatcherExpressions("source", pair.source)
+	destExp := generateCIDRMatcherExpressions("destination", pair.destination)
+
+	forward4Exp := append(sourceExp, append(destExp, exprCounterAccept...)...)
+	forward4RuleKey := genKey(forwardingFormat, pair.ID)
+	inserted4Forwarding := nftablesTestingClient.InsertRule(&nftables.Rule{
+		Table:    manager.tableIPv4,
+		Chain:    manager.chains[ipv4][nftablesRoutingForwardingChain],
+		Exprs:    forward4Exp,
+		UserData: []byte(forward4RuleKey),
+	})
+
+	nat4Exp := append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...)
+	nat4RuleKey := genKey(natFormat, pair.ID)
+
+	inserted4Nat := nftablesTestingClient.InsertRule(&nftables.Rule{
+		Table:    manager.tableIPv4,
+		Chain:    manager.chains[ipv4][nftablesRoutingNatChain],
+		Exprs:    nat4Exp,
+		UserData: []byte(nat4RuleKey),
+	})
+
+	err = nftablesTestingClient.Flush()
+	require.NoError(t, err, "shouldn't return error")
+
+	pair = routerPair{
+		ID:          "xyz",
+		source:      "fc00::1/128",
+		destination: "fc11::/64",
+		masquerade:  true,
+	}
+
+	sourceExp = generateCIDRMatcherExpressions("source", pair.source)
+	destExp = generateCIDRMatcherExpressions("destination", pair.destination)
+
+	forward6Exp := append(sourceExp, append(destExp, exprCounterAccept...)...)
+	forward6RuleKey := genKey(forwardingFormat, pair.ID)
+	inserted6Forwarding := nftablesTestingClient.InsertRule(&nftables.Rule{
+		Table:    manager.tableIPv6,
+		Chain:    manager.chains[ipv6][nftablesRoutingForwardingChain],
+		Exprs:    forward6Exp,
+		UserData: []byte(forward6RuleKey),
+	})
+
+	nat6Exp := append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...)
+	nat6RuleKey := genKey(natFormat, pair.ID)
+
+	inserted6Nat := nftablesTestingClient.InsertRule(&nftables.Rule{
+		Table:    manager.tableIPv6,
+		Chain:    manager.chains[ipv6][nftablesRoutingNatChain],
+		Exprs:    nat6Exp,
+		UserData: []byte(nat6RuleKey),
+	})
+
+	err = nftablesTestingClient.Flush()
+	require.NoError(t, err, "shouldn't return error")
+
+	manager.tableIPv4 = nil
+	manager.tableIPv6 = nil
+
+	err = manager.RestoreOrCreateContainers()
+	require.NoError(t, err, "shouldn't return error")
+
+	require.Len(t, manager.chains, 2, "should have created chains for ipv4 and ipv6")
+	require.Len(t, manager.chains[ipv4], 2, "should have created chains for ipv4")
+	require.Len(t, manager.chains[ipv4], 2, "should have created chains for ipv6")
+	require.Len(t, manager.rules, 6, "should have restored all rules for ipv4 and ipv6")
+
+	foundRule, found := manager.rules[forward4RuleKey]
+	require.True(t, found, "forwarding rule should exist in the map")
+	assert.Equal(t, inserted4Forwarding.Exprs, foundRule.Exprs, "stored forwarding rule expressions should match")
+
+	foundRule, found = manager.rules[nat4RuleKey]
+	require.True(t, found, "nat rule should exist in the map")
+	// match len of output as nftables client doesn't return expressions with masquerade expression
+	assert.ElementsMatch(t, inserted4Nat.Exprs[:len(foundRule.Exprs)], foundRule.Exprs, "stored nat rule expressions should match")
+
+	foundRule, found = manager.rules[forward6RuleKey]
+	require.True(t, found, "forwarding rule should exist in the map")
+	assert.Equal(t, inserted6Forwarding.Exprs, foundRule.Exprs, "stored forward rule should match")
+
+	foundRule, found = manager.rules[nat6RuleKey]
+	require.True(t, found, "nat rule should exist in the map")
+	// match len of output as nftables client doesn't return expressions with masquerade expression
+	assert.ElementsMatch(t, inserted6Nat.Exprs[:len(foundRule.Exprs)], foundRule.Exprs, "stored nat rule should match")
+}
+
+func TestNftablesManager_InsertRoutingRules(t *testing.T) {
+
+	for _, testCase := range insertRuleTestCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.TODO())
+
+			manager := &nftablesManager{
+				ctx:    ctx,
+				stop:   cancel,
+				conn:   &nftables.Conn{},
+				chains: make(map[string]map[string]*nftables.Chain),
+				rules:  make(map[string]*nftables.Rule),
+			}
+
+			nftablesTestingClient := &nftables.Conn{}
+
+			defer manager.CleanRoutingRules()
+
+			err := manager.RestoreOrCreateContainers()
+			require.NoError(t, err, "shouldn't return error")
+
+			err = manager.InsertRoutingRules(testCase.inputPair)
+			require.NoError(t, err, "forwarding pair should be inserted")
+
+			sourceExp := generateCIDRMatcherExpressions("source", testCase.inputPair.source)
+			destExp := generateCIDRMatcherExpressions("destination", testCase.inputPair.destination)
+			testingExpression := append(sourceExp, destExp...)
+			fwdRuleKey := genKey(forwardingFormat, testCase.inputPair.ID)
+
+			found := 0
+			for _, registeredChains := range manager.chains {
+				for _, chain := range registeredChains {
+					rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+					require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+					for _, rule := range rules {
+						if len(rule.UserData) > 0 && string(rule.UserData) == fwdRuleKey {
+							require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "forwarding rule elements should match")
+							found = 1
+						}
+					}
+				}
+			}
+
+			require.Equal(t, 1, found, "should find at least 1 rule to test")
+
+			if testCase.inputPair.masquerade {
+				natRuleKey := genKey(natFormat, testCase.inputPair.ID)
+				found := 0
+				for _, registeredChains := range manager.chains {
+					for _, chain := range registeredChains {
+						rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+						require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+						for _, rule := range rules {
+							if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+								require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "nat rule elements should match")
+								found = 1
+							}
+						}
+					}
+				}
+				require.Equal(t, 1, found, "should find at least 1 rule to test")
+			}
+		})
+	}
+}
+
+func TestNftablesManager_RemoveRoutingRules(t *testing.T) {
+
+	for _, testCase := range removeRuleTestCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.TODO())
+
+			manager := &nftablesManager{
+				ctx:    ctx,
+				stop:   cancel,
+				conn:   &nftables.Conn{},
+				chains: make(map[string]map[string]*nftables.Chain),
+				rules:  make(map[string]*nftables.Rule),
+			}
+
+			nftablesTestingClient := &nftables.Conn{}
+
+			defer manager.CleanRoutingRules()
+
+			err := manager.RestoreOrCreateContainers()
+			require.NoError(t, err, "shouldn't return error")
+
+			table := manager.tableIPv4
+			if testCase.ipVersion == ipv6 {
+				table = manager.tableIPv6
+			}
+
+			sourceExp := generateCIDRMatcherExpressions("source", testCase.inputPair.source)
+			destExp := generateCIDRMatcherExpressions("destination", testCase.inputPair.destination)
+
+			forwardExp := append(sourceExp, append(destExp, exprCounterAccept...)...)
+			forwardRuleKey := genKey(forwardingFormat, testCase.inputPair.ID)
+			insertedForwarding := nftablesTestingClient.InsertRule(&nftables.Rule{
+				Table:    table,
+				Chain:    manager.chains[testCase.ipVersion][nftablesRoutingForwardingChain],
+				Exprs:    forwardExp,
+				UserData: []byte(forwardRuleKey),
+			})
+
+			natExp := append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...)
+			natRuleKey := genKey(natFormat, testCase.inputPair.ID)
+
+			insertedNat := nftablesTestingClient.InsertRule(&nftables.Rule{
+				Table:    table,
+				Chain:    manager.chains[testCase.ipVersion][nftablesRoutingNatChain],
+				Exprs:    natExp,
+				UserData: []byte(natRuleKey),
+			})
+
+			err = nftablesTestingClient.Flush()
+			require.NoError(t, err, "shouldn't return error")
+
+			manager.tableIPv4 = nil
+			manager.tableIPv6 = nil
+
+			err = manager.RestoreOrCreateContainers()
+			require.NoError(t, err, "shouldn't return error")
+
+			err = manager.RemoveRoutingRules(testCase.inputPair)
+			require.NoError(t, err, "shouldn't return error")
+
+			for _, registeredChains := range manager.chains {
+				for _, chain := range registeredChains {
+					rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+					require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+					for _, rule := range rules {
+						if len(rule.UserData) > 0 {
+							require.NotEqual(t, insertedForwarding.UserData, rule.UserData, "forwarding rule should exist")
+							require.NotEqual(t, insertedNat.UserData, rule.UserData, "nat rule should exist")
+						}
+					}
+				}
+			}
+		})
+	}
+}

client/internal/routemanager/server.go

@@ -0,0 +1,67 @@
+package routemanager
+
+import (
+	"github.com/netbirdio/netbird/route"
+	log "github.com/sirupsen/logrus"
+	"net/netip"
+	"sync"
+)
+
+type serverRouter struct {
+	routes map[string]*route.Route
+	// best effort to keep net forward configuration as it was
+	netForwardHistoryEnabled bool
+	mux                      sync.Mutex
+	firewall                 firewallManager
+}
+
+type routerPair struct {
+	ID          string
+	source      string
+	destination string
+	masquerade  bool
+}
+
+func routeToRouterPair(source string, route *route.Route) routerPair {
+	parsed := netip.MustParsePrefix(source).Masked()
+	return routerPair{
+		ID:          route.ID,
+		source:      parsed.String(),
+		destination: route.Network.Masked().String(),
+		masquerade:  route.Masquerade,
+	}
+}
+
+func (m *DefaultManager) removeFromServerNetwork(route *route.Route) error {
+	select {
+	case <-m.ctx.Done():
+		log.Infof("not removing from server network because context is done")
+		return m.ctx.Err()
+	default:
+		m.serverRouter.mux.Lock()
+		defer m.serverRouter.mux.Unlock()
+		err := m.serverRouter.firewall.RemoveRoutingRules(routeToRouterPair(m.wgInterface.Address.String(), route))
+		if err != nil {
+			return err
+		}
+		delete(m.serverRouter.routes, route.ID)
+		return nil
+	}
+}
+
+func (m *DefaultManager) addToServerNetwork(route *route.Route) error {
+	select {
+	case <-m.ctx.Done():
+		log.Infof("not adding to server network because context is done")
+		return m.ctx.Err()
+	default:
+		m.serverRouter.mux.Lock()
+		defer m.serverRouter.mux.Unlock()
+		err := m.serverRouter.firewall.InsertRoutingRules(routeToRouterPair(m.wgInterface.Address.String(), route))
+		if err != nil {
+			return err
+		}
+		m.serverRouter.routes[route.ID] = route
+		return nil
+	}
+}

client/internal/routemanager/systemops.go

@@ -0,0 +1,55 @@
+package routemanager
+
+import (
+	"fmt"
+	"github.com/libp2p/go-netroute"
+	log "github.com/sirupsen/logrus"
+	"net"
+	"net/netip"
+)
+
+var errRouteNotFound = fmt.Errorf("route not found")
+
+func addToRouteTableIfNoExists(prefix netip.Prefix, addr string) error {
+	gateway, err := getExistingRIBRouteGateway(netip.MustParsePrefix("0.0.0.0/0"))
+	if err != nil && err != errRouteNotFound {
+		return err
+	}
+	prefixGateway, err := getExistingRIBRouteGateway(prefix)
+	if err != nil && err != errRouteNotFound {
+		return err
+	}
+
+	if prefixGateway != nil && !prefixGateway.Equal(gateway) {
+		log.Warnf("route for network %s already exist and is pointing to the gateway: %s, won't add another one", prefix, prefixGateway)
+		return nil
+	}
+	return addToRouteTable(prefix, addr)
+}
+
+func removeFromRouteTableIfNonSystem(prefix netip.Prefix, addr string) error {
+	addrIP := net.ParseIP(addr)
+	prefixGateway, err := getExistingRIBRouteGateway(prefix)
+	if err != nil {
+		return err
+	}
+	if prefixGateway != nil && !prefixGateway.Equal(addrIP) {
+		log.Warnf("route for network %s is pointing to a different gateway: %s, should be pointing to: %s, not removing", prefix, prefixGateway, addrIP)
+		return nil
+	}
+	return removeFromRouteTable(prefix)
+}
+
+func getExistingRIBRouteGateway(prefix netip.Prefix) (net.IP, error) {
+	r, err := netroute.New()
+	if err != nil {
+		return nil, err
+	}
+	_, _, localGatewayAddress, err := r.Route(prefix.Addr().AsSlice())
+	if err != nil {
+		log.Errorf("getting routes returned an error: %v", err)
+		return nil, errRouteNotFound
+	}
+
+	return localGatewayAddress, nil
+}

client/internal/routemanager/systemops_linux.go

@@ -0,0 +1,73 @@
+package routemanager
+
+import (
+	"github.com/vishvananda/netlink"
+	"io/ioutil"
+	"net"
+	"net/netip"
+)
+
+const ipv4ForwardingPath = "/proc/sys/net/ipv4/ip_forward"
+
+func addToRouteTable(prefix netip.Prefix, addr string) error {
+	_, ipNet, err := net.ParseCIDR(prefix.String())
+	if err != nil {
+		return err
+	}
+
+	addrMask := "/32"
+	if prefix.Addr().Unmap().Is6() {
+		addrMask = "/128"
+	}
+
+	ip, _, err := net.ParseCIDR(addr + addrMask)
+	if err != nil {
+		return err
+	}
+
+	route := &netlink.Route{
+		Scope: netlink.SCOPE_UNIVERSE,
+		Dst:   ipNet,
+		Gw:    ip,
+	}
+
+	err = netlink.RouteAdd(route)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removeFromRouteTable(prefix netip.Prefix) error {
+	_, ipNet, err := net.ParseCIDR(prefix.String())
+	if err != nil {
+		return err
+	}
+
+	route := &netlink.Route{
+		Scope: netlink.SCOPE_UNIVERSE,
+		Dst:   ipNet,
+	}
+
+	err = netlink.RouteDel(route)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func enableIPForwarding() error {
+	err := ioutil.WriteFile(ipv4ForwardingPath, []byte("1"), 0644)
+	return err
+}
+
+func isNetForwardHistoryEnabled() bool {
+	out, err := ioutil.ReadFile(ipv4ForwardingPath)
+	if err != nil {
+		// todo
+		panic(err)
+	}
+	return string(out) == "1"
+}

client/internal/routemanager/systemops_nonlinux.go

@@ -0,0 +1,41 @@
+//go:build !linux
+// +build !linux
+
+package routemanager
+
+import (
+	log "github.com/sirupsen/logrus"
+	"net/netip"
+	"os/exec"
+	"runtime"
+)
+
+func addToRouteTable(prefix netip.Prefix, addr string) error {
+	cmd := exec.Command("route", "add", prefix.String(), addr)
+	out, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+	log.Debugf(string(out))
+	return nil
+}
+
+func removeFromRouteTable(prefix netip.Prefix) error {
+	cmd := exec.Command("route", "delete", prefix.String())
+	out, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+	log.Debugf(string(out))
+	return nil
+}
+
+func enableIPForwarding() error {
+	log.Infof("enable IP forwarding is not implemented on %s", runtime.GOOS)
+	return nil
+}
+
+func isNetForwardHistoryEnabled() bool {
+	log.Infof("check netforwad history is not implemented on %s", runtime.GOOS)
+	return false
+}

client/internal/routemanager/systemops_test.go

@@ -0,0 +1,68 @@
+package routemanager
+
+import (
+	"fmt"
+	"github.com/netbirdio/netbird/iface"
+	"github.com/stretchr/testify/require"
+	"net/netip"
+	"testing"
+)
+
+func TestAddRemoveRoutes(t *testing.T) {
+	testCases := []struct {
+		name                   string
+		prefix                 netip.Prefix
+		shouldRouteToWireguard bool
+		shouldBeRemoved        bool
+	}{
+		{
+			name:                   "Should Add And Remove Route",
+			prefix:                 netip.MustParsePrefix("100.66.120.0/24"),
+			shouldRouteToWireguard: true,
+			shouldBeRemoved:        true,
+		},
+		{
+			name:                   "Should Not Add Or Remove Route",
+			prefix:                 netip.MustParsePrefix("127.0.0.1/32"),
+			shouldRouteToWireguard: false,
+			shouldBeRemoved:        false,
+		},
+	}
+
+	for n, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", iface.DefaultMTU)
+			require.NoError(t, err, "should create testing WGIface interface")
+			defer wgInterface.Close()
+
+			err = wgInterface.Create()
+			require.NoError(t, err, "should create testing wireguard interface")
+
+			err = addToRouteTableIfNoExists(testCase.prefix, wgInterface.GetAddress().IP.String())
+			require.NoError(t, err, "should not return err")
+
+			prefixGateway, err := getExistingRIBRouteGateway(testCase.prefix)
+			require.NoError(t, err, "should not return err")
+			if testCase.shouldRouteToWireguard {
+				require.Equal(t, wgInterface.GetAddress().IP.String(), prefixGateway.String(), "route should point to wireguard interface IP")
+			} else {
+				require.NotEqual(t, wgInterface.GetAddress().IP.String(), prefixGateway.String(), "route should point to a different interface")
+			}
+
+			err = removeFromRouteTableIfNonSystem(testCase.prefix, wgInterface.GetAddress().IP.String())
+			require.NoError(t, err, "should not return err")
+
+			prefixGateway, err = getExistingRIBRouteGateway(testCase.prefix)
+			require.NoError(t, err, "should not return err")
+
+			internetGateway, err := getExistingRIBRouteGateway(netip.MustParsePrefix("0.0.0.0/0"))
+			require.NoError(t, err)
+
+			if testCase.shouldBeRemoved {
+				require.Equal(t, internetGateway, prefixGateway, "route should be pointing to default internet gateway")
+			} else {
+				require.NotEqual(t, internetGateway, prefixGateway, "route should be pointing to a different gateway than the internet gateway")
+			}
+		})
+	}
+}

client/main.go

@@ -1,9 +1,8 @@
 package main
 
 import (
-	"os"
-
 	"github.com/netbirdio/netbird/client/cmd"
+	"os"
 )
 
 func main() {

client/server/server.go

@@ -208,7 +208,8 @@ func (s *Server) Login(callerCtx context.Context, msg *proto.LoginRequest) (*pro
 		hostedClient := internal.NewHostedDeviceFlow(
 			providerConfig.ProviderConfig.Audience,
 			providerConfig.ProviderConfig.ClientID,
-			providerConfig.ProviderConfig.Domain,
+			providerConfig.ProviderConfig.TokenEndpoint,
+			providerConfig.ProviderConfig.DeviceAuthEndpoint,
 		)
 
 		if s.oauthAuthFlow.client != nil && s.oauthAuthFlow.client.GetClientID(ctx) == hostedClient.GetClientID(context.TODO()) {

client/status/status.go

@@ -47,17 +47,19 @@ type FullStatus struct {
 
 // Status holds a state of peers, signal and management connections
 type Status struct {
-	mux        sync.Mutex
+	mux          sync.Mutex
-	peers      map[string]PeerState
+	peers        map[string]PeerState
-	signal     SignalState
+	changeNotify map[string]chan struct{}
-	management ManagementState
+	signal       SignalState
-	localPeer  LocalPeerState
+	management   ManagementState
+	localPeer    LocalPeerState
 }
 
 // NewRecorder returns a new Status instance
 func NewRecorder() *Status {
 	return &Status{
-		peers: make(map[string]PeerState),
+		peers:        make(map[string]PeerState),
+		changeNotify: make(map[string]chan struct{}),
 	}
 }
 
@@ -74,6 +76,18 @@ func (d *Status) AddPeer(peerPubKey string) error {
 	return nil
 }
 
+// GetPeer adds peer to Daemon status map
+func (d *Status) GetPeer(peerPubKey string) (PeerState, error) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	state, ok := d.peers[peerPubKey]
+	if !ok {
+		return PeerState{}, errors.New("peer not found")
+	}
+	return state, nil
+}
+
 // RemovePeer removes peer from Daemon status map
 func (d *Status) RemovePeer(peerPubKey string) error {
 	d.mux.Lock()
@@ -113,9 +127,27 @@ func (d *Status) UpdatePeerState(receivedState PeerState) error {
 
 	d.peers[receivedState.PubKey] = peerState
 
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
 	return nil
 }
 
+// GetPeerStateChangeNotifier returns a change notifier channel for a peer
+func (d *Status) GetPeerStateChangeNotifier(peer string) <-chan struct{} {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	ch, found := d.changeNotify[peer]
+	if !found || ch == nil {
+		ch = make(chan struct{})
+		d.changeNotify[peer] = ch
+	}
+	return ch
+}
+
 // UpdateLocalPeerState updates local peer status
 func (d *Status) UpdateLocalPeerState(localPeerState LocalPeerState) {
 	d.mux.Lock()

client/status/status_test.go

@@ -19,6 +19,21 @@ func TestAddPeer(t *testing.T) {
 	assert.Error(t, err, "should return error on duplicate")
 }
 
+func TestGetPeer(t *testing.T) {
+	key := "abc"
+	status := NewRecorder()
+	err := status.AddPeer(key)
+	assert.NoError(t, err, "shouldn't return error")
+
+	peerStatus, err := status.GetPeer(key)
+	assert.NoError(t, err, "shouldn't return error on getting peer")
+
+	assert.Equal(t, key, peerStatus.PubKey, "retrieved public key should match")
+
+	_, err = status.GetPeer("non_existing_key")
+	assert.Error(t, err, "should return error when peer doesn't exist")
+}
+
 func TestUpdatePeerState(t *testing.T) {
 	key := "abc"
 	ip := "10.10.10.10"
@@ -39,6 +54,31 @@ func TestUpdatePeerState(t *testing.T) {
 	assert.Equal(t, ip, state.IP, "ip should be equal")
 }
 
+func TestGetPeerStateChangeNotifierLogic(t *testing.T) {
+	key := "abc"
+	ip := "10.10.10.10"
+	status := NewRecorder()
+	peerState := PeerState{
+		PubKey: key,
+	}
+
+	status.peers[key] = peerState
+
+	ch := status.GetPeerStateChangeNotifier(key)
+	assert.NotNil(t, ch, "channel shouldn't be nil")
+
+	peerState.IP = ip
+
+	err := status.UpdatePeerState(peerState)
+	assert.NoError(t, err, "shouldn't return error")
+
+	select {
+	case <-ch:
+	default:
+		t.Errorf("channel wasn't closed after update")
+	}
+}
+
 func TestRemovePeer(t *testing.T) {
 	key := "abc"
 	status := NewRecorder()

client/ui/client_ui.go

@@ -9,7 +9,6 @@ import (
 	"flag"
 	"fmt"
 	"github.com/netbirdio/netbird/client/system"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -501,7 +500,7 @@ func (s *serviceClient) getSrvConfig() {
 // checkPIDFile exists and return error, or write new.
 func checkPIDFile() error {
 	pidFile := path.Join(os.TempDir(), "wiretrustee-ui.pid")
-	if piddata, err := ioutil.ReadFile(pidFile); err == nil {
+	if piddata, err := os.ReadFile(pidFile); err == nil {
 		if pid, err := strconv.Atoi(string(piddata)); err == nil {
 			if process, err := os.FindProcess(pid); err == nil {
 				if err := process.Signal(syscall.Signal(0)); err == nil {
@@ -511,5 +510,5 @@ func checkPIDFile() error {
 		}
 	}
 
-	return ioutil.WriteFile(pidFile, []byte(fmt.Sprintf("%d", os.Getpid())), 0o664)
+	return os.WriteFile(pidFile, []byte(fmt.Sprintf("%d", os.Getpid())), 0o664)
 }

go.mod

@@ -30,15 +30,22 @@ require (
 require (
 	fyne.io/fyne/v2 v2.1.4
 	github.com/c-robinson/iplib v1.0.3
+	github.com/coreos/go-iptables v0.6.0
 	github.com/creack/pty v1.1.18
 	github.com/eko/gocache/v2 v2.3.1
 	github.com/getlantern/systray v1.2.1
 	github.com/gliderlabs/ssh v0.3.4
+	github.com/google/nftables v0.0.0-20220808154552-2eca00135732
+	github.com/libp2p/go-netroute v0.2.0
 	github.com/magiconair/properties v1.8.5
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/pion/logging v0.2.2
+	github.com/pion/stun v0.3.5
+	github.com/pion/transport v0.13.0
 	github.com/rs/xid v1.3.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/stretchr/testify v1.7.1
+	go.uber.org/zap v1.17.0
 	golang.org/x/net v0.0.0-20220513224357-95641704303c
 	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
 )
@@ -67,6 +74,7 @@ require (
 	github.com/godbus/dbus/v5 v5.0.4 // indirect
 	github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
@@ -77,11 +85,8 @@ require (
 	github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
 	github.com/pegasus-kv/thrift v0.13.0 // indirect
 	github.com/pion/dtls/v2 v2.1.2 // indirect
-	github.com/pion/logging v0.2.2 // indirect
 	github.com/pion/mdns v0.0.5 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
-	github.com/pion/stun v0.3.5 // indirect
-	github.com/pion/transport v0.13.0 // indirect
 	github.com/pion/turn/v2 v2.0.7 // indirect
 	github.com/pion/udp v0.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -95,6 +100,8 @@ require (
 	github.com/srwiley/rasterx v0.0.0-20200120212402-85cb7272f5e9 // indirect
 	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
 	github.com/yuin/goldmark v1.4.1 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/image v0.0.0-20200430140353-33d19683fad8 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
@@ -114,3 +121,5 @@ require (
 )
 
 replace github.com/pion/ice/v2 => github.com/wiretrustee/ice/v2 v2.1.21-0.20220218121004-dc81faead4bb
+
+replace github.com/kardianos/service => github.com/netbirdio/service v0.0.0-20220905002524-6ac14ad5ea84

go.sum

@@ -115,6 +115,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/coocood/freecache v1.2.1 h1:/v1CqMq45NFH9mp/Pt142reundeBM0dVUD3osQBeu/U=
+github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
+github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -283,10 +285,14 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
+github.com/google/nftables v0.0.0-20220808154552-2eca00135732 h1:csc7dT82JiSLvq4aMyQMIQDL7986NH6Wxf/QrvOj55A=
+github.com/google/nftables v0.0.0-20220808154552-2eca00135732/go.mod h1:b97ulCCFipUC+kSin+zygkvUVpx0vyIAwxXFdY3PlNc=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -383,8 +389,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kardianos/service v1.2.1-0.20210728001519-a323c3813bc7 h1:oohm9Rk9JAxxmp2NLZa7Kebgz9h4+AJDcc64txg3dQ0=
-github.com/kardianos/service v1.2.1-0.20210728001519-a323c3813bc7/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -401,6 +405,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
+github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4nWRE=
+github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI=
 github.com/lucor/goinfo v0.0.0-20210802170112-c078a2b0f08b/go.mod h1:PRq09yoB+Q2OJReAmwzKivcYyremnibWGbK7WfftHzc=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
 github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
@@ -466,6 +472,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/netbirdio/service v0.0.0-20220905002524-6ac14ad5ea84 h1:u8kpzR9ld1uAeH/BAXsS0SfcnhooLWeO7UgHSBVPD9I=
+github.com/netbirdio/service v0.0.0-20220905002524-6ac14ad5ea84/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -638,8 +646,11 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -748,6 +759,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -870,6 +882,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

iface/bind.go

@@ -0,0 +1,185 @@
+package iface
+
+import (
+	"errors"
+	"fmt"
+	"github.com/pion/stun"
+	log "github.com/sirupsen/logrus"
+	"golang.zx2c4.com/wireguard/conn"
+	"net"
+	"net/netip"
+	"sync"
+	"syscall"
+)
+
+type ICEBind struct {
+	sharedConn net.PacketConn
+	udpMux     *UniversalUDPMuxDefault
+	iceHostMux *UDPMuxDefault
+
+	mu sync.Mutex // protects following fields
+}
+
+func (b *ICEBind) GetICEMux() (UniversalUDPMux, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.udpMux == nil {
+		return nil, fmt.Errorf("ICEBind has not been initialized yet")
+	}
+
+	return b.udpMux, nil
+}
+
+func (b *ICEBind) GetICEHostMux() (UDPMux, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.iceHostMux == nil {
+		return nil, fmt.Errorf("ICEBind has not been initialized yet")
+	}
+
+	return b.iceHostMux, nil
+}
+
+func (b *ICEBind) Open(uport uint16) ([]conn.ReceiveFunc, uint16, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	if b.sharedConn != nil {
+		return nil, 0, conn.ErrBindAlreadyOpen
+	}
+
+	port := int(uport)
+	ipv4Conn, port, err := listenNet("udp4", port)
+	if err != nil && !errors.Is(err, syscall.EAFNOSUPPORT) {
+		return nil, 0, err
+	}
+	b.sharedConn = ipv4Conn
+	b.udpMux = NewUniversalUDPMuxDefault(UniversalUDPMuxParams{UDPConn: b.sharedConn})
+
+	portAddr1, err := netip.ParseAddrPort(ipv4Conn.LocalAddr().String())
+	if err != nil {
+		return nil, 0, err
+	}
+
+	log.Infof("opened ICEBind on %s", ipv4Conn.LocalAddr().String())
+
+	return []conn.ReceiveFunc{
+			b.makeReceiveIPv4(b.sharedConn),
+		},
+		portAddr1.Port(), nil
+}
+
+func listenNet(network string, port int) (*net.UDPConn, int, error) {
+	conn, err := net.ListenUDP(network, &net.UDPAddr{Port: port})
+	if err != nil {
+		return nil, 0, err
+	}
+
+	// Retrieve port.
+	laddr := conn.LocalAddr()
+	uaddr, err := net.ResolveUDPAddr(
+		laddr.Network(),
+		laddr.String(),
+	)
+	if err != nil {
+		return nil, 0, err
+	}
+	return conn, uaddr.Port, nil
+}
+
+func parseSTUNMessage(raw []byte) (*stun.Message, error) {
+	msg := &stun.Message{
+		Raw: append([]byte{}, raw...),
+	}
+	if err := msg.Decode(); err != nil {
+		return nil, err
+	}
+
+	return msg, nil
+}
+
+func (b *ICEBind) makeReceiveIPv4(c net.PacketConn) conn.ReceiveFunc {
+	return func(buff []byte) (int, conn.Endpoint, error) {
+		n, endpoint, err := c.ReadFrom(buff)
+		if err != nil {
+			return 0, nil, err
+		}
+		e, err := netip.ParseAddrPort(endpoint.String())
+		if err != nil {
+			return 0, nil, err
+		}
+		if !stun.IsMessage(buff[:20]) {
+			// WireGuard traffic
+			return n, (*conn.StdNetEndpoint)(&net.UDPAddr{
+				IP:   e.Addr().AsSlice(),
+				Port: int(e.Port()),
+				Zone: e.Addr().Zone(),
+			}), nil
+		}
+
+		msg, err := parseSTUNMessage(buff[:n])
+		if err != nil {
+			return 0, nil, err
+		}
+
+		err = b.udpMux.HandleSTUNMessage(msg, endpoint)
+		if err != nil {
+			return 0, nil, err
+		}
+		if err != nil {
+			log.Warnf("failed to handle packet")
+		}
+
+		// discard packets because they are STUN related
+		return 0, nil, nil //todo proper return
+	}
+}
+
+func (b *ICEBind) Close() error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	var err1, err2 error
+	if b.sharedConn != nil {
+		c := b.sharedConn
+		b.sharedConn = nil
+		err1 = c.Close()
+	}
+
+	if b.udpMux != nil {
+		m := b.udpMux
+		b.udpMux = nil
+		err2 = m.Close()
+	}
+
+	if err1 != nil {
+		return err1
+	}
+
+	return err2
+}
+
+// SetMark sets the mark for each packet sent through this Bind.
+// This mark is passed to the kernel as the socket option SO_MARK.
+func (b *ICEBind) SetMark(mark uint32) error {
+	return nil
+}
+
+func (b *ICEBind) Send(buff []byte, endpoint conn.Endpoint) error {
+	nend, ok := endpoint.(*conn.StdNetEndpoint)
+	if !ok {
+		return conn.ErrWrongEndpointType
+	}
+	_, err := b.sharedConn.WriteTo(buff, (*net.UDPAddr)(nend))
+	return err
+}
+
+// ParseEndpoint creates a new endpoint from a string.
+func (b *ICEBind) ParseEndpoint(s string) (ep conn.Endpoint, err error) {
+	e, err := netip.ParseAddrPort(s)
+	return (*conn.StdNetEndpoint)(&net.UDPAddr{
+		IP:   e.Addr().AsSlice(),
+		Port: int(e.Port()),
+		Zone: e.Addr().Zone(),
+	}), err
+}

iface/configuration.go

@@ -9,6 +9,16 @@ import (
 	"time"
 )
 
+// GetName returns the interface name
+func (w *WGIface) GetName() string {
+	return w.Name
+}
+
+// GetAddress returns the interface address
+func (w *WGIface) GetAddress() WGAddress {
+	return w.Address
+}
+
 // configureDevice configures the wireguard device
 func (w *WGIface) configureDevice(config wgtypes.Config) error {
 	wg, err := wgctrl.New()
@@ -45,7 +55,6 @@ func (w *WGIface) Configure(privateKey string, port int) error {
 		PrivateKey:   &key,
 		ReplacePeers: true,
 		FirewallMark: &fwmark,
-		ListenPort:   &port,
 	}
 
 	err = w.configureDevice(config)
@@ -112,6 +121,114 @@ func (w *WGIface) UpdatePeer(peerKey string, allowedIps string, keepAlive time.D
 	return nil
 }
 
+// AddAllowedIP adds a prefix to the allowed IPs list of peer
+func (w *WGIface) AddAllowedIP(peerKey string, allowedIP string) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	log.Debugf("adding allowed IP to interface %s and peer %s: allowed IP %s ", w.Name, peerKey, allowedIP)
+
+	_, ipNet, err := net.ParseCIDR(allowedIP)
+	if err != nil {
+		return err
+	}
+
+	peerKeyParsed, err := wgtypes.ParseKey(peerKey)
+	if err != nil {
+		return err
+	}
+	peer := wgtypes.PeerConfig{
+		PublicKey:         peerKeyParsed,
+		UpdateOnly:        true,
+		ReplaceAllowedIPs: false,
+		AllowedIPs:        []net.IPNet{*ipNet},
+	}
+
+	config := wgtypes.Config{
+		Peers: []wgtypes.PeerConfig{peer},
+	}
+	err = w.configureDevice(config)
+	if err != nil {
+		return fmt.Errorf("received error \"%v\" while adding allowed Ip to peer on interface %s with settings: allowed ips %s", err, w.Name, allowedIP)
+	}
+	return nil
+}
+
+// RemoveAllowedIP removes a prefix from the allowed IPs list of peer
+func (w *WGIface) RemoveAllowedIP(peerKey string, allowedIP string) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	log.Debugf("removing allowed IP from interface %s and peer %s: allowed IP %s ", w.Name, peerKey, allowedIP)
+
+	_, ipNet, err := net.ParseCIDR(allowedIP)
+	if err != nil {
+		return err
+	}
+
+	peerKeyParsed, err := wgtypes.ParseKey(peerKey)
+	if err != nil {
+		return err
+	}
+
+	existingPeer, err := getPeer(w.Name, peerKey)
+	if err != nil {
+		return err
+	}
+
+	newAllowedIPs := existingPeer.AllowedIPs
+
+	for i, existingAllowedIP := range existingPeer.AllowedIPs {
+		if existingAllowedIP.String() == ipNet.String() {
+			newAllowedIPs = append(existingPeer.AllowedIPs[:i], existingPeer.AllowedIPs[i+1:]...)
+			break
+		}
+	}
+
+	if err != nil {
+		return err
+	}
+	peer := wgtypes.PeerConfig{
+		PublicKey:         peerKeyParsed,
+		UpdateOnly:        true,
+		ReplaceAllowedIPs: true,
+		AllowedIPs:        newAllowedIPs,
+	}
+
+	config := wgtypes.Config{
+		Peers: []wgtypes.PeerConfig{peer},
+	}
+	err = w.configureDevice(config)
+	if err != nil {
+		return fmt.Errorf("received error \"%v\" while removing allowed IP from peer on interface %s with settings: allowed ips %s", err, w.Name, allowedIP)
+	}
+	return nil
+}
+
+func getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) {
+	wg, err := wgctrl.New()
+	if err != nil {
+		return wgtypes.Peer{}, err
+	}
+	defer func() {
+		err = wg.Close()
+		if err != nil {
+			log.Errorf("got error while closing wgctl: %v", err)
+		}
+	}()
+
+	wgDevice, err := wg.Device(ifaceName)
+	if err != nil {
+		return wgtypes.Peer{}, err
+	}
+	for _, peer := range wgDevice.Peers {
+		if peer.PublicKey.String() == peerPubKey {
+			return peer, nil
+		}
+	}
+	return wgtypes.Peer{}, fmt.Errorf("peer not found")
+}
+
 // RemovePeer removes a Wireguard Peer from the interface iface
 func (w *WGIface) RemovePeer(peerKey string) error {
 	w.mu.Lock()

iface/iface.go

@@ -2,6 +2,10 @@ package iface
 
 import (
 	"fmt"
+	log "github.com/sirupsen/logrus"
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun"
 	"net"
 	"os"
 	"runtime"
@@ -21,6 +25,7 @@ type WGIface struct {
 	Address   WGAddress
 	Interface NetInterface
 	mu        sync.Mutex
+	Bind      *ICEBind
 }
 
 // WGAddress Wireguard parsed address
@@ -91,3 +96,49 @@ func (w *WGIface) Close() error {
 
 	return nil
 }
+
+func (w *WGIface) CreateNew(bind conn.Bind) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	return w.createWithUserspaceNew(bind)
+}
+
+func (w *WGIface) createWithUserspaceNew(bind conn.Bind) error {
+	tunIface, err := tun.CreateTUN(w.Name, w.MTU)
+	if err != nil {
+		return err
+	}
+
+	w.Interface = tunIface
+
+	// We need to create a wireguard-go device and listen to configuration requests
+	tunDevice := device.NewDevice(tunIface, bind, device.NewLogger(device.LogLevelSilent, "[wiretrustee] "))
+	err = tunDevice.Up()
+	if err != nil {
+		return err
+	}
+	uapi, err := getUAPI(w.Name)
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		for {
+			uapiConn, uapiErr := uapi.Accept()
+			if uapiErr != nil {
+				log.Traceln("uapi Accept failed with error: ", uapiErr)
+				continue
+			}
+			go tunDevice.IpcHandle(uapiConn)
+		}
+	}()
+
+	log.Debugln("UAPI listener started")
+
+	err = w.assignAddr()
+	if err != nil {
+		return err
+	}
+	return nil
+}

iface/iface_linux.go

@@ -39,13 +39,7 @@ func (w *WGIface) Create() error {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 
-	if WireguardModExists() {
+	return w.createWithUserspace()
-		log.Info("using kernel WireGuard")
-		return w.createWithKernel()
-	} else {
-		log.Info("using userspace WireGuard")
-		return w.createWithUserspace()
-	}
 }
 
 // createWithKernel Creates a new Wireguard interface using kernel Wireguard module.

iface/iface_test.go

@@ -229,7 +229,7 @@ func Test_UpdatePeer(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	peer, err := getPeer(ifaceName, peerPubKey, t)
+	peer, err := getPeer(ifaceName, peerPubKey)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -289,7 +289,7 @@ func Test_RemovePeer(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = getPeer(ifaceName, peerPubKey, t)
+	_, err = getPeer(ifaceName, peerPubKey)
 	if err.Error() != "peer not found" {
 		t.Fatal(err)
 	}
@@ -378,7 +378,7 @@ func Test_ConnectPeers(t *testing.T) {
 			t.Fatalf("waiting for peer handshake timeout after %s", timeout.String())
 		default:
 		}
-		peer, gpErr := getPeer(peer1ifaceName, peer2Key.PublicKey().String(), t)
+		peer, gpErr := getPeer(peer1ifaceName, peer2Key.PublicKey().String())
 		if gpErr != nil {
 			t.Fatal(gpErr)
 		}
@@ -389,28 +389,3 @@ func Test_ConnectPeers(t *testing.T) {
 	}
 
 }
-
-func getPeer(ifaceName, peerPubKey string, t *testing.T) (wgtypes.Peer, error) {
-	emptyPeer := wgtypes.Peer{}
-	wg, err := wgctrl.New()
-	if err != nil {
-		return emptyPeer, err
-	}
-	defer func() {
-		err = wg.Close()
-		if err != nil {
-			t.Error(err)
-		}
-	}()
-
-	wgDevice, err := wg.Device(ifaceName)
-	if err != nil {
-		return emptyPeer, err
-	}
-	for _, peer := range wgDevice.Peers {
-		if peer.PublicKey.String() == peerPubKey {
-			return peer, nil
-		}
-	}
-	return emptyPeer, fmt.Errorf("peer not found")
-}

iface/iface_windows.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows"
+	"golang.zx2c4.com/wireguard/ipc"
 	"golang.zx2c4.com/wireguard/windows/driver"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	"net"
@@ -62,3 +63,8 @@ func (w *WGIface) UpdateAddr(newAddr string) error {
 func WireguardModExists() bool {
 	return false
 }
+
+// getUAPI returns a Listener
+func getUAPI(iface string) (net.Listener, error) {
+	return ipc.UAPIListen(iface)
+}

iface/udp_mux.go

@@ -0,0 +1,288 @@
+package iface
+
+import (
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	"io"
+	"net"
+	"strings"
+	"sync"
+
+	"github.com/pion/logging"
+	"github.com/pion/stun"
+)
+
+const receiveMTU = 8192
+
+// UDPMux allows multiple connections to go over a single UDP port
+type UDPMux interface {
+	io.Closer
+	GetConn(ufrag string) (net.PacketConn, error)
+	RemoveConnByUfrag(ufrag string)
+}
+
+// UDPMuxDefault is an implementation of the interface
+type UDPMuxDefault struct {
+	params UDPMuxParams
+
+	closedChan chan struct{}
+	closeOnce  sync.Once
+
+	// conns is a map of all udpMuxedConn indexed by ufrag|network|candidateType
+	conns map[string]*udpMuxedConn
+
+	addressMapMu sync.RWMutex
+	addressMap   map[string][]*udpMuxedConn
+
+	// buffer pool to recycle buffers for net.UDPAddr encodes/decodes
+	pool *sync.Pool
+
+	mu sync.Mutex
+}
+
+const maxAddrSize = 512
+
+// UDPMuxParams are parameters for UDPMux.
+type UDPMuxParams struct {
+	Logger  logging.LeveledLogger
+	UDPConn net.PacketConn
+}
+
+// NewUDPMuxDefault creates an implementation of UDPMux
+func NewUDPMuxDefault(params UDPMuxParams) *UDPMuxDefault {
+	if params.Logger == nil {
+		params.Logger = logging.NewDefaultLoggerFactory().NewLogger("ice")
+	}
+
+	return &UDPMuxDefault{
+		addressMap: map[string][]*udpMuxedConn{},
+		params:     params,
+		conns:      make(map[string]*udpMuxedConn),
+		closedChan: make(chan struct{}, 1),
+		pool: &sync.Pool{
+			New: func() interface{} {
+				// big enough buffer to fit both packet and address
+				return newBufferHolder(receiveMTU + maxAddrSize)
+			},
+		},
+	}
+}
+
+func (m *UDPMuxDefault) HandleSTUNMessage(msg *stun.Message, addr net.Addr) error {
+
+	remoteAddr, ok := addr.(*net.UDPAddr)
+	if !ok {
+		return fmt.Errorf("underlying PacketConn did not return a UDPAddr")
+	}
+
+	// If we have already seen this address dispatch to the appropriate destination
+	// If you are using the same socket for the Host and SRFLX candidates, it might be that there are more than one
+	// muxed connection - one for the SRFLX candidate and the other one for the HOST one.
+	// We will then forward STUN packets to each of these connections.
+	m.addressMapMu.Lock()
+	var destinationConnList []*udpMuxedConn
+	if storedConns, ok := m.addressMap[addr.String()]; ok {
+		for _, conn := range storedConns {
+			destinationConnList = append(destinationConnList, conn)
+		}
+	}
+	m.addressMapMu.Unlock()
+
+	// This block is needed to discover Peer Reflexive Candidates for which we don't know the Endpoint upfront.
+	// However, we can take a username attribute from the STUN message which contains ufrag.
+	// We can use ufrag to identify the destination conn to route packet to.
+	attr, stunAttrErr := msg.Get(stun.AttrUsername)
+	if stunAttrErr == nil {
+		ufrag := strings.Split(string(attr), ":")[0]
+
+		m.mu.Lock()
+		if destinationConn, ok := m.conns[ufrag]; ok {
+			exists := false
+			for _, conn := range destinationConnList {
+				if conn.params.Key == destinationConn.params.Key {
+					exists = true
+					break
+				}
+			}
+			if !exists {
+				destinationConnList = append(destinationConnList, destinationConn)
+			}
+		}
+		m.mu.Unlock()
+	}
+
+	// Forward STUN packets to each destination connections even thought the STUN packet might not belong there.
+	// It will be discarded by the further ICE candidate logic if so.
+	for _, conn := range destinationConnList {
+		if err := conn.writePacket(msg.Raw, remoteAddr); err != nil {
+			log.Errorf("could not write packet: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// LocalAddr returns the listening address of this UDPMuxDefault
+func (m *UDPMuxDefault) LocalAddr() net.Addr {
+	return m.params.UDPConn.LocalAddr()
+}
+
+// GetConn returns a PacketConn given the connection's ufrag and network
+// creates the connection if an existing one can't be found
+func (m *UDPMuxDefault) GetConn(ufrag string) (net.PacketConn, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	log.Debugf("ICE: getting muxed connection for %s", ufrag)
+
+	if m.IsClosed() {
+		return nil, io.ErrClosedPipe
+	}
+
+	if c, ok := m.conns[ufrag]; ok {
+		return c, nil
+	}
+
+	c := m.createMuxedConn(ufrag)
+	go func() {
+		<-c.CloseChannel()
+		m.removeConn(ufrag)
+	}()
+	m.conns[ufrag] = c
+	return c, nil
+}
+
+// RemoveConnByUfrag stops and removes the muxed packet connection
+func (m *UDPMuxDefault) RemoveConnByUfrag(ufrag string) {
+	m.mu.Lock()
+	removedConns := make([]*udpMuxedConn, 0)
+	for key := range m.conns {
+		if key != ufrag {
+			continue
+		}
+
+		c := m.conns[key]
+		delete(m.conns, key)
+		if c != nil {
+			removedConns = append(removedConns, c)
+		}
+	}
+	// keep lock section small to avoid deadlock with conn lock
+	m.mu.Unlock()
+
+	m.addressMapMu.Lock()
+	defer m.addressMapMu.Unlock()
+
+	for _, c := range removedConns {
+		addresses := c.getAddresses()
+		for _, addr := range addresses {
+			if connList, ok := m.addressMap[addr]; ok {
+				var newList []*udpMuxedConn
+				for _, conn := range connList {
+					if conn.params.Key != ufrag {
+						newList = append(newList, conn)
+					}
+				}
+				m.addressMap[addr] = newList
+			}
+		}
+	}
+}
+
+// IsClosed returns true if the mux had been closed
+func (m *UDPMuxDefault) IsClosed() bool {
+	select {
+	case <-m.closedChan:
+		return true
+	default:
+		return false
+	}
+}
+
+// Close the mux, no further connections could be created
+func (m *UDPMuxDefault) Close() error {
+	var err error
+	m.closeOnce.Do(func() {
+		m.mu.Lock()
+		defer m.mu.Unlock()
+
+		for _, c := range m.conns {
+			_ = c.Close()
+		}
+		m.conns = make(map[string]*udpMuxedConn)
+		close(m.closedChan)
+	})
+	return err
+}
+
+func (m *UDPMuxDefault) removeConn(key string) {
+	m.mu.Lock()
+	c := m.conns[key]
+	delete(m.conns, key)
+	// keep lock section small to avoid deadlock with conn lock
+	m.mu.Unlock()
+
+	if c == nil {
+		return
+	}
+
+	m.addressMapMu.Lock()
+	defer m.addressMapMu.Unlock()
+
+	addresses := c.getAddresses()
+	for _, addr := range addresses {
+		if connList, ok := m.addressMap[addr]; ok {
+			var newList []*udpMuxedConn
+			for _, conn := range connList {
+				if conn.params.Key != key {
+					newList = append(newList, conn)
+				}
+			}
+			m.addressMap[addr] = newList
+		}
+	}
+}
+
+func (m *UDPMuxDefault) writeTo(buf []byte, raddr net.Addr) (n int, err error) {
+	return m.params.UDPConn.WriteTo(buf, raddr)
+}
+
+func (m *UDPMuxDefault) registerConnForAddress(conn *udpMuxedConn, addr string) {
+	if m.IsClosed() {
+		return
+	}
+
+	m.addressMapMu.Lock()
+	defer m.addressMapMu.Unlock()
+
+	existing, ok := m.addressMap[addr]
+	if !ok {
+		existing = []*udpMuxedConn{}
+	}
+	existing = append(existing, conn)
+	m.addressMap[addr] = existing
+
+	log.Debugf("ICE: registered %s for %s", addr, conn.params.Key)
+}
+
+func (m *UDPMuxDefault) createMuxedConn(key string) *udpMuxedConn {
+	c := newUDPMuxedConn(&udpMuxedConnParams{
+		Mux:       m,
+		Key:       key,
+		AddrPool:  m.pool,
+		LocalAddr: m.LocalAddr(),
+		Logger:    m.params.Logger,
+	})
+	log.Debugf("ICE: created muxed connection %s for %s", c.LocalAddr().String(), key)
+	return c
+}
+
+type bufferHolder struct {
+	buffer []byte
+}
+
+func newBufferHolder(size int) *bufferHolder {
+	return &bufferHolder{
+		buffer: make([]byte, size),
+	}
+}

iface/udp_mux_universal.go

@@ -0,0 +1,235 @@
+package iface
+
+import (
+	"errors"
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	"net"
+	"time"
+
+	"github.com/pion/logging"
+	"github.com/pion/stun"
+)
+
+// UniversalUDPMux allows multiple connections to go over a single UDP port for
+// host, server reflexive and relayed candidates.
+// Actual connection muxing is happening in the UDPMux.
+type UniversalUDPMux interface {
+	UDPMux
+	GetXORMappedAddr(stunAddr net.Addr, deadline time.Duration) (*stun.XORMappedAddress, error)
+	GetRelayedAddr(turnAddr net.Addr, deadline time.Duration) (*net.Addr, error)
+	GetConnForURL(ufrag string, url string) (net.PacketConn, error)
+}
+
+// UniversalUDPMuxDefault handles STUN and TURN servers packets by wrapping the original UDPConn overriding ReadFrom.
+// It the passes packets to the UDPMux that does the actual connection muxing.
+type UniversalUDPMuxDefault struct {
+	*UDPMuxDefault
+	params UniversalUDPMuxParams
+
+	// since we have a shared socket, for srflx candidates it makes sense to have a shared mapped address across all the agents
+	// stun.XORMappedAddress indexed by the STUN server addr
+	xorMappedMap map[string]*xorMapped
+}
+
+// UniversalUDPMuxParams are parameters for UniversalUDPMux server reflexive.
+type UniversalUDPMuxParams struct {
+	Logger                logging.LeveledLogger
+	UDPConn               net.PacketConn
+	XORMappedAddrCacheTTL time.Duration
+}
+
+// NewUniversalUDPMuxDefault creates an implementation of UniversalUDPMux embedding UDPMux
+func NewUniversalUDPMuxDefault(params UniversalUDPMuxParams) *UniversalUDPMuxDefault {
+	if params.Logger == nil {
+		params.Logger = logging.NewDefaultLoggerFactory().NewLogger("ice")
+	}
+	if params.XORMappedAddrCacheTTL == 0 {
+		params.XORMappedAddrCacheTTL = time.Second * 25
+	}
+
+	m := &UniversalUDPMuxDefault{
+		params:       params,
+		xorMappedMap: make(map[string]*xorMapped),
+	}
+
+	// embed UDPMux
+	udpMuxParams := UDPMuxParams{
+		Logger:  params.Logger,
+		UDPConn: m.params.UDPConn,
+	}
+	m.UDPMuxDefault = NewUDPMuxDefault(udpMuxParams)
+
+	return m
+}
+
+// GetRelayedAddr creates relayed connection to the given TURN service and returns the relayed addr.
+// Not implemented yet.
+func (m *UniversalUDPMuxDefault) GetRelayedAddr(turnAddr net.Addr, deadline time.Duration) (*net.Addr, error) {
+	return nil, errors.New("not implemented yet")
+}
+
+// GetConnForURL add uniques to the muxed connection by concatenating ufrag and URL (e.g. STUN URL) to be able to support multiple STUN/TURN servers
+// and return a unique connection per server.
+func (m *UniversalUDPMuxDefault) GetConnForURL(ufrag string, url string) (net.PacketConn, error) {
+	return m.UDPMuxDefault.GetConn(fmt.Sprintf("%s%s", ufrag, url))
+}
+
+func (m *UniversalUDPMuxDefault) HandleSTUNMessage(msg *stun.Message, addr net.Addr) error {
+
+	udpAddr, ok := addr.(*net.UDPAddr)
+	if !ok {
+		// message about this err will be logged in the UDPMux
+		return nil
+	}
+
+	if m.isXORMappedResponse(msg, udpAddr.String()) {
+		err := m.handleXORMappedResponse(udpAddr, msg)
+		if err != nil {
+			log.Debugf("%w: %v", errors.New("failed to get XOR-MAPPED-ADDRESS response"), err)
+			return nil
+		}
+		return nil
+	}
+	return m.UDPMuxDefault.HandleSTUNMessage(msg, addr)
+}
+
+// isXORMappedResponse indicates whether the message is a XORMappedAddress and is coming from the known STUN server.
+func (m *UniversalUDPMuxDefault) isXORMappedResponse(msg *stun.Message, stunAddr string) bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	// check first if it is a STUN server address because remote peer can also send similar messages but as a BindingSuccess
+	_, ok := m.xorMappedMap[stunAddr]
+	_, err := msg.Get(stun.AttrXORMappedAddress)
+	return err == nil && ok
+}
+
+// handleXORMappedResponse parses response from the STUN server, extracts XORMappedAddress attribute
+// and set the mapped address for the server
+func (m *UniversalUDPMuxDefault) handleXORMappedResponse(stunAddr *net.UDPAddr, msg *stun.Message) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	mappedAddr, ok := m.xorMappedMap[stunAddr.String()]
+	if !ok {
+		return errors.New("no address mapping")
+	}
+
+	var addr stun.XORMappedAddress
+	if err := addr.GetFrom(msg); err != nil {
+		return err
+	}
+
+	m.xorMappedMap[stunAddr.String()] = mappedAddr
+	mappedAddr.SetAddr(&addr)
+
+	return nil
+}
+
+// GetXORMappedAddr returns *stun.XORMappedAddress if already present for a given STUN server.
+// Makes a STUN binding request to discover mapped address otherwise.
+// Blocks until the stun.XORMappedAddress has been discovered or deadline.
+// Method is safe for concurrent use.
+func (m *UniversalUDPMuxDefault) GetXORMappedAddr(serverAddr net.Addr, deadline time.Duration) (*stun.XORMappedAddress, error) {
+	m.mu.Lock()
+	mappedAddr, ok := m.xorMappedMap[serverAddr.String()]
+	// if we already have a mapping for this STUN server (address already received)
+	// and if it is not too old we return it without making a new request to STUN server
+	if ok {
+		if mappedAddr.expired() {
+			mappedAddr.closeWaiters()
+			delete(m.xorMappedMap, serverAddr.String())
+			ok = false
+		} else if mappedAddr.pending() {
+			ok = false
+		}
+	}
+	m.mu.Unlock()
+	if ok {
+		return mappedAddr.addr, nil
+	}
+
+	// otherwise, make a STUN request to discover the address
+	// or wait for already sent request to complete
+	waitAddrReceived, err := m.sendStun(serverAddr)
+	if err != nil {
+		return nil, errors.New("failed to send STUN packet")
+	}
+
+	// block until response was handled by the connWorker routine and XORMappedAddress was updated
+	select {
+	case <-waitAddrReceived:
+		// when channel closed, addr was obtained
+		m.mu.Lock()
+		mappedAddr := *m.xorMappedMap[serverAddr.String()]
+		m.mu.Unlock()
+		if mappedAddr.addr == nil {
+			return nil, errors.New("no address mapping")
+		}
+		return mappedAddr.addr, nil
+	case <-time.After(deadline):
+		return nil, errors.New("timeout while waiting for XORMappedAddr")
+	}
+}
+
+// sendStun sends a STUN request via UDP conn.
+//
+// The returned channel is closed when the STUN response has been received.
+// Method is safe for concurrent use.
+func (m *UniversalUDPMuxDefault) sendStun(serverAddr net.Addr) (chan struct{}, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	// if record present in the map, we already sent a STUN request,
+	// just wait when waitAddrReceived will be closed
+	addrMap, ok := m.xorMappedMap[serverAddr.String()]
+	if !ok {
+		addrMap = &xorMapped{
+			expiresAt:        time.Now().Add(m.params.XORMappedAddrCacheTTL),
+			waitAddrReceived: make(chan struct{}),
+		}
+		m.xorMappedMap[serverAddr.String()] = addrMap
+	}
+
+	req, err := stun.Build(stun.BindingRequest, stun.TransactionID)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = m.params.UDPConn.WriteTo(req.Raw, serverAddr); err != nil {
+		return nil, err
+	}
+
+	return addrMap.waitAddrReceived, nil
+}
+
+type xorMapped struct {
+	addr             *stun.XORMappedAddress
+	waitAddrReceived chan struct{}
+	expiresAt        time.Time
+}
+
+func (a *xorMapped) closeWaiters() {
+	select {
+	case <-a.waitAddrReceived:
+		// notify was close, ok, that means we received duplicate response
+		// just exit
+		break
+	default:
+		// notify tha twe have a new addr
+		close(a.waitAddrReceived)
+	}
+}
+
+func (a *xorMapped) pending() bool {
+	return a.addr == nil
+}
+
+func (a *xorMapped) expired() bool {
+	return a.expiresAt.Before(time.Now())
+}
+
+func (a *xorMapped) SetAddr(addr *stun.XORMappedAddress) {
+	a.addr = addr
+	a.closeWaiters()
+}

iface/udp_muxed_conn.go

@@ -0,0 +1,246 @@
+package iface
+
+import (
+	"encoding/binary"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/pion/logging"
+	"github.com/pion/transport/packetio"
+)
+
+type udpMuxedConnParams struct {
+	Mux       *UDPMuxDefault
+	AddrPool  *sync.Pool
+	Key       string
+	LocalAddr net.Addr
+	Logger    logging.LeveledLogger
+}
+
+// udpMuxedConn represents a logical packet conn for a single remote as identified by ufrag
+type udpMuxedConn struct {
+	params *udpMuxedConnParams
+	// remote addresses that we have sent to on this conn
+	addresses []string
+
+	// channel holding incoming packets
+	buffer     *packetio.Buffer
+	closedChan chan struct{}
+	closeOnce  sync.Once
+	mu         sync.Mutex
+}
+
+func newUDPMuxedConn(params *udpMuxedConnParams) *udpMuxedConn {
+	p := &udpMuxedConn{
+		params:     params,
+		buffer:     packetio.NewBuffer(),
+		closedChan: make(chan struct{}),
+	}
+
+	return p
+}
+
+func (c *udpMuxedConn) ReadFrom(b []byte) (n int, raddr net.Addr, err error) {
+	buf := c.params.AddrPool.Get().(*bufferHolder)
+	defer c.params.AddrPool.Put(buf)
+
+	// read address
+	total, err := c.buffer.Read(buf.buffer)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	dataLen := int(binary.LittleEndian.Uint16(buf.buffer[:2]))
+	if dataLen > total || dataLen > len(b) {
+		return 0, nil, io.ErrShortBuffer
+	}
+
+	// read data and then address
+	offset := 2
+	copy(b, buf.buffer[offset:offset+dataLen])
+	offset += dataLen
+
+	// read address len & decode address
+	addrLen := int(binary.LittleEndian.Uint16(buf.buffer[offset : offset+2]))
+	offset += 2
+
+	if raddr, err = decodeUDPAddr(buf.buffer[offset : offset+addrLen]); err != nil {
+		return 0, nil, err
+	}
+
+	return dataLen, raddr, nil
+}
+
+func (c *udpMuxedConn) WriteTo(buf []byte, raddr net.Addr) (n int, err error) {
+	if c.isClosed() {
+		return 0, io.ErrClosedPipe
+	}
+	// each time we write to a new address, we'll register it with the mux
+	addr := raddr.String()
+	if !c.containsAddress(addr) {
+		c.addAddress(addr)
+	}
+
+	return c.params.Mux.writeTo(buf, raddr)
+}
+
+func (c *udpMuxedConn) LocalAddr() net.Addr {
+	return c.params.LocalAddr
+}
+
+func (c *udpMuxedConn) SetDeadline(tm time.Time) error {
+	return nil
+}
+
+func (c *udpMuxedConn) SetReadDeadline(tm time.Time) error {
+	return nil
+}
+
+func (c *udpMuxedConn) SetWriteDeadline(tm time.Time) error {
+	return nil
+}
+
+func (c *udpMuxedConn) CloseChannel() <-chan struct{} {
+	return c.closedChan
+}
+
+func (c *udpMuxedConn) Close() error {
+	var err error
+	c.closeOnce.Do(func() {
+		err = c.buffer.Close()
+		close(c.closedChan)
+	})
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.addresses = nil
+	return err
+}
+
+func (c *udpMuxedConn) isClosed() bool {
+	select {
+	case <-c.closedChan:
+		return true
+	default:
+		return false
+	}
+}
+
+func (c *udpMuxedConn) getAddresses() []string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	addresses := make([]string, len(c.addresses))
+	copy(addresses, c.addresses)
+	return addresses
+}
+
+func (c *udpMuxedConn) addAddress(addr string) {
+	c.mu.Lock()
+	c.addresses = append(c.addresses, addr)
+	c.mu.Unlock()
+
+	// map it on mux
+	c.params.Mux.registerConnForAddress(c, addr)
+}
+
+func (c *udpMuxedConn) removeAddress(addr string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	newAddresses := make([]string, 0, len(c.addresses))
+	for _, a := range c.addresses {
+		if a != addr {
+			newAddresses = append(newAddresses, a)
+		}
+	}
+
+	c.addresses = newAddresses
+}
+
+func (c *udpMuxedConn) containsAddress(addr string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for _, a := range c.addresses {
+		if addr == a {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *udpMuxedConn) writePacket(data []byte, addr *net.UDPAddr) error {
+	// write two packets, address and data
+	buf := c.params.AddrPool.Get().(*bufferHolder)
+	defer c.params.AddrPool.Put(buf)
+
+	// format of buffer | data len | data bytes | addr len | addr bytes |
+	if len(buf.buffer) < len(data)+maxAddrSize {
+		return io.ErrShortBuffer
+	}
+	// data len
+	binary.LittleEndian.PutUint16(buf.buffer, uint16(len(data)))
+	offset := 2
+
+	// data
+	copy(buf.buffer[offset:], data)
+	offset += len(data)
+
+	// write address first, leaving room for its length
+	n, err := encodeUDPAddr(addr, buf.buffer[offset+2:])
+	if err != nil {
+		return nil
+	}
+	total := offset + n + 2
+
+	// address len
+	binary.LittleEndian.PutUint16(buf.buffer[offset:], uint16(n))
+
+	if _, err := c.buffer.Write(buf.buffer[:total]); err != nil {
+		return err
+	}
+	return nil
+}
+
+func encodeUDPAddr(addr *net.UDPAddr, buf []byte) (int, error) {
+	ipdata, err := addr.IP.MarshalText()
+	if err != nil {
+		return 0, err
+	}
+	total := 2 + len(ipdata) + 2 + len(addr.Zone)
+	if total > len(buf) {
+		return 0, io.ErrShortBuffer
+	}
+
+	binary.LittleEndian.PutUint16(buf, uint16(len(ipdata)))
+	offset := 2
+	n := copy(buf[offset:], ipdata)
+	offset += n
+	binary.LittleEndian.PutUint16(buf[offset:], uint16(addr.Port))
+	offset += 2
+	copy(buf[offset:], addr.Zone)
+	return total, nil
+}
+
+func decodeUDPAddr(buf []byte) (*net.UDPAddr, error) {
+	addr := net.UDPAddr{}
+
+	offset := 0
+	ipLen := int(binary.LittleEndian.Uint16(buf[:2]))
+	offset += 2
+	// basic bounds checking
+	if ipLen+offset > len(buf) {
+		return nil, io.ErrShortBuffer
+	}
+	if err := addr.IP.UnmarshalText(buf[offset : offset+ipLen]); err != nil {
+		return nil, err
+	}
+	offset += ipLen
+	addr.Port = int(binary.LittleEndian.Uint16(buf[offset : offset+2]))
+	offset += 2
+	zone := make([]byte, len(buf[offset:]))
+	copy(zone, buf[offset:])
+	addr.Zone = string(zone)
+
+	return &addr, nil
+}

infrastructure_files/base.setup.env

@@ -27,16 +27,24 @@ MGMT_VOLUMESUFFIX="mgmt"
 SIGNAL_VOLUMESUFFIX="signal"
 LETSENCRYPT_VOLUMESUFFIX="letsencrypt"
 
+NETBIRD_AUTH_DEVICE_AUTH_PROVIDER="none"
+
 # exports
 export NETBIRD_DOMAIN
-export NETBIRD_AUTH0_DOMAIN
+export NETBIRD_AUTH_CLIENT_ID
-export NETBIRD_AUTH0_CLIENT_ID
+export NETBIRD_AUTH_AUDIENCE
-export NETBIRD_AUTH0_AUDIENCE
+export NETBIRD_AUTH_AUTHORITY
+export NETBIRD_USE_AUTH0
+export NETBIRD_AUTH_SUPPORTED_SCOPES
+export NETBIRD_AUTH_JWT_CERTS
 export NETBIRD_LETSENCRYPT_EMAIL
 export NETBIRD_MGMT_API_PORT
 export NETBIRD_MGMT_API_ENDPOINT
 export NETBIRD_MGMT_API_CERT_FILE
 export NETBIRD_MGMT_API_CERT_KEY_FILE
+export NETBIRD_AUTH_DEVICE_AUTH_PROVIDER
+export NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID
+export NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT
 export TURN_USER
 export TURN_PASSWORD
 export TURN_MIN_PORT

infrastructure_files/configure.sh

@@ -1,5 +1,21 @@
 #!/bin/bash
 
+if ! which curl > /dev/null 2>&1
+then
+    echo "This script uses curl fetch OpenID configuration from IDP."
+    echo "Please install curl and re-run the script https://curl.se/"
+    echo ""
+    exit 1
+fi
+
+if ! which jq > /dev/null 2>&1
+then
+    echo "This script uses jq to load OpenID configuration from IDP."
+    echo "Please install jq and re-run the script https://stedolan.github.io/jq/"
+    echo ""
+    exit 1
+fi
+
 source setup.env
 source base.setup.env
 
@@ -63,6 +79,49 @@ export MGMT_VOLUMENAME
 export SIGNAL_VOLUMENAME
 export LETSENCRYPT_VOLUMENAME
 
+#backwards compatibility after migrating to generic OIDC with Auth0
+if [[ -z "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" ]]; then
+
+    if [[ -z "${NETBIRD_AUTH0_DOMAIN}" ]]; then
+       # not a backward compatible state
+       echo "NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT property must be set in the setup.env file"
+       exit 1
+    fi
+
+    echo "It seems like you provided an old setup.env file."
+    echo "Since the release of v0.8.10, we introduced a new set of properties."
+    echo "The script is backward compatible and will continue automatically."
+    echo "In the future versions it will be deprecated. Please refer to the documentation to learn about the changes http://netbird.io/docs/getting-started/self-hosting"
+
+    export NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT="https://${NETBIRD_AUTH0_DOMAIN}/.well-known/openid-configuration"
+    export NETBIRD_USE_AUTH0="true"
+    export NETBIRD_AUTH_AUDIENCE=${NETBIRD_AUTH0_AUDIENCE}
+    export NETBIRD_AUTH_CLIENT_ID=${NETBIRD_AUTH0_CLIENT_ID}
+fi
+
+echo "loading OpenID configuration from ${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT} to the openid-configuration.json file"
+curl "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" -q -o openid-configuration.json
+
+export NETBIRD_AUTH_AUTHORITY=$( jq -r  '.issuer' openid-configuration.json )
+export NETBIRD_AUTH_JWT_CERTS=$( jq -r  '.jwks_uri' openid-configuration.json )
+export NETBIRD_AUTH_SUPPORTED_SCOPES=$( jq -r '.scopes_supported | join(" ")' openid-configuration.json )
+export NETBIRD_AUTH_TOKEN_ENDPOINT=$( jq -r  '.token_endpoint' openid-configuration.json )
+export NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT=$( jq -r  '.device_authorization_endpoint' openid-configuration.json )
+
+if [ $NETBIRD_USE_AUTH0 == "true" ]
+then
+    export NETBIRD_AUTH_SUPPORTED_SCOPES="openid profile email offline_access api email_verified"
+else
+    export NETBIRD_AUTH_SUPPORTED_SCOPES="openid profile email offline_access api"
+fi
+
+if [[ ! -z "${NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID}" ]]; then
+    # user enabled Device Authorization Grant feature
+    export NETBIRD_AUTH_DEVICE_AUTH_PROVIDER="hosted"
+fi
+
+env | grep NETBIRD
+
 envsubst < docker-compose.yml.tmpl > docker-compose.yml
 envsubst < management.json.tmpl > management.json
 envsubst < turnserver.conf.tmpl > turnserver.conf

infrastructure_files/docker-compose.yml.tmpl

@@ -8,9 +8,11 @@ services:
       - 80:80
       - 443:443
     environment:
-      - AUTH0_DOMAIN=$NETBIRD_AUTH0_DOMAIN
+      - AUTH_AUDIENCE=$NETBIRD_AUTH_AUDIENCE
-      - AUTH0_CLIENT_ID=$NETBIRD_AUTH0_CLIENT_ID
+      - AUTH_CLIENT_ID=$NETBIRD_AUTH_CLIENT_ID
-      - AUTH0_AUDIENCE=$NETBIRD_AUTH0_AUDIENCE
+      - AUTH_AUTHORITY=$NETBIRD_AUTH_AUTHORITY
+      - USE_AUTH0=$NETBIRD_USE_AUTH0
+      - AUTH_SUPPORTED_SCOPES=$NETBIRD_AUTH_SUPPORTED_SCOPES
       - NETBIRD_MGMT_API_ENDPOINT=$NETBIRD_MGMT_API_ENDPOINT
       - NETBIRD_MGMT_GRPC_API_ENDPOINT=$NETBIRD_MGMT_API_ENDPOINT
       - NGINX_SSL_PORT=443

infrastructure_files/management.json.tmpl

@@ -29,13 +29,24 @@
     "Datadir": "",
     "HttpConfig": {
         "Address": "0.0.0.0:$NETBIRD_MGMT_API_PORT",
-        "AuthIssuer": "https://$NETBIRD_AUTH0_DOMAIN/",
+        "AuthIssuer": "$NETBIRD_AUTH_AUTHORITY",
-        "AuthAudience": "$NETBIRD_AUTH0_AUDIENCE",
+        "AuthAudience": "$NETBIRD_AUTH_AUDIENCE",
-        "AuthKeysLocation": "https://$NETBIRD_AUTH0_DOMAIN/.well-known/jwks.json",
+        "AuthKeysLocation": "$NETBIRD_AUTH_JWT_CERTS",
         "CertFile":"$NETBIRD_MGMT_API_CERT_FILE",
-        "CertKey":"$NETBIRD_MGMT_API_CERT_KEY_FILE"
+        "CertKey":"$NETBIRD_MGMT_API_CERT_KEY_FILE",
+        "OIDCConfigEndpoint":"$NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT"
     },
     "IdpManagerConfig": {
         "Manager": "none"
-     }
+     },
+    "DeviceAuthorizationFlow": {
+        "Provider": "$NETBIRD_AUTH_DEVICE_AUTH_PROVIDER",
+        "ProviderConfig": {
+          "Audience": "$NETBIRD_AUTH_AUDIENCE",
+          "Domain": "$NETBIRD_AUTH0_DOMAIN",
+          "ClientID": "$NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID",
+          "TokenEndpoint": "$NETBIRD_AUTH_TOKEN_ENDPOINT",
+          "DeviceAuthEndpoint": "$NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT"
+         }
+    }
 }

infrastructure_files/setup.env.example

@@ -1,16 +1,15 @@
 ## example file, you can copy this file to setup.env and update its values
 ##
-# Dashboard domain and auth0 configuration
-
 # Dashboard domain. e.g. app.mydomain.com
 NETBIRD_DOMAIN=""
-# e.g. dev-24vkclam.us.auth0.com
+# OIDC configuration e.g., https://example.eu.auth0.com/.well-known/openid-configuration
-NETBIRD_AUTH0_DOMAIN=""
+NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT=""
-# e.g. 61u3JMXRO0oOevc7gCkZLCwePQvT4lL0
+NETBIRD_AUTH_AUDIENCE=""
-NETBIRD_AUTH0_CLIENT_ID=""
+# e.g. netbird-client
-# e.g. https://app.mydomain.com/ or https://app.mydomain.com,
+NETBIRD_AUTH_CLIENT_ID=""
-# Make sure you used the exact same value for Identifier
+# indicates whether to use Auth0 or not: true or false
-# you used when creating your Auth0 API
+NETBIRD_USE_AUTH0="false"
-NETBIRD_AUTH0_AUDIENCE=""
+NETBIRD_AUTH_DEVICE_AUTH_PROVIDER="none"
+NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID=""
 # e.g. hello@mydomain.com
 NETBIRD_LETSENCRYPT_EMAIL=""

infrastructure_files/tests/setup.env

@@ -1,16 +1,13 @@
 ## example file, you can copy this file to setup.env and update its values
 ##
-# Dashboard domain and auth0 configuration
-
 # Dashboard domain. e.g. app.mydomain.com
 NETBIRD_DOMAIN="localhost"
-# e.g. dev-24vkclam.us.auth0.com
+# e.g. https://dev-24vkclam.us.auth0.com/ or https://YOUR-KEYCLOAK-HOST:8080/realms/netbird
-NETBIRD_AUTH0_DOMAIN=$CI_NETBIRD_AUTH0_DOMAIN
+NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT="https://example.eu.auth0.com/.well-known/openid-configuration"
-# e.g. 61u3JMXRO0oOevc7gCkZLCwePQvT4lL0
+# e.g. netbird-client
-NETBIRD_AUTH0_CLIENT_ID=$CI_NETBIRD_AUTH0_CLIENT_ID
+NETBIRD_AUTH_CLIENT_ID=$CI_NETBIRD_AUTH_CLIENT_ID
-# e.g. https://app.mydomain.com/ or https://app.mydomain.com,
+# indicates whether to use Auth0 or not: true or false
-# Make sure you used the exact same value for Identifier
+NETBIRD_USE_AUTH0=$CI_NETBIRD_USE_AUTH0
-# you used when creating your Auth0 API
+NETBIRD_AUTH_AUDIENCE=$CI_NETBIRD_AUTH_AUDIENCE
-NETBIRD_AUTH0_AUDIENCE=$CI_NETBIRD_AUTH0_AUDIENCE
 # e.g. hello@mydomain.com
 NETBIRD_LETSENCRYPT_EMAIL=""

management/client/grpc.go

@@ -109,7 +109,7 @@ func (c *GrpcClient) Sync(msgHandler func(msg *proto.SyncResponse) error) error
 			return err
 		}
 
-		stream, err := c.connectToStream(*serverPubKey)
+		cancel, stream, err := c.connectToStream(*serverPubKey)
 		if err != nil {
 			log.Debugf("failed to open Management Service stream: %s", err)
 			if s, ok := gstatus.FromError(err); ok && s.Code() == codes.PermissionDenied {
@@ -117,6 +117,7 @@ func (c *GrpcClient) Sync(msgHandler func(msg *proto.SyncResponse) error) error
 			}
 			return err
 		}
+		defer cancel()
 
 		log.Infof("connected to the Management Service stream")
 
@@ -145,7 +146,7 @@ func (c *GrpcClient) Sync(msgHandler func(msg *proto.SyncResponse) error) error
 	return nil
 }
 
-func (c *GrpcClient) connectToStream(serverPubKey wgtypes.Key) (proto.ManagementService_SyncClient, error) {
+func (c *GrpcClient) connectToStream(serverPubKey wgtypes.Key) (context.CancelFunc, proto.ManagementService_SyncClient, error) {
 	req := &proto.SyncRequest{}
 
 	myPrivateKey := c.key
@@ -154,11 +155,16 @@ func (c *GrpcClient) connectToStream(serverPubKey wgtypes.Key) (proto.Management
 	encryptedReq, err := encryption.EncryptMessage(serverPubKey, myPrivateKey, req)
 	if err != nil {
 		log.Errorf("failed encrypting message: %s", err)
-		return nil, err
+		return nil, nil, err
 	}
-
+	ctx, cancel := context.WithCancel(c.ctx)
 	syncReq := &proto.EncryptedMessage{WgPubKey: myPublicKey.String(), Body: encryptedReq}
-	return c.realClient.Sync(c.ctx, syncReq)
+	sync, err := c.realClient.Sync(ctx, syncReq)
+	if err != nil {
+		cancel()
+		return nil, nil, err
+	}
+	return cancel, sync, nil
 }
 
 func (c *GrpcClient) receiveEvents(stream proto.ManagementService_SyncClient, serverPubKey wgtypes.Key, msgHandler func(msg *proto.SyncResponse) error) error {

management/cmd/management.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -11,9 +12,9 @@ import (
 	"golang.org/x/net/http2/h2c"
 	"io"
 	"io/fs"
-	"io/ioutil"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"strings"
@@ -305,9 +306,88 @@ func loadMgmtConfig(mgmtConfigPath string) (*server.Config, error) {
 		config.HttpConfig.CertKey = certKey
 	}
 
+	oidcEndpoint := config.HttpConfig.OIDCConfigEndpoint
+	if oidcEndpoint != "" {
+		// if OIDCConfigEndpoint is specified, we can load DeviceAuthEndpoint and TokenEndpoint automatically
+		log.Infof("loading OIDC configuration from the provided IDP configuration endpoint %s", oidcEndpoint)
+		oidcConfig, err := fetchOIDCConfig(oidcEndpoint)
+		if err != nil {
+			return nil, err
+		}
+		log.Infof("loaded OIDC configuration from the provided IDP configuration endpoint: %s", oidcEndpoint)
+
+		log.Infof("overriding HttpConfig.AuthIssuer with a new value %s, previously configured value: %s",
+			oidcConfig.Issuer, config.HttpConfig.AuthIssuer)
+		config.HttpConfig.AuthIssuer = oidcConfig.Issuer
+
+		log.Infof("overriding HttpConfig.AuthKeysLocation (JWT certs) with a new value %s, previously configured value: %s",
+			oidcConfig.JwksURI, config.HttpConfig.AuthKeysLocation)
+		config.HttpConfig.AuthKeysLocation = oidcConfig.JwksURI
+
+		if !(config.DeviceAuthorizationFlow == nil || strings.ToLower(config.DeviceAuthorizationFlow.Provider) == string(server.NONE)) {
+			log.Infof("overriding DeviceAuthorizationFlow.TokenEndpoint with a new value: %s, previously configured value: %s",
+				oidcConfig.TokenEndpoint, config.DeviceAuthorizationFlow.ProviderConfig.TokenEndpoint)
+			config.DeviceAuthorizationFlow.ProviderConfig.TokenEndpoint = oidcConfig.TokenEndpoint
+			log.Infof("overriding DeviceAuthorizationFlow.DeviceAuthEndpoint with a new value: %s, previously configured value: %s",
+				oidcConfig.DeviceAuthEndpoint, config.DeviceAuthorizationFlow.ProviderConfig.DeviceAuthEndpoint)
+			config.DeviceAuthorizationFlow.ProviderConfig.DeviceAuthEndpoint = oidcConfig.DeviceAuthEndpoint
+
+			u, err := url.Parse(oidcEndpoint)
+			if err != nil {
+				return nil, err
+			}
+			log.Infof("overriding DeviceAuthorizationFlow.ProviderConfig.Domain with a new value: %s, previously configured value: %s",
+				u.Host, config.DeviceAuthorizationFlow.ProviderConfig.Domain)
+			config.DeviceAuthorizationFlow.ProviderConfig.Domain = u.Host
+		}
+	}
+
 	return config, err
 }
 
+// OIDCConfigResponse used for parsing OIDC config response
+type OIDCConfigResponse struct {
+	Issuer             string `json:"issuer"`
+	TokenEndpoint      string `json:"token_endpoint"`
+	DeviceAuthEndpoint string `json:"device_authorization_endpoint"`
+	JwksURI            string `json:"jwks_uri"`
+}
+
+// fetchOIDCConfig fetches OIDC configuration from the IDP
+func fetchOIDCConfig(oidcEndpoint string) (OIDCConfigResponse, error) {
+
+	res, err := http.Get(oidcEndpoint)
+	if err != nil {
+		return OIDCConfigResponse{}, fmt.Errorf("failed fetching OIDC configuration fro mendpoint %s %v", oidcEndpoint, err)
+	}
+
+	defer func() {
+		err := res.Body.Close()
+		if err != nil {
+			log.Debugf("failed closing response body %v", err)
+		}
+	}()
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return OIDCConfigResponse{}, fmt.Errorf("failed reading OIDC configuration response body: %v", err)
+	}
+
+	if res.StatusCode != 200 {
+		return OIDCConfigResponse{}, fmt.Errorf("OIDC configuration request returned status %d with response: %s",
+			res.StatusCode, string(body))
+	}
+
+	config := OIDCConfigResponse{}
+	err = json.Unmarshal(body, &config)
+	if err != nil {
+		return OIDCConfigResponse{}, fmt.Errorf("failed unmarshaling OIDC configuration response: %v", err)
+	}
+
+	return config, nil
+
+}
+
 func loadTLSConfig(certFile string, certKey string) (*tls.Config, error) {
 	// Load server's certificate and private key
 	serverCert, err := tls.LoadX509KeyPair(certFile, certKey)
@@ -319,6 +399,9 @@ func loadTLSConfig(certFile string, certKey string) (*tls.Config, error) {
 	config := &tls.Config{
 		Certificates: []tls.Certificate{serverCert},
 		ClientAuth:   tls.NoClientCert,
+		NextProtos: []string{
+			"h2", "http/1.1", // enable HTTP/2
+		},
 	}
 
 	return config, nil
@@ -391,7 +474,7 @@ func copySymLink(source, dest string) error {
 
 func cpDir(src string, dst string) error {
 	var err error
-	var fds []os.FileInfo
+	var fds []os.DirEntry
 	var srcinfo os.FileInfo
 
 	if srcinfo, err = os.Stat(src); err != nil {
@@ -402,7 +485,7 @@ func cpDir(src string, dst string) error {
 		return err
 	}
 
-	if fds, err = ioutil.ReadDir(src); err != nil {
+	if fds, err = os.ReadDir(src); err != nil {
 		return err
 	}
 	for _, fd := range fds {

management/proto/management.pb.go

@@ -980,6 +980,8 @@ type NetworkMap struct {
 	RemotePeers []*RemotePeerConfig `protobuf:"bytes,3,rep,name=remotePeers,proto3" json:"remotePeers,omitempty"`
 	// Indicates whether remotePeers array is empty or not to bypass protobuf null and empty array equality.
 	RemotePeersIsEmpty bool `protobuf:"varint,4,opt,name=remotePeersIsEmpty,proto3" json:"remotePeersIsEmpty,omitempty"`
+	// List of routes to be applied
+	Routes []*Route `protobuf:"bytes,5,rep,name=Routes,proto3" json:"Routes,omitempty"`
 }
 
 func (x *NetworkMap) Reset() {
@@ -1042,6 +1044,13 @@ func (x *NetworkMap) GetRemotePeersIsEmpty() bool {
 	return false
 }
 
+func (x *NetworkMap) GetRoutes() []*Route {
+	if x != nil {
+		return x.Routes
+	}
+	return nil
+}
+
 // RemotePeerConfig represents a configuration of a remote peer.
 // The properties are used to configure Wireguard Peers sections
 type RemotePeerConfig struct {
@@ -1278,9 +1287,14 @@ type ProviderConfig struct {
 	// An IDP application client secret
 	ClientSecret string `protobuf:"bytes,2,opt,name=ClientSecret,proto3" json:"ClientSecret,omitempty"`
 	// An IDP API domain
+	// Deprecated. Use a DeviceAuthEndpoint and TokenEndpoint
 	Domain string `protobuf:"bytes,3,opt,name=Domain,proto3" json:"Domain,omitempty"`
 	// An Audience for validation
 	Audience string `protobuf:"bytes,4,opt,name=Audience,proto3" json:"Audience,omitempty"`
+	// DeviceAuthEndpoint is an endpoint to request device authentication code.
+	DeviceAuthEndpoint string `protobuf:"bytes,5,opt,name=DeviceAuthEndpoint,proto3" json:"DeviceAuthEndpoint,omitempty"`
+	// TokenEndpoint is an endpoint to request auth token.
+	TokenEndpoint string `protobuf:"bytes,6,opt,name=TokenEndpoint,proto3" json:"TokenEndpoint,omitempty"`
 }
 
 func (x *ProviderConfig) Reset() {
@@ -1343,6 +1357,116 @@ func (x *ProviderConfig) GetAudience() string {
 	return ""
 }
 
+func (x *ProviderConfig) GetDeviceAuthEndpoint() string {
+	if x != nil {
+		return x.DeviceAuthEndpoint
+	}
+	return ""
+}
+
+func (x *ProviderConfig) GetTokenEndpoint() string {
+	if x != nil {
+		return x.TokenEndpoint
+	}
+	return ""
+}
+
+// Route represents a route.Route object
+type Route struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	ID          string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"`
+	Network     string `protobuf:"bytes,2,opt,name=Network,proto3" json:"Network,omitempty"`
+	NetworkType int64  `protobuf:"varint,3,opt,name=NetworkType,proto3" json:"NetworkType,omitempty"`
+	Peer        string `protobuf:"bytes,4,opt,name=Peer,proto3" json:"Peer,omitempty"`
+	Metric      int64  `protobuf:"varint,5,opt,name=Metric,proto3" json:"Metric,omitempty"`
+	Masquerade  bool   `protobuf:"varint,6,opt,name=Masquerade,proto3" json:"Masquerade,omitempty"`
+	NetID       string `protobuf:"bytes,7,opt,name=NetID,proto3" json:"NetID,omitempty"`
+}
+
+func (x *Route) Reset() {
+	*x = Route{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_management_proto_msgTypes[19]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Route) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Route) ProtoMessage() {}
+
+func (x *Route) ProtoReflect() protoreflect.Message {
+	mi := &file_management_proto_msgTypes[19]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Route.ProtoReflect.Descriptor instead.
+func (*Route) Descriptor() ([]byte, []int) {
+	return file_management_proto_rawDescGZIP(), []int{19}
+}
+
+func (x *Route) GetID() string {
+	if x != nil {
+		return x.ID
+	}
+	return ""
+}
+
+func (x *Route) GetNetwork() string {
+	if x != nil {
+		return x.Network
+	}
+	return ""
+}
+
+func (x *Route) GetNetworkType() int64 {
+	if x != nil {
+		return x.NetworkType
+	}
+	return 0
+}
+
+func (x *Route) GetPeer() string {
+	if x != nil {
+		return x.Peer
+	}
+	return ""
+}
+
+func (x *Route) GetMetric() int64 {
+	if x != nil {
+		return x.Metric
+	}
+	return 0
+}
+
+func (x *Route) GetMasquerade() bool {
+	if x != nil {
+		return x.Masquerade
+	}
+	return false
+}
+
+func (x *Route) GetNetID() string {
+	if x != nil {
+		return x.NetID
+	}
+	return ""
+}
+
 var File_management_proto protoreflect.FileDescriptor
 
 var file_management_proto_rawDesc = []byte{
@@ -1459,7 +1583,7 @@ var file_management_proto_rawDesc = []byte{
 	0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28,
 	0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53,
 	0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e,
-	0x66, 0x69, 0x67, 0x22, 0xcc, 0x01, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d,
+	0x66, 0x69, 0x67, 0x22, 0xf7, 0x01, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d,
 	0x61, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01,
 	0x28, 0x04, 0x52, 0x06, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65,
 	0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16,
@@ -1472,67 +1596,87 @@ var file_management_proto_rawDesc = []byte{
 	0x72, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72,
 	0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12,
 	0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70,
-	0x74, 0x79, 0x22, 0x83, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65,
+	0x74, 0x79, 0x12, 0x29, 0x0a, 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03,
-	0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62,
+	0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e,
-	0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62,
+	0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x83, 0x01,
-	0x4b, 0x65, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70,
+	0x0a, 0x10, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66,
-	0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64,
+	0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x01,
-	0x49, 0x70, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x1e,
-	0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d,
+	0x0a, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, 0x73, 0x18, 0x02, 0x20, 0x03,
-	0x65, 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73,
+	0x28, 0x09, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, 0x73, 0x12, 0x33,
-	0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x49, 0x0a, 0x09, 0x53, 0x53, 0x48, 0x43,
+	0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x73, 0x68, 0x45, 0x6e, 0x61, 0x62,
+	0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53,
-	0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x73, 0x68, 0x45, 0x6e,
+	0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e,
-	0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, 0x62, 0x4b,
+	0x66, 0x69, 0x67, 0x22, 0x49, 0x0a, 0x09, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
-	0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, 0x62,
+	0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x73, 0x68, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01,
-	0x4b, 0x65, 0x79, 0x22, 0x20, 0x0a, 0x1e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74,
+	0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x73, 0x68, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64,
-	0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x52, 0x65,
+	0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbf, 0x01, 0x0a, 0x17, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65,
+	0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x22, 0x20,
-	0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f,
+	0x0a, 0x1e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a,
-	0x77, 0x12, 0x48, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74,
+	0x22, 0xbf, 0x01, 0x0a, 0x17, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f,
-	0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61,
+	0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x48, 0x0a, 0x08,
-	0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65,
+	0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c,
-	0x72, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x42, 0x0a, 0x0e, 0x50,
+	0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x65, 0x76, 0x69,
-	0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20,
+	0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46,
-	0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74,
+	0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x52, 0x08, 0x50, 0x72,
-	0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52,
+	0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x42, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64,
-	0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22,
+	0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a,
-	0x16, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x0a, 0x0a, 0x06, 0x48,
+	0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76,
-	0x4f, 0x53, 0x54, 0x45, 0x44, 0x10, 0x00, 0x22, 0x84, 0x01, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76,
+	0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76,
-	0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x43, 0x6c,
+	0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x16, 0x0a, 0x08, 0x70, 0x72,
-	0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x43, 0x6c,
+	0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x0a, 0x0a, 0x06, 0x48, 0x4f, 0x53, 0x54, 0x45, 0x44,
-	0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x10, 0x00, 0x22, 0xda, 0x01, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43,
-	0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x6c,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49,
-	0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f,
+	0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49,
-	0x6d, 0x61, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61,
+	0x44, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65,
-	0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04,
+	0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x32, 0xf7,
+	0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18,
-	0x02, 0x0a, 0x11, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72,
+	0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1a, 0x0a,
-	0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1c, 0x2e,
+	0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79,
+	0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x44, 0x65, 0x76,
-	0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61,
+	0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18,
-	0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74,
+	0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74,
-	0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x04, 0x53,
+	0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x6f, 0x6b,
-	0x79, 0x6e, 0x63, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74,
+	0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x22,
+	0xb5, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x4e, 0x65, 0x74,
+	0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x4e, 0x65, 0x74, 0x77,
+	0x6f, 0x72, 0x6b, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x79,
+	0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72,
+	0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, 0x18, 0x04, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x4d, 0x65, 0x74,
+	0x72, 0x69, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69,
+	0x63, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, 0x73, 0x71, 0x75, 0x65, 0x72, 0x61, 0x64, 0x65, 0x18,
+	0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x4d, 0x61, 0x73, 0x71, 0x75, 0x65, 0x72, 0x61, 0x64,
+	0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x32, 0xf7, 0x02, 0x0a, 0x11, 0x4d, 0x61, 0x6e, 0x61,
+	0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a,
+	0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d,
+	0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73,
+	0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e,
+	0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61,
+	0x67, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x1c, 0x2e, 0x6d,
+	0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70,
+	0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e,
+	0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65,
+	0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c,
+	0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x11, 0x2e, 0x6d,
+	0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a,
+	0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
+	0x12, 0x33, 0x0a, 0x09, 0x69, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x12, 0x11, 0x2e,
+	0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
+	0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d,
+	0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x44, 0x65, 0x76, 0x69,
+	0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46,
+	0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74,
 	0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67,
 	0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45,
 	0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22,
-	0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f,
-	0x4b, 0x65, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74,
+	0x74, 0x6f, 0x33,
-	0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d,
-	0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x69, 0x73, 0x48, 0x65, 0x61,
-	0x6c, 0x74, 0x68, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e,
-	0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65,
-	0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x1a,
-	0x47, 0x65, 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
-	0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e,
-	0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65,
-	0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67,
-	0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d,
-	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -1548,7 +1692,7 @@ func file_management_proto_rawDescGZIP() []byte {
 }
 
 var file_management_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
-var file_management_proto_msgTypes = make([]protoimpl.MessageInfo, 19)
+var file_management_proto_msgTypes = make([]protoimpl.MessageInfo, 20)
 var file_management_proto_goTypes = []interface{}{
 	(HostConfig_Protocol)(0),               // 0: management.HostConfig.Protocol
 	(DeviceAuthorizationFlowProvider)(0),   // 1: management.DeviceAuthorizationFlow.provider
@@ -1571,7 +1715,8 @@ var file_management_proto_goTypes = []interface{}{
 	(*DeviceAuthorizationFlowRequest)(nil), // 18: management.DeviceAuthorizationFlowRequest
 	(*DeviceAuthorizationFlow)(nil),        // 19: management.DeviceAuthorizationFlow
 	(*ProviderConfig)(nil),                 // 20: management.ProviderConfig
-	(*timestamp.Timestamp)(nil),            // 21: google.protobuf.Timestamp
+	(*Route)(nil),                          // 21: management.Route
+	(*timestamp.Timestamp)(nil),            // 22: google.protobuf.Timestamp
 }
 var file_management_proto_depIdxs = []int32{
 	11, // 0: management.SyncResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig
@@ -1582,7 +1727,7 @@ var file_management_proto_depIdxs = []int32{
 	6,  // 5: management.LoginRequest.peerKeys:type_name -> management.PeerKeys
 	11, // 6: management.LoginResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig
 	14, // 7: management.LoginResponse.peerConfig:type_name -> management.PeerConfig
-	21, // 8: management.ServerKeyResponse.expiresAt:type_name -> google.protobuf.Timestamp
+	22, // 8: management.ServerKeyResponse.expiresAt:type_name -> google.protobuf.Timestamp
 	12, // 9: management.WiretrusteeConfig.stuns:type_name -> management.HostConfig
 	13, // 10: management.WiretrusteeConfig.turns:type_name -> management.ProtectedHostConfig
 	12, // 11: management.WiretrusteeConfig.signal:type_name -> management.HostConfig
@@ -1591,24 +1736,25 @@ var file_management_proto_depIdxs = []int32{
 	17, // 14: management.PeerConfig.sshConfig:type_name -> management.SSHConfig
 	14, // 15: management.NetworkMap.peerConfig:type_name -> management.PeerConfig
 	16, // 16: management.NetworkMap.remotePeers:type_name -> management.RemotePeerConfig
-	17, // 17: management.RemotePeerConfig.sshConfig:type_name -> management.SSHConfig
+	21, // 17: management.NetworkMap.Routes:type_name -> management.Route
-	1,  // 18: management.DeviceAuthorizationFlow.Provider:type_name -> management.DeviceAuthorizationFlow.provider
+	17, // 18: management.RemotePeerConfig.sshConfig:type_name -> management.SSHConfig
-	20, // 19: management.DeviceAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig
+	1,  // 19: management.DeviceAuthorizationFlow.Provider:type_name -> management.DeviceAuthorizationFlow.provider
-	2,  // 20: management.ManagementService.Login:input_type -> management.EncryptedMessage
+	20, // 20: management.DeviceAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig
-	2,  // 21: management.ManagementService.Sync:input_type -> management.EncryptedMessage
+	2,  // 21: management.ManagementService.Login:input_type -> management.EncryptedMessage
-	10, // 22: management.ManagementService.GetServerKey:input_type -> management.Empty
+	2,  // 22: management.ManagementService.Sync:input_type -> management.EncryptedMessage
-	10, // 23: management.ManagementService.isHealthy:input_type -> management.Empty
+	10, // 23: management.ManagementService.GetServerKey:input_type -> management.Empty
-	2,  // 24: management.ManagementService.GetDeviceAuthorizationFlow:input_type -> management.EncryptedMessage
+	10, // 24: management.ManagementService.isHealthy:input_type -> management.Empty
-	2,  // 25: management.ManagementService.Login:output_type -> management.EncryptedMessage
+	2,  // 25: management.ManagementService.GetDeviceAuthorizationFlow:input_type -> management.EncryptedMessage
-	2,  // 26: management.ManagementService.Sync:output_type -> management.EncryptedMessage
+	2,  // 26: management.ManagementService.Login:output_type -> management.EncryptedMessage
-	9,  // 27: management.ManagementService.GetServerKey:output_type -> management.ServerKeyResponse
+	2,  // 27: management.ManagementService.Sync:output_type -> management.EncryptedMessage
-	10, // 28: management.ManagementService.isHealthy:output_type -> management.Empty
+	9,  // 28: management.ManagementService.GetServerKey:output_type -> management.ServerKeyResponse
-	2,  // 29: management.ManagementService.GetDeviceAuthorizationFlow:output_type -> management.EncryptedMessage
+	10, // 29: management.ManagementService.isHealthy:output_type -> management.Empty
-	25, // [25:30] is the sub-list for method output_type
+	2,  // 30: management.ManagementService.GetDeviceAuthorizationFlow:output_type -> management.EncryptedMessage
-	20, // [20:25] is the sub-list for method input_type
+	26, // [26:31] is the sub-list for method output_type
-	20, // [20:20] is the sub-list for extension type_name
+	21, // [21:26] is the sub-list for method input_type
-	20, // [20:20] is the sub-list for extension extendee
+	21, // [21:21] is the sub-list for extension type_name
-	0,  // [0:20] is the sub-list for field type_name
+	21, // [21:21] is the sub-list for extension extendee
+	0,  // [0:21] is the sub-list for field type_name
 }
 
 func init() { file_management_proto_init() }
@@ -1845,6 +1991,18 @@ func file_management_proto_init() {
 				return nil
 			}
 		}
+		file_management_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Route); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -1852,7 +2010,7 @@ func file_management_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_management_proto_rawDesc,
 			NumEnums:      2,
-			NumMessages:   19,
+			NumMessages:   20,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

management/proto/management.proto

@@ -176,6 +176,8 @@ message NetworkMap {
   // Indicates whether remotePeers array is empty or not to bypass protobuf null and empty array equality.
   bool remotePeersIsEmpty = 4;
 
+  // List of routes to be applied
+  repeated Route Routes = 5;
 }
 
 // RemotePeerConfig represents a configuration of a remote peer.
@@ -225,7 +227,23 @@ message ProviderConfig {
   // An IDP application client secret
   string ClientSecret = 2;
   // An IDP API domain
-  string Domain =3;
+  // Deprecated. Use a DeviceAuthEndpoint and TokenEndpoint
+  string Domain = 3;
   // An Audience for validation
   string Audience = 4;
+  // DeviceAuthEndpoint is an endpoint to request device authentication code.
+  string DeviceAuthEndpoint = 5;
+  // TokenEndpoint is an endpoint to request auth token.
+  string TokenEndpoint = 6;
+}
+
+// Route represents a route.Route object
+message Route {
+  string ID = 1;
+  string Network = 2;
+  int64  NetworkType = 3;
+  string Peer = 4;
+  int64  Metric = 5;
+  bool   Masquerade = 6;
+  string NetID = 7;
 }

management/server/account.go

@@ -7,6 +7,7 @@ import (
 	cacheStore "github.com/eko/gocache/v2/store"
 	"github.com/netbirdio/netbird/management/server/idp"
 	"github.com/netbirdio/netbird/management/server/jwtclaims"
+	"github.com/netbirdio/netbird/route"
 	gocache "github.com/patrickmn/go-cache"
 	"github.com/rs/xid"
 	log "github.com/sirupsen/logrus"
@@ -48,6 +49,7 @@ type AccountManager interface {
 	RenamePeer(accountId string, peerKey string, newName string) (*Peer, error)
 	DeletePeer(accountId string, peerKey string) (*Peer, error)
 	GetPeerByIP(accountId string, peerIP string) (*Peer, error)
+	UpdatePeer(accountID string, peer *Peer) (*Peer, error)
 	GetNetworkMap(peerKey string) (*NetworkMap, error)
 	GetPeerNetwork(peerKey string) (*Network, error)
 	AddPeer(setupKey string, userId string, peer *Peer) (*Peer, error)
@@ -67,7 +69,12 @@ type AccountManager interface {
 	UpdateRule(accountID string, ruleID string, operations []RuleUpdateOperation) (*Rule, error)
 	DeleteRule(accountId, ruleID string) error
 	ListRules(accountId string) ([]*Rule, error)
-	UpdatePeer(accountID string, peer *Peer) (*Peer, error)
+	GetRoute(accountID, routeID string) (*route.Route, error)
+	CreateRoute(accountID string, prefix, peer, description, netID string, masquerade bool, metric int, enabled bool) (*route.Route, error)
+	SaveRoute(accountID string, route *route.Route) error
+	UpdateRoute(accountID string, routeID string, operations []RouteUpdateOperation) (*route.Route, error)
+	DeleteRoute(accountID, routeID string) error
+	ListRoutes(accountID string) ([]*route.Route, error)
 }
 
 type DefaultAccountManager struct {
@@ -94,6 +101,7 @@ type Account struct {
 	Users                  map[string]*User
 	Groups                 map[string]*Group
 	Rules                  map[string]*Rule
+	Routes                 map[string]*route.Route
 }
 
 type UserInfo struct {
@@ -686,6 +694,7 @@ func newAccountWithId(accountId, userId, domain string) *Account {
 	network := NewNetwork()
 	peers := make(map[string]*Peer)
 	users := make(map[string]*User)
+	routes := make(map[string]*route.Route)
 	users[userId] = NewAdminUser(userId)
 	log.Debugf("created new account %s with setup key %s", accountId, defaultKey.Key)
 
@@ -697,6 +706,7 @@ func newAccountWithId(accountId, userId, domain string) *Account {
 		Users:     users,
 		CreatedBy: userId,
 		Domain:    domain,
+		Routes:    routes,
 	}
 
 	addAllGroup(acc)

management/server/config.go

@@ -16,7 +16,7 @@ const (
 	TCP   Protocol = "tcp"
 	HTTP  Protocol = "http"
 	HTTPS Protocol = "https"
-	AUTH0 Provider = "auth0"
+	NONE  Provider = "none"
 )
 
 // Config of the Management service
@@ -55,6 +55,8 @@ type HttpServerConfig struct {
 	AuthIssuer string
 	// AuthKeysLocation is a location of JWT key set containing the public keys used to verify JWT
 	AuthKeysLocation string
+	// OIDCConfigEndpoint is the endpoint of an IDP manager to get OIDC configuration
+	OIDCConfigEndpoint string
 }
 
 // Host represents a Wiretrustee host (e.g. STUN, TURN, Signal)
@@ -81,9 +83,14 @@ type ProviderConfig struct {
 	// ClientSecret An IDP application client secret
 	ClientSecret string
 	// Domain An IDP API domain
+	// Deprecated. Use TokenEndpoint and DeviceAuthEndpoint
 	Domain string
 	// Audience An Audience for to authorization validation
 	Audience string
+	// TokenEndpoint is the endpoint of an IDP manager where clients can obtain access token
+	TokenEndpoint string
+	// DeviceAuthEndpoint is the endpoint of an IDP manager where clients can obtain device authorization code
+	DeviceAuthEndpoint string
 }
 
 // validateURL validates input http url

management/server/file_store.go

@@ -2,6 +2,8 @@ package server
 
 import (
 	"fmt"
+	"github.com/netbirdio/netbird/route"
+	"net/netip"
 	"os"
 	"path/filepath"
 	"strings"
@@ -25,6 +27,8 @@ type FileStore struct {
 	PrivateDomain2AccountId map[string]string              `json:"-"`
 	PeerKeyId2SrcRulesId    map[string]map[string]struct{} `json:"-"`
 	PeerKeyId2DstRulesId    map[string]map[string]struct{} `json:"-"`
+	PeerKeyID2RouteIDs      map[string]map[string]struct{} `json:"-"`
+	AccountPrefix2RouteIDs  map[string]map[string][]string `json:"-"`
 
 	// mutex to synchronise Store read/write operations
 	mux       sync.Mutex `json:"-"`
@@ -51,7 +55,9 @@ func restore(file string) (*FileStore, error) {
 			UserId2AccountId:        make(map[string]string),
 			PrivateDomain2AccountId: make(map[string]string),
 			PeerKeyId2SrcRulesId:    make(map[string]map[string]struct{}),
+			PeerKeyID2RouteIDs:      make(map[string]map[string]struct{}),
 			PeerKeyId2DstRulesId:    make(map[string]map[string]struct{}),
+			AccountPrefix2RouteIDs:  make(map[string]map[string][]string),
 			storeFile:               file,
 		}
 
@@ -74,8 +80,10 @@ func restore(file string) (*FileStore, error) {
 	store.PeerKeyId2AccountId = make(map[string]string)
 	store.UserId2AccountId = make(map[string]string)
 	store.PrivateDomain2AccountId = make(map[string]string)
-	store.PeerKeyId2SrcRulesId = map[string]map[string]struct{}{}
+	store.PeerKeyId2SrcRulesId = make(map[string]map[string]struct{})
-	store.PeerKeyId2DstRulesId = map[string]map[string]struct{}{}
+	store.PeerKeyId2DstRulesId = make(map[string]map[string]struct{})
+	store.PeerKeyID2RouteIDs = make(map[string]map[string]struct{})
+	store.AccountPrefix2RouteIDs = make(map[string]map[string][]string)
 
 	for accountId, account := range store.Accounts {
 		for setupKeyId := range account.SetupKeys {
@@ -116,6 +124,25 @@ func restore(file string) (*FileStore, error) {
 		for _, user := range account.Users {
 			store.UserId2AccountId[user.Id] = accountId
 		}
+		for _, route := range account.Routes {
+			if route.Peer == "" {
+				continue
+			}
+			if store.PeerKeyID2RouteIDs[route.Peer] == nil {
+				store.PeerKeyID2RouteIDs[route.Peer] = make(map[string]struct{})
+			}
+			store.PeerKeyID2RouteIDs[route.Peer][route.ID] = struct{}{}
+			if store.AccountPrefix2RouteIDs[account.Id] == nil {
+				store.AccountPrefix2RouteIDs[account.Id] = make(map[string][]string)
+			}
+			if _, ok := store.AccountPrefix2RouteIDs[account.Id][route.Network.String()]; !ok {
+				store.AccountPrefix2RouteIDs[account.Id][route.Network.String()] = make([]string, 0)
+			}
+			store.AccountPrefix2RouteIDs[account.Id][route.Network.String()] = append(
+				store.AccountPrefix2RouteIDs[account.Id][route.Network.String()],
+				route.ID,
+			)
+		}
 		if account.Domain != "" && account.DomainCategory == PrivateCategory &&
 			account.IsDomainPrimaryAccount {
 			store.PrivateDomain2AccountId[account.Domain] = accountId
@@ -177,11 +204,12 @@ func (s *FileStore) DeletePeer(accountId string, peerKey string) (*Peer, error)
 	if peer == nil {
 		return nil, status.Errorf(codes.NotFound, "peer not found")
 	}
-
+	peerRoutes := s.PeerKeyID2RouteIDs[peerKey]
 	delete(account.Peers, peerKey)
 	delete(s.PeerKeyId2AccountId, peerKey)
 	delete(s.PeerKeyId2DstRulesId, peerKey)
 	delete(s.PeerKeyId2SrcRulesId, peerKey)
+	delete(s.PeerKeyID2RouteIDs, peerKey)
 
 	// cleanup groups
 	for _, g := range account.Groups {
@@ -194,6 +222,11 @@ func (s *FileStore) DeletePeer(accountId string, peerKey string) (*Peer, error)
 		g.Peers = peers
 	}
 
+	for routeID := range peerRoutes {
+		account.Routes[routeID].Enabled = false
+		account.Routes[routeID].Peer = ""
+	}
+
 	err = s.persist(s.storeFile)
 	if err != nil {
 		return nil, err
@@ -238,10 +271,14 @@ func (s *FileStore) SaveAccount(account *Account) error {
 		s.SetupKeyId2AccountId[strings.ToUpper(keyId)] = account.Id
 	}
 
+	// enforce peer to account index and delete peer to route indexes for rebuild
 	for _, peer := range account.Peers {
 		s.PeerKeyId2AccountId[peer.Key] = account.Id
+		delete(s.PeerKeyID2RouteIDs, peer.Key)
 	}
 
+	delete(s.AccountPrefix2RouteIDs, account.Id)
+
 	// remove all peers related to account from rules indexes
 	cleanIDs := make([]string, 0)
 	for key := range s.PeerKeyId2SrcRulesId {
@@ -294,6 +331,26 @@ func (s *FileStore) SaveAccount(account *Account) error {
 		}
 	}
 
+	for _, route := range account.Routes {
+		if route.Peer == "" {
+			continue
+		}
+		if s.PeerKeyID2RouteIDs[route.Peer] == nil {
+			s.PeerKeyID2RouteIDs[route.Peer] = make(map[string]struct{})
+		}
+		s.PeerKeyID2RouteIDs[route.Peer][route.ID] = struct{}{}
+		if s.AccountPrefix2RouteIDs[account.Id] == nil {
+			s.AccountPrefix2RouteIDs[account.Id] = make(map[string][]string)
+		}
+		if _, ok := s.AccountPrefix2RouteIDs[account.Id][route.Network.String()]; !ok {
+			s.AccountPrefix2RouteIDs[account.Id][route.Network.String()] = make([]string, 0)
+		}
+		s.AccountPrefix2RouteIDs[account.Id][route.Network.String()] = append(
+			s.AccountPrefix2RouteIDs[account.Id][route.Network.String()],
+			route.ID,
+		)
+	}
+
 	for _, user := range account.Users {
 		s.UserId2AccountId[user.Id] = account.Id
 	}
@@ -305,6 +362,7 @@ func (s *FileStore) SaveAccount(account *Account) error {
 	return s.persist(s.storeFile)
 }
 
+// GetAccountByPrivateDomain returns account by private domain
 func (s *FileStore) GetAccountByPrivateDomain(domain string) (*Account, error) {
 	accountId, accountIdFound := s.PrivateDomain2AccountId[strings.ToLower(domain)]
 	if !accountIdFound {
@@ -322,6 +380,7 @@ func (s *FileStore) GetAccountByPrivateDomain(domain string) (*Account, error) {
 	return account, nil
 }
 
+// GetAccountBySetupKey returns account by setup key id
 func (s *FileStore) GetAccountBySetupKey(setupKey string) (*Account, error) {
 	accountId, accountIdFound := s.SetupKeyId2AccountId[strings.ToUpper(setupKey)]
 	if !accountIdFound {
@@ -336,6 +395,7 @@ func (s *FileStore) GetAccountBySetupKey(setupKey string) (*Account, error) {
 	return account, nil
 }
 
+// GetAccountPeers returns account peers
 func (s *FileStore) GetAccountPeers(accountId string) ([]*Peer, error) {
 	s.mux.Lock()
 	defer s.mux.Unlock()
@@ -353,6 +413,7 @@ func (s *FileStore) GetAccountPeers(accountId string) ([]*Peer, error) {
 	return peers, nil
 }
 
+// GetAllAccounts returns all accounts
 func (s *FileStore) GetAllAccounts() (all []*Account) {
 	for _, a := range s.Accounts {
 		all = append(all, a)
@@ -361,6 +422,7 @@ func (s *FileStore) GetAllAccounts() (all []*Account) {
 	return all
 }
 
+// GetAccount returns an account for id
 func (s *FileStore) GetAccount(accountId string) (*Account, error) {
 	account, accountFound := s.Accounts[accountId]
 	if !accountFound {
@@ -370,6 +432,7 @@ func (s *FileStore) GetAccount(accountId string) (*Account, error) {
 	return account, nil
 }
 
+// GetUserAccount returns a user account
 func (s *FileStore) GetUserAccount(userId string) (*Account, error) {
 	s.mux.Lock()
 	defer s.mux.Unlock()
@@ -382,10 +445,7 @@ func (s *FileStore) GetUserAccount(userId string) (*Account, error) {
 	return s.GetAccount(accountId)
 }
 
-func (s *FileStore) GetPeerAccount(peerKey string) (*Account, error) {
+func (s *FileStore) getPeerAccount(peerKey string) (*Account, error) {
-	s.mux.Lock()
-	defer s.mux.Unlock()
-
 	accountId, accountIdFound := s.PeerKeyId2AccountId[peerKey]
 	if !accountIdFound {
 		return nil, status.Errorf(codes.NotFound, "Provided peer key doesn't exists %s", peerKey)
@@ -394,6 +454,15 @@ func (s *FileStore) GetPeerAccount(peerKey string) (*Account, error) {
 	return s.GetAccount(accountId)
 }
 
+// GetPeerAccount returns user account if exists
+func (s *FileStore) GetPeerAccount(peerKey string) (*Account, error) {
+	s.mux.Lock()
+	defer s.mux.Unlock()
+
+	return s.getPeerAccount(peerKey)
+}
+
+// GetPeerSrcRules return list of source rules for peer
 func (s *FileStore) GetPeerSrcRules(accountId, peerKey string) ([]*Rule, error) {
 	s.mux.Lock()
 	defer s.mux.Unlock()
@@ -419,6 +488,7 @@ func (s *FileStore) GetPeerSrcRules(accountId, peerKey string) ([]*Rule, error)
 	return rules, nil
 }
 
+// GetPeerDstRules return list of destination rules for peer
 func (s *FileStore) GetPeerDstRules(accountId, peerKey string) ([]*Rule, error) {
 	s.mux.Lock()
 	defer s.mux.Unlock()
@@ -443,3 +513,56 @@ func (s *FileStore) GetPeerDstRules(accountId, peerKey string) ([]*Rule, error)
 
 	return rules, nil
 }
+
+// GetPeerRoutes return list of routes for peer
+func (s *FileStore) GetPeerRoutes(peerKey string) ([]*route.Route, error) {
+	s.mux.Lock()
+	defer s.mux.Unlock()
+
+	account, err := s.getPeerAccount(peerKey)
+	if err != nil {
+		return nil, err
+	}
+
+	var routes []*route.Route
+
+	routeIDs, ok := s.PeerKeyID2RouteIDs[peerKey]
+	if !ok {
+		return routes, nil
+	}
+
+	for id := range routeIDs {
+		route, found := account.Routes[id]
+		if found {
+			routes = append(routes, route)
+		}
+	}
+
+	return routes, nil
+}
+
+// GetRoutesByPrefix return list of routes by account and route prefix
+func (s *FileStore) GetRoutesByPrefix(accountID string, prefix netip.Prefix) ([]*route.Route, error) {
+	s.mux.Lock()
+	defer s.mux.Unlock()
+
+	account, err := s.GetAccount(accountID)
+	if err != nil {
+		return nil, err
+	}
+
+	routeIDs, ok := s.AccountPrefix2RouteIDs[accountID][prefix.String()]
+	if !ok {
+		return nil, status.Errorf(codes.NotFound, "no routes for prefix: %v", prefix.String())
+	}
+
+	var routes []*route.Route
+	for _, id := range routeIDs {
+		route, found := account.Routes[id]
+		if found {
+			routes = append(routes, route)
+		}
+	}
+
+	return routes, nil
+}

management/server/grpcserver.go

@@ -3,6 +3,8 @@ package server
 import (
 	"context"
 	"fmt"
+	"github.com/netbirdio/netbird/route"
+	gPeer "google.golang.org/grpc/peer"
 	"strings"
 	"time"
 
@@ -87,17 +89,24 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi
 
 	peer, err := s.accountManager.GetPeer(peerKey.String())
 	if err != nil {
-		return status.Errorf(codes.PermissionDenied, "provided peer with the key wgPubKey %s is not registered", peerKey.String())
+		p, _ := gPeer.FromContext(srv.Context())
+		msg := status.Errorf(codes.PermissionDenied, "provided peer with the key wgPubKey %s is not registered, remote addr is %s", peerKey.String(), p.Addr.String())
+		log.Debug(msg)
+		return msg
 	}
 
 	syncReq := &proto.SyncRequest{}
 	err = encryption.DecryptMessage(peerKey, s.wgKey, req.Body, syncReq)
 	if err != nil {
-		return status.Errorf(codes.InvalidArgument, "invalid request message")
+		p, _ := gPeer.FromContext(srv.Context())
+		msg := status.Errorf(codes.InvalidArgument, "invalid request message from %s,remote addr is %s", peerKey.String(), p.Addr.String())
+		log.Debug(msg)
+		return msg
 	}
 
 	err = s.sendInitialSync(peerKey, peer, srv)
 	if err != nil {
+		log.Debugf("error while sending initial sync for %s: %v", peerKey.String(), err)
 		return err
 	}
 
@@ -116,7 +125,7 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi
 		// condition when there are some updates
 		case update, open := <-updates:
 			if !open {
-				// updates channel has been closed
+				log.Debugf("updates channel for peer %s was closed", peerKey.String())
 				return nil
 			}
 			log.Debugf("recevied an update for peer %s", peerKey.String())
@@ -230,7 +239,7 @@ func (s *GRPCServer) registerPeer(peerKey wgtypes.Key, req *proto.LoginRequest)
 				peersToSend = append(peersToSend, p)
 			}
 		}
-		update := toSyncResponse(s.config, remotePeer, peersToSend, nil, networkMap.Network.CurrentSerial(), networkMap.Network)
+		update := toSyncResponse(s.config, remotePeer, peersToSend, networkMap.Routes, nil, networkMap.Network.CurrentSerial(), networkMap.Network)
 		err = s.peersUpdateManager.SendUpdate(remotePeer.Key, &UpdateMessage{Update: update})
 		if err != nil {
 			// todo rethink if we should keep this return
@@ -265,8 +274,13 @@ func (s *GRPCServer) Login(ctx context.Context, req *proto.EncryptedMessage) (*p
 		if errStatus, ok := status.FromError(err); ok && errStatus.Code() == codes.NotFound {
 			// peer doesn't exist -> check if setup key was provided
 			if loginReq.GetJwtToken() == "" && loginReq.GetSetupKey() == "" {
-				// absent setup key -> permission denied
+				// absent setup key or jwt -> permission denied
-				return nil, status.Errorf(codes.PermissionDenied, "provided peer with the key wgPubKey %s is not registered and no setup key or jwt was provided", peerKey.String())
+				p, _ := gPeer.FromContext(ctx)
+				msg := status.Errorf(codes.PermissionDenied,
+					"provided peer with the key wgPubKey %s is not registered and no setup key or jwt was provided,"+
+						" remote addr is %s", peerKey.String(), p.Addr.String())
+				log.Debug(msg)
+				return nil, msg
 			}
 
 			// setup key or jwt is present -> try normal registration flow
@@ -407,13 +421,15 @@ func toRemotePeerConfig(peers []*Peer) []*proto.RemotePeerConfig {
 	return remotePeers
 }
 
-func toSyncResponse(config *Config, peer *Peer, peers []*Peer, turnCredentials *TURNCredentials, serial uint64, network *Network) *proto.SyncResponse {
+func toSyncResponse(config *Config, peer *Peer, peers []*Peer, routes []*route.Route, turnCredentials *TURNCredentials, serial uint64, network *Network) *proto.SyncResponse {
 	wtConfig := toWiretrusteeConfig(config, turnCredentials)
 
 	pConfig := toPeerConfig(peer, network)
 
 	remotePeers := toRemotePeerConfig(peers)
 
+	routesUpdate := toProtocolRoutes(routes)
+
 	return &proto.SyncResponse{
 		WiretrusteeConfig:  wtConfig,
 		PeerConfig:         pConfig,
@@ -424,6 +440,7 @@ func toSyncResponse(config *Config, peer *Peer, peers []*Peer, turnCredentials *
 			PeerConfig:         pConfig,
 			RemotePeers:        remotePeers,
 			RemotePeersIsEmpty: len(remotePeers) == 0,
+			Routes:             routesUpdate,
 		},
 	}
 }
@@ -449,7 +466,7 @@ func (s *GRPCServer) sendInitialSync(peerKey wgtypes.Key, peer *Peer, srv proto.
 	} else {
 		turnCredentials = nil
 	}
-	plainResp := toSyncResponse(s.config, peer, networkMap.Peers, turnCredentials, networkMap.Network.CurrentSerial(), networkMap.Network)
+	plainResp := toSyncResponse(s.config, peer, networkMap.Peers, networkMap.Routes, turnCredentials, networkMap.Network.CurrentSerial(), networkMap.Network)
 
 	encryptedResp, err := encryption.EncryptMessage(peerKey, s.wgKey, plainResp)
 	if err != nil {
@@ -487,7 +504,7 @@ func (s *GRPCServer) GetDeviceAuthorizationFlow(ctx context.Context, req *proto.
 		return nil, status.Error(codes.InvalidArgument, errMSG)
 	}
 
-	if s.config.DeviceAuthorizationFlow == nil {
+	if s.config.DeviceAuthorizationFlow == nil || s.config.DeviceAuthorizationFlow.Provider == string(NONE) {
 		return nil, status.Error(codes.NotFound, "no device authorization flow information available")
 	}
 
@@ -499,10 +516,12 @@ func (s *GRPCServer) GetDeviceAuthorizationFlow(ctx context.Context, req *proto.
 	flowInfoResp := &proto.DeviceAuthorizationFlow{
 		Provider: proto.DeviceAuthorizationFlowProvider(provider),
 		ProviderConfig: &proto.ProviderConfig{
-			ClientID:     s.config.DeviceAuthorizationFlow.ProviderConfig.ClientID,
+			ClientID:           s.config.DeviceAuthorizationFlow.ProviderConfig.ClientID,
-			ClientSecret: s.config.DeviceAuthorizationFlow.ProviderConfig.ClientSecret,
+			ClientSecret:       s.config.DeviceAuthorizationFlow.ProviderConfig.ClientSecret,
-			Domain:       s.config.DeviceAuthorizationFlow.ProviderConfig.Domain,
+			Domain:             s.config.DeviceAuthorizationFlow.ProviderConfig.Domain,
-			Audience:     s.config.DeviceAuthorizationFlow.ProviderConfig.Audience,
+			Audience:           s.config.DeviceAuthorizationFlow.ProviderConfig.Audience,
+			DeviceAuthEndpoint: s.config.DeviceAuthorizationFlow.ProviderConfig.DeviceAuthEndpoint,
+			TokenEndpoint:      s.config.DeviceAuthorizationFlow.ProviderConfig.TokenEndpoint,
 		},
 	}
 

management/server/http/api/openapi.yml

@@ -14,6 +14,8 @@ tags:
     description: Interact with and view information about groups.
   - name: Rules
     description: Interact with and view information about rules.
+  - name: Routes
+    description: Interact with and view information about routes.
 components:
   schemas:
     User:
@@ -191,17 +193,13 @@ components:
                 $ref: '#/components/schemas/PeerMinimum'
           required:
           - peers
-    GroupPatchOperation:
+    PatchMinimum:
       type: object
       properties:
         op:
           description: Patch operation type
           type: string
           enum: [ "replace","add","remove" ]
-        path:
-          description: Group field to update in form /<field>
-          type: string
-          enum: [ "name","peers" ]
         value:
           description: Values to be applied
           type: array
@@ -209,8 +207,19 @@ components:
             type: string
       required:
         - op
-        - path
         - value
+    GroupPatchOperation:
+      allOf:
+        - $ref: '#/components/schemas/PatchMinimum'
+        - type: object
+          properties:
+            path:
+              description: Group field to update in form /<field>
+              type: string
+              enum: [ "name","peers" ]
+          required:
+            - path
+
     RuleMinimum:
       type: object
       properties:
@@ -257,25 +266,79 @@ components:
           - sources
           - destinations
     RulePatchOperation:
+      allOf:
+        - $ref: '#/components/schemas/PatchMinimum'
+        - type: object
+          properties:
+            path:
+              description: Rule field to update in form /<field>
+              type: string
+              enum: [ "name","description","disabled","flow","sources","destinations" ]
+          required:
+            - path
+    RouteRequest:
       type: object
       properties:
-        op:
+        description:
-          description: Patch operation type
+          description: Route description
           type: string
-          enum: [ "replace","add","remove" ]
+        network_id:
-        path:
+          description: Route network identifier, to group HA routes
-          description: Rule field to update in form /<field>
           type: string
-          enum: [ "name","description","disabled","flow","sources","destinations" ]
+          maxLength: 40
-        value:
+          minLength: 1
-          description: Values to be applied
+        enabled:
-          type: array
+          description: Route status
-          items:
+          type: boolean
-            type: string
+        peer:
+          description: Peer Identifier associated with route
+          type: string
+        network:
+          description: Network range in CIDR format
+          type: string
+        metric:
+          description: Route metric number. Lowest number has higher priority
+          type: integer
+          maximum: 9999
+          minimum: 1
+        masquerade:
+          description: Indicate if peer should masquerade traffic to this route's prefix
+          type: boolean
       required:
-        - op
+        - id
-        - path
+        - description
-        - value
+        - network_id
+        - enabled
+        - peer
+        - network
+        - metric
+        - masquerade
+    Route:
+      allOf:
+        - type: object
+          properties:
+            id:
+              description: Route Id
+              type: string
+            network_type:
+              description: Network type indicating if it is IPv4 or IPv6
+              type: string
+          required:
+            - id
+            - network_type
+        - $ref: '#/components/schemas/RouteRequest'
+    RoutePatchOperation:
+      allOf:
+        - $ref: '#/components/schemas/PatchMinimum'
+        - type: object
+          properties:
+            path:
+              description: Route field to update in form /<field>
+              type: string
+              enum: [ "network","network_id","description","enabled","peer","metric","masquerade" ]
+          required:
+            - path
+
   responses:
     not_found:
       description: Resource not found
@@ -947,3 +1010,174 @@ paths:
           "$ref": "#/components/responses/forbidden"
         '500':
           "$ref": "#/components/responses/internal_error"
+
+  /api/routes:
+    get:
+      summary: Returns a list of all routes
+      tags: [ Routes ]
+      security:
+        - BearerAuth: [ ]
+      responses:
+        '200':
+          description: A JSON Array of Routes
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/Route'
+        '400':
+          "$ref": "#/components/responses/bad_request"
+        '401':
+          "$ref": "#/components/responses/requires_authentication"
+        '403':
+          "$ref": "#/components/responses/forbidden"
+        '500':
+          "$ref": "#/components/responses/internal_error"
+    post:
+      summary: Creates a Route
+      tags: [ Routes ]
+      security:
+        - BearerAuth: [ ]
+      requestBody:
+        description: New Routes request
+        content:
+          'application/json':
+            schema:
+              $ref: '#/components/schemas/RouteRequest'
+      responses:
+        '200':
+          description: A Route Object
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Route'
+        '400':
+          "$ref": "#/components/responses/bad_request"
+        '401':
+          "$ref": "#/components/responses/requires_authentication"
+        '403':
+          "$ref": "#/components/responses/forbidden"
+        '500':
+          "$ref": "#/components/responses/internal_error"
+
+  /api/routes/{id}:
+    get:
+      summary: Get information about a Routes
+      tags: [ Routes ]
+      security:
+        - BearerAuth: [ ]
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: The Route ID
+      responses:
+        '200':
+          description: A Route object
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Route'
+        '400':
+          "$ref": "#/components/responses/bad_request"
+        '401':
+          "$ref": "#/components/responses/requires_authentication"
+        '403':
+          "$ref": "#/components/responses/forbidden"
+        '500':
+          "$ref": "#/components/responses/internal_error"
+    put:
+      summary: Update/Replace a Route
+      tags: [ Routes ]
+      security:
+        - BearerAuth: [ ]
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: The Route ID
+      requestBody:
+        description: Update Route request
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/RouteRequest'
+      responses:
+        '200':
+          description: A Route object
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Route'
+        '400':
+          "$ref": "#/components/responses/bad_request"
+        '401':
+          "$ref": "#/components/responses/requires_authentication"
+        '403':
+          "$ref": "#/components/responses/forbidden"
+        '500':
+          "$ref": "#/components/responses/internal_error"
+    patch:
+      summary: Update information about a Route
+      tags: [ Routes ]
+      security:
+        - BearerAuth: [ ]
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: The Route ID
+      requestBody:
+        description: Update Route request using a list of json patch objects
+        content:
+          'application/json':
+            schema:
+              type: array
+              items:
+                $ref: '#/components/schemas/RoutePatchOperation'
+      responses:
+        '200':
+          description: A Route object
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Route'
+        '400':
+          "$ref": "#/components/responses/bad_request"
+        '401':
+          "$ref": "#/components/responses/requires_authentication"
+        '403':
+          "$ref": "#/components/responses/forbidden"
+        '500':
+          "$ref": "#/components/responses/internal_error"
+    delete:
+      summary: Delete a Route
+      tags: [ Routes ]
+      security:
+        - BearerAuth: [ ]
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: The Route ID
+      responses:
+        '200':
+          description: Delete status code
+          content: { }
+        '400':
+          "$ref": "#/components/responses/bad_request"
+        '401':
+          "$ref": "#/components/responses/requires_authentication"
+        '403':
+          "$ref": "#/components/responses/forbidden"
+        '500':
+          "$ref": "#/components/responses/internal_error"

management/server/http/api/types.gen.go

@@ -24,6 +24,31 @@ const (
 	GroupPatchOperationPathPeers GroupPatchOperationPath = "peers"
 )
 
+// Defines values for PatchMinimumOp.
+const (
+	PatchMinimumOpAdd     PatchMinimumOp = "add"
+	PatchMinimumOpRemove  PatchMinimumOp = "remove"
+	PatchMinimumOpReplace PatchMinimumOp = "replace"
+)
+
+// Defines values for RoutePatchOperationOp.
+const (
+	RoutePatchOperationOpAdd     RoutePatchOperationOp = "add"
+	RoutePatchOperationOpRemove  RoutePatchOperationOp = "remove"
+	RoutePatchOperationOpReplace RoutePatchOperationOp = "replace"
+)
+
+// Defines values for RoutePatchOperationPath.
+const (
+	RoutePatchOperationPathDescription RoutePatchOperationPath = "description"
+	RoutePatchOperationPathEnabled     RoutePatchOperationPath = "enabled"
+	RoutePatchOperationPathMasquerade  RoutePatchOperationPath = "masquerade"
+	RoutePatchOperationPathMetric      RoutePatchOperationPath = "metric"
+	RoutePatchOperationPathNetwork     RoutePatchOperationPath = "network"
+	RoutePatchOperationPathNetworkId   RoutePatchOperationPath = "network_id"
+	RoutePatchOperationPathPeer        RoutePatchOperationPath = "peer"
+)
+
 // Defines values for RulePatchOperationOp.
 const (
 	RulePatchOperationOpAdd     RulePatchOperationOp = "add"
@@ -86,6 +111,18 @@ type GroupPatchOperationOp string
 // Group field to update in form /<field>
 type GroupPatchOperationPath string
 
+// PatchMinimum defines model for PatchMinimum.
+type PatchMinimum struct {
+	// Patch operation type
+	Op PatchMinimumOp `json:"op"`
+
+	// Values to be applied
+	Value []string `json:"value"`
+}
+
+// Patch operation type
+type PatchMinimumOp string
+
 // Peer defines model for Peer.
 type Peer struct {
 	// Provides information of who activated the Peer. User or Setup Key
@@ -131,6 +168,78 @@ type PeerMinimum struct {
 	Name string `json:"name"`
 }
 
+// Route defines model for Route.
+type Route struct {
+	// Route description
+	Description string `json:"description"`
+
+	// Route status
+	Enabled bool `json:"enabled"`
+
+	// Route Id
+	Id string `json:"id"`
+
+	// Indicate if peer should masquerade traffic to this route's prefix
+	Masquerade bool `json:"masquerade"`
+
+	// Route metric number. Lowest number has higher priority
+	Metric int `json:"metric"`
+
+	// Network range in CIDR format
+	Network string `json:"network"`
+
+	// Route network identifier, to group HA routes
+	NetworkId string `json:"network_id"`
+
+	// Network type indicating if it is IPv4 or IPv6
+	NetworkType string `json:"network_type"`
+
+	// Peer Identifier associated with route
+	Peer string `json:"peer"`
+}
+
+// RoutePatchOperation defines model for RoutePatchOperation.
+type RoutePatchOperation struct {
+	// Patch operation type
+	Op RoutePatchOperationOp `json:"op"`
+
+	// Route field to update in form /<field>
+	Path RoutePatchOperationPath `json:"path"`
+
+	// Values to be applied
+	Value []string `json:"value"`
+}
+
+// Patch operation type
+type RoutePatchOperationOp string
+
+// Route field to update in form /<field>
+type RoutePatchOperationPath string
+
+// RouteRequest defines model for RouteRequest.
+type RouteRequest struct {
+	// Route description
+	Description string `json:"description"`
+
+	// Route status
+	Enabled bool `json:"enabled"`
+
+	// Indicate if peer should masquerade traffic to this route's prefix
+	Masquerade bool `json:"masquerade"`
+
+	// Route metric number. Lowest number has higher priority
+	Metric int `json:"metric"`
+
+	// Network range in CIDR format
+	Network string `json:"network"`
+
+	// Route network identifier, to group HA routes
+	NetworkId string `json:"network_id"`
+
+	// Peer Identifier associated with route
+	Peer string `json:"peer"`
+}
+
 // Rule defines model for Rule.
 type Rule struct {
 	// Rule friendly description
@@ -272,6 +381,15 @@ type PutApiPeersIdJSONBody struct {
 	SshEnabled bool   `json:"ssh_enabled"`
 }
 
+// PostApiRoutesJSONBody defines parameters for PostApiRoutes.
+type PostApiRoutesJSONBody = RouteRequest
+
+// PatchApiRoutesIdJSONBody defines parameters for PatchApiRoutesId.
+type PatchApiRoutesIdJSONBody = []RoutePatchOperation
+
+// PutApiRoutesIdJSONBody defines parameters for PutApiRoutesId.
+type PutApiRoutesIdJSONBody = RouteRequest
+
 // PostApiRulesJSONBody defines parameters for PostApiRules.
 type PostApiRulesJSONBody struct {
 	// Rule friendly description
@@ -327,6 +445,15 @@ type PutApiGroupsIdJSONRequestBody PutApiGroupsIdJSONBody
 // PutApiPeersIdJSONRequestBody defines body for PutApiPeersId for application/json ContentType.
 type PutApiPeersIdJSONRequestBody PutApiPeersIdJSONBody
 
+// PostApiRoutesJSONRequestBody defines body for PostApiRoutes for application/json ContentType.
+type PostApiRoutesJSONRequestBody = PostApiRoutesJSONBody
+
+// PatchApiRoutesIdJSONRequestBody defines body for PatchApiRoutesId for application/json ContentType.
+type PatchApiRoutesIdJSONRequestBody = PatchApiRoutesIdJSONBody
+
+// PutApiRoutesIdJSONRequestBody defines body for PutApiRoutesId for application/json ContentType.
+type PutApiRoutesIdJSONRequestBody = PutApiRoutesIdJSONBody
+
 // PostApiRulesJSONRequestBody defines body for PostApiRules for application/json ContentType.
 type PostApiRulesJSONRequestBody PostApiRulesJSONBody
 

management/server/http/handler.go

@@ -33,6 +33,7 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience
 	peersHandler := NewPeers(accountManager, authAudience)
 	keysHandler := NewSetupKeysHandler(accountManager, authAudience)
 	userHandler := NewUserHandler(accountManager, authAudience)
+	routesHandler := NewRoutes(accountManager, authAudience)
 
 	apiHandler.HandleFunc("/api/peers", peersHandler.GetPeers).Methods("GET", "OPTIONS")
 	apiHandler.HandleFunc("/api/peers/{id}", peersHandler.HandlePeer).
@@ -59,6 +60,13 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience
 	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.GetGroupHandler).Methods("GET", "OPTIONS")
 	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.DeleteGroupHandler).Methods("DELETE", "OPTIONS")
 
+	apiHandler.HandleFunc("/api/routes", routesHandler.GetAllRoutesHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/api/routes", routesHandler.CreateRouteHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.UpdateRouteHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.PatchRouteHandler).Methods("PATCH", "OPTIONS")
+	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.GetRouteHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.DeleteRouteHandler).Methods("DELETE", "OPTIONS")
+
 	return apiHandler, nil
 
 }

management/server/http/routes.go

@@ -0,0 +1,403 @@
+package http
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/gorilla/mux"
+	"github.com/netbirdio/netbird/management/server"
+	"github.com/netbirdio/netbird/management/server/http/api"
+	"github.com/netbirdio/netbird/management/server/jwtclaims"
+	"github.com/netbirdio/netbird/route"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"net/http"
+	"unicode/utf8"
+)
+
+// Routes is the routes handler of the account
+type Routes struct {
+	jwtExtractor   jwtclaims.ClaimsExtractor
+	accountManager server.AccountManager
+	authAudience   string
+}
+
+// NewRoutes returns a new instance of Routes handler
+func NewRoutes(accountManager server.AccountManager, authAudience string) *Routes {
+	return &Routes{
+		accountManager: accountManager,
+		authAudience:   authAudience,
+		jwtExtractor:   *jwtclaims.NewClaimsExtractor(nil),
+	}
+}
+
+// GetAllRoutesHandler returns the list of routes for the account
+func (h *Routes) GetAllRoutesHandler(w http.ResponseWriter, r *http.Request) {
+	account, err := getJWTAccount(h.accountManager, h.jwtExtractor, h.authAudience, r)
+	if err != nil {
+		log.Error(err)
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	routes, err := h.accountManager.ListRoutes(account.Id)
+	if err != nil {
+		log.Error(err)
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+	apiRoutes := make([]*api.Route, 0)
+	for _, r := range routes {
+		apiRoutes = append(apiRoutes, toRouteResponse(account, r))
+	}
+
+	writeJSONObject(w, apiRoutes)
+}
+
+// CreateRouteHandler handles route creation request
+func (h *Routes) CreateRouteHandler(w http.ResponseWriter, r *http.Request) {
+	account, err := getJWTAccount(h.accountManager, h.jwtExtractor, h.authAudience, r)
+	if err != nil {
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	var req api.PostApiRoutesJSONRequestBody
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	peerKey := req.Peer
+	if req.Peer != "" {
+		peer, err := h.accountManager.GetPeerByIP(account.Id, req.Peer)
+		if err != nil {
+			log.Error(err)
+			http.Redirect(w, r, "/", http.StatusUnprocessableEntity)
+			return
+		}
+		peerKey = peer.Key
+	}
+
+	_, newPrefix, err := route.ParseNetwork(req.Network)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("couldn't parse update prefix %s", req.Network), http.StatusBadRequest)
+		return
+	}
+
+	if utf8.RuneCountInString(req.NetworkId) > route.MaxNetIDChar || req.NetworkId == "" {
+		http.Error(w, fmt.Sprintf("identifier should be between 1 and %d", route.MaxNetIDChar), http.StatusBadRequest)
+		return
+	}
+
+	newRoute, err := h.accountManager.CreateRoute(account.Id, newPrefix.String(), peerKey, req.Description, req.NetworkId, req.Masquerade, req.Metric, req.Enabled)
+	if err != nil {
+		log.Error(err)
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	resp := toRouteResponse(account, newRoute)
+
+	writeJSONObject(w, &resp)
+}
+
+// UpdateRouteHandler handles update to a route identified by a given ID
+func (h *Routes) UpdateRouteHandler(w http.ResponseWriter, r *http.Request) {
+	account, err := getJWTAccount(h.accountManager, h.jwtExtractor, h.authAudience, r)
+	if err != nil {
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	vars := mux.Vars(r)
+	routeID := vars["id"]
+	if len(routeID) == 0 {
+		http.Error(w, "invalid route Id", http.StatusBadRequest)
+		return
+	}
+
+	_, err = h.accountManager.GetRoute(account.Id, routeID)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("couldn't find route for ID %s", routeID), http.StatusNotFound)
+		return
+	}
+
+	var req api.PutApiRoutesIdJSONBody
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	prefixType, newPrefix, err := route.ParseNetwork(req.Network)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("couldn't parse update prefix %s for route ID %s", req.Network, routeID), http.StatusBadRequest)
+		return
+	}
+
+	peerKey := req.Peer
+	if req.Peer != "" {
+		peer, err := h.accountManager.GetPeerByIP(account.Id, req.Peer)
+		if err != nil {
+			log.Error(err)
+			http.Redirect(w, r, "/", http.StatusUnprocessableEntity)
+			return
+		}
+		peerKey = peer.Key
+	}
+
+	if utf8.RuneCountInString(req.NetworkId) > route.MaxNetIDChar || req.NetworkId == "" {
+		http.Error(w, fmt.Sprintf("identifier should be between 1 and %d", route.MaxNetIDChar), http.StatusBadRequest)
+		return
+	}
+
+	newRoute := &route.Route{
+		ID:          routeID,
+		Network:     newPrefix,
+		NetID:       req.NetworkId,
+		NetworkType: prefixType,
+		Masquerade:  req.Masquerade,
+		Peer:        peerKey,
+		Metric:      req.Metric,
+		Description: req.Description,
+		Enabled:     req.Enabled,
+	}
+
+	err = h.accountManager.SaveRoute(account.Id, newRoute)
+	if err != nil {
+		log.Errorf("failed updating route \"%s\" under account %s %v", routeID, account.Id, err)
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	resp := toRouteResponse(account, newRoute)
+
+	writeJSONObject(w, &resp)
+}
+
+// PatchRouteHandler handles patch updates to a route identified by a given ID
+func (h *Routes) PatchRouteHandler(w http.ResponseWriter, r *http.Request) {
+	account, err := getJWTAccount(h.accountManager, h.jwtExtractor, h.authAudience, r)
+	if err != nil {
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	vars := mux.Vars(r)
+	routeID := vars["id"]
+	if len(routeID) == 0 {
+		http.Error(w, "invalid route ID", http.StatusBadRequest)
+		return
+	}
+
+	_, err = h.accountManager.GetRoute(account.Id, routeID)
+	if err != nil {
+		log.Error(err)
+		http.Error(w, fmt.Sprintf("couldn't find route ID %s", routeID), http.StatusNotFound)
+		return
+	}
+
+	var req api.PatchApiRoutesIdJSONRequestBody
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if len(req) == 0 {
+		http.Error(w, "no patch instruction received", http.StatusBadRequest)
+		return
+	}
+
+	var operations []server.RouteUpdateOperation
+
+	for _, patch := range req {
+		switch patch.Path {
+		case api.RoutePatchOperationPathNetwork:
+			if patch.Op != api.RoutePatchOperationOpReplace {
+				http.Error(w, fmt.Sprintf("Network field only accepts replace operation, got %s", patch.Op),
+					http.StatusBadRequest)
+				return
+			}
+			operations = append(operations, server.RouteUpdateOperation{
+				Type:   server.UpdateRouteNetwork,
+				Values: patch.Value,
+			})
+		case api.RoutePatchOperationPathDescription:
+			if patch.Op != api.RoutePatchOperationOpReplace {
+				http.Error(w, fmt.Sprintf("Description field only accepts replace operation, got %s", patch.Op),
+					http.StatusBadRequest)
+				return
+			}
+			operations = append(operations, server.RouteUpdateOperation{
+				Type:   server.UpdateRouteDescription,
+				Values: patch.Value,
+			})
+		case api.RoutePatchOperationPathNetworkId:
+			if patch.Op != api.RoutePatchOperationOpReplace {
+				http.Error(w, fmt.Sprintf("Network Identifier field only accepts replace operation, got %s", patch.Op),
+					http.StatusBadRequest)
+				return
+			}
+			operations = append(operations, server.RouteUpdateOperation{
+				Type:   server.UpdateRouteNetworkIdentifier,
+				Values: patch.Value,
+			})
+		case api.RoutePatchOperationPathPeer:
+			if patch.Op != api.RoutePatchOperationOpReplace {
+				http.Error(w, fmt.Sprintf("Peer field only accepts replace operation, got %s", patch.Op),
+					http.StatusBadRequest)
+				return
+			}
+			if len(patch.Value) > 1 {
+				http.Error(w, fmt.Sprintf("Value field only accepts 1 value, got %d", len(patch.Value)),
+					http.StatusBadRequest)
+				return
+			}
+			peerValue := patch.Value
+			if patch.Value[0] != "" {
+				peer, err := h.accountManager.GetPeerByIP(account.Id, patch.Value[0])
+				if err != nil {
+					log.Error(err)
+					http.Redirect(w, r, "/", http.StatusUnprocessableEntity)
+					return
+				}
+				peerValue = []string{peer.Key}
+			}
+			operations = append(operations, server.RouteUpdateOperation{
+				Type:   server.UpdateRoutePeer,
+				Values: peerValue,
+			})
+		case api.RoutePatchOperationPathMetric:
+			if patch.Op != api.RoutePatchOperationOpReplace {
+				http.Error(w, fmt.Sprintf("Metric field only accepts replace operation, got %s", patch.Op),
+					http.StatusBadRequest)
+				return
+			}
+			operations = append(operations, server.RouteUpdateOperation{
+				Type:   server.UpdateRouteMetric,
+				Values: patch.Value,
+			})
+		case api.RoutePatchOperationPathMasquerade:
+			if patch.Op != api.RoutePatchOperationOpReplace {
+				http.Error(w, fmt.Sprintf("Masquerade field only accepts replace operation, got %s", patch.Op),
+					http.StatusBadRequest)
+				return
+			}
+			operations = append(operations, server.RouteUpdateOperation{
+				Type:   server.UpdateRouteMasquerade,
+				Values: patch.Value,
+			})
+		case api.RoutePatchOperationPathEnabled:
+			if patch.Op != api.RoutePatchOperationOpReplace {
+				http.Error(w, fmt.Sprintf("Enabled field only accepts replace operation, got %s", patch.Op),
+					http.StatusBadRequest)
+				return
+			}
+			operations = append(operations, server.RouteUpdateOperation{
+				Type:   server.UpdateRouteEnabled,
+				Values: patch.Value,
+			})
+		default:
+			http.Error(w, "invalid patch path", http.StatusBadRequest)
+			return
+		}
+	}
+
+	route, err := h.accountManager.UpdateRoute(account.Id, routeID, operations)
+
+	if err != nil {
+		errStatus, ok := status.FromError(err)
+		if ok && errStatus.Code() == codes.Internal {
+			http.Error(w, errStatus.String(), http.StatusInternalServerError)
+			return
+		}
+
+		if ok && errStatus.Code() == codes.NotFound {
+			http.Error(w, errStatus.String(), http.StatusNotFound)
+			return
+		}
+
+		if ok && errStatus.Code() == codes.InvalidArgument {
+			http.Error(w, errStatus.String(), http.StatusBadRequest)
+			return
+		}
+
+		log.Errorf("failed updating route %s under account %s %v", routeID, account.Id, err)
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	resp := toRouteResponse(account, route)
+
+	writeJSONObject(w, &resp)
+}
+
+// DeleteRouteHandler handles route deletion request
+func (h *Routes) DeleteRouteHandler(w http.ResponseWriter, r *http.Request) {
+	account, err := getJWTAccount(h.accountManager, h.jwtExtractor, h.authAudience, r)
+	if err != nil {
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	routeID := mux.Vars(r)["id"]
+	if len(routeID) == 0 {
+		http.Error(w, "invalid route ID", http.StatusBadRequest)
+		return
+	}
+
+	err = h.accountManager.DeleteRoute(account.Id, routeID)
+	if err != nil {
+		log.Errorf("failed delete route %s under account %s %v", routeID, account.Id, err)
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	writeJSONObject(w, "")
+}
+
+// GetRouteHandler handles a route Get request identified by ID
+func (h *Routes) GetRouteHandler(w http.ResponseWriter, r *http.Request) {
+	account, err := getJWTAccount(h.accountManager, h.jwtExtractor, h.authAudience, r)
+	if err != nil {
+		http.Redirect(w, r, "/", http.StatusInternalServerError)
+		return
+	}
+
+	routeID := mux.Vars(r)["id"]
+	if len(routeID) == 0 {
+		http.Error(w, "invalid route ID", http.StatusBadRequest)
+		return
+	}
+
+	foundRoute, err := h.accountManager.GetRoute(account.Id, routeID)
+	if err != nil {
+		http.Error(w, "route not found", http.StatusNotFound)
+		return
+	}
+
+	writeJSONObject(w, toRouteResponse(account, foundRoute))
+}
+
+func toRouteResponse(account *server.Account, serverRoute *route.Route) *api.Route {
+	var peerIP string
+	if serverRoute.Peer != "" {
+		peer, found := account.Peers[serverRoute.Peer]
+		if !found {
+			panic("peer ID not found")
+		}
+		peerIP = peer.IP.String()
+	}
+
+	return &api.Route{
+		Id:          serverRoute.ID,
+		Description: serverRoute.Description,
+		NetworkId:   serverRoute.NetID,
+		Enabled:     serverRoute.Enabled,
+		Peer:        peerIP,
+		Network:     serverRoute.Network.String(),
+		NetworkType: serverRoute.NetworkType.String(),
+		Masquerade:  serverRoute.Masquerade,
+		Metric:      serverRoute.Metric,
+	}
+}

management/server/http/routes_test.go

@@ -0,0 +1,362 @@
+package http
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/netbirdio/netbird/management/server/http/api"
+	"github.com/netbirdio/netbird/route"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"strconv"
+	"testing"
+
+	"github.com/gorilla/mux"
+	"github.com/magiconair/properties/assert"
+	"github.com/netbirdio/netbird/management/server"
+	"github.com/netbirdio/netbird/management/server/jwtclaims"
+	"github.com/netbirdio/netbird/management/server/mock_server"
+)
+
+const (
+	existingRouteID = "existingRouteID"
+	notFoundRouteID = "notFoundRouteID"
+	existingPeerID  = "100.64.0.100"
+	notFoundPeerID  = "100.64.0.200"
+	existingPeerKey = "existingPeerKey"
+	testAccountID   = "test_id"
+)
+
+var baseExistingRoute = &route.Route{
+	ID:          existingRouteID,
+	Description: "base route",
+	NetID:       "awesomeNet",
+	Network:     netip.MustParsePrefix("192.168.0.0/24"),
+	NetworkType: route.IPv4Network,
+	Metric:      9999,
+	Masquerade:  false,
+	Enabled:     true,
+}
+
+var testingAccount = &server.Account{
+	Id:     testAccountID,
+	Domain: "hotmail.com",
+	Peers: map[string]*server.Peer{
+		existingPeerKey: {
+			Key: existingPeerID,
+			IP:  netip.MustParseAddr(existingPeerID).AsSlice(),
+		},
+	},
+}
+
+func initRoutesTestData() *Routes {
+	return &Routes{
+		accountManager: &mock_server.MockAccountManager{
+			GetRouteFunc: func(_, routeID string) (*route.Route, error) {
+				if routeID == existingRouteID {
+					return baseExistingRoute, nil
+				}
+				return nil, status.Errorf(codes.NotFound, "route with ID %s not found", routeID)
+			},
+			CreateRouteFunc: func(accountID string, network, peer, description, netID string, masquerade bool, metric int, enabled bool) (*route.Route, error) {
+				networkType, p, _ := route.ParseNetwork(network)
+				return &route.Route{
+					ID:          existingRouteID,
+					NetID:       netID,
+					Peer:        peer,
+					Network:     p,
+					NetworkType: networkType,
+					Description: description,
+					Masquerade:  masquerade,
+					Enabled:     enabled,
+				}, nil
+			},
+			SaveRouteFunc: func(_ string, _ *route.Route) error {
+				return nil
+			},
+			DeleteRouteFunc: func(_ string, _ string) error {
+				return nil
+			},
+			GetPeerByIPFunc: func(_ string, peerIP string) (*server.Peer, error) {
+				if peerIP != existingPeerID {
+					return nil, status.Errorf(codes.NotFound, "Peer with ID %s not found", peerIP)
+				}
+				return &server.Peer{
+					Key: existingPeerKey,
+					IP:  netip.MustParseAddr(existingPeerID).AsSlice(),
+				}, nil
+			},
+			UpdateRouteFunc: func(_ string, routeID string, operations []server.RouteUpdateOperation) (*route.Route, error) {
+				routeToUpdate := baseExistingRoute
+				if routeID != routeToUpdate.ID {
+					return nil, status.Errorf(codes.NotFound, "route %s no longer exists", routeID)
+				}
+				for _, operation := range operations {
+					switch operation.Type {
+					case server.UpdateRouteNetwork:
+						routeToUpdate.NetworkType, routeToUpdate.Network, _ = route.ParseNetwork(operation.Values[0])
+					case server.UpdateRouteDescription:
+						routeToUpdate.Description = operation.Values[0]
+					case server.UpdateRouteNetworkIdentifier:
+						routeToUpdate.NetID = operation.Values[0]
+					case server.UpdateRoutePeer:
+						routeToUpdate.Peer = operation.Values[0]
+					case server.UpdateRouteMetric:
+						routeToUpdate.Metric, _ = strconv.Atoi(operation.Values[0])
+					case server.UpdateRouteMasquerade:
+						routeToUpdate.Masquerade, _ = strconv.ParseBool(operation.Values[0])
+					case server.UpdateRouteEnabled:
+						routeToUpdate.Enabled, _ = strconv.ParseBool(operation.Values[0])
+					default:
+						return nil, fmt.Errorf("no operation")
+					}
+				}
+				return routeToUpdate, nil
+			},
+			GetAccountWithAuthorizationClaimsFunc: func(_ jwtclaims.AuthorizationClaims) (*server.Account, error) {
+				return testingAccount, nil
+			},
+		},
+		authAudience: "",
+		jwtExtractor: jwtclaims.ClaimsExtractor{
+			ExtractClaimsFromRequestContext: func(r *http.Request, authAudiance string) jwtclaims.AuthorizationClaims {
+				return jwtclaims.AuthorizationClaims{
+					UserId:    "test_user",
+					Domain:    "hotmail.com",
+					AccountId: testAccountID,
+				}
+			},
+		},
+	}
+}
+
+func TestRoutesHandlers(t *testing.T) {
+	tt := []struct {
+		name           string
+		expectedStatus int
+		expectedBody   bool
+		expectedRoute  *api.Route
+		requestType    string
+		requestPath    string
+		requestBody    io.Reader
+	}{
+		{
+			name:           "Get Existing Route",
+			requestType:    http.MethodGet,
+			requestPath:    "/api/routes/" + existingRouteID,
+			expectedStatus: http.StatusOK,
+			expectedBody:   true,
+			expectedRoute:  toRouteResponse(testingAccount, baseExistingRoute),
+		},
+		{
+			name:           "Get Not Existing Route",
+			requestType:    http.MethodGet,
+			requestPath:    "/api/rules/" + notFoundRouteID,
+			expectedStatus: http.StatusNotFound,
+		},
+		{
+			name:           "Delete Existing Route",
+			requestType:    http.MethodDelete,
+			requestPath:    "/api/routes/" + existingRouteID,
+			expectedStatus: http.StatusOK,
+			expectedBody:   false,
+		},
+		{
+			name:           "Delete Not Existing Route",
+			requestType:    http.MethodDelete,
+			requestPath:    "/api/rules/" + notFoundRouteID,
+			expectedStatus: http.StatusNotFound,
+		},
+		{
+			name:        "POST OK",
+			requestType: http.MethodPost,
+			requestPath: "/api/routes",
+			requestBody: bytes.NewBuffer(
+				[]byte(fmt.Sprintf("{\"Description\":\"Post\",\"Network\":\"192.168.0.0/16\",\"network_id\":\"awesomeNet\",\"Peer\":\"%s\"}", existingPeerID))),
+			expectedStatus: http.StatusOK,
+			expectedBody:   true,
+			expectedRoute: &api.Route{
+				Id:          existingRouteID,
+				Description: "Post",
+				NetworkId:   "awesomeNet",
+				Network:     "192.168.0.0/16",
+				Peer:        existingPeerID,
+				NetworkType: route.IPv4NetworkString,
+				Masquerade:  false,
+				Enabled:     false,
+			},
+		},
+		{
+			name:           "POST Not Found Peer",
+			requestType:    http.MethodPost,
+			requestPath:    "/api/routes",
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("{\"Description\":\"Post\",\"Network\":\"192.168.0.0/16\",\"network_id\":\"awesomeNet\",\"Peer\":\"%s\"}", notFoundPeerID)),
+			expectedStatus: http.StatusUnprocessableEntity,
+			expectedBody:   false,
+		},
+		{
+			name:           "POST Not Invalid Network Identifier",
+			requestType:    http.MethodPost,
+			requestPath:    "/api/routes",
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("{\"Description\":\"Post\",\"Network\":\"192.168.0.0/16\",\"network_id\":\"12345678901234567890qwertyuiopqwertyuiop1\",\"Peer\":\"%s\"}", existingPeerID)),
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   false,
+		},
+		{
+			name:           "POST Invalid Network",
+			requestType:    http.MethodPost,
+			requestPath:    "/api/routes",
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("{\"Description\":\"Post\",\"Network\":\"192.168.0.0/34\",\"network_id\":\"awesomeNet\",\"Peer\":\"%s\"}", existingPeerID)),
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   false,
+		},
+		{
+			name:           "PUT OK",
+			requestType:    http.MethodPut,
+			requestPath:    "/api/routes/" + existingRouteID,
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("{\"Description\":\"Post\",\"Network\":\"192.168.0.0/16\",\"network_id\":\"awesomeNet\",\"Peer\":\"%s\"}", existingPeerID)),
+			expectedStatus: http.StatusOK,
+			expectedBody:   true,
+			expectedRoute: &api.Route{
+				Id:          existingRouteID,
+				Description: "Post",
+				NetworkId:   "awesomeNet",
+				Network:     "192.168.0.0/16",
+				Peer:        existingPeerID,
+				NetworkType: route.IPv4NetworkString,
+				Masquerade:  false,
+				Enabled:     false,
+			},
+		},
+		{
+			name:           "PUT Not Found Route",
+			requestType:    http.MethodPut,
+			requestPath:    "/api/routes/" + notFoundRouteID,
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("{\"Description\":\"Post\",\"Network\":\"192.168.0.0/16\",\"network_id\":\"awesomeNet\",\"Peer\":\"%s\"}", existingPeerID)),
+			expectedStatus: http.StatusNotFound,
+			expectedBody:   false,
+		},
+		{
+			name:           "PUT Not Found Peer",
+			requestType:    http.MethodPut,
+			requestPath:    "/api/routes/" + existingRouteID,
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("{\"Description\":\"Post\",\"Network\":\"192.168.0.0/16\",\"network_id\":\"awesomeNet\",\"Peer\":\"%s\"}", notFoundPeerID)),
+			expectedStatus: http.StatusUnprocessableEntity,
+			expectedBody:   false,
+		},
+		{
+			name:           "PUT Invalid Network Identifier",
+			requestType:    http.MethodPut,
+			requestPath:    "/api/routes/" + existingRouteID,
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("{\"Description\":\"Post\",\"Network\":\"192.168.0.0/16\",\"network_id\":\"12345678901234567890qwertyuiopqwertyuiop1\",\"Peer\":\"%s\"}", existingPeerID)),
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   false,
+		},
+		{
+			name:           "PUT Invalid Network",
+			requestType:    http.MethodPut,
+			requestPath:    "/api/routes/" + existingRouteID,
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("{\"Description\":\"Post\",\"Network\":\"192.168.0.0/34\",\"network_id\":\"awesomeNet\",\"Peer\":\"%s\"}", existingPeerID)),
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   false,
+		},
+		{
+			name:           "PATCH Description OK",
+			requestType:    http.MethodPatch,
+			requestPath:    "/api/routes/" + existingRouteID,
+			requestBody:    bytes.NewBufferString("[{\"op\":\"replace\",\"path\":\"description\",\"value\":[\"NewDesc\"]}]"),
+			expectedStatus: http.StatusOK,
+			expectedBody:   true,
+			expectedRoute: &api.Route{
+				Id:          existingRouteID,
+				Description: "NewDesc",
+				NetworkId:   "awesomeNet",
+				Network:     baseExistingRoute.Network.String(),
+				NetworkType: route.IPv4NetworkString,
+				Masquerade:  baseExistingRoute.Masquerade,
+				Enabled:     baseExistingRoute.Enabled,
+				Metric:      baseExistingRoute.Metric,
+			},
+		},
+		{
+			name:           "PATCH Peer OK",
+			requestType:    http.MethodPatch,
+			requestPath:    "/api/routes/" + existingRouteID,
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("[{\"op\":\"replace\",\"path\":\"peer\",\"value\":[\"%s\"]}]", existingPeerID)),
+			expectedStatus: http.StatusOK,
+			expectedBody:   true,
+			expectedRoute: &api.Route{
+				Id:          existingRouteID,
+				Description: "NewDesc",
+				NetworkId:   "awesomeNet",
+				Network:     baseExistingRoute.Network.String(),
+				NetworkType: route.IPv4NetworkString,
+				Peer:        existingPeerID,
+				Masquerade:  baseExistingRoute.Masquerade,
+				Enabled:     baseExistingRoute.Enabled,
+				Metric:      baseExistingRoute.Metric,
+			},
+		},
+		{
+			name:           "PATCH Not Found Peer",
+			requestType:    http.MethodPatch,
+			requestPath:    "/api/routes/" + existingRouteID,
+			requestBody:    bytes.NewBufferString(fmt.Sprintf("[{\"op\":\"replace\",\"path\":\"peer\",\"value\":[\"%s\"]}]", notFoundPeerID)),
+			expectedStatus: http.StatusUnprocessableEntity,
+			expectedBody:   false,
+		},
+		{
+			name:           "PATCH Not Found Route",
+			requestType:    http.MethodPatch,
+			requestPath:    "/api/routes/" + notFoundRouteID,
+			requestBody:    bytes.NewBufferString("[{\"op\":\"replace\",\"path\":\"network\",\"value\":[\"192.168.0.0/34\"]}]"),
+			expectedStatus: http.StatusNotFound,
+			expectedBody:   false,
+		},
+	}
+
+	p := initRoutesTestData()
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			recorder := httptest.NewRecorder()
+			req := httptest.NewRequest(tc.requestType, tc.requestPath, tc.requestBody)
+
+			router := mux.NewRouter()
+			router.HandleFunc("/api/routes/{id}", p.GetRouteHandler).Methods("GET")
+			router.HandleFunc("/api/routes/{id}", p.DeleteRouteHandler).Methods("DELETE")
+			router.HandleFunc("/api/routes", p.CreateRouteHandler).Methods("POST")
+			router.HandleFunc("/api/routes/{id}", p.UpdateRouteHandler).Methods("PUT")
+			router.HandleFunc("/api/routes/{id}", p.PatchRouteHandler).Methods("PATCH")
+			router.ServeHTTP(recorder, req)
+
+			res := recorder.Result()
+			defer res.Body.Close()
+
+			content, err := io.ReadAll(res.Body)
+			if err != nil {
+				t.Fatalf("I don't know what I expected; %v", err)
+			}
+
+			if status := recorder.Code; status != tc.expectedStatus {
+				t.Errorf("handler returned wrong status code: got %v want %v, content: %s",
+					status, tc.expectedStatus, string(content))
+				return
+			}
+
+			if !tc.expectedBody {
+				return
+			}
+
+			got := &api.Route{}
+			if err = json.Unmarshal(content, &got); err != nil {
+				t.Fatalf("Sent content is not in correct json format; %v", err)
+			}
+			assert.Equal(t, got, tc.expectedRoute)
+		})
+	}
+}

management/server/mock_server/account_mock.go

@@ -3,6 +3,7 @@ package mock_server
 import (
 	"github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/jwtclaims"
+	"github.com/netbirdio/netbird/route"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"time"
@@ -44,6 +45,12 @@ type MockAccountManager struct {
 	UpdatePeerMetaFunc                    func(peerKey string, meta server.PeerSystemMeta) error
 	UpdatePeerSSHKeyFunc                  func(peerKey string, sshKey string) error
 	UpdatePeerFunc                        func(accountID string, peer *server.Peer) (*server.Peer, error)
+	CreateRouteFunc                       func(accountID string, prefix, peer, description, netID string, masquerade bool, metric int, enabled bool) (*route.Route, error)
+	GetRouteFunc                          func(accountID, routeID string) (*route.Route, error)
+	SaveRouteFunc                         func(accountID string, route *route.Route) error
+	UpdateRouteFunc                       func(accountID string, routeID string, operations []server.RouteUpdateOperation) (*route.Route, error)
+	DeleteRouteFunc                       func(accountID, routeID string) error
+	ListRoutesFunc                        func(accountID string) ([]*route.Route, error)
 }
 
 // GetUsersFromAccount mock implementation of GetUsersFromAccount from server.AccountManager interface
@@ -360,3 +367,51 @@ func (am *MockAccountManager) UpdatePeer(accountID string, peer *server.Peer) (*
 	}
 	return nil, status.Errorf(codes.Unimplemented, "method UpdatePeerFunc is is not implemented")
 }
+
+// CreateRoute mock implementation of CreateRoute from server.AccountManager interface
+func (am *MockAccountManager) CreateRoute(accountID string, network, peer, description, netID string, masquerade bool, metric int, enabled bool) (*route.Route, error) {
+	if am.GetRouteFunc != nil {
+		return am.CreateRouteFunc(accountID, network, peer, description, netID, masquerade, metric, enabled)
+	}
+	return nil, status.Errorf(codes.Unimplemented, "method CreateRoute is not implemented")
+}
+
+// GetRoute mock implementation of GetRoute from server.AccountManager interface
+func (am *MockAccountManager) GetRoute(accountID, routeID string) (*route.Route, error) {
+	if am.GetRouteFunc != nil {
+		return am.GetRouteFunc(accountID, routeID)
+	}
+	return nil, status.Errorf(codes.Unimplemented, "method GetRoute is not implemented")
+}
+
+// SaveRoute mock implementation of SaveRoute from server.AccountManager interface
+func (am *MockAccountManager) SaveRoute(accountID string, route *route.Route) error {
+	if am.SaveRouteFunc != nil {
+		return am.SaveRouteFunc(accountID, route)
+	}
+	return status.Errorf(codes.Unimplemented, "method SaveRoute is not implemented")
+}
+
+// UpdateRoute mock implementation of UpdateRoute from server.AccountManager interface
+func (am *MockAccountManager) UpdateRoute(accountID string, ruleID string, operations []server.RouteUpdateOperation) (*route.Route, error) {
+	if am.UpdateRouteFunc != nil {
+		return am.UpdateRouteFunc(accountID, ruleID, operations)
+	}
+	return nil, status.Errorf(codes.Unimplemented, "method UpdateRoute not implemented")
+}
+
+// DeleteRoute mock implementation of DeleteRoute from server.AccountManager interface
+func (am *MockAccountManager) DeleteRoute(accountID, routeID string) error {
+	if am.DeleteRouteFunc != nil {
+		return am.DeleteRouteFunc(accountID, routeID)
+	}
+	return status.Errorf(codes.Unimplemented, "method DeleteRoute is not implemented")
+}
+
+// ListRoutes mock implementation of ListRoutes from server.AccountManager interface
+func (am *MockAccountManager) ListRoutes(accountID string) ([]*route.Route, error) {
+	if am.ListRoutesFunc != nil {
+		return am.ListRoutesFunc(accountID)
+	}
+	return nil, status.Errorf(codes.Unimplemented, "method ListRoutes is not implemented")
+}

management/server/network.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"github.com/c-robinson/iplib"
+	"github.com/netbirdio/netbird/route"
 	"github.com/rs/xid"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -24,6 +25,7 @@ const (
 type NetworkMap struct {
 	Peers   []*Peer
 	Network *Network
+	Routes  []*route.Route
 }
 
 type Network struct {

management/server/peer.go

@@ -251,9 +251,13 @@ func (am *DefaultAccountManager) GetNetworkMap(peerKey string) (*NetworkMap, err
 		return nil, status.Errorf(codes.Internal, "Invalid peer key %s", peerKey)
 	}
 
+	aclPeers := am.getPeersByACL(account, peerKey)
+	routesUpdate := am.getPeersRoutes(append(aclPeers, account.Peers[peerKey]))
+
 	return &NetworkMap{
-		Peers:   am.getPeersByACL(account, peerKey),
+		Peers:   aclPeers,
 		Network: account.Network.Copy(),
+		Routes:  routesUpdate,
 	}, err
 }
 
@@ -386,6 +390,11 @@ func (am *DefaultAccountManager) UpdatePeerSSHKey(peerKey string, sshKey string)
 		return err
 	}
 
+	if peer.SSHKey == sshKey {
+		log.Debugf("same SSH key provided for peer %s, skipping update", peerKey)
+		return nil
+	}
+
 	account, err := am.Store.GetPeerAccount(peerKey)
 	if err != nil {
 		return err
@@ -508,20 +517,23 @@ func (am *DefaultAccountManager) updateAccountPeers(account *Account) error {
 
 	network := account.Network.Copy()
 
-	for _, p := range peers {
+	for _, peer := range peers {
-		update := toRemotePeerConfig(am.getPeersByACL(account, p.Key))
+		aclPeers := am.getPeersByACL(account, peer.Key)
-		err = am.peersUpdateManager.SendUpdate(p.Key,
+		peersUpdate := toRemotePeerConfig(aclPeers)
+		routesUpdate := toProtocolRoutes(am.getPeersRoutes(append(aclPeers, peer)))
+		err = am.peersUpdateManager.SendUpdate(peer.Key,
 			&UpdateMessage{
 				Update: &proto.SyncResponse{
 					// fill deprecated fields for backward compatibility
-					RemotePeers:        update,
+					RemotePeers:        peersUpdate,
-					RemotePeersIsEmpty: len(update) == 0,
+					RemotePeersIsEmpty: len(peersUpdate) == 0,
 					// new field
 					NetworkMap: &proto.NetworkMap{
 						Serial:             account.Network.CurrentSerial(),
-						RemotePeers:        update,
+						RemotePeers:        peersUpdate,
-						RemotePeersIsEmpty: len(update) == 0,
+						RemotePeersIsEmpty: len(peersUpdate) == 0,
-						PeerConfig:         toPeerConfig(p, network),
+						PeerConfig:         toPeerConfig(peer, network),
+						Routes:             routesUpdate,
 					},
 				},
 			})

management/server/route.go

@@ -0,0 +1,387 @@
+package server
+
+import (
+	"github.com/netbirdio/netbird/management/proto"
+	"github.com/netbirdio/netbird/route"
+	"github.com/rs/xid"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"net/netip"
+	"strconv"
+	"unicode/utf8"
+)
+
+const (
+	// UpdateRouteDescription indicates a route description update operation
+	UpdateRouteDescription RouteUpdateOperationType = iota
+	// UpdateRouteNetwork indicates a route IP update operation
+	UpdateRouteNetwork
+	// UpdateRoutePeer indicates a route peer update operation
+	UpdateRoutePeer
+	// UpdateRouteMetric indicates a route metric update operation
+	UpdateRouteMetric
+	// UpdateRouteMasquerade indicates a route masquerade update operation
+	UpdateRouteMasquerade
+	// UpdateRouteEnabled indicates a route enabled update operation
+	UpdateRouteEnabled
+	// UpdateRouteNetworkIdentifier indicates a route net ID update operation
+	UpdateRouteNetworkIdentifier
+)
+
+// RouteUpdateOperationType operation type
+type RouteUpdateOperationType int
+
+func (t RouteUpdateOperationType) String() string {
+	switch t {
+	case UpdateRouteDescription:
+		return "UpdateRouteDescription"
+	case UpdateRouteNetwork:
+		return "UpdateRouteNetwork"
+	case UpdateRoutePeer:
+		return "UpdateRoutePeer"
+	case UpdateRouteMetric:
+		return "UpdateRouteMetric"
+	case UpdateRouteMasquerade:
+		return "UpdateRouteMasquerade"
+	case UpdateRouteEnabled:
+		return "UpdateRouteEnabled"
+	case UpdateRouteNetworkIdentifier:
+		return "UpdateRouteNetworkIdentifier"
+	default:
+		return "InvalidOperation"
+	}
+}
+
+// RouteUpdateOperation operation object with type and values to be applied
+type RouteUpdateOperation struct {
+	Type   RouteUpdateOperationType
+	Values []string
+}
+
+// GetRoute gets a route object from account and route IDs
+func (am *DefaultAccountManager) GetRoute(accountID, routeID string) (*route.Route, error) {
+	am.mux.Lock()
+	defer am.mux.Unlock()
+
+	account, err := am.Store.GetAccount(accountID)
+	if err != nil {
+		return nil, status.Errorf(codes.NotFound, "account not found")
+	}
+
+	wantedRoute, found := account.Routes[routeID]
+	if found {
+		return wantedRoute, nil
+	}
+
+	return nil, status.Errorf(codes.NotFound, "route with ID %s not found", routeID)
+}
+
+// checkPrefixPeerExists checks the combination of prefix and peer id, if it exists returns an error, otherwise returns nil
+func (am *DefaultAccountManager) checkPrefixPeerExists(accountID, peer string, prefix netip.Prefix) error {
+
+	if peer == "" {
+		return nil
+	}
+
+	routesWithPrefix, err := am.Store.GetRoutesByPrefix(accountID, prefix)
+
+	if err != nil {
+		if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound {
+			return nil
+		}
+		return status.Errorf(codes.InvalidArgument, "failed to parse prefix %s", prefix.String())
+	}
+	for _, prefixRoute := range routesWithPrefix {
+		if prefixRoute.Peer == peer {
+			return status.Errorf(codes.AlreadyExists, "failed a route with prefix %s and peer already exist", prefix.String())
+		}
+	}
+	return nil
+}
+
+// CreateRoute creates and saves a new route
+func (am *DefaultAccountManager) CreateRoute(accountID string, network, peer, description, netID string, masquerade bool, metric int, enabled bool) (*route.Route, error) {
+	am.mux.Lock()
+	defer am.mux.Unlock()
+
+	account, err := am.Store.GetAccount(accountID)
+	if err != nil {
+		return nil, status.Errorf(codes.NotFound, "account not found")
+	}
+
+	var newRoute route.Route
+	prefixType, newPrefix, err := route.ParseNetwork(network)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "failed to parse IP %s", network)
+	}
+	err = am.checkPrefixPeerExists(accountID, peer, newPrefix)
+	if err != nil {
+		return nil, err
+	}
+
+	if peer != "" {
+		_, peerExist := account.Peers[peer]
+		if !peerExist {
+			return nil, status.Errorf(codes.InvalidArgument, "failed to find Peer %s", peer)
+		}
+	}
+
+	if metric < route.MinMetric || metric > route.MaxMetric {
+		return nil, status.Errorf(codes.InvalidArgument, "metric should be between %d and %d", route.MinMetric, route.MaxMetric)
+	}
+
+	if utf8.RuneCountInString(netID) > route.MaxNetIDChar || netID == "" {
+		return nil, status.Errorf(codes.InvalidArgument, "identifier should be between 1 and %d", route.MaxNetIDChar)
+	}
+
+	newRoute.Peer = peer
+	newRoute.ID = xid.New().String()
+	newRoute.Network = newPrefix
+	newRoute.NetworkType = prefixType
+	newRoute.Description = description
+	newRoute.NetID = netID
+	newRoute.Masquerade = masquerade
+	newRoute.Metric = metric
+	newRoute.Enabled = enabled
+
+	if account.Routes == nil {
+		account.Routes = make(map[string]*route.Route)
+	}
+
+	account.Routes[newRoute.ID] = &newRoute
+
+	account.Network.IncSerial()
+	if err = am.Store.SaveAccount(account); err != nil {
+		return nil, err
+	}
+
+	err = am.updateAccountPeers(account)
+	if err != nil {
+		log.Error(err)
+		return &newRoute, status.Errorf(codes.Unavailable, "failed to update peers after create route %s", newPrefix)
+	}
+	return &newRoute, nil
+}
+
+// SaveRoute saves route
+func (am *DefaultAccountManager) SaveRoute(accountID string, routeToSave *route.Route) error {
+	am.mux.Lock()
+	defer am.mux.Unlock()
+
+	if routeToSave == nil {
+		return status.Errorf(codes.InvalidArgument, "route provided is nil")
+	}
+
+	if !routeToSave.Network.IsValid() {
+		return status.Errorf(codes.InvalidArgument, "invalid Prefix %s", routeToSave.Network.String())
+	}
+
+	if routeToSave.Metric < route.MinMetric || routeToSave.Metric > route.MaxMetric {
+		return status.Errorf(codes.InvalidArgument, "metric should be between %d and %d", route.MinMetric, route.MaxMetric)
+	}
+
+	if utf8.RuneCountInString(routeToSave.NetID) > route.MaxNetIDChar || routeToSave.NetID == "" {
+		return status.Errorf(codes.InvalidArgument, "identifier should be between 1 and %d", route.MaxNetIDChar)
+	}
+
+	account, err := am.Store.GetAccount(accountID)
+	if err != nil {
+		return status.Errorf(codes.NotFound, "account not found")
+	}
+
+	if routeToSave.Peer != "" {
+		_, peerExist := account.Peers[routeToSave.Peer]
+		if !peerExist {
+			return status.Errorf(codes.InvalidArgument, "failed to find Peer %s", routeToSave.Peer)
+		}
+	}
+
+	account.Routes[routeToSave.ID] = routeToSave
+
+	account.Network.IncSerial()
+	if err = am.Store.SaveAccount(account); err != nil {
+		return err
+	}
+
+	return am.updateAccountPeers(account)
+}
+
+// UpdateRoute updates existing route with set of operations
+func (am *DefaultAccountManager) UpdateRoute(accountID, routeID string, operations []RouteUpdateOperation) (*route.Route, error) {
+	am.mux.Lock()
+	defer am.mux.Unlock()
+
+	account, err := am.Store.GetAccount(accountID)
+	if err != nil {
+		return nil, status.Errorf(codes.NotFound, "account not found")
+	}
+
+	routeToUpdate, ok := account.Routes[routeID]
+	if !ok {
+		return nil, status.Errorf(codes.NotFound, "route %s no longer exists", routeID)
+	}
+
+	newRoute := routeToUpdate.Copy()
+
+	for _, operation := range operations {
+
+		if len(operation.Values) != 1 {
+			return nil, status.Errorf(codes.InvalidArgument, "operation %s contains invalid number of values, it should be 1", operation.Type.String())
+		}
+
+		switch operation.Type {
+		case UpdateRouteDescription:
+			newRoute.Description = operation.Values[0]
+		case UpdateRouteNetworkIdentifier:
+			if utf8.RuneCountInString(operation.Values[0]) > route.MaxNetIDChar || operation.Values[0] == "" {
+				return nil, status.Errorf(codes.InvalidArgument, "identifier should be between 1 and %d", route.MaxNetIDChar)
+			}
+			newRoute.NetID = operation.Values[0]
+		case UpdateRouteNetwork:
+			prefixType, prefix, err := route.ParseNetwork(operation.Values[0])
+			if err != nil {
+				return nil, status.Errorf(codes.InvalidArgument, "failed to parse IP %s", operation.Values[0])
+			}
+			err = am.checkPrefixPeerExists(accountID, routeToUpdate.Peer, prefix)
+			if err != nil {
+				return nil, err
+			}
+			newRoute.Network = prefix
+			newRoute.NetworkType = prefixType
+		case UpdateRoutePeer:
+			if operation.Values[0] != "" {
+				_, peerExist := account.Peers[operation.Values[0]]
+				if !peerExist {
+					return nil, status.Errorf(codes.InvalidArgument, "failed to find Peer %s", operation.Values[0])
+				}
+			}
+
+			err = am.checkPrefixPeerExists(accountID, operation.Values[0], routeToUpdate.Network)
+			if err != nil {
+				return nil, err
+			}
+			newRoute.Peer = operation.Values[0]
+		case UpdateRouteMetric:
+			metric, err := strconv.Atoi(operation.Values[0])
+			if err != nil {
+				return nil, status.Errorf(codes.InvalidArgument, "failed to parse metric %s, not int", operation.Values[0])
+			}
+			if metric < route.MinMetric || metric > route.MaxMetric {
+				return nil, status.Errorf(codes.InvalidArgument, "failed to parse metric %s, value should be %d > N < %d",
+					operation.Values[0],
+					route.MinMetric,
+					route.MaxMetric,
+				)
+			}
+			newRoute.Metric = metric
+		case UpdateRouteMasquerade:
+			masquerade, err := strconv.ParseBool(operation.Values[0])
+			if err != nil {
+				return nil, status.Errorf(codes.InvalidArgument, "failed to parse masquerade %s, not boolean", operation.Values[0])
+			}
+			newRoute.Masquerade = masquerade
+		case UpdateRouteEnabled:
+			enabled, err := strconv.ParseBool(operation.Values[0])
+			if err != nil {
+				return nil, status.Errorf(codes.InvalidArgument, "failed to parse enabled %s, not boolean", operation.Values[0])
+			}
+			newRoute.Enabled = enabled
+		}
+	}
+
+	account.Routes[routeID] = newRoute
+
+	account.Network.IncSerial()
+	if err = am.Store.SaveAccount(account); err != nil {
+		return nil, err
+	}
+
+	err = am.updateAccountPeers(account)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to update account peers")
+	}
+	return newRoute, nil
+}
+
+// DeleteRoute deletes route with routeID
+func (am *DefaultAccountManager) DeleteRoute(accountID, routeID string) error {
+	am.mux.Lock()
+	defer am.mux.Unlock()
+
+	account, err := am.Store.GetAccount(accountID)
+	if err != nil {
+		return status.Errorf(codes.NotFound, "account not found")
+	}
+
+	delete(account.Routes, routeID)
+
+	account.Network.IncSerial()
+	if err = am.Store.SaveAccount(account); err != nil {
+		return err
+	}
+
+	return am.updateAccountPeers(account)
+}
+
+// ListRoutes returns a list of routes from account
+func (am *DefaultAccountManager) ListRoutes(accountID string) ([]*route.Route, error) {
+	am.mux.Lock()
+	defer am.mux.Unlock()
+
+	account, err := am.Store.GetAccount(accountID)
+	if err != nil {
+		return nil, status.Errorf(codes.NotFound, "account not found")
+	}
+
+	routes := make([]*route.Route, 0, len(account.Routes))
+	for _, item := range account.Routes {
+		routes = append(routes, item)
+	}
+
+	return routes, nil
+}
+
+func toProtocolRoute(route *route.Route) *proto.Route {
+	return &proto.Route{
+		ID:          route.ID,
+		NetID:       route.NetID,
+		Network:     route.Network.String(),
+		NetworkType: int64(route.NetworkType),
+		Peer:        route.Peer,
+		Metric:      int64(route.Metric),
+		Masquerade:  route.Masquerade,
+	}
+}
+
+func (am *DefaultAccountManager) getPeersRoutes(peers []*Peer) []*route.Route {
+	routes := make([]*route.Route, 0)
+	for _, peer := range peers {
+		peerRoutes, err := am.Store.GetPeerRoutes(peer.Key)
+		if err != nil {
+			errorStatus, ok := status.FromError(err)
+			if !ok && errorStatus.Code() != codes.NotFound {
+				log.Error(err)
+			}
+			continue
+		}
+		activeRoutes := make([]*route.Route, 0)
+		for _, pr := range peerRoutes {
+			if pr.Enabled {
+				activeRoutes = append(activeRoutes, pr)
+			}
+		}
+		if len(activeRoutes) > 0 {
+			routes = append(routes, activeRoutes...)
+		}
+	}
+	return routes
+}
+
+func toProtocolRoutes(routes []*route.Route) []*proto.Route {
+	protoRoutes := make([]*proto.Route, 0)
+	for _, r := range routes {
+		protoRoutes = append(protoRoutes, toProtocolRoute(r))
+	}
+	return protoRoutes
+}

management/server/route_test.go

@@ -0,0 +1,844 @@
+package server
+
+import (
+	"github.com/netbirdio/netbird/route"
+	"github.com/rs/xid"
+	"github.com/stretchr/testify/require"
+	"net/netip"
+	"testing"
+)
+
+const peer1Key = "BhRPtynAAYRDy08+q4HTMsos8fs4plTP4NOSh7C1ry8="
+const peer2Key = "/yF0+vCfv+mRR5k0dca0TrGdO/oiNeAI58gToZm5NyI="
+
+func TestCreateRoute(t *testing.T) {
+
+	type input struct {
+		network     string
+		netID       string
+		peer        string
+		description string
+		masquerade  bool
+		metric      int
+		enabled     bool
+	}
+
+	testCases := []struct {
+		name          string
+		inputArgs     input
+		shouldCreate  bool
+		errFunc       require.ErrorAssertionFunc
+		expectedRoute *route.Route
+	}{
+		{
+			name: "Happy Path",
+			inputArgs: input{
+				network:     "192.168.0.0/16",
+				netID:       "happy",
+				peer:        peer1Key,
+				description: "super",
+				masquerade:  false,
+				metric:      9999,
+				enabled:     true,
+			},
+			errFunc:      require.NoError,
+			shouldCreate: true,
+			expectedRoute: &route.Route{
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetworkType: route.IPv4Network,
+				NetID:       "happy",
+				Peer:        peer1Key,
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     true,
+			},
+		},
+		{
+			name: "Bad Prefix",
+			inputArgs: input{
+				network:     "192.168.0.0/34",
+				netID:       "happy",
+				peer:        peer1Key,
+				description: "super",
+				masquerade:  false,
+				metric:      9999,
+				enabled:     true,
+			},
+			errFunc:      require.Error,
+			shouldCreate: false,
+		},
+		{
+			name: "Bad Peer",
+			inputArgs: input{
+				network:     "192.168.0.0/16",
+				netID:       "happy",
+				peer:        "notExistingPeer",
+				description: "super",
+				masquerade:  false,
+				metric:      9999,
+				enabled:     true,
+			},
+			errFunc:      require.Error,
+			shouldCreate: false,
+		},
+		{
+			name: "Empty Peer",
+			inputArgs: input{
+				network:     "192.168.0.0/16",
+				netID:       "happy",
+				peer:        "",
+				description: "super",
+				masquerade:  false,
+				metric:      9999,
+				enabled:     false,
+			},
+			errFunc:      require.NoError,
+			shouldCreate: true,
+			expectedRoute: &route.Route{
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetworkType: route.IPv4Network,
+				NetID:       "happy",
+				Peer:        "",
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     false,
+			},
+		},
+		{
+			name: "Large Metric",
+			inputArgs: input{
+				network:     "192.168.0.0/16",
+				peer:        peer1Key,
+				netID:       "happy",
+				description: "super",
+				masquerade:  false,
+				metric:      99999,
+				enabled:     true,
+			},
+			errFunc:      require.Error,
+			shouldCreate: false,
+		},
+		{
+			name: "Small Metric",
+			inputArgs: input{
+				network:     "192.168.0.0/16",
+				netID:       "happy",
+				peer:        peer1Key,
+				description: "super",
+				masquerade:  false,
+				metric:      0,
+				enabled:     true,
+			},
+			errFunc:      require.Error,
+			shouldCreate: false,
+		},
+		{
+			name: "Large NetID",
+			inputArgs: input{
+				network:     "192.168.0.0/16",
+				peer:        peer1Key,
+				netID:       "12345678901234567890qwertyuiopqwertyuiop1",
+				description: "super",
+				masquerade:  false,
+				metric:      9999,
+				enabled:     true,
+			},
+			errFunc:      require.Error,
+			shouldCreate: false,
+		},
+		{
+			name: "Small NetID",
+			inputArgs: input{
+				network:     "192.168.0.0/16",
+				netID:       "",
+				peer:        peer1Key,
+				description: "",
+				masquerade:  false,
+				metric:      9999,
+				enabled:     true,
+			},
+			errFunc:      require.Error,
+			shouldCreate: false,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			am, err := createRouterManager(t)
+			if err != nil {
+				t.Error("failed to create account manager")
+			}
+
+			account, err := initTestRouteAccount(t, am)
+			if err != nil {
+				t.Error("failed to init testing account")
+			}
+
+			outRoute, err := am.CreateRoute(
+				account.Id,
+				testCase.inputArgs.network,
+				testCase.inputArgs.peer,
+				testCase.inputArgs.description,
+				testCase.inputArgs.netID,
+				testCase.inputArgs.masquerade,
+				testCase.inputArgs.metric,
+				testCase.inputArgs.enabled,
+			)
+
+			testCase.errFunc(t, err)
+
+			if !testCase.shouldCreate {
+				return
+			}
+
+			// assign generated ID
+			testCase.expectedRoute.ID = outRoute.ID
+
+			if !testCase.expectedRoute.IsEqual(outRoute) {
+				t.Errorf("new route didn't match expected route:\nGot %#v\nExpected:%#v\n", outRoute, testCase.expectedRoute)
+			}
+
+		})
+	}
+}
+
+func TestSaveRoute(t *testing.T) {
+
+	validPeer := peer2Key
+	invalidPeer := "nonExisting"
+	validPrefix := netip.MustParsePrefix("192.168.0.0/24")
+	invalidPrefix, _ := netip.ParsePrefix("192.168.0.0/34")
+	validMetric := 1000
+	invalidMetric := 99999
+	validNetID := "12345678901234567890qw"
+	invalidNetID := "12345678901234567890qwertyuiopqwertyuiop1"
+
+	testCases := []struct {
+		name          string
+		existingRoute *route.Route
+		newPeer       *string
+		newMetric     *int
+		newPrefix     *netip.Prefix
+		skipCopying   bool
+		shouldCreate  bool
+		errFunc       require.ErrorAssertionFunc
+		expectedRoute *route.Route
+	}{
+		{
+			name: "Happy Path",
+			existingRoute: &route.Route{
+				ID:          "testingRoute",
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetID:       validNetID,
+				NetworkType: route.IPv4Network,
+				Peer:        peer1Key,
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     true,
+			},
+			newPeer:      &validPeer,
+			newMetric:    &validMetric,
+			newPrefix:    &validPrefix,
+			errFunc:      require.NoError,
+			shouldCreate: true,
+			expectedRoute: &route.Route{
+				ID:          "testingRoute",
+				Network:     validPrefix,
+				NetID:       validNetID,
+				NetworkType: route.IPv4Network,
+				Peer:        validPeer,
+				Description: "super",
+				Masquerade:  false,
+				Metric:      validMetric,
+				Enabled:     true,
+			},
+		},
+		{
+			name: "Bad Prefix",
+			existingRoute: &route.Route{
+				ID:          "testingRoute",
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetID:       validNetID,
+				NetworkType: route.IPv4Network,
+				Peer:        peer1Key,
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     true,
+			},
+			newPrefix: &invalidPrefix,
+			errFunc:   require.Error,
+		},
+		{
+			name: "Bad Peer",
+			existingRoute: &route.Route{
+				ID:          "testingRoute",
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetID:       validNetID,
+				NetworkType: route.IPv4Network,
+				Peer:        peer1Key,
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     true,
+			},
+			newPeer: &invalidPeer,
+			errFunc: require.Error,
+		},
+		{
+			name: "Invalid Metric",
+			existingRoute: &route.Route{
+				ID:          "testingRoute",
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetID:       validNetID,
+				NetworkType: route.IPv4Network,
+				Peer:        peer1Key,
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     true,
+			},
+			newMetric: &invalidMetric,
+			errFunc:   require.Error,
+		},
+		{
+			name: "Invalid NetID",
+			existingRoute: &route.Route{
+				ID:          "testingRoute",
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetID:       invalidNetID,
+				NetworkType: route.IPv4Network,
+				Peer:        peer1Key,
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     true,
+			},
+			newMetric: &invalidMetric,
+			errFunc:   require.Error,
+		},
+		{
+			name: "Nil Route",
+			existingRoute: &route.Route{
+				ID:          "testingRoute",
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetID:       validNetID,
+				NetworkType: route.IPv4Network,
+				Peer:        peer1Key,
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     true,
+			},
+			skipCopying: true,
+			errFunc:     require.Error,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			am, err := createRouterManager(t)
+			if err != nil {
+				t.Error("failed to create account manager")
+			}
+
+			account, err := initTestRouteAccount(t, am)
+			if err != nil {
+				t.Error("failed to init testing account")
+			}
+
+			account.Routes[testCase.existingRoute.ID] = testCase.existingRoute
+
+			err = am.Store.SaveAccount(account)
+			if err != nil {
+				t.Error("account should be saved")
+			}
+
+			var routeToSave *route.Route
+
+			if !testCase.skipCopying {
+				routeToSave = testCase.existingRoute.Copy()
+				if testCase.newPeer != nil {
+					routeToSave.Peer = *testCase.newPeer
+				}
+
+				if testCase.newMetric != nil {
+					routeToSave.Metric = *testCase.newMetric
+				}
+
+				if testCase.newPrefix != nil {
+					routeToSave.Network = *testCase.newPrefix
+				}
+			}
+
+			err = am.SaveRoute(account.Id, routeToSave)
+
+			testCase.errFunc(t, err)
+
+			if !testCase.shouldCreate {
+				return
+			}
+
+			savedRoute, saved := account.Routes[testCase.expectedRoute.ID]
+			require.True(t, saved)
+
+			if !testCase.expectedRoute.IsEqual(savedRoute) {
+				t.Errorf("new route didn't match expected route:\nGot %#v\nExpected:%#v\n", savedRoute, testCase.expectedRoute)
+			}
+
+		})
+	}
+}
+
+func TestUpdateRoute(t *testing.T) {
+	routeID := "testingRouteID"
+
+	existingRoute := &route.Route{
+		ID:          routeID,
+		Network:     netip.MustParsePrefix("192.168.0.0/16"),
+		NetID:       "superRoute",
+		NetworkType: route.IPv4Network,
+		Peer:        peer1Key,
+		Description: "super",
+		Masquerade:  false,
+		Metric:      9999,
+		Enabled:     true,
+	}
+
+	testCases := []struct {
+		name          string
+		existingRoute *route.Route
+		operations    []RouteUpdateOperation
+		shouldCreate  bool
+		errFunc       require.ErrorAssertionFunc
+		expectedRoute *route.Route
+	}{
+		{
+			name:          "Happy Path Single OPS",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRoutePeer,
+					Values: []string{peer2Key},
+				},
+			},
+			errFunc:      require.NoError,
+			shouldCreate: true,
+			expectedRoute: &route.Route{
+				ID:          routeID,
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetID:       "superRoute",
+				NetworkType: route.IPv4Network,
+				Peer:        peer2Key,
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     true,
+			},
+		},
+		{
+			name:          "Happy Path Multiple OPS",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRouteDescription,
+					Values: []string{"great"},
+				},
+				RouteUpdateOperation{
+					Type:   UpdateRouteNetwork,
+					Values: []string{"192.168.0.0/24"},
+				},
+				RouteUpdateOperation{
+					Type:   UpdateRoutePeer,
+					Values: []string{peer2Key},
+				},
+				RouteUpdateOperation{
+					Type:   UpdateRouteMetric,
+					Values: []string{"3030"},
+				},
+				RouteUpdateOperation{
+					Type:   UpdateRouteMasquerade,
+					Values: []string{"true"},
+				},
+				RouteUpdateOperation{
+					Type:   UpdateRouteEnabled,
+					Values: []string{"false"},
+				},
+				RouteUpdateOperation{
+					Type:   UpdateRouteNetworkIdentifier,
+					Values: []string{"megaRoute"},
+				},
+			},
+			errFunc:      require.NoError,
+			shouldCreate: true,
+			expectedRoute: &route.Route{
+				ID:          routeID,
+				Network:     netip.MustParsePrefix("192.168.0.0/24"),
+				NetID:       "megaRoute",
+				NetworkType: route.IPv4Network,
+				Peer:        peer2Key,
+				Description: "great",
+				Masquerade:  true,
+				Metric:      3030,
+				Enabled:     false,
+			},
+		},
+		{
+			name:          "Empty Values",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type: UpdateRoutePeer,
+				},
+			},
+			errFunc: require.Error,
+		},
+		{
+			name:          "Multiple Values",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRoutePeer,
+					Values: []string{peer2Key, peer1Key},
+				},
+			},
+			errFunc: require.Error,
+		},
+		{
+			name:          "Bad Prefix",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRouteNetwork,
+					Values: []string{"192.168.0.0/34"},
+				},
+			},
+			errFunc: require.Error,
+		},
+		{
+			name:          "Bad Peer",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRoutePeer,
+					Values: []string{"non existing Peer"},
+				},
+			},
+			errFunc: require.Error,
+		},
+		{
+			name:          "Empty Peer",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRoutePeer,
+					Values: []string{""},
+				},
+			},
+			errFunc:      require.NoError,
+			shouldCreate: true,
+			expectedRoute: &route.Route{
+				ID:          routeID,
+				Network:     netip.MustParsePrefix("192.168.0.0/16"),
+				NetID:       "superRoute",
+				NetworkType: route.IPv4Network,
+				Peer:        "",
+				Description: "super",
+				Masquerade:  false,
+				Metric:      9999,
+				Enabled:     true,
+			},
+		},
+		{
+			name:          "Large Network ID",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRouteNetworkIdentifier,
+					Values: []string{"12345678901234567890qwertyuiopqwertyuiop1"},
+				},
+			},
+			errFunc: require.Error,
+		},
+		{
+			name:          "Empty Network ID",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRouteNetworkIdentifier,
+					Values: []string{""},
+				},
+			},
+			errFunc: require.Error,
+		},
+		{
+			name:          "Invalid Metric",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRouteMetric,
+					Values: []string{"999999"},
+				},
+			},
+			errFunc: require.Error,
+		},
+		{
+			name:          "Invalid Boolean",
+			existingRoute: existingRoute,
+			operations: []RouteUpdateOperation{
+				RouteUpdateOperation{
+					Type:   UpdateRouteMasquerade,
+					Values: []string{"yes"},
+				},
+			},
+			errFunc: require.Error,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			am, err := createRouterManager(t)
+			if err != nil {
+				t.Error("failed to create account manager")
+			}
+
+			account, err := initTestRouteAccount(t, am)
+			if err != nil {
+				t.Error("failed to init testing account")
+			}
+
+			account.Routes[testCase.existingRoute.ID] = testCase.existingRoute
+
+			err = am.Store.SaveAccount(account)
+			if err != nil {
+				t.Error("account should be saved")
+			}
+
+			updatedRoute, err := am.UpdateRoute(account.Id, testCase.existingRoute.ID, testCase.operations)
+
+			testCase.errFunc(t, err)
+
+			if !testCase.shouldCreate {
+				return
+			}
+
+			testCase.expectedRoute.ID = updatedRoute.ID
+
+			if !testCase.expectedRoute.IsEqual(updatedRoute) {
+				t.Errorf("new route didn't match expected route:\nGot %#v\nExpected:%#v\n", updatedRoute, testCase.expectedRoute)
+			}
+
+		})
+	}
+}
+
+func TestDeleteRoute(t *testing.T) {
+
+	testingRoute := &route.Route{
+		ID:          "testingRoute",
+		Network:     netip.MustParsePrefix("192.168.0.0/16"),
+		NetworkType: route.IPv4Network,
+		Peer:        peer1Key,
+		Description: "super",
+		Masquerade:  false,
+		Metric:      9999,
+		Enabled:     true,
+	}
+
+	am, err := createRouterManager(t)
+	if err != nil {
+		t.Error("failed to create account manager")
+	}
+
+	account, err := initTestRouteAccount(t, am)
+	if err != nil {
+		t.Error("failed to init testing account")
+	}
+
+	account.Routes[testingRoute.ID] = testingRoute
+
+	err = am.Store.SaveAccount(account)
+	if err != nil {
+		t.Error("failed to save account")
+	}
+
+	err = am.DeleteRoute(account.Id, testingRoute.ID)
+	if err != nil {
+		t.Error("deleting route failed with error: ", err)
+	}
+
+	savedAccount, err := am.Store.GetAccount(account.Id)
+	if err != nil {
+		t.Error("failed to retrieve saved account with error: ", err)
+	}
+
+	_, found := savedAccount.Routes[testingRoute.ID]
+	if found {
+		t.Error("route shouldn't be found after delete")
+	}
+}
+
+func TestGetNetworkMap_RouteSync(t *testing.T) {
+	// no routes for peer in different groups
+	// no routes when route is deleted
+
+	baseRoute := &route.Route{
+		ID:          "testingRoute",
+		Network:     netip.MustParsePrefix("192.168.0.0/16"),
+		NetID:       "superNet",
+		NetworkType: route.IPv4Network,
+		Peer:        peer1Key,
+		Description: "super",
+		Masquerade:  false,
+		Metric:      9999,
+		Enabled:     true,
+	}
+
+	am, err := createRouterManager(t)
+	if err != nil {
+		t.Error("failed to create account manager")
+	}
+
+	account, err := initTestRouteAccount(t, am)
+	if err != nil {
+		t.Error("failed to init testing account")
+	}
+
+	newAccountRoutes, err := am.GetNetworkMap(peer1Key)
+	require.NoError(t, err)
+	require.Len(t, newAccountRoutes.Routes, 0, "new accounts should have no routes")
+
+	createdRoute, err := am.CreateRoute(account.Id, baseRoute.Network.String(), baseRoute.Peer,
+		baseRoute.Description, baseRoute.NetID, baseRoute.Masquerade, baseRoute.Metric, false)
+	require.NoError(t, err)
+
+	noDisabledRoutes, err := am.GetNetworkMap(peer1Key)
+	require.NoError(t, err)
+	require.Len(t, noDisabledRoutes.Routes, 0, "no routes for disabled routes")
+
+	enabledRoute := createdRoute.Copy()
+	enabledRoute.Enabled = true
+
+	err = am.SaveRoute(account.Id, enabledRoute)
+	require.NoError(t, err)
+
+	peer1Routes, err := am.GetNetworkMap(peer1Key)
+	require.NoError(t, err)
+	require.Len(t, peer1Routes.Routes, 1, "we should receive one route for peer1")
+	require.True(t, enabledRoute.IsEqual(peer1Routes.Routes[0]), "received route should be equal")
+
+	peer2Routes, err := am.GetNetworkMap(peer2Key)
+	require.NoError(t, err)
+	require.Len(t, peer2Routes.Routes, 1, "we should receive one route for peer2")
+	require.True(t, peer1Routes.Routes[0].IsEqual(peer2Routes.Routes[0]), "routes should be the same for peers in the same group")
+
+	newGroup := &Group{
+		ID:    xid.New().String(),
+		Name:  "peer1 group",
+		Peers: []string{peer1Key},
+	}
+	err = am.SaveGroup(account.Id, newGroup)
+	require.NoError(t, err)
+
+	rules, err := am.ListRules(account.Id)
+	require.NoError(t, err)
+
+	defaultRule := rules[0]
+	newRule := defaultRule.Copy()
+	newRule.ID = xid.New().String()
+	newRule.Name = "peer1 only"
+	newRule.Source = []string{newGroup.ID}
+	newRule.Destination = []string{newGroup.ID}
+
+	err = am.SaveRule(account.Id, newRule)
+	require.NoError(t, err)
+
+	err = am.DeleteRule(account.Id, defaultRule.ID)
+	require.NoError(t, err)
+
+	peer1GroupRoutes, err := am.GetNetworkMap(peer1Key)
+	require.NoError(t, err)
+	require.Len(t, peer1GroupRoutes.Routes, 1, "we should receive one route for peer1")
+
+	peer2GroupRoutes, err := am.GetNetworkMap(peer2Key)
+	require.NoError(t, err)
+	require.Len(t, peer2GroupRoutes.Routes, 0, "we should not receive routes for peer2")
+
+	err = am.DeleteRoute(account.Id, enabledRoute.ID)
+	require.NoError(t, err)
+
+	peer1DeletedRoute, err := am.GetNetworkMap(peer1Key)
+	require.NoError(t, err)
+	require.Len(t, peer1DeletedRoute.Routes, 0, "we should receive one route for peer1")
+
+}
+
+func createRouterManager(t *testing.T) (*DefaultAccountManager, error) {
+	store, err := createRouterStore(t)
+	if err != nil {
+		return nil, err
+	}
+	return BuildManager(store, NewPeersUpdateManager(), nil)
+}
+
+func createRouterStore(t *testing.T) (Store, error) {
+	dataDir := t.TempDir()
+	store, err := NewStore(dataDir)
+	if err != nil {
+		return nil, err
+	}
+
+	return store, nil
+}
+
+func initTestRouteAccount(t *testing.T, am *DefaultAccountManager) (*Account, error) {
+	peer1 := &Peer{
+		Key:  peer1Key,
+		Name: "test-host1@netbird.io",
+		Meta: PeerSystemMeta{
+			Hostname:  "test-host1@netbird.io",
+			GoOS:      "linux",
+			Kernel:    "Linux",
+			Core:      "21.04",
+			Platform:  "x86_64",
+			OS:        "Ubuntu",
+			WtVersion: "development",
+			UIVersion: "development",
+		},
+	}
+	peer2 := &Peer{
+		Key:  peer2Key,
+		Name: "test-host2@netbird.io",
+		Meta: PeerSystemMeta{
+			Hostname:  "test-host2@netbird.io",
+			GoOS:      "linux",
+			Kernel:    "Linux",
+			Core:      "21.04",
+			Platform:  "x86_64",
+			OS:        "Ubuntu",
+			WtVersion: "development",
+			UIVersion: "development",
+		},
+	}
+
+	accountID := "testingAcc"
+	userID := "testingUser"
+	domain := "example.com"
+
+	account := newAccountWithId(accountID, userID, domain)
+	err := am.Store.SaveAccount(account)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = am.AddPeer("", userID, peer1)
+	if err != nil {
+		return nil, err
+	}
+	_, err = am.AddPeer("", userID, peer2)
+	if err != nil {
+		return nil, err
+	}
+
+	return account, nil
+}

management/server/store.go

@@ -1,5 +1,10 @@
 package server
 
+import (
+	"github.com/netbirdio/netbird/route"
+	"net/netip"
+)
+
 type Store interface {
 	GetPeer(peerKey string) (*Peer, error)
 	DeletePeer(accountId string, peerKey string) (*Peer, error)
@@ -14,4 +19,6 @@ type Store interface {
 	GetAccountBySetupKey(setupKey string) (*Account, error)
 	GetAccountByPrivateDomain(domain string) (*Account, error)
 	SaveAccount(account *Account) error
+	GetPeerRoutes(peerKey string) ([]*route.Route, error)
+	GetRoutesByPrefix(accountID string, prefix netip.Prefix) ([]*route.Route, error)
 }

management/server/turncredentials.go

@@ -85,15 +85,18 @@ func (m *TimeBasedAuthSecretsManager) SetupRefresh(peerKey string) {
 	m.cancel(peerKey)
 	cancel := make(chan struct{}, 1)
 	m.cancelMap[peerKey] = cancel
+	log.Debugf("starting turn refresh for %s", peerKey)
+
 	go func() {
+		//we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL)
+		ticker := time.NewTicker(m.config.CredentialsTTL.Duration / 4 * 3)
+
 		for {
 			select {
 			case <-cancel:
+				log.Debugf("stopping turn refresh for %s", peerKey)
 				return
-			default:
+			case <-ticker.C:
-				//we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL)
-				time.Sleep(m.config.CredentialsTTL.Duration / 4 * 3)
-
 				c := m.GenerateCredentials()
 				var turns []*proto.ProtectedHostConfig
 				for _, host := range m.config.Turns {

management/server/updatechannel.go

@@ -6,6 +6,8 @@ import (
 	"sync"
 )
 
+const channelBufferSize = 100
+
 type UpdateMessage struct {
 	Update *proto.SyncResponse
 }
@@ -28,7 +30,12 @@ func (p *PeersUpdateManager) SendUpdate(peer string, update *UpdateMessage) erro
 	p.channelsMux.Lock()
 	defer p.channelsMux.Unlock()
 	if channel, ok := p.peerChannels[peer]; ok {
-		channel <- update
+		select {
+		case channel <- update:
+			log.Infof("update was sent to channel for peer %s", peer)
+		default:
+			log.Warnf("channel for peer %s is %d full", peer, len(channel))
+		}
 		return nil
 	}
 	log.Debugf("peer %s has no channel", peer)
@@ -45,7 +52,7 @@ func (p *PeersUpdateManager) CreateChannel(peerKey string) chan *UpdateMessage {
 		close(channel)
 	}
 	//mbragin: todo shouldn't it be more? or configurable?
-	channel := make(chan *UpdateMessage, 100)
+	channel := make(chan *UpdateMessage, channelBufferSize)
 	p.peerChannels[peerKey] = channel
 
 	log.Debugf("opened updates channel for a peer %s", peerKey)

management/server/updatechannel_test.go

@@ -3,13 +3,14 @@ package server
 import (
 	"github.com/netbirdio/netbird/management/proto"
 	"testing"
+	"time"
 )
 
-var peersUpdater *PeersUpdateManager
+//var peersUpdater *PeersUpdateManager
 
 func TestCreateChannel(t *testing.T) {
 	peer := "test-create"
-	peersUpdater = NewPeersUpdateManager()
+	peersUpdater := NewPeersUpdateManager()
 	defer peersUpdater.CloseChannel(peer)
 
 	_ = peersUpdater.CreateChannel(peer)
@@ -20,12 +21,17 @@ func TestCreateChannel(t *testing.T) {
 
 func TestSendUpdate(t *testing.T) {
 	peer := "test-sendupdate"
-	update := &UpdateMessage{Update: &proto.SyncResponse{}}
+	peersUpdater := NewPeersUpdateManager()
+	update1 := &UpdateMessage{Update: &proto.SyncResponse{
+		NetworkMap: &proto.NetworkMap{
+			Serial: 0,
+		},
+	}}
 	_ = peersUpdater.CreateChannel(peer)
 	if _, ok := peersUpdater.peerChannels[peer]; !ok {
 		t.Error("Error creating the channel")
 	}
-	err := peersUpdater.SendUpdate(peer, update)
+	err := peersUpdater.SendUpdate(peer, update1)
 	if err != nil {
 		t.Error("Error sending update: ", err)
 	}
@@ -34,10 +40,41 @@ func TestSendUpdate(t *testing.T) {
 	default:
 		t.Error("Update wasn't send")
 	}
+
+	for range [channelBufferSize]int{} {
+		err = peersUpdater.SendUpdate(peer, update1)
+		if err != nil {
+			t.Errorf("got an early error sending update: %v ", err)
+		}
+	}
+
+	update2 := &UpdateMessage{Update: &proto.SyncResponse{
+		NetworkMap: &proto.NetworkMap{
+			Serial: 10,
+		},
+	}}
+
+	err = peersUpdater.SendUpdate(peer, update2)
+	if err != nil {
+		t.Error("update shouldn't return an error when channel buffer is full")
+	}
+	timeout := time.After(5 * time.Second)
+	for range [channelBufferSize]int{} {
+		select {
+		case <-timeout:
+			t.Error("timed out reading previously sent updates")
+		case updateReader := <-peersUpdater.peerChannels[peer]:
+			if updateReader.Update.NetworkMap.Serial == update2.Update.NetworkMap.Serial {
+				t.Error("got the update that shouldn't have been sent")
+			}
+		}
+	}
+
 }
 
 func TestCloseChannel(t *testing.T) {
 	peer := "test-close"
+	peersUpdater := NewPeersUpdateManager()
 	_ = peersUpdater.CreateChannel(peer)
 	if _, ok := peersUpdater.peerChannels[peer]; !ok {
 		t.Error("Error creating the channel")

route/route.go

@@ -0,0 +1,125 @@
+package route
+
+import (
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"net/netip"
+)
+
+// Windows has some limitation regarding metric size that differ from Unix-like systems.
+// Because of that we are limiting the min and max metric size based on Windows limits:
+// see based on info from https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/route_ws2008
+const (
+	// MinMetric max metric input
+	MinMetric = 1
+	// MaxMetric max metric input
+	MaxMetric = 9999
+	// MaxNetIDChar Max Network Identifier
+	MaxNetIDChar = 40
+)
+
+const (
+	// InvalidNetworkString invalid network type string
+	InvalidNetworkString = "Invalid"
+	// IPv4NetworkString IPv4 network type string
+	IPv4NetworkString = "IPv4"
+	// IPv6NetworkString IPv6 network type string
+	IPv6NetworkString = "IPv6"
+)
+
+const (
+	// InvalidNetwork invalid network type
+	InvalidNetwork NetworkType = iota
+	// IPv4Network IPv4 network type
+	IPv4Network
+	// IPv6Network IPv6 network type
+	IPv6Network
+)
+
+// NetworkType route network type
+type NetworkType int
+
+// String returns prefix type string
+func (p NetworkType) String() string {
+	switch p {
+	case IPv4Network:
+		return IPv4NetworkString
+	case IPv6Network:
+		return IPv6NetworkString
+	default:
+		return InvalidNetworkString
+	}
+}
+
+// ToPrefixType returns a prefix type
+func ToPrefixType(prefix string) NetworkType {
+	switch prefix {
+	case IPv4NetworkString:
+		return IPv4Network
+	case IPv6NetworkString:
+		return IPv6Network
+	default:
+		return InvalidNetwork
+	}
+}
+
+// Route represents a route
+type Route struct {
+	ID          string
+	Network     netip.Prefix
+	NetID       string
+	Description string
+	Peer        string
+	NetworkType NetworkType
+	Masquerade  bool
+	Metric      int
+	Enabled     bool
+}
+
+// Copy copies a route object
+func (r *Route) Copy() *Route {
+	return &Route{
+		ID:          r.ID,
+		Description: r.Description,
+		NetID:       r.NetID,
+		Network:     r.Network,
+		NetworkType: r.NetworkType,
+		Peer:        r.Peer,
+		Metric:      r.Metric,
+		Masquerade:  r.Masquerade,
+		Enabled:     r.Enabled,
+	}
+}
+
+// IsEqual compares one route with the other
+func (r *Route) IsEqual(other *Route) bool {
+	return other.ID == r.ID &&
+		other.Description == r.Description &&
+		other.NetID == r.NetID &&
+		other.Network == r.Network &&
+		other.NetworkType == r.NetworkType &&
+		other.Peer == r.Peer &&
+		other.Metric == r.Metric &&
+		other.Masquerade == r.Masquerade &&
+		other.Enabled == r.Enabled
+}
+
+// ParseNetwork Parses a network prefix string and returns a netip.Prefix object and if is invalid, IPv4 or IPv6
+func ParseNetwork(networkString string) (NetworkType, netip.Prefix, error) {
+	prefix, err := netip.ParsePrefix(networkString)
+	if err != nil {
+		return InvalidNetwork, netip.Prefix{}, err
+	}
+
+	masked := prefix.Masked()
+
+	if !masked.IsValid() {
+		return InvalidNetwork, netip.Prefix{}, status.Errorf(codes.InvalidArgument, "invalid range %s", networkString)
+	}
+
+	if masked.Addr().Is6() {
+		return IPv6Network, masked, nil
+	}
+
+	return IPv4Network, masked, nil
+}

signal/client/client.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"fmt"
+	"github.com/netbirdio/netbird/client/system"
 	"github.com/netbirdio/netbird/signal/proto"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"io"
@@ -41,13 +42,15 @@ func UnMarshalCredential(msg *proto.Message) (*Credential, error) {
 }
 
 // MarshalCredential marsharl a Credential instance and returns a Message object
-func MarshalCredential(myKey wgtypes.Key, remoteKey wgtypes.Key, credential *Credential, t proto.Body_Type) (*proto.Message, error) {
+func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey wgtypes.Key, credential *Credential, t proto.Body_Type) (*proto.Message, error) {
 	return &proto.Message{
 		Key:       myKey.PublicKey().String(),
 		RemoteKey: remoteKey.String(),
 		Body: &proto.Body{
-			Type:    t,
+			Type:           t,
-			Payload: fmt.Sprintf("%s:%s", credential.UFrag, credential.Pwd),
+			Payload:        fmt.Sprintf("%s:%s", credential.UFrag, credential.Pwd),
+			WgListenPort:   uint32(myPort),
+			NetBirdVersion: system.NetbirdVersion(),
 		},
 	}, nil
 }

signal/client/grpc.go

@@ -139,6 +139,7 @@ func (c *GrpcClient) Receive(msgHandler func(msg *proto.Message) error) error {
 			// we need this reset because after a successful connection and a consequent error, backoff lib doesn't
 			// reset times and next try will start with a long delay
 			backOff.Reset()
+			log.Warnf("disconnected from the Signal service but will retry silently. Reason: %v", err)
 			return err
 		}
 

signal/peer/peer.go

@@ -4,6 +4,7 @@ import (
 	"github.com/netbirdio/netbird/signal/proto"
 	log "github.com/sirupsen/logrus"
 	"sync"
+	"time"
 )
 
 // Peer representation of a connected Peer
@@ -11,6 +12,8 @@ type Peer struct {
 	// a unique id of the Peer (e.g. sha256 fingerprint of the Wireguard public key)
 	Id string
 
+	StreamID int64
+
 	//a gRpc connection stream to the Peer
 	Stream proto.SignalExchange_ConnectStreamServer
 }
@@ -18,20 +21,25 @@ type Peer struct {
 // NewPeer creates a new instance of a connected Peer
 func NewPeer(id string, stream proto.SignalExchange_ConnectStreamServer) *Peer {
 	return &Peer{
-		Id:     id,
+		Id:       id,
-		Stream: stream,
+		Stream:   stream,
+		StreamID: time.Now().UnixNano(),
 	}
 }
 
-// Registry registry that holds all currently connected Peers
+// Registry that holds all currently connected Peers
 type Registry struct {
 	// Peer.key -> Peer
 	Peers sync.Map
+	// regMutex ensures that registration and de-registrations are safe
+	regMutex sync.Mutex
 }
 
 // NewRegistry creates a new connected Peer registry
 func NewRegistry() *Registry {
-	return &Registry{}
+	return &Registry{
+		regMutex: sync.Mutex{},
+	}
 }
 
 // Get gets a peer from the registry
@@ -52,20 +60,34 @@ func (registry *Registry) IsPeerRegistered(peerId string) bool {
 
 // Register registers peer in the registry
 func (registry *Registry) Register(peer *Peer) {
-	// can be that peer already exists but it is fine (e.g. reconnect)
+	registry.regMutex.Lock()
-	// todo investigate what happens to the old peer (especially Peer.Stream) when we override it
+	defer registry.regMutex.Unlock()
-	registry.Peers.Store(peer.Id, peer)
-	log.Debugf("peer registered [%s]", peer.Id)
 
-}
+	// can be that peer already exists, but it is fine (e.g. reconnect)
-
+	p, loaded := registry.Peers.LoadOrStore(peer.Id, peer)
-// Deregister deregister Peer from the Registry (usually once it disconnects)
-func (registry *Registry) Deregister(peer *Peer) {
-	_, loaded := registry.Peers.LoadAndDelete(peer.Id)
 	if loaded {
-		log.Debugf("peer deregistered [%s]", peer.Id)
+		pp := p.(*Peer)
-	} else {
+		log.Warnf("peer [%s] is already registered [new streamID %d, previous StreamID %d]. Will override stream.",
-		log.Warnf("attempted to remove non-existent peer [%s]", peer.Id)
+			peer.Id, peer.StreamID, pp.StreamID)
+		registry.Peers.Store(peer.Id, peer)
 	}
-
+	log.Debugf("peer registered [%s]", peer.Id)
+}
+
+// Deregister Peer from the Registry (usually once it disconnects)
+func (registry *Registry) Deregister(peer *Peer) {
+	registry.regMutex.Lock()
+	defer registry.regMutex.Unlock()
+
+	p, loaded := registry.Peers.LoadAndDelete(peer.Id)
+	if loaded {
+		pp := p.(*Peer)
+		if peer.StreamID < pp.StreamID {
+			registry.Peers.Store(peer.Id, p)
+			log.Warnf("attempted to remove newer registered stream of a peer [%s] [newer streamID %d, previous StreamID %d]. Ignoring.",
+				peer.Id, pp.StreamID, peer.StreamID)
+			return
+		}
+	}
+	log.Debugf("peer deregistered [%s]", peer.Id)
 }

signal/peer/peer_test.go

@@ -1,9 +1,34 @@
 package peer
 
 import (
+	"github.com/stretchr/testify/assert"
 	"testing"
+	"time"
 )
 
+func TestRegistry_ShouldNotDeregisterWhenHasNewerStreamRegistered(t *testing.T) {
+	r := NewRegistry()
+
+	peerID := "peer"
+
+	olderPeer := NewPeer(peerID, nil)
+	r.Register(olderPeer)
+	time.Sleep(time.Nanosecond)
+
+	newerPeer := NewPeer(peerID, nil)
+	r.Register(newerPeer)
+	registered, _ := r.Get(olderPeer.Id)
+
+	assert.NotNil(t, registered, "peer can't be nil")
+	assert.Equal(t, newerPeer, registered)
+
+	r.Deregister(olderPeer)
+	registered, _ = r.Get(olderPeer.Id)
+
+	assert.NotNil(t, registered, "peer can't be nil")
+	assert.Equal(t, newerPeer, registered)
+}
+
 func TestRegistry_GetNonExistentPeer(t *testing.T) {
 	r := NewRegistry()
 

signal/proto/generate.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
+go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
+protoc -I proto/ proto/signalexchange.proto --go_out=. --go-grpc_out=.

signal/proto/signalexchange.pb.go

@@ -214,6 +214,9 @@ type Body struct {
 
 	Type    Body_Type `protobuf:"varint,1,opt,name=type,proto3,enum=signalexchange.Body_Type" json:"type,omitempty"`
 	Payload string    `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"`
+	// wgListenPort is an actual WireGuard listen port
+	WgListenPort   uint32 `protobuf:"varint,3,opt,name=wgListenPort,proto3" json:"wgListenPort,omitempty"`
+	NetBirdVersion string `protobuf:"bytes,4,opt,name=netBirdVersion,proto3" json:"netBirdVersion,omitempty"`
 }
 
 func (x *Body) Reset() {
@@ -262,6 +265,20 @@ func (x *Body) GetPayload() string {
 	return ""
 }
 
+func (x *Body) GetWgListenPort() uint32 {
+	if x != nil {
+		return x.WgListenPort
+	}
+	return 0
+}
+
+func (x *Body) GetNetBirdVersion() string {
+	if x != nil {
+		return x.NetBirdVersion
+	}
+	return ""
+}
+
 var File_signalexchange_proto protoreflect.FileDescriptor
 
 var file_signalexchange_proto_rawDesc = []byte{
@@ -281,28 +298,32 @@ var file_signalexchange_proto_rawDesc = []byte{
 	0x52, 0x09, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x04, 0x62,
 	0x6f, 0x64, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x69, 0x67, 0x6e,
 	0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52,
-	0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x7d, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a,
+	0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0xc9, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d,
-	0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x73, 0x69,
+	0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x73,
-	0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64,
+	0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f,
-	0x79, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07,
+	0x64, 0x79, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a,
-	0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70,
+	0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
-	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x2c, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09,
+	0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x77, 0x67, 0x4c, 0x69, 0x73,
-	0x0a, 0x05, 0x4f, 0x46, 0x46, 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4e, 0x53,
+	0x74, 0x65, 0x6e, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x77,
-	0x57, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, 0x49, 0x44, 0x41,
+	0x67, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x6e,
-	0x54, 0x45, 0x10, 0x02, 0x32, 0xb9, 0x01, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x45,
+	0x65, 0x74, 0x42, 0x69, 0x72, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20,
-	0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x4c, 0x0a, 0x04, 0x53, 0x65, 0x6e, 0x64, 0x12,
+	0x01, 0x28, 0x09, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x42, 0x69, 0x72, 0x64, 0x56, 0x65, 0x72, 0x73,
-	0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65,
+	0x69, 0x6f, 0x6e, 0x22, 0x2c, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x4f,
-	0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67,
+	0x46, 0x46, 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4e, 0x53, 0x57, 0x45, 0x52,
-	0x65, 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e,
+	0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, 0x49, 0x44, 0x41, 0x54, 0x45, 0x10,
-	0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73,
+	0x02, 0x32, 0xb9, 0x01, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x45, 0x78, 0x63, 0x68,
-	0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x59, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74,
+	0x61, 0x6e, 0x67, 0x65, 0x12, 0x4c, 0x0a, 0x04, 0x53, 0x65, 0x6e, 0x64, 0x12, 0x20, 0x2e, 0x73,
-	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65,
+	0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e,
-	0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65,
+	0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20,
-	0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61,
+	0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e,
-	0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70,
+	0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
-	0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01,
+	0x22, 0x00, 0x12, 0x59, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x53, 0x74, 0x72,
-	0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+	0x65, 0x61, 0x6d, 0x12, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68,
-	0x6f, 0x33,
+	0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65,
+	0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78,
+	0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64,
+	0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x08, 0x5a,
+	0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

signal/proto/signalexchange.proto

@@ -49,4 +49,7 @@ message Body {
   }
   Type type = 1;
   string payload = 2;
+  // wgListenPort is an actual WireGuard listen port
+  uint32 wgListenPort = 3;
+  string netBirdVersion = 4;
 }

signal/server/signal.go

@@ -55,7 +55,7 @@ func (s *Server) ConnectStream(stream proto.SignalExchange_ConnectStreamServer)
 	}
 
 	defer func() {
-		log.Infof("peer disconnected [%s] ", p.Id)
+		log.Infof("peer disconnected [%s] [streamID %d] ", p.Id, p.StreamID)
 		s.registry.Deregister(p)
 	}()
 
@@ -66,7 +66,7 @@ func (s *Server) ConnectStream(stream proto.SignalExchange_ConnectStreamServer)
 		return err
 	}
 
-	log.Infof("peer connected [%s]", p.Id)
+	log.Infof("peer connected [%s] [streamID %d] ", p.Id, p.StreamID)
 
 	for {
 		//read incoming messages