add mysql support [WIP]

Port the conn close call to eBPF proxy

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.go]
+indent_style = tab

.github/ISSUE_TEMPLATE/bug-issue-report.md

@@ -31,9 +31,14 @@ Please specify whether you use NetBird Cloud or self-host NetBird's control plan
 
 `netbird version`
 
-**NetBird status -d output:**
+**NetBird status -dA output:**
 
-If applicable, add the `netbird status -d' command output.
+If applicable, add the `netbird status -dA' command output.
+
+**Do you face any (non-mobile) client issues?**
+
+Please provide the file created by `netbird debug for 1m -AS`.
+We advise reviewing the anonymized files for any remaining PII.
 
 **Screenshots**
 

.github/workflows/golang-test-darwin.yml

@@ -18,14 +18,14 @@ jobs:
     runs-on: macos-latest
     steps:
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: macos-go-${{ hashFiles('**/go.sum') }}

.github/workflows/golang-test-freebsd.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - name: Test in FreeBSD
@@ -21,19 +21,26 @@ jobs:
         uses: vmactions/freebsd-vm@v1
         with:
           usesh: true
+          copyback: false
+          release: "14.1"
           prepare: |
-            pkg install -y curl
+            pkg install -y go
-            pkg install -y git
 
+          # -x - to print all executed commands
+          # -e - to faile on first error
           run: |
-            set -x
+            set -e -x
-            curl -o go.tar.gz https://go.dev/dl/go1.21.11.freebsd-amd64.tar.gz -L
+            time go build -o netbird client/main.go
-            tar zxf go.tar.gz
+            # check all component except management, since we do not support management server on freebsd
-            mv go /usr/local/go
+            time go test -timeout 1m -failfast ./base62/...
-            ln -s /usr/local/go/bin/go /usr/local/bin/go
+            # NOTE: without -p1 `client/internal/dns` will fail becasue of `listen udp4 :33100: bind: address already in use`
-            go mod tidy
+            time go test -timeout 8m -failfast -p 1 ./client/...
-            go test -timeout 5m -p 1 ./iface/...
+            time go test -timeout 1m -failfast ./dns/...
-            go test -timeout 5m -p 1 ./client/... 
+            time go test -timeout 1m -failfast ./encryption/...
-            cd client
+            time go test -timeout 1m -failfast ./formatter/...
-            go build .
+            time go test -timeout 1m -failfast ./iface/...
-            cd ..
+            time go test -timeout 1m -failfast ./route/...
+            time go test -timeout 1m -failfast ./sharedsock/...
+            time go test -timeout 1m -failfast ./signal/...
+            time go test -timeout 1m -failfast ./util/...
+            time go test -timeout 1m -failfast ./version/...

.github/workflows/golang-test-linux.yml

@@ -19,13 +19,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
 
 
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -33,7 +33,7 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
@@ -49,18 +49,18 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Test
-        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 6m -p 1 ./...
 
   test_client_on_docker:
     runs-on: ubuntu-20.04
     steps:
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
 
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -68,7 +68,7 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev

.github/workflows/golang-test-windows.yml

@@ -17,13 +17,13 @@ jobs:
     runs-on: windows-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         id: go
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
 
       - name: Download wintun
         uses: carlosperate/download-file-action@v2

.github/workflows/golangci-lint.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: codespell
         uses: codespell-project/actions-codespell@v2
         with:
@@ -32,15 +32,15 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Check for duplicate constants
         if: matrix.os == 'ubuntu-latest'
         run: |
           ! awk '/const \(/,/)/{print $0}' management/server/activity/codes.go | grep -o '= [0-9]*' | sort | uniq -d | grep .
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
           cache: false
       - name: Install dependencies
         if: matrix.os == 'ubuntu-latest'

.github/workflows/install-script-test.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: run install script
         env:

.github/workflows/mobile-build-validation.yml

@@ -15,23 +15,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
       - name: Setup Android SDK
         uses: android-actions/setup-android@v3
         with:
           cmdline-tools-version: 8512546
       - name: Setup Java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: "11"
           distribution: "adopt"
       - name: NDK Cache
         id: ndk-cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /usr/local/lib/android/sdk/ndk
           key: ndk-cache-23.1.7779620
@@ -50,11 +50,11 @@ jobs:
     runs-on: macos-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
       - name: install gomobile
         run: go install golang.org/x/mobile/cmd/gomobile@v0.0.0-20240404231514-09dbf07665ed
       - name: gomobile init

.github/workflows/release.yml

@@ -10,8 +10,10 @@ on:
 
 
 env:
-  SIGN_PIPE_VER: "v0.0.11"
+  SIGN_PIPE_VER: "v0.0.14"
   GORELEASER_VER: "v1.14.1"
+  PRODUCT_NAME: "NetBird"
+  COPYRIGHT: "Wiretrustee UG (haftungsbeschreankt)"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
@@ -23,22 +25,29 @@ jobs:
     env:
       flags: ""
     steps:
+      - name: Parse semver string
+        id: semver_parser
+        uses: booxmedialtd/ws-action-parse-semver@v1
+        with:
+          input_string: ${{ (startsWith(github.ref, 'refs/tags/v') && github.ref) || 'refs/tags/v0.0.0' }}
+          version_extractor_regex: '\/v(.*)$'
+
       - if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
         run: echo "flags=--snapshot" >> $GITHUB_ENV
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # It is required for GoReleaser to work properly
       -
         name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.23"
           cache: false
       -
         name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/go/pkg/mod
@@ -68,18 +77,11 @@ jobs:
       - name: Install OS build dependencies
         run: sudo apt update && sudo apt install -y -q gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
 
-      - name: Install rsrc
+      - name: Install goversioninfo
-        run: go install github.com/akavel/rsrc@v0.10.2
+        run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@233067e
-      - name: Generate windows rsrc amd64
+      - name: Generate windows syso amd64
-        run: rsrc -arch amd64 -ico client/ui/netbird.ico -manifest client/manifest.xml -o client/resources_windows_amd64.syso
+        run: goversioninfo  -icon client/ui/netbird.ico -manifest client/manifest.xml -product-name ${{ env.PRODUCT_NAME }} -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/resources_windows_amd64.syso
-      - name: Generate windows rsrc arm64
+      - name: Run GoReleaser
-        run: rsrc -arch arm64 -ico client/ui/netbird.ico -manifest client/manifest.xml -o client/resources_windows_arm64.syso
-      - name: Generate windows rsrc arm
-        run: rsrc -arch arm -ico client/ui/netbird.ico -manifest client/manifest.xml -o client/resources_windows_arm.syso
-      - name: Generate windows rsrc 386
-        run: rsrc -arch 386 -ico client/ui/netbird.ico -manifest client/manifest.xml -o client/resources_windows_386.syso
-      -
-        name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:
           version: ${{ env.GORELEASER_VER }}
@@ -91,28 +93,28 @@ jobs:
           UPLOAD_YUM_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
       -
         name: upload non tags for debug purposes
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: release
           path: dist/
           retention-days: 3
       -
         name: upload linux packages
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: linux-packages
           path: dist/netbird_linux**
           retention-days: 3
       -
         name: upload windows packages
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: windows-packages
           path: dist/netbird_windows**
           retention-days: 3
       -
         name: upload macos packages
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: macos-packages
           path: dist/netbird_darwin**
@@ -121,20 +123,27 @@ jobs:
   release_ui:
     runs-on: ubuntu-latest
     steps:
+      - name: Parse semver string
+        id: semver_parser
+        uses: booxmedialtd/ws-action-parse-semver@v1
+        with:
+          input_string: ${{ (startsWith(github.ref, 'refs/tags/v') && github.ref) || 'refs/tags/v0.0.0' }}
+          version_extractor_regex: '\/v(.*)$'
+
       - if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
         run: echo "flags=--snapshot" >> $GITHUB_ENV
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # It is required for GoReleaser to work properly
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.23"
           cache: false
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: | 
             ~/go/pkg/mod
@@ -151,10 +160,11 @@ jobs:
 
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libappindicator3-dev gir1.2-appindicator3-0.1 libxxf86vm-dev gcc-mingw-w64-x86-64
-      - name: Install rsrc
+      - name: Install goversioninfo
-        run: go install github.com/akavel/rsrc@v0.10.2
+        run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@233067e
-      - name: Generate windows rsrc
+      - name: Generate windows syso amd64
-        run: rsrc -arch amd64 -ico client/ui/netbird.ico -manifest client/ui/manifest.xml -o client/ui/resources_windows_amd64.syso
+        run: goversioninfo -64 -icon client/ui/netbird.ico -manifest client/ui/manifest.xml -product-name ${{ env.PRODUCT_NAME }}-"UI" -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/ui/resources_windows_amd64.syso
+
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:
@@ -166,7 +176,7 @@ jobs:
           UPLOAD_DEBIAN_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
           UPLOAD_YUM_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
       - name: upload non tags for debug purposes
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: release-ui
           path: dist/
@@ -179,18 +189,18 @@ jobs:
         run: echo "flags=--snapshot" >> $GITHUB_ENV
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # It is required for GoReleaser to work properly
       -
         name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.23"
           cache: false
       -
         name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/go/pkg/mod
@@ -215,35 +225,21 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       -
         name: upload non tags for debug purposes
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: release-ui-darwin
           path: dist/
           retention-days: 3
 
-  trigger_windows_signer:
+  trigger_signer:
     runs-on: ubuntu-latest
-    needs: [release,release_ui]
+    needs: [release,release_ui,release_ui_darwin]
     if: startsWith(github.ref, 'refs/tags/')
     steps:
-      - name: Trigger Windows binaries sign pipeline
+      - name: Trigger binaries sign pipelines
         uses: benc-uk/workflow-dispatch@v1
         with:
-          workflow: Sign windows bin and installer
+          workflow: Sign bin and installer
-          repo: netbirdio/sign-pipelines
-          ref: ${{ env.SIGN_PIPE_VER }}
-          token: ${{ secrets.SIGN_GITHUB_TOKEN }}
-          inputs: '{ "tag": "${{ github.ref }}" }'
-
-  trigger_darwin_signer:
-    runs-on: ubuntu-latest
-    needs: [release,release_ui_darwin]
-    if: startsWith(github.ref, 'refs/tags/')
-    steps:
-      - name: Trigger Darwin App binaries sign pipeline
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: Sign darwin ui app with dispatch
           repo: netbirdio/sign-pipelines
           ref: ${{ env.SIGN_PIPE_VER }}
           token: ${{ secrets.SIGN_GITHUB_TOKEN }}

.github/workflows/test-infrastructure-files.yml

@@ -18,7 +18,31 @@ concurrency:
 jobs:
   test-docker-compose:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        store: [ 'sqlite', 'postgres' ]
+    services:
+      postgres:
+        image: ${{ (matrix.store == 'postgres') && 'postgres' || '' }}
+        env:
+          POSTGRES_USER: netbird
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: netbird
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+        ports:
+          - 5432:5432
     steps:
+      - name: Set Database Connection String
+        run: |
+          if [ "${{ matrix.store }}" == "postgres" ]; then
+            echo "NETBIRD_STORE_ENGINE_POSTGRES_DSN=host=$(hostname -I | awk '{print $1}') user=netbird password=postgres dbname=netbird port=5432" >> $GITHUB_ENV
+          else
+            echo "NETBIRD_STORE_ENGINE_POSTGRES_DSN==" >> $GITHUB_ENV
+          fi
+
       - name: Install jq
         run: sudo apt-get install -y jq
 
@@ -26,12 +50,12 @@ jobs:
         run: sudo apt-get install -y curl
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
 
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -39,7 +63,7 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: cp setup.env
         run: cp infrastructure_files/tests/setup.env infrastructure_files/
@@ -58,7 +82,8 @@ jobs:
           CI_NETBIRD_IDP_MGMT_CLIENT_ID: testing.client.id
           CI_NETBIRD_IDP_MGMT_CLIENT_SECRET: testing.client.secret
           CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified"
-          CI_NETBIRD_STORE_CONFIG_ENGINE: "sqlite"
+          CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }}
+          NETBIRD_STORE_ENGINE_POSTGRES_DSN: ${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }}
           CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
 
       - name: check values
@@ -85,7 +110,8 @@ jobs:
           CI_NETBIRD_IDP_MGMT_CLIENT_ID: testing.client.id
           CI_NETBIRD_IDP_MGMT_CLIENT_SECRET: testing.client.secret
           CI_NETBIRD_SIGNAL_PORT: 12345
-          CI_NETBIRD_STORE_CONFIG_ENGINE: "sqlite"
+          CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }}
+          NETBIRD_STORE_ENGINE_POSTGRES_DSN: '${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }}$'
           CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
           CI_NETBIRD_TURN_EXTERNAL_IP: "1.2.3.4"
 
@@ -123,6 +149,14 @@ jobs:
           grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep Scope | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES"
           grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep -A 3 RedirectURLs | grep "http://localhost:53000"
           grep "external-ip" turnserver.conf | grep $CI_NETBIRD_TURN_EXTERNAL_IP
+          grep NETBIRD_STORE_ENGINE_POSTGRES_DSN docker-compose.yml | egrep "$NETBIRD_STORE_ENGINE_POSTGRES_DSN"
+          # check relay values
+          grep "NB_EXPOSED_ADDRESS=$CI_NETBIRD_DOMAIN:33445" docker-compose.yml
+          grep "NB_LISTEN_ADDRESS=:33445" docker-compose.yml
+          grep '33445:33445' docker-compose.yml
+          grep -A 10 'relay:' docker-compose.yml | egrep 'NB_AUTH_SECRET=.+$'
+          grep -A 7 Relay management.json | grep "rel://$CI_NETBIRD_DOMAIN:33445"
+          grep -A 7 Relay management.json | egrep '"Secret": ".+"'
 
       - name: Install modules
         run: go mod tidy
@@ -148,26 +182,35 @@ jobs:
         run: |
           docker build -t netbirdio/signal:latest .
 
+      - name: Build relay binary
+        working-directory: relay
+        run: CGO_ENABLED=0 go build -o netbird-relay main.go
+
+      - name: Build relay docker image
+        working-directory: relay
+        run: |
+          docker build -t netbirdio/relay:latest .
+
       - name: run docker compose up
         working-directory: infrastructure_files/artifacts
         run: |
-          docker-compose up -d
+          docker compose up -d
           sleep 5
-          docker-compose ps
+          docker compose ps
-          docker-compose logs --tail=20
+          docker compose logs --tail=20
 
       - name: test running containers
         run: |
           count=$(docker compose ps --format json | jq '. | select(.Name | contains("artifacts")) | .State' | grep -c running)
-          test $count -eq 4
+          test $count -eq 5 || docker compose logs
         working-directory: infrastructure_files/artifacts
 
       - name: test geolocation databases
         working-directory: infrastructure_files/artifacts
         run: |
           sleep 30
-          docker compose exec management ls -l /var/lib/netbird/ | grep -i GeoLite2-City.mmdb
+          docker compose exec management ls -l /var/lib/netbird/ | grep -i GeoLite2-City_[0-9]*.mmdb
-          docker compose exec management ls -l /var/lib/netbird/ | grep -i geonames.db
+          docker compose exec management ls -l /var/lib/netbird/ | grep -i geonames_[0-9]*.db
 
   test-getting-started-script:
     runs-on: ubuntu-latest
@@ -176,7 +219,7 @@ jobs:
         run: sudo apt-get install -y jq
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: run script with Zitadel PostgreSQL
         run: NETBIRD_DOMAIN=use-ip bash -x infrastructure_files/getting-started-with-zitadel.sh
@@ -202,12 +245,15 @@ jobs:
       - name: test dashboard.env file gen postgres
         run: test -f dashboard.env
 
+      - name: test relay.env file gen postgres
+        run: test -f relay.env
+
       - name: test zdb.env file gen postgres
         run: test -f zdb.env
 
       - name: Postgres run cleanup
         run: |
-          docker-compose down --volumes --rmi all
+          docker compose down --volumes --rmi all
           rm -rf docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json zdb.env
 
       - name: run script with Zitadel CockroachDB
@@ -237,20 +283,5 @@ jobs:
       - name: test dashboard.env file gen CockroachDB
         run: test -f dashboard.env
 
-  test-download-geolite2-script:
+      - name: test relay.env file gen CockroachDB
-    runs-on: ubuntu-latest
+        run: test -f relay.env
-    steps:
-      - name: Install jq
-        run: sudo apt-get update && sudo apt-get install -y unzip sqlite3
-
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: test script
-        run: bash -x infrastructure_files/download-geolite2.sh
-
-      - name: test mmdb file exists
-        run: test -f GeoLite2-City.mmdb
-
-      - name: test geonames file exists
-        run: test -f geonames.db

.gitignore

@@ -29,4 +29,3 @@ infrastructure_files/setup.env
 infrastructure_files/setup-*.env
 .vscode
 .DS_Store
-GeoLite2-City*

.goreleaser.yaml

@@ -80,6 +80,20 @@ builds:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
     mod_timestamp: '{{ .CommitTimestamp }}'
 
+  - id: netbird-relay
+    dir: relay
+    env: [CGO_ENABLED=0]
+    binary: netbird-relay
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+      - arm
+    ldflags:
+      - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
+    mod_timestamp: '{{ .CommitTimestamp }}'
+
 archives:
   - builds:
       - netbird
@@ -161,6 +175,52 @@ dockers:
       - "--label=org.opencontainers.image.revision={{.FullCommit}}"
       - "--label=org.opencontainers.image.version={{.Version}}"
       - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-amd64
+    ids:
+      - netbird-relay
+    goarch: amd64
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/amd64"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+    ids:
+      - netbird-relay
+    goarch: arm64
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/arm64"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-arm
+    ids:
+      - netbird-relay
+    goarch: arm
+    goarm: 6
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/arm"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
   - image_templates:
       - netbirdio/signal:{{ .Version }}-amd64
     ids:
@@ -313,6 +373,18 @@ docker_manifests:
       - netbirdio/netbird:{{ .Version }}-arm
       - netbirdio/netbird:{{ .Version }}-amd64
 
+  - name_template: netbirdio/relay:{{ .Version }}
+    image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+      - netbirdio/relay:{{ .Version }}-arm
+      - netbirdio/relay:{{ .Version }}-amd64
+
+  - name_template: netbirdio/relay:latest
+    image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+      - netbirdio/relay:{{ .Version }}-arm
+      - netbirdio/relay:{{ .Version }}-amd64
+
   - name_template: netbirdio/signal:{{ .Version }}
     image_templates:
       - netbirdio/signal:{{ .Version }}-arm64v8

.goreleaser_ui.yaml

@@ -11,8 +11,6 @@ builds:
       - amd64
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    tags:
-      - legacy_appindicator
     mod_timestamp: '{{ .CommitTimestamp }}'
 
   - id: netbird-ui-windows

README.md

@@ -10,12 +10,14 @@
   <img width="234" src="docs/media/logo-full.png"/>
 </p>
   <p>
+   <a href="https://img.shields.io/badge/license-BSD--3-blue)">
+       <img src="https://sonarcloud.io/api/project_badges/measure?project=netbirdio_netbird&metric=alert_status" />
+     </a> 
      <a href="https://github.com/netbirdio/netbird/blob/main/LICENSE">
        <img src="https://img.shields.io/badge/license-BSD--3-blue" />
      </a> 
-   <a href="https://www.codacy.com/gh/netbirdio/netbird/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=netbirdio/netbird&amp;utm_campaign=Badge_Grade"><img src="https://app.codacy.com/project/badge/Grade/e3013d046aec44cdb7462c8673b00976"/></a>
     <br>
-    <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-vrahf41g-ik1v7fV8du6t0RwxSrJ96A">
+    <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2p5zwhm4g-8fHollzrQa5y4PZF5AEpvQ">
         <img src="https://img.shields.io/badge/slack-@netbird-red.svg?logo=slack"/>
      </a>    
   </p>
@@ -28,7 +30,7 @@
   <br/>
   See <a href="https://netbird.io/docs/">Documentation</a>
   <br/>
-   Join our <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-vrahf41g-ik1v7fV8du6t0RwxSrJ96A">Slack channel</a>
+   Join our <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2p5zwhm4g-8fHollzrQa5y4PZF5AEpvQ">Slack channel</a>
   <br/>
  
 </strong>

client/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM alpine:3.20
 RUN apk add --no-cache ca-certificates iptables ip6tables
 ENV NB_FOREGROUND_MODE=true
 ENTRYPOINT [ "/usr/local/bin/netbird","up"]

client/android/login.go

@@ -84,7 +84,7 @@ func (a *Auth) SaveConfigIfSSOSupported(listener SSOListener) {
 func (a *Auth) saveConfigIfSSOSupported() (bool, error) {
 	supportsSSO := true
 	err := a.withBackOff(a.ctx, func() (err error) {
-		_, err = internal.GetPKCEAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL)
+		_, err = internal.GetPKCEAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL, nil)
 		if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.NotFound || s.Code() == codes.Unimplemented) {
 			_, err = internal.GetDeviceAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL)
 			s, ok := gstatus.FromError(err)

client/anonymize/anonymize.go

@@ -178,6 +178,21 @@ func (a *Anonymizer) AnonymizeDNSLogLine(logEntry string) string {
 	})
 }
 
+// AnonymizeRoute anonymizes a route string by replacing IP addresses with anonymized versions and
+// domain names with random strings.
+func (a *Anonymizer) AnonymizeRoute(route string) string {
+	prefix, err := netip.ParsePrefix(route)
+	if err == nil {
+		ip := a.AnonymizeIPString(prefix.Addr().String())
+		return fmt.Sprintf("%s/%d", ip, prefix.Bits())
+	}
+	domains := strings.Split(route, ", ")
+	for i, domain := range domains {
+		domains[i] = a.AnonymizeDomain(domain)
+	}
+	return strings.Join(domains, ", ")
+}
+
 func isWellKnown(addr netip.Addr) bool {
 	wellKnown := []string{
 		"8.8.8.8", "8.8.4.4", // Google DNS IPv4

client/cmd/debug.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc/status"
 
@@ -13,6 +14,8 @@ import (
 	"github.com/netbirdio/netbird/client/server"
 )
 
+const errCloseConnection = "Failed to close connection: %v"
+
 var debugCmd = &cobra.Command{
 	Use:   "debug",
 	Short: "Debugging commands",
@@ -63,12 +66,17 @@ func debugBundle(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
 
 	client := proto.NewDaemonServiceClient(conn)
 	resp, err := client.DebugBundle(cmd.Context(), &proto.DebugBundleRequest{
 		Anonymize:  anonymizeFlag,
 		Status:     getStatusOutput(cmd),
+		SystemInfo: debugSystemInfoFlag,
 	})
 	if err != nil {
 		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
@@ -84,7 +92,11 @@ func setLogLevel(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
 
 	client := proto.NewDaemonServiceClient(conn)
 	level := server.ParseLogLevel(args[0])
@@ -113,7 +125,11 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
 
 	client := proto.NewDaemonServiceClient(conn)
 
@@ -122,17 +138,20 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to get status: %v", status.Convert(err).Message())
 	}
 
-	restoreUp := stat.Status == string(internal.StatusConnected) || stat.Status == string(internal.StatusConnecting)
+	stateWasDown := stat.Status != string(internal.StatusConnected) && stat.Status != string(internal.StatusConnecting)
 
 	initialLogLevel, err := client.GetLogLevel(cmd.Context(), &proto.GetLogLevelRequest{})
 	if err != nil {
 		return fmt.Errorf("failed to get log level: %v", status.Convert(err).Message())
 	}
 
-	if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
+	if stateWasDown {
-		return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
+		if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
+			return fmt.Errorf("failed to up: %v", status.Convert(err).Message())
+		}
+		cmd.Println("Netbird up")
+		time.Sleep(time.Second * 10)
 	}
-	cmd.Println("Netbird down")
 
 	initialLevelTrace := initialLogLevel.GetLevel() >= proto.LogLevel_TRACE
 	if !initialLevelTrace {
@@ -145,6 +164,11 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		cmd.Println("Log level set to trace.")
 	}
 
+	if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
+		return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
+	}
+	cmd.Println("Netbird down")
+
 	time.Sleep(1 * time.Second)
 
 	if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
@@ -162,21 +186,25 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 	}
 	cmd.Println("\nDuration completed")
 
+	cmd.Println("Creating debug bundle...")
+
 	headerPreDown := fmt.Sprintf("----- Netbird pre-down - Timestamp: %s - Duration: %s", time.Now().Format(time.RFC3339), duration)
 	statusOutput = fmt.Sprintf("%s\n%s\n%s", statusOutput, headerPreDown, getStatusOutput(cmd))
 
+	resp, err := client.DebugBundle(cmd.Context(), &proto.DebugBundleRequest{
+		Anonymize:  anonymizeFlag,
+		Status:     statusOutput,
+		SystemInfo: debugSystemInfoFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
+	}
+
+	if stateWasDown {
 		if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
 			return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
 		}
 		cmd.Println("Netbird down")
-
-	time.Sleep(1 * time.Second)
-
-	if restoreUp {
-		if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
-			return fmt.Errorf("failed to up: %v", status.Convert(err).Message())
-		}
-		cmd.Println("Netbird up")
 	}
 
 	if !initialLevelTrace {
@@ -186,16 +214,6 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		cmd.Println("Log level restored to", initialLogLevel.GetLevel())
 	}
 
-	cmd.Println("Creating debug bundle...")
-
-	resp, err := client.DebugBundle(cmd.Context(), &proto.DebugBundleRequest{
-		Anonymize: anonymizeFlag,
-		Status:    statusOutput,
-	})
-	if err != nil {
-		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
-	}
-
 	cmd.Println(resp.GetPath())
 
 	return nil

client/cmd/down.go

@@ -26,7 +26,7 @@ var downCmd = &cobra.Command{
 			return err
 		}
 
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*7)
 		defer cancel()
 
 		conn, err := DialClientGRPCServer(ctx, daemonAddr)
@@ -42,6 +42,8 @@ var downCmd = &cobra.Command{
 			log.Errorf("call service down method: %v", err)
 			return err
 		}
+
+		cmd.Println("Disconnected")
 		return nil
 	},
 }

client/cmd/login.go

@@ -39,6 +39,11 @@ var loginCmd = &cobra.Command{
 			ctx = context.WithValue(ctx, system.DeviceNameCtxKey, hostName)
 		}
 
+		providedSetupKey, err := getSetupKey()
+		if err != nil {
+			return err
+		}
+
 		// workaround to run without service
 		if logFile == "console" {
 			err = handleRebrand(cmd)
@@ -62,7 +67,7 @@ var loginCmd = &cobra.Command{
 
 			config, _ = internal.UpdateOldManagementURL(ctx, config, configPath)
 
-			err = foregroundLogin(ctx, cmd, config, setupKey)
+			err = foregroundLogin(ctx, cmd, config, providedSetupKey)
 			if err != nil {
 				return fmt.Errorf("foreground login failed: %v", err)
 			}
@@ -81,7 +86,7 @@ var loginCmd = &cobra.Command{
 		client := proto.NewDaemonServiceClient(conn)
 
 		loginRequest := proto.LoginRequest{
-			SetupKey:             setupKey,
+			SetupKey:             providedSetupKey,
 			ManagementUrl:        managementURL,
 			IsLinuxDesktopClient: isLinuxRunningDesktop(),
 			Hostname:             hostName,

client/cmd/root.go

@@ -37,6 +37,7 @@ const (
 	serverSSHAllowedFlag    = "allow-server-ssh"
 	extraIFaceBlackListFlag = "extra-iface-blacklist"
 	dnsRouteIntervalFlag    = "dns-router-interval"
+	systemInfoFlag          = "system-info"
 )
 
 var (
@@ -55,6 +56,7 @@ var (
 	managementURL           string
 	adminURL                string
 	setupKey                string
+	setupKeyPath            string
 	hostName                string
 	preSharedKey            string
 	natExternalIPs          []string
@@ -69,6 +71,7 @@ var (
 	autoConnectDisabled     bool
 	extraIFaceBlackList     []string
 	anonymizeFlag           bool
+	debugSystemInfoFlag     bool
 	dnsRouteInterval        time.Duration
 
 	rootCmd = &cobra.Command{
@@ -91,12 +94,15 @@ func init() {
 	oldDefaultConfigPathDir = "/etc/wiretrustee/"
 	oldDefaultLogFileDir = "/var/log/wiretrustee/"
 
-	if runtime.GOOS == "windows" {
+	switch runtime.GOOS {
+	case "windows":
 		defaultConfigPathDir = os.Getenv("PROGRAMDATA") + "\\Netbird\\"
 		defaultLogFileDir = os.Getenv("PROGRAMDATA") + "\\Netbird\\"
 
 		oldDefaultConfigPathDir = os.Getenv("PROGRAMDATA") + "\\Wiretrustee\\"
 		oldDefaultLogFileDir = os.Getenv("PROGRAMDATA") + "\\Wiretrustee\\"
+	case "freebsd":
+		defaultConfigPathDir = "/var/db/netbird/"
 	}
 
 	defaultConfigPath = defaultConfigPathDir + "config.json"
@@ -121,8 +127,10 @@ func init() {
 	rootCmd.PersistentFlags().StringVarP(&serviceName, "service", "s", defaultServiceName, "Netbird system service name")
 	rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", defaultConfigPath, "Netbird config file location")
 	rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "info", "sets Netbird log level")
-	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", defaultLogFile, "sets Netbird log path. If console is specified the log will be output to stdout")
+	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", defaultLogFile, "sets Netbird log path. If console is specified the log will be output to stdout. If syslog is specified the log will be sent to syslog daemon.")
 	rootCmd.PersistentFlags().StringVarP(&setupKey, "setup-key", "k", "", "Setup key obtained from the Management Service Dashboard (used to register peer)")
+	rootCmd.PersistentFlags().StringVar(&setupKeyPath, "setup-key-file", "", "The path to a setup key obtained from the Management Service Dashboard (used to register peer) This is ignored if the setup-key flag is provided.")
+	rootCmd.MarkFlagsMutuallyExclusive("setup-key", "setup-key-file")
 	rootCmd.PersistentFlags().StringVar(&preSharedKey, preSharedKeyFlag, "", "Sets Wireguard PreSharedKey property. If set, then only peers that have the same key can communicate.")
 	rootCmd.PersistentFlags().StringVarP(&hostName, "hostname", "n", "", "Sets a custom hostname for the device")
 	rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize IP addresses and non-netbird.io domains in logs and status output")
@@ -165,6 +173,8 @@ func init() {
 	upCmd.PersistentFlags().BoolVar(&rosenpassPermissive, rosenpassPermissiveFlag, false, "[Experimental] Enable Rosenpass in permissive mode to allow this peer to accept WireGuard connections without requiring Rosenpass functionality from peers that do not have Rosenpass enabled.")
 	upCmd.PersistentFlags().BoolVar(&serverSSHAllowed, serverSSHAllowedFlag, false, "Allow SSH server on peer. If enabled, the SSH server will be permitted")
 	upCmd.PersistentFlags().BoolVar(&autoConnectDisabled, disableAutoConnectFlag, false, "Disables auto-connect feature. If enabled, then the client won't connect automatically when the service starts.")
+
+	debugCmd.PersistentFlags().BoolVarP(&debugSystemInfoFlag, systemInfoFlag, "S", false, "Adds system information to the debug bundle")
 }
 
 // SetupCloseHandler handles SIGTERM signal and exits with success
@@ -246,6 +256,21 @@ var CLIBackOffSettings = &backoff.ExponentialBackOff{
 	Clock:               backoff.SystemClock,
 }
 
+func getSetupKey() (string, error) {
+	if setupKeyPath != "" && setupKey == "" {
+		return getSetupKeyFromFile(setupKeyPath)
+	}
+	return setupKey, nil
+}
+
+func getSetupKeyFromFile(setupKeyPath string) (string, error) {
+	data, err := os.ReadFile(setupKeyPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read setup key file: %v", err)
+	}
+	return strings.TrimSpace(string(data)), nil
+}
+
 func handleRebrand(cmd *cobra.Command) error {
 	var err error
 	if logFile == defaultLogFile {

client/cmd/root_test.go

@@ -4,6 +4,10 @@ import (
 	"fmt"
 	"io"
 	"testing"
+
+	"github.com/spf13/cobra"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 func TestInitCommands(t *testing.T) {
@@ -34,3 +38,44 @@ func TestInitCommands(t *testing.T) {
 		})
 	}
 }
+
+func TestSetFlagsFromEnvVars(t *testing.T) {
+	var cmd = &cobra.Command{
+		Use:          "netbird",
+		Long:         "test",
+		SilenceUsage: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			SetFlagsFromEnvVars(cmd)
+		},
+	}
+
+	cmd.PersistentFlags().StringSliceVar(&natExternalIPs, externalIPMapFlag, nil,
+		`comma separated list of external IPs to map to the Wireguard interface`)
+	cmd.PersistentFlags().StringVar(&interfaceName, interfaceNameFlag, iface.WgInterfaceDefault, "Wireguard interface name")
+	cmd.PersistentFlags().BoolVar(&rosenpassEnabled, enableRosenpassFlag, false, "Enable Rosenpass feature Rosenpass.")
+	cmd.PersistentFlags().Uint16Var(&wireguardPort, wireguardPortFlag, iface.DefaultWgPort, "Wireguard interface listening port")
+
+	t.Setenv("NB_EXTERNAL_IP_MAP", "abc,dec")
+	t.Setenv("NB_INTERFACE_NAME", "test-name")
+	t.Setenv("NB_ENABLE_ROSENPASS", "true")
+	t.Setenv("NB_WIREGUARD_PORT", "10000")
+	err := cmd.Execute()
+	if err != nil {
+		t.Fatalf("expected no error while running netbird command, got %v", err)
+	}
+	if len(natExternalIPs) != 2 {
+		t.Errorf("expected 2 external ips, got %d", len(natExternalIPs))
+	}
+	if natExternalIPs[0] != "abc" || natExternalIPs[1] != "dec" {
+		t.Errorf("expected abc,dec, got %s,%s", natExternalIPs[0], natExternalIPs[1])
+	}
+	if interfaceName != "test-name" {
+		t.Errorf("expected test-name, got %s", interfaceName)
+	}
+	if !rosenpassEnabled {
+		t.Errorf("expected rosenpassEnabled to be true, got false")
+	}
+	if wireguardPort != 10000 {
+		t.Errorf("expected wireguardPort to be 10000, got %d", wireguardPort)
+	}
+}

client/cmd/service.go

@@ -2,18 +2,21 @@ package cmd
 
 import (
 	"context"
+
 	"github.com/kardianos/service"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
 
 	"github.com/netbirdio/netbird/client/internal"
+	"github.com/netbirdio/netbird/client/server"
 )
 
 type program struct {
 	ctx            context.Context
 	cancel         context.CancelFunc
 	serv           *grpc.Server
+	serverInstance *server.Server
 }
 
 func newProgram(ctx context.Context, cancel context.CancelFunc) *program {

client/cmd/service_controller.go

@@ -61,6 +61,8 @@ func (p *program) Start(svc service.Service) error {
 		}
 		proto.RegisterDaemonServiceServer(p.serv, serverInstance)
 
+		p.serverInstance = serverInstance
+
 		log.Printf("started daemon server: %v", split[1])
 		if err := p.serv.Serve(listen); err != nil {
 			log.Errorf("failed to serve daemon requests: %v", err)
@@ -70,6 +72,14 @@ func (p *program) Start(svc service.Service) error {
 }
 
 func (p *program) Stop(srv service.Service) error {
+	if p.serverInstance != nil {
+		in := new(proto.DownRequest)
+		_, err := p.serverInstance.Down(p.ctx, in)
+		if err != nil {
+			log.Errorf("failed to stop daemon: %v", err)
+		}
+	}
+
 	p.cancel()
 
 	if p.serv != nil {

client/cmd/service_installer.go

@@ -31,6 +31,8 @@ var installCmd = &cobra.Command{
 			configPath,
 			"--log-level",
 			logLevel,
+			"--daemon-addr",
+			daemonAddr,
 		}
 
 		if managementURL != "" {

client/cmd/status.go

@@ -31,9 +31,9 @@ type peerStateDetailOutput struct {
 	Status                 string           `json:"status" yaml:"status"`
 	LastStatusUpdate       time.Time        `json:"lastStatusUpdate" yaml:"lastStatusUpdate"`
 	ConnType               string           `json:"connectionType" yaml:"connectionType"`
-	Direct                 bool             `json:"direct" yaml:"direct"`
 	IceCandidateType       iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"`
 	IceCandidateEndpoint   iceCandidateType `json:"iceCandidateEndpoint" yaml:"iceCandidateEndpoint"`
+	RelayAddress           string           `json:"relayAddress" yaml:"relayAddress"`
 	LastWireguardHandshake time.Time        `json:"lastWireguardHandshake" yaml:"lastWireguardHandshake"`
 	TransferReceived       int64            `json:"transferReceived" yaml:"transferReceived"`
 	TransferSent           int64            `json:"transferSent" yaml:"transferSent"`
@@ -335,16 +335,18 @@ func mapNSGroups(servers []*proto.NSGroupState) []nsServerGroupStateOutput {
 
 func mapPeers(peers []*proto.PeerState) peersStateOutput {
 	var peersStateDetail []peerStateDetailOutput
+	peersConnected := 0
+	for _, pbPeerState := range peers {
 		localICE := ""
 		remoteICE := ""
 		localICEEndpoint := ""
 		remoteICEEndpoint := ""
+		relayServerAddress := ""
 		connType := ""
-	peersConnected := 0
 		lastHandshake := time.Time{}
 		transferReceived := int64(0)
 		transferSent := int64(0)
-	for _, pbPeerState := range peers {
+
 		isPeerConnected := pbPeerState.ConnStatus == peer.StatusConnected.String()
 		if skipDetailByFilters(pbPeerState, isPeerConnected) {
 			continue
@@ -360,6 +362,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 			if pbPeerState.Relayed {
 				connType = "Relayed"
 			}
+			relayServerAddress = pbPeerState.GetRelayAddress()
 			lastHandshake = pbPeerState.GetLastWireguardHandshake().AsTime().Local()
 			transferReceived = pbPeerState.GetBytesRx()
 			transferSent = pbPeerState.GetBytesTx()
@@ -372,7 +375,6 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 			Status:           pbPeerState.GetConnStatus(),
 			LastStatusUpdate: timeLocal,
 			ConnType:         connType,
-			Direct:           pbPeerState.GetDirect(),
 			IceCandidateType: iceCandidateType{
 				Local:  localICE,
 				Remote: remoteICE,
@@ -381,6 +383,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 				Local:  localICEEndpoint,
 				Remote: remoteICEEndpoint,
 			},
+			RelayAddress:           relayServerAddress,
 			FQDN:                   pbPeerState.GetFqdn(),
 			LastWireguardHandshake: lastHandshake,
 			TransferReceived:       transferReceived,
@@ -641,9 +644,9 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 				"  Status: %s\n"+
 				"  -- detail --\n"+
 				"  Connection type: %s\n"+
-				"  Direct: %t\n"+
 				"  ICE candidate (Local/Remote): %s/%s\n"+
 				"  ICE candidate endpoints (Local/Remote): %s/%s\n"+
+				"  Relay server address: %s\n"+
 				"  Last connection update: %s\n"+
 				"  Last WireGuard handshake: %s\n"+
 				"  Transfer status (received/sent) %s/%s\n"+
@@ -655,11 +658,11 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 			peerState.PubKey,
 			peerState.Status,
 			peerState.ConnType,
-			peerState.Direct,
 			localICE,
 			remoteICE,
 			localICEEndpoint,
 			remoteICEEndpoint,
+			peerState.RelayAddress,
 			timeAgo(peerState.LastStatusUpdate),
 			timeAgo(peerState.LastWireguardHandshake),
 			toIEC(peerState.TransferReceived),
@@ -802,12 +805,15 @@ func anonymizePeerDetail(a *anonymize.Anonymizer, peer *peerStateDetailOutput) {
 	if remoteIP, port, err := net.SplitHostPort(peer.IceCandidateEndpoint.Remote); err == nil {
 		peer.IceCandidateEndpoint.Remote = fmt.Sprintf("%s:%s", a.AnonymizeIPString(remoteIP), port)
 	}
+
+	peer.RelayAddress = a.AnonymizeURI(peer.RelayAddress)
+
 	for i, route := range peer.Routes {
 		peer.Routes[i] = a.AnonymizeIPString(route)
 	}
 
 	for i, route := range peer.Routes {
-		peer.Routes[i] = anonymizeRoute(a, route)
+		peer.Routes[i] = a.AnonymizeRoute(route)
 	}
 }
 
@@ -843,21 +849,8 @@ func anonymizeOverview(a *anonymize.Anonymizer, overview *statusOutputOverview)
 	}
 
 	for i, route := range overview.Routes {
-		overview.Routes[i] = anonymizeRoute(a, route)
+		overview.Routes[i] = a.AnonymizeRoute(route)
 	}
 
 	overview.FQDN = a.AnonymizeDomain(overview.FQDN)
 }
-
-func anonymizeRoute(a *anonymize.Anonymizer, route string) string {
-	prefix, err := netip.ParsePrefix(route)
-	if err == nil {
-		ip := a.AnonymizeIPString(prefix.Addr().String())
-		return fmt.Sprintf("%s/%d", ip, prefix.Bits())
-	}
-	domains := strings.Split(route, ", ")
-	for i, domain := range domains {
-		domains[i] = a.AnonymizeDomain(domain)
-	}
-	return strings.Join(domains, ", ")
-}

client/cmd/status_test.go

@@ -37,7 +37,6 @@ var resp = &proto.StatusResponse{
 				ConnStatus:                 "Connected",
 				ConnStatusUpdate:           timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 1, 0, time.UTC)),
 				Relayed:                    false,
-				Direct:                     true,
 				LocalIceCandidateType:      "",
 				RemoteIceCandidateType:     "",
 				LocalIceCandidateEndpoint:  "",
@@ -57,7 +56,6 @@ var resp = &proto.StatusResponse{
 				ConnStatus:                 "Connected",
 				ConnStatusUpdate:           timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 2, 0, time.UTC)),
 				Relayed:                    true,
-				Direct:                     false,
 				LocalIceCandidateType:      "relay",
 				RemoteIceCandidateType:     "prflx",
 				LocalIceCandidateEndpoint:  "10.0.0.1:10001",
@@ -137,7 +135,6 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2001, 1, 1, 1, 1, 1, 0, time.UTC),
 				ConnType:         "P2P",
-				Direct:           true,
 				IceCandidateType: iceCandidateType{
 					Local:  "",
 					Remote: "",
@@ -161,7 +158,6 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2002, 2, 2, 2, 2, 2, 0, time.UTC),
 				ConnType:         "Relayed",
-				Direct:           false,
 				IceCandidateType: iceCandidateType{
 					Local:  "relay",
 					Remote: "prflx",
@@ -283,7 +279,6 @@ func TestParsingToJSON(t *testing.T) {
                 "status": "Connected",
                 "lastStatusUpdate": "2001-01-01T01:01:01Z",
                 "connectionType": "P2P",
-                "direct": true,
                 "iceCandidateType": {
                   "local": "",
                   "remote": ""
@@ -292,6 +287,7 @@ func TestParsingToJSON(t *testing.T) {
                   "local": "",
                   "remote": ""
                 },
+				"relayAddress": "",
                 "lastWireguardHandshake": "2001-01-01T01:01:02Z",
                 "transferReceived": 200,
                 "transferSent": 100,
@@ -308,7 +304,6 @@ func TestParsingToJSON(t *testing.T) {
                 "status": "Connected",
                 "lastStatusUpdate": "2002-02-02T02:02:02Z",
                 "connectionType": "Relayed",
-                "direct": false,
                 "iceCandidateType": {
                   "local": "relay",
                   "remote": "prflx"
@@ -317,6 +312,7 @@ func TestParsingToJSON(t *testing.T) {
                   "local": "10.0.0.1:10001",
                   "remote": "10.0.10.1:10002"
                 },
+				"relayAddress": "",
                 "lastWireguardHandshake": "2002-02-02T02:02:03Z",
                 "transferReceived": 2000,
                 "transferSent": 1000,
@@ -408,13 +404,13 @@ func TestParsingToYAML(t *testing.T) {
           status: Connected
           lastStatusUpdate: 2001-01-01T01:01:01Z
           connectionType: P2P
-          direct: true
           iceCandidateType:
             local: ""
             remote: ""
           iceCandidateEndpoint:
             local: ""
             remote: ""
+          relayAddress: ""
           lastWireguardHandshake: 2001-01-01T01:01:02Z
           transferReceived: 200
           transferSent: 100
@@ -428,13 +424,13 @@ func TestParsingToYAML(t *testing.T) {
           status: Connected
           lastStatusUpdate: 2002-02-02T02:02:02Z
           connectionType: Relayed
-          direct: false
           iceCandidateType:
             local: relay
             remote: prflx
           iceCandidateEndpoint:
             local: 10.0.0.1:10001
             remote: 10.0.10.1:10002
+          relayAddress: ""
           lastWireguardHandshake: 2002-02-02T02:02:03Z
           transferReceived: 2000
           transferSent: 1000
@@ -505,9 +501,9 @@ func TestParsingToDetail(t *testing.T) {
   Status: Connected
   -- detail --
   Connection type: P2P
-  Direct: true
   ICE candidate (Local/Remote): -/-
   ICE candidate endpoints (Local/Remote): -/-
+  Relay server address: 
   Last connection update: %s
   Last WireGuard handshake: %s
   Transfer status (received/sent) 200 B/100 B
@@ -521,9 +517,9 @@ func TestParsingToDetail(t *testing.T) {
   Status: Connected
   -- detail --
   Connection type: Relayed
-  Direct: false
   ICE candidate (Local/Remote): relay/prflx
   ICE candidate endpoints (Local/Remote): 10.0.0.1:10001/10.0.10.1:10002
+  Relay server address: 
   Last connection update: %s
   Last WireGuard handshake: %s
   Transfer status (received/sent) 2.0 KiB/1000 B

client/cmd/testutil_test.go

@@ -11,6 +11,7 @@ import (
 	"go.opentelemetry.io/otel"
 
 	"github.com/netbirdio/netbird/management/server/activity"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 
 	"github.com/netbirdio/netbird/util"
 
@@ -71,6 +72,7 @@ func startSignal(t *testing.T) (*grpc.Server, net.Listener) {
 
 func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Listener) {
 	t.Helper()
+
 	lis, err := net.Listen("tcp", ":0")
 	if err != nil {
 		t.Fatal(err)
@@ -88,12 +90,17 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 		return nil, nil
 	}
 	iv, _ := integrations.NewIntegratedValidator(context.Background(), eventStore)
-	accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, iv)
+
+	metrics, err := telemetry.NewDefaultAppMetrics(context.Background())
+	require.NoError(t, err)
+
+	accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, iv, metrics)
 	if err != nil {
 		t.Fatal(err)
 	}
-	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
-	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	secretsManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
+	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

client/cmd/up.go

@@ -147,6 +147,11 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 		ic.DNSRouteInterval = &dnsRouteInterval
 	}
 
+	providedSetupKey, err := getSetupKey()
+	if err != nil {
+		return err
+	}
+
 	config, err := internal.UpdateOrCreateConfig(ic)
 	if err != nil {
 		return fmt.Errorf("get config file: %v", err)
@@ -154,7 +159,7 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 
 	config, _ = internal.UpdateOldManagementURL(ctx, config, configPath)
 
-	err = foregroundLogin(ctx, cmd, config, setupKey)
+	err = foregroundLogin(ctx, cmd, config, providedSetupKey)
 	if err != nil {
 		return fmt.Errorf("foreground login failed: %v", err)
 	}
@@ -163,7 +168,10 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 	ctx, cancel = context.WithCancel(ctx)
 	SetupCloseHandler(ctx, cancel)
 
-	connectClient := internal.NewConnectClient(ctx, config, peer.NewRecorder(config.ManagementURL.String()))
+	r := peer.NewRecorder(config.ManagementURL.String())
+	r.GetFullStatus()
+
+	connectClient := internal.NewConnectClient(ctx, config, r)
 	return connectClient.Run()
 }
 
@@ -199,8 +207,13 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command) error {
 		return nil
 	}
 
+	providedSetupKey, err := getSetupKey()
+	if err != nil {
+		return err
+	}
+
 	loginRequest := proto.LoginRequest{
-		SetupKey:             setupKey,
+		SetupKey:             providedSetupKey,
 		ManagementUrl:        managementURL,
 		AdminURL:             adminURL,
 		NatExternalIPs:       natExternalIPs,

client/cmd/up_daemon_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"os"
 	"testing"
 	"time"
 
@@ -40,6 +41,36 @@ func TestUpDaemon(t *testing.T) {
 		return
 	}
 
+	// Test the setup-key-file flag.
+	tempFile, err := os.CreateTemp("", "setup-key")
+	if err != nil {
+		t.Errorf("could not create temp file, got error %v", err)
+		return
+	}
+	defer os.Remove(tempFile.Name())
+	if _, err := tempFile.Write([]byte("A2C8E62B-38F5-4553-B31E-DD66C696CEBB")); err != nil {
+		t.Errorf("could not write to temp file, got error %v", err)
+		return
+	}
+	if err := tempFile.Close(); err != nil {
+		t.Errorf("unable to close file, got error %v", err)
+	}
+	rootCmd.SetArgs([]string{
+		"login",
+		"--daemon-addr", "tcp://" + cliAddr,
+		"--setup-key-file", tempFile.Name(),
+		"--log-file", "",
+	})
+	if err := rootCmd.Execute(); err != nil {
+		t.Errorf("expected no error while running up command, got %v", err)
+		return
+	}
+	time.Sleep(time.Second * 3)
+	if status, err := state.Status(); err != nil && status != internal.StatusIdle {
+		t.Errorf("wrong status after login: %s, %v", internal.StatusIdle, err)
+		return
+	}
+
 	rootCmd.SetArgs([]string{
 		"up",
 		"--daemon-addr", "tcp://" + cliAddr,

client/errors/errors.go

@@ -8,8 +8,8 @@ import (
 )
 
 func formatError(es []error) string {
-	if len(es) == 0 {
+	if len(es) == 1 {
-		return fmt.Sprintf("0 error occurred:\n\t* %s", es[0])
+		return fmt.Sprintf("1 error occurred:\n\t* %s", es[0])
 	}
 
 	points := make([]string, len(es))

client/firewall/uspfilter/uspfilter.go

@@ -337,7 +337,6 @@ func validateRule(ip net.IP, packetData []byte, rules map[string]Rule, d *decode
 			if rule.dPort != 0 && rule.dPort == uint16(d.udp.DstPort) {
 				return rule.drop, true
 			}
-			return rule.drop, true
 		case layers.LayerTypeICMPv4, layers.LayerTypeICMPv6:
 			return rule.drop, true
 		}

client/internal/auth/device_flow.go

@@ -3,6 +3,7 @@ package auth
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -180,7 +181,7 @@ func (d *DeviceAuthorizationFlow) WaitToken(ctx context.Context, info AuthFlowIn
 					continue
 				}
 
-				return TokenInfo{}, fmt.Errorf(tokenResponse.ErrorDescription)
+				return TokenInfo{}, errors.New(tokenResponse.ErrorDescription)
 			}
 
 			tokenInfo := TokenInfo{

client/internal/auth/oauth.go

@@ -69,6 +69,11 @@ func NewOAuthFlow(ctx context.Context, config *internal.Config, isLinuxDesktopCl
 		return authenticateWithDeviceCodeFlow(ctx, config)
 	}
 
+	// On FreeBSD we currently do not support desktop environments and offer only Device Code Flow (#2384)
+	if runtime.GOOS == "freebsd" {
+		return authenticateWithDeviceCodeFlow(ctx, config)
+	}
+
 	pkceFlow, err := authenticateWithPKCEFlow(ctx, config)
 	if err != nil {
 		// fallback to device code flow
@@ -81,7 +86,7 @@ func NewOAuthFlow(ctx context.Context, config *internal.Config, isLinuxDesktopCl
 
 // authenticateWithPKCEFlow initializes the Proof Key for Code Exchange flow auth flow
 func authenticateWithPKCEFlow(ctx context.Context, config *internal.Config) (OAuthFlow, error) {
-	pkceFlowInfo, err := internal.GetPKCEAuthorizationFlowInfo(ctx, config.PrivateKey, config.ManagementURL)
+	pkceFlowInfo, err := internal.GetPKCEAuthorizationFlowInfo(ctx, config.PrivateKey, config.ManagementURL, config.ClientCertKeyPair)
 	if err != nil {
 		return nil, fmt.Errorf("getting pkce authorization flow info failed with error: %v", err)
 	}

client/internal/auth/pkce_flow.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"crypto/subtle"
+	"crypto/tls"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -143,6 +144,18 @@ func (p *PKCEAuthorizationFlow) WaitToken(ctx context.Context, _ AuthFlowInfo) (
 func (p *PKCEAuthorizationFlow) startServer(server *http.Server, tokenChan chan<- *oauth2.Token, errChan chan<- error) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
+		cert := p.providerConfig.ClientCertPair
+		if cert != nil {
+			tr := &http.Transport{
+				TLSClientConfig: &tls.Config{
+					Certificates: []tls.Certificate{*cert},
+				},
+			}
+			sslClient := &http.Client{Transport: tr}
+			ctx := context.WithValue(req.Context(), oauth2.HTTPClient, sslClient)
+			req = req.WithContext(ctx)
+		}
+
 		token, err := p.handleRequest(req)
 		if err != nil {
 			renderPKCEFlowTmpl(w, err)

client/internal/config.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/url"
 	"os"
@@ -57,6 +58,8 @@ type ConfigInput struct {
 	DisableAutoConnect  *bool
 	ExtraIFaceBlackList []string
 	DNSRouteInterval    *time.Duration
+	ClientCertPath      string
+	ClientCertKeyPath   string
 }
 
 // Config Configuration type
@@ -102,11 +105,23 @@ type Config struct {
 
 	// DNSRouteInterval is the interval in which the DNS routes are updated
 	DNSRouteInterval time.Duration
+	//Path to a certificate used for mTLS authentication
+	ClientCertPath string
+
+	//Path to corresponding private key of ClientCertPath
+	ClientCertKeyPath string
+
+	ClientCertKeyPair *tls.Certificate `json:"-"`
 }
 
 // ReadConfig read config file and return with Config. If it is not exists create a new with default values
 func ReadConfig(configPath string) (*Config, error) {
 	if configFileIsExists(configPath) {
+		err := util.EnforcePermission(configPath)
+		if err != nil {
+			log.Errorf("failed to enforce permission on config dir: %v", err)
+		}
+
 		config := &Config{}
 		if _, err := util.ReadJson(configPath, config); err != nil {
 			return nil, err
@@ -149,13 +164,17 @@ func UpdateOrCreateConfig(input ConfigInput) (*Config, error) {
 		if err != nil {
 			return nil, err
 		}
-		err = WriteOutConfig(input.ConfigPath, cfg)
+		err = util.WriteJsonWithRestrictedPermission(input.ConfigPath, cfg)
 		return cfg, err
 	}
 
 	if isPreSharedKeyHidden(input.PreSharedKey) {
 		input.PreSharedKey = nil
 	}
+	err := util.EnforcePermission(input.ConfigPath)
+	if err != nil {
+		log.Errorf("failed to enforce permission on config dir: %v", err)
+	}
 	return update(input)
 }
 
@@ -385,6 +404,26 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) {
 
 	}
 
+	if input.ClientCertKeyPath != "" {
+		config.ClientCertKeyPath = input.ClientCertKeyPath
+		updated = true
+	}
+
+	if input.ClientCertPath != "" {
+		config.ClientCertPath = input.ClientCertPath
+		updated = true
+	}
+
+	if config.ClientCertPath != "" && config.ClientCertKeyPath != "" {
+		cert, err := tls.LoadX509KeyPair(config.ClientCertPath, config.ClientCertKeyPath)
+		if err != nil {
+			log.Error("Failed to load mTLS cert/key pair: ", err)
+		} else {
+			config.ClientCertKeyPair = &cert
+			log.Info("Loaded client mTLS cert/key pair")
+		}
+	}
+
 	return updated, nil
 }
 

client/internal/connect.go

@@ -26,6 +26,8 @@ import (
 	"github.com/netbirdio/netbird/iface"
 	mgm "github.com/netbirdio/netbird/management/client"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
+	"github.com/netbirdio/netbird/relay/auth/hmac"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	signal "github.com/netbirdio/netbird/signal/client"
 	"github.com/netbirdio/netbird/util"
 	"github.com/netbirdio/netbird/version"
@@ -55,17 +57,15 @@ func NewConnectClient(
 
 // Run with main logic.
 func (c *ConnectClient) Run() error {
-	return c.run(MobileDependency{}, nil, nil, nil, nil)
+	return c.run(MobileDependency{}, nil, nil)
 }
 
 // RunWithProbes runs the client's main logic with probes attached
 func (c *ConnectClient) RunWithProbes(
-	mgmProbe *Probe,
+	probes *ProbeHolder,
-	signalProbe *Probe,
+	runningChan chan error,
-	relayProbe *Probe,
-	wgProbe *Probe,
 ) error {
-	return c.run(MobileDependency{}, mgmProbe, signalProbe, relayProbe, wgProbe)
+	return c.run(MobileDependency{}, probes, runningChan)
 }
 
 // RunOnAndroid with main logic on mobile system
@@ -84,7 +84,7 @@ func (c *ConnectClient) RunOnAndroid(
 		HostDNSAddresses:      dnsAddresses,
 		DnsReadyListener:      dnsReadyListener,
 	}
-	return c.run(mobileDependency, nil, nil, nil, nil)
+	return c.run(mobileDependency, nil, nil)
 }
 
 func (c *ConnectClient) RunOniOS(
@@ -100,15 +100,13 @@ func (c *ConnectClient) RunOniOS(
 		NetworkChangeListener: networkChangeListener,
 		DnsManager:            dnsManager,
 	}
-	return c.run(mobileDependency, nil, nil, nil, nil)
+	return c.run(mobileDependency, nil, nil)
 }
 
 func (c *ConnectClient) run(
 	mobileDependency MobileDependency,
-	mgmProbe *Probe,
+	probes *ProbeHolder,
-	signalProbe *Probe,
+	runningChan chan error,
-	relayProbe *Probe,
-	wgProbe *Probe,
 ) error {
 	defer func() {
 		if r := recover(); r != nil {
@@ -160,12 +158,11 @@ func (c *ConnectClient) run(
 	}
 
 	defer c.statusRecorder.ClientStop()
+	runningChanOpen := true
 	operation := func() error {
 		// if context cancelled we not start new backoff cycle
-		select {
+		if c.isContextCancelled() {
-		case <-c.ctx.Done():
 			return nil
-		default:
 		}
 
 		state.Set(StatusConnecting)
@@ -187,8 +184,7 @@ func (c *ConnectClient) run(
 
 		log.Debugf("connected to the Management service %s", c.config.ManagementURL.Host)
 		defer func() {
-			err = mgmClient.Close()
+			if err = mgmClient.Close(); err != nil {
-			if err != nil {
 				log.Warnf("failed to close the Management service client %v", err)
 			}
 		}()
@@ -199,6 +195,7 @@ func (c *ConnectClient) run(
 			log.Debug(err)
 			if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.PermissionDenied) {
 				state.Set(StatusNeedsLogin)
+				_ = c.Stop()
 				return backoff.Permanent(wrapErr(err)) // unrecoverable error
 			}
 			return wrapErr(err)
@@ -211,7 +208,6 @@ func (c *ConnectClient) run(
 			KernelInterface: iface.WireGuardModuleIsLoaded(),
 			FQDN:            loginResp.GetPeerConfig().GetFqdn(),
 		}
-
 		c.statusRecorder.UpdateLocalPeerState(localPeerState)
 
 		signalURL := fmt.Sprintf("%s://%s",
@@ -244,6 +240,23 @@ func (c *ConnectClient) run(
 
 		c.statusRecorder.MarkSignalConnected()
 
+		relayURLs, token := parseRelayInfo(loginResp)
+		relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String())
+		if len(relayURLs) > 0 {
+			if token != nil {
+				if err := relayManager.UpdateToken(token); err != nil {
+					log.Errorf("failed to update token: %s", err)
+					return wrapErr(err)
+				}
+			}
+			log.Infof("connecting to the Relay service(s): %s", strings.Join(relayURLs, ", "))
+			if err = relayManager.Serve(); err != nil {
+				log.Error(err)
+				return wrapErr(err)
+			}
+			c.statusRecorder.SetRelayMgr(relayManager)
+		}
+
 		peerConfig := loginResp.GetPeerConfig()
 
 		engineConfig, err := createEngineConfig(myPrivateKey, c.config, peerConfig)
@@ -255,11 +268,17 @@ func (c *ConnectClient) run(
 		checks := loginResp.GetChecks()
 
 		c.engineMutex.Lock()
-		c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, engineConfig, mobileDependency, c.statusRecorder, mgmProbe, signalProbe, relayProbe, wgProbe, checks)
+		if c.engine != nil && c.engine.ctx.Err() != nil {
+			log.Info("Stopping Netbird Engine")
+			if err := c.engine.Stop(); err != nil {
+				log.Errorf("Failed to stop engine: %v", err)
+			}
+		}
+		c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, relayManager, engineConfig, mobileDependency, c.statusRecorder, probes, checks)
+
 		c.engineMutex.Unlock()
 
-		err = c.engine.Start()
+		if err := c.engine.Start(); err != nil {
-		if err != nil {
 			log.Errorf("error while starting Netbird Connection Engine: %s", err)
 			return wrapErr(err)
 		}
@@ -267,17 +286,17 @@ func (c *ConnectClient) run(
 		log.Infof("Netbird engine started, the IP is: %s", peerConfig.GetAddress())
 		state.Set(StatusConnected)
 
+		if runningChan != nil && runningChanOpen {
+			runningChan <- nil
+			close(runningChan)
+			runningChanOpen = false
+		}
+
 		<-engineCtx.Done()
 		c.statusRecorder.ClientTeardown()
 
 		backOff.Reset()
 
-		err = c.engine.Stop()
-		if err != nil {
-			log.Errorf("failed stopping engine %v", err)
-			return wrapErr(err)
-		}
-
 		log.Info("stopped NetBird client")
 
 		if _, err := state.Status(); errors.Is(err, ErrResetConnection) {
@@ -293,13 +312,31 @@ func (c *ConnectClient) run(
 		log.Debugf("exiting client retry loop due to unrecoverable error: %s", err)
 		if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.PermissionDenied) {
 			state.Set(StatusNeedsLogin)
+			_ = c.Stop()
 		}
 		return err
 	}
 	return nil
 }
 
+func parseRelayInfo(loginResp *mgmProto.LoginResponse) ([]string, *hmac.Token) {
+	relayCfg := loginResp.GetWiretrusteeConfig().GetRelay()
+	if relayCfg == nil {
+		return nil, nil
+	}
+
+	token := &hmac.Token{
+		Payload:   relayCfg.GetTokenPayload(),
+		Signature: relayCfg.GetTokenSignature(),
+	}
+
+	return relayCfg.GetUrls(), token
+}
+
 func (c *ConnectClient) Engine() *Engine {
+	if c == nil {
+		return nil
+	}
 	var e *Engine
 	c.engineMutex.Lock()
 	e = c.engine
@@ -307,6 +344,28 @@ func (c *ConnectClient) Engine() *Engine {
 	return e
 }
 
+func (c *ConnectClient) Stop() error {
+	if c == nil {
+		return nil
+	}
+	c.engineMutex.Lock()
+	defer c.engineMutex.Unlock()
+
+	if c.engine == nil {
+		return nil
+	}
+	return c.engine.Stop()
+}
+
+func (c *ConnectClient) isContextCancelled() bool {
+	select {
+	case <-c.ctx.Done():
+		return true
+	default:
+		return false
+	}
+}
+
 // createEngineConfig converts configuration received from Management Service to EngineConfig
 func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.PeerConfig) (*EngineConfig, error) {
 	nm := false
@@ -397,19 +456,43 @@ func statusRecorderToSignalConnStateNotifier(statusRecorder *peer.Status) signal
 	return notifier
 }
 
-func freePort(start int) (int, error) {
+// freePort attempts to determine if the provided port is available, if not it will ask the system for a free port.
+func freePort(initPort int) (int, error) {
 	addr := net.UDPAddr{}
-	if start == 0 {
+	if initPort == 0 {
-		start = iface.DefaultWgPort
+		initPort = iface.DefaultWgPort
 	}
-	for x := start; x <= 65535; x++ {
+
-		addr.Port = x
+	addr.Port = initPort
+
 	conn, err := net.ListenUDP("udp", &addr)
+	if err == nil {
+		closeConnWithLog(conn)
+		return initPort, nil
+	}
+
+	// if the port is already in use, ask the system for a free port
+	addr.Port = 0
+	conn, err = net.ListenUDP("udp", &addr)
 	if err != nil {
-			continue
+		return 0, fmt.Errorf("unable to get a free port: %v", err)
 	}
-		conn.Close()
+
-		return x, nil
+	udpAddr, ok := conn.LocalAddr().(*net.UDPAddr)
+	if !ok {
+		return 0, errors.New("wrong address type when getting a free port")
+	}
+	closeConnWithLog(conn)
+	return udpAddr.Port, nil
+}
+
+func closeConnWithLog(conn *net.UDPConn) {
+	startClosing := time.Now()
+	err := conn.Close()
+	if err != nil {
+		log.Warnf("closing probe port %d failed: %v. NetBird will still attempt to use this port for connection.", conn.LocalAddr().(*net.UDPAddr).Port, err)
+	}
+	if time.Since(startClosing) > time.Second {
+		log.Warnf("closing the testing port %d took %s. Usually it is safe to ignore, but continuous warnings may indicate a problem.", conn.LocalAddr().(*net.UDPAddr).Port, time.Since(startClosing))
 	}
-	return 0, errors.New("no free ports")
 }

client/internal/connect_test.go

@@ -10,48 +10,52 @@ func Test_freePort(t *testing.T) {
 		name        string
 		port        int
 		want        int
-		wantErr bool
+		shouldMatch bool
 	}{
 		{
-			name:    "available",
+			name:        "not provided, fallback to default",
-			port:    51820,
+			port:        0,
 			want:        51820,
-			wantErr: false,
+			shouldMatch: true,
 		},
 		{
-			name:    "notavailable",
+			name:        "provided and available",
+			port:        51821,
+			want:        51821,
+			shouldMatch: true,
+		},
+		{
+			name:        "provided and not available",
 			port:        51830,
-			want:    51831,
+			want:        51830,
-			wantErr: false,
+			shouldMatch: false,
-		},
-		{
-			name:    "noports",
-			port:    65535,
-			want:    0,
-			wantErr: true,
 		},
 	}
-	for _, tt := range tests {
-
 	c1, err := net.ListenUDP("udp", &net.UDPAddr{Port: 51830})
 	if err != nil {
 		t.Errorf("freePort error = %v", err)
 	}
-		c2, err := net.ListenUDP("udp", &net.UDPAddr{Port: 65535})
+	defer func(c1 *net.UDPConn) {
-		if err != nil {
+		_ = c1.Close()
-			t.Errorf("freePort error = %v", err)
+	}(c1)
-		}
+
+	for _, tt := range tests {
+
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := freePort(tt.port)
-			if (err != nil) != tt.wantErr {
+
-				t.Errorf("freePort() error = %v, wantErr %v", err, tt.wantErr)
+			if err != nil {
-				return
+				t.Errorf("got an error while getting free port: %v", err)
 			}
-			if got != tt.want {
+
-				t.Errorf("freePort() = %v, want %v", got, tt.want)
+			if tt.shouldMatch && got != tt.want {
+				t.Errorf("got a different port %v, want %v", got, tt.want)
+			}
+
+			if !tt.shouldMatch && got == tt.want {
+				t.Errorf("got the same port %v, want a different port", tt.want)
 			}
 		})
-		c1.Close()
+
-		c2.Close()
 	}
 }

client/internal/dns/host.go

@@ -15,6 +15,12 @@ type hostManager interface {
 	restoreUncleanShutdownDNS(storedDNSAddress *netip.Addr) error
 }
 
+type SystemDNSSettings struct {
+	Domains    []string
+	ServerIP   string
+	ServerPort int
+}
+
 type HostDNSConfig struct {
 	Domains    []DomainConfig `json:"domains"`
 	RouteAll   bool           `json:"routeAll"`

client/internal/dns/host_darwin.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"net"
 	"net/netip"
 	"os/exec"
 	"strconv"
@@ -18,7 +19,7 @@ import (
 const (
 	netbirdDNSStateKeyFormat            = "State:/Network/Service/NetBird-%s/DNS"
 	globalIPv4State                     = "State:/Network/Global/IPv4"
-	primaryServiceSetupKeyFormat        = "Setup:/Network/Service/%s/DNS"
+	primaryServiceStateKeyFormat        = "State:/Network/Service/%s/DNS"
 	keySupplementalMatchDomains         = "SupplementalMatchDomains"
 	keySupplementalMatchDomainsNoSearch = "SupplementalMatchDomainsNoSearch"
 	keyServerAddresses                  = "ServerAddresses"
@@ -28,12 +29,12 @@ const (
 	scutilPath                          = "/usr/sbin/scutil"
 	searchSuffix                        = "Search"
 	matchSuffix                         = "Match"
+	localSuffix                         = "Local"
 )
 
 type systemConfigurator struct {
-	// primaryServiceID primary interface in the system. AKA the interface with the default route
-	primaryServiceID string
 	createdKeys       map[string]struct{}
+	systemDNSSettings SystemDNSSettings
 }
 
 func newHostManager() (hostManager, error) {
@@ -49,20 +50,6 @@ func (s *systemConfigurator) supportCustomPort() bool {
 func (s *systemConfigurator) applyDNSConfig(config HostDNSConfig) error {
 	var err error
 
-	if config.RouteAll {
-		err = s.addDNSSetupForAll(config.ServerIP, config.ServerPort)
-		if err != nil {
-			return fmt.Errorf("add dns setup for all: %w", err)
-		}
-	} else if s.primaryServiceID != "" {
-		err = s.removeKeyFromSystemConfig(getKeyWithInput(primaryServiceSetupKeyFormat, s.primaryServiceID))
-		if err != nil {
-			return fmt.Errorf("remote key from system config: %w", err)
-		}
-		s.primaryServiceID = ""
-		log.Infof("removed %s:%d as main DNS resolver for this peer", config.ServerIP, config.ServerPort)
-	}
-
 	// create a file for unclean shutdown detection
 	if err := createUncleanShutdownIndicator(); err != nil {
 		log.Errorf("failed to create unclean shutdown file: %s", err)
@@ -73,6 +60,19 @@ func (s *systemConfigurator) applyDNSConfig(config HostDNSConfig) error {
 		matchDomains  []string
 	)
 
+	err = s.recordSystemDNSSettings(true)
+	if err != nil {
+		log.Errorf("unable to update record of System's DNS config: %s", err.Error())
+	}
+
+	if config.RouteAll {
+		searchDomains = append(searchDomains, "\"\"")
+		err = s.addLocalDNS()
+		if err != nil {
+			log.Infof("failed to enable split DNS")
+		}
+	}
+
 	for _, dConf := range config.Domains {
 		if dConf.Disabled {
 			continue
@@ -110,23 +110,17 @@ func (s *systemConfigurator) applyDNSConfig(config HostDNSConfig) error {
 }
 
 func (s *systemConfigurator) restoreHostDNS() error {
-	lines := ""
+	keys := s.getRemovableKeysWithDefaults()
-	for key := range s.createdKeys {
+	for _, key := range keys {
-		lines += buildRemoveKeyOperation(key)
 		keyType := "search"
 		if strings.Contains(key, matchSuffix) {
 			keyType = "match"
 		}
 		log.Infof("removing %s domains from system", keyType)
-	}
+		err := s.removeKeyFromSystemConfig(key)
-	if s.primaryServiceID != "" {
-		lines += buildRemoveKeyOperation(getKeyWithInput(primaryServiceSetupKeyFormat, s.primaryServiceID))
-		log.Infof("restoring DNS resolver configuration for system")
-	}
-	_, err := runSystemConfigCommand(wrapCommand(lines))
 		if err != nil {
-		log.Errorf("got an error while cleaning the system configuration: %s", err)
+			log.Errorf("failed to remove %s domains from system: %s", keyType, err)
-		return fmt.Errorf("clean system: %w", err)
+		}
 	}
 
 	if err := removeUncleanShutdownIndicator(); err != nil {
@@ -136,6 +130,19 @@ func (s *systemConfigurator) restoreHostDNS() error {
 	return nil
 }
 
+func (s *systemConfigurator) getRemovableKeysWithDefaults() []string {
+	if len(s.createdKeys) == 0 {
+		// return defaults for startup calls
+		return []string{getKeyWithInput(netbirdDNSStateKeyFormat, searchSuffix), getKeyWithInput(netbirdDNSStateKeyFormat, matchSuffix)}
+	}
+
+	keys := make([]string, 0, len(s.createdKeys))
+	for key := range s.createdKeys {
+		keys = append(keys, key)
+	}
+	return keys
+}
+
 func (s *systemConfigurator) removeKeyFromSystemConfig(key string) error {
 	line := buildRemoveKeyOperation(key)
 	_, err := runSystemConfigCommand(wrapCommand(line))
@@ -148,6 +155,97 @@ func (s *systemConfigurator) removeKeyFromSystemConfig(key string) error {
 	return nil
 }
 
+func (s *systemConfigurator) addLocalDNS() error {
+	if s.systemDNSSettings.ServerIP == "" || len(s.systemDNSSettings.Domains) == 0 {
+		err := s.recordSystemDNSSettings(true)
+		log.Errorf("Unable to get system DNS configuration")
+		return err
+	}
+	localKey := getKeyWithInput(netbirdDNSStateKeyFormat, localSuffix)
+	if s.systemDNSSettings.ServerIP != "" && len(s.systemDNSSettings.Domains) != 0 {
+		err := s.addSearchDomains(localKey, strings.Join(s.systemDNSSettings.Domains, " "), s.systemDNSSettings.ServerIP, s.systemDNSSettings.ServerPort)
+		if err != nil {
+			return fmt.Errorf("couldn't add local network DNS conf: %w", err)
+		}
+	} else {
+		log.Info("Not enabling local DNS server")
+	}
+
+	return nil
+}
+
+func (s *systemConfigurator) recordSystemDNSSettings(force bool) error {
+	if s.systemDNSSettings.ServerIP != "" && len(s.systemDNSSettings.Domains) != 0 && !force {
+		return nil
+	}
+
+	systemDNSSettings, err := s.getSystemDNSSettings()
+	if err != nil {
+		return fmt.Errorf("couldn't get current DNS config: %w", err)
+	}
+	s.systemDNSSettings = systemDNSSettings
+
+	return nil
+}
+
+func (s *systemConfigurator) getSystemDNSSettings() (SystemDNSSettings, error) {
+	primaryServiceKey, _, err := s.getPrimaryService()
+	if err != nil || primaryServiceKey == "" {
+		return SystemDNSSettings{}, fmt.Errorf("couldn't find the primary service key: %w", err)
+	}
+	dnsServiceKey := getKeyWithInput(primaryServiceStateKeyFormat, primaryServiceKey)
+	line := buildCommandLine("show", dnsServiceKey, "")
+	stdinCommands := wrapCommand(line)
+
+	b, err := runSystemConfigCommand(stdinCommands)
+	if err != nil {
+		return SystemDNSSettings{}, fmt.Errorf("sending the command: %w", err)
+	}
+
+	var dnsSettings SystemDNSSettings
+	inSearchDomainsArray := false
+	inServerAddressesArray := false
+
+	scanner := bufio.NewScanner(bytes.NewReader(b))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		switch {
+		case strings.HasPrefix(line, "DomainName :"):
+			domainName := strings.TrimSpace(strings.Split(line, ":")[1])
+			dnsSettings.Domains = append(dnsSettings.Domains, domainName)
+		case line == "SearchDomains : <array> {":
+			inSearchDomainsArray = true
+			continue
+		case line == "ServerAddresses : <array> {":
+			inServerAddressesArray = true
+			continue
+		case line == "}":
+			inSearchDomainsArray = false
+			inServerAddressesArray = false
+		}
+
+		if inSearchDomainsArray {
+			searchDomain := strings.Split(line, " : ")[1]
+			dnsSettings.Domains = append(dnsSettings.Domains, searchDomain)
+		} else if inServerAddressesArray {
+			address := strings.Split(line, " : ")[1]
+			if ip := net.ParseIP(address); ip != nil && ip.To4() != nil {
+				dnsSettings.ServerIP = address
+				inServerAddressesArray = false // Stop reading after finding the first IPv4 address
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return dnsSettings, err
+	}
+
+	// default to 53 port
+	dnsSettings.ServerPort = 53
+
+	return dnsSettings, nil
+}
+
 func (s *systemConfigurator) addSearchDomains(key, domains string, ip string, port int) error {
 	err := s.addDNSState(key, domains, ip, port, true)
 	if err != nil {
@@ -194,23 +292,6 @@ func (s *systemConfigurator) addDNSState(state, domains, dnsServer string, port
 	return nil
 }
 
-func (s *systemConfigurator) addDNSSetupForAll(dnsServer string, port int) error {
-	primaryServiceKey, existingNameserver, err := s.getPrimaryService()
-	if err != nil || primaryServiceKey == "" {
-		return fmt.Errorf("couldn't find the primary service key: %w", err)
-	}
-
-	err = s.addDNSSetup(getKeyWithInput(primaryServiceSetupKeyFormat, primaryServiceKey), dnsServer, port, existingNameserver)
-	if err != nil {
-		return fmt.Errorf("add dns setup: %w", err)
-	}
-
-	log.Infof("configured %s:%d as main DNS resolver for this peer", dnsServer, port)
-	s.primaryServiceID = primaryServiceKey
-
-	return nil
-}
-
 func (s *systemConfigurator) getPrimaryService() (string, string, error) {
 	line := buildCommandLine("show", globalIPv4State, "")
 	stdinCommands := wrapCommand(line)
@@ -239,19 +320,6 @@ func (s *systemConfigurator) getPrimaryService() (string, string, error) {
 	return primaryService, router, nil
 }
 
-func (s *systemConfigurator) addDNSSetup(setupKey, dnsServer string, port int, existingDNSServer string) error {
-	lines := buildAddCommandLine(keySupplementalMatchDomainsNoSearch, digitSymbol+strconv.Itoa(0))
-	lines += buildAddCommandLine(keyServerAddresses, arraySymbol+dnsServer+" "+existingDNSServer)
-	lines += buildAddCommandLine(keyServerPort, digitSymbol+strconv.Itoa(port))
-	addDomainCommand := buildCreateStateWithOperation(setupKey, lines)
-	stdinCommands := wrapCommand(addDomainCommand)
-	_, err := runSystemConfigCommand(stdinCommands)
-	if err != nil {
-		return fmt.Errorf("applying dns setup, error: %w", err)
-	}
-	return nil
-}
-
 func (s *systemConfigurator) restoreUncleanShutdownDNS(*netip.Addr) error {
 	if err := s.restoreHostDNS(); err != nil {
 		return fmt.Errorf("restoring dns via scutil: %w", err)

client/internal/dns/server.go

@@ -94,7 +94,7 @@ func NewDefaultServer(
 
 	var dnsService service
 	if wgInterface.IsUserspaceBind() {
-		dnsService = newServiceViaMemory(wgInterface)
+		dnsService = NewServiceViaMemory(wgInterface)
 	} else {
 		dnsService = newServiceViaListener(wgInterface, addrPort)
 	}
@@ -112,7 +112,7 @@ func NewDefaultServerPermanentUpstream(
 	statusRecorder *peer.Status,
 ) *DefaultServer {
 	log.Debugf("host dns address list is: %v", hostsDnsList)
-	ds := newDefaultServer(ctx, wgInterface, newServiceViaMemory(wgInterface), statusRecorder)
+	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder)
 	ds.hostsDNSHolder.set(hostsDnsList)
 	ds.permanent = true
 	ds.addHostRootZone()
@@ -130,7 +130,7 @@ func NewDefaultServerIos(
 	iosDnsManager IosDnsManager,
 	statusRecorder *peer.Status,
 ) *DefaultServer {
-	ds := newDefaultServer(ctx, wgInterface, newServiceViaMemory(wgInterface), statusRecorder)
+	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder)
 	ds.iosDnsManager = iosDnsManager
 	return ds
 }

client/internal/dns/server_test.go

@@ -534,7 +534,7 @@ func TestDNSServerStartStop(t *testing.T) {
 func TestDNSServerUpstreamDeactivateCallback(t *testing.T) {
 	hostManager := &mockHostConfigurator{}
 	server := DefaultServer{
-		service: newServiceViaMemory(&mocWGIface{}),
+		service: NewServiceViaMemory(&mocWGIface{}),
 		localResolver: &localResolver{
 			registeredMap: make(registrationMap),
 		},

client/internal/dns/service_listener.go

@@ -128,6 +128,9 @@ func (s *serviceViaListener) RuntimeIP() string {
 }
 
 func (s *serviceViaListener) setListenerStatus(running bool) {
+	s.listenerFlagLock.Lock()
+	defer s.listenerFlagLock.Unlock()
+
 	s.listenerIsRunning = running
 }
 

client/internal/dns/service_memory.go

@@ -12,7 +12,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-type serviceViaMemory struct {
+type ServiceViaMemory struct {
 	wgInterface       WGIface
 	dnsMux            *dns.ServeMux
 	runtimeIP         string
@@ -22,8 +22,8 @@ type serviceViaMemory struct {
 	listenerFlagLock  sync.Mutex
 }
 
-func newServiceViaMemory(wgIface WGIface) *serviceViaMemory {
+func NewServiceViaMemory(wgIface WGIface) *ServiceViaMemory {
-	s := &serviceViaMemory{
+	s := &ServiceViaMemory{
 		wgInterface: wgIface,
 		dnsMux:      dns.NewServeMux(),
 
@@ -33,7 +33,7 @@ func newServiceViaMemory(wgIface WGIface) *serviceViaMemory {
 	return s
 }
 
-func (s *serviceViaMemory) Listen() error {
+func (s *ServiceViaMemory) Listen() error {
 	s.listenerFlagLock.Lock()
 	defer s.listenerFlagLock.Unlock()
 
@@ -52,7 +52,7 @@ func (s *serviceViaMemory) Listen() error {
 	return nil
 }
 
-func (s *serviceViaMemory) Stop() {
+func (s *ServiceViaMemory) Stop() {
 	s.listenerFlagLock.Lock()
 	defer s.listenerFlagLock.Unlock()
 
@@ -67,23 +67,23 @@ func (s *serviceViaMemory) Stop() {
 	s.listenerIsRunning = false
 }
 
-func (s *serviceViaMemory) RegisterMux(pattern string, handler dns.Handler) {
+func (s *ServiceViaMemory) RegisterMux(pattern string, handler dns.Handler) {
 	s.dnsMux.Handle(pattern, handler)
 }
 
-func (s *serviceViaMemory) DeregisterMux(pattern string) {
+func (s *ServiceViaMemory) DeregisterMux(pattern string) {
 	s.dnsMux.HandleRemove(pattern)
 }
 
-func (s *serviceViaMemory) RuntimePort() int {
+func (s *ServiceViaMemory) RuntimePort() int {
 	return s.runtimePort
 }
 
-func (s *serviceViaMemory) RuntimeIP() string {
+func (s *ServiceViaMemory) RuntimeIP() string {
 	return s.runtimeIP
 }
 
-func (s *serviceViaMemory) filterDNSTraffic() (string, error) {
+func (s *ServiceViaMemory) filterDNSTraffic() (string, error) {
 	filter := s.wgInterface.GetFilter()
 	if filter == nil {
 		return "", fmt.Errorf("can't set DNS filter, filter not initialized")

client/internal/dns/upstream.go

@@ -24,7 +24,7 @@ const (
 	probeTimeout     = 2 * time.Second
 )
 
-const testRecord = "."
+const testRecord = "com."
 
 type upstreamClient interface {
 	exchange(ctx context.Context, upstream string, r *dns.Msg) (*dns.Msg, time.Duration, error)
@@ -42,6 +42,7 @@ type upstreamResolverBase struct {
 	upstreamServers  []string
 	disabled         bool
 	failsCount       atomic.Int32
+	successCount     atomic.Int32
 	failsTillDeact   int32
 	mutex            sync.Mutex
 	reactivatePeriod time.Duration
@@ -124,6 +125,7 @@ func (u *upstreamResolverBase) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 			return
 		}
 
+		u.successCount.Add(1)
 		log.Tracef("took %s to query the upstream %s", t, upstream)
 
 		err = w.WriteMsg(rm)
@@ -172,6 +174,11 @@ func (u *upstreamResolverBase) probeAvailability() {
 	default:
 	}
 
+	// avoid probe if upstreams could resolve at least one query and fails count is less than failsTillDeact
+	if u.successCount.Load() > 0 && u.failsCount.Load() < u.failsTillDeact {
+		return
+	}
+
 	var success bool
 	var mu sync.Mutex
 	var wg sync.WaitGroup
@@ -183,7 +190,7 @@ func (u *upstreamResolverBase) probeAvailability() {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err := u.testNameserver(upstream)
+			err := u.testNameserver(upstream, 500*time.Millisecond)
 			if err != nil {
 				errors = multierror.Append(errors, err)
 				log.Warnf("probing upstream nameserver %s: %s", upstream, err)
@@ -224,7 +231,7 @@ func (u *upstreamResolverBase) waitUntilResponse() {
 		}
 
 		for _, upstream := range u.upstreamServers {
-			if err := u.testNameserver(upstream); err != nil {
+			if err := u.testNameserver(upstream, probeTimeout); err != nil {
 				log.Tracef("upstream check for %s: %s", upstream, err)
 			} else {
 				// at least one upstream server is available, stop probing
@@ -244,6 +251,7 @@ func (u *upstreamResolverBase) waitUntilResponse() {
 
 	log.Infof("upstreams %s are responsive again. Adding them back to system", u.upstreamServers)
 	u.failsCount.Store(0)
+	u.successCount.Add(1)
 	u.reactivate()
 	u.disabled = false
 }
@@ -265,13 +273,14 @@ func (u *upstreamResolverBase) disable(err error) {
 	}
 
 	log.Warnf("Upstream resolving is Disabled for %v", reactivatePeriod)
+	u.successCount.Store(0)
 	u.deactivate(err)
 	u.disabled = true
 	go u.waitUntilResponse()
 }
 
-func (u *upstreamResolverBase) testNameserver(server string) error {
+func (u *upstreamResolverBase) testNameserver(server string, timeout time.Duration) error {
-	ctx, cancel := context.WithTimeout(u.ctx, probeTimeout)
+	ctx, cancel := context.WithTimeout(u.ctx, timeout)
 	defer cancel()
 
 	r := new(dns.Msg).SetQuestion(testRecord, dns.TypeSOA)

client/internal/dns/upstream_ios.go

@@ -4,6 +4,7 @@ package dns
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"syscall"
 	"time"
@@ -19,7 +20,7 @@ type upstreamResolverIOS struct {
 	*upstreamResolverBase
 	lIP           net.IP
 	lNet          *net.IPNet
-	iIndex int
+	interfaceName string
 }
 
 func newUpstreamResolver(
@@ -32,17 +33,11 @@ func newUpstreamResolver(
 ) (*upstreamResolverIOS, error) {
 	upstreamResolverBase := newUpstreamResolverBase(ctx, statusRecorder)
 
-	index, err := getInterfaceIndex(interfaceName)
-	if err != nil {
-		log.Debugf("unable to get interface index for %s: %s", interfaceName, err)
-		return nil, err
-	}
-
 	ios := &upstreamResolverIOS{
 		upstreamResolverBase: upstreamResolverBase,
 		lIP:                  ip,
 		lNet:                 net,
-		iIndex:               index,
+		interfaceName:        interfaceName,
 	}
 	ios.upstreamClient = ios
 
@@ -53,7 +48,7 @@ func (u *upstreamResolverIOS) exchange(ctx context.Context, upstream string, r *
 	client := &dns.Client{}
 	upstreamHost, _, err := net.SplitHostPort(upstream)
 	if err != nil {
-		log.Errorf("error while parsing upstream host: %s", err)
+		return nil, 0, fmt.Errorf("error while parsing upstream host: %s", err)
 	}
 
 	timeout := upstreamTimeout
@@ -65,26 +60,35 @@ func (u *upstreamResolverIOS) exchange(ctx context.Context, upstream string, r *
 	upstreamIP := net.ParseIP(upstreamHost)
 	if u.lNet.Contains(upstreamIP) || net.IP.IsPrivate(upstreamIP) {
 		log.Debugf("using private client to query upstream: %s", upstream)
-		client = u.getClientPrivate(timeout)
+		client, err = GetClientPrivate(u.lIP, u.interfaceName, timeout)
+		if err != nil {
+			return nil, 0, fmt.Errorf("error while creating private client: %s", err)
+		}
 	}
 
 	// Cannot use client.ExchangeContext because it overwrites our Dialer
 	return client.Exchange(r, upstream)
 }
 
-// getClientPrivate returns a new DNS client bound to the local IP address of the Netbird interface
+// GetClientPrivate returns a new DNS client bound to the local IP address of the Netbird interface
 // This method is needed for iOS
-func (u *upstreamResolverIOS) getClientPrivate(dialTimeout time.Duration) *dns.Client {
+func GetClientPrivate(ip net.IP, interfaceName string, dialTimeout time.Duration) (*dns.Client, error) {
+	index, err := getInterfaceIndex(interfaceName)
+	if err != nil {
+		log.Debugf("unable to get interface index for %s: %s", interfaceName, err)
+		return nil, err
+	}
+
 	dialer := &net.Dialer{
 		LocalAddr: &net.UDPAddr{
-			IP:   u.lIP,
+			IP:   ip,
 			Port: 0, // Let the OS pick a free port
 		},
 		Timeout: dialTimeout,
 		Control: func(network, address string, c syscall.RawConn) error {
 			var operr error
 			fn := func(s uintptr) {
-				operr = unix.SetsockoptInt(int(s), unix.IPPROTO_IP, unix.IP_BOUND_IF, u.iIndex)
+				operr = unix.SetsockoptInt(int(s), unix.IPPROTO_IP, unix.IP_BOUND_IF, index)
 			}
 
 			if err := c.Control(fn); err != nil {
@@ -101,7 +105,7 @@ func (u *upstreamResolverIOS) getClientPrivate(dialTimeout time.Duration) *dns.C
 	client := &dns.Client{
 		Dialer: dialer,
 	}
-	return client
+	return client, nil
 }
 
 func getInterfaceIndex(interfaceName string) (int, error) {

client/internal/engine.go

@@ -13,6 +13,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/pion/ice/v3"
@@ -24,6 +25,7 @@ import (
 	"github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/acl"
 	"github.com/netbirdio/netbird/client/internal/dns"
+
 	"github.com/netbirdio/netbird/client/internal/networkmonitor"
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/internal/relay"
@@ -39,6 +41,8 @@ import (
 	mgm "github.com/netbirdio/netbird/management/client"
 	"github.com/netbirdio/netbird/management/domain"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
+	auth "github.com/netbirdio/netbird/relay/auth/hmac"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	signal "github.com/netbirdio/netbird/signal/client"
 	sProto "github.com/netbirdio/netbird/signal/proto"
@@ -102,6 +106,7 @@ type EngineConfig struct {
 type Engine struct {
 	// signal is a Signal Service client
 	signal   signal.Client
+	signaler *peer.Signaler
 	// mgmClient is a Management Service client
 	mgmClient mgm.Client
 	// peerConns is a map that holds all the peers that are known to this peer
@@ -123,6 +128,7 @@ type Engine struct {
 	STUNs []*stun.URI
 	// TURNs is a list of STUN servers used by ICE
 	TURNs    []*stun.URI
+	stunTurn atomic.Value
 
 	// clientRoutes is the most recent list of clientRoutes received from the Management Service
 	clientRoutes   route.HAMap
@@ -134,7 +140,7 @@ type Engine struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 
-	wgInterface    *iface.WGIface
+	wgInterface    iface.IWGIface
 	wgProxyFactory *wgproxy.Factory
 
 	udpMux *bind.UniversalUDPMuxDefault
@@ -155,15 +161,12 @@ type Engine struct {
 
 	dnsServer dns.Server
 
-	mgmProbe    *Probe
+	probes *ProbeHolder
-	signalProbe *Probe
-	relayProbe  *Probe
-	wgProbe     *Probe
-
-	wgConnWorker sync.WaitGroup
 
 	// checks are the client-applied posture checks that need to be evaluated on the client
 	checks []*mgmProto.Checks
+
+	relayManager *relayClient.Manager
 }
 
 // Peer is an instance of the Connection Peer
@@ -178,6 +181,7 @@ func NewEngine(
 	clientCancel context.CancelFunc,
 	signalClient signal.Client,
 	mgmClient mgm.Client,
+	relayManager *relayClient.Manager,
 	config *EngineConfig,
 	mobileDep MobileDependency,
 	statusRecorder *peer.Status,
@@ -188,13 +192,11 @@ func NewEngine(
 		clientCancel,
 		signalClient,
 		mgmClient,
+		relayManager,
 		config,
 		mobileDep,
 		statusRecorder,
 		nil,
-		nil,
-		nil,
-		nil,
 		checks,
 	)
 }
@@ -205,21 +207,20 @@ func NewEngineWithProbes(
 	clientCancel context.CancelFunc,
 	signalClient signal.Client,
 	mgmClient mgm.Client,
+	relayManager *relayClient.Manager,
 	config *EngineConfig,
 	mobileDep MobileDependency,
 	statusRecorder *peer.Status,
-	mgmProbe *Probe,
+	probes *ProbeHolder,
-	signalProbe *Probe,
-	relayProbe *Probe,
-	wgProbe *Probe,
 	checks []*mgmProto.Checks,
 ) *Engine {
-
 	return &Engine{
 		clientCtx:      clientCtx,
 		clientCancel:   clientCancel,
 		signal:         signalClient,
+		signaler:       peer.NewSignaler(signalClient, config.WgPrivateKey),
 		mgmClient:      mgmClient,
+		relayManager:   relayManager,
 		peerConns:      make(map[string]*peer.Conn),
 		syncMsgMux:     &sync.Mutex{},
 		config:         config,
@@ -229,22 +230,20 @@ func NewEngineWithProbes(
 		networkSerial:  0,
 		sshServerFunc:  nbssh.DefaultSSHServer,
 		statusRecorder: statusRecorder,
-		mgmProbe:       mgmProbe,
+		probes:         probes,
-		signalProbe:    signalProbe,
-		relayProbe:     relayProbe,
-		wgProbe:        wgProbe,
 		checks:         checks,
 	}
 }
 
 func (e *Engine) Stop() error {
+	if e == nil {
+		// this seems to be a very odd case but there was the possibility if the netbird down command comes before the engine is fully started
+		log.Debugf("tried stopping engine that is nil")
+		return nil
+	}
 	e.syncMsgMux.Lock()
 	defer e.syncMsgMux.Unlock()
 
-	if e.cancel != nil {
-		e.cancel()
-	}
-
 	// stopping network monitor first to avoid starting the engine again
 	if e.networkMonitor != nil {
 		e.networkMonitor.Stop()
@@ -253,19 +252,22 @@ func (e *Engine) Stop() error {
 
 	err := e.removeAllPeers()
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to remove all peers: %s", err)
 	}
 
 	e.clientRoutesMu.Lock()
 	e.clientRoutes = nil
 	e.clientRoutesMu.Unlock()
 
+	if e.cancel != nil {
+		e.cancel()
+	}
+
 	// very ugly but we want to remove peers from the WireGuard interface first before removing interface.
 	// Removing peers happens in the conn.Close() asynchronously
 	time.Sleep(500 * time.Millisecond)
 
 	e.close()
-	e.wgConnWorker.Wait()
 	log.Infof("stopped Netbird Engine")
 	return nil
 }
@@ -290,7 +292,7 @@ func (e *Engine) Start() error {
 	e.wgInterface = wgIface
 
 	userspace := e.wgInterface.IsUserspaceBind()
-	e.wgProxyFactory = wgproxy.NewFactory(e.ctx, userspace, e.config.WgPort)
+	e.wgProxyFactory = wgproxy.NewFactory(userspace, e.config.WgPort)
 
 	if e.config.RosenpassEnabled {
 		log.Infof("rosenpass is enabled")
@@ -316,7 +318,7 @@ func (e *Engine) Start() error {
 	}
 	e.dnsServer = dnsServer
 
-	e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.config.DNSRouteInterval, e.wgInterface, e.statusRecorder, initialRoutes)
+	e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.config.DNSRouteInterval, e.wgInterface, e.statusRecorder, e.relayManager, initialRoutes)
 	beforePeerHook, afterPeerHook, err := e.routeManager.Init()
 	if err != nil {
 		log.Errorf("Failed to initialize route manager: %s", err)
@@ -465,62 +467,8 @@ func (e *Engine) removePeer(peerKey string) error {
 	conn, exists := e.peerConns[peerKey]
 	if exists {
 		delete(e.peerConns, peerKey)
-		err := conn.Close()
+		conn.Close()
-		if err != nil {
-			switch err.(type) {
-			case *peer.ConnectionAlreadyClosedError:
-				return nil
-			default:
-				return err
 	}
-		}
-	}
-	return nil
-}
-
-func signalCandidate(candidate ice.Candidate, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client) error {
-	err := s.Send(&sProto.Message{
-		Key:       myKey.PublicKey().String(),
-		RemoteKey: remoteKey.String(),
-		Body: &sProto.Body{
-			Type:    sProto.Body_CANDIDATE,
-			Payload: candidate.Marshal(),
-		},
-	})
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func sendSignal(message *sProto.Message, s signal.Client) error {
-	return s.Send(message)
-}
-
-// SignalOfferAnswer signals either an offer or an answer to remote peer
-func SignalOfferAnswer(offerAnswer peer.OfferAnswer, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client,
-	isAnswer bool) error {
-	var t sProto.Body_Type
-	if isAnswer {
-		t = sProto.Body_ANSWER
-	} else {
-		t = sProto.Body_OFFER
-	}
-
-	msg, err := signal.MarshalCredential(myKey, offerAnswer.WgListenPort, remoteKey, &signal.Credential{
-		UFrag: offerAnswer.IceCredentials.UFrag,
-		Pwd:   offerAnswer.IceCredentials.Pwd,
-	}, t, offerAnswer.RosenpassPubKey, offerAnswer.RosenpassAddr)
-	if err != nil {
-		return err
-	}
-
-	err = s.Send(msg)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -529,16 +477,35 @@ func (e *Engine) handleSync(update *mgmProto.SyncResponse) error {
 	defer e.syncMsgMux.Unlock()
 
 	if update.GetWiretrusteeConfig() != nil {
-		err := e.updateTURNs(update.GetWiretrusteeConfig().GetTurns())
+		wCfg := update.GetWiretrusteeConfig()
+		err := e.updateTURNs(wCfg.GetTurns())
 		if err != nil {
-			return err
+			return fmt.Errorf("update TURNs: %w", err)
 		}
 
-		err = e.updateSTUNs(update.GetWiretrusteeConfig().GetStuns())
+		err = e.updateSTUNs(wCfg.GetStuns())
 		if err != nil {
-			return err
+			return fmt.Errorf("update STUNs: %w", err)
 		}
 
+		var stunTurn []*stun.URI
+		stunTurn = append(stunTurn, e.STUNs...)
+		stunTurn = append(stunTurn, e.TURNs...)
+		e.stunTurn.Store(stunTurn)
+
+		relayMsg := wCfg.GetRelay()
+		if relayMsg != nil {
+			c := &auth.Token{
+				Payload:   relayMsg.GetTokenPayload(),
+				Signature: relayMsg.GetTokenSignature(),
+			}
+			if err := e.relayManager.UpdateToken(c); err != nil {
+				log.Errorf("failed to update relay token: %v", err)
+				return fmt.Errorf("update relay token: %w", err)
+			}
+		}
+
+		// todo update relay address in the relay manager
 		// todo update signal
 	}
 
@@ -934,68 +901,13 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error {
 			log.Warnf("error adding peer %s to status recorder, got error: %v", peerKey, err)
 		}
 
-		e.wgConnWorker.Add(1)
+		conn.Open()
-		go e.connWorker(conn, peerKey)
 	}
 	return nil
 }
 
-func (e *Engine) connWorker(conn *peer.Conn, peerKey string) {
-	defer e.wgConnWorker.Done()
-	for {
-
-		// randomize starting time a bit
-		min := 500
-		max := 2000
-		duration := time.Duration(rand.Intn(max-min)+min) * time.Millisecond
-		select {
-		case <-e.ctx.Done():
-			return
-		case <-time.After(duration):
-		}
-
-		// if peer has been removed -> give up
-		if !e.peerExists(peerKey) {
-			log.Debugf("peer %s doesn't exist anymore, won't retry connection", peerKey)
-			return
-		}
-
-		if !e.signal.Ready() {
-			log.Infof("signal client isn't ready, skipping connection attempt %s", peerKey)
-			continue
-		}
-
-		// we might have received new STUN and TURN servers meanwhile, so update them
-		e.syncMsgMux.Lock()
-		conn.UpdateStunTurn(append(e.STUNs, e.TURNs...))
-		e.syncMsgMux.Unlock()
-
-		err := conn.Open(e.ctx)
-		if err != nil {
-			log.Debugf("connection to peer %s failed: %v", peerKey, err)
-			var connectionClosedError *peer.ConnectionClosedError
-			switch {
-			case errors.As(err, &connectionClosedError):
-				// conn has been forced to close, so we exit the loop
-				return
-			default:
-			}
-		}
-	}
-}
-
-func (e *Engine) peerExists(peerKey string) bool {
-	e.syncMsgMux.Lock()
-	defer e.syncMsgMux.Unlock()
-	_, ok := e.peerConns[peerKey]
-	return ok
-}
-
 func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, error) {
 	log.Debugf("creating peer connection %s", pubKey)
-	var stunTurn []*stun.URI
-	stunTurn = append(stunTurn, e.STUNs...)
-	stunTurn = append(stunTurn, e.TURNs...)
 
 	wgConfig := peer.WgConfig{
 		RemoteKey:    pubKey,
@@ -1030,50 +942,27 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e
 	config := peer.ConnConfig{
 		Key:             pubKey,
 		LocalKey:        e.config.WgPrivateKey.PublicKey().String(),
-		StunTurn:             stunTurn,
-		InterfaceBlackList:   e.config.IFaceBlackList,
-		DisableIPv6Discovery: e.config.DisableIPv6Discovery,
 		Timeout:         timeout,
-		UDPMux:               e.udpMux.UDPMuxDefault,
-		UDPMuxSrflx:          e.udpMux,
 		WgConfig:        wgConfig,
 		LocalWgPort:     e.config.WgPort,
-		NATExternalIPs:       e.parseNATExternalIPMappings(),
 		RosenpassPubKey: e.getRosenpassPubKey(),
 		RosenpassAddr:   e.getRosenpassAddr(),
+		ICEConfig: peer.ICEConfig{
+			StunTurn:             &e.stunTurn,
+			InterfaceBlackList:   e.config.IFaceBlackList,
+			DisableIPv6Discovery: e.config.DisableIPv6Discovery,
+			UDPMux:               e.udpMux.UDPMuxDefault,
+			UDPMuxSrflx:          e.udpMux,
+			NATExternalIPs:       e.parseNATExternalIPMappings(),
+		},
 	}
 
-	peerConn, err := peer.NewConn(config, e.statusRecorder, e.wgProxyFactory, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover)
+	peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.wgProxyFactory, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager)
 	if err != nil {
 		return nil, err
 	}
 
-	wgPubKey, err := wgtypes.ParseKey(pubKey)
-	if err != nil {
-		return nil, err
-	}
-
-	signalOffer := func(offerAnswer peer.OfferAnswer) error {
-		return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, false)
-	}
-
-	signalCandidate := func(candidate ice.Candidate) error {
-		return signalCandidate(candidate, e.config.WgPrivateKey, wgPubKey, e.signal)
-	}
-
-	signalAnswer := func(offerAnswer peer.OfferAnswer) error {
-		return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, true)
-	}
-
-	peerConn.SetSignalCandidate(signalCandidate)
-	peerConn.SetSignalOffer(signalOffer)
-	peerConn.SetSignalAnswer(signalAnswer)
-	peerConn.SetSendSignalMessage(func(message *sProto.Message) error {
-		return sendSignal(message, e.signal)
-	})
-
 	if e.rpManager != nil {
-
 		peerConn.SetOnConnected(e.rpManager.OnConnected)
 		peerConn.SetOnDisconnected(e.rpManager.OnDisconnected)
 	}
@@ -1116,6 +1005,7 @@ func (e *Engine) receiveSignalEvents() {
 					Version:         msg.GetBody().GetNetBirdVersion(),
 					RosenpassPubKey: rosenpassPubKey,
 					RosenpassAddr:   rosenpassAddr,
+					RelaySrvAddress: msg.GetBody().GetRelayServerAddress(),
 				})
 			case sProto.Body_ANSWER:
 				remoteCred, err := signal.UnMarshalCredential(msg)
@@ -1138,6 +1028,7 @@ func (e *Engine) receiveSignalEvents() {
 					Version:         msg.GetBody().GetNetBirdVersion(),
 					RosenpassPubKey: rosenpassPubKey,
 					RosenpassAddr:   rosenpassAddr,
+					RelaySrvAddress: msg.GetBody().GetRelayServerAddress(),
 				})
 			case sProto.Body_CANDIDATE:
 				candidate, err := ice.UnmarshalCandidate(msg.GetBody().Payload)
@@ -1146,7 +1037,7 @@ func (e *Engine) receiveSignalEvents() {
 					return err
 				}
 
-				conn.OnRemoteCandidate(candidate, e.GetClientRoutes())
+				go conn.OnRemoteCandidate(candidate, e.GetClientRoutes())
 			case sProto.Body_MODE:
 			}
 
@@ -1224,10 +1115,7 @@ func (e *Engine) close() {
 	}
 
 	// stop/restore DNS first so dbus and friends don't complain because of a missing interface
-	if e.dnsServer != nil {
+	e.stopDNSServer()
-		e.dnsServer.Stop()
-		e.dnsServer = nil
-	}
 
 	if e.routeManager != nil {
 		e.routeManager.Stop()
@@ -1400,24 +1288,27 @@ func (e *Engine) getRosenpassAddr() string {
 }
 
 func (e *Engine) receiveProbeEvents() {
-	if e.signalProbe != nil {
+	if e.probes == nil {
-		go e.signalProbe.Receive(e.ctx, func() bool {
+		return
+	}
+	if e.probes.SignalProbe != nil {
+		go e.probes.SignalProbe.Receive(e.ctx, func() bool {
 			healthy := e.signal.IsHealthy()
 			log.Debugf("received signal probe request, healthy: %t", healthy)
 			return healthy
 		})
 	}
 
-	if e.mgmProbe != nil {
+	if e.probes.MgmProbe != nil {
-		go e.mgmProbe.Receive(e.ctx, func() bool {
+		go e.probes.MgmProbe.Receive(e.ctx, func() bool {
 			healthy := e.mgmClient.IsHealthy()
 			log.Debugf("received management probe request, healthy: %t", healthy)
 			return healthy
 		})
 	}
 
-	if e.relayProbe != nil {
+	if e.probes.RelayProbe != nil {
-		go e.relayProbe.Receive(e.ctx, func() bool {
+		go e.probes.RelayProbe.Receive(e.ctx, func() bool {
 			healthy := true
 
 			results := append(e.probeSTUNs(), e.probeTURNs()...)
@@ -1436,13 +1327,13 @@ func (e *Engine) receiveProbeEvents() {
 		})
 	}
 
-	if e.wgProbe != nil {
+	if e.probes.WgProbe != nil {
-		go e.wgProbe.Receive(e.ctx, func() bool {
+		go e.probes.WgProbe.Receive(e.ctx, func() bool {
 			log.Debug("received wg probe request")
 
 			for _, peer := range e.peerConns {
 				key := peer.GetKey()
-				wgStats, err := peer.GetConf().WgConfig.WgInterface.GetStats(key)
+				wgStats, err := peer.WgConfig().WgInterface.GetStats(key)
 				if err != nil {
 					log.Debugf("failed to get wg stats for peer %s: %s", key, err)
 				}
@@ -1466,12 +1357,16 @@ func (e *Engine) probeTURNs() []relay.ProbeResult {
 }
 
 func (e *Engine) restartEngine() {
+	log.Info("restarting engine")
+	CtxGetState(e.ctx).Set(StatusConnecting)
+
 	if err := e.Stop(); err != nil {
 		log.Errorf("Failed to stop engine: %v", err)
 	}
-	if err := e.Start(); err != nil {
+
-		log.Errorf("Failed to start engine: %v", err)
+	_ = CtxGetState(e.ctx).Wrap(ErrResetConnection)
-	}
+	log.Infof("cancelling client, engine will be recreated")
+	e.clientCancel()
 }
 
 func (e *Engine) startNetworkMonitor() {
@@ -1493,6 +1388,7 @@ func (e *Engine) startNetworkMonitor() {
 			defer mu.Unlock()
 
 			if debounceTimer != nil {
+				log.Infof("Network monitor: detected network change, reset debounceTimer")
 				debounceTimer.Stop()
 			}
 
@@ -1502,7 +1398,7 @@ func (e *Engine) startNetworkMonitor() {
 				mu.Lock()
 				defer mu.Unlock()
 
-				log.Infof("Network monitor detected network change, restarting engine")
+				log.Infof("Network monitor: detected network change, restarting engine")
 				e.restartEngine()
 			})
 		})
@@ -1527,6 +1423,20 @@ func (e *Engine) addrViaRoutes(addr netip.Addr) (bool, netip.Prefix, error) {
 	return false, netip.Prefix{}, nil
 }
 
+func (e *Engine) stopDNSServer() {
+	err := fmt.Errorf("DNS server stopped")
+	nsGroupStates := e.statusRecorder.GetDNSStates()
+	for i := range nsGroupStates {
+		nsGroupStates[i].Enabled = false
+		nsGroupStates[i].Error = err
+	}
+	e.statusRecorder.UpdateDNSStates(nsGroupStates)
+	if e.dnsServer != nil {
+		e.dnsServer.Stop()
+		e.dnsServer = nil
+	}
+}
+
 // isChecksEqual checks if two slices of checks are equal.
 func isChecksEqual(checks []*mgmProto.Checks, oChecks []*mgmProto.Checks) bool {
 	return slices.EqualFunc(checks, oChecks, func(checks, oChecks *mgmProto.Checks) bool {

client/internal/engine_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/pion/transport/v3/stdnet"
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -36,6 +37,8 @@ import (
 	mgmtProto "github.com/netbirdio/netbird/management/proto"
 	"github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/activity"
+	"github.com/netbirdio/netbird/management/server/telemetry"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	signal "github.com/netbirdio/netbird/signal/client"
 	"github.com/netbirdio/netbird/signal/proto"
@@ -57,6 +60,12 @@ var (
 	}
 )
 
+func TestMain(m *testing.M) {
+	_ = util.InitLog("debug", "console")
+	code := m.Run()
+	os.Exit(code)
+}
+
 func TestEngine_SSH(t *testing.T) {
 	// todo resolve test execution on freebsd
 	if runtime.GOOS == "windows" || runtime.GOOS == "freebsd" {
@@ -72,13 +81,23 @@ func TestEngine_SSH(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+	relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
+	engine := NewEngine(
+		ctx, cancel,
+		&signal.MockClient{},
+		&mgmt.MockClient{},
+		relayMgr,
+		&EngineConfig{
 			WgIfaceName:      "utun101",
 			WgAddr:           "100.64.0.1/24",
 			WgPrivateKey:     key,
 			WgPort:           33100,
 			ServerSSHAllowed: true,
-	}, MobileDependency{}, peer.NewRecorder("https://mgm"), nil)
+		},
+		MobileDependency{},
+		peer.NewRecorder("https://mgm"),
+		nil,
+	)
 
 	engine.dnsServer = &dns.MockServer{
 		UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil },
@@ -207,21 +226,29 @@ func TestEngine_UpdateNetworkMap(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+	relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
+	engine := NewEngine(
+		ctx, cancel,
+		&signal.MockClient{},
+		&mgmt.MockClient{},
+		relayMgr,
+		&EngineConfig{
 			WgIfaceName:  "utun102",
 			WgAddr:       "100.64.0.1/24",
 			WgPrivateKey: key,
 			WgPort:       33100,
-	}, MobileDependency{}, peer.NewRecorder("https://mgm"), nil)
+		},
-	newNet, err := stdnet.NewNet()
+		MobileDependency{},
-	if err != nil {
+		peer.NewRecorder("https://mgm"),
-		t.Fatal(err)
+		nil)
+
+	wgIface := &iface.MockWGIface{
+		RemovePeerFunc: func(peerKey string) error {
+			return nil
+		},
 	}
-	engine.wgInterface, err = iface.NewWGIFace("utun102", "100.64.0.1/24", engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil, nil)
+	engine.wgInterface = wgIface
-	if err != nil {
+	engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), time.Minute, engine.wgInterface, engine.statusRecorder, relayMgr, nil)
-		t.Fatal(err)
-	}
-	engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), time.Minute, engine.wgInterface, engine.statusRecorder, nil)
 	engine.dnsServer = &dns.MockServer{
 		UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil },
 	}
@@ -403,8 +430,8 @@ func TestEngine_Sync(t *testing.T) {
 		}
 		return nil
 	}
-
+	relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
-	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, &EngineConfig{
+	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, relayMgr, &EngineConfig{
 		WgIfaceName:  "utun103",
 		WgAddr:       "100.64.0.1/24",
 		WgPrivateKey: key,
@@ -563,7 +590,8 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) {
 			wgIfaceName := fmt.Sprintf("utun%d", 104+n)
 			wgAddr := fmt.Sprintf("100.66.%d.1/24", n)
 
-			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+			relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
+			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{
 				WgIfaceName:  wgIfaceName,
 				WgAddr:       wgAddr,
 				WgPrivateKey: key,
@@ -733,7 +761,8 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) {
 			wgIfaceName := fmt.Sprintf("utun%d", 104+n)
 			wgAddr := fmt.Sprintf("100.66.%d.1/24", n)
 
-			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+			relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
+			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{
 				WgIfaceName:  wgIfaceName,
 				WgAddr:       wgAddr,
 				WgPrivateKey: key,
@@ -844,6 +873,8 @@ func TestEngine_MultiplePeers(t *testing.T) {
 			engine.dnsServer = &dns.MockServer{}
 			mu.Lock()
 			defer mu.Unlock()
+			guid := fmt.Sprintf("{%s}", uuid.New().String())
+			iface.CustomWindowsGUIDString = strings.ToLower(guid)
 			err = engine.Start()
 			if err != nil {
 				t.Errorf("unable to start engine for peer %d with error %v", j, err)
@@ -1009,7 +1040,8 @@ func createEngine(ctx context.Context, cancel context.CancelFunc, setupKey strin
 		WgPort:       wgPort,
 	}
 
-	e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil
+	relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
+	e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, relayMgr, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil
 	e.ctx = ctx
 	return e, err
 }
@@ -1043,6 +1075,11 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error)
 	config := &server.Config{
 		Stuns:      []*server.Host{},
 		TURNConfig: &server.TURNConfig{},
+		Relay: &server.Relay{
+			Addresses:      []string{"127.0.0.1:1234"},
+			CredentialsTTL: util.Duration{Duration: time.Hour},
+			Secret:         "222222222222222222",
+		},
 		Signal: &server.Host{
 			Proto: "http",
 			URI:   "localhost:10000",
@@ -1069,12 +1106,17 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error)
 		return nil, "", err
 	}
 	ia, _ := integrations.NewIntegratedValidator(context.Background(), eventStore)
-	accountManager, err := server.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia)
+
+	metrics, err := telemetry.NewDefaultAppMetrics(context.Background())
+	require.NoError(t, err)
+
+	accountManager, err := server.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia, metrics)
 	if err != nil {
 		return nil, "", err
 	}
-	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
-	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
+	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil)
 	if err != nil {
 		return nil, "", err
 	}

client/internal/networkmonitor/monitor_bsd.go

@@ -4,6 +4,7 @@ package networkmonitor
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"syscall"
 	"unsafe"
@@ -21,8 +22,17 @@ func checkChange(ctx context.Context, nexthopv4, nexthopv6 systemops.Nexthop, ca
 		return fmt.Errorf("failed to open routing socket: %v", err)
 	}
 	defer func() {
-		if err := unix.Close(fd); err != nil {
+		err := unix.Close(fd)
-			log.Errorf("Network monitor: failed to close routing socket: %v", err)
+		if err != nil && !errors.Is(err, unix.EBADF) {
+			log.Warnf("Network monitor: failed to close routing socket: %v", err)
+		}
+	}()
+
+	go func() {
+		<-ctx.Done()
+		err := unix.Close(fd)
+		if err != nil && !errors.Is(err, unix.EBADF) {
+			log.Debugf("Network monitor: closed routing socket: %v", err)
 		}
 	}()
 
@@ -34,11 +44,13 @@ func checkChange(ctx context.Context, nexthopv4, nexthopv6 systemops.Nexthop, ca
 			buf := make([]byte, 2048)
 			n, err := unix.Read(fd, buf)
 			if err != nil {
-				log.Errorf("Network monitor: failed to read from routing socket: %v", err)
+				if !errors.Is(err, unix.EBADF) && !errors.Is(err, unix.EINVAL) {
+					log.Warnf("Network monitor: failed to read from routing socket: %v", err)
+				}
 				continue
 			}
 			if n < unix.SizeofRtMsghdr {
-				log.Errorf("Network monitor: read from routing socket returned less than expected: %d bytes", n)
+				log.Debugf("Network monitor: read from routing socket returned less than expected: %d bytes", n)
 				continue
 			}
 
@@ -49,11 +61,11 @@ func checkChange(ctx context.Context, nexthopv4, nexthopv6 systemops.Nexthop, ca
 			case unix.RTM_ADD, syscall.RTM_DELETE:
 				route, err := parseRouteMessage(buf[:n])
 				if err != nil {
-					log.Errorf("Network monitor: error parsing routing message: %v", err)
+					log.Debugf("Network monitor: error parsing routing message: %v", err)
 					continue
 				}
 
-				if !route.Dst.Addr().IsUnspecified() {
+				if route.Dst.Bits() != 0 {
 					continue
 				}
 

client/internal/networkmonitor/monitor_generic.go

@@ -59,7 +59,7 @@ func (nw *NetworkMonitor) Start(ctx context.Context, callback func()) (err error
 	// recover in case sys ops panic
 	defer func() {
 		if r := recover(); r != nil {
-			err = fmt.Errorf("panic occurred: %v, stack trace: %s", r, string(debug.Stack()))
+			err = fmt.Errorf("panic occurred: %v, stack trace: %s", r, debug.Stack())
 		}
 	}()
 

client/internal/networkmonitor/monitor_windows.go

@@ -3,243 +3,73 @@ package networkmonitor
 import (
 	"context"
 	"fmt"
-	"net"
-	"net/netip"
 	"strings"
-	"time"
 
 	log "github.com/sirupsen/logrus"
 
 	"github.com/netbirdio/netbird/client/internal/routemanager/systemops"
 )
 
-const (
-	unreachable = 0
-	incomplete  = 1
-	probe       = 2
-	delay       = 3
-	stale       = 4
-	reachable   = 5
-	permanent   = 6
-	tbd         = 7
-)
-
-const interval = 10 * time.Second
-
 func checkChange(ctx context.Context, nexthopv4, nexthopv6 systemops.Nexthop, callback func()) error {
-	var neighborv4, neighborv6 *systemops.Neighbor
+	routeMonitor, err := systemops.NewRouteMonitor(ctx)
-	{
-		initialNeighbors, err := getNeighbors()
 	if err != nil {
-			return fmt.Errorf("get neighbors: %w", err)
+		return fmt.Errorf("failed to create route monitor: %w", err)
 	}
-
+	defer func() {
-		neighborv4 = assignNeighbor(nexthopv4, initialNeighbors)
+		if err := routeMonitor.Stop(); err != nil {
-		neighborv6 = assignNeighbor(nexthopv6, initialNeighbors)
+			log.Errorf("Network monitor: failed to stop route monitor: %v", err)
 		}
-	log.Debugf("Network monitor: initial IPv4 neighbor: %v, IPv6 neighbor: %v", neighborv4, neighborv6)
+	}()
-
-	ticker := time.NewTicker(interval)
-	defer ticker.Stop()
 
 	for {
 		select {
 		case <-ctx.Done():
 			return ErrStopped
-		case <-ticker.C:
+		case route := <-routeMonitor.RouteUpdates():
-			if changed(nexthopv4, neighborv4, nexthopv6, neighborv6) {
+			if route.Destination.Bits() != 0 {
+				continue
+			}
+
+			if routeChanged(route, nexthopv4, nexthopv6, callback) {
+				break
+			}
+		}
+	}
+}
+
+func routeChanged(route systemops.RouteUpdate, nexthopv4, nexthopv6 systemops.Nexthop, callback func()) bool {
+	intf := "<nil>"
+	if route.Interface != nil {
+		intf = route.Interface.Name
+		if isSoftInterface(intf) {
+			log.Debugf("Network monitor: ignoring default route change for soft interface %s", intf)
+			return false
+		}
+	}
+
+	switch route.Type {
+	case systemops.RouteModified:
+		// TODO: get routing table to figure out if our route is affected for modified routes
+		log.Infof("Network monitor: default route changed: via %s, interface %s", route.NextHop, intf)
+		go callback()
+		return true
+	case systemops.RouteAdded:
+		if route.NextHop.Is4() && route.NextHop != nexthopv4.IP || route.NextHop.Is6() && route.NextHop != nexthopv6.IP {
+			log.Infof("Network monitor: default route added: via %s, interface %s", route.NextHop, intf)
 			go callback()
-				return nil
-			}
-		}
-	}
-}
-
-func assignNeighbor(nexthop systemops.Nexthop, initialNeighbors map[netip.Addr]systemops.Neighbor) *systemops.Neighbor {
-	if n, ok := initialNeighbors[nexthop.IP]; ok &&
-		n.State != unreachable &&
-		n.State != incomplete &&
-		n.State != tbd {
-		return &n
-	}
-	return nil
-}
-
-func changed(
-	nexthopv4 systemops.Nexthop,
-	neighborv4 *systemops.Neighbor,
-	nexthopv6 systemops.Nexthop,
-	neighborv6 *systemops.Neighbor,
-) bool {
-	neighbors, err := getNeighbors()
-	if err != nil {
-		log.Errorf("network monitor: error fetching current neighbors: %v", err)
-		return false
-	}
-	if neighborChanged(nexthopv4, neighborv4, neighbors) || neighborChanged(nexthopv6, neighborv6, neighbors) {
 			return true
 		}
-
+	case systemops.RouteDeleted:
-	routes, err := getRoutes()
+		if nexthopv4.Intf != nil && route.NextHop == nexthopv4.IP || nexthopv6.Intf != nil && route.NextHop == nexthopv6.IP {
-	if err != nil {
+			log.Infof("Network monitor: default route removed: via %s, interface %s", route.NextHop, intf)
-		log.Errorf("network monitor: error fetching current routes: %v", err)
+			go callback()
-		return false
-	}
-
-	if routeChanged(nexthopv4, nexthopv4.Intf, routes) || routeChanged(nexthopv6, nexthopv6.Intf, routes) {
 			return true
 		}
-
-	return false
-}
-
-// routeChanged checks if the default routes still point to our nexthop/interface
-func routeChanged(nexthop systemops.Nexthop, intf *net.Interface, routes []systemops.Route) bool {
-	if !nexthop.IP.IsValid() {
-		return false
-	}
-
-	unspec := getUnspecifiedPrefix(nexthop.IP)
-	defaultRoutes, foundMatchingRoute := processRoutes(nexthop, intf, routes, unspec)
-
-	log.Tracef("network monitor: all default routes:\n%s", strings.Join(defaultRoutes, "\n"))
-
-	if !foundMatchingRoute {
-		logRouteChange(nexthop.IP, intf)
-		return true
 	}
 
 	return false
 }
 
-func getUnspecifiedPrefix(ip netip.Addr) netip.Prefix {
+func isSoftInterface(name string) bool {
-	if ip.Is6() {
+	return strings.Contains(strings.ToLower(name), "isatap") || strings.Contains(strings.ToLower(name), "teredo")
-		return netip.PrefixFrom(netip.IPv6Unspecified(), 0)
-	}
-	return netip.PrefixFrom(netip.IPv4Unspecified(), 0)
-}
-
-func processRoutes(nexthop systemops.Nexthop, intf *net.Interface, routes []systemops.Route, unspec netip.Prefix) ([]string, bool) {
-	var defaultRoutes []string
-	foundMatchingRoute := false
-
-	for _, r := range routes {
-		if r.Destination == unspec {
-			routeInfo := formatRouteInfo(r)
-			defaultRoutes = append(defaultRoutes, routeInfo)
-
-			if r.Nexthop == nexthop.IP && compareIntf(r.Interface, intf) == 0 {
-				foundMatchingRoute = true
-				log.Debugf("network monitor: found matching default route: %s", routeInfo)
-			}
-		}
-	}
-
-	return defaultRoutes, foundMatchingRoute
-}
-
-func formatRouteInfo(r systemops.Route) string {
-	newIntf := "<nil>"
-	if r.Interface != nil {
-		newIntf = r.Interface.Name
-	}
-	return fmt.Sprintf("Nexthop: %s, Interface: %s", r.Nexthop, newIntf)
-}
-
-func logRouteChange(ip netip.Addr, intf *net.Interface) {
-	oldIntf := "<nil>"
-	if intf != nil {
-		oldIntf = intf.Name
-	}
-	log.Infof("network monitor: default route for %s (%s) is gone or changed", ip, oldIntf)
-}
-
-func neighborChanged(nexthop systemops.Nexthop, neighbor *systemops.Neighbor, neighbors map[netip.Addr]systemops.Neighbor) bool {
-	if neighbor == nil {
-		return false
-	}
-
-	// TODO: consider non-local nexthops, e.g. on point-to-point interfaces
-	if n, ok := neighbors[nexthop.IP]; ok {
-		if n.State == unreachable || n.State == incomplete {
-			log.Infof("network monitor: neighbor %s (%s) is not reachable: %s", neighbor.IPAddress, neighbor.LinkLayerAddress, stateFromInt(n.State))
-			return true
-		} else if n.InterfaceIndex != neighbor.InterfaceIndex {
-			log.Infof(
-				"network monitor: neighbor %s (%s) changed interface from '%s' (%d) to '%s' (%d): %s",
-				neighbor.IPAddress,
-				neighbor.LinkLayerAddress,
-				neighbor.InterfaceAlias,
-				neighbor.InterfaceIndex,
-				n.InterfaceAlias,
-				n.InterfaceIndex,
-				stateFromInt(n.State),
-			)
-			return true
-		}
-	} else {
-		log.Infof("network monitor: neighbor %s (%s) is gone", neighbor.IPAddress, neighbor.LinkLayerAddress)
-		return true
-	}
-
-	return false
-}
-
-func getNeighbors() (map[netip.Addr]systemops.Neighbor, error) {
-	entries, err := systemops.GetNeighbors()
-	if err != nil {
-		return nil, fmt.Errorf("get neighbors: %w", err)
-	}
-
-	neighbours := make(map[netip.Addr]systemops.Neighbor, len(entries))
-	for _, entry := range entries {
-		neighbours[entry.IPAddress] = entry
-	}
-
-	return neighbours, nil
-}
-
-func getRoutes() ([]systemops.Route, error) {
-	entries, err := systemops.GetRoutes()
-	if err != nil {
-		return nil, fmt.Errorf("get routes: %w", err)
-	}
-
-	return entries, nil
-}
-
-func stateFromInt(state uint8) string {
-	switch state {
-	case unreachable:
-		return "unreachable"
-	case incomplete:
-		return "incomplete"
-	case probe:
-		return "probe"
-	case delay:
-		return "delay"
-	case stale:
-		return "stale"
-	case reachable:
-		return "reachable"
-	case permanent:
-		return "permanent"
-	case tbd:
-		return "tbd"
-	default:
-		return "unknown"
-	}
-}
-
-func compareIntf(a, b *net.Interface) int {
-	if a == nil && b == nil {
-		return 0
-	}
-	if a == nil {
-		return -1
-	}
-	if b == nil {
-		return 1
-	}
-	return a.Index - b.Index
 }

client/internal/peer/conn_status.go

@@ -1,6 +1,10 @@
 package peer
 
-import log "github.com/sirupsen/logrus"
+import (
+	"sync/atomic"
+
+	log "github.com/sirupsen/logrus"
+)
 
 const (
 	// StatusConnected indicate the peer is in connected state
@@ -12,7 +16,34 @@ const (
 )
 
 // ConnStatus describe the status of a peer's connection
-type ConnStatus int
+type ConnStatus int32
+
+// AtomicConnStatus is a thread-safe wrapper for ConnStatus
+type AtomicConnStatus struct {
+	status atomic.Int32
+}
+
+// NewAtomicConnStatus creates a new AtomicConnStatus with the given initial status
+func NewAtomicConnStatus() *AtomicConnStatus {
+	acs := &AtomicConnStatus{}
+	acs.Set(StatusDisconnected)
+	return acs
+}
+
+// Get returns the current connection status
+func (acs *AtomicConnStatus) Get() ConnStatus {
+	return ConnStatus(acs.status.Load())
+}
+
+// Set updates the connection status
+func (acs *AtomicConnStatus) Set(status ConnStatus) {
+	acs.status.Store(int32(status))
+}
+
+// String returns the string representation of the current status
+func (acs *AtomicConnStatus) String() string {
+	return acs.Get().String()
+}
 
 func (s ConnStatus) String() string {
 	switch s {

client/internal/peer/conn_test.go

@@ -2,25 +2,33 @@ package peer
 
 import (
 	"context"
+	"os"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/magiconair/properties/assert"
-	"github.com/pion/stun/v2"
 
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	"github.com/netbirdio/netbird/client/internal/wgproxy"
 	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/util"
 )
 
 var connConf = ConnConfig{
 	Key:         "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
 	LocalKey:    "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
-	StunTurn:           []*stun.URI{},
-	InterfaceBlackList: nil,
 	Timeout:     time.Second,
 	LocalWgPort: 51820,
+	ICEConfig: ICEConfig{
+		InterfaceBlackList: nil,
+	},
+}
+
+func TestMain(m *testing.M) {
+	_ = util.InitLog("trace", "console")
+	code := m.Run()
+	os.Exit(code)
 }
 
 func TestNewConn_interfaceFilter(t *testing.T) {
@@ -36,11 +44,11 @@ func TestNewConn_interfaceFilter(t *testing.T) {
 }
 
 func TestConn_GetKey(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(context.Background(), false, connConf.LocalWgPort)
+	wgProxyFactory := wgproxy.NewFactory(false, connConf.LocalWgPort)
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, nil, wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, nil, wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
@@ -51,11 +59,11 @@ func TestConn_GetKey(t *testing.T) {
 }
 
 func TestConn_OnRemoteOffer(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(context.Background(), false, connConf.LocalWgPort)
+	wgProxyFactory := wgproxy.NewFactory(false, connConf.LocalWgPort)
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
@@ -63,7 +71,7 @@ func TestConn_OnRemoteOffer(t *testing.T) {
 	wg := sync.WaitGroup{}
 	wg.Add(2)
 	go func() {
-		<-conn.remoteOffersCh
+		<-conn.handshaker.remoteOffersCh
 		wg.Done()
 	}()
 
@@ -88,11 +96,11 @@ func TestConn_OnRemoteOffer(t *testing.T) {
 }
 
 func TestConn_OnRemoteAnswer(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(context.Background(), false, connConf.LocalWgPort)
+	wgProxyFactory := wgproxy.NewFactory(false, connConf.LocalWgPort)
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
@@ -100,7 +108,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) {
 	wg := sync.WaitGroup{}
 	wg.Add(2)
 	go func() {
-		<-conn.remoteAnswerCh
+		<-conn.handshaker.remoteAnswerCh
 		wg.Done()
 	}()
 
@@ -124,62 +132,42 @@ func TestConn_OnRemoteAnswer(t *testing.T) {
 	wg.Wait()
 }
 func TestConn_Status(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(context.Background(), false, connConf.LocalWgPort)
+	wgProxyFactory := wgproxy.NewFactory(false, connConf.LocalWgPort)
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
 
 	tables := []struct {
 		name        string
-		status ConnStatus
+		statusIce   ConnStatus
+		statusRelay ConnStatus
 		want        ConnStatus
 	}{
-		{"StatusConnected", StatusConnected, StatusConnected},
+		{"StatusConnected", StatusConnected, StatusConnected, StatusConnected},
-		{"StatusDisconnected", StatusDisconnected, StatusDisconnected},
+		{"StatusDisconnected", StatusDisconnected, StatusDisconnected, StatusDisconnected},
-		{"StatusConnecting", StatusConnecting, StatusConnecting},
+		{"StatusConnecting", StatusConnecting, StatusConnecting, StatusConnecting},
+		{"StatusConnectingIce", StatusConnecting, StatusDisconnected, StatusConnecting},
+		{"StatusConnectingIceAlternative", StatusConnecting, StatusConnected, StatusConnected},
+		{"StatusConnectingRelay", StatusDisconnected, StatusConnecting, StatusConnecting},
+		{"StatusConnectingRelayAlternative", StatusConnected, StatusConnecting, StatusConnected},
 	}
 
 	for _, table := range tables {
 		t.Run(table.name, func(t *testing.T) {
-			conn.status = table.status
+			si := NewAtomicConnStatus()
+			si.Set(table.statusIce)
+			conn.statusICE = si
+
+			sr := NewAtomicConnStatus()
+			sr.Set(table.statusRelay)
+			conn.statusRelay = sr
 
 			got := conn.Status()
 			assert.Equal(t, got, table.want, "they should be equal")
 		})
 	}
 }
-
-func TestConn_Close(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(context.Background(), false, connConf.LocalWgPort)
-	defer func() {
-		_ = wgProxyFactory.Free()
-	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
-	if err != nil {
-		return
-	}
-
-	wg := sync.WaitGroup{}
-	wg.Add(1)
-	go func() {
-		<-conn.closeCh
-		wg.Done()
-	}()
-
-	go func() {
-		for {
-			err := conn.Close()
-			if err != nil {
-				continue
-			} else {
-				return
-			}
-		}
-	}()
-
-	wg.Wait()
-}

client/internal/peer/handshaker.go

@@ -0,0 +1,192 @@
+package peer
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/version"
+)
+
+var (
+	ErrSignalIsNotReady = errors.New("signal is not ready")
+)
+
+// IceCredentials ICE protocol credentials struct
+type IceCredentials struct {
+	UFrag string
+	Pwd   string
+}
+
+// OfferAnswer represents a session establishment offer or answer
+type OfferAnswer struct {
+	IceCredentials IceCredentials
+	// WgListenPort is a remote WireGuard listen port.
+	// This field is used when establishing a direct WireGuard connection without any proxy.
+	// We can set the remote peer's endpoint with this port.
+	WgListenPort int
+
+	// Version of NetBird Agent
+	Version string
+	// RosenpassPubKey is the Rosenpass public key of the remote peer when receiving this message
+	// This value is the local Rosenpass server public key when sending the message
+	RosenpassPubKey []byte
+	// RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message
+	// This value is the local Rosenpass server address when sending the message
+	RosenpassAddr string
+
+	// relay server address
+	RelaySrvAddress string
+}
+
+type Handshaker struct {
+	mu                  sync.Mutex
+	ctx                 context.Context
+	log                 *log.Entry
+	config              ConnConfig
+	signaler            *Signaler
+	ice                 *WorkerICE
+	relay               *WorkerRelay
+	onNewOfferListeners []func(*OfferAnswer)
+
+	// remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection
+	remoteOffersCh chan OfferAnswer
+	// remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection
+	remoteAnswerCh chan OfferAnswer
+}
+
+func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ice *WorkerICE, relay *WorkerRelay) *Handshaker {
+	return &Handshaker{
+		ctx:            ctx,
+		log:            log,
+		config:         config,
+		signaler:       signaler,
+		ice:            ice,
+		relay:          relay,
+		remoteOffersCh: make(chan OfferAnswer),
+		remoteAnswerCh: make(chan OfferAnswer),
+	}
+}
+
+func (h *Handshaker) AddOnNewOfferListener(offer func(remoteOfferAnswer *OfferAnswer)) {
+	h.onNewOfferListeners = append(h.onNewOfferListeners, offer)
+}
+
+func (h *Handshaker) Listen() {
+	for {
+		h.log.Debugf("wait for remote offer confirmation")
+		remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation()
+		if err != nil {
+			var connectionClosedError *ConnectionClosedError
+			if errors.As(err, &connectionClosedError) {
+				h.log.Tracef("stop handshaker")
+				return
+			}
+			h.log.Errorf("failed to received remote offer confirmation: %s", err)
+			continue
+		}
+
+		h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort)
+		for _, listener := range h.onNewOfferListeners {
+			go listener(remoteOfferAnswer)
+		}
+	}
+}
+
+func (h *Handshaker) SendOffer() error {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.sendOffer()
+}
+
+// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
+// doesn't block, discards the message if connection wasn't ready
+func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool {
+	select {
+	case h.remoteOffersCh <- offer:
+		return true
+	default:
+		h.log.Debugf("OnRemoteOffer skipping message because is not ready")
+		// connection might not be ready yet to receive so we ignore the message
+		return false
+	}
+}
+
+// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
+// doesn't block, discards the message if connection wasn't ready
+func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool {
+	select {
+	case h.remoteAnswerCh <- answer:
+		return true
+	default:
+		// connection might not be ready yet to receive so we ignore the message
+		h.log.Debugf("OnRemoteAnswer skipping message because is not ready")
+		return false
+	}
+}
+
+func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) {
+	select {
+	case remoteOfferAnswer := <-h.remoteOffersCh:
+		// received confirmation from the remote peer -> ready to proceed
+		err := h.sendAnswer()
+		if err != nil {
+			return nil, err
+		}
+		return &remoteOfferAnswer, nil
+	case remoteOfferAnswer := <-h.remoteAnswerCh:
+		return &remoteOfferAnswer, nil
+	case <-h.ctx.Done():
+		// closed externally
+		return nil, NewConnectionClosedError(h.config.Key)
+	}
+}
+
+// sendOffer prepares local user credentials and signals them to the remote peer
+func (h *Handshaker) sendOffer() error {
+	if !h.signaler.Ready() {
+		return ErrSignalIsNotReady
+	}
+
+	iceUFrag, icePwd := h.ice.GetLocalUserCredentials()
+	offer := OfferAnswer{
+		IceCredentials:  IceCredentials{iceUFrag, icePwd},
+		WgListenPort:    h.config.LocalWgPort,
+		Version:         version.NetbirdVersion(),
+		RosenpassPubKey: h.config.RosenpassPubKey,
+		RosenpassAddr:   h.config.RosenpassAddr,
+	}
+
+	addr, err := h.relay.RelayInstanceAddress()
+	if err == nil {
+		offer.RelaySrvAddress = addr
+	}
+
+	return h.signaler.SignalOffer(offer, h.config.Key)
+}
+
+func (h *Handshaker) sendAnswer() error {
+	h.log.Debugf("sending answer")
+	uFrag, pwd := h.ice.GetLocalUserCredentials()
+
+	answer := OfferAnswer{
+		IceCredentials:  IceCredentials{uFrag, pwd},
+		WgListenPort:    h.config.LocalWgPort,
+		Version:         version.NetbirdVersion(),
+		RosenpassPubKey: h.config.RosenpassPubKey,
+		RosenpassAddr:   h.config.RosenpassAddr,
+	}
+	addr, err := h.relay.RelayInstanceAddress()
+	if err == nil {
+		answer.RelaySrvAddress = addr
+	}
+
+	err = h.signaler.SignalAnswer(answer, h.config.Key)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

client/internal/peer/signaler.go

@@ -0,0 +1,70 @@
+package peer
+
+import (
+	"github.com/pion/ice/v3"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	signal "github.com/netbirdio/netbird/signal/client"
+	sProto "github.com/netbirdio/netbird/signal/proto"
+)
+
+type Signaler struct {
+	signal       signal.Client
+	wgPrivateKey wgtypes.Key
+}
+
+func NewSignaler(signal signal.Client, wgPrivateKey wgtypes.Key) *Signaler {
+	return &Signaler{
+		signal:       signal,
+		wgPrivateKey: wgPrivateKey,
+	}
+}
+
+func (s *Signaler) SignalOffer(offer OfferAnswer, remoteKey string) error {
+	return s.signalOfferAnswer(offer, remoteKey, sProto.Body_OFFER)
+}
+
+func (s *Signaler) SignalAnswer(offer OfferAnswer, remoteKey string) error {
+	return s.signalOfferAnswer(offer, remoteKey, sProto.Body_ANSWER)
+}
+
+func (s *Signaler) SignalICECandidate(candidate ice.Candidate, remoteKey string) error {
+	return s.signal.Send(&sProto.Message{
+		Key:       s.wgPrivateKey.PublicKey().String(),
+		RemoteKey: remoteKey,
+		Body: &sProto.Body{
+			Type:    sProto.Body_CANDIDATE,
+			Payload: candidate.Marshal(),
+		},
+	})
+}
+
+func (s *Signaler) Ready() bool {
+	return s.signal.Ready()
+}
+
+// SignalOfferAnswer signals either an offer or an answer to remote peer
+func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, bodyType sProto.Body_Type) error {
+	msg, err := signal.MarshalCredential(
+		s.wgPrivateKey,
+		offerAnswer.WgListenPort,
+		remoteKey,
+		&signal.Credential{
+			UFrag: offerAnswer.IceCredentials.UFrag,
+			Pwd:   offerAnswer.IceCredentials.Pwd,
+		},
+		bodyType,
+		offerAnswer.RosenpassPubKey,
+		offerAnswer.RosenpassAddr,
+		offerAnswer.RelaySrvAddress)
+	if err != nil {
+		return err
+	}
+
+	err = s.signal.Send(msg)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

client/internal/peer/status.go

@@ -3,6 +3,7 @@ package peer
 import (
 	"errors"
 	"net/netip"
+	"slices"
 	"sync"
 	"time"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/relay"
 	"github.com/netbirdio/netbird/iface"
 	"github.com/netbirdio/netbird/management/domain"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 )
 
 // State contains the latest state of a peer
@@ -24,11 +26,11 @@ type State struct {
 	ConnStatus                 ConnStatus
 	ConnStatusUpdate           time.Time
 	Relayed                    bool
-	Direct                     bool
 	LocalIceCandidateType      string
 	RemoteIceCandidateType     string
 	LocalIceCandidateEndpoint  string
 	RemoteIceCandidateEndpoint string
+	RelayServerAddress         string
 	LastWireguardHandshake     time.Time
 	BytesTx                    int64
 	BytesRx                    int64
@@ -142,6 +144,8 @@ type Status struct {
 	// Some Peer actions mostly used by in a batch when the network map has been synchronized. In these type of events
 	// set to true this variable and at the end of the processing we will reset it by the FinishPeerListModifications()
 	peerListChangedForNotification bool
+
+	relayMgr *relayClient.Manager
 }
 
 // NewRecorder returns a new Status instance
@@ -156,6 +160,12 @@ func NewRecorder(mgmAddress string) *Status {
 	}
 }
 
+func (d *Status) SetRelayMgr(manager *relayClient.Manager) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	d.relayMgr = manager
+}
+
 // ReplaceOfflinePeers replaces
 func (d *Status) ReplaceOfflinePeers(replacement []State) {
 	d.mux.Lock()
@@ -231,20 +241,160 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 		peerState.SetRoutes(receivedState.GetRoutes())
 	}
 
-	skipNotification := shouldSkipNotify(receivedState, peerState)
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
 
 	if receivedState.ConnStatus != peerState.ConnStatus {
 		peerState.ConnStatus = receivedState.ConnStatus
 		peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
-		peerState.Direct = receivedState.Direct
+		peerState.Relayed = receivedState.Relayed
+		peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType
+		peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
+		peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
+		peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
+		peerState.RelayServerAddress = receivedState.RelayServerAddress
+		peerState.RosenpassEnabled = receivedState.RosenpassEnabled
+	}
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerICEState(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	if receivedState.IP != "" {
+		peerState.IP = receivedState.IP
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
 	peerState.Relayed = receivedState.Relayed
 	peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType
 	peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
 	peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
 	peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
 	peerState.RosenpassEnabled = receivedState.RosenpassEnabled
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
 	}
 
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerRelayedState(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.Relayed = receivedState.Relayed
+	peerState.RelayServerAddress = receivedState.RelayServerAddress
+	peerState.RosenpassEnabled = receivedState.RosenpassEnabled
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.Relayed = receivedState.Relayed
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.RelayServerAddress = ""
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.Relayed = receivedState.Relayed
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType
+	peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
+	peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
+	peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
+
 	d.peers[receivedState.PubKey] = peerState
 
 	if skipNotification {
@@ -280,13 +430,13 @@ func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats)
 	return nil
 }
 
-func shouldSkipNotify(received, curr State) bool {
+func shouldSkipNotify(receivedConnStatus ConnStatus, curr State) bool {
 	switch {
-	case received.ConnStatus == StatusConnecting:
+	case receivedConnStatus == StatusConnecting:
 		return true
-	case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting:
+	case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting:
 		return true
-	case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected:
+	case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected:
 		return curr.IP != ""
 	default:
 		return false
@@ -447,6 +597,8 @@ func (d *Status) DeleteResolvedDomainsStates(domain domain.Domain) {
 }
 
 func (d *Status) GetRosenpassState() RosenpassState {
+	d.mux.Lock()
+	defer d.mux.Unlock()
 	return RosenpassState{
 		d.rosenpassEnabled,
 		d.rosenpassPermissive,
@@ -454,6 +606,8 @@ func (d *Status) GetRosenpassState() RosenpassState {
 }
 
 func (d *Status) GetManagementState() ManagementState {
+	d.mux.Lock()
+	defer d.mux.Unlock()
 	return ManagementState{
 		d.mgmAddress,
 		d.managementState,
@@ -495,6 +649,8 @@ func (d *Status) IsLoginRequired() bool {
 }
 
 func (d *Status) GetSignalState() SignalState {
+	d.mux.Lock()
+	defer d.mux.Unlock()
 	return SignalState{
 		d.signalAddress,
 		d.signalState,
@@ -502,11 +658,42 @@ func (d *Status) GetSignalState() SignalState {
 	}
 }
 
+// GetRelayStates returns the stun/turn/permanent relay states
 func (d *Status) GetRelayStates() []relay.ProbeResult {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	if d.relayMgr == nil {
 		return d.relayStates
 	}
 
+	// extend the list of stun, turn servers with relay address
+	relayStates := slices.Clone(d.relayStates)
+
+	var relayState relay.ProbeResult
+
+	// if the server connection is not established then we will use the general address
+	// in case of connection we will use the instance specific address
+	instanceAddr, err := d.relayMgr.RelayInstanceAddress()
+	if err != nil {
+		// TODO add their status
+		if errors.Is(err, relayClient.ErrRelayClientNotConnected) {
+			for _, r := range d.relayMgr.ServerURLs() {
+				relayStates = append(relayStates, relay.ProbeResult{
+					URI: r,
+				})
+			}
+			return relayStates
+		}
+		relayState.Err = err
+	}
+
+	relayState.URI = instanceAddr
+	return append(relayStates, relayState)
+}
+
 func (d *Status) GetDNSStates() []NSGroupState {
+	d.mux.Lock()
+	defer d.mux.Unlock()
 	return d.nsGroupStates
 }
 
@@ -518,24 +705,24 @@ func (d *Status) GetResolvedDomainsStates() map[domain.Domain][]netip.Prefix {
 
 // GetFullStatus gets full status
 func (d *Status) GetFullStatus() FullStatus {
-	d.mux.Lock()
-	defer d.mux.Unlock()
-
 	fullStatus := FullStatus{
 		ManagementState: d.GetManagementState(),
 		SignalState:     d.GetSignalState(),
-		LocalPeerState:  d.localPeer,
 		Relays:          d.GetRelayStates(),
 		RosenpassState:  d.GetRosenpassState(),
 		NSGroupStates:   d.GetDNSStates(),
 	}
 
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	fullStatus.LocalPeerState = d.localPeer
+
 	for _, status := range d.peers {
 		fullStatus.Peers = append(fullStatus.Peers, status)
 	}
 
 	fullStatus.Peers = append(fullStatus.Peers, d.offlinePeers...)
-
 	return fullStatus
 }
 

client/internal/peer/status_test.go

@@ -2,8 +2,8 @@ package peer
 
 import (
 	"errors"
-	"testing"
 	"sync"
+	"testing"
 
 	"github.com/stretchr/testify/assert"
 )

client/internal/peer/stdnet.go

@@ -6,6 +6,6 @@ import (
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 )
 
-func (conn *Conn) newStdNet() (*stdnet.Net, error) {
+func (w *WorkerICE) newStdNet() (*stdnet.Net, error) {
-	return stdnet.NewNet(conn.config.InterfaceBlackList)
+	return stdnet.NewNet(w.config.ICEConfig.InterfaceBlackList)
 }

client/internal/peer/stdnet_android.go

@@ -2,6 +2,6 @@ package peer
 
 import "github.com/netbirdio/netbird/client/internal/stdnet"
 
-func (conn *Conn) newStdNet() (*stdnet.Net, error) {
+func (w *WorkerICE) newStdNet() (*stdnet.Net, error) {
-	return stdnet.NewNetWithDiscover(conn.iFaceDiscover, conn.config.InterfaceBlackList)
+	return stdnet.NewNetWithDiscover(w.iFaceDiscover, w.config.ICEConfig.InterfaceBlackList)
 }

client/internal/peer/worker_ice.go

@@ -0,0 +1,470 @@
+package peer
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pion/ice/v3"
+	"github.com/pion/randutil"
+	"github.com/pion/stun/v2"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/stdnet"
+	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/iface/bind"
+	"github.com/netbirdio/netbird/route"
+)
+
+const (
+	iceKeepAliveDefault           = 4 * time.Second
+	iceDisconnectedTimeoutDefault = 6 * time.Second
+	// iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package
+	iceRelayAcceptanceMinWaitDefault = 2 * time.Second
+
+	lenUFrag   = 16
+	lenPwd     = 32
+	runesAlpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+)
+
+var (
+	failedTimeout = 6 * time.Second
+)
+
+type ICEConfig struct {
+	// StunTurn is a list of STUN and TURN URLs
+	StunTurn *atomic.Value // []*stun.URI
+
+	// InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering
+	// (e.g. if eth0 is in the list, host candidate of this interface won't be used)
+	InterfaceBlackList   []string
+	DisableIPv6Discovery bool
+
+	UDPMux      ice.UDPMux
+	UDPMuxSrflx ice.UniversalUDPMux
+
+	NATExternalIPs []string
+}
+
+type ICEConnInfo struct {
+	RemoteConn                 net.Conn
+	RosenpassPubKey            []byte
+	RosenpassAddr              string
+	LocalIceCandidateType      string
+	RemoteIceCandidateType     string
+	RemoteIceCandidateEndpoint string
+	LocalIceCandidateEndpoint  string
+	Relayed                    bool
+	RelayedOnLocal             bool
+}
+
+type WorkerICECallbacks struct {
+	OnConnReady     func(ConnPriority, ICEConnInfo)
+	OnStatusChanged func(ConnStatus)
+}
+
+type WorkerICE struct {
+	ctx               context.Context
+	log               *log.Entry
+	config            ConnConfig
+	signaler          *Signaler
+	iFaceDiscover     stdnet.ExternalIFaceDiscover
+	statusRecorder    *Status
+	hasRelayOnLocally bool
+	conn              WorkerICECallbacks
+
+	selectedPriority ConnPriority
+
+	agent    *ice.Agent
+	muxAgent sync.Mutex
+
+	StunTurn []*stun.URI
+
+	sentExtraSrflx bool
+
+	localUfrag string
+	localPwd   string
+}
+
+func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool, callBacks WorkerICECallbacks) (*WorkerICE, error) {
+	w := &WorkerICE{
+		ctx:               ctx,
+		log:               log,
+		config:            config,
+		signaler:          signaler,
+		iFaceDiscover:     ifaceDiscover,
+		statusRecorder:    statusRecorder,
+		hasRelayOnLocally: hasRelayOnLocally,
+		conn:              callBacks,
+	}
+
+	localUfrag, localPwd, err := generateICECredentials()
+	if err != nil {
+		return nil, err
+	}
+	w.localUfrag = localUfrag
+	w.localPwd = localPwd
+	return w, nil
+}
+
+func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
+	w.log.Debugf("OnNewOffer for ICE")
+	w.muxAgent.Lock()
+
+	if w.agent != nil {
+		w.log.Debugf("agent already exists, skipping the offer")
+		w.muxAgent.Unlock()
+		return
+	}
+
+	var preferredCandidateTypes []ice.CandidateType
+	if w.hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" {
+		w.selectedPriority = connPriorityICEP2P
+		preferredCandidateTypes = candidateTypesP2P()
+	} else {
+		w.selectedPriority = connPriorityICETurn
+		preferredCandidateTypes = candidateTypes()
+	}
+
+	w.log.Debugf("recreate ICE agent")
+	agentCtx, agentCancel := context.WithCancel(w.ctx)
+	agent, err := w.reCreateAgent(agentCancel, preferredCandidateTypes)
+	if err != nil {
+		w.log.Errorf("failed to recreate ICE Agent: %s", err)
+		w.muxAgent.Unlock()
+		return
+	}
+	w.agent = agent
+	w.muxAgent.Unlock()
+
+	w.log.Debugf("gather candidates")
+	err = w.agent.GatherCandidates()
+	if err != nil {
+		w.log.Debugf("failed to gather candidates: %s", err)
+		return
+	}
+
+	// will block until connection succeeded
+	// but it won't release if ICE Agent went into Disconnected or Failed state,
+	// so we have to cancel it with the provided context once agent detected a broken connection
+	w.log.Debugf("turn agent dial")
+	remoteConn, err := w.turnAgentDial(agentCtx, remoteOfferAnswer)
+	if err != nil {
+		w.log.Debugf("failed to dial the remote peer: %s", err)
+		return
+	}
+	w.log.Debugf("agent dial succeeded")
+
+	pair, err := w.agent.GetSelectedCandidatePair()
+	if err != nil {
+		return
+	}
+
+	if !isRelayCandidate(pair.Local) {
+		// dynamically set remote WireGuard port if other side specified a different one from the default one
+		remoteWgPort := iface.DefaultWgPort
+		if remoteOfferAnswer.WgListenPort != 0 {
+			remoteWgPort = remoteOfferAnswer.WgListenPort
+		}
+
+		// To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port
+		go w.punchRemoteWGPort(pair, remoteWgPort)
+	}
+
+	ci := ICEConnInfo{
+		RemoteConn:                 remoteConn,
+		RosenpassPubKey:            remoteOfferAnswer.RosenpassPubKey,
+		RosenpassAddr:              remoteOfferAnswer.RosenpassAddr,
+		LocalIceCandidateType:      pair.Local.Type().String(),
+		RemoteIceCandidateType:     pair.Remote.Type().String(),
+		LocalIceCandidateEndpoint:  fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()),
+		RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()),
+		Relayed:                    isRelayed(pair),
+		RelayedOnLocal:             isRelayCandidate(pair.Local),
+	}
+	w.log.Debugf("on ICE conn read to use ready")
+	go w.conn.OnConnReady(w.selectedPriority, ci)
+}
+
+// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer.
+func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+	w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String())
+	if w.agent == nil {
+		w.log.Warnf("ICE Agent is not initialized yet")
+		return
+	}
+
+	if candidateViaRoutes(candidate, haRoutes) {
+		return
+	}
+
+	err := w.agent.AddRemoteCandidate(candidate)
+	if err != nil {
+		w.log.Errorf("error while handling remote candidate")
+		return
+	}
+}
+
+func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string) {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+	return w.localUfrag, w.localPwd
+}
+
+func (w *WorkerICE) Close() {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+
+	if w.agent == nil {
+		return
+	}
+
+	err := w.agent.Close()
+	if err != nil {
+		w.log.Warnf("failed to close ICE agent: %s", err)
+	}
+}
+
+func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) {
+	transportNet, err := w.newStdNet()
+	if err != nil {
+		w.log.Errorf("failed to create pion's stdnet: %s", err)
+	}
+
+	iceKeepAlive := iceKeepAlive()
+	iceDisconnectedTimeout := iceDisconnectedTimeout()
+	iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait()
+
+	agentConfig := &ice.AgentConfig{
+		MulticastDNSMode:       ice.MulticastDNSModeDisabled,
+		NetworkTypes:           []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6},
+		Urls:                   w.config.ICEConfig.StunTurn.Load().([]*stun.URI),
+		CandidateTypes:         relaySupport,
+		InterfaceFilter:        stdnet.InterfaceFilter(w.config.ICEConfig.InterfaceBlackList),
+		UDPMux:                 w.config.ICEConfig.UDPMux,
+		UDPMuxSrflx:            w.config.ICEConfig.UDPMuxSrflx,
+		NAT1To1IPs:             w.config.ICEConfig.NATExternalIPs,
+		Net:                    transportNet,
+		FailedTimeout:          &failedTimeout,
+		DisconnectedTimeout:    &iceDisconnectedTimeout,
+		KeepaliveInterval:      &iceKeepAlive,
+		RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait,
+		LocalUfrag:             w.localUfrag,
+		LocalPwd:               w.localPwd,
+	}
+
+	if w.config.ICEConfig.DisableIPv6Discovery {
+		agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4}
+	}
+
+	w.sentExtraSrflx = false
+	agent, err := ice.NewAgent(agentConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnCandidate(w.onICECandidate)
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnConnectionStateChange(func(state ice.ConnectionState) {
+		w.log.Debugf("ICE ConnectionState has changed to %s", state.String())
+		if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected {
+			w.conn.OnStatusChanged(StatusDisconnected)
+
+			w.muxAgent.Lock()
+			agentCancel()
+			_ = agent.Close()
+			w.agent = nil
+
+			w.muxAgent.Unlock()
+		}
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair)
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) {
+		err := w.statusRecorder.UpdateLatency(w.config.Key, p.Latency())
+		if err != nil {
+			w.log.Debugf("failed to update latency for peer: %s", err)
+			return
+		}
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed setting binding response callback: %w", err)
+	}
+
+	return agent, nil
+}
+
+func (w *WorkerICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) {
+	// wait local endpoint configuration
+	time.Sleep(time.Second)
+	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort))
+	if err != nil {
+		w.log.Warnf("got an error while resolving the udp address, err: %s", err)
+		return
+	}
+
+	mux, ok := w.config.ICEConfig.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault)
+	if !ok {
+		w.log.Warn("invalid udp mux conversion")
+		return
+	}
+	_, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr)
+	if err != nil {
+		w.log.Warnf("got an error while sending the punch packet, err: %s", err)
+	}
+}
+
+// onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates
+// and then signals them to the remote peer
+func (w *WorkerICE) onICECandidate(candidate ice.Candidate) {
+	// nil means candidate gathering has been ended
+	if candidate == nil {
+		return
+	}
+
+	// TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored
+	w.log.Debugf("discovered local candidate %s", candidate.String())
+	go func() {
+		err := w.signaler.SignalICECandidate(candidate, w.config.Key)
+		if err != nil {
+			w.log.Errorf("failed signaling candidate to the remote peer %s %s", w.config.Key, err)
+		}
+	}()
+
+	if !w.shouldSendExtraSrflxCandidate(candidate) {
+		return
+	}
+
+	// sends an extra server reflexive candidate to the remote peer with our related port (usually the wireguard port)
+	// this is useful when network has an existing port forwarding rule for the wireguard port and this peer
+	extraSrflx, err := extraSrflxCandidate(candidate)
+	if err != nil {
+		w.log.Errorf("failed creating extra server reflexive candidate %s", err)
+		return
+	}
+	w.sentExtraSrflx = true
+
+	go func() {
+		err = w.signaler.SignalICECandidate(extraSrflx, w.config.Key)
+		if err != nil {
+			w.log.Errorf("failed signaling the extra server reflexive candidate: %s", err)
+		}
+	}()
+}
+
+func (w *WorkerICE) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) {
+	w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(),
+		w.config.Key)
+}
+
+func (w *WorkerICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool {
+	if !w.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port {
+		return true
+	}
+	return false
+}
+
+func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) {
+	isControlling := w.config.LocalKey > w.config.Key
+	if isControlling {
+		return w.agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
+	} else {
+		return w.agent.Accept(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
+	}
+}
+
+func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) {
+	relatedAdd := candidate.RelatedAddress()
+	return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{
+		Network:   candidate.NetworkType().String(),
+		Address:   candidate.Address(),
+		Port:      relatedAdd.Port,
+		Component: candidate.Component(),
+		RelAddr:   relatedAdd.Address,
+		RelPort:   relatedAdd.Port,
+	})
+}
+
+func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool {
+	var routePrefixes []netip.Prefix
+	for _, routes := range clientRoutes {
+		if len(routes) > 0 && routes[0] != nil {
+			routePrefixes = append(routePrefixes, routes[0].Network)
+		}
+	}
+
+	addr, err := netip.ParseAddr(candidate.Address())
+	if err != nil {
+		log.Errorf("Failed to parse IP address %s: %v", candidate.Address(), err)
+		return false
+	}
+
+	for _, prefix := range routePrefixes {
+		// default route is
+		if prefix.Bits() == 0 {
+			continue
+		}
+
+		if prefix.Contains(addr) {
+			log.Debugf("Ignoring candidate [%s], its address is part of routed network %s", candidate.String(), prefix)
+			return true
+		}
+	}
+	return false
+}
+
+func candidateTypes() []ice.CandidateType {
+	if hasICEForceRelayConn() {
+		return []ice.CandidateType{ice.CandidateTypeRelay}
+	}
+	// TODO: remove this once we have refactored userspace proxy into the bind package
+	if runtime.GOOS == "ios" {
+		return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
+	}
+	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay}
+}
+
+func candidateTypesP2P() []ice.CandidateType {
+	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
+}
+
+func isRelayCandidate(candidate ice.Candidate) bool {
+	return candidate.Type() == ice.CandidateTypeRelay
+}
+
+func isRelayed(pair *ice.CandidatePair) bool {
+	if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay {
+		return true
+	}
+	return false
+}
+
+func generateICECredentials() (string, string, error) {
+	ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha)
+	if err != nil {
+		return "", "", err
+	}
+
+	pwd, err := randutil.GenerateCryptoRandomString(lenPwd, runesAlpha)
+	if err != nil {
+		return "", "", err
+	}
+	return ufrag, pwd, nil
+}

client/internal/peer/worker_relay.go

@@ -0,0 +1,237 @@
+package peer
+
+import (
+	"context"
+	"errors"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	relayClient "github.com/netbirdio/netbird/relay/client"
+)
+
+var (
+	wgHandshakePeriod   = 3 * time.Minute
+	wgHandshakeOvertime = 30 * time.Second
+)
+
+type RelayConnInfo struct {
+	relayedConn     net.Conn
+	rosenpassPubKey []byte
+	rosenpassAddr   string
+}
+
+type WorkerRelayCallbacks struct {
+	OnConnReady    func(RelayConnInfo)
+	OnDisconnected func()
+}
+
+type WorkerRelay struct {
+	log          *log.Entry
+	config       ConnConfig
+	relayManager relayClient.ManagerService
+	callBacks    WorkerRelayCallbacks
+
+	relayedConn      net.Conn
+	relayLock        sync.Mutex
+	ctxWgWatch       context.Context
+	ctxCancelWgWatch context.CancelFunc
+	ctxLock          sync.Mutex
+
+	relaySupportedOnRemotePeer atomic.Bool
+}
+
+func NewWorkerRelay(log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay {
+	r := &WorkerRelay{
+		log:          log,
+		config:       config,
+		relayManager: relayManager,
+		callBacks:    callbacks,
+	}
+	return r
+}
+
+func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
+	if !w.isRelaySupported(remoteOfferAnswer) {
+		w.log.Infof("Relay is not supported by remote peer")
+		w.relaySupportedOnRemotePeer.Store(false)
+		return
+	}
+	w.relaySupportedOnRemotePeer.Store(true)
+
+	// the relayManager will return with error in case if the connection has lost with relay server
+	currentRelayAddress, err := w.relayManager.RelayInstanceAddress()
+	if err != nil {
+		w.log.Errorf("failed to handle new offer: %s", err)
+		return
+	}
+
+	srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress)
+
+	relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key)
+	if err != nil {
+		if errors.Is(err, relayClient.ErrConnAlreadyExists) {
+			w.log.Infof("do not need to reopen relay connection")
+			return
+		}
+		w.log.Errorf("failed to open connection via Relay: %s", err)
+		return
+	}
+	w.relayLock.Lock()
+	w.relayedConn = relayedConn
+	w.relayLock.Unlock()
+
+	err = w.relayManager.AddCloseListener(srv, w.onRelayMGDisconnected)
+	if err != nil {
+		log.Errorf("failed to add close listener: %s", err)
+		_ = relayedConn.Close()
+		return
+	}
+
+	w.log.Debugf("peer conn opened via Relay: %s", srv)
+	go w.callBacks.OnConnReady(RelayConnInfo{
+		relayedConn:     relayedConn,
+		rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey,
+		rosenpassAddr:   remoteOfferAnswer.RosenpassAddr,
+	})
+}
+
+func (w *WorkerRelay) EnableWgWatcher(ctx context.Context) {
+	w.log.Debugf("enable WireGuard watcher")
+	w.ctxLock.Lock()
+	defer w.ctxLock.Unlock()
+
+	if w.ctxWgWatch != nil && w.ctxWgWatch.Err() == nil {
+		return
+	}
+
+	ctx, ctxCancel := context.WithCancel(ctx)
+	w.ctxWgWatch = ctx
+	w.ctxCancelWgWatch = ctxCancel
+
+	w.wgStateCheck(ctx, ctxCancel)
+}
+
+func (w *WorkerRelay) DisableWgWatcher() {
+	w.ctxLock.Lock()
+	defer w.ctxLock.Unlock()
+
+	if w.ctxCancelWgWatch == nil {
+		return
+	}
+
+	w.log.Debugf("disable WireGuard watcher")
+
+	w.ctxCancelWgWatch()
+}
+
+func (w *WorkerRelay) RelayInstanceAddress() (string, error) {
+	return w.relayManager.RelayInstanceAddress()
+}
+
+func (w *WorkerRelay) IsRelayConnectionSupportedWithPeer() bool {
+	return w.relaySupportedOnRemotePeer.Load() && w.RelayIsSupportedLocally()
+}
+
+func (w *WorkerRelay) IsController() bool {
+	return w.config.LocalKey > w.config.Key
+}
+
+func (w *WorkerRelay) RelayIsSupportedLocally() bool {
+	return w.relayManager.HasRelayAddress()
+}
+
+func (w *WorkerRelay) CloseConn() {
+	w.relayLock.Lock()
+	defer w.relayLock.Unlock()
+	if w.relayedConn == nil {
+		return
+	}
+
+	err := w.relayedConn.Close()
+	if err != nil {
+		w.log.Warnf("failed to close relay connection: %v", err)
+	}
+}
+
+// wgStateCheck help to check the state of the WireGuard handshake and relay connection
+func (w *WorkerRelay) wgStateCheck(ctx context.Context, ctxCancel context.CancelFunc) {
+	w.log.Debugf("WireGuard watcher started")
+	lastHandshake, err := w.wgState()
+	if err != nil {
+		w.log.Warnf("failed to read wg stats: %v", err)
+		lastHandshake = time.Time{}
+	}
+
+	go func(lastHandshake time.Time) {
+		timer := time.NewTimer(wgHandshakeOvertime)
+		defer timer.Stop()
+		defer ctxCancel()
+
+		for {
+			select {
+			case <-timer.C:
+				handshake, err := w.wgState()
+				if err != nil {
+					w.log.Errorf("failed to read wg stats: %v", err)
+					timer.Reset(wgHandshakeOvertime)
+					continue
+				}
+
+				w.log.Tracef("previous handshake, handshake: %v, %v", lastHandshake, handshake)
+
+				if handshake.Equal(lastHandshake) {
+					w.log.Infof("WireGuard handshake timed out, closing relay connection: %v", handshake)
+					w.relayLock.Lock()
+					_ = w.relayedConn.Close()
+					w.relayLock.Unlock()
+					w.callBacks.OnDisconnected()
+					return
+				}
+
+				resetTime := time.Until(handshake.Add(wgHandshakePeriod + wgHandshakeOvertime))
+				lastHandshake = handshake
+				timer.Reset(resetTime)
+			case <-ctx.Done():
+				w.log.Debugf("WireGuard watcher stopped")
+				return
+			}
+		}
+	}(lastHandshake)
+
+}
+
+func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool {
+	if !w.relayManager.HasRelayAddress() {
+		return false
+	}
+	return answer.RelaySrvAddress != ""
+}
+
+func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string {
+	if w.IsController() {
+		return myRelayAddress
+	}
+	return remoteRelayAddress
+}
+
+func (w *WorkerRelay) wgState() (time.Time, error) {
+	wgState, err := w.config.WgConfig.WgInterface.GetStats(w.config.Key)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return wgState.LastHandshake, nil
+}
+
+func (w *WorkerRelay) onRelayMGDisconnected() {
+	w.ctxLock.Lock()
+	defer w.ctxLock.Unlock()
+
+	if w.ctxCancelWgWatch != nil {
+		w.ctxCancelWgWatch()
+	}
+	go w.callBacks.OnDisconnected()
+}

client/internal/pkce_auth.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/url"
 
@@ -36,10 +37,12 @@ type PKCEAuthProviderConfig struct {
 	RedirectURLs []string
 	// UseIDToken indicates if the id token should be used for authentication
 	UseIDToken bool
+	//ClientCertPair is used for mTLS authentication to the IDP
+	ClientCertPair *tls.Certificate
 }
 
 // GetPKCEAuthorizationFlowInfo initialize a PKCEAuthorizationFlow instance and return with it
-func GetPKCEAuthorizationFlowInfo(ctx context.Context, privateKey string, mgmURL *url.URL) (PKCEAuthorizationFlow, error) {
+func GetPKCEAuthorizationFlowInfo(ctx context.Context, privateKey string, mgmURL *url.URL, clientCert *tls.Certificate) (PKCEAuthorizationFlow, error) {
 	// validate our peer's Wireguard PRIVATE key
 	myPrivateKey, err := wgtypes.ParseKey(privateKey)
 	if err != nil {
@@ -93,6 +96,7 @@ func GetPKCEAuthorizationFlowInfo(ctx context.Context, privateKey string, mgmURL
 			Scope:                 protoPKCEAuthorizationFlow.GetProviderConfig().GetScope(),
 			RedirectURLs:          protoPKCEAuthorizationFlow.GetProviderConfig().GetRedirectURLs(),
 			UseIDToken:            protoPKCEAuthorizationFlow.GetProviderConfig().GetUseIDToken(),
+			ClientCertPair:        clientCert,
 		},
 	}
 

client/internal/probe.go

@@ -2,6 +2,13 @@ package internal
 
 import "context"
 
+type ProbeHolder struct {
+	MgmProbe    *Probe
+	SignalProbe *Probe
+	RelayProbe  *Probe
+	WgProbe     *Probe
+}
+
 // Probe allows to run on-demand callbacks from different code locations.
 // Pass the probe to a receiving and a sending end. The receiving end starts listening
 // to requests with Receive and executes a callback when the sending end requests it

client/internal/relay/relay.go

@@ -17,7 +17,7 @@ import (
 
 // ProbeResult holds the info about the result of a relay probe request
 type ProbeResult struct {
-	URI  *stun.URI
+	URI  string
 	Err  error
 	Addr string
 }
@@ -176,7 +176,7 @@ func ProbeAll(
 		wg.Add(1)
 		go func(res *ProbeResult, stunURI *stun.URI) {
 			defer wg.Done()
-			res.URI = stunURI
+			res.URI = stunURI.String()
 			res.Addr, res.Err = fn(ctx, stunURI)
 		}(&results[i], uri)
 	}

client/internal/routemanager/client.go

@@ -3,12 +3,14 @@ package routemanager
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"time"
 
 	"github.com/hashicorp/go-multierror"
 	log "github.com/sirupsen/logrus"
 
 	nberrors "github.com/netbirdio/netbird/client/errors"
+	nbdns "github.com/netbirdio/netbird/client/internal/dns"
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/internal/routemanager/dynamic"
 	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
@@ -20,7 +22,6 @@ import (
 type routerPeerStatus struct {
 	connected bool
 	relayed   bool
-	direct    bool
 	latency   time.Duration
 }
 
@@ -42,7 +43,7 @@ type clientNetwork struct {
 	ctx                 context.Context
 	cancel              context.CancelFunc
 	statusRecorder      *peer.Status
-	wgInterface         *iface.WGIface
+	wgInterface         iface.IWGIface
 	routes              map[route.ID]*route.Route
 	routeUpdate         chan routesUpdate
 	peerStateUpdate     chan struct{}
@@ -52,7 +53,7 @@ type clientNetwork struct {
 	updateSerial        uint64
 }
 
-func newClientNetworkWatcher(ctx context.Context, dnsRouteInterval time.Duration, wgInterface *iface.WGIface, statusRecorder *peer.Status, rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter) *clientNetwork {
+func newClientNetworkWatcher(ctx context.Context, dnsRouteInterval time.Duration, wgInterface iface.IWGIface, statusRecorder *peer.Status, rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter) *clientNetwork {
 	ctx, cancel := context.WithCancel(ctx)
 
 	client := &clientNetwork{
@@ -64,7 +65,7 @@ func newClientNetworkWatcher(ctx context.Context, dnsRouteInterval time.Duration
 		routePeersNotifiers: make(map[string]chan struct{}),
 		routeUpdate:         make(chan routesUpdate),
 		peerStateUpdate:     make(chan struct{}),
-		handler:             handlerFromRoute(rt, routeRefCounter, allowedIPsRefCounter, dnsRouteInterval, statusRecorder),
+		handler:             handlerFromRoute(rt, routeRefCounter, allowedIPsRefCounter, dnsRouteInterval, statusRecorder, wgInterface),
 	}
 	return client
 }
@@ -80,7 +81,6 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus {
 		routePeerStatuses[r.ID] = routerPeerStatus{
 			connected: peerStatus.ConnStatus == peer.StatusConnected,
 			relayed:   peerStatus.Relayed,
-			direct:    peerStatus.Direct,
 			latency:   peerStatus.Latency,
 		}
 	}
@@ -95,8 +95,8 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus {
 // * Connected peers: Only routes with connected peers are considered.
 // * Metric: Routes with lower metrics (better) are prioritized.
 // * Non-relayed: Routes without relays are preferred.
-// * Direct connections: Routes with direct peer connections are favored.
 // * Latency: Routes with lower latency are prioritized.
+// * we compare the current score + 10ms to the chosen score to avoid flapping between routes
 // * Stability: In case of equal scores, the currently active route (if any) is maintained.
 //
 // It returns the ID of the selected optimal route.
@@ -135,10 +135,6 @@ func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID]
 			tempScore++
 		}
 
-		if peerStatus.direct {
-			tempScore++
-		}
-
 		if tempScore > chosenScore || (tempScore == chosenScore && chosen == "") {
 			chosen = r.ID
 			chosenScore = tempScore
@@ -309,22 +305,33 @@ func (c *clientNetwork) sendUpdateToClientNetworkWatcher(update routesUpdate) {
 	}()
 }
 
-func (c *clientNetwork) handleUpdate(update routesUpdate) {
+func (c *clientNetwork) handleUpdate(update routesUpdate) bool {
+	isUpdateMapDifferent := false
 	updateMap := make(map[route.ID]*route.Route)
 
 	for _, r := range update.routes {
 		updateMap[r.ID] = r
 	}
 
+	if len(c.routes) != len(updateMap) {
+		isUpdateMapDifferent = true
+	}
+
 	for id, r := range c.routes {
 		_, found := updateMap[id]
 		if !found {
 			close(c.routePeersNotifiers[r.Peer])
 			delete(c.routePeersNotifiers, r.Peer)
+			isUpdateMapDifferent = true
+			continue
+		}
+		if !reflect.DeepEqual(c.routes[id], updateMap[id]) {
+			isUpdateMapDifferent = true
 		}
 	}
 
 	c.routes = updateMap
+	return isUpdateMapDifferent
 }
 
 // peersStateAndUpdateWatcher is the main point of reacting on client network routing events.
@@ -351,23 +358,30 @@ func (c *clientNetwork) peersStateAndUpdateWatcher() {
 
 			log.Debugf("Received a new client network route update for [%v]", c.handler)
 
-			c.handleUpdate(update)
+			// hash update somehow
+			isTrueRouteUpdate := c.handleUpdate(update)
 
 			c.updateSerial = update.updateSerial
 
+			if isTrueRouteUpdate {
+				log.Debug("Client network update contains different routes, recalculating routes")
 				err := c.recalculateRouteAndUpdatePeerAndSystem()
 				if err != nil {
 					log.Errorf("Failed to recalculate routes for network [%v]: %v", c.handler, err)
 				}
+			} else {
+				log.Debug("Route update is not different, skipping route recalculation")
+			}
 
 			c.startPeersStatusChangeWatcher()
 		}
 	}
 }
 
-func handlerFromRoute(rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, dnsRouterInteval time.Duration, statusRecorder *peer.Status) RouteHandler {
+func handlerFromRoute(rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, dnsRouterInteval time.Duration, statusRecorder *peer.Status, wgInterface iface.IWGIface) RouteHandler {
 	if rt.IsDynamic() {
-		return dynamic.NewRoute(rt, routeRefCounter, allowedIPsRefCounter, dnsRouterInteval, statusRecorder)
+		dns := nbdns.NewServiceViaMemory(wgInterface)
+		return dynamic.NewRoute(rt, routeRefCounter, allowedIPsRefCounter, dnsRouterInteval, statusRecorder, wgInterface, fmt.Sprintf("%s:%d", dns.RuntimeIP(), dns.RuntimePort()))
 	}
 	return static.NewRoute(rt, routeRefCounter, allowedIPsRefCounter)
 }

client/internal/routemanager/client_test.go

@@ -24,7 +24,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -43,7 +42,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   true,
-					direct:    true,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -62,7 +60,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   true,
-					direct:    false,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -81,7 +78,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: false,
 					relayed:   false,
-					direct:    false,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -100,12 +96,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -129,41 +123,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 				"route2": {
 					connected: true,
 					relayed:   true,
-					direct:    true,
-				},
-			},
-			existingRoutes: map[route.ID]*route.Route{
-				"route1": {
-					ID:     "route1",
-					Metric: route.MaxMetric,
-					Peer:   "peer1",
-				},
-				"route2": {
-					ID:     "route2",
-					Metric: route.MaxMetric,
-					Peer:   "peer2",
-				},
-			},
-			currentRoute:    "",
-			expectedRouteID: "route1",
-		},
-		{
-			name: "multiple connected peers with one direct",
-			statuses: map[route.ID]routerPeerStatus{
-				"route1": {
-					connected: true,
-					relayed:   false,
-					direct:    true,
-				},
-				"route2": {
-					connected: true,
-					relayed:   false,
-					direct:    false,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -241,13 +204,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   15 * time.Millisecond,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   10 * time.Millisecond,
 				},
 			},
@@ -272,13 +233,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   200 * time.Millisecond,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   10 * time.Millisecond,
 				},
 			},
@@ -303,13 +262,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   20 * time.Millisecond,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   10 * time.Millisecond,
 				},
 			},

client/internal/routemanager/dynamic/route.go

@@ -16,6 +16,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
 	"github.com/netbirdio/netbird/client/internal/routemanager/util"
+	"github.com/netbirdio/netbird/iface"
 	"github.com/netbirdio/netbird/management/domain"
 	"github.com/netbirdio/netbird/route"
 )
@@ -47,6 +48,8 @@ type Route struct {
 	currentPeerKey       string
 	cancel               context.CancelFunc
 	statusRecorder       *peer.Status
+	wgInterface          iface.IWGIface
+	resolverAddr         string
 }
 
 func NewRoute(
@@ -55,6 +58,8 @@ func NewRoute(
 	allowedIPsRefCounter *refcounter.AllowedIPsRefCounter,
 	interval time.Duration,
 	statusRecorder *peer.Status,
+	wgInterface iface.IWGIface,
+	resolverAddr string,
 ) *Route {
 	return &Route{
 		route:                rt,
@@ -63,6 +68,8 @@ func NewRoute(
 		interval:             interval,
 		dynamicDomains:       domainMap{},
 		statusRecorder:       statusRecorder,
+		wgInterface:          wgInterface,
+		resolverAddr:         resolverAddr,
 	}
 }
 
@@ -189,9 +196,14 @@ func (r *Route) startResolver(ctx context.Context) {
 }
 
 func (r *Route) update(ctx context.Context) error {
-	if resolved, err := r.resolveDomains(); err != nil {
+	resolved, err := r.resolveDomains()
+	if err != nil {
+		if len(resolved) == 0 {
 			return fmt.Errorf("resolve domains: %w", err)
-	} else if err := r.updateDynamicRoutes(ctx, resolved); err != nil {
+		}
+		log.Warnf("Failed to resolve domains: %v", err)
+	}
+	if err := r.updateDynamicRoutes(ctx, resolved); err != nil {
 		return fmt.Errorf("update dynamic routes: %w", err)
 	}
 
@@ -223,11 +235,17 @@ func (r *Route) resolve(results chan resolveResult) {
 		wg.Add(1)
 		go func(domain domain.Domain) {
 			defer wg.Done()
-			ips, err := net.LookupIP(string(domain))
+
+			ips, err := r.getIPsFromResolver(domain)
+			if err != nil {
+				log.Tracef("Failed to resolve domain %s with private resolver: %v", domain.SafeString(), err)
+				ips, err = net.LookupIP(string(domain))
 				if err != nil {
 					results <- resolveResult{domain: domain, err: fmt.Errorf("resolve d %s: %w", domain.SafeString(), err)}
 					return
 				}
+			}
+
 			for _, ip := range ips {
 				prefix, err := util.GetPrefixFromIP(ip)
 				if err != nil {

client/internal/routemanager/dynamic/route_generic.go

@@ -0,0 +1,13 @@
+//go:build !ios
+
+package dynamic
+
+import (
+	"net"
+
+	"github.com/netbirdio/netbird/management/domain"
+)
+
+func (r *Route) getIPsFromResolver(domain domain.Domain) ([]net.IP, error) {
+	return net.LookupIP(string(domain))
+}

client/internal/routemanager/dynamic/route_ios.go

@@ -0,0 +1,55 @@
+//go:build ios
+
+package dynamic
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/miekg/dns"
+
+	nbdns "github.com/netbirdio/netbird/client/internal/dns"
+
+	"github.com/netbirdio/netbird/management/domain"
+)
+
+const dialTimeout = 10 * time.Second
+
+func (r *Route) getIPsFromResolver(domain domain.Domain) ([]net.IP, error) {
+	privateClient, err := nbdns.GetClientPrivate(r.wgInterface.Address().IP, r.wgInterface.Name(), dialTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("error while creating private client: %s", err)
+	}
+
+	msg := new(dns.Msg)
+	msg.SetQuestion(dns.Fqdn(string(domain)), dns.TypeA)
+
+	startTime := time.Now()
+
+	response, _, err := privateClient.Exchange(msg, r.resolverAddr)
+	if err != nil {
+		return nil, fmt.Errorf("DNS query for %s failed after %s: %s ", domain.SafeString(), time.Since(startTime), err)
+	}
+
+	if response.Rcode != dns.RcodeSuccess {
+		return nil, fmt.Errorf("dns response code: %s", dns.RcodeToString[response.Rcode])
+	}
+
+	ips := make([]net.IP, 0)
+
+	for _, answ := range response.Answer {
+		if aRecord, ok := answ.(*dns.A); ok {
+			ips = append(ips, aRecord.A)
+		}
+		if aaaaRecord, ok := answ.(*dns.AAAA); ok {
+			ips = append(ips, aaaaRecord.AAAA)
+		}
+	}
+
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("no A or AAAA records found for %s", domain.SafeString())
+	}
+
+	return ips, nil
+}

client/internal/routemanager/manager.go

@@ -16,11 +16,13 @@ import (
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/listener"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/routemanager/notifier"
 	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
 	"github.com/netbirdio/netbird/client/internal/routemanager/systemops"
 	"github.com/netbirdio/netbird/client/internal/routemanager/vars"
 	"github.com/netbirdio/netbird/client/internal/routeselector"
 	"github.com/netbirdio/netbird/iface"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	nbnet "github.com/netbirdio/netbird/util/net"
 	"github.com/netbirdio/netbird/version"
@@ -48,9 +50,10 @@ type DefaultManager struct {
 	serverRouter         serverRouter
 	sysOps               *systemops.SysOps
 	statusRecorder       *peer.Status
-	wgInterface          *iface.WGIface
+	relayMgr             *relayClient.Manager
+	wgInterface          iface.IWGIface
 	pubKey               string
-	notifier             *notifier
+	notifier             *notifier.Notifier
 	routeRefCounter      *refcounter.RouteRefCounter
 	allowedIPsRefCounter *refcounter.AllowedIPsRefCounter
 	dnsRouteInterval     time.Duration
@@ -60,24 +63,27 @@ func NewManager(
 	ctx context.Context,
 	pubKey string,
 	dnsRouteInterval time.Duration,
-	wgInterface *iface.WGIface,
+	wgInterface iface.IWGIface,
 	statusRecorder *peer.Status,
+	relayMgr *relayClient.Manager,
 	initialRoutes []*route.Route,
 ) *DefaultManager {
 	mCTX, cancel := context.WithCancel(ctx)
-	sysOps := systemops.NewSysOps(wgInterface)
+	notifier := notifier.NewNotifier()
+	sysOps := systemops.NewSysOps(wgInterface, notifier)
 
 	dm := &DefaultManager{
 		ctx:              mCTX,
 		stop:             cancel,
 		dnsRouteInterval: dnsRouteInterval,
 		clientNetworks:   make(map[route.HAUniqueID]*clientNetwork),
+		relayMgr:         relayMgr,
 		routeSelector:    routeselector.NewRouteSelector(),
 		sysOps:           sysOps,
 		statusRecorder:   statusRecorder,
 		wgInterface:      wgInterface,
 		pubKey:           pubKey,
-		notifier:         newNotifier(),
+		notifier:         notifier,
 	}
 
 	dm.routeRefCounter = refcounter.New(
@@ -107,7 +113,7 @@ func NewManager(
 
 	if runtime.GOOS == "android" {
 		cr := dm.clientRoutes(initialRoutes)
-		dm.notifier.setInitialClientRoutes(cr)
+		dm.notifier.SetInitialClientRoutes(cr)
 	}
 	return dm
 }
@@ -122,9 +128,12 @@ func (m *DefaultManager) Init() (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error)
 		log.Warnf("Failed cleaning up routing: %v", err)
 	}
 
-	mgmtAddress := m.statusRecorder.GetManagementState().URL
+	initialAddresses := []string{m.statusRecorder.GetManagementState().URL, m.statusRecorder.GetSignalState().URL}
-	signalAddress := m.statusRecorder.GetSignalState().URL
+	if m.relayMgr != nil {
-	ips := resolveURLsToIPs([]string{mgmtAddress, signalAddress})
+		initialAddresses = append(initialAddresses, m.relayMgr.ServerURLs()...)
+	}
+
+	ips := resolveURLsToIPs(initialAddresses)
 
 	beforePeerHook, afterPeerHook, err := m.sysOps.SetupRouting(ips)
 	if err != nil {
@@ -186,7 +195,7 @@ func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Ro
 
 		filteredClientRoutes := m.routeSelector.FilterSelected(newClientRoutesIDMap)
 		m.updateClientNetworks(updateSerial, filteredClientRoutes)
-		m.notifier.onNewRoutes(filteredClientRoutes)
+		m.notifier.OnNewRoutes(filteredClientRoutes)
 
 		if m.serverRouter != nil {
 			err := m.serverRouter.updateRoutes(newServerRoutesMap)
@@ -199,14 +208,14 @@ func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Ro
 	}
 }
 
-// SetRouteChangeListener set RouteListener for route change notifier
+// SetRouteChangeListener set RouteListener for route change Notifier
 func (m *DefaultManager) SetRouteChangeListener(listener listener.NetworkChangeListener) {
-	m.notifier.setListener(listener)
+	m.notifier.SetListener(listener)
 }
 
 // InitialRouteRange return the list of initial routes. It used by mobile systems
 func (m *DefaultManager) InitialRouteRange() []string {
-	return m.notifier.getInitialRouteRanges()
+	return m.notifier.GetInitialRouteRanges()
 }
 
 // GetRouteSelector returns the route selector
@@ -226,7 +235,7 @@ func (m *DefaultManager) TriggerSelection(networks route.HAMap) {
 
 	networks = m.routeSelector.FilterSelected(networks)
 
-	m.notifier.onNewRoutes(networks)
+	m.notifier.OnNewRoutes(networks)
 
 	m.stopObsoleteClients(networks)
 

client/internal/routemanager/manager_test.go

@@ -416,7 +416,7 @@ func TestManagerUpdateRoutes(t *testing.T) {
 
 			statusRecorder := peer.NewRecorder("https://mgm")
 			ctx := context.TODO()
-			routeManager := NewManager(ctx, localPeerKey, 0, wgInterface, statusRecorder, nil)
+			routeManager := NewManager(ctx, localPeerKey, 0, wgInterface, statusRecorder, nil, nil)
 
 			_, _, err = routeManager.Init()
 

client/internal/routemanager/notifier/notifier.go

@@ -1,6 +1,7 @@
-package routemanager
+package notifier
 
 import (
+	"net/netip"
 	"runtime"
 	"sort"
 	"strings"
@@ -10,7 +11,7 @@ import (
 	"github.com/netbirdio/netbird/route"
 )
 
-type notifier struct {
+type Notifier struct {
 	initialRouteRanges []string
 	routeRanges        []string
 
@@ -18,17 +19,17 @@ type notifier struct {
 	listenerMux sync.Mutex
 }
 
-func newNotifier() *notifier {
+func NewNotifier() *Notifier {
-	return &notifier{}
+	return &Notifier{}
 }
 
-func (n *notifier) setListener(listener listener.NetworkChangeListener) {
+func (n *Notifier) SetListener(listener listener.NetworkChangeListener) {
 	n.listenerMux.Lock()
 	defer n.listenerMux.Unlock()
 	n.listener = listener
 }
 
-func (n *notifier) setInitialClientRoutes(clientRoutes []*route.Route) {
+func (n *Notifier) SetInitialClientRoutes(clientRoutes []*route.Route) {
 	nets := make([]string, 0)
 	for _, r := range clientRoutes {
 		nets = append(nets, r.Network.String())
@@ -37,7 +38,10 @@ func (n *notifier) setInitialClientRoutes(clientRoutes []*route.Route) {
 	n.initialRouteRanges = nets
 }
 
-func (n *notifier) onNewRoutes(idMap route.HAMap) {
+func (n *Notifier) OnNewRoutes(idMap route.HAMap) {
+	if runtime.GOOS != "android" {
+		return
+	}
 	newNets := make([]string, 0)
 	for _, routes := range idMap {
 		for _, r := range routes {
@@ -62,7 +66,30 @@ func (n *notifier) onNewRoutes(idMap route.HAMap) {
 	n.notify()
 }
 
-func (n *notifier) notify() {
+func (n *Notifier) OnNewPrefixes(prefixes []netip.Prefix) {
+	newNets := make([]string, 0)
+	for _, prefix := range prefixes {
+		newNets = append(newNets, prefix.String())
+	}
+
+	sort.Strings(newNets)
+	switch runtime.GOOS {
+	case "android":
+		if !n.hasDiff(n.initialRouteRanges, newNets) {
+			return
+		}
+	default:
+		if !n.hasDiff(n.routeRanges, newNets) {
+			return
+		}
+	}
+
+	n.routeRanges = newNets
+
+	n.notify()
+}
+
+func (n *Notifier) notify() {
 	n.listenerMux.Lock()
 	defer n.listenerMux.Unlock()
 	if n.listener == nil {
@@ -74,7 +101,7 @@ func (n *notifier) notify() {
 	}(n.listener)
 }
 
-func (n *notifier) hasDiff(a []string, b []string) bool {
+func (n *Notifier) hasDiff(a []string, b []string) bool {
 	if len(a) != len(b) {
 		return true
 	}
@@ -86,7 +113,7 @@ func (n *notifier) hasDiff(a []string, b []string) bool {
 	return false
 }
 
-func (n *notifier) getInitialRouteRanges() []string {
+func (n *Notifier) GetInitialRouteRanges() []string {
 	return addIPv6RangeIfNeeded(n.initialRouteRanges)
 }
 

client/internal/routemanager/server_android.go

@@ -11,6 +11,6 @@ import (
 	"github.com/netbirdio/netbird/iface"
 )
 
-func newServerRouter(context.Context, *iface.WGIface, firewall.Manager, *peer.Status) (serverRouter, error) {
+func newServerRouter(context.Context, iface.IWGIface, firewall.Manager, *peer.Status) (serverRouter, error) {
 	return nil, fmt.Errorf("server route not supported on this os")
 }

client/internal/routemanager/server_nonandroid.go

@@ -22,11 +22,11 @@ type defaultServerRouter struct {
 	ctx            context.Context
 	routes         map[route.ID]*route.Route
 	firewall       firewall.Manager
-	wgInterface    *iface.WGIface
+	wgInterface    iface.IWGIface
 	statusRecorder *peer.Status
 }
 
-func newServerRouter(ctx context.Context, wgInterface *iface.WGIface, firewall firewall.Manager, statusRecorder *peer.Status) (serverRouter, error) {
+func newServerRouter(ctx context.Context, wgInterface iface.IWGIface, firewall firewall.Manager, statusRecorder *peer.Status) (serverRouter, error) {
 	return &defaultServerRouter{
 		ctx:            ctx,
 		routes:         make(map[route.ID]*route.Route),

client/internal/routemanager/sysctl/sysctl_linux.go

@@ -1,4 +1,5 @@
 //go:build !android
+
 package sysctl
 
 import (
@@ -22,7 +23,7 @@ const (
 )
 
 // Setup configures sysctl settings for RP filtering and source validation.
-func Setup(wgIface *iface.WGIface) (map[string]int, error) {
+func Setup(wgIface iface.IWGIface) (map[string]int, error) {
 	keys := map[string]int{}
 	var result *multierror.Error
 

client/internal/routemanager/systemops/systemops.go

@@ -3,7 +3,9 @@ package systemops
 import (
 	"net"
 	"net/netip"
+	"sync"
 
+	"github.com/netbirdio/netbird/client/internal/routemanager/notifier"
 	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
 	"github.com/netbirdio/netbird/iface"
 )
@@ -17,11 +19,20 @@ type ExclusionCounter = refcounter.Counter[any, Nexthop]
 
 type SysOps struct {
 	refCounter  *ExclusionCounter
-	wgInterface *iface.WGIface
+	wgInterface iface.IWGIface
+	// prefixes is tracking all the current added prefixes im memory
+	// (this is used in iOS as all route updates require a full table update)
+	//nolint
+	prefixes map[netip.Prefix]struct{}
+	//nolint
+	mu sync.Mutex
+	// notifier is used to notify the system of route changes (also used on mobile)
+	notifier *notifier.Notifier
 }
 
-func NewSysOps(wgInterface *iface.WGIface) *SysOps {
+func NewSysOps(wgInterface iface.IWGIface, notifier *notifier.Notifier) *SysOps {
 	return &SysOps{
 		wgInterface: wgInterface,
+		notifier:    notifier,
 	}
 }

client/internal/routemanager/systemops/systemops_android.go

@@ -1,4 +1,4 @@
-//go:build ios || android
+//go:build android
 
 package systemops
 

client/internal/routemanager/systemops/systemops_bsd.go

@@ -22,7 +22,7 @@ type Route struct {
 	Interface *net.Interface
 }
 
-func getRoutesFromTable() ([]netip.Prefix, error) {
+func GetRoutesFromTable() ([]netip.Prefix, error) {
 	tab, err := retryFetchRIB()
 	if err != nil {
 		return nil, fmt.Errorf("fetch RIB: %v", err)

client/internal/routemanager/systemops/systemops_bsd_test.go

@@ -36,7 +36,7 @@ func TestConcurrentRoutes(t *testing.T) {
 	baseIP := netip.MustParseAddr("192.0.2.0")
 	intf := &net.Interface{Name: "lo0"}
 
-	r := NewSysOps(nil)
+	r := NewSysOps(nil, nil)
 
 	var wg sync.WaitGroup
 	for i := 0; i < 1024; i++ {

client/internal/routemanager/systemops/systemops_generic.go

@@ -50,7 +50,7 @@ func (r *SysOps) setupRefCounter(initAddresses []net.IP) (nbnet.AddHookFunc, nbn
 			nexthop, err := r.addRouteToNonVPNIntf(prefix, r.wgInterface, initialNexthop)
 			if errors.Is(err, vars.ErrRouteNotAllowed) || errors.Is(err, vars.ErrRouteNotFound) {
 				log.Tracef("Adding for prefix %s: %v", prefix, err)
-				// These errors are not critical but also we should not track and try to remove the routes either.
+				// These errors are not critical, but also we should not track and try to remove the routes either.
 				return nexthop, refcounter.ErrIgnore
 			}
 			return nexthop, err
@@ -122,7 +122,7 @@ func (r *SysOps) addRouteForCurrentDefaultGateway(prefix netip.Prefix) error {
 
 // addRouteToNonVPNIntf adds a new route to the routing table for the given prefix and returns the next hop and interface.
 // If the next hop or interface is pointing to the VPN interface, it will return the initial values.
-func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf *iface.WGIface, initialNextHop Nexthop) (Nexthop, error) {
+func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf iface.IWGIface, initialNextHop Nexthop) (Nexthop, error) {
 	addr := prefix.Addr()
 	switch {
 	case addr.IsLoopback(),
@@ -135,6 +135,11 @@ func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf *iface.WGIfac
 		return Nexthop{}, vars.ErrRouteNotAllowed
 	}
 
+	// Check if the prefix is part of any local subnets
+	if isLocal, subnet := r.isPrefixInLocalSubnets(prefix); isLocal {
+		return Nexthop{}, fmt.Errorf("prefix %s is part of local subnet %s: %w", prefix, subnet, vars.ErrRouteNotAllowed)
+	}
+
 	// Determine the exit interface and next hop for the prefix, so we can add a specific route
 	nexthop, err := GetNextHop(addr)
 	if err != nil {
@@ -167,6 +172,36 @@ func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf *iface.WGIfac
 	return exitNextHop, nil
 }
 
+func (r *SysOps) isPrefixInLocalSubnets(prefix netip.Prefix) (bool, *net.IPNet) {
+	localInterfaces, err := net.Interfaces()
+	if err != nil {
+		log.Errorf("Failed to get local interfaces: %v", err)
+		return false, nil
+	}
+
+	for _, intf := range localInterfaces {
+		addrs, err := intf.Addrs()
+		if err != nil {
+			log.Errorf("Failed to get addresses for interface %s: %v", intf.Name, err)
+			continue
+		}
+
+		for _, addr := range addrs {
+			ipnet, ok := addr.(*net.IPNet)
+			if !ok {
+				log.Errorf("Failed to convert address to IPNet: %v", addr)
+				continue
+			}
+
+			if ipnet.Contains(prefix.Addr().AsSlice()) {
+				return true, ipnet
+			}
+		}
+	}
+
+	return false, nil
+}
+
 // genericAddVPNRoute adds a new route to the vpn interface, it splits the default prefix
 // in two /1 prefixes to avoid replacing the existing default route
 func (r *SysOps) genericAddVPNRoute(prefix netip.Prefix, intf *net.Interface) error {
@@ -392,7 +427,7 @@ func ipToAddr(ip net.IP, intf *net.Interface) (netip.Addr, error) {
 }
 
 func existsInRouteTable(prefix netip.Prefix) (bool, error) {
-	routes, err := getRoutesFromTable()
+	routes, err := GetRoutesFromTable()
 	if err != nil {
 		return false, fmt.Errorf("get routes from table: %w", err)
 	}
@@ -405,7 +440,7 @@ func existsInRouteTable(prefix netip.Prefix) (bool, error) {
 }
 
 func isSubRange(prefix netip.Prefix) (bool, error) {
-	routes, err := getRoutesFromTable()
+	routes, err := GetRoutesFromTable()
 	if err != nil {
 		return false, fmt.Errorf("get routes from table: %w", err)
 	}

client/internal/routemanager/systemops/systemops_generic_test.go

@@ -68,7 +68,7 @@ func TestAddRemoveRoutes(t *testing.T) {
 			err = wgInterface.Create()
 			require.NoError(t, err, "should create testing wireguard interface")
 
-			r := NewSysOps(wgInterface)
+			r := NewSysOps(wgInterface, nil)
 
 			_, _, err = r.SetupRouting(nil)
 			require.NoError(t, err)
@@ -224,7 +224,7 @@ func TestAddExistAndRemoveRoute(t *testing.T) {
 			require.NoError(t, err, "InterfaceByName should not return err")
 			intf := &net.Interface{Index: index.Index, Name: wgInterface.Name()}
 
-			r := NewSysOps(wgInterface)
+			r := NewSysOps(wgInterface, nil)
 
 			// Prepare the environment
 			if testCase.preExistingPrefix.IsValid() {
@@ -379,7 +379,7 @@ func setupTestEnv(t *testing.T) {
 		assert.NoError(t, wgInterface.Close())
 	})
 
-	r := NewSysOps(wgInterface)
+	r := NewSysOps(wgInterface, nil)
 	_, _, err := r.SetupRouting(nil)
 	require.NoError(t, err, "setupRouting should not return err")
 	t.Cleanup(func() {

client/internal/routemanager/systemops/systemops_ios.go

@@ -0,0 +1,64 @@
+//go:build ios
+
+package systemops
+
+import (
+	"net"
+	"net/netip"
+	"runtime"
+
+	log "github.com/sirupsen/logrus"
+
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+func (r *SysOps) SetupRouting([]net.IP) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.prefixes = make(map[netip.Prefix]struct{})
+	return nil, nil, nil
+}
+
+func (r *SysOps) CleanupRouting() error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	r.prefixes = make(map[netip.Prefix]struct{})
+	r.notify()
+	return nil
+}
+
+func (r *SysOps) AddVPNRoute(prefix netip.Prefix, _ *net.Interface) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	r.prefixes[prefix] = struct{}{}
+	r.notify()
+	return nil
+}
+
+func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, _ *net.Interface) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	delete(r.prefixes, prefix)
+	r.notify()
+	return nil
+}
+
+func EnableIPForwarding() error {
+	log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
+	return nil
+}
+
+func IsAddrRouted(netip.Addr, []netip.Prefix) (bool, netip.Prefix) {
+	return false, netip.Prefix{}
+}
+
+func (r *SysOps) notify() {
+	prefixes := make([]netip.Prefix, 0, len(r.prefixes))
+	for prefix := range r.prefixes {
+		prefixes = append(prefixes, prefix)
+	}
+	r.notifier.OnNewPrefixes(prefixes)
+}

client/internal/routemanager/systemops/systemops_linux.go

@@ -206,7 +206,7 @@ func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, intf *net.Interface) error
 	return nil
 }
 
-func getRoutesFromTable() ([]netip.Prefix, error) {
+func GetRoutesFromTable() ([]netip.Prefix, error) {
 	v4Routes, err := getRoutes(syscall.RT_TABLE_MAIN, netlink.FAMILY_V4)
 	if err != nil {
 		return nil, fmt.Errorf("get v4 routes: %w", err)
@@ -504,7 +504,7 @@ func getAddressFamily(prefix netip.Prefix) int {
 
 func hasSeparateRouting() ([]netip.Prefix, error) {
 	if isLegacy() {
-		return getRoutesFromTable()
+		return GetRoutesFromTable()
 	}
 	return nil, ErrRoutingIsSeparate
 }

client/internal/routemanager/systemops/systemops_nonlinux.go

@@ -24,5 +24,5 @@ func EnableIPForwarding() error {
 }
 
 func hasSeparateRouting() ([]netip.Prefix, error) {
-	return getRoutesFromTable()
+	return GetRoutesFromTable()
 }

client/internal/routemanager/systemops/systemops_windows.go

@@ -3,6 +3,8 @@
 package systemops
 
 import (
+	"context"
+	"encoding/binary"
 	"fmt"
 	"net"
 	"net/netip"
@@ -11,15 +13,43 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
+	"unsafe"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/yusufpapurcu/wmi"
+	"golang.org/x/sys/windows"
 
 	"github.com/netbirdio/netbird/client/firewall/uspfilter"
 	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
+type RouteUpdateType int
+
+// RouteUpdate represents a change in the routing table.
+// The interface field contains the index only.
+type RouteUpdate struct {
+	Type        RouteUpdateType
+	Destination netip.Prefix
+	NextHop     netip.Addr
+	Interface   *net.Interface
+}
+
+// RouteMonitor provides a way to monitor changes in the routing table.
+type RouteMonitor struct {
+	updates chan RouteUpdate
+	handle  windows.Handle
+	done    chan struct{}
+}
+
+// Route represents a single routing table entry.
+type Route struct {
+	Destination netip.Prefix
+	Nexthop     netip.Addr
+	Interface   *net.Interface
+}
+
 type MSFT_NetRoute struct {
 	DestinationPrefix string
 	NextHop           string
@@ -28,33 +58,77 @@ type MSFT_NetRoute struct {
 	AddressFamily     uint16
 }
 
-type Route struct {
+// MIB_IPFORWARD_ROW2 is defined in https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_ipforward_row2
-	Destination netip.Prefix
+type MIB_IPFORWARD_ROW2 struct {
-	Nexthop     netip.Addr
+	InterfaceLuid        uint64
-	Interface   *net.Interface
-}
-
-type MSFT_NetNeighbor struct {
-	IPAddress        string
-	LinkLayerAddress string
-	State            uint8
-	AddressFamily    uint16
 	InterfaceIndex       uint32
-	InterfaceAlias   string
+	DestinationPrefix    IP_ADDRESS_PREFIX
+	NextHop              SOCKADDR_INET_NEXTHOP
+	SitePrefixLength     uint8
+	ValidLifetime        uint32
+	PreferredLifetime    uint32
+	Metric               uint32
+	Protocol             uint32
+	Loopback             uint8
+	AutoconfigureAddress uint8
+	Publish              uint8
+	Immortal             uint8
+	Age                  uint32
+	Origin               uint32
 }
 
-type Neighbor struct {
+// IP_ADDRESS_PREFIX is defined in https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-ip_address_prefix
-	IPAddress        netip.Addr
+type IP_ADDRESS_PREFIX struct {
-	LinkLayerAddress string
+	Prefix       SOCKADDR_INET
-	State            uint8
+	PrefixLength uint8
-	AddressFamily    uint16
-	InterfaceIndex   uint32
-	InterfaceAlias   string
 }
 
-var prefixList []netip.Prefix
+// SOCKADDR_INET is defined in https://learn.microsoft.com/en-us/windows/win32/api/ws2ipdef/ns-ws2ipdef-sockaddr_inet
-var lastUpdate time.Time
+// It represents the union of IPv4 and IPv6 socket addresses
-var mux = sync.Mutex{}
+type SOCKADDR_INET struct {
+	sin6_family int16
+	// nolint:unused
+	sin6_port uint16
+	// 4 bytes ipv4 or 4 bytes flowinfo + 16 bytes ipv6 + 4 bytes scope_id
+	data [24]byte
+}
+
+// SOCKADDR_INET_NEXTHOP is the same as SOCKADDR_INET but offset by 2 bytes
+type SOCKADDR_INET_NEXTHOP struct {
+	// nolint:unused
+	pad         [2]byte
+	sin6_family int16
+	// nolint:unused
+	sin6_port uint16
+	// 4 bytes ipv4 or 4 bytes flowinfo + 16 bytes ipv6 + 4 bytes scope_id
+	data [24]byte
+}
+
+// MIB_NOTIFICATION_TYPE is defined in https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ne-netioapi-mib_notification_type
+type MIB_NOTIFICATION_TYPE int32
+
+var (
+	modiphlpapi                = windows.NewLazyDLL("iphlpapi.dll")
+	procNotifyRouteChange2     = modiphlpapi.NewProc("NotifyRouteChange2")
+	procCancelMibChangeNotify2 = modiphlpapi.NewProc("CancelMibChangeNotify2")
+
+	prefixList []netip.Prefix
+	lastUpdate time.Time
+	mux        sync.Mutex
+)
+
+const (
+	MibParemeterModification MIB_NOTIFICATION_TYPE = iota
+	MibAddInstance
+	MibDeleteInstance
+	MibInitialNotification
+)
+
+const (
+	RouteModified RouteUpdateType = iota
+	RouteAdded
+	RouteDeleted
+)
 
 func (r *SysOps) SetupRouting(initAddresses []net.IP) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
 	return r.setupRefCounter(initAddresses)
@@ -94,7 +168,156 @@ func (r *SysOps) removeFromRouteTable(prefix netip.Prefix, nexthop Nexthop) erro
 	return nil
 }
 
-func getRoutesFromTable() ([]netip.Prefix, error) {
+// NewRouteMonitor creates and starts a new RouteMonitor.
+// It returns a pointer to the RouteMonitor and an error if the monitor couldn't be started.
+func NewRouteMonitor(ctx context.Context) (*RouteMonitor, error) {
+	rm := &RouteMonitor{
+		updates: make(chan RouteUpdate, 5),
+		done:    make(chan struct{}),
+	}
+
+	if err := rm.start(ctx); err != nil {
+		return nil, err
+	}
+
+	return rm, nil
+}
+
+func (rm *RouteMonitor) start(ctx context.Context) error {
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
+	callbackPtr := windows.NewCallback(func(callerContext uintptr, row *MIB_IPFORWARD_ROW2, notificationType MIB_NOTIFICATION_TYPE) uintptr {
+		if ctx.Err() != nil {
+			return 0
+		}
+
+		update, err := rm.parseUpdate(row, notificationType)
+		if err != nil {
+			log.Errorf("Failed to parse route update: %v", err)
+			return 0
+		}
+
+		select {
+		case <-rm.done:
+			return 0
+		case rm.updates <- update:
+		default:
+			log.Warn("Route update channel is full, dropping update")
+		}
+		return 0
+	})
+
+	var handle windows.Handle
+	if err := notifyRouteChange2(windows.AF_UNSPEC, callbackPtr, 0, false, &handle); err != nil {
+		return fmt.Errorf("NotifyRouteChange2 failed: %w", err)
+	}
+
+	rm.handle = handle
+
+	return nil
+}
+
+func (rm *RouteMonitor) parseUpdate(row *MIB_IPFORWARD_ROW2, notificationType MIB_NOTIFICATION_TYPE) (RouteUpdate, error) {
+	// destination prefix, next hop, interface index, interface luid are guaranteed to be there
+	// GetIpForwardEntry2 is not needed
+
+	var update RouteUpdate
+
+	idx := int(row.InterfaceIndex)
+	if idx != 0 {
+		intf, err := net.InterfaceByIndex(idx)
+		if err != nil {
+			return update, fmt.Errorf("get interface name: %w", err)
+		}
+
+		update.Interface = intf
+	}
+
+	log.Tracef("Received route update with destination %v, next hop %v, interface %v", row.DestinationPrefix, row.NextHop, update.Interface)
+	dest := parseIPPrefix(row.DestinationPrefix, idx)
+	if !dest.Addr().IsValid() {
+		return RouteUpdate{}, fmt.Errorf("invalid destination: %v", row)
+	}
+
+	nexthop := parseIPNexthop(row.NextHop, idx)
+	if !nexthop.IsValid() {
+		return RouteUpdate{}, fmt.Errorf("invalid next hop %v", row)
+	}
+
+	updateType := RouteModified
+	switch notificationType {
+	case MibParemeterModification:
+		updateType = RouteModified
+	case MibAddInstance:
+		updateType = RouteAdded
+	case MibDeleteInstance:
+		updateType = RouteDeleted
+	}
+
+	update.Type = updateType
+	update.Destination = dest
+	update.NextHop = nexthop
+
+	return update, nil
+}
+
+// Stop stops the RouteMonitor.
+func (rm *RouteMonitor) Stop() error {
+	if rm.handle != 0 {
+		if err := cancelMibChangeNotify2(rm.handle); err != nil {
+			return fmt.Errorf("CancelMibChangeNotify2 failed: %w", err)
+		}
+		rm.handle = 0
+	}
+	close(rm.done)
+	close(rm.updates)
+	return nil
+}
+
+// RouteUpdates returns a channel that receives RouteUpdate messages.
+func (rm *RouteMonitor) RouteUpdates() <-chan RouteUpdate {
+	return rm.updates
+}
+
+func notifyRouteChange2(family uint32, callback uintptr, callerContext uintptr, initialNotification bool, handle *windows.Handle) error {
+	var initNotif uint32
+	if initialNotification {
+		initNotif = 1
+	}
+
+	r1, _, e1 := syscall.SyscallN(
+		procNotifyRouteChange2.Addr(),
+		uintptr(family),
+		callback,
+		callerContext,
+		uintptr(initNotif),
+		uintptr(unsafe.Pointer(handle)),
+	)
+	if r1 != 0 {
+		if e1 != 0 {
+			return e1
+		}
+		return syscall.EINVAL
+	}
+	return nil
+}
+
+func cancelMibChangeNotify2(handle windows.Handle) error {
+	r1, _, e1 := syscall.SyscallN(procCancelMibChangeNotify2.Addr(), uintptr(handle))
+	if r1 != 0 {
+		if e1 != 0 {
+			return e1
+		}
+		return syscall.EINVAL
+	}
+	return nil
+}
+
+// GetRoutesFromTable returns the current routing table from with prefixes only.
+// It ccaches the result for 2 seconds to avoid blocking the caller.
+func GetRoutesFromTable() ([]netip.Prefix, error) {
 	mux.Lock()
 	defer mux.Unlock()
 
@@ -117,6 +340,7 @@ func getRoutesFromTable() ([]netip.Prefix, error) {
 	return prefixList, nil
 }
 
+// GetRoutes retrieves the current routing table using WMI.
 func GetRoutes() ([]Route, error) {
 	var entries []MSFT_NetRoute
 
@@ -146,8 +370,8 @@ func GetRoutes() ([]Route, error) {
 				Name:  entry.InterfaceAlias,
 			}
 
-			if nexthop.Is6() && (nexthop.IsLinkLocalUnicast() || nexthop.IsLinkLocalMulticast()) {
+			if nexthop.Is6() {
-				nexthop = nexthop.WithZone(strconv.Itoa(int(entry.InterfaceIndex)))
+				nexthop = addZone(nexthop, int(entry.InterfaceIndex))
 			}
 		}
 
@@ -161,33 +385,6 @@ func GetRoutes() ([]Route, error) {
 	return routes, nil
 }
 
-func GetNeighbors() ([]Neighbor, error) {
-	var entries []MSFT_NetNeighbor
-	query := `SELECT IPAddress, LinkLayerAddress, State, AddressFamily, InterfaceIndex, InterfaceAlias FROM MSFT_NetNeighbor`
-	if err := wmi.QueryNamespace(query, &entries, `ROOT\StandardCimv2`); err != nil {
-		return nil, fmt.Errorf("failed to query MSFT_NetNeighbor: %w", err)
-	}
-
-	var neighbors []Neighbor
-	for _, entry := range entries {
-		addr, err := netip.ParseAddr(entry.IPAddress)
-		if err != nil {
-			log.Warnf("Unable to parse neighbor IP address %s: %v", entry.IPAddress, err)
-			continue
-		}
-		neighbors = append(neighbors, Neighbor{
-			IPAddress:        addr,
-			LinkLayerAddress: entry.LinkLayerAddress,
-			State:            entry.State,
-			AddressFamily:    entry.AddressFamily,
-			InterfaceIndex:   entry.InterfaceIndex,
-			InterfaceAlias:   entry.InterfaceAlias,
-		})
-	}
-
-	return neighbors, nil
-}
-
 func addRouteCmd(prefix netip.Prefix, nexthop Nexthop) error {
 	args := []string{"add", prefix.String()}
 
@@ -220,3 +417,54 @@ func addRouteCmd(prefix netip.Prefix, nexthop Nexthop) error {
 func isCacheDisabled() bool {
 	return os.Getenv("NB_DISABLE_ROUTE_CACHE") == "true"
 }
+
+func parseIPPrefix(prefix IP_ADDRESS_PREFIX, idx int) netip.Prefix {
+	ip := parseIP(prefix.Prefix, idx)
+	return netip.PrefixFrom(ip, int(prefix.PrefixLength))
+}
+
+func parseIP(addr SOCKADDR_INET, idx int) netip.Addr {
+	return parseIPGeneric(addr.sin6_family, addr.data, idx)
+}
+
+func parseIPNexthop(addr SOCKADDR_INET_NEXTHOP, idx int) netip.Addr {
+	return parseIPGeneric(addr.sin6_family, addr.data, idx)
+}
+
+func parseIPGeneric(family int16, data [24]byte, interfaceIndex int) netip.Addr {
+	switch family {
+	case windows.AF_INET:
+		ipv4 := binary.BigEndian.Uint32(data[:4])
+		return netip.AddrFrom4([4]byte{
+			byte(ipv4 >> 24),
+			byte(ipv4 >> 16),
+			byte(ipv4 >> 8),
+			byte(ipv4),
+		})
+
+	case windows.AF_INET6:
+		// The IPv6 address is stored after the 4-byte flowinfo field
+		var ipv6 [16]byte
+		copy(ipv6[:], data[4:20])
+		ip := netip.AddrFrom16(ipv6)
+
+		// Check if there's a non-zero scope_id
+		scopeID := binary.BigEndian.Uint32(data[20:24])
+		if scopeID != 0 {
+			ip = ip.WithZone(strconv.FormatUint(uint64(scopeID), 10))
+		} else if interfaceIndex != 0 {
+			ip = addZone(ip, interfaceIndex)
+		}
+
+		return ip
+	}
+
+	return netip.IPv4Unspecified()
+}
+
+func addZone(ip netip.Addr, interfaceIndex int) netip.Addr {
+	if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+		ip = ip.WithZone(strconv.Itoa(interfaceIndex))
+	}
+	return ip
+}

client/internal/routemanager/systemops/systemops_windows_test.go

@@ -73,7 +73,7 @@ var testCases = []testCase{
 	{
 		name:               "To duplicate internal route without custom dialer via physical interface", // local route takes precedence
 		destination:        "10.0.0.2:53",
-		expectedSourceIP:   "10.0.0.1",
+		expectedSourceIP:   "127.0.0.1",
 		expectedDestPrefix: "10.0.0.0/8",
 		expectedNextHop:    "0.0.0.0",
 		expectedInterface:  "Loopback Pseudo-Interface 1",
@@ -110,7 +110,7 @@ var testCases = []testCase{
 	{
 		name:               "To more specific route (local) without custom dialer via physical interface",
 		destination:        "127.0.10.2:53",
-		expectedSourceIP:   "10.0.0.1",
+		expectedSourceIP:   "127.0.0.1",
 		expectedDestPrefix: "127.0.0.0/8",
 		expectedNextHop:    "0.0.0.0",
 		expectedInterface:  "Loopback Pseudo-Interface 1",
@@ -181,31 +181,6 @@ func testRoute(t *testing.T, destination string, dialer dialer) *FindNetRouteOut
 	return combinedOutput
 }
 
-func createAndSetupDummyInterface(t *testing.T, interfaceName, ipAddressCIDR string) string {
-	t.Helper()
-
-	ip, ipNet, err := net.ParseCIDR(ipAddressCIDR)
-	require.NoError(t, err)
-	subnetMaskSize, _ := ipNet.Mask.Size()
-	script := fmt.Sprintf(`New-NetIPAddress -InterfaceAlias "%s" -IPAddress "%s" -PrefixLength %d -PolicyStore ActiveStore -Confirm:$False`, interfaceName, ip.String(), subnetMaskSize)
-	_, err = exec.Command("powershell", "-Command", script).CombinedOutput()
-	require.NoError(t, err, "Failed to assign IP address to loopback adapter")
-
-	// Wait for the IP address to be applied
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
-	defer cancel()
-	err = waitForIPAddress(ctx, interfaceName, ip.String())
-	require.NoError(t, err, "IP address not applied within timeout")
-
-	t.Cleanup(func() {
-		script = fmt.Sprintf(`Remove-NetIPAddress -InterfaceAlias "%s" -IPAddress "%s" -Confirm:$False`, interfaceName, ip.String())
-		_, err = exec.Command("powershell", "-Command", script).CombinedOutput()
-		require.NoError(t, err, "Failed to remove IP address from loopback adapter")
-	})
-
-	return interfaceName
-}
-
 func fetchOriginalGateway() (*RouteInfo, error) {
 	cmd := exec.Command("powershell", "-Command", "Get-NetRoute -DestinationPrefix 0.0.0.0/0 | Select-Object Nexthop, RouteMetric, InterfaceAlias | ConvertTo-Json")
 	output, err := cmd.CombinedOutput()
@@ -231,30 +206,6 @@ func verifyOutput(t *testing.T, output *FindNetRouteOutput, sourceIP, destPrefix
 	assert.Equal(t, intf, output.InterfaceAlias, "Interface mismatch")
 }
 
-func waitForIPAddress(ctx context.Context, interfaceAlias, expectedIPAddress string) error {
-	ticker := time.NewTicker(1 * time.Second)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-ticker.C:
-			out, err := exec.Command("powershell", "-Command", fmt.Sprintf(`Get-NetIPAddress -InterfaceAlias "%s" | Select-Object -ExpandProperty IPAddress`, interfaceAlias)).CombinedOutput()
-			if err != nil {
-				return err
-			}
-
-			ipAddresses := strings.Split(strings.TrimSpace(string(out)), "\n")
-			for _, ip := range ipAddresses {
-				if strings.TrimSpace(ip) == expectedIPAddress {
-					return nil
-				}
-			}
-		}
-	}
-}
-
 func combineOutputs(outputs []FindNetRouteOutput) *FindNetRouteOutput {
 	var combined FindNetRouteOutput
 
@@ -285,5 +236,25 @@ func combineOutputs(outputs []FindNetRouteOutput) *FindNetRouteOutput {
 func setupDummyInterfacesAndRoutes(t *testing.T) {
 	t.Helper()
 
-	createAndSetupDummyInterface(t, "Loopback Pseudo-Interface 1", "10.0.0.1/8")
+	addDummyRoute(t, "10.0.0.0/8")
+}
+
+func addDummyRoute(t *testing.T, dstCIDR string) {
+	t.Helper()
+
+	script := fmt.Sprintf(`New-NetRoute -DestinationPrefix "%s" -InterfaceIndex 1 -PolicyStore ActiveStore`, dstCIDR)
+
+	output, err := exec.Command("powershell", "-Command", script).CombinedOutput()
+	if err != nil {
+		t.Logf("Failed to add dummy route: %v\nOutput: %s", err, output)
+		t.FailNow()
+	}
+
+	t.Cleanup(func() {
+		script = fmt.Sprintf(`Remove-NetRoute -DestinationPrefix  "%s" -InterfaceIndex 1 -Confirm:$false`, dstCIDR)
+		output, err := exec.Command("powershell", "-Command", script).CombinedOutput()
+		if err != nil {
+			t.Logf("Failed to remove dummy route: %v\nOutput: %s", err, output)
+		}
+	})
 }

client/internal/wgproxy/ebpf/portlookup.go

@@ -1,4 +1,4 @@
-package wgproxy
+package ebpf
 
 import (
 	"fmt"

client/internal/wgproxy/ebpf/portlookup_test.go

@@ -1,4 +1,4 @@
-package wgproxy
+package ebpf
 
 import (
 	"fmt"

client/internal/wgproxy/ebpf/proxy.go

@@ -1,6 +1,6 @@
 //go:build linux && !android
 
-package wgproxy
+package ebpf
 
 import (
 	"context"
@@ -13,47 +13,49 @@ import (
 
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
+	"github.com/hashicorp/go-multierror"
 	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 
+	nberrors "github.com/netbirdio/netbird/client/errors"
 	"github.com/netbirdio/netbird/client/internal/ebpf"
 	ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager"
 	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
+const (
+	loopbackAddr = "127.0.0.1"
+)
+
 // WGEBPFProxy definition for proxy with EBPF support
 type WGEBPFProxy struct {
-	ebpfManager ebpfMgr.Manager
-
-	ctx    context.Context
-	cancel context.CancelFunc
-
-	lastUsedPort      uint16
 	localWGListenPort int
 
+	ebpfManager   ebpfMgr.Manager
 	turnConnStore map[uint16]net.Conn
 	turnConnMutex sync.Mutex
 
+	lastUsedPort uint16
 	rawConn      net.PacketConn
 	conn         transport.UDPConn
+
+	ctx       context.Context
+	ctxCancel context.CancelFunc
 }
 
 // NewWGEBPFProxy create new WGEBPFProxy instance
-func NewWGEBPFProxy(ctx context.Context, wgPort int) *WGEBPFProxy {
+func NewWGEBPFProxy(wgPort int) *WGEBPFProxy {
 	log.Debugf("instantiate ebpf proxy")
 	wgProxy := &WGEBPFProxy{
 		localWGListenPort: wgPort,
 		ebpfManager:       ebpf.GetEbpfManagerInstance(),
-		lastUsedPort:      0,
 		turnConnStore:     make(map[uint16]net.Conn),
 	}
-	wgProxy.ctx, wgProxy.cancel = context.WithCancel(ctx)
-
 	return wgProxy
 }
 
-// listen load ebpf program and listen the proxy
+// Listen load ebpf program and listen the proxy
-func (p *WGEBPFProxy) listen() error {
+func (p *WGEBPFProxy) Listen() error {
 	pl := portLookup{}
 	wgPorxyPort, err := pl.searchFreePort()
 	if err != nil {
@@ -72,9 +74,11 @@ func (p *WGEBPFProxy) listen() error {
 
 	addr := net.UDPAddr{
 		Port: wgPorxyPort,
-		IP:   net.ParseIP("127.0.0.1"),
+		IP:   net.ParseIP(loopbackAddr),
 	}
 
+	p.ctx, p.ctxCancel = context.WithCancel(context.Background())
+
 	conn, err := nbnet.ListenUDP("udp", &addr)
 	if err != nil {
 		cErr := p.Free()
@@ -91,106 +95,110 @@ func (p *WGEBPFProxy) listen() error {
 }
 
 // AddTurnConn add new turn connection for the proxy
-func (p *WGEBPFProxy) AddTurnConn(turnConn net.Conn) (net.Addr, error) {
+func (p *WGEBPFProxy) AddTurnConn(ctx context.Context, turnConn net.Conn) (net.Addr, error) {
 	wgEndpointPort, err := p.storeTurnConn(turnConn)
 	if err != nil {
 		return nil, err
 	}
 
-	go p.proxyToLocal(wgEndpointPort, turnConn)
+	go p.proxyToLocal(ctx, wgEndpointPort, turnConn)
 	log.Infof("turn conn added to wg proxy store: %s, endpoint port: :%d", turnConn.RemoteAddr(), wgEndpointPort)
 
 	wgEndpoint := &net.UDPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
+		IP:   net.ParseIP(loopbackAddr),
 		Port: int(wgEndpointPort),
 	}
 	return wgEndpoint, nil
 }
 
-// CloseConn doing nothing because this type of proxy implementation does not store the connection
+// Free resources except the remoteConns will be keep open.
-func (p *WGEBPFProxy) CloseConn() error {
+func (p *WGEBPFProxy) Free() error {
+	log.Debugf("free up ebpf wg proxy")
+	if p.ctx != nil && p.ctx.Err() != nil {
+		//nolint
 		return nil
 	}
 
-// Free resources
+	p.ctxCancel()
-func (p *WGEBPFProxy) Free() error {
+
-	log.Debugf("free up ebpf wg proxy")
+	var result *multierror.Error
-	var err1, err2, err3 error
+	if err := p.conn.Close(); err != nil {
-	if p.conn != nil {
+		result = multierror.Append(result, err)
-		err1 = p.conn.Close()
 	}
 
-	err2 = p.ebpfManager.FreeWGProxy()
+	if err := p.ebpfManager.FreeWGProxy(); err != nil {
-	if p.rawConn != nil {
+		result = multierror.Append(result, err)
-		err3 = p.rawConn.Close()
 	}
 
-	if err1 != nil {
+	if err := p.rawConn.Close(); err != nil {
-		return err1
+		result = multierror.Append(result, err)
+	}
+	return nberrors.FormatErrorOrNil(result)
 }
 
-	if err2 != nil {
+func (p *WGEBPFProxy) proxyToLocal(ctx context.Context, endpointPort uint16, remoteConn net.Conn) {
-		return err2
+	defer p.removeTurnConn(endpointPort)
-	}
 
-	return err3
+	var (
-}
+		err error
-
+		n   int
-func (p *WGEBPFProxy) proxyToLocal(endpointPort uint16, remoteConn net.Conn) {
+	)
 	buf := make([]byte, 1500)
-	var err error
+	for ctx.Err() == nil {
-	defer func() {
-		p.removeTurnConn(endpointPort)
-	}()
-	for {
-		select {
-		case <-p.ctx.Done():
-			return
-		default:
-			var n int
 		n, err = remoteConn.Read(buf)
 		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
 			if err != io.EOF {
 				log.Errorf("failed to read from turn conn (endpoint: :%d): %s", endpointPort, err)
 			}
 			return
 		}
-			err = p.sendPkg(buf[:n], endpointPort)
+
-			if err != nil {
+		if err := p.sendPkg(buf[:n], endpointPort); err != nil {
-				log.Errorf("failed to write out turn pkg to local conn: %v", err)
+			if ctx.Err() != nil || p.ctx.Err() != nil {
+				return
 			}
+			log.Errorf("failed to write out turn pkg to local conn: %v", err)
 		}
 	}
 }
 
 // proxyToRemote read messages from local WireGuard interface and forward it to remote conn
+// From this go routine has only one instance.
 func (p *WGEBPFProxy) proxyToRemote() {
 	buf := make([]byte, 1500)
-	for {
+	for p.ctx.Err() == nil {
-		select {
+		if err := p.readAndForwardPacket(buf); err != nil {
-		case <-p.ctx.Done():
+			if p.ctx.Err() != nil {
 				return
-		default:
+			}
+			log.Errorf("failed to proxy packet to remote conn: %s", err)
+		}
+	}
+}
+
+func (p *WGEBPFProxy) readAndForwardPacket(buf []byte) error {
 	n, addr, err := p.conn.ReadFromUDP(buf)
 	if err != nil {
-				log.Errorf("failed to read UDP pkg from WG: %s", err)
+		return fmt.Errorf("failed to read UDP packet from WG: %w", err)
-				return
 	}
 
 	p.turnConnMutex.Lock()
 	conn, ok := p.turnConnStore[uint16(addr.Port)]
 	p.turnConnMutex.Unlock()
 	if !ok {
-				log.Infof("turn conn not found by port: %d", addr.Port)
+		if p.ctx.Err() == nil {
-				continue
+			log.Debugf("turn conn not found by port because conn already has been closed: %d", addr.Port)
+		}
+		return nil
 	}
 
-			_, err = conn.Write(buf[:n])
+	if _, err := conn.Write(buf[:n]); err != nil {
-			if err != nil {
+		return fmt.Errorf("failed to forward local WG packet (%d) to remote turn conn: %w", addr.Port, err)
-				log.Debugf("failed to forward local wg pkg (%d) to remote turn conn: %s", addr.Port, err)
-			}
-		}
 	}
+	return nil
 }
 
 func (p *WGEBPFProxy) storeTurnConn(turnConn net.Conn) (uint16, error) {
@@ -206,11 +214,14 @@ func (p *WGEBPFProxy) storeTurnConn(turnConn net.Conn) (uint16, error) {
 }
 
 func (p *WGEBPFProxy) removeTurnConn(turnConnID uint16) {
-	log.Tracef("remove turn conn from store by port: %d", turnConnID)
 	p.turnConnMutex.Lock()
 	defer p.turnConnMutex.Unlock()
-	delete(p.turnConnStore, turnConnID)
 
+	_, ok := p.turnConnStore[turnConnID]
+	if ok {
+		log.Debugf("remove turn conn from store by port: %d", turnConnID)
+	}
+	delete(p.turnConnStore, turnConnID)
 }
 
 func (p *WGEBPFProxy) nextFreePort() (uint16, error) {

client/internal/wgproxy/ebpf/proxy_test.go

@@ -1,14 +1,13 @@
 //go:build linux && !android
 
-package wgproxy
+package ebpf
 
 import (
-	"context"
 	"testing"
 )
 
 func TestWGEBPFProxy_connStore(t *testing.T) {
-	wgProxy := NewWGEBPFProxy(context.Background(), 1)
+	wgProxy := NewWGEBPFProxy(1)
 
 	p, _ := wgProxy.storeTurnConn(nil)
 	if p != 1 {
@@ -28,7 +27,7 @@ func TestWGEBPFProxy_connStore(t *testing.T) {
 }
 
 func TestWGEBPFProxy_portCalculation_overflow(t *testing.T) {
-	wgProxy := NewWGEBPFProxy(context.Background(), 1)
+	wgProxy := NewWGEBPFProxy(1)
 
 	_, _ = wgProxy.storeTurnConn(nil)
 	wgProxy.lastUsedPort = 65535
@@ -44,7 +43,7 @@ func TestWGEBPFProxy_portCalculation_overflow(t *testing.T) {
 }
 
 func TestWGEBPFProxy_portCalculation_maxConn(t *testing.T) {
-	wgProxy := NewWGEBPFProxy(context.Background(), 1)
+	wgProxy := NewWGEBPFProxy(1)
 
 	for i := 0; i < 65535; i++ {
 		_, _ = wgProxy.storeTurnConn(nil)

client/internal/wgproxy/ebpf/wrapper.go

@@ -0,0 +1,44 @@
+//go:build linux && !android
+
+package ebpf
+
+import (
+	"context"
+	"fmt"
+	"net"
+)
+
+// ProxyWrapper help to keep the remoteConn instance for net.Conn.Close function call
+type ProxyWrapper struct {
+	WgeBPFProxy *WGEBPFProxy
+
+	remoteConn net.Conn
+	cancel     context.CancelFunc // with thic cancel function, we stop remoteToLocal thread
+}
+
+func (e *ProxyWrapper) AddTurnConn(ctx context.Context, remoteConn net.Conn) (net.Addr, error) {
+	ctxConn, cancel := context.WithCancel(ctx)
+	addr, err := e.WgeBPFProxy.AddTurnConn(ctxConn, remoteConn)
+
+	if err != nil {
+		cancel()
+		return nil, fmt.Errorf("add turn conn: %w", err)
+	}
+	e.remoteConn = remoteConn
+	e.cancel = cancel
+	return addr, err
+}
+
+// CloseConn close the remoteConn and automatically remove the conn instance from the map
+func (e *ProxyWrapper) CloseConn() error {
+	if e.cancel == nil {
+		return fmt.Errorf("proxy not started")
+	}
+
+	e.cancel()
+
+	if err := e.remoteConn.Close(); err != nil {
+		return fmt.Errorf("failed to close remote conn: %w", err)
+	}
+	return nil
+}

client/internal/wgproxy/factory.go

@@ -1,22 +0,0 @@
-package wgproxy
-
-import "context"
-
-type Factory struct {
-	wgPort    int
-	ebpfProxy Proxy
-}
-
-func (w *Factory) GetProxy(ctx context.Context) Proxy {
-	if w.ebpfProxy != nil {
-		return w.ebpfProxy
-	}
-	return NewWGUserSpaceProxy(ctx, w.wgPort)
-}
-
-func (w *Factory) Free() error {
-	if w.ebpfProxy != nil {
-		return w.ebpfProxy.Free()
-	}
-	return nil
-}

client/internal/wgproxy/factory_linux.go

@@ -3,20 +3,26 @@
 package wgproxy
 
 import (
-	"context"
-
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/wgproxy/ebpf"
+	"github.com/netbirdio/netbird/client/internal/wgproxy/usp"
 )
 
-func NewFactory(ctx context.Context, userspace bool, wgPort int) *Factory {
+type Factory struct {
+	wgPort    int
+	ebpfProxy *ebpf.WGEBPFProxy
+}
+
+func NewFactory(userspace bool, wgPort int) *Factory {
 	f := &Factory{wgPort: wgPort}
 
 	if userspace {
 		return f
 	}
 
-	ebpfProxy := NewWGEBPFProxy(ctx, wgPort)
+	ebpfProxy := ebpf.NewWGEBPFProxy(wgPort)
-	err := ebpfProxy.listen()
+	err := ebpfProxy.Listen()
 	if err != nil {
 		log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err)
 		return f
@@ -25,3 +31,20 @@ func NewFactory(ctx context.Context, userspace bool, wgPort int) *Factory {
 	f.ebpfProxy = ebpfProxy
 	return f
 }
+
+func (w *Factory) GetProxy() Proxy {
+	if w.ebpfProxy != nil {
+		p := &ebpf.ProxyWrapper{
+			WgeBPFProxy: w.ebpfProxy,
+		}
+		return p
+	}
+	return usp.NewWGUserSpaceProxy(w.wgPort)
+}
+
+func (w *Factory) Free() error {
+	if w.ebpfProxy == nil {
+		return nil
+	}
+	return w.ebpfProxy.Free()
+}

client/internal/wgproxy/factory_nonlinux.go

@@ -2,8 +2,20 @@
 
 package wgproxy
 
-import "context"
+import "github.com/netbirdio/netbird/client/internal/wgproxy/usp"
 
-func NewFactory(ctx context.Context, _ bool, wgPort int) *Factory {
+type Factory struct {
+	wgPort int
+}
+
+func NewFactory(_ bool, wgPort int) *Factory {
 	return &Factory{wgPort: wgPort}
 }
+
+func (w *Factory) GetProxy() Proxy {
+	return usp.NewWGUserSpaceProxy(w.wgPort)
+}
+
+func (w *Factory) Free() error {
+	return nil
+}

client/internal/wgproxy/proxy.go

@@ -1,12 +1,12 @@
 package wgproxy
 
 import (
+	"context"
 	"net"
 )
 
-// Proxy is a transfer layer between the Turn connection and the WireGuard
+// Proxy is a transfer layer between the relayed connection and the WireGuard
 type Proxy interface {
-	AddTurnConn(turnConn net.Conn) (net.Addr, error)
+	AddTurnConn(ctx context.Context, turnConn net.Conn) (net.Addr, error)
 	CloseConn() error
-	Free() error
 }

client/internal/wgproxy/proxy_test.go

@@ -0,0 +1,128 @@
+//go:build linux
+
+package wgproxy
+
+import (
+	"context"
+	"io"
+	"net"
+	"os"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/netbirdio/netbird/client/internal/wgproxy/ebpf"
+	"github.com/netbirdio/netbird/client/internal/wgproxy/usp"
+	"github.com/netbirdio/netbird/util"
+)
+
+func TestMain(m *testing.M) {
+	_ = util.InitLog("trace", "console")
+	code := m.Run()
+	os.Exit(code)
+}
+
+type mocConn struct {
+	closeChan chan struct{}
+	closed    bool
+}
+
+func newMockConn() *mocConn {
+	return &mocConn{
+		closeChan: make(chan struct{}),
+	}
+}
+
+func (m *mocConn) Read(b []byte) (n int, err error) {
+	<-m.closeChan
+	return 0, io.EOF
+}
+
+func (m *mocConn) Write(b []byte) (n int, err error) {
+	<-m.closeChan
+	return 0, io.EOF
+}
+
+func (m *mocConn) Close() error {
+	if m.closed == true {
+		return nil
+	}
+
+	m.closed = true
+	close(m.closeChan)
+	return nil
+}
+
+func (m *mocConn) LocalAddr() net.Addr {
+	panic("implement me")
+}
+
+func (m *mocConn) RemoteAddr() net.Addr {
+	return &net.UDPAddr{
+		IP: net.ParseIP("172.16.254.1"),
+	}
+}
+
+func (m *mocConn) SetDeadline(t time.Time) error {
+	panic("implement me")
+}
+
+func (m *mocConn) SetReadDeadline(t time.Time) error {
+	panic("implement me")
+}
+
+func (m *mocConn) SetWriteDeadline(t time.Time) error {
+	panic("implement me")
+}
+
+func TestProxyCloseByRemoteConn(t *testing.T) {
+	ctx := context.Background()
+
+	tests := []struct {
+		name  string
+		proxy Proxy
+	}{
+		{
+			name:  "userspace proxy",
+			proxy: usp.NewWGUserSpaceProxy(51830),
+		},
+	}
+
+	if runtime.GOOS == "linux" && os.Getenv("GITHUB_ACTIONS") != "true" {
+		ebpfProxy := ebpf.NewWGEBPFProxy(51831)
+		if err := ebpfProxy.Listen(); err != nil {
+			t.Fatalf("failed to initialize ebpf proxy: %s", err)
+		}
+		defer func() {
+			if err := ebpfProxy.Free(); err != nil {
+				t.Errorf("failed to free ebpf proxy: %s", err)
+			}
+		}()
+		proxyWrapper := &ebpf.ProxyWrapper{
+			WgeBPFProxy: ebpfProxy,
+		}
+
+		tests = append(tests, struct {
+			name  string
+			proxy Proxy
+		}{
+			name:  "ebpf proxy",
+			proxy: proxyWrapper,
+		})
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			relayedConn := newMockConn()
+			_, err := tt.proxy.AddTurnConn(ctx, relayedConn)
+			if err != nil {
+				t.Errorf("error: %v", err)
+			}
+
+			_ = relayedConn.Close()
+			if err := tt.proxy.CloseConn(); err != nil {
+				t.Errorf("error: %v", err)
+			}
+		})
+	}
+}

client/internal/wgproxy/proxy_userspace.go

@@ -1,108 +0,0 @@
-package wgproxy
-
-import (
-	"context"
-	"fmt"
-	"net"
-
-	log "github.com/sirupsen/logrus"
-
-	nbnet "github.com/netbirdio/netbird/util/net"
-)
-
-// WGUserSpaceProxy proxies
-type WGUserSpaceProxy struct {
-	localWGListenPort int
-	ctx               context.Context
-	cancel            context.CancelFunc
-
-	remoteConn net.Conn
-	localConn  net.Conn
-}
-
-// NewWGUserSpaceProxy instantiate a user space WireGuard proxy
-func NewWGUserSpaceProxy(ctx context.Context, wgPort int) *WGUserSpaceProxy {
-	log.Debugf("Initializing new user space proxy with port %d", wgPort)
-	p := &WGUserSpaceProxy{
-		localWGListenPort: wgPort,
-	}
-	p.ctx, p.cancel = context.WithCancel(ctx)
-	return p
-}
-
-// AddTurnConn start the proxy with the given remote conn
-func (p *WGUserSpaceProxy) AddTurnConn(turnConn net.Conn) (net.Addr, error) {
-	p.remoteConn = turnConn
-
-	var err error
-	p.localConn, err = nbnet.NewDialer().DialContext(p.ctx, "udp", fmt.Sprintf(":%d", p.localWGListenPort))
-	if err != nil {
-		log.Errorf("failed dialing to local Wireguard port %s", err)
-		return nil, err
-	}
-
-	go p.proxyToRemote()
-	go p.proxyToLocal()
-
-	return p.localConn.LocalAddr(), err
-}
-
-// CloseConn close the localConn
-func (p *WGUserSpaceProxy) CloseConn() error {
-	p.cancel()
-	if p.localConn == nil {
-		return nil
-	}
-	return p.localConn.Close()
-}
-
-// Free doing nothing because this implementation of proxy does not have global state
-func (p *WGUserSpaceProxy) Free() error {
-	return nil
-}
-
-// proxyToRemote proxies everything from Wireguard to the RemoteKey peer
-// blocks
-func (p *WGUserSpaceProxy) proxyToRemote() {
-
-	buf := make([]byte, 1500)
-	for {
-		select {
-		case <-p.ctx.Done():
-			return
-		default:
-			n, err := p.localConn.Read(buf)
-			if err != nil {
-				continue
-			}
-
-			_, err = p.remoteConn.Write(buf[:n])
-			if err != nil {
-				continue
-			}
-		}
-	}
-}
-
-// proxyToLocal proxies everything from the RemoteKey peer to local Wireguard
-// blocks
-func (p *WGUserSpaceProxy) proxyToLocal() {
-
-	buf := make([]byte, 1500)
-	for {
-		select {
-		case <-p.ctx.Done():
-			return
-		default:
-			n, err := p.remoteConn.Read(buf)
-			if err != nil {
-				continue
-			}
-
-			_, err = p.localConn.Write(buf[:n])
-			if err != nil {
-				continue
-			}
-		}
-	}
-}

client/internal/wgproxy/usp/proxy.go

@@ -0,0 +1,146 @@
+package usp
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/errors"
+)
+
+// WGUserSpaceProxy proxies
+type WGUserSpaceProxy struct {
+	localWGListenPort int
+	ctx               context.Context
+	cancel            context.CancelFunc
+
+	remoteConn net.Conn
+	localConn  net.Conn
+	closeMu    sync.Mutex
+	closed     bool
+}
+
+// NewWGUserSpaceProxy instantiate a user space WireGuard proxy. This is not a thread safe implementation
+func NewWGUserSpaceProxy(wgPort int) *WGUserSpaceProxy {
+	log.Debugf("Initializing new user space proxy with port %d", wgPort)
+	p := &WGUserSpaceProxy{
+		localWGListenPort: wgPort,
+	}
+	return p
+}
+
+// AddTurnConn start the proxy with the given remote conn
+func (p *WGUserSpaceProxy) AddTurnConn(ctx context.Context, remoteConn net.Conn) (net.Addr, error) {
+	p.ctx, p.cancel = context.WithCancel(ctx)
+
+	p.remoteConn = remoteConn
+
+	var err error
+	dialer := net.Dialer{}
+	p.localConn, err = dialer.DialContext(p.ctx, "udp", fmt.Sprintf(":%d", p.localWGListenPort))
+	if err != nil {
+		log.Errorf("failed dialing to local Wireguard port %s", err)
+		return nil, err
+	}
+
+	go p.proxyToRemote()
+	go p.proxyToLocal()
+
+	return p.localConn.LocalAddr(), err
+}
+
+// CloseConn close the localConn
+func (p *WGUserSpaceProxy) CloseConn() error {
+	if p.cancel == nil {
+		return fmt.Errorf("proxy not started")
+	}
+	return p.close()
+}
+
+func (p *WGUserSpaceProxy) close() error {
+	p.closeMu.Lock()
+	defer p.closeMu.Unlock()
+
+	// prevent double close
+	if p.closed {
+		return nil
+	}
+	p.closed = true
+
+	p.cancel()
+
+	var result *multierror.Error
+	if err := p.remoteConn.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("remote conn: %s", err))
+	}
+
+	if err := p.localConn.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("local conn: %s", err))
+	}
+	return errors.FormatErrorOrNil(result)
+}
+
+// proxyToRemote proxies from Wireguard to the RemoteKey
+func (p *WGUserSpaceProxy) proxyToRemote() {
+	defer func() {
+		if err := p.close(); err != nil {
+			log.Warnf("error in proxy to remote loop: %s", err)
+		}
+	}()
+
+	buf := make([]byte, 1500)
+	for p.ctx.Err() == nil {
+		n, err := p.localConn.Read(buf)
+		if err != nil {
+			if p.ctx.Err() != nil {
+				return
+			}
+			log.Debugf("failed to read from wg interface conn: %s", err)
+			return
+		}
+
+		_, err = p.remoteConn.Write(buf[:n])
+		if err != nil {
+			if p.ctx.Err() != nil {
+				return
+			}
+
+			log.Debugf("failed to write to remote conn: %s", err)
+			return
+		}
+	}
+}
+
+// proxyToLocal proxies from the Remote peer to local WireGuard
+func (p *WGUserSpaceProxy) proxyToLocal() {
+	defer func() {
+		if err := p.close(); err != nil {
+			log.Warnf("error in proxy to local loop: %s", err)
+		}
+	}()
+
+	buf := make([]byte, 1500)
+	for p.ctx.Err() == nil {
+		n, err := p.remoteConn.Read(buf)
+		if err != nil {
+			if p.ctx.Err() != nil {
+				return
+			}
+			log.Errorf("failed to read from remote conn: %s, %s", p.remoteConn.RemoteAddr(), err)
+			return
+		}
+
+		_, err = p.localConn.Write(buf[:n])
+		if err != nil {
+			if p.ctx.Err() != nil {
+				return
+			}
+			log.Debugf("failed to write to wg interface conn: %s", err)
+			continue
+		}
+	}
+}